@xyo-network/xl1-rest-block-publisher 2.1.1 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -14
- package/dist/node/AbstractS3PublishRunner.d.ts +31 -0
- package/dist/node/AbstractS3PublishRunner.d.ts.map +1 -0
- package/dist/node/S3BlockPublishRunner.d.ts +50 -0
- package/dist/node/S3BlockPublishRunner.d.ts.map +1 -0
- package/dist/node/S3ChainStatePublishRunner.d.ts +26 -0
- package/dist/node/S3ChainStatePublishRunner.d.ts.map +1 -0
- package/dist/node/S3IndexPublishRunner.d.ts +28 -0
- package/dist/node/S3IndexPublishRunner.d.ts.map +1 -0
- package/dist/node/index.d.ts +4 -1
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.mjs +134 -95
- package/dist/node/index.mjs.map +4 -4
- package/package.json +4 -4
- package/dist/node/RestBlockPublisher.d.ts +0 -83
- package/dist/node/RestBlockPublisher.d.ts.map +0 -1
package/README.md
CHANGED
|
@@ -5,26 +5,31 @@
|
|
|
5
5
|
[![npm-badge][]][npm-link]
|
|
6
6
|
[![license-badge][]][license-link]
|
|
7
7
|
|
|
8
|
-
> Publishes
|
|
8
|
+
> Publishes an XL1 chain as the static REST file layout the Rest viewers read — pre-compressed, CDN-friendly, one runner per concern.
|
|
9
9
|
|
|
10
10
|
## About
|
|
11
11
|
|
|
12
|
-
Finalized blocks and completed steps never change, so the chain can be materialized once as static files on S3-compatible object storage (e.g. Cloudflare R2) and served forever over plain HTTP. `
|
|
12
|
+
Finalized blocks and completed steps never change, so the chain can be materialized once as static files on S3-compatible object storage (e.g. Cloudflare R2) and served forever over plain HTTP. Three locator-configurable runners (implementing the `BlockPublishRunner`, `IndexPublishRunner`, and `ChainStatePublishRunner` interfaces from `@xyo-network/xl1-protocol-lib`) write that layout from any source `BlockViewer`:
|
|
13
|
+
|
|
14
|
+
- **`S3BlockPublishRunner`** — finalized blocks and their payloads
|
|
15
|
+
- **`S3IndexPublishRunner`** — completed steps' block summaries (the chain index)
|
|
16
|
+
- **`S3ChainStatePublishRunner`** — the mutable chain state (the head pointer), targeting the chain-state bucket
|
|
13
17
|
|
|
14
18
|
```text
|
|
15
|
-
/block/number/<n>.json ← immutable
|
|
16
|
-
/block/hash/<hash>.json ← immutable, same content
|
|
17
|
-
/blocks/step/<level>/<index>.json ← immutable, completed steps only
|
|
18
|
-
/payload/<hash>.json ← immutable
|
|
19
|
+
/block/number/<n>.json ← immutable (block runner)
|
|
20
|
+
/block/hash/<hash>.json ← immutable, same content (block runner)
|
|
21
|
+
/blocks/step/<level>/<index>.json ← immutable, completed steps only (index runner)
|
|
22
|
+
/payload/<hash>.json ← immutable (block runner)
|
|
23
|
+
/chain/head.json ← mutable, chain-state bucket (chain state runner)
|
|
19
24
|
```
|
|
20
25
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
Bodies are pre-compressed at write time (brotli by default, gzip or none configurable) and stored with the matching `Content-Encoding` plus `application/json`, so HTTP clients decompress transparently — keys keep their `.json` extension. Everything written gets `Cache-Control: public, max-age=31536000, immutable`.
|
|
26
|
+
Bodies are pre-compressed at write time (brotli by default, gzip or none configurable) and stored with the matching `Content-Encoding` plus `application/json`, so HTTP clients decompress transparently — keys keep their `.json` extension. Finalized files get `Cache-Control: public, max-age=31536000, immutable`; the head pointer gets `must-revalidate`.
|
|
24
27
|
|
|
25
28
|
```ts
|
|
26
29
|
import { S3Client } from '@aws-sdk/client-s3'
|
|
27
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
S3BlockPublishRunner, S3ChainStatePublishRunner, S3IndexPublishRunner,
|
|
32
|
+
} from '@xyo-network/xl1-rest-block-publisher'
|
|
28
33
|
|
|
29
34
|
const client = new S3Client({
|
|
30
35
|
region: 'auto',
|
|
@@ -32,11 +37,18 @@ const client = new S3Client({
|
|
|
32
37
|
credentials: { accessKeyId, secretAccessKey },
|
|
33
38
|
})
|
|
34
39
|
|
|
35
|
-
const
|
|
36
|
-
const
|
|
40
|
+
const blocks = await S3BlockPublishRunner.create({ bucket: 'finalized', client, context, source })
|
|
41
|
+
const index = await S3IndexPublishRunner.create({ bucket: 'finalized', client, context, source })
|
|
42
|
+
const chainState = await S3ChainStatePublishRunner.create({ bucket: 'chain-state', client, context, source })
|
|
43
|
+
|
|
44
|
+
const range = await blocks.sync(from) // publishes [from, sourceHead]; null when up to date
|
|
45
|
+
if (range) {
|
|
46
|
+
for (let n = range[0]; n <= range[1]; n++) await index.publishStepsCompletedBy(n)
|
|
47
|
+
await chainState.publishHead() // last, so readers never see a head referencing missing files
|
|
48
|
+
}
|
|
37
49
|
```
|
|
38
50
|
|
|
39
|
-
`sync(from)` resumes from a caller-supplied cursor — typically the block after the last published head pointer
|
|
51
|
+
`sync(from)` resumes from a caller-supplied cursor — typically the block after the last published head pointer. Omitting `from` republishes from genesis, which is safe (idempotent — deterministic keys, immutable content) but slow, so re-running after a crash is always safe. The path builders come from [`@xyo-network/xl1-rest-block-viewer`](https://www.npmjs.com/package/@xyo-network/xl1-rest-block-viewer), so readers and writers share one layout contract.
|
|
40
52
|
|
|
41
53
|
## Install
|
|
42
54
|
|
|
@@ -68,7 +80,10 @@ bun add @xyo-network/xl1-rest-block-publisher
|
|
|
68
80
|
|
|
69
81
|
## What's Inside
|
|
70
82
|
|
|
71
|
-
- **`
|
|
83
|
+
- **`S3BlockPublishRunner`** — `sync(from)` (cursor-resumed publish) and `publishBlock`; implements `BlockPublishRunner`.
|
|
84
|
+
- **`S3IndexPublishRunner`** — `publishStep`/`publishStepsCompletedBy` (completed steps only); implements `IndexPublishRunner`.
|
|
85
|
+
- **`S3ChainStatePublishRunner`** — `publishHead` with must-revalidate caching; implements `ChainStatePublishRunner`.
|
|
86
|
+
- **`AbstractS3PublishRunner`** — shared S3 target params (bucket, client, prefix, encoding) for custom runners.
|
|
72
87
|
- **`encodeBody`/`decodeBody`** — brotli/gzip/none body codecs used for storage and read-back.
|
|
73
88
|
|
|
74
89
|
## Building Locally
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { S3Client } from '@aws-sdk/client-s3';
|
|
2
|
+
import type { CreatableProviderParams } from '@xyo-network/xl1-protocol-sdk';
|
|
3
|
+
import { AbstractCreatableProvider } from '@xyo-network/xl1-protocol-sdk';
|
|
4
|
+
import type { RestContentEncoding } from './encoding.ts';
|
|
5
|
+
/** Shared parameters for S3 publish runners: where and how published files are written. */
|
|
6
|
+
export interface S3PublishRunnerParams extends CreatableProviderParams {
|
|
7
|
+
bucket: string;
|
|
8
|
+
/** S3-compatible client pointed at the target endpoint (e.g. Cloudflare R2). */
|
|
9
|
+
client: S3Client;
|
|
10
|
+
/** Storage/wire encoding for published files. Defaults to 'br' (brotli). */
|
|
11
|
+
contentEncoding?: RestContentEncoding;
|
|
12
|
+
/** Optional key prefix, allowing the layout to live in a shared bucket. */
|
|
13
|
+
prefix?: string;
|
|
14
|
+
}
|
|
15
|
+
/** Finalized files never change once written. */
|
|
16
|
+
export declare const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable";
|
|
17
|
+
/** Chain state is rewritten as the chain advances. */
|
|
18
|
+
export declare const MUTABLE_CACHE_CONTROL = "public, max-age=0, must-revalidate";
|
|
19
|
+
/**
|
|
20
|
+
* Base for publish runners targeting S3-compatible storage. Bodies are pre-compressed at
|
|
21
|
+
* write time and stored with the matching Content-Encoding plus application/json, so HTTP
|
|
22
|
+
* clients decompress transparently — keys keep their `.json` extension.
|
|
23
|
+
*/
|
|
24
|
+
export declare abstract class AbstractS3PublishRunner<TParams extends S3PublishRunnerParams = S3PublishRunnerParams> extends AbstractCreatableProvider<TParams> {
|
|
25
|
+
get bucket(): string;
|
|
26
|
+
get client(): S3Client;
|
|
27
|
+
get contentEncoding(): RestContentEncoding;
|
|
28
|
+
get prefix(): string;
|
|
29
|
+
protected putJson(path: string, json: string, cacheControl: string): Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=AbstractS3PublishRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AbstractS3PublishRunner.d.ts","sourceRoot":"","sources":["../../src/AbstractS3PublishRunner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AAEzE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AAGxD,2FAA2F;AAC3F,MAAM,WAAW,qBAAsB,SAAQ,uBAAuB;IACpE,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,MAAM,EAAE,QAAQ,CAAA;IAChB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,mBAAmB,CAAA;IACrC,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,iDAAiD;AACjD,eAAO,MAAM,uBAAuB,wCAAwC,CAAA;AAC5E,sDAAsD;AACtD,eAAO,MAAM,qBAAqB,uCAAuC,CAAA;AAEzE;;;;GAIG;AACH,8BAAsB,uBAAuB,CAAC,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,CACzG,SAAQ,yBAAyB,CAAC,OAAO,CAAC;IAC1C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,MAAM,IAAI,QAAQ,CAErB;IAED,IAAI,eAAe,IAAI,mBAAmB,CAEzC;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;cAEe,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAWzF"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { BlockPublishRunner, BlockViewer, SignedHydratedBlockWithHashMeta, XL1BlockNumber, XL1BlockRange } from '@xyo-network/xl1-protocol-lib';
|
|
2
|
+
import type { S3PublishRunnerParams } from './AbstractS3PublishRunner.ts';
|
|
3
|
+
import { AbstractS3PublishRunner } from './AbstractS3PublishRunner.ts';
|
|
4
|
+
/** A progress event emitted while publishing blocks. */
|
|
5
|
+
export interface BlockPublishProgress {
|
|
6
|
+
blockNumber: XL1BlockNumber;
|
|
7
|
+
published: number;
|
|
8
|
+
total: number;
|
|
9
|
+
}
|
|
10
|
+
/** Parameters for S3BlockPublishRunner. */
|
|
11
|
+
export interface S3BlockPublishRunnerParams extends S3PublishRunnerParams {
|
|
12
|
+
/** Maximum concurrent block publishes during sync. */
|
|
13
|
+
concurrency?: number;
|
|
14
|
+
/**
|
|
15
|
+
* Called every `progressInterval` blocks (and at pass completion). When omitted, the same
|
|
16
|
+
* events are logged via the params logger so long backfills are observable out of the box.
|
|
17
|
+
*/
|
|
18
|
+
onProgress?: (event: BlockPublishProgress) => void;
|
|
19
|
+
/** How many block publishes between progress reports during sync. */
|
|
20
|
+
progressInterval?: number;
|
|
21
|
+
/** The viewer to publish from (only finalized chains should be published). */
|
|
22
|
+
source: BlockViewer;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Publishes finalized blocks (and their payloads) as the static REST file layout that
|
|
26
|
+
* `RestBlockViewer` reads (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).
|
|
27
|
+
*
|
|
28
|
+
* Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing
|
|
29
|
+
* any range is safe. `sync(from)` resumes from a caller-supplied cursor (typically the block
|
|
30
|
+
* after the last published head); omitting it republishes from genesis, which is safe.
|
|
31
|
+
*/
|
|
32
|
+
export declare class S3BlockPublishRunner extends AbstractS3PublishRunner<S3BlockPublishRunnerParams> implements BlockPublishRunner {
|
|
33
|
+
static readonly defaultMoniker: "BlockPublishRunner";
|
|
34
|
+
static readonly dependencies: never[];
|
|
35
|
+
static readonly monikers: "BlockPublishRunner"[];
|
|
36
|
+
moniker: "BlockPublishRunner";
|
|
37
|
+
get concurrency(): number;
|
|
38
|
+
get progressInterval(): number;
|
|
39
|
+
get source(): BlockViewer;
|
|
40
|
+
/** Publishes one block at its by-number and by-hash paths, plus its payload files. */
|
|
41
|
+
publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null>;
|
|
42
|
+
/**
|
|
43
|
+
* Publishes every block from `from` (inclusive, default genesis) through the source head.
|
|
44
|
+
* Returns the published range, or null when already up to date.
|
|
45
|
+
*/
|
|
46
|
+
sync(from?: XL1BlockNumber): Promise<XL1BlockRange | null>;
|
|
47
|
+
/** Emits a progress event to the onProgress callback, or logs it when no callback is set. */
|
|
48
|
+
private report;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=S3BlockPublishRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"S3BlockPublishRunner.d.ts","sourceRoot":"","sources":["../../src/S3BlockPublishRunner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAAE,WAAW,EAAE,+BAA+B,EAAE,cAAc,EAAE,aAAa,EAChG,MAAM,+BAA+B,CAAA;AAQtC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAA;AACzE,OAAO,EAAE,uBAAuB,EAA2B,MAAM,8BAA8B,CAAA;AAE/F,wDAAwD;AACxD,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,cAAc,CAAA;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,2CAA2C;AAC3C,MAAM,WAAW,0BAA2B,SAAQ,qBAAqB;IACvE,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAA;IAClD,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,8EAA8E;IAC9E,MAAM,EAAE,WAAW,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,qBACa,oBAAqB,SAAQ,uBAAuB,CAAC,0BAA0B,CAAE,YAAW,kBAAkB;IACzH,MAAM,CAAC,QAAQ,CAAC,cAAc,uBAA4B;IAC1D,MAAM,CAAC,QAAQ,CAAC,YAAY,UAAK;IACjC,MAAM,CAAC,QAAQ,CAAC,QAAQ,yBAA8B;IACtD,OAAO,uBAAsC;IAE7C,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,sFAAsF;IAChF,YAAY,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,+BAA+B,GAAG,IAAI,CAAC;IAYhG;;;OAGG;IACG,IAAI,CAAC,IAAI,GAAE,cAA0C,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAqB3F,6FAA6F;IAC7F,OAAO,CAAC,MAAM;CAQf"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { ChainStatePublishRunner, ChainStateViewerMethods, SignedHydratedBlockWithHashMeta } from '@xyo-network/xl1-protocol-lib';
|
|
2
|
+
import type { S3PublishRunnerParams } from './AbstractS3PublishRunner.ts';
|
|
3
|
+
import { AbstractS3PublishRunner } from './AbstractS3PublishRunner.ts';
|
|
4
|
+
/** Parameters for S3ChainStatePublishRunner. */
|
|
5
|
+
export interface S3ChainStatePublishRunnerParams extends S3PublishRunnerParams {
|
|
6
|
+
/** Where the current head is read from (any BlockViewer satisfies this). */
|
|
7
|
+
source: ChainStateViewerMethods;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Publishes the mutable chain state — the head pointer (`chain/head.json`) — that
|
|
11
|
+
* `RestChainStateViewer` reads. This is the one mutable file in the REST layout, so it is
|
|
12
|
+
* stored with must-revalidate caching and should target the chain-state bucket.
|
|
13
|
+
*
|
|
14
|
+
* Callers must publish the finalized files for a head before publishing the head itself,
|
|
15
|
+
* so readers never see a head that references missing files.
|
|
16
|
+
*/
|
|
17
|
+
export declare class S3ChainStatePublishRunner extends AbstractS3PublishRunner<S3ChainStatePublishRunnerParams> implements ChainStatePublishRunner {
|
|
18
|
+
static readonly defaultMoniker: "ChainStatePublishRunner";
|
|
19
|
+
static readonly dependencies: never[];
|
|
20
|
+
static readonly monikers: "ChainStatePublishRunner"[];
|
|
21
|
+
moniker: "ChainStatePublishRunner";
|
|
22
|
+
get source(): ChainStateViewerMethods;
|
|
23
|
+
/** Publishes the source's current head to the mutable chain state pointer. */
|
|
24
|
+
publishHead(): Promise<SignedHydratedBlockWithHashMeta>;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=S3ChainStatePublishRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"S3ChainStatePublishRunner.d.ts","sourceRoot":"","sources":["../../src/S3ChainStatePublishRunner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,uBAAuB,EAAE,uBAAuB,EAAE,+BAA+B,EAClF,MAAM,+BAA+B,CAAA;AAKtC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAA;AACzE,OAAO,EAAE,uBAAuB,EAAyB,MAAM,8BAA8B,CAAA;AAE7F,gDAAgD;AAChD,MAAM,WAAW,+BAAgC,SAAQ,qBAAqB;IAC5E,4EAA4E;IAC5E,MAAM,EAAE,uBAAuB,CAAA;CAChC;AAED;;;;;;;GAOG;AACH,qBACa,yBAA0B,SAAQ,uBAAuB,CAAC,+BAA+B,CAAE,YAAW,uBAAuB;IACxI,MAAM,CAAC,QAAQ,CAAC,cAAc,4BAAiC;IAC/D,MAAM,CAAC,QAAQ,CAAC,YAAY,UAAK;IACjC,MAAM,CAAC,QAAQ,CAAC,QAAQ,8BAAmC;IAC3D,OAAO,4BAA2C;IAElD,IAAI,MAAM,IAAI,uBAAuB,CAEpC;IAED,8EAA8E;IACxE,WAAW,IAAI,OAAO,CAAC,+BAA+B,CAAC;CAM9D"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { BlockViewer, IndexPublishRunner, XL1BlockNumber } from '@xyo-network/xl1-protocol-lib';
|
|
2
|
+
import type { S3PublishRunnerParams } from './AbstractS3PublishRunner.ts';
|
|
3
|
+
import { AbstractS3PublishRunner } from './AbstractS3PublishRunner.ts';
|
|
4
|
+
/** Parameters for S3IndexPublishRunner. */
|
|
5
|
+
export interface S3IndexPublishRunnerParams extends S3PublishRunnerParams {
|
|
6
|
+
/** The viewer to publish from (only finalized chains should be published). */
|
|
7
|
+
source: BlockViewer;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Publishes completed steps' block summaries (the chain index) as the static REST file
|
|
11
|
+
* layout that `RestIndexViewer` reads. Step files are immutable: only complete steps are
|
|
12
|
+
* ever published, and a completed step's blocks never change.
|
|
13
|
+
*/
|
|
14
|
+
export declare class S3IndexPublishRunner extends AbstractS3PublishRunner<S3IndexPublishRunnerParams> implements IndexPublishRunner {
|
|
15
|
+
static readonly defaultMoniker: "IndexPublishRunner";
|
|
16
|
+
static readonly dependencies: never[];
|
|
17
|
+
static readonly monikers: "IndexPublishRunner"[];
|
|
18
|
+
moniker: "IndexPublishRunner";
|
|
19
|
+
get source(): BlockViewer;
|
|
20
|
+
/**
|
|
21
|
+
* Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).
|
|
22
|
+
* Asserts the step is complete — partial tail steps are never published.
|
|
23
|
+
*/
|
|
24
|
+
publishStep(stepLevel: number, stepIndex: number): Promise<void>;
|
|
25
|
+
/** Publishes every step (at every level) completed by the given block. */
|
|
26
|
+
publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=S3IndexPublishRunner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"S3IndexPublishRunner.d.ts","sourceRoot":"","sources":["../../src/S3IndexPublishRunner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACS,WAAW,EAAE,kBAAkB,EAAE,cAAc,EACnE,MAAM,+BAA+B,CAAA;AAOtC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAA;AACzE,OAAO,EAAE,uBAAuB,EAA2B,MAAM,8BAA8B,CAAA;AAE/F,2CAA2C;AAC3C,MAAM,WAAW,0BAA2B,SAAQ,qBAAqB;IACvE,8EAA8E;IAC9E,MAAM,EAAE,WAAW,CAAA;CACpB;AAED;;;;GAIG;AACH,qBACa,oBAAqB,SAAQ,uBAAuB,CAAC,0BAA0B,CAAE,YAAW,kBAAkB;IACzH,MAAM,CAAC,QAAQ,CAAC,cAAc,uBAA4B;IAC1D,MAAM,CAAC,QAAQ,CAAC,YAAY,UAAK;IACjC,MAAM,CAAC,QAAQ,CAAC,QAAQ,yBAA8B;IACtD,OAAO,uBAAsC;IAE7C,IAAI,MAAM,IAAI,WAAW,CAExB;IAED;;;OAGG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBtE,0EAA0E;IACpE,uBAAuB,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;CAQ1E"}
|
package/dist/node/index.d.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
export * from './AbstractS3PublishRunner.ts';
|
|
1
2
|
export * from './encoding.ts';
|
|
2
|
-
export * from './
|
|
3
|
+
export * from './S3BlockPublishRunner.ts';
|
|
4
|
+
export * from './S3ChainStatePublishRunner.ts';
|
|
5
|
+
export * from './S3IndexPublishRunner.ts';
|
|
3
6
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/node/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,8BAA8B,CAAA;AAC5C,cAAc,eAAe,CAAA;AAC7B,cAAc,2BAA2B,CAAA;AACzC,cAAc,gCAAgC,CAAA;AAC9C,cAAc,2BAA2B,CAAA"}
|
package/dist/node/index.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
3
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
4
|
var __decorateClass = (decorators, target, key, kind) => {
|
|
4
5
|
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
5
6
|
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
@@ -8,6 +9,12 @@ var __decorateClass = (decorators, target, key, kind) => {
|
|
|
8
9
|
if (kind && result) __defProp(target, key, result);
|
|
9
10
|
return result;
|
|
10
11
|
};
|
|
12
|
+
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
13
|
+
|
|
14
|
+
// src/AbstractS3PublishRunner.ts
|
|
15
|
+
import { PutObjectCommand } from "@aws-sdk/client-s3";
|
|
16
|
+
import { assertEx } from "@xylabs/sdk-js";
|
|
17
|
+
import { AbstractCreatableProvider } from "@xyo-network/xl1-protocol-sdk";
|
|
11
18
|
|
|
12
19
|
// src/encoding.ts
|
|
13
20
|
import ZLIB from "node:zlib";
|
|
@@ -38,100 +45,71 @@ function decodeBody(body, contentEncoding) {
|
|
|
38
45
|
}
|
|
39
46
|
}
|
|
40
47
|
|
|
41
|
-
// src/
|
|
42
|
-
import { PutObjectCommand } from "@aws-sdk/client-s3";
|
|
43
|
-
import {
|
|
44
|
-
AbstractCreatable,
|
|
45
|
-
assertEx,
|
|
46
|
-
creatable,
|
|
47
|
-
isNull
|
|
48
|
-
} from "@xylabs/sdk-js";
|
|
49
|
-
import { asXL1BlockNumber, StepSizes } from "@xyo-network/xl1-protocol-lib";
|
|
50
|
-
import { blocksMaxStep, BlocksStepSummarySchema } from "@xyo-network/xl1-protocol-sdk";
|
|
51
|
-
import {
|
|
52
|
-
blockHashPath,
|
|
53
|
-
blockNumberPath,
|
|
54
|
-
blocksStepPath,
|
|
55
|
-
payloadPath
|
|
56
|
-
} from "@xyo-network/xl1-rest-block-viewer";
|
|
57
|
-
import { Semaphore } from "async-mutex";
|
|
48
|
+
// src/AbstractS3PublishRunner.ts
|
|
58
49
|
var IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable";
|
|
59
|
-
var
|
|
50
|
+
var MUTABLE_CACHE_CONTROL = "public, max-age=0, must-revalidate";
|
|
51
|
+
var AbstractS3PublishRunner = class extends AbstractCreatableProvider {
|
|
60
52
|
get bucket() {
|
|
61
53
|
return assertEx(this.params.bucket, () => "No bucket specified");
|
|
62
54
|
}
|
|
63
55
|
get client() {
|
|
64
56
|
return assertEx(this.params.client, () => "No client specified");
|
|
65
57
|
}
|
|
66
|
-
get concurrency() {
|
|
67
|
-
return this.params.concurrency ?? 8;
|
|
68
|
-
}
|
|
69
58
|
get contentEncoding() {
|
|
70
59
|
return this.params.contentEncoding ?? "br";
|
|
71
60
|
}
|
|
72
61
|
get prefix() {
|
|
73
62
|
return this.params.prefix ?? "";
|
|
74
63
|
}
|
|
64
|
+
async putJson(path, json, cacheControl) {
|
|
65
|
+
const { body, contentEncoding } = encodeBody(json, this.contentEncoding);
|
|
66
|
+
await this.client.send(new PutObjectCommand({
|
|
67
|
+
Bucket: this.bucket,
|
|
68
|
+
Key: `${this.prefix}${path}`,
|
|
69
|
+
Body: body,
|
|
70
|
+
CacheControl: cacheControl,
|
|
71
|
+
ContentEncoding: contentEncoding,
|
|
72
|
+
ContentType: "application/json"
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
// src/S3BlockPublishRunner.ts
|
|
78
|
+
import { assertEx as assertEx2, isNull } from "@xylabs/sdk-js";
|
|
79
|
+
import { asXL1BlockNumber, BlockPublishRunnerMoniker } from "@xyo-network/xl1-protocol-lib";
|
|
80
|
+
import { creatableProvider } from "@xyo-network/xl1-protocol-sdk";
|
|
81
|
+
import {
|
|
82
|
+
blockHashPath,
|
|
83
|
+
blockNumberPath,
|
|
84
|
+
payloadPath
|
|
85
|
+
} from "@xyo-network/xl1-rest-block-viewer";
|
|
86
|
+
import { Semaphore } from "async-mutex";
|
|
87
|
+
var S3BlockPublishRunner = class extends AbstractS3PublishRunner {
|
|
88
|
+
moniker = S3BlockPublishRunner.defaultMoniker;
|
|
89
|
+
get concurrency() {
|
|
90
|
+
return this.params.concurrency ?? 8;
|
|
91
|
+
}
|
|
75
92
|
get progressInterval() {
|
|
76
93
|
return this.params.progressInterval ?? 100;
|
|
77
94
|
}
|
|
78
95
|
get source() {
|
|
79
|
-
return
|
|
96
|
+
return assertEx2(this.params.source, () => "No source specified");
|
|
80
97
|
}
|
|
81
98
|
/** Publishes one block at its by-number and by-hash paths, plus its payload files. */
|
|
82
99
|
async publishBlock(blockNumber) {
|
|
83
100
|
const block = await this.source.blockByNumber(blockNumber);
|
|
84
101
|
if (isNull(block)) return null;
|
|
85
102
|
const json = JSON.stringify(block);
|
|
86
|
-
await this.putJson(blockNumberPath(block[0].block), json);
|
|
87
|
-
await this.putJson(blockHashPath(block[0]._hash), json);
|
|
103
|
+
await this.putJson(blockNumberPath(block[0].block), json, IMMUTABLE_CACHE_CONTROL);
|
|
104
|
+
await this.putJson(blockHashPath(block[0]._hash), json, IMMUTABLE_CACHE_CONTROL);
|
|
88
105
|
for (const payload of block[1]) {
|
|
89
|
-
await this.putJson(payloadPath(payload._hash), JSON.stringify(payload));
|
|
106
|
+
await this.putJson(payloadPath(payload._hash), JSON.stringify(payload), IMMUTABLE_CACHE_CONTROL);
|
|
90
107
|
}
|
|
91
108
|
return block;
|
|
92
109
|
}
|
|
93
110
|
/**
|
|
94
|
-
* Publishes
|
|
95
|
-
*
|
|
96
|
-
*/
|
|
97
|
-
async publishStep(stepLevel, stepIndex) {
|
|
98
|
-
const size = StepSizes[stepLevel];
|
|
99
|
-
assertEx(stepLevel <= blocksMaxStep, () => `publishStep does not support step levels above ${blocksMaxStep} (requested ${stepLevel})`);
|
|
100
|
-
const lastBlockNumber = (stepIndex + 1) * size - 1;
|
|
101
|
-
const headNumber = await this.source.currentBlockNumber();
|
|
102
|
-
assertEx(lastBlockNumber <= headNumber, () => `Step ${stepLevel}/${stepIndex} is not complete (head ${headNumber})`);
|
|
103
|
-
const newestFirst = await this.source.blocksByStep(stepLevel, stepIndex);
|
|
104
|
-
const blocks = newestFirst.toReversed();
|
|
105
|
-
const summary = {
|
|
106
|
-
schema: BlocksStepSummarySchema,
|
|
107
|
-
hash: assertEx(blocks.at(-1), () => `No blocks for step ${stepLevel}/${stepIndex}`)[0]._hash,
|
|
108
|
-
stepSize: size,
|
|
109
|
-
blocks
|
|
110
|
-
};
|
|
111
|
-
await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary));
|
|
112
|
-
this.report({
|
|
113
|
-
stepIndex,
|
|
114
|
-
stepLevel,
|
|
115
|
-
type: "step"
|
|
116
|
-
});
|
|
117
|
-
}
|
|
118
|
-
/** Publishes every step (at every level) completed by the given block. */
|
|
119
|
-
async publishStepsCompletedBy(blockNumber) {
|
|
120
|
-
for (let stepLevel = 0; stepLevel <= blocksMaxStep; stepLevel++) {
|
|
121
|
-
const size = StepSizes[stepLevel];
|
|
122
|
-
if ((blockNumber + 1) % size === 0) {
|
|
123
|
-
await this.publishStep(stepLevel, (blockNumber + 1) / size - 1);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
/**
|
|
128
|
-
* Publishes everything from `from` (inclusive, default genesis) to the source head
|
|
129
|
-
* (inclusive), then the completed steps in that range. Returns the published range, or
|
|
130
|
-
* null when already up to date.
|
|
131
|
-
*
|
|
132
|
-
* The caller supplies the resume cursor — typically the block after the last published
|
|
133
|
-
* head pointer, which is written by the finalizer rather than this publisher. Omitting
|
|
134
|
-
* `from` republishes from genesis, which is safe (idempotent) but slow.
|
|
111
|
+
* Publishes every block from `from` (inclusive, default genesis) through the source head.
|
|
112
|
+
* Returns the published range, or null when already up to date.
|
|
135
113
|
*/
|
|
136
114
|
async sync(from = asXL1BlockNumber(0, true)) {
|
|
137
115
|
const sourceHead = await this.source.currentBlock();
|
|
@@ -142,33 +120,18 @@ var RestBlockPublisher = class extends AbstractCreatable {
|
|
|
142
120
|
const total = numbers.length;
|
|
143
121
|
let publishedCount = 0;
|
|
144
122
|
await Promise.all(numbers.map((blockNumber) => semaphore.runExclusive(async () => {
|
|
145
|
-
|
|
123
|
+
assertEx2(await this.publishBlock(blockNumber), () => `Block not found in source [${blockNumber}]`);
|
|
146
124
|
publishedCount += 1;
|
|
147
125
|
if (publishedCount % this.progressInterval === 0 || publishedCount === total) {
|
|
148
126
|
this.report({
|
|
149
127
|
blockNumber,
|
|
150
128
|
published: publishedCount,
|
|
151
|
-
total
|
|
152
|
-
type: "block"
|
|
129
|
+
total
|
|
153
130
|
});
|
|
154
131
|
}
|
|
155
132
|
})));
|
|
156
|
-
for (const blockNumber of numbers) {
|
|
157
|
-
await this.publishStepsCompletedBy(blockNumber);
|
|
158
|
-
}
|
|
159
133
|
return [from, end];
|
|
160
134
|
}
|
|
161
|
-
async putJson(path, json) {
|
|
162
|
-
const { body, contentEncoding } = encodeBody(json, this.contentEncoding);
|
|
163
|
-
await this.client.send(new PutObjectCommand({
|
|
164
|
-
Bucket: this.bucket,
|
|
165
|
-
Key: `${this.prefix}${path}`,
|
|
166
|
-
Body: body,
|
|
167
|
-
CacheControl: IMMUTABLE_CACHE_CONTROL,
|
|
168
|
-
ContentEncoding: contentEncoding,
|
|
169
|
-
ContentType: "application/json"
|
|
170
|
-
}));
|
|
171
|
-
}
|
|
172
135
|
/** Emits a progress event to the onProgress callback, or logs it when no callback is set. */
|
|
173
136
|
report(event) {
|
|
174
137
|
const onProgress = this.params.onProgress;
|
|
@@ -176,23 +139,99 @@ var RestBlockPublisher = class extends AbstractCreatable {
|
|
|
176
139
|
onProgress(event);
|
|
177
140
|
return;
|
|
178
141
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
142
|
+
this.logger?.info(`S3BlockPublishRunner: published block ${event.blockNumber} (${event.published}/${event.total})`);
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
__publicField(S3BlockPublishRunner, "defaultMoniker", BlockPublishRunnerMoniker);
|
|
146
|
+
__publicField(S3BlockPublishRunner, "dependencies", []);
|
|
147
|
+
__publicField(S3BlockPublishRunner, "monikers", [BlockPublishRunnerMoniker]);
|
|
148
|
+
S3BlockPublishRunner = __decorateClass([
|
|
149
|
+
creatableProvider()
|
|
150
|
+
], S3BlockPublishRunner);
|
|
151
|
+
|
|
152
|
+
// src/S3ChainStatePublishRunner.ts
|
|
153
|
+
import { assertEx as assertEx3 } from "@xylabs/sdk-js";
|
|
154
|
+
import { ChainStatePublishRunnerMoniker } from "@xyo-network/xl1-protocol-lib";
|
|
155
|
+
import { creatableProvider as creatableProvider2 } from "@xyo-network/xl1-protocol-sdk";
|
|
156
|
+
import { headPath } from "@xyo-network/xl1-rest-block-viewer";
|
|
157
|
+
var S3ChainStatePublishRunner = class extends AbstractS3PublishRunner {
|
|
158
|
+
moniker = S3ChainStatePublishRunner.defaultMoniker;
|
|
159
|
+
get source() {
|
|
160
|
+
return assertEx3(this.params.source, () => "No source specified");
|
|
161
|
+
}
|
|
162
|
+
/** Publishes the source's current head to the mutable chain state pointer. */
|
|
163
|
+
async publishHead() {
|
|
164
|
+
const head = await this.source.currentBlock();
|
|
165
|
+
await this.putJson(headPath(), JSON.stringify(head), MUTABLE_CACHE_CONTROL);
|
|
166
|
+
this.logger?.info(`S3ChainStatePublishRunner: published head ${head[0].block}`);
|
|
167
|
+
return head;
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
__publicField(S3ChainStatePublishRunner, "defaultMoniker", ChainStatePublishRunnerMoniker);
|
|
171
|
+
__publicField(S3ChainStatePublishRunner, "dependencies", []);
|
|
172
|
+
__publicField(S3ChainStatePublishRunner, "monikers", [ChainStatePublishRunnerMoniker]);
|
|
173
|
+
S3ChainStatePublishRunner = __decorateClass([
|
|
174
|
+
creatableProvider2()
|
|
175
|
+
], S3ChainStatePublishRunner);
|
|
176
|
+
|
|
177
|
+
// src/S3IndexPublishRunner.ts
|
|
178
|
+
import { assertEx as assertEx4 } from "@xylabs/sdk-js";
|
|
179
|
+
import {
|
|
180
|
+
BlocksStepSummarySchema,
|
|
181
|
+
IndexPublishRunnerMoniker,
|
|
182
|
+
StepSizes
|
|
183
|
+
} from "@xyo-network/xl1-protocol-lib";
|
|
184
|
+
import { blocksMaxStep, creatableProvider as creatableProvider3 } from "@xyo-network/xl1-protocol-sdk";
|
|
185
|
+
import { blocksStepPath } from "@xyo-network/xl1-rest-block-viewer";
|
|
186
|
+
var S3IndexPublishRunner = class extends AbstractS3PublishRunner {
|
|
187
|
+
moniker = S3IndexPublishRunner.defaultMoniker;
|
|
188
|
+
get source() {
|
|
189
|
+
return assertEx4(this.params.source, () => "No source specified");
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).
|
|
193
|
+
* Asserts the step is complete — partial tail steps are never published.
|
|
194
|
+
*/
|
|
195
|
+
async publishStep(stepLevel, stepIndex) {
|
|
196
|
+
const size = StepSizes[stepLevel];
|
|
197
|
+
assertEx4(stepLevel <= blocksMaxStep, () => `publishStep does not support step levels above ${blocksMaxStep} (requested ${stepLevel})`);
|
|
198
|
+
const lastBlockNumber = (stepIndex + 1) * size - 1;
|
|
199
|
+
const headNumber = await this.source.currentBlockNumber();
|
|
200
|
+
assertEx4(lastBlockNumber <= headNumber, () => `Step ${stepLevel}/${stepIndex} is not complete (head ${headNumber})`);
|
|
201
|
+
const newestFirst = await this.source.blocksByStep(stepLevel, stepIndex);
|
|
202
|
+
const blocks = newestFirst.toReversed();
|
|
203
|
+
const summary = {
|
|
204
|
+
schema: BlocksStepSummarySchema,
|
|
205
|
+
hash: assertEx4(blocks.at(-1), () => `No blocks for step ${stepLevel}/${stepIndex}`)[0]._hash,
|
|
206
|
+
stepSize: size,
|
|
207
|
+
blocks
|
|
208
|
+
};
|
|
209
|
+
await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary), IMMUTABLE_CACHE_CONTROL);
|
|
210
|
+
this.logger?.info(`S3IndexPublishRunner: published step ${stepLevel}/${stepIndex}`);
|
|
211
|
+
}
|
|
212
|
+
/** Publishes every step (at every level) completed by the given block. */
|
|
213
|
+
async publishStepsCompletedBy(blockNumber) {
|
|
214
|
+
for (let stepLevel = 0; stepLevel <= blocksMaxStep; stepLevel++) {
|
|
215
|
+
const size = StepSizes[stepLevel];
|
|
216
|
+
if ((blockNumber + 1) % size === 0) {
|
|
217
|
+
await this.publishStep(stepLevel, (blockNumber + 1) / size - 1);
|
|
187
218
|
}
|
|
188
219
|
}
|
|
189
220
|
}
|
|
190
221
|
};
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
222
|
+
__publicField(S3IndexPublishRunner, "defaultMoniker", IndexPublishRunnerMoniker);
|
|
223
|
+
__publicField(S3IndexPublishRunner, "dependencies", []);
|
|
224
|
+
__publicField(S3IndexPublishRunner, "monikers", [IndexPublishRunnerMoniker]);
|
|
225
|
+
S3IndexPublishRunner = __decorateClass([
|
|
226
|
+
creatableProvider3()
|
|
227
|
+
], S3IndexPublishRunner);
|
|
194
228
|
export {
|
|
195
|
-
|
|
229
|
+
AbstractS3PublishRunner,
|
|
230
|
+
IMMUTABLE_CACHE_CONTROL,
|
|
231
|
+
MUTABLE_CACHE_CONTROL,
|
|
232
|
+
S3BlockPublishRunner,
|
|
233
|
+
S3ChainStatePublishRunner,
|
|
234
|
+
S3IndexPublishRunner,
|
|
196
235
|
decodeBody,
|
|
197
236
|
encodeBody
|
|
198
237
|
};
|
package/dist/node/index.mjs.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
|
-
"sources": ["../../src/encoding.ts", "../../src/
|
|
4
|
-
"sourcesContent": ["import ZLIB from 'node:zlib'\n\n/** Wire/storage encoding for published files. */\nexport type RestContentEncoding = 'br' | 'gzip' | 'none'\n\n/** An encoded file body plus the HTTP Content-Encoding it should be served with. */\nexport interface EncodedBody {\n body: Uint8Array\n contentEncoding?: 'br' | 'gzip'\n}\n\n/**\n * Encodes a JSON string for storage. Bodies are pre-compressed at write time and stored\n * with the matching Content-Encoding so HTTP clients decompress transparently \u2014 keys keep\n * their .json extension and Content-Type stays application/json.\n */\nexport function encodeBody(json: string, encoding: RestContentEncoding): EncodedBody {\n switch (encoding) {\n case 'br': {\n return { body: ZLIB.brotliCompressSync(json), contentEncoding: 'br' }\n }\n case 'gzip': {\n return { body: ZLIB.gzipSync(json), contentEncoding: 'gzip' }\n }\n case 'none': {\n return { body: new TextEncoder().encode(json) }\n }\n }\n}\n\n/** Decodes a stored body back to its JSON string using the object's Content-Encoding. */\nexport function decodeBody(body: Uint8Array, contentEncoding?: string): string {\n switch (contentEncoding) {\n case 'br': {\n return ZLIB.brotliDecompressSync(body).toString('utf8')\n }\n case 'gzip': {\n return ZLIB.gunzipSync(body).toString('utf8')\n }\n default: {\n return new TextDecoder().decode(body)\n }\n }\n}\n", "import type { S3Client } from '@aws-sdk/client-s3'\nimport { PutObjectCommand } from '@aws-sdk/client-s3'\nimport type { CreatableParams } from '@xylabs/sdk-js'\nimport {\n AbstractCreatable, assertEx, creatable, isNull,\n} from '@xylabs/sdk-js'\nimport type {\n BlockViewer, SignedHydratedBlockWithHashMeta, XL1BlockNumber, XL1BlockRange,\n} from '@xyo-network/xl1-protocol-lib'\nimport { asXL1BlockNumber, StepSizes } from '@xyo-network/xl1-protocol-lib'\nimport type { BlocksStepSummary } from '@xyo-network/xl1-protocol-sdk'\nimport { blocksMaxStep, BlocksStepSummarySchema } from '@xyo-network/xl1-protocol-sdk'\nimport {\n blockHashPath, blockNumberPath, blocksStepPath, payloadPath,\n} from '@xyo-network/xl1-rest-block-viewer'\nimport { Semaphore } from 'async-mutex'\n\nimport type { RestContentEncoding } from './encoding.ts'\nimport { encodeBody } from './encoding.ts'\n\n/** A progress event emitted while publishing. */\nexport type RestPublishEvent\n = | { blockNumber: XL1BlockNumber; published: number; total: number; type: 'block' }\n | { stepIndex: number; stepLevel: number; type: 'step' }\n\n/** Parameters for RestBlockPublisher. */\nexport interface RestBlockPublisherParams extends CreatableParams {\n bucket: string\n /** S3-compatible client pointed at the target endpoint (e.g. Cloudflare R2). */\n client: S3Client\n /** Maximum concurrent block publishes during sync. */\n concurrency?: number\n /** Storage/wire encoding for published files. Defaults to 'br' (brotli). */\n contentEncoding?: RestContentEncoding\n /**\n * Called as publishing progresses: every `progressInterval` blocks (and at pass completion)\n * and per step file. When omitted, the same events are logged via the params logger so\n * long backfills are observable out of the box.\n */\n onProgress?: (event: RestPublishEvent) => void\n /** Optional key prefix, allowing the layout to live in a shared bucket. */\n prefix?: string\n /** How many block publishes between progress reports during sync. */\n progressInterval?: number\n /** The viewer to publish from (only finalized chains should be published). */\n source: BlockViewer\n}\n\n/** Everything this publisher writes is immutable once written. */\nconst IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'\n\n/**\n * Publishes a chain's immutable files as the static REST file layout that `RestBlockViewer`\n * reads (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).\n *\n * The mutable chain state (the head pointer, `chain/head.json`) is NOT written here \u2014 it\n * lives in a separate chain-state bucket and is written by the chain's finalizer after the\n * finalized files for that head exist, so readers never see a head that references missing\n * files.\n *\n * Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing\n * any range is safe. `sync(from)` resumes from a caller-supplied cursor (typically the block\n * after the last published head); omitting it republishes from genesis, which is safe.\n */\n@creatable()\nexport class RestBlockPublisher extends AbstractCreatable<RestBlockPublisherParams> {\n get bucket(): string {\n return assertEx(this.params.bucket, () => 'No bucket specified')\n }\n\n get client(): S3Client {\n return assertEx(this.params.client, () => 'No client specified')\n }\n\n get concurrency(): number {\n return this.params.concurrency ?? 8\n }\n\n get contentEncoding(): RestContentEncoding {\n return this.params.contentEncoding ?? 'br'\n }\n\n get prefix(): string {\n return this.params.prefix ?? ''\n }\n\n get progressInterval(): number {\n return this.params.progressInterval ?? 100\n }\n\n get source(): BlockViewer {\n return assertEx(this.params.source, () => 'No source specified')\n }\n\n /** Publishes one block at its by-number and by-hash paths, plus its payload files. */\n async publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null> {\n const block = await this.source.blockByNumber(blockNumber)\n if (isNull(block)) return null\n const json = JSON.stringify(block)\n await this.putJson(blockNumberPath(block[0].block), json)\n await this.putJson(blockHashPath(block[0]._hash), json)\n for (const payload of block[1]) {\n await this.putJson(payloadPath(payload._hash), JSON.stringify(payload))\n }\n return block\n }\n\n /**\n * Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).\n * Asserts the step is complete \u2014 partial tail steps are never published.\n */\n async publishStep(stepLevel: number, stepIndex: number): Promise<void> {\n const size = StepSizes[stepLevel]\n assertEx(stepLevel <= blocksMaxStep, () => `publishStep does not support step levels above ${blocksMaxStep} (requested ${stepLevel})`)\n const lastBlockNumber = (stepIndex + 1) * size - 1\n const headNumber = await this.source.currentBlockNumber()\n assertEx(lastBlockNumber <= headNumber, () => `Step ${stepLevel}/${stepIndex} is not complete (head ${headNumber})`)\n const newestFirst = await this.source.blocksByStep(stepLevel, stepIndex)\n const blocks = newestFirst.toReversed()\n const summary: BlocksStepSummary = {\n schema: BlocksStepSummarySchema,\n hash: assertEx(blocks.at(-1), () => `No blocks for step ${stepLevel}/${stepIndex}`)[0]._hash,\n stepSize: size,\n blocks,\n }\n await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary))\n this.report({\n stepIndex, stepLevel, type: 'step',\n })\n }\n\n /** Publishes every step (at every level) completed by the given block. */\n async publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void> {\n for (let stepLevel = 0; stepLevel <= blocksMaxStep; stepLevel++) {\n const size = StepSizes[stepLevel]\n if ((blockNumber + 1) % size === 0) {\n await this.publishStep(stepLevel, (blockNumber + 1) / size - 1)\n }\n }\n }\n\n /**\n * Publishes everything from `from` (inclusive, default genesis) to the source head\n * (inclusive), then the completed steps in that range. Returns the published range, or\n * null when already up to date.\n *\n * The caller supplies the resume cursor \u2014 typically the block after the last published\n * head pointer, which is written by the finalizer rather than this publisher. Omitting\n * `from` republishes from genesis, which is safe (idempotent) but slow.\n */\n async sync(from: XL1BlockNumber = asXL1BlockNumber(0, true)): Promise<XL1BlockRange | null> {\n const sourceHead = await this.source.currentBlock()\n const end = sourceHead[0].block\n if (from > end) return null\n\n const semaphore = new Semaphore(this.concurrency)\n const numbers = Array.from({ length: end - from + 1 }, (_, i) => asXL1BlockNumber(from + i, true))\n const total = numbers.length\n let publishedCount = 0\n await Promise.all(numbers.map(blockNumber => semaphore.runExclusive(async () => {\n assertEx(await this.publishBlock(blockNumber), () => `Block not found in source [${blockNumber}]`)\n publishedCount += 1\n if (publishedCount % this.progressInterval === 0 || publishedCount === total) {\n this.report({\n blockNumber, published: publishedCount, total, type: 'block',\n })\n }\n })))\n for (const blockNumber of numbers) {\n await this.publishStepsCompletedBy(blockNumber)\n }\n return [from, end]\n }\n\n private async putJson(path: string, json: string): Promise<void> {\n const { body, contentEncoding } = encodeBody(json, this.contentEncoding)\n await this.client.send(new PutObjectCommand({\n Bucket: this.bucket,\n Key: `${this.prefix}${path}`,\n Body: body,\n CacheControl: IMMUTABLE_CACHE_CONTROL,\n ContentEncoding: contentEncoding,\n ContentType: 'application/json',\n }))\n }\n\n /** Emits a progress event to the onProgress callback, or logs it when no callback is set. */\n private report(event: RestPublishEvent): void {\n const onProgress = this.params.onProgress\n if (onProgress) {\n onProgress(event)\n return\n }\n switch (event.type) {\n case 'block': {\n this.logger?.info(`RestBlockPublisher: published block ${event.blockNumber} (${event.published}/${event.total})`)\n break\n }\n case 'step': {\n this.logger?.info(`RestBlockPublisher: published step ${event.stepLevel}/${event.stepIndex}`)\n break\n }\n }\n }\n}\n"],
|
|
5
|
-
"mappings": "
|
|
6
|
-
"names": []
|
|
3
|
+
"sources": ["../../src/AbstractS3PublishRunner.ts", "../../src/encoding.ts", "../../src/S3BlockPublishRunner.ts", "../../src/S3ChainStatePublishRunner.ts", "../../src/S3IndexPublishRunner.ts"],
|
|
4
|
+
"sourcesContent": ["import type { S3Client } from '@aws-sdk/client-s3'\nimport { PutObjectCommand } from '@aws-sdk/client-s3'\nimport { assertEx } from '@xylabs/sdk-js'\nimport type { CreatableProviderParams } from '@xyo-network/xl1-protocol-sdk'\nimport { AbstractCreatableProvider } from '@xyo-network/xl1-protocol-sdk'\n\nimport type { RestContentEncoding } from './encoding.ts'\nimport { encodeBody } from './encoding.ts'\n\n/** Shared parameters for S3 publish runners: where and how published files are written. */\nexport interface S3PublishRunnerParams extends CreatableProviderParams {\n bucket: string\n /** S3-compatible client pointed at the target endpoint (e.g. Cloudflare R2). */\n client: S3Client\n /** Storage/wire encoding for published files. Defaults to 'br' (brotli). */\n contentEncoding?: RestContentEncoding\n /** Optional key prefix, allowing the layout to live in a shared bucket. */\n prefix?: string\n}\n\n/** Finalized files never change once written. */\nexport const IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'\n/** Chain state is rewritten as the chain advances. */\nexport const MUTABLE_CACHE_CONTROL = 'public, max-age=0, must-revalidate'\n\n/**\n * Base for publish runners targeting S3-compatible storage. Bodies are pre-compressed at\n * write time and stored with the matching Content-Encoding plus application/json, so HTTP\n * clients decompress transparently \u2014 keys keep their `.json` extension.\n */\nexport abstract class AbstractS3PublishRunner<TParams extends S3PublishRunnerParams = S3PublishRunnerParams>\n extends AbstractCreatableProvider<TParams> {\n get bucket(): string {\n return assertEx(this.params.bucket, () => 'No bucket specified')\n }\n\n get client(): S3Client {\n return assertEx(this.params.client, () => 'No client specified')\n }\n\n get contentEncoding(): RestContentEncoding {\n return this.params.contentEncoding ?? 'br'\n }\n\n get prefix(): string {\n return this.params.prefix ?? ''\n }\n\n protected async putJson(path: string, json: string, cacheControl: string): Promise<void> {\n const { body, contentEncoding } = encodeBody(json, this.contentEncoding)\n await this.client.send(new PutObjectCommand({\n Bucket: this.bucket,\n Key: `${this.prefix}${path}`,\n Body: body,\n CacheControl: cacheControl,\n ContentEncoding: contentEncoding,\n ContentType: 'application/json',\n }))\n }\n}\n", "import ZLIB from 'node:zlib'\n\n/** Wire/storage encoding for published files. */\nexport type RestContentEncoding = 'br' | 'gzip' | 'none'\n\n/** An encoded file body plus the HTTP Content-Encoding it should be served with. */\nexport interface EncodedBody {\n body: Uint8Array\n contentEncoding?: 'br' | 'gzip'\n}\n\n/**\n * Encodes a JSON string for storage. Bodies are pre-compressed at write time and stored\n * with the matching Content-Encoding so HTTP clients decompress transparently \u2014 keys keep\n * their .json extension and Content-Type stays application/json.\n */\nexport function encodeBody(json: string, encoding: RestContentEncoding): EncodedBody {\n switch (encoding) {\n case 'br': {\n return { body: ZLIB.brotliCompressSync(json), contentEncoding: 'br' }\n }\n case 'gzip': {\n return { body: ZLIB.gzipSync(json), contentEncoding: 'gzip' }\n }\n case 'none': {\n return { body: new TextEncoder().encode(json) }\n }\n }\n}\n\n/** Decodes a stored body back to its JSON string using the object's Content-Encoding. */\nexport function decodeBody(body: Uint8Array, contentEncoding?: string): string {\n switch (contentEncoding) {\n case 'br': {\n return ZLIB.brotliDecompressSync(body).toString('utf8')\n }\n case 'gzip': {\n return ZLIB.gunzipSync(body).toString('utf8')\n }\n default: {\n return new TextDecoder().decode(body)\n }\n }\n}\n", "import { assertEx, isNull } from '@xylabs/sdk-js'\nimport type {\n BlockPublishRunner, BlockViewer, SignedHydratedBlockWithHashMeta, XL1BlockNumber, XL1BlockRange,\n} from '@xyo-network/xl1-protocol-lib'\nimport { asXL1BlockNumber, BlockPublishRunnerMoniker } from '@xyo-network/xl1-protocol-lib'\nimport { creatableProvider } from '@xyo-network/xl1-protocol-sdk'\nimport {\n blockHashPath, blockNumberPath, payloadPath,\n} from '@xyo-network/xl1-rest-block-viewer'\nimport { Semaphore } from 'async-mutex'\n\nimport type { S3PublishRunnerParams } from './AbstractS3PublishRunner.ts'\nimport { AbstractS3PublishRunner, IMMUTABLE_CACHE_CONTROL } from './AbstractS3PublishRunner.ts'\n\n/** A progress event emitted while publishing blocks. */\nexport interface BlockPublishProgress {\n blockNumber: XL1BlockNumber\n published: number\n total: number\n}\n\n/** Parameters for S3BlockPublishRunner. */\nexport interface S3BlockPublishRunnerParams extends S3PublishRunnerParams {\n /** Maximum concurrent block publishes during sync. */\n concurrency?: number\n /**\n * Called every `progressInterval` blocks (and at pass completion). When omitted, the same\n * events are logged via the params logger so long backfills are observable out of the box.\n */\n onProgress?: (event: BlockPublishProgress) => void\n /** How many block publishes between progress reports during sync. */\n progressInterval?: number\n /** The viewer to publish from (only finalized chains should be published). */\n source: BlockViewer\n}\n\n/**\n * Publishes finalized blocks (and their payloads) as the static REST file layout that\n * `RestBlockViewer` reads (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).\n *\n * Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing\n * any range is safe. `sync(from)` resumes from a caller-supplied cursor (typically the block\n * after the last published head); omitting it republishes from genesis, which is safe.\n */\n@creatableProvider()\nexport class S3BlockPublishRunner extends AbstractS3PublishRunner<S3BlockPublishRunnerParams> implements BlockPublishRunner {\n static readonly defaultMoniker = BlockPublishRunnerMoniker\n static readonly dependencies = []\n static readonly monikers = [BlockPublishRunnerMoniker]\n moniker = S3BlockPublishRunner.defaultMoniker\n\n get concurrency(): number {\n return this.params.concurrency ?? 8\n }\n\n get progressInterval(): number {\n return this.params.progressInterval ?? 100\n }\n\n get source(): BlockViewer {\n return assertEx(this.params.source, () => 'No source specified')\n }\n\n /** Publishes one block at its by-number and by-hash paths, plus its payload files. */\n async publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null> {\n const block = await this.source.blockByNumber(blockNumber)\n if (isNull(block)) return null\n const json = JSON.stringify(block)\n await this.putJson(blockNumberPath(block[0].block), json, IMMUTABLE_CACHE_CONTROL)\n await this.putJson(blockHashPath(block[0]._hash), json, IMMUTABLE_CACHE_CONTROL)\n for (const payload of block[1]) {\n await this.putJson(payloadPath(payload._hash), JSON.stringify(payload), IMMUTABLE_CACHE_CONTROL)\n }\n return block\n }\n\n /**\n * Publishes every block from `from` (inclusive, default genesis) through the source head.\n * Returns the published range, or null when already up to date.\n */\n async sync(from: XL1BlockNumber = asXL1BlockNumber(0, true)): Promise<XL1BlockRange | null> {\n const sourceHead = await this.source.currentBlock()\n const end = sourceHead[0].block\n if (from > end) return null\n\n const semaphore = new Semaphore(this.concurrency)\n const numbers = Array.from({ length: end - from + 1 }, (_, i) => asXL1BlockNumber(from + i, true))\n const total = numbers.length\n let publishedCount = 0\n await Promise.all(numbers.map(blockNumber => semaphore.runExclusive(async () => {\n assertEx(await this.publishBlock(blockNumber), () => `Block not found in source [${blockNumber}]`)\n publishedCount += 1\n if (publishedCount % this.progressInterval === 0 || publishedCount === total) {\n this.report({\n blockNumber, published: publishedCount, total,\n })\n }\n })))\n return [from, end]\n }\n\n /** Emits a progress event to the onProgress callback, or logs it when no callback is set. */\n private report(event: BlockPublishProgress): void {\n const onProgress = this.params.onProgress\n if (onProgress) {\n onProgress(event)\n return\n }\n this.logger?.info(`S3BlockPublishRunner: published block ${event.blockNumber} (${event.published}/${event.total})`)\n }\n}\n", "import { assertEx } from '@xylabs/sdk-js'\nimport type {\n ChainStatePublishRunner, ChainStateViewerMethods, SignedHydratedBlockWithHashMeta,\n} from '@xyo-network/xl1-protocol-lib'\nimport { ChainStatePublishRunnerMoniker } from '@xyo-network/xl1-protocol-lib'\nimport { creatableProvider } from '@xyo-network/xl1-protocol-sdk'\nimport { headPath } from '@xyo-network/xl1-rest-block-viewer'\n\nimport type { S3PublishRunnerParams } from './AbstractS3PublishRunner.ts'\nimport { AbstractS3PublishRunner, MUTABLE_CACHE_CONTROL } from './AbstractS3PublishRunner.ts'\n\n/** Parameters for S3ChainStatePublishRunner. */\nexport interface S3ChainStatePublishRunnerParams extends S3PublishRunnerParams {\n /** Where the current head is read from (any BlockViewer satisfies this). */\n source: ChainStateViewerMethods\n}\n\n/**\n * Publishes the mutable chain state \u2014 the head pointer (`chain/head.json`) \u2014 that\n * `RestChainStateViewer` reads. This is the one mutable file in the REST layout, so it is\n * stored with must-revalidate caching and should target the chain-state bucket.\n *\n * Callers must publish the finalized files for a head before publishing the head itself,\n * so readers never see a head that references missing files.\n */\n@creatableProvider()\nexport class S3ChainStatePublishRunner extends AbstractS3PublishRunner<S3ChainStatePublishRunnerParams> implements ChainStatePublishRunner {\n static readonly defaultMoniker = ChainStatePublishRunnerMoniker\n static readonly dependencies = []\n static readonly monikers = [ChainStatePublishRunnerMoniker]\n moniker = S3ChainStatePublishRunner.defaultMoniker\n\n get source(): ChainStateViewerMethods {\n return assertEx(this.params.source, () => 'No source specified')\n }\n\n /** Publishes the source's current head to the mutable chain state pointer. */\n async publishHead(): Promise<SignedHydratedBlockWithHashMeta> {\n const head = await this.source.currentBlock()\n await this.putJson(headPath(), JSON.stringify(head), MUTABLE_CACHE_CONTROL)\n this.logger?.info(`S3ChainStatePublishRunner: published head ${head[0].block}`)\n return head\n }\n}\n", "import { assertEx } from '@xylabs/sdk-js'\nimport type {\n BlocksStepSummary, BlockViewer, IndexPublishRunner, XL1BlockNumber,\n} from '@xyo-network/xl1-protocol-lib'\nimport {\n BlocksStepSummarySchema, IndexPublishRunnerMoniker, StepSizes,\n} from '@xyo-network/xl1-protocol-lib'\nimport { blocksMaxStep, creatableProvider } from '@xyo-network/xl1-protocol-sdk'\nimport { blocksStepPath } from '@xyo-network/xl1-rest-block-viewer'\n\nimport type { S3PublishRunnerParams } from './AbstractS3PublishRunner.ts'\nimport { AbstractS3PublishRunner, IMMUTABLE_CACHE_CONTROL } from './AbstractS3PublishRunner.ts'\n\n/** Parameters for S3IndexPublishRunner. */\nexport interface S3IndexPublishRunnerParams extends S3PublishRunnerParams {\n /** The viewer to publish from (only finalized chains should be published). */\n source: BlockViewer\n}\n\n/**\n * Publishes completed steps' block summaries (the chain index) as the static REST file\n * layout that `RestIndexViewer` reads. Step files are immutable: only complete steps are\n * ever published, and a completed step's blocks never change.\n */\n@creatableProvider()\nexport class S3IndexPublishRunner extends AbstractS3PublishRunner<S3IndexPublishRunnerParams> implements IndexPublishRunner {\n static readonly defaultMoniker = IndexPublishRunnerMoniker\n static readonly dependencies = []\n static readonly monikers = [IndexPublishRunnerMoniker]\n moniker = S3IndexPublishRunner.defaultMoniker\n\n get source(): BlockViewer {\n return assertEx(this.params.source, () => 'No source specified')\n }\n\n /**\n * Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).\n * Asserts the step is complete \u2014 partial tail steps are never published.\n */\n async publishStep(stepLevel: number, stepIndex: number): Promise<void> {\n const size = StepSizes[stepLevel]\n assertEx(stepLevel <= blocksMaxStep, () => `publishStep does not support step levels above ${blocksMaxStep} (requested ${stepLevel})`)\n const lastBlockNumber = (stepIndex + 1) * size - 1\n const headNumber = await this.source.currentBlockNumber()\n assertEx(lastBlockNumber <= headNumber, () => `Step ${stepLevel}/${stepIndex} is not complete (head ${headNumber})`)\n const newestFirst = await this.source.blocksByStep(stepLevel, stepIndex)\n const blocks = newestFirst.toReversed()\n const summary: BlocksStepSummary = {\n schema: BlocksStepSummarySchema,\n hash: assertEx(blocks.at(-1), () => `No blocks for step ${stepLevel}/${stepIndex}`)[0]._hash,\n stepSize: size,\n blocks,\n }\n await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary), IMMUTABLE_CACHE_CONTROL)\n this.logger?.info(`S3IndexPublishRunner: published step ${stepLevel}/${stepIndex}`)\n }\n\n /** Publishes every step (at every level) completed by the given block. */\n async publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void> {\n for (let stepLevel = 0; stepLevel <= blocksMaxStep; stepLevel++) {\n const size = StepSizes[stepLevel]\n if ((blockNumber + 1) % size === 0) {\n await this.publishStep(stepLevel, (blockNumber + 1) / size - 1)\n }\n }\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;AACA,SAAS,wBAAwB;AACjC,SAAS,gBAAgB;AAEzB,SAAS,iCAAiC;;;ACJ1C,OAAO,UAAU;AAgBV,SAAS,WAAW,MAAc,UAA4C;AACnF,UAAQ,UAAU;AAAA,IAChB,KAAK,MAAM;AACT,aAAO,EAAE,MAAM,KAAK,mBAAmB,IAAI,GAAG,iBAAiB,KAAK;AAAA,IACtE;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,EAAE,MAAM,KAAK,SAAS,IAAI,GAAG,iBAAiB,OAAO;AAAA,IAC9D;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,EAAE,MAAM,IAAI,YAAY,EAAE,OAAO,IAAI,EAAE;AAAA,IAChD;AAAA,EACF;AACF;AAGO,SAAS,WAAW,MAAkB,iBAAkC;AAC7E,UAAQ,iBAAiB;AAAA,IACvB,KAAK,MAAM;AACT,aAAO,KAAK,qBAAqB,IAAI,EAAE,SAAS,MAAM;AAAA,IACxD;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,KAAK,WAAW,IAAI,EAAE,SAAS,MAAM;AAAA,IAC9C;AAAA,IACA,SAAS;AACP,aAAO,IAAI,YAAY,EAAE,OAAO,IAAI;AAAA,IACtC;AAAA,EACF;AACF;;;ADtBO,IAAM,0BAA0B;AAEhC,IAAM,wBAAwB;AAO9B,IAAe,0BAAf,cACG,0BAAmC;AAAA,EAC3C,IAAI,SAAiB;AACnB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,SAAmB;AACrB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,kBAAuC;AACzC,WAAO,KAAK,OAAO,mBAAmB;AAAA,EACxC;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,OAAO,UAAU;AAAA,EAC/B;AAAA,EAEA,MAAgB,QAAQ,MAAc,MAAc,cAAqC;AACvF,UAAM,EAAE,MAAM,gBAAgB,IAAI,WAAW,MAAM,KAAK,eAAe;AACvE,UAAM,KAAK,OAAO,KAAK,IAAI,iBAAiB;AAAA,MAC1C,QAAQ,KAAK;AAAA,MACb,KAAK,GAAG,KAAK,MAAM,GAAG,IAAI;AAAA,MAC1B,MAAM;AAAA,MACN,cAAc;AAAA,MACd,iBAAiB;AAAA,MACjB,aAAa;AAAA,IACf,CAAC,CAAC;AAAA,EACJ;AACF;;;AE3DA,SAAS,YAAAA,WAAU,cAAc;AAIjC,SAAS,kBAAkB,iCAAiC;AAC5D,SAAS,yBAAyB;AAClC;AAAA,EACE;AAAA,EAAe;AAAA,EAAiB;AAAA,OAC3B;AACP,SAAS,iBAAiB;AAoCnB,IAAM,uBAAN,cAAmC,wBAAkF;AAAA,EAI1H,UAAU,qBAAqB;AAAA,EAE/B,IAAI,cAAsB;AACxB,WAAO,KAAK,OAAO,eAAe;AAAA,EACpC;AAAA,EAEA,IAAI,mBAA2B;AAC7B,WAAO,KAAK,OAAO,oBAAoB;AAAA,EACzC;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAOC,UAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA;AAAA,EAGA,MAAM,aAAa,aAA8E;AAC/F,UAAM,QAAQ,MAAM,KAAK,OAAO,cAAc,WAAW;AACzD,QAAI,OAAO,KAAK,EAAG,QAAO;AAC1B,UAAM,OAAO,KAAK,UAAU,KAAK;AACjC,UAAM,KAAK,QAAQ,gBAAgB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,uBAAuB;AACjF,UAAM,KAAK,QAAQ,cAAc,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,uBAAuB;AAC/E,eAAW,WAAW,MAAM,CAAC,GAAG;AAC9B,YAAM,KAAK,QAAQ,YAAY,QAAQ,KAAK,GAAG,KAAK,UAAU,OAAO,GAAG,uBAAuB;AAAA,IACjG;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,KAAK,OAAuB,iBAAiB,GAAG,IAAI,GAAkC;AAC1F,UAAM,aAAa,MAAM,KAAK,OAAO,aAAa;AAClD,UAAM,MAAM,WAAW,CAAC,EAAE;AAC1B,QAAI,OAAO,IAAK,QAAO;AAEvB,UAAM,YAAY,IAAI,UAAU,KAAK,WAAW;AAChD,UAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,MAAM,OAAO,EAAE,GAAG,CAAC,GAAG,MAAM,iBAAiB,OAAO,GAAG,IAAI,CAAC;AACjG,UAAM,QAAQ,QAAQ;AACtB,QAAI,iBAAiB;AACrB,UAAM,QAAQ,IAAI,QAAQ,IAAI,iBAAe,UAAU,aAAa,YAAY;AAC9E,MAAAA,UAAS,MAAM,KAAK,aAAa,WAAW,GAAG,MAAM,8BAA8B,WAAW,GAAG;AACjG,wBAAkB;AAClB,UAAI,iBAAiB,KAAK,qBAAqB,KAAK,mBAAmB,OAAO;AAC5E,aAAK,OAAO;AAAA,UACV;AAAA,UAAa,WAAW;AAAA,UAAgB;AAAA,QAC1C,CAAC;AAAA,MACH;AAAA,IACF,CAAC,CAAC,CAAC;AACH,WAAO,CAAC,MAAM,GAAG;AAAA,EACnB;AAAA;AAAA,EAGQ,OAAO,OAAmC;AAChD,UAAM,aAAa,KAAK,OAAO;AAC/B,QAAI,YAAY;AACd,iBAAW,KAAK;AAChB;AAAA,IACF;AACA,SAAK,QAAQ,KAAK,yCAAyC,MAAM,WAAW,KAAK,MAAM,SAAS,IAAI,MAAM,KAAK,GAAG;AAAA,EACpH;AACF;AAhEE,cADW,sBACK,kBAAiB;AACjC,cAFW,sBAEK,gBAAe,CAAC;AAChC,cAHW,sBAGK,YAAW,CAAC,yBAAyB;AAH1C,uBAAN;AAAA,EADN,kBAAkB;AAAA,GACN;;;AC7Cb,SAAS,YAAAC,iBAAgB;AAIzB,SAAS,sCAAsC;AAC/C,SAAS,qBAAAC,0BAAyB;AAClC,SAAS,gBAAgB;AAoBlB,IAAM,4BAAN,cAAwC,wBAA4F;AAAA,EAIzI,UAAU,0BAA0B;AAAA,EAEpC,IAAI,SAAkC;AACpC,WAAOC,UAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA;AAAA,EAGA,MAAM,cAAwD;AAC5D,UAAM,OAAO,MAAM,KAAK,OAAO,aAAa;AAC5C,UAAM,KAAK,QAAQ,SAAS,GAAG,KAAK,UAAU,IAAI,GAAG,qBAAqB;AAC1E,SAAK,QAAQ,KAAK,6CAA6C,KAAK,CAAC,EAAE,KAAK,EAAE;AAC9E,WAAO;AAAA,EACT;AACF;AAhBE,cADW,2BACK,kBAAiB;AACjC,cAFW,2BAEK,gBAAe,CAAC;AAChC,cAHW,2BAGK,YAAW,CAAC,8BAA8B;AAH/C,4BAAN;AAAA,EADNC,mBAAkB;AAAA,GACN;;;AC1Bb,SAAS,YAAAC,iBAAgB;AAIzB;AAAA,EACE;AAAA,EAAyB;AAAA,EAA2B;AAAA,OAC/C;AACP,SAAS,eAAe,qBAAAC,0BAAyB;AACjD,SAAS,sBAAsB;AAiBxB,IAAM,uBAAN,cAAmC,wBAAkF;AAAA,EAI1H,UAAU,qBAAqB;AAAA,EAE/B,IAAI,SAAsB;AACxB,WAAOC,UAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,YAAY,WAAmB,WAAkC;AACrE,UAAM,OAAO,UAAU,SAAS;AAChC,IAAAA,UAAS,aAAa,eAAe,MAAM,kDAAkD,aAAa,eAAe,SAAS,GAAG;AACrI,UAAM,mBAAmB,YAAY,KAAK,OAAO;AACjD,UAAM,aAAa,MAAM,KAAK,OAAO,mBAAmB;AACxD,IAAAA,UAAS,mBAAmB,YAAY,MAAM,QAAQ,SAAS,IAAI,SAAS,0BAA0B,UAAU,GAAG;AACnH,UAAM,cAAc,MAAM,KAAK,OAAO,aAAa,WAAW,SAAS;AACvE,UAAM,SAAS,YAAY,WAAW;AACtC,UAAM,UAA6B;AAAA,MACjC,QAAQ;AAAA,MACR,MAAMA,UAAS,OAAO,GAAG,EAAE,GAAG,MAAM,sBAAsB,SAAS,IAAI,SAAS,EAAE,EAAE,CAAC,EAAE;AAAA,MACvF,UAAU;AAAA,MACV;AAAA,IACF;AACA,UAAM,KAAK,QAAQ,eAAe,WAAW,SAAS,GAAG,KAAK,UAAU,OAAO,GAAG,uBAAuB;AACzG,SAAK,QAAQ,KAAK,wCAAwC,SAAS,IAAI,SAAS,EAAE;AAAA,EACpF;AAAA;AAAA,EAGA,MAAM,wBAAwB,aAA4C;AACxE,aAAS,YAAY,GAAG,aAAa,eAAe,aAAa;AAC/D,YAAM,OAAO,UAAU,SAAS;AAChC,WAAK,cAAc,KAAK,SAAS,GAAG;AAClC,cAAM,KAAK,YAAY,YAAY,cAAc,KAAK,OAAO,CAAC;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AACF;AAxCE,cADW,sBACK,kBAAiB;AACjC,cAFW,sBAEK,gBAAe,CAAC;AAChC,cAHW,sBAGK,YAAW,CAAC,yBAAyB;AAH1C,uBAAN;AAAA,EADNC,mBAAkB;AAAA,GACN;",
|
|
6
|
+
"names": ["assertEx", "assertEx", "assertEx", "creatableProvider", "assertEx", "creatableProvider", "assertEx", "creatableProvider", "assertEx", "creatableProvider"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "http://json.schemastore.org/package.json",
|
|
3
3
|
"name": "@xyo-network/xl1-rest-block-publisher",
|
|
4
|
-
"version": "2.1.
|
|
4
|
+
"version": "2.1.3",
|
|
5
5
|
"description": "XYO Layer One static REST chain layout publisher",
|
|
6
6
|
"homepage": "https://xylabs.com",
|
|
7
7
|
"bugs": {
|
|
@@ -35,9 +35,9 @@
|
|
|
35
35
|
"README.md"
|
|
36
36
|
],
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@xyo-network/xl1-protocol-
|
|
39
|
-
"@xyo-network/xl1-protocol-
|
|
40
|
-
"@xyo-network/xl1-rest-block-viewer": "~2.1.
|
|
38
|
+
"@xyo-network/xl1-protocol-lib": "~2.1.3",
|
|
39
|
+
"@xyo-network/xl1-protocol-sdk": "~2.1.3",
|
|
40
|
+
"@xyo-network/xl1-rest-block-viewer": "~2.1.3"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@aws-sdk/client-s3": "^3.1065.0",
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
import type { S3Client } from '@aws-sdk/client-s3';
|
|
2
|
-
import type { CreatableParams } from '@xylabs/sdk-js';
|
|
3
|
-
import { AbstractCreatable } from '@xylabs/sdk-js';
|
|
4
|
-
import type { BlockViewer, SignedHydratedBlockWithHashMeta, XL1BlockNumber, XL1BlockRange } from '@xyo-network/xl1-protocol-lib';
|
|
5
|
-
import type { RestContentEncoding } from './encoding.ts';
|
|
6
|
-
/** A progress event emitted while publishing. */
|
|
7
|
-
export type RestPublishEvent = {
|
|
8
|
-
blockNumber: XL1BlockNumber;
|
|
9
|
-
published: number;
|
|
10
|
-
total: number;
|
|
11
|
-
type: 'block';
|
|
12
|
-
} | {
|
|
13
|
-
stepIndex: number;
|
|
14
|
-
stepLevel: number;
|
|
15
|
-
type: 'step';
|
|
16
|
-
};
|
|
17
|
-
/** Parameters for RestBlockPublisher. */
|
|
18
|
-
export interface RestBlockPublisherParams extends CreatableParams {
|
|
19
|
-
bucket: string;
|
|
20
|
-
/** S3-compatible client pointed at the target endpoint (e.g. Cloudflare R2). */
|
|
21
|
-
client: S3Client;
|
|
22
|
-
/** Maximum concurrent block publishes during sync. */
|
|
23
|
-
concurrency?: number;
|
|
24
|
-
/** Storage/wire encoding for published files. Defaults to 'br' (brotli). */
|
|
25
|
-
contentEncoding?: RestContentEncoding;
|
|
26
|
-
/**
|
|
27
|
-
* Called as publishing progresses: every `progressInterval` blocks (and at pass completion)
|
|
28
|
-
* and per step file. When omitted, the same events are logged via the params logger so
|
|
29
|
-
* long backfills are observable out of the box.
|
|
30
|
-
*/
|
|
31
|
-
onProgress?: (event: RestPublishEvent) => void;
|
|
32
|
-
/** Optional key prefix, allowing the layout to live in a shared bucket. */
|
|
33
|
-
prefix?: string;
|
|
34
|
-
/** How many block publishes between progress reports during sync. */
|
|
35
|
-
progressInterval?: number;
|
|
36
|
-
/** The viewer to publish from (only finalized chains should be published). */
|
|
37
|
-
source: BlockViewer;
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Publishes a chain's immutable files as the static REST file layout that `RestBlockViewer`
|
|
41
|
-
* reads (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).
|
|
42
|
-
*
|
|
43
|
-
* The mutable chain state (the head pointer, `chain/head.json`) is NOT written here — it
|
|
44
|
-
* lives in a separate chain-state bucket and is written by the chain's finalizer after the
|
|
45
|
-
* finalized files for that head exist, so readers never see a head that references missing
|
|
46
|
-
* files.
|
|
47
|
-
*
|
|
48
|
-
* Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing
|
|
49
|
-
* any range is safe. `sync(from)` resumes from a caller-supplied cursor (typically the block
|
|
50
|
-
* after the last published head); omitting it republishes from genesis, which is safe.
|
|
51
|
-
*/
|
|
52
|
-
export declare class RestBlockPublisher extends AbstractCreatable<RestBlockPublisherParams> {
|
|
53
|
-
get bucket(): string;
|
|
54
|
-
get client(): S3Client;
|
|
55
|
-
get concurrency(): number;
|
|
56
|
-
get contentEncoding(): RestContentEncoding;
|
|
57
|
-
get prefix(): string;
|
|
58
|
-
get progressInterval(): number;
|
|
59
|
-
get source(): BlockViewer;
|
|
60
|
-
/** Publishes one block at its by-number and by-hash paths, plus its payload files. */
|
|
61
|
-
publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null>;
|
|
62
|
-
/**
|
|
63
|
-
* Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).
|
|
64
|
-
* Asserts the step is complete — partial tail steps are never published.
|
|
65
|
-
*/
|
|
66
|
-
publishStep(stepLevel: number, stepIndex: number): Promise<void>;
|
|
67
|
-
/** Publishes every step (at every level) completed by the given block. */
|
|
68
|
-
publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void>;
|
|
69
|
-
/**
|
|
70
|
-
* Publishes everything from `from` (inclusive, default genesis) to the source head
|
|
71
|
-
* (inclusive), then the completed steps in that range. Returns the published range, or
|
|
72
|
-
* null when already up to date.
|
|
73
|
-
*
|
|
74
|
-
* The caller supplies the resume cursor — typically the block after the last published
|
|
75
|
-
* head pointer, which is written by the finalizer rather than this publisher. Omitting
|
|
76
|
-
* `from` republishes from genesis, which is safe (idempotent) but slow.
|
|
77
|
-
*/
|
|
78
|
-
sync(from?: XL1BlockNumber): Promise<XL1BlockRange | null>;
|
|
79
|
-
private putJson;
|
|
80
|
-
/** Emits a progress event to the onProgress callback, or logs it when no callback is set. */
|
|
81
|
-
private report;
|
|
82
|
-
}
|
|
83
|
-
//# sourceMappingURL=RestBlockPublisher.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"RestBlockPublisher.d.ts","sourceRoot":"","sources":["../../src/RestBlockPublisher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,EACL,iBAAiB,EAClB,MAAM,gBAAgB,CAAA;AACvB,OAAO,KAAK,EACV,WAAW,EAAE,+BAA+B,EAAE,cAAc,EAAE,aAAa,EAC5E,MAAM,+BAA+B,CAAA;AAStC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AAGxD,iDAAiD;AACjD,MAAM,MAAM,gBAAgB,GACtB;IAAE,WAAW,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAChF;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAE5D,yCAAyC;AACzC,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,MAAM,EAAE,QAAQ,CAAA;IAChB,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,mBAAmB,CAAA;IACrC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAA;IAC9C,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,8EAA8E;IAC9E,MAAM,EAAE,WAAW,CAAA;CACpB;AAKD;;;;;;;;;;;;GAYG;AACH,qBACa,kBAAmB,SAAQ,iBAAiB,CAAC,wBAAwB,CAAC;IACjF,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,MAAM,IAAI,QAAQ,CAErB;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,eAAe,IAAI,mBAAmB,CAEzC;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,sFAAsF;IAChF,YAAY,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,+BAA+B,GAAG,IAAI,CAAC;IAYhG;;;OAGG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBtE,0EAA0E;IACpE,uBAAuB,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE;;;;;;;;OAQG;IACG,IAAI,CAAC,IAAI,GAAE,cAA0C,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;YAwB7E,OAAO;IAYrB,6FAA6F;IAC7F,OAAO,CAAC,MAAM;CAiBf"}
|