@xyo-network/xl1-rest-block-publisher 2.1.0 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -5,21 +5,22 @@
5
5
  [![npm-badge][]][npm-link]
6
6
  [![license-badge][]][license-link]
7
7
 
8
- > Publishes a finalized XL1 chain as the static REST file layout that `RestBlockViewer` reads — pre-compressed, immutable, CDN-friendly.
8
+ > Publishes a finalized XL1 chain's immutable files as the static REST file layout that `RestBlockViewer` reads — pre-compressed, immutable, CDN-friendly.
9
9
 
10
10
  ## About
11
11
 
12
- Finalized blocks and completed steps never change, so the chain can be materialized once as static files on S3-compatible object storage (e.g. Cloudflare R2) and served forever over plain HTTP. `RestBlockPublisher` writes that layout from any source `BlockViewer`:
12
+ Finalized blocks and completed steps never change, so the chain can be materialized once as static files on S3-compatible object storage (e.g. Cloudflare R2) and served forever over plain HTTP. `RestBlockPublisher` writes the finalized (immutable) part of that layout from any source `BlockViewer`:
13
13
 
14
14
  ```text
15
- /chain/head.json ← written last on every sync (must-revalidate)
16
15
  /block/number/<n>.json ← immutable
17
16
  /block/hash/<hash>.json ← immutable, same content
18
17
  /blocks/step/<level>/<index>.json ← immutable, completed steps only
19
18
  /payload/<hash>.json ← immutable
20
19
  ```
21
20
 
22
- Bodies are pre-compressed at write time (brotli by default, gzip or none configurable) and stored with the matching `Content-Encoding` plus `application/json`, so HTTP clients decompress transparently keys keep their `.json` extension. Immutable files get `Cache-Control: public, max-age=31536000, immutable`; the head pointer gets `must-revalidate`.
21
+ The mutable chain state (the head pointer, `/chain/head.json`) is **not** written by this publisher it lives in a separate chain-state bucket and is written by the chain's finalizer (the `FinalizerActor` in xyo-chain) after the finalized files for that head exist, so readers never see a head that references missing files.
22
+
23
+ Bodies are pre-compressed at write time (brotli by default, gzip or none configurable) and stored with the matching `Content-Encoding` plus `application/json`, so HTTP clients decompress transparently — keys keep their `.json` extension. Everything written gets `Cache-Control: public, max-age=31536000, immutable`.
23
24
 
24
25
  ```ts
25
26
  import { S3Client } from '@aws-sdk/client-s3'
@@ -32,10 +33,10 @@ const client = new S3Client({
32
33
  })
33
34
 
34
35
  const publisher = await RestBlockPublisher.create({ bucket: 'chain', client, source: blockViewer })
35
- const range = await publisher.sync() // publishes everything new, head last; null when up to date
36
+ const range = await publisher.sync(from) // publishes [from, sourceHead]; null when up to date
36
37
  ```
37
38
 
38
- `sync()` resumes from the published head pointer, so the same call serves both initial backfill and incremental tailing. Publishing is idempotent — deterministic keys, immutable content so re-running after a crash is always safe. The path builders come from [`@xyo-network/xl1-rest-block-viewer`](https://www.npmjs.com/package/@xyo-network/xl1-rest-block-viewer), so reader and writer share one layout contract.
39
+ `sync(from)` resumes from a caller-supplied cursor — typically the block after the last published head pointer, which the finalizer owns. Omitting `from` republishes from genesis, which is safe (idempotent — deterministic keys, immutable content) but slow, so re-running after a crash is always safe. The path builders come from [`@xyo-network/xl1-rest-block-viewer`](https://www.npmjs.com/package/@xyo-network/xl1-rest-block-viewer), so reader and writer share one layout contract.
39
40
 
40
41
  ## Install
41
42
 
@@ -67,7 +68,7 @@ bun add @xyo-network/xl1-rest-block-publisher
67
68
 
68
69
  ## What's Inside
69
70
 
70
- - **`RestBlockPublisher`** — `sync()` (cursor-resumed publish, head written last), `publishBlock`, `publishStep`/`publishStepsCompletedBy` (completed steps only), `publishHead`, `publishedHead`.
71
+ - **`RestBlockPublisher`** — `sync(from)` (cursor-resumed publish), `publishBlock`, `publishStep`/`publishStepsCompletedBy` (completed steps only).
71
72
  - **`encodeBody`/`decodeBody`** — brotli/gzip/none body codecs used for storage and read-back.
72
73
 
73
74
  ## Building Locally
@@ -13,9 +13,6 @@ export type RestPublishEvent = {
13
13
  stepIndex: number;
14
14
  stepLevel: number;
15
15
  type: 'step';
16
- } | {
17
- blockNumber: XL1BlockNumber;
18
- type: 'head';
19
16
  };
20
17
  /** Parameters for RestBlockPublisher. */
21
18
  export interface RestBlockPublisherParams extends CreatableParams {
@@ -27,9 +24,9 @@ export interface RestBlockPublisherParams extends CreatableParams {
27
24
  /** Storage/wire encoding for published files. Defaults to 'br' (brotli). */
28
25
  contentEncoding?: RestContentEncoding;
29
26
  /**
30
- * Called as publishing progresses: every `progressInterval` blocks (and at pass completion),
31
- * per step file, and on the head write. When omitted, the same events are logged via the
32
- * params logger so long backfills are observable out of the box.
27
+ * Called as publishing progresses: every `progressInterval` blocks (and at pass completion)
28
+ * and per step file. When omitted, the same events are logged via the params logger so
29
+ * long backfills are observable out of the box.
33
30
  */
34
31
  onProgress?: (event: RestPublishEvent) => void;
35
32
  /** Optional key prefix, allowing the layout to live in a shared bucket. */
@@ -40,12 +37,17 @@ export interface RestBlockPublisherParams extends CreatableParams {
40
37
  source: BlockViewer;
41
38
  }
42
39
  /**
43
- * Publishes a chain as the static REST file layout that `RestBlockViewer` reads
44
- * (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).
40
+ * Publishes a chain's immutable files as the static REST file layout that `RestBlockViewer`
41
+ * reads (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).
42
+ *
43
+ * The mutable chain state (the head pointer, `chain/head.json`) is NOT written here — it
44
+ * lives in a separate chain-state bucket and is written by the chain's finalizer after the
45
+ * finalized files for that head exist, so readers never see a head that references missing
46
+ * files.
45
47
  *
46
48
  * Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing
47
- * any range is safe. `sync()` writes the head pointer last readers never see a head that
48
- * references files that do not exist yet and uses the published head as its resume cursor.
49
+ * any range is safe. `sync(from)` resumes from a caller-supplied cursor (typically the block
50
+ * after the last published head); omitting it republishes from genesis, which is safe.
49
51
  */
50
52
  export declare class RestBlockPublisher extends AbstractCreatable<RestBlockPublisherParams> {
51
53
  get bucket(): string;
@@ -57,8 +59,6 @@ export declare class RestBlockPublisher extends AbstractCreatable<RestBlockPubli
57
59
  get source(): BlockViewer;
58
60
  /** Publishes one block at its by-number and by-hash paths, plus its payload files. */
59
61
  publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null>;
60
- /** Publishes the current source head to the mutable head pointer. */
61
- publishHead(): Promise<SignedHydratedBlockWithHashMeta>;
62
62
  /**
63
63
  * Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).
64
64
  * Asserts the step is complete — partial tail steps are never published.
@@ -66,14 +66,16 @@ export declare class RestBlockPublisher extends AbstractCreatable<RestBlockPubli
66
66
  publishStep(stepLevel: number, stepIndex: number): Promise<void>;
67
67
  /** Publishes every step (at every level) completed by the given block. */
68
68
  publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void>;
69
- /** Reads the published head pointer back from the bucket, or undefined if absent. */
70
- publishedHead(): Promise<SignedHydratedBlockWithHashMeta | undefined>;
71
69
  /**
72
- * Publishes everything from the published head (exclusive) to the source head (inclusive),
73
- * then the completed steps in that range, then the head pointer last. Returns the published
74
- * range, or null when already up to date.
70
+ * Publishes everything from `from` (inclusive, default genesis) to the source head
71
+ * (inclusive), then the completed steps in that range. Returns the published range, or
72
+ * null when already up to date.
73
+ *
74
+ * The caller supplies the resume cursor — typically the block after the last published
75
+ * head pointer, which is written by the finalizer rather than this publisher. Omitting
76
+ * `from` republishes from genesis, which is safe (idempotent) but slow.
75
77
  */
76
- sync(): Promise<XL1BlockRange | null>;
78
+ sync(from?: XL1BlockNumber): Promise<XL1BlockRange | null>;
77
79
  private putJson;
78
80
  /** Emits a progress event to the onProgress callback, or logs it when no callback is set. */
79
81
  private report;
@@ -1 +1 @@
1
- {"version":3,"file":"RestBlockPublisher.d.ts","sourceRoot":"","sources":["../../src/RestBlockPublisher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,EACL,iBAAiB,EAClB,MAAM,gBAAgB,CAAA;AACvB,OAAO,KAAK,EACV,WAAW,EAAE,+BAA+B,EAAE,cAAc,EAAE,aAAa,EAC5E,MAAM,+BAA+B,CAAA;AAWtC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AAGxD,iDAAiD;AACjD,MAAM,MAAM,gBAAgB,GACtB;IAAE,WAAW,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAChF;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACtD;IAAE,WAAW,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAEnD,yCAAyC;AACzC,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,MAAM,EAAE,QAAQ,CAAA;IAChB,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,mBAAmB,CAAA;IACrC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAA;IAC9C,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,8EAA8E;IAC9E,MAAM,EAAE,WAAW,CAAA;CACpB;AAOD;;;;;;;GAOG;AACH,qBACa,kBAAmB,SAAQ,iBAAiB,CAAC,wBAAwB,CAAC;IACjF,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,MAAM,IAAI,QAAQ,CAErB;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,eAAe,IAAI,mBAAmB,CAEzC;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,sFAAsF;IAChF,YAAY,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,+BAA+B,GAAG,IAAI,CAAC;IAYhG,qEAAqE;IAC/D,WAAW,IAAI,OAAO,CAAC,+BAA+B,CAAC;IAO7D;;;OAGG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBtE,0EAA0E;IACpE,uBAAuB,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE,qFAAqF;IAC/E,aAAa,IAAI,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC;IAa3E;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;YA2B7B,OAAO;IAYrB,6FAA6F;IAC7F,OAAO,CAAC,MAAM;CAqBf"}
1
+ {"version":3,"file":"RestBlockPublisher.d.ts","sourceRoot":"","sources":["../../src/RestBlockPublisher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,EACL,iBAAiB,EAClB,MAAM,gBAAgB,CAAA;AACvB,OAAO,KAAK,EACV,WAAW,EAAE,+BAA+B,EAAE,cAAc,EAAE,aAAa,EAC5E,MAAM,+BAA+B,CAAA;AAStC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AAGxD,iDAAiD;AACjD,MAAM,MAAM,gBAAgB,GACtB;IAAE,WAAW,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAChF;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAE5D,yCAAyC;AACzC,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,MAAM,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,MAAM,EAAE,QAAQ,CAAA;IAChB,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,mBAAmB,CAAA;IACrC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAA;IAC9C,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,8EAA8E;IAC9E,MAAM,EAAE,WAAW,CAAA;CACpB;AAKD;;;;;;;;;;;;GAYG;AACH,qBACa,kBAAmB,SAAQ,iBAAiB,CAAC,wBAAwB,CAAC;IACjF,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,MAAM,IAAI,QAAQ,CAErB;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,eAAe,IAAI,mBAAmB,CAEzC;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,sFAAsF;IAChF,YAAY,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,+BAA+B,GAAG,IAAI,CAAC;IAYhG;;;OAGG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBtE,0EAA0E;IACpE,uBAAuB,CAAC,WAAW,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE;;;;;;;;OAQG;IACG,IAAI,CAAC,IAAI,GAAE,cAA0C,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;YAwB7E,OAAO;IAYrB,6FAA6F;IAC7F,OAAO,CAAC,MAAM;CAiBf"}
@@ -39,29 +39,23 @@ function decodeBody(body, contentEncoding) {
39
39
  }
40
40
 
41
41
  // src/RestBlockPublisher.ts
42
- import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
42
+ import { PutObjectCommand } from "@aws-sdk/client-s3";
43
43
  import {
44
44
  AbstractCreatable,
45
45
  assertEx,
46
46
  creatable,
47
47
  isNull
48
48
  } from "@xylabs/sdk-js";
49
- import {
50
- asSignedHydratedBlockWithHashMeta,
51
- asXL1BlockNumber,
52
- StepSizes
53
- } from "@xyo-network/xl1-protocol-lib";
49
+ import { asXL1BlockNumber, StepSizes } from "@xyo-network/xl1-protocol-lib";
54
50
  import { blocksMaxStep, BlocksStepSummarySchema } from "@xyo-network/xl1-protocol-sdk";
55
51
  import {
56
52
  blockHashPath,
57
53
  blockNumberPath,
58
54
  blocksStepPath,
59
- headPath,
60
55
  payloadPath
61
56
  } from "@xyo-network/xl1-rest-block-viewer";
62
57
  import { Semaphore } from "async-mutex";
63
58
  var IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable";
64
- var HEAD_CACHE_CONTROL = "public, max-age=0, must-revalidate";
65
59
  var RestBlockPublisher = class extends AbstractCreatable {
66
60
  get bucket() {
67
61
  return assertEx(this.params.bucket, () => "No bucket specified");
@@ -89,20 +83,13 @@ var RestBlockPublisher = class extends AbstractCreatable {
89
83
  const block = await this.source.blockByNumber(blockNumber);
90
84
  if (isNull(block)) return null;
91
85
  const json = JSON.stringify(block);
92
- await this.putJson(blockNumberPath(block[0].block), json, IMMUTABLE_CACHE_CONTROL);
93
- await this.putJson(blockHashPath(block[0]._hash), json, IMMUTABLE_CACHE_CONTROL);
86
+ await this.putJson(blockNumberPath(block[0].block), json);
87
+ await this.putJson(blockHashPath(block[0]._hash), json);
94
88
  for (const payload of block[1]) {
95
- await this.putJson(payloadPath(payload._hash), JSON.stringify(payload), IMMUTABLE_CACHE_CONTROL);
89
+ await this.putJson(payloadPath(payload._hash), JSON.stringify(payload));
96
90
  }
97
91
  return block;
98
92
  }
99
- /** Publishes the current source head to the mutable head pointer. */
100
- async publishHead() {
101
- const head = await this.source.currentBlock();
102
- await this.putJson(headPath(), JSON.stringify(head), HEAD_CACHE_CONTROL);
103
- this.report({ blockNumber: head[0].block, type: "head" });
104
- return head;
105
- }
106
93
  /**
107
94
  * Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).
108
95
  * Asserts the step is complete — partial tail steps are never published.
@@ -121,7 +108,7 @@ var RestBlockPublisher = class extends AbstractCreatable {
121
108
  stepSize: size,
122
109
  blocks
123
110
  };
124
- await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary), IMMUTABLE_CACHE_CONTROL);
111
+ await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary));
125
112
  this.report({
126
113
  stepIndex,
127
114
  stepLevel,
@@ -137,32 +124,21 @@ var RestBlockPublisher = class extends AbstractCreatable {
137
124
  }
138
125
  }
139
126
  }
140
- /** Reads the published head pointer back from the bucket, or undefined if absent. */
141
- async publishedHead() {
142
- try {
143
- const response = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: `${this.prefix}${headPath()}` }));
144
- const body = await response.Body?.transformToByteArray();
145
- if (body === void 0) return void 0;
146
- const parsed = JSON.parse(decodeBody(body, response.ContentEncoding));
147
- return asSignedHydratedBlockWithHashMeta(parsed, true);
148
- } catch (error) {
149
- if (isNotFoundError(error)) return void 0;
150
- throw error;
151
- }
152
- }
153
127
  /**
154
- * Publishes everything from the published head (exclusive) to the source head (inclusive),
155
- * then the completed steps in that range, then the head pointer last. Returns the published
156
- * range, or null when already up to date.
128
+ * Publishes everything from `from` (inclusive, default genesis) to the source head
129
+ * (inclusive), then the completed steps in that range. Returns the published range, or
130
+ * null when already up to date.
131
+ *
132
+ * The caller supplies the resume cursor — typically the block after the last published
133
+ * head pointer, which is written by the finalizer rather than this publisher. Omitting
134
+ * `from` republishes from genesis, which is safe (idempotent) but slow.
157
135
  */
158
- async sync() {
136
+ async sync(from = asXL1BlockNumber(0, true)) {
159
137
  const sourceHead = await this.source.currentBlock();
160
- const published = await this.publishedHead();
161
- const start = published === void 0 ? 0 : published[0].block + 1;
162
138
  const end = sourceHead[0].block;
163
- if (start > end) return null;
139
+ if (from > end) return null;
164
140
  const semaphore = new Semaphore(this.concurrency);
165
- const numbers = Array.from({ length: end - start + 1 }, (_, i) => asXL1BlockNumber(start + i, true));
141
+ const numbers = Array.from({ length: end - from + 1 }, (_, i) => asXL1BlockNumber(from + i, true));
166
142
  const total = numbers.length;
167
143
  let publishedCount = 0;
168
144
  await Promise.all(numbers.map((blockNumber) => semaphore.runExclusive(async () => {
@@ -180,16 +156,15 @@ var RestBlockPublisher = class extends AbstractCreatable {
180
156
  for (const blockNumber of numbers) {
181
157
  await this.publishStepsCompletedBy(blockNumber);
182
158
  }
183
- await this.publishHead();
184
- return [asXL1BlockNumber(start, true), end];
159
+ return [from, end];
185
160
  }
186
- async putJson(path, json, cacheControl) {
161
+ async putJson(path, json) {
187
162
  const { body, contentEncoding } = encodeBody(json, this.contentEncoding);
188
163
  await this.client.send(new PutObjectCommand({
189
164
  Bucket: this.bucket,
190
165
  Key: `${this.prefix}${path}`,
191
166
  Body: body,
192
- CacheControl: cacheControl,
167
+ CacheControl: IMMUTABLE_CACHE_CONTROL,
193
168
  ContentEncoding: contentEncoding,
194
169
  ContentType: "application/json"
195
170
  }));
@@ -210,21 +185,12 @@ var RestBlockPublisher = class extends AbstractCreatable {
210
185
  this.logger?.info(`RestBlockPublisher: published step ${event.stepLevel}/${event.stepIndex}`);
211
186
  break;
212
187
  }
213
- case "head": {
214
- this.logger?.info(`RestBlockPublisher: published head ${event.blockNumber}`);
215
- break;
216
- }
217
188
  }
218
189
  }
219
190
  };
220
191
  RestBlockPublisher = __decorateClass([
221
192
  creatable()
222
193
  ], RestBlockPublisher);
223
- var isNotFoundError = (error) => {
224
- if (typeof error !== "object" || error === null) return false;
225
- const { name, $metadata } = error;
226
- return name === "NoSuchKey" || name === "NotFound" || $metadata?.httpStatusCode === 404;
227
- };
228
194
  export {
229
195
  RestBlockPublisher,
230
196
  decodeBody,
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../src/encoding.ts", "../../src/RestBlockPublisher.ts"],
4
- "sourcesContent": ["import ZLIB from 'node:zlib'\n\n/** Wire/storage encoding for published files. */\nexport type RestContentEncoding = 'br' | 'gzip' | 'none'\n\n/** An encoded file body plus the HTTP Content-Encoding it should be served with. */\nexport interface EncodedBody {\n body: Uint8Array\n contentEncoding?: 'br' | 'gzip'\n}\n\n/**\n * Encodes a JSON string for storage. Bodies are pre-compressed at write time and stored\n * with the matching Content-Encoding so HTTP clients decompress transparently \u2014 keys keep\n * their .json extension and Content-Type stays application/json.\n */\nexport function encodeBody(json: string, encoding: RestContentEncoding): EncodedBody {\n switch (encoding) {\n case 'br': {\n return { body: ZLIB.brotliCompressSync(json), contentEncoding: 'br' }\n }\n case 'gzip': {\n return { body: ZLIB.gzipSync(json), contentEncoding: 'gzip' }\n }\n case 'none': {\n return { body: new TextEncoder().encode(json) }\n }\n }\n}\n\n/** Decodes a stored body back to its JSON string using the object's Content-Encoding. */\nexport function decodeBody(body: Uint8Array, contentEncoding?: string): string {\n switch (contentEncoding) {\n case 'br': {\n return ZLIB.brotliDecompressSync(body).toString('utf8')\n }\n case 'gzip': {\n return ZLIB.gunzipSync(body).toString('utf8')\n }\n default: {\n return new TextDecoder().decode(body)\n }\n }\n}\n", "import type { S3Client } from '@aws-sdk/client-s3'\nimport { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'\nimport type { CreatableParams } from '@xylabs/sdk-js'\nimport {\n AbstractCreatable, assertEx, creatable, isNull,\n} from '@xylabs/sdk-js'\nimport type {\n BlockViewer, SignedHydratedBlockWithHashMeta, XL1BlockNumber, XL1BlockRange,\n} from '@xyo-network/xl1-protocol-lib'\nimport {\n asSignedHydratedBlockWithHashMeta, asXL1BlockNumber, StepSizes,\n} from '@xyo-network/xl1-protocol-lib'\nimport type { BlocksStepSummary } from '@xyo-network/xl1-protocol-sdk'\nimport { blocksMaxStep, BlocksStepSummarySchema } from '@xyo-network/xl1-protocol-sdk'\nimport {\n blockHashPath, blockNumberPath, blocksStepPath, headPath, payloadPath,\n} from '@xyo-network/xl1-rest-block-viewer'\nimport { Semaphore } from 'async-mutex'\n\nimport type { RestContentEncoding } from './encoding.ts'\nimport { decodeBody, encodeBody } from './encoding.ts'\n\n/** A progress event emitted while publishing. */\nexport type RestPublishEvent\n = | { blockNumber: XL1BlockNumber; published: number; total: number; type: 'block' }\n | { stepIndex: number; stepLevel: number; type: 'step' }\n | { blockNumber: XL1BlockNumber; type: 'head' }\n\n/** Parameters for RestBlockPublisher. */\nexport interface RestBlockPublisherParams extends CreatableParams {\n bucket: string\n /** S3-compatible client pointed at the target endpoint (e.g. Cloudflare R2). */\n client: S3Client\n /** Maximum concurrent block publishes during sync. */\n concurrency?: number\n /** Storage/wire encoding for published files. Defaults to 'br' (brotli). */\n contentEncoding?: RestContentEncoding\n /**\n * Called as publishing progresses: every `progressInterval` blocks (and at pass completion),\n * per step file, and on the head write. When omitted, the same events are logged via the\n * params logger so long backfills are observable out of the box.\n */\n onProgress?: (event: RestPublishEvent) => void\n /** Optional key prefix, allowing the layout to live in a shared bucket. */\n prefix?: string\n /** How many block publishes between progress reports during sync. */\n progressInterval?: number\n /** The viewer to publish from (only finalized chains should be published). */\n source: BlockViewer\n}\n\n/** Everything except the head pointer is immutable once written. */\nconst IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'\n/** The head pointer is rewritten as the chain advances. */\nconst HEAD_CACHE_CONTROL = 'public, max-age=0, must-revalidate'\n\n/**\n * Publishes a chain as the static REST file layout that `RestBlockViewer` reads\n * (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).\n *\n * Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing\n * any range is safe. `sync()` writes the head pointer last \u2014 readers never see a head that\n * references files that do not exist yet \u2014 and uses the published head as its resume cursor.\n */\n@creatable()\nexport class RestBlockPublisher extends AbstractCreatable<RestBlockPublisherParams> {\n get bucket(): string {\n return assertEx(this.params.bucket, () => 'No bucket specified')\n }\n\n get client(): S3Client {\n return assertEx(this.params.client, () => 'No client specified')\n }\n\n get concurrency(): number {\n return this.params.concurrency ?? 8\n }\n\n get contentEncoding(): RestContentEncoding {\n return this.params.contentEncoding ?? 'br'\n }\n\n get prefix(): string {\n return this.params.prefix ?? ''\n }\n\n get progressInterval(): number {\n return this.params.progressInterval ?? 100\n }\n\n get source(): BlockViewer {\n return assertEx(this.params.source, () => 'No source specified')\n }\n\n /** Publishes one block at its by-number and by-hash paths, plus its payload files. */\n async publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null> {\n const block = await this.source.blockByNumber(blockNumber)\n if (isNull(block)) return null\n const json = JSON.stringify(block)\n await this.putJson(blockNumberPath(block[0].block), json, IMMUTABLE_CACHE_CONTROL)\n await this.putJson(blockHashPath(block[0]._hash), json, IMMUTABLE_CACHE_CONTROL)\n for (const payload of block[1]) {\n await this.putJson(payloadPath(payload._hash), JSON.stringify(payload), IMMUTABLE_CACHE_CONTROL)\n }\n return block\n }\n\n /** Publishes the current source head to the mutable head pointer. */\n async publishHead(): Promise<SignedHydratedBlockWithHashMeta> {\n const head = await this.source.currentBlock()\n await this.putJson(headPath(), JSON.stringify(head), HEAD_CACHE_CONTROL)\n this.report({ blockNumber: head[0].block, type: 'head' })\n return head\n }\n\n /**\n * Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).\n * Asserts the step is complete \u2014 partial tail steps are never published.\n */\n async publishStep(stepLevel: number, stepIndex: number): Promise<void> {\n const size = StepSizes[stepLevel]\n assertEx(stepLevel <= blocksMaxStep, () => `publishStep does not support step levels above ${blocksMaxStep} (requested ${stepLevel})`)\n const lastBlockNumber = (stepIndex + 1) * size - 1\n const headNumber = await this.source.currentBlockNumber()\n assertEx(lastBlockNumber <= headNumber, () => `Step ${stepLevel}/${stepIndex} is not complete (head ${headNumber})`)\n const newestFirst = await this.source.blocksByStep(stepLevel, stepIndex)\n const blocks = newestFirst.toReversed()\n const summary: BlocksStepSummary = {\n schema: BlocksStepSummarySchema,\n hash: assertEx(blocks.at(-1), () => `No blocks for step ${stepLevel}/${stepIndex}`)[0]._hash,\n stepSize: size,\n blocks,\n }\n await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary), IMMUTABLE_CACHE_CONTROL)\n this.report({\n stepIndex, stepLevel, type: 'step',\n })\n }\n\n /** Publishes every step (at every level) completed by the given block. */\n async publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void> {\n for (let stepLevel = 0; stepLevel <= blocksMaxStep; stepLevel++) {\n const size = StepSizes[stepLevel]\n if ((blockNumber + 1) % size === 0) {\n await this.publishStep(stepLevel, (blockNumber + 1) / size - 1)\n }\n }\n }\n\n /** Reads the published head pointer back from the bucket, or undefined if absent. */\n async publishedHead(): Promise<SignedHydratedBlockWithHashMeta | undefined> {\n try {\n const response = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: `${this.prefix}${headPath()}` }))\n const body = await response.Body?.transformToByteArray()\n if (body === undefined) return undefined\n const parsed: unknown = JSON.parse(decodeBody(body, response.ContentEncoding))\n return asSignedHydratedBlockWithHashMeta(parsed, true)\n } catch (error) {\n if (isNotFoundError(error)) return undefined\n throw error\n }\n }\n\n /**\n * Publishes everything from the published head (exclusive) to the source head (inclusive),\n * then the completed steps in that range, then the head pointer last. Returns the published\n * range, or null when already up to date.\n */\n async sync(): Promise<XL1BlockRange | null> {\n const sourceHead = await this.source.currentBlock()\n const published = await this.publishedHead()\n const start = published === undefined ? 0 : published[0].block + 1\n const end = sourceHead[0].block\n if (start > end) return null\n\n const semaphore = new Semaphore(this.concurrency)\n const numbers = Array.from({ length: end - start + 1 }, (_, i) => asXL1BlockNumber(start + i, true))\n const total = numbers.length\n let publishedCount = 0\n await Promise.all(numbers.map(blockNumber => semaphore.runExclusive(async () => {\n assertEx(await this.publishBlock(blockNumber), () => `Block not found in source [${blockNumber}]`)\n publishedCount += 1\n if (publishedCount % this.progressInterval === 0 || publishedCount === total) {\n this.report({\n blockNumber, published: publishedCount, total, type: 'block',\n })\n }\n })))\n for (const blockNumber of numbers) {\n await this.publishStepsCompletedBy(blockNumber)\n }\n await this.publishHead()\n return [asXL1BlockNumber(start, true), end]\n }\n\n private async putJson(path: string, json: string, cacheControl: string): Promise<void> {\n const { body, contentEncoding } = encodeBody(json, this.contentEncoding)\n await this.client.send(new PutObjectCommand({\n Bucket: this.bucket,\n Key: `${this.prefix}${path}`,\n Body: body,\n CacheControl: cacheControl,\n ContentEncoding: contentEncoding,\n ContentType: 'application/json',\n }))\n }\n\n /** Emits a progress event to the onProgress callback, or logs it when no callback is set. */\n private report(event: RestPublishEvent): void {\n const onProgress = this.params.onProgress\n if (onProgress) {\n onProgress(event)\n return\n }\n switch (event.type) {\n case 'block': {\n this.logger?.info(`RestBlockPublisher: published block ${event.blockNumber} (${event.published}/${event.total})`)\n break\n }\n case 'step': {\n this.logger?.info(`RestBlockPublisher: published step ${event.stepLevel}/${event.stepIndex}`)\n break\n }\n case 'head': {\n this.logger?.info(`RestBlockPublisher: published head ${event.blockNumber}`)\n break\n }\n }\n }\n}\n\nconst isNotFoundError = (error: unknown): boolean => {\n if (typeof error !== 'object' || error === null) return false\n const { name, $metadata } = error as { $metadata?: { httpStatusCode?: number }; name?: string }\n return name === 'NoSuchKey' || name === 'NotFound' || $metadata?.httpStatusCode === 404\n}\n"],
5
- "mappings": ";;;;;;;;;;;;AAAA,OAAO,UAAU;AAgBV,SAAS,WAAW,MAAc,UAA4C;AACnF,UAAQ,UAAU;AAAA,IAChB,KAAK,MAAM;AACT,aAAO,EAAE,MAAM,KAAK,mBAAmB,IAAI,GAAG,iBAAiB,KAAK;AAAA,IACtE;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,EAAE,MAAM,KAAK,SAAS,IAAI,GAAG,iBAAiB,OAAO;AAAA,IAC9D;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,EAAE,MAAM,IAAI,YAAY,EAAE,OAAO,IAAI,EAAE;AAAA,IAChD;AAAA,EACF;AACF;AAGO,SAAS,WAAW,MAAkB,iBAAkC;AAC7E,UAAQ,iBAAiB;AAAA,IACvB,KAAK,MAAM;AACT,aAAO,KAAK,qBAAqB,IAAI,EAAE,SAAS,MAAM;AAAA,IACxD;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,KAAK,WAAW,IAAI,EAAE,SAAS,MAAM;AAAA,IAC9C;AAAA,IACA,SAAS;AACP,aAAO,IAAI,YAAY,EAAE,OAAO,IAAI;AAAA,IACtC;AAAA,EACF;AACF;;;AC1CA,SAAS,kBAAkB,wBAAwB;AAEnD;AAAA,EACE;AAAA,EAAmB;AAAA,EAAU;AAAA,EAAW;AAAA,OACnC;AAIP;AAAA,EACE;AAAA,EAAmC;AAAA,EAAkB;AAAA,OAChD;AAEP,SAAS,eAAe,+BAA+B;AACvD;AAAA,EACE;AAAA,EAAe;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAU;AAAA,OACrD;AACP,SAAS,iBAAiB;AAmC1B,IAAM,0BAA0B;AAEhC,IAAM,qBAAqB;AAWpB,IAAM,qBAAN,cAAiC,kBAA4C;AAAA,EAClF,IAAI,SAAiB;AACnB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,SAAmB;AACrB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,KAAK,OAAO,eAAe;AAAA,EACpC;AAAA,EAEA,IAAI,kBAAuC;AACzC,WAAO,KAAK,OAAO,mBAAmB;AAAA,EACxC;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,OAAO,UAAU;AAAA,EAC/B;AAAA,EAEA,IAAI,mBAA2B;AAC7B,WAAO,KAAK,OAAO,oBAAoB;AAAA,EACzC;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA;AAAA,EAGA,MAAM,aAAa,aAA8E;AAC/F,UAAM,QAAQ,MAAM,KAAK,OAAO,cAAc,WAAW;AACzD,QAAI,OAAO,KAAK,EAAG,QAAO;AAC1B,UAAM,OAAO,KAAK,UAAU,KAAK;AACjC,UAAM,KAAK,QAAQ,gBAAgB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,uBAAuB;AACjF,UAAM,KAAK,QAAQ,cAAc,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,uBAAuB;AAC/E,eAAW,WAAW,MAAM,CAAC,GAAG;AAC9B,YAAM,KAAK,QAAQ,YAAY,QAAQ,KAAK,GAAG,KAAK,UAAU,OAAO,GAAG,uBAAuB;AAAA,IACjG;AACA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAM,cAAwD;AAC5D,UAAM,OAAO,MAAM,KAAK,OAAO,aAAa;AAC5C,UAAM,KAAK,QAAQ,SAAS,GAAG,KAAK,UAAU,IAAI,GAAG,kBAAkB;AACvE,SAAK,OAAO,EAAE,aAAa,KAAK,CAAC,EAAE,OAAO,MAAM,OAAO,CAAC;AACxD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,YAAY,WAAmB,WAAkC;AACrE,UAAM,OAAO,UAAU,SAAS;AAChC,aAAS,aAAa,eAAe,MAAM,kDAAkD,aAAa,eAAe,SAAS,GAAG;AACrI,UAAM,mBAAmB,YAAY,KAAK,OAAO;AACjD,UAAM,aAAa,MAAM,KAAK,OAAO,mBAAmB;AACxD,aAAS,mBAAmB,YAAY,MAAM,QAAQ,SAAS,IAAI,SAAS,0BAA0B,UAAU,GAAG;AACnH,UAAM,cAAc,MAAM,KAAK,OAAO,aAAa,WAAW,SAAS;AACvE,UAAM,SAAS,YAAY,WAAW;AACtC,UAAM,UAA6B;AAAA,MACjC,QAAQ;AAAA,MACR,MAAM,SAAS,OAAO,GAAG,EAAE,GAAG,MAAM,sBAAsB,SAAS,IAAI,SAAS,EAAE,EAAE,CAAC,EAAE;AAAA,MACvF,UAAU;AAAA,MACV;AAAA,IACF;AACA,UAAM,KAAK,QAAQ,eAAe,WAAW,SAAS,GAAG,KAAK,UAAU,OAAO,GAAG,uBAAuB;AACzG,SAAK,OAAO;AAAA,MACV;AAAA,MAAW;AAAA,MAAW,MAAM;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAM,wBAAwB,aAA4C;AACxE,aAAS,YAAY,GAAG,aAAa,eAAe,aAAa;AAC/D,YAAM,OAAO,UAAU,SAAS;AAChC,WAAK,cAAc,KAAK,SAAS,GAAG;AAClC,cAAM,KAAK,YAAY,YAAY,cAAc,KAAK,OAAO,CAAC;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,gBAAsE;AAC1E,QAAI;AACF,YAAM,WAAW,MAAM,KAAK,OAAO,KAAK,IAAI,iBAAiB,EAAE,QAAQ,KAAK,QAAQ,KAAK,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;AACzH,YAAM,OAAO,MAAM,SAAS,MAAM,qBAAqB;AACvD,UAAI,SAAS,OAAW,QAAO;AAC/B,YAAM,SAAkB,KAAK,MAAM,WAAW,MAAM,SAAS,eAAe,CAAC;AAC7E,aAAO,kCAAkC,QAAQ,IAAI;AAAA,IACvD,SAAS,OAAO;AACd,UAAI,gBAAgB,KAAK,EAAG,QAAO;AACnC,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,OAAsC;AAC1C,UAAM,aAAa,MAAM,KAAK,OAAO,aAAa;AAClD,UAAM,YAAY,MAAM,KAAK,cAAc;AAC3C,UAAM,QAAQ,cAAc,SAAY,IAAI,UAAU,CAAC,EAAE,QAAQ;AACjE,UAAM,MAAM,WAAW,CAAC,EAAE;AAC1B,QAAI,QAAQ,IAAK,QAAO;AAExB,UAAM,YAAY,IAAI,UAAU,KAAK,WAAW;AAChD,UAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,MAAM,QAAQ,EAAE,GAAG,CAAC,GAAG,MAAM,iBAAiB,QAAQ,GAAG,IAAI,CAAC;AACnG,UAAM,QAAQ,QAAQ;AACtB,QAAI,iBAAiB;AACrB,UAAM,QAAQ,IAAI,QAAQ,IAAI,iBAAe,UAAU,aAAa,YAAY;AAC9E,eAAS,MAAM,KAAK,aAAa,WAAW,GAAG,MAAM,8BAA8B,WAAW,GAAG;AACjG,wBAAkB;AAClB,UAAI,iBAAiB,KAAK,qBAAqB,KAAK,mBAAmB,OAAO;AAC5E,aAAK,OAAO;AAAA,UACV;AAAA,UAAa,WAAW;AAAA,UAAgB;AAAA,UAAO,MAAM;AAAA,QACvD,CAAC;AAAA,MACH;AAAA,IACF,CAAC,CAAC,CAAC;AACH,eAAW,eAAe,SAAS;AACjC,YAAM,KAAK,wBAAwB,WAAW;AAAA,IAChD;AACA,UAAM,KAAK,YAAY;AACvB,WAAO,CAAC,iBAAiB,OAAO,IAAI,GAAG,GAAG;AAAA,EAC5C;AAAA,EAEA,MAAc,QAAQ,MAAc,MAAc,cAAqC;AACrF,UAAM,EAAE,MAAM,gBAAgB,IAAI,WAAW,MAAM,KAAK,eAAe;AACvE,UAAM,KAAK,OAAO,KAAK,IAAI,iBAAiB;AAAA,MAC1C,QAAQ,KAAK;AAAA,MACb,KAAK,GAAG,KAAK,MAAM,GAAG,IAAI;AAAA,MAC1B,MAAM;AAAA,MACN,cAAc;AAAA,MACd,iBAAiB;AAAA,MACjB,aAAa;AAAA,IACf,CAAC,CAAC;AAAA,EACJ;AAAA;AAAA,EAGQ,OAAO,OAA+B;AAC5C,UAAM,aAAa,KAAK,OAAO;AAC/B,QAAI,YAAY;AACd,iBAAW,KAAK;AAChB;AAAA,IACF;AACA,YAAQ,MAAM,MAAM;AAAA,MAClB,KAAK,SAAS;AACZ,aAAK,QAAQ,KAAK,uCAAuC,MAAM,WAAW,KAAK,MAAM,SAAS,IAAI,MAAM,KAAK,GAAG;AAChH;AAAA,MACF;AAAA,MACA,KAAK,QAAQ;AACX,aAAK,QAAQ,KAAK,sCAAsC,MAAM,SAAS,IAAI,MAAM,SAAS,EAAE;AAC5F;AAAA,MACF;AAAA,MACA,KAAK,QAAQ;AACX,aAAK,QAAQ,KAAK,sCAAsC,MAAM,WAAW,EAAE;AAC3E;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AApKa,qBAAN;AAAA,EADN,UAAU;AAAA,GACE;AAsKb,IAAM,kBAAkB,CAAC,UAA4B;AACnD,MAAI,OAAO,UAAU,YAAY,UAAU,KAAM,QAAO;AACxD,QAAM,EAAE,MAAM,UAAU,IAAI;AAC5B,SAAO,SAAS,eAAe,SAAS,cAAc,WAAW,mBAAmB;AACtF;",
4
+ "sourcesContent": ["import ZLIB from 'node:zlib'\n\n/** Wire/storage encoding for published files. */\nexport type RestContentEncoding = 'br' | 'gzip' | 'none'\n\n/** An encoded file body plus the HTTP Content-Encoding it should be served with. */\nexport interface EncodedBody {\n body: Uint8Array\n contentEncoding?: 'br' | 'gzip'\n}\n\n/**\n * Encodes a JSON string for storage. Bodies are pre-compressed at write time and stored\n * with the matching Content-Encoding so HTTP clients decompress transparently \u2014 keys keep\n * their .json extension and Content-Type stays application/json.\n */\nexport function encodeBody(json: string, encoding: RestContentEncoding): EncodedBody {\n switch (encoding) {\n case 'br': {\n return { body: ZLIB.brotliCompressSync(json), contentEncoding: 'br' }\n }\n case 'gzip': {\n return { body: ZLIB.gzipSync(json), contentEncoding: 'gzip' }\n }\n case 'none': {\n return { body: new TextEncoder().encode(json) }\n }\n }\n}\n\n/** Decodes a stored body back to its JSON string using the object's Content-Encoding. */\nexport function decodeBody(body: Uint8Array, contentEncoding?: string): string {\n switch (contentEncoding) {\n case 'br': {\n return ZLIB.brotliDecompressSync(body).toString('utf8')\n }\n case 'gzip': {\n return ZLIB.gunzipSync(body).toString('utf8')\n }\n default: {\n return new TextDecoder().decode(body)\n }\n }\n}\n", "import type { S3Client } from '@aws-sdk/client-s3'\nimport { PutObjectCommand } from '@aws-sdk/client-s3'\nimport type { CreatableParams } from '@xylabs/sdk-js'\nimport {\n AbstractCreatable, assertEx, creatable, isNull,\n} from '@xylabs/sdk-js'\nimport type {\n BlockViewer, SignedHydratedBlockWithHashMeta, XL1BlockNumber, XL1BlockRange,\n} from '@xyo-network/xl1-protocol-lib'\nimport { asXL1BlockNumber, StepSizes } from '@xyo-network/xl1-protocol-lib'\nimport type { BlocksStepSummary } from '@xyo-network/xl1-protocol-sdk'\nimport { blocksMaxStep, BlocksStepSummarySchema } from '@xyo-network/xl1-protocol-sdk'\nimport {\n blockHashPath, blockNumberPath, blocksStepPath, payloadPath,\n} from '@xyo-network/xl1-rest-block-viewer'\nimport { Semaphore } from 'async-mutex'\n\nimport type { RestContentEncoding } from './encoding.ts'\nimport { encodeBody } from './encoding.ts'\n\n/** A progress event emitted while publishing. */\nexport type RestPublishEvent\n = | { blockNumber: XL1BlockNumber; published: number; total: number; type: 'block' }\n | { stepIndex: number; stepLevel: number; type: 'step' }\n\n/** Parameters for RestBlockPublisher. */\nexport interface RestBlockPublisherParams extends CreatableParams {\n bucket: string\n /** S3-compatible client pointed at the target endpoint (e.g. Cloudflare R2). */\n client: S3Client\n /** Maximum concurrent block publishes during sync. */\n concurrency?: number\n /** Storage/wire encoding for published files. Defaults to 'br' (brotli). */\n contentEncoding?: RestContentEncoding\n /**\n * Called as publishing progresses: every `progressInterval` blocks (and at pass completion)\n * and per step file. When omitted, the same events are logged via the params logger so\n * long backfills are observable out of the box.\n */\n onProgress?: (event: RestPublishEvent) => void\n /** Optional key prefix, allowing the layout to live in a shared bucket. */\n prefix?: string\n /** How many block publishes between progress reports during sync. */\n progressInterval?: number\n /** The viewer to publish from (only finalized chains should be published). */\n source: BlockViewer\n}\n\n/** Everything this publisher writes is immutable once written. */\nconst IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'\n\n/**\n * Publishes a chain's immutable files as the static REST file layout that `RestBlockViewer`\n * reads (see `paths.ts` in @xyo-network/xl1-rest-block-viewer).\n *\n * The mutable chain state (the head pointer, `chain/head.json`) is NOT written here \u2014 it\n * lives in a separate chain-state bucket and is written by the chain's finalizer after the\n * finalized files for that head exist, so readers never see a head that references missing\n * files.\n *\n * Publishing is idempotent: keys are deterministic and contents immutable, so re-publishing\n * any range is safe. `sync(from)` resumes from a caller-supplied cursor (typically the block\n * after the last published head); omitting it republishes from genesis, which is safe.\n */\n@creatable()\nexport class RestBlockPublisher extends AbstractCreatable<RestBlockPublisherParams> {\n get bucket(): string {\n return assertEx(this.params.bucket, () => 'No bucket specified')\n }\n\n get client(): S3Client {\n return assertEx(this.params.client, () => 'No client specified')\n }\n\n get concurrency(): number {\n return this.params.concurrency ?? 8\n }\n\n get contentEncoding(): RestContentEncoding {\n return this.params.contentEncoding ?? 'br'\n }\n\n get prefix(): string {\n return this.params.prefix ?? ''\n }\n\n get progressInterval(): number {\n return this.params.progressInterval ?? 100\n }\n\n get source(): BlockViewer {\n return assertEx(this.params.source, () => 'No source specified')\n }\n\n /** Publishes one block at its by-number and by-hash paths, plus its payload files. */\n async publishBlock(blockNumber: XL1BlockNumber): Promise<SignedHydratedBlockWithHashMeta | null> {\n const block = await this.source.blockByNumber(blockNumber)\n if (isNull(block)) return null\n const json = JSON.stringify(block)\n await this.putJson(blockNumberPath(block[0].block), json)\n await this.putJson(blockHashPath(block[0]._hash), json)\n for (const payload of block[1]) {\n await this.putJson(payloadPath(payload._hash), JSON.stringify(payload))\n }\n return block\n }\n\n /**\n * Publishes one completed step's blocks as a BlocksStepSummary file (blocks oldest-first).\n * Asserts the step is complete \u2014 partial tail steps are never published.\n */\n async publishStep(stepLevel: number, stepIndex: number): Promise<void> {\n const size = StepSizes[stepLevel]\n assertEx(stepLevel <= blocksMaxStep, () => `publishStep does not support step levels above ${blocksMaxStep} (requested ${stepLevel})`)\n const lastBlockNumber = (stepIndex + 1) * size - 1\n const headNumber = await this.source.currentBlockNumber()\n assertEx(lastBlockNumber <= headNumber, () => `Step ${stepLevel}/${stepIndex} is not complete (head ${headNumber})`)\n const newestFirst = await this.source.blocksByStep(stepLevel, stepIndex)\n const blocks = newestFirst.toReversed()\n const summary: BlocksStepSummary = {\n schema: BlocksStepSummarySchema,\n hash: assertEx(blocks.at(-1), () => `No blocks for step ${stepLevel}/${stepIndex}`)[0]._hash,\n stepSize: size,\n blocks,\n }\n await this.putJson(blocksStepPath(stepLevel, stepIndex), JSON.stringify(summary))\n this.report({\n stepIndex, stepLevel, type: 'step',\n })\n }\n\n /** Publishes every step (at every level) completed by the given block. */\n async publishStepsCompletedBy(blockNumber: XL1BlockNumber): Promise<void> {\n for (let stepLevel = 0; stepLevel <= blocksMaxStep; stepLevel++) {\n const size = StepSizes[stepLevel]\n if ((blockNumber + 1) % size === 0) {\n await this.publishStep(stepLevel, (blockNumber + 1) / size - 1)\n }\n }\n }\n\n /**\n * Publishes everything from `from` (inclusive, default genesis) to the source head\n * (inclusive), then the completed steps in that range. Returns the published range, or\n * null when already up to date.\n *\n * The caller supplies the resume cursor \u2014 typically the block after the last published\n * head pointer, which is written by the finalizer rather than this publisher. Omitting\n * `from` republishes from genesis, which is safe (idempotent) but slow.\n */\n async sync(from: XL1BlockNumber = asXL1BlockNumber(0, true)): Promise<XL1BlockRange | null> {\n const sourceHead = await this.source.currentBlock()\n const end = sourceHead[0].block\n if (from > end) return null\n\n const semaphore = new Semaphore(this.concurrency)\n const numbers = Array.from({ length: end - from + 1 }, (_, i) => asXL1BlockNumber(from + i, true))\n const total = numbers.length\n let publishedCount = 0\n await Promise.all(numbers.map(blockNumber => semaphore.runExclusive(async () => {\n assertEx(await this.publishBlock(blockNumber), () => `Block not found in source [${blockNumber}]`)\n publishedCount += 1\n if (publishedCount % this.progressInterval === 0 || publishedCount === total) {\n this.report({\n blockNumber, published: publishedCount, total, type: 'block',\n })\n }\n })))\n for (const blockNumber of numbers) {\n await this.publishStepsCompletedBy(blockNumber)\n }\n return [from, end]\n }\n\n private async putJson(path: string, json: string): Promise<void> {\n const { body, contentEncoding } = encodeBody(json, this.contentEncoding)\n await this.client.send(new PutObjectCommand({\n Bucket: this.bucket,\n Key: `${this.prefix}${path}`,\n Body: body,\n CacheControl: IMMUTABLE_CACHE_CONTROL,\n ContentEncoding: contentEncoding,\n ContentType: 'application/json',\n }))\n }\n\n /** Emits a progress event to the onProgress callback, or logs it when no callback is set. */\n private report(event: RestPublishEvent): void {\n const onProgress = this.params.onProgress\n if (onProgress) {\n onProgress(event)\n return\n }\n switch (event.type) {\n case 'block': {\n this.logger?.info(`RestBlockPublisher: published block ${event.blockNumber} (${event.published}/${event.total})`)\n break\n }\n case 'step': {\n this.logger?.info(`RestBlockPublisher: published step ${event.stepLevel}/${event.stepIndex}`)\n break\n }\n }\n }\n}\n"],
5
+ "mappings": ";;;;;;;;;;;;AAAA,OAAO,UAAU;AAgBV,SAAS,WAAW,MAAc,UAA4C;AACnF,UAAQ,UAAU;AAAA,IAChB,KAAK,MAAM;AACT,aAAO,EAAE,MAAM,KAAK,mBAAmB,IAAI,GAAG,iBAAiB,KAAK;AAAA,IACtE;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,EAAE,MAAM,KAAK,SAAS,IAAI,GAAG,iBAAiB,OAAO;AAAA,IAC9D;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,EAAE,MAAM,IAAI,YAAY,EAAE,OAAO,IAAI,EAAE;AAAA,IAChD;AAAA,EACF;AACF;AAGO,SAAS,WAAW,MAAkB,iBAAkC;AAC7E,UAAQ,iBAAiB;AAAA,IACvB,KAAK,MAAM;AACT,aAAO,KAAK,qBAAqB,IAAI,EAAE,SAAS,MAAM;AAAA,IACxD;AAAA,IACA,KAAK,QAAQ;AACX,aAAO,KAAK,WAAW,IAAI,EAAE,SAAS,MAAM;AAAA,IAC9C;AAAA,IACA,SAAS;AACP,aAAO,IAAI,YAAY,EAAE,OAAO,IAAI;AAAA,IACtC;AAAA,EACF;AACF;;;AC1CA,SAAS,wBAAwB;AAEjC;AAAA,EACE;AAAA,EAAmB;AAAA,EAAU;AAAA,EAAW;AAAA,OACnC;AAIP,SAAS,kBAAkB,iBAAiB;AAE5C,SAAS,eAAe,+BAA+B;AACvD;AAAA,EACE;AAAA,EAAe;AAAA,EAAiB;AAAA,EAAgB;AAAA,OAC3C;AACP,SAAS,iBAAiB;AAkC1B,IAAM,0BAA0B;AAgBzB,IAAM,qBAAN,cAAiC,kBAA4C;AAAA,EAClF,IAAI,SAAiB;AACnB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,SAAmB;AACrB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,KAAK,OAAO,eAAe;AAAA,EACpC;AAAA,EAEA,IAAI,kBAAuC;AACzC,WAAO,KAAK,OAAO,mBAAmB;AAAA,EACxC;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,OAAO,UAAU;AAAA,EAC/B;AAAA,EAEA,IAAI,mBAA2B;AAC7B,WAAO,KAAK,OAAO,oBAAoB;AAAA,EACzC;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,SAAS,KAAK,OAAO,QAAQ,MAAM,qBAAqB;AAAA,EACjE;AAAA;AAAA,EAGA,MAAM,aAAa,aAA8E;AAC/F,UAAM,QAAQ,MAAM,KAAK,OAAO,cAAc,WAAW;AACzD,QAAI,OAAO,KAAK,EAAG,QAAO;AAC1B,UAAM,OAAO,KAAK,UAAU,KAAK;AACjC,UAAM,KAAK,QAAQ,gBAAgB,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI;AACxD,UAAM,KAAK,QAAQ,cAAc,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI;AACtD,eAAW,WAAW,MAAM,CAAC,GAAG;AAC9B,YAAM,KAAK,QAAQ,YAAY,QAAQ,KAAK,GAAG,KAAK,UAAU,OAAO,CAAC;AAAA,IACxE;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,YAAY,WAAmB,WAAkC;AACrE,UAAM,OAAO,UAAU,SAAS;AAChC,aAAS,aAAa,eAAe,MAAM,kDAAkD,aAAa,eAAe,SAAS,GAAG;AACrI,UAAM,mBAAmB,YAAY,KAAK,OAAO;AACjD,UAAM,aAAa,MAAM,KAAK,OAAO,mBAAmB;AACxD,aAAS,mBAAmB,YAAY,MAAM,QAAQ,SAAS,IAAI,SAAS,0BAA0B,UAAU,GAAG;AACnH,UAAM,cAAc,MAAM,KAAK,OAAO,aAAa,WAAW,SAAS;AACvE,UAAM,SAAS,YAAY,WAAW;AACtC,UAAM,UAA6B;AAAA,MACjC,QAAQ;AAAA,MACR,MAAM,SAAS,OAAO,GAAG,EAAE,GAAG,MAAM,sBAAsB,SAAS,IAAI,SAAS,EAAE,EAAE,CAAC,EAAE;AAAA,MACvF,UAAU;AAAA,MACV;AAAA,IACF;AACA,UAAM,KAAK,QAAQ,eAAe,WAAW,SAAS,GAAG,KAAK,UAAU,OAAO,CAAC;AAChF,SAAK,OAAO;AAAA,MACV;AAAA,MAAW;AAAA,MAAW,MAAM;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAM,wBAAwB,aAA4C;AACxE,aAAS,YAAY,GAAG,aAAa,eAAe,aAAa;AAC/D,YAAM,OAAO,UAAU,SAAS;AAChC,WAAK,cAAc,KAAK,SAAS,GAAG;AAClC,cAAM,KAAK,YAAY,YAAY,cAAc,KAAK,OAAO,CAAC;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,KAAK,OAAuB,iBAAiB,GAAG,IAAI,GAAkC;AAC1F,UAAM,aAAa,MAAM,KAAK,OAAO,aAAa;AAClD,UAAM,MAAM,WAAW,CAAC,EAAE;AAC1B,QAAI,OAAO,IAAK,QAAO;AAEvB,UAAM,YAAY,IAAI,UAAU,KAAK,WAAW;AAChD,UAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,MAAM,OAAO,EAAE,GAAG,CAAC,GAAG,MAAM,iBAAiB,OAAO,GAAG,IAAI,CAAC;AACjG,UAAM,QAAQ,QAAQ;AACtB,QAAI,iBAAiB;AACrB,UAAM,QAAQ,IAAI,QAAQ,IAAI,iBAAe,UAAU,aAAa,YAAY;AAC9E,eAAS,MAAM,KAAK,aAAa,WAAW,GAAG,MAAM,8BAA8B,WAAW,GAAG;AACjG,wBAAkB;AAClB,UAAI,iBAAiB,KAAK,qBAAqB,KAAK,mBAAmB,OAAO;AAC5E,aAAK,OAAO;AAAA,UACV;AAAA,UAAa,WAAW;AAAA,UAAgB;AAAA,UAAO,MAAM;AAAA,QACvD,CAAC;AAAA,MACH;AAAA,IACF,CAAC,CAAC,CAAC;AACH,eAAW,eAAe,SAAS;AACjC,YAAM,KAAK,wBAAwB,WAAW;AAAA,IAChD;AACA,WAAO,CAAC,MAAM,GAAG;AAAA,EACnB;AAAA,EAEA,MAAc,QAAQ,MAAc,MAA6B;AAC/D,UAAM,EAAE,MAAM,gBAAgB,IAAI,WAAW,MAAM,KAAK,eAAe;AACvE,UAAM,KAAK,OAAO,KAAK,IAAI,iBAAiB;AAAA,MAC1C,QAAQ,KAAK;AAAA,MACb,KAAK,GAAG,KAAK,MAAM,GAAG,IAAI;AAAA,MAC1B,MAAM;AAAA,MACN,cAAc;AAAA,MACd,iBAAiB;AAAA,MACjB,aAAa;AAAA,IACf,CAAC,CAAC;AAAA,EACJ;AAAA;AAAA,EAGQ,OAAO,OAA+B;AAC5C,UAAM,aAAa,KAAK,OAAO;AAC/B,QAAI,YAAY;AACd,iBAAW,KAAK;AAChB;AAAA,IACF;AACA,YAAQ,MAAM,MAAM;AAAA,MAClB,KAAK,SAAS;AACZ,aAAK,QAAQ,KAAK,uCAAuC,MAAM,WAAW,KAAK,MAAM,SAAS,IAAI,MAAM,KAAK,GAAG;AAChH;AAAA,MACF;AAAA,MACA,KAAK,QAAQ;AACX,aAAK,QAAQ,KAAK,sCAAsC,MAAM,SAAS,IAAI,MAAM,SAAS,EAAE;AAC5F;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AA3Ia,qBAAN;AAAA,EADN,UAAU;AAAA,GACE;",
6
6
  "names": []
7
7
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "$schema": "http://json.schemastore.org/package.json",
3
3
  "name": "@xyo-network/xl1-rest-block-publisher",
4
- "version": "2.1.0",
4
+ "version": "2.1.2",
5
5
  "description": "XYO Layer One static REST chain layout publisher",
6
6
  "homepage": "https://xylabs.com",
7
7
  "bugs": {
@@ -35,9 +35,9 @@
35
35
  "README.md"
36
36
  ],
37
37
  "dependencies": {
38
- "@xyo-network/xl1-protocol-lib": "~2.1.0",
39
- "@xyo-network/xl1-rest-block-viewer": "~2.1.0",
40
- "@xyo-network/xl1-protocol-sdk": "~2.1.0"
38
+ "@xyo-network/xl1-protocol-lib": "~2.1.2",
39
+ "@xyo-network/xl1-protocol-sdk": "~2.1.2",
40
+ "@xyo-network/xl1-rest-block-viewer": "~2.1.2"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@aws-sdk/client-s3": "^3.1065.0",