@saga-sync/producer 0.0.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/chunk-builder/accumulator.d.ts +29 -0
  2. package/dist/chunk-builder/accumulator.d.ts.map +1 -0
  3. package/dist/chunk-builder/accumulator.js +72 -0
  4. package/dist/chunk-builder/accumulator.js.map +1 -0
  5. package/dist/chunk-builder/archive.d.ts +18 -0
  6. package/dist/chunk-builder/archive.d.ts.map +1 -0
  7. package/dist/chunk-builder/archive.js +68 -0
  8. package/dist/chunk-builder/archive.js.map +1 -0
  9. package/dist/chunk-builder/cli.d.ts +28 -0
  10. package/dist/chunk-builder/cli.d.ts.map +1 -0
  11. package/dist/chunk-builder/cli.js +167 -0
  12. package/dist/chunk-builder/cli.js.map +1 -0
  13. package/dist/index.d.ts +3 -0
  14. package/dist/index.d.ts.map +1 -0
  15. package/dist/index.js +6 -0
  16. package/dist/index.js.map +1 -0
  17. package/dist/keygen.d.ts +3 -0
  18. package/dist/keygen.d.ts.map +1 -0
  19. package/dist/keygen.js +10 -0
  20. package/dist/keygen.js.map +1 -0
  21. package/dist/orchestrator/cli.d.ts +37 -0
  22. package/dist/orchestrator/cli.d.ts.map +1 -0
  23. package/dist/orchestrator/cli.js +336 -0
  24. package/dist/orchestrator/cli.js.map +1 -0
  25. package/dist/orchestrator/pipeline.d.ts +32 -0
  26. package/dist/orchestrator/pipeline.d.ts.map +1 -0
  27. package/dist/orchestrator/pipeline.js +32 -0
  28. package/dist/orchestrator/pipeline.js.map +1 -0
  29. package/dist/scraper/cli.d.ts +5 -0
  30. package/dist/scraper/cli.d.ts.map +1 -0
  31. package/dist/scraper/cli.js +153 -0
  32. package/dist/scraper/cli.js.map +1 -0
  33. package/dist/scraper/config.d.ts +15 -0
  34. package/dist/scraper/config.d.ts.map +1 -0
  35. package/dist/scraper/config.js +87 -0
  36. package/dist/scraper/config.js.map +1 -0
  37. package/dist/scraper/cursor.d.ts +12 -0
  38. package/dist/scraper/cursor.d.ts.map +1 -0
  39. package/dist/scraper/cursor.js +38 -0
  40. package/dist/scraper/cursor.js.map +1 -0
  41. package/dist/scraper/normalize.d.ts +5 -0
  42. package/dist/scraper/normalize.d.ts.map +1 -0
  43. package/dist/scraper/normalize.js +19 -0
  44. package/dist/scraper/normalize.js.map +1 -0
  45. package/dist/scraper/scrape.d.ts +10 -0
  46. package/dist/scraper/scrape.d.ts.map +1 -0
  47. package/dist/scraper/scrape.js +70 -0
  48. package/dist/scraper/scrape.js.map +1 -0
  49. package/dist/storage/dry-run-store.d.ts +10 -0
  50. package/dist/storage/dry-run-store.d.ts.map +1 -0
  51. package/dist/storage/dry-run-store.js +22 -0
  52. package/dist/storage/dry-run-store.js.map +1 -0
  53. package/dist/storage/gcs-store.d.ts +44 -0
  54. package/dist/storage/gcs-store.d.ts.map +1 -0
  55. package/dist/storage/gcs-store.js +85 -0
  56. package/dist/storage/gcs-store.js.map +1 -0
  57. package/dist/storage/index.d.ts +15 -0
  58. package/dist/storage/index.d.ts.map +1 -0
  59. package/dist/storage/index.js +52 -0
  60. package/dist/storage/index.js.map +1 -0
  61. package/package.json +38 -16
@@ -0,0 +1,29 @@
1
+ import type { CanonicalEvent } from "../scraper/normalize.js";
2
+ export type CompletedChunk = {
3
+ events: CanonicalEvent[];
4
+ from: bigint;
5
+ to: bigint;
6
+ };
7
+ export type Trailing = {
8
+ events: CanonicalEvent[];
9
+ fromBlock: bigint;
10
+ };
11
+ export declare class ChunkAccumulator {
12
+ private readonly sizeLimit;
13
+ private chunkFrom;
14
+ private accumulated;
15
+ private accumulatedBytes;
16
+ private pending;
17
+ private pendingBytes;
18
+ private pendingBlock;
19
+ constructor(sizeLimit: number, chunkFrom: bigint);
20
+ add(event: CanonicalEvent): CompletedChunk | null;
21
+ finish(): {
22
+ completed: CompletedChunk | null;
23
+ trailing: Trailing;
24
+ };
25
+ private wouldOverflow;
26
+ private commitPending;
27
+ private cut;
28
+ }
29
+ //# sourceMappingURL=accumulator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"accumulator.d.ts","sourceRoot":"","sources":["../../src/chunk-builder/accumulator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAE9D,MAAM,MAAM,cAAc,GAAG;IAAE,MAAM,EAAE,cAAc,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG;IAAE,MAAM,EAAE,cAAc,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAYvE,qBAAa,gBAAgB;IAQzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,SAAS;IARnB,OAAO,CAAC,WAAW,CAAwB;IAC3C,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,YAAY,CAAuB;gBAGxB,SAAS,EAAE,MAAM,EAC1B,SAAS,EAAE,MAAM;IAG3B,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,cAAc,GAAG,IAAI;IAoBjD,MAAM,IAAI;QAAE,SAAS,EAAE,cAAc,GAAG,IAAI,CAAC;QAAC,QAAQ,EAAE,QAAQ,CAAA;KAAE;IAclE,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,GAAG;CAOZ"}
@@ -0,0 +1,72 @@
1
+ // Block-aligned chunk partitioning — pure, no I/O. Events are fed in
2
+ // (blockNumber, logIndex) order; the accumulator buffers the in-progress block
3
+ // separately and only commits it to the current chunk once the next block
4
+ // arrives. That guarantees chunk boundaries always fall *between* blocks, so a
5
+ // multi-event block can never be split across two chunks.
6
+ //
7
+ // `add` emits a completed chunk when feeding an event closed a block and the
8
+ // accumulated bytes had crossed the size limit. `finish` flushes the last
9
+ // in-progress block and hands back the trailing accumulator — the caller seals
10
+ // it as a final chunk or carries it forward as a hot head.
11
+ export class ChunkAccumulator {
12
+ sizeLimit;
13
+ chunkFrom;
14
+ accumulated = [];
15
+ accumulatedBytes = 0;
16
+ pending = [];
17
+ pendingBytes = 0;
18
+ pendingBlock = null;
19
+ constructor(sizeLimit, chunkFrom) {
20
+ this.sizeLimit = sizeLimit;
21
+ this.chunkFrom = chunkFrom;
22
+ }
23
+ add(event) {
24
+ const eventBlock = BigInt(event.blockNumber);
25
+ const lineBytes = Buffer.byteLength(JSON.stringify(event) + "\n", "utf8");
26
+ let completed = null;
27
+ if (this.pendingBlock !== null && eventBlock !== this.pendingBlock) {
28
+ // The pending block is finished — commit it to the current chunk,
29
+ // sealing first if doing so would overflow the size limit.
30
+ if (this.wouldOverflow()) {
31
+ completed = this.cut(this.pendingBlock);
32
+ }
33
+ this.commitPending();
34
+ }
35
+ this.pendingBlock = eventBlock;
36
+ this.pending.push(event);
37
+ this.pendingBytes += lineBytes;
38
+ return completed;
39
+ }
40
+ finish() {
41
+ let completed = null;
42
+ if (this.pending.length > 0 && this.pendingBlock !== null) {
43
+ if (this.wouldOverflow()) {
44
+ completed = this.cut(this.pendingBlock);
45
+ }
46
+ this.commitPending();
47
+ }
48
+ return {
49
+ completed,
50
+ trailing: { events: this.accumulated, fromBlock: this.chunkFrom },
51
+ };
52
+ }
53
+ wouldOverflow() {
54
+ return (this.accumulatedBytes + this.pendingBytes > this.sizeLimit &&
55
+ this.accumulated.length > 0);
56
+ }
57
+ commitPending() {
58
+ this.accumulated.push(...this.pending);
59
+ this.accumulatedBytes += this.pendingBytes;
60
+ this.pending = [];
61
+ this.pendingBytes = 0;
62
+ }
63
+ // Close the current chunk at `to` and start a fresh one.
64
+ cut(to) {
65
+ const chunk = { events: this.accumulated, from: this.chunkFrom, to };
66
+ this.chunkFrom = to;
67
+ this.accumulated = [];
68
+ this.accumulatedBytes = 0;
69
+ return chunk;
70
+ }
71
+ }
72
+ //# sourceMappingURL=accumulator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"accumulator.js","sourceRoot":"","sources":["../../src/chunk-builder/accumulator.ts"],"names":[],"mappings":"AAKA,qEAAqE;AACrE,+EAA+E;AAC/E,0EAA0E;AAC1E,+EAA+E;AAC/E,0DAA0D;AAC1D,EAAE;AACF,6EAA6E;AAC7E,0EAA0E;AAC1E,+EAA+E;AAC/E,2DAA2D;AAC3D,MAAM,OAAO,gBAAgB;IAQR;IACT;IARF,WAAW,GAAqB,EAAE,CAAC;IACnC,gBAAgB,GAAG,CAAC,CAAC;IACrB,OAAO,GAAqB,EAAE,CAAC;IAC/B,YAAY,GAAG,CAAC,CAAC;IACjB,YAAY,GAAkB,IAAI,CAAC;IAE3C,YACmB,SAAiB,EAC1B,SAAiB;QADR,cAAS,GAAT,SAAS,CAAQ;QAC1B,cAAS,GAAT,SAAS,CAAQ;IACxB,CAAC;IAEJ,GAAG,CAAC,KAAqB;QACvB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1E,IAAI,SAAS,GAA0B,IAAI,CAAC;QAE5C,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YACnE,kEAAkE;YAClE,2DAA2D;YAC3D,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;gBACzB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC;QAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;QAC/B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM;QACJ,IAAI,SAAS,GAA0B,IAAI,CAAC;QAC5C,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;gBACzB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,CAAC;QACD,OAAO;YACL,SAAS;YACT,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE;SAClE,CAAC;IACJ,CAAC;IAEO,aAAa;QACnB,OAAO,CACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS;YAC1D,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAC5B,CAAC;IACJ,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,yDAAyD;IACjD,GAAG,CAAC,EAAU;QACpB,MAAM,KAAK,GAAmB,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,EAAE,CAAC;QACrF,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
@@ -0,0 +1,18 @@
1
+ import type { CanonicalEvent } from "../scraper/normalize.js";
2
+ import type { Store } from "@saga-sync/core";
3
+ import type { ChunkMeta } from "@saga-sync/core";
4
+ export type Range = {
5
+ from: bigint;
6
+ to: bigint;
7
+ };
8
+ export declare function buildJsonl(events: CanonicalEvent[]): Buffer;
9
+ export declare class ChunkArchive {
10
+ private readonly store;
11
+ constructor(store: Store);
12
+ seal(protocolId: string, events: CanonicalEvent[], range: Range): Promise<ChunkMeta>;
13
+ writeHotHead(protocolId: string, events: CanonicalEvent[], range: Range): Promise<ChunkMeta>;
14
+ private write;
15
+ readEvents(meta: ChunkMeta): Promise<CanonicalEvent[]>;
16
+ delete(meta: ChunkMeta): Promise<void>;
17
+ }
18
+ //# sourceMappingURL=archive.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"archive.d.ts","sourceRoot":"","sources":["../../src/chunk-builder/archive.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,MAAM,KAAK,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAKjD,wBAAgB,UAAU,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAG3D;AAUD,qBAAa,YAAY;IACX,OAAO,CAAC,QAAQ,CAAC,KAAK;gBAAL,KAAK,EAAE,KAAK;IAGzC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC;IAKpF,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC;YAI9E,KAAK;IA4Bb,UAAU,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAWtD,MAAM,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;CAG7C"}
@@ -0,0 +1,68 @@
1
+ import { gzipSync, gunzipSync } from "node:zlib";
2
+ import { numberToHex } from "viem";
3
+ import { sha256Hex } from "@saga-sync/core";
4
+ // Build the JSONL bytes for a chunk: one CanonicalEvent per line, trailing
5
+ // newline. Empty events list produces zero bytes so an empty chunk file is
6
+ // genuinely empty rather than a single blank line.
7
+ export function buildJsonl(events) {
8
+ if (events.length === 0)
9
+ return Buffer.alloc(0);
10
+ return Buffer.from(events.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
11
+ }
12
+ // Encodes/decodes chunk files over a Store. A chunk's digest is the sha256 of
13
+ // the *uncompressed* JSONL bytes — clients verify integrity after decompressing
14
+ // without trusting the gzip wrapper. Sealed and hot files are byte-identical for
15
+ // the same events; only the filename suffix differs (`.jsonl.gz` vs
16
+ // `.hot.jsonl.gz`), a hint that a hot file is the protocol's mutable tail.
17
+ //
18
+ // Dry-run is handled at the Store layer (DryRunStore no-ops writes), so this
19
+ // class needs no dry-run awareness.
20
+ export class ChunkArchive {
21
+ store;
22
+ constructor(store) {
23
+ this.store = store;
24
+ }
25
+ // Seal an immutable chunk.
26
+ seal(protocolId, events, range) {
27
+ return this.write(protocolId, events, range, false);
28
+ }
29
+ // Write the protocol's mutable hot head.
30
+ writeHotHead(protocolId, events, range) {
31
+ return this.write(protocolId, events, range, true);
32
+ }
33
+ async write(protocolId, events, range, hot) {
34
+ const uncompressed = buildJsonl(events);
35
+ const digest = sha256Hex(uncompressed);
36
+ const compressed = gzipSync(uncompressed);
37
+ const fromHex = numberToHex(range.from);
38
+ const toHex = numberToHex(range.to);
39
+ const file = `${protocolId}-[${fromHex},${toHex})${hot ? ".hot.jsonl.gz" : ".jsonl.gz"}`;
40
+ await this.store.put(file, compressed);
41
+ return {
42
+ fromBlock: fromHex,
43
+ toBlock: toHex,
44
+ file,
45
+ size: numberToHex(compressed.length),
46
+ digest: { type: "sha256", data: digest },
47
+ };
48
+ }
49
+ // Inverse of write(): fetch a chunk file and parse its JSONL back into events.
50
+ // Used to load a previous hot head into the accumulator (and by the future
51
+ // client library to materialize chunk contents).
52
+ async readEvents(meta) {
53
+ const compressed = await this.store.get(meta.file);
54
+ if (!compressed || compressed.length === 0)
55
+ return [];
56
+ const uncompressed = gunzipSync(compressed).toString("utf8");
57
+ if (uncompressed.length === 0)
58
+ return [];
59
+ return uncompressed
60
+ .split("\n")
61
+ .filter((line) => line.length > 0)
62
+ .map((line) => JSON.parse(line));
63
+ }
64
+ async delete(meta) {
65
+ await this.store.delete(meta.file);
66
+ }
67
+ }
68
+ //# sourceMappingURL=archive.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"archive.js","sourceRoot":"","sources":["../../src/chunk-builder/archive.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAO5C,2EAA2E;AAC3E,2EAA2E;AAC3E,mDAAmD;AACnD,MAAM,UAAU,UAAU,CAAC,MAAwB;IACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;AACrF,CAAC;AAED,8EAA8E;AAC9E,gFAAgF;AAChF,iFAAiF;AACjF,oEAAoE;AACpE,2EAA2E;AAC3E,EAAE;AACF,6EAA6E;AAC7E,oCAAoC;AACpC,MAAM,OAAO,YAAY;IACM;IAA7B,YAA6B,KAAY;QAAZ,UAAK,GAAL,KAAK,CAAO;IAAG,CAAC;IAE7C,2BAA2B;IAC3B,IAAI,CAAC,UAAkB,EAAE,MAAwB,EAAE,KAAY;QAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAED,yCAAyC;IACzC,YAAY,CAAC,UAAkB,EAAE,MAAwB,EAAE,KAAY;QACrE,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,UAAkB,EAClB,MAAwB,EACxB,KAAY,EACZ,GAAY;QAEZ,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,GAAG,UAAU,KAAK,OAAO,IAAI,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAEzF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEvC,OAAO;YACL,SAAS,EAAE,OAAO;YAClB,OAAO,EAAE,KAAK;YACd,IAAI;YACJ,IAAI,EAAE,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;YACpC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE;SACzC,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,2EAA2E;IAC3E,iDAAiD;IACjD,KAAK,CAAC,UAAU,CAAC,IAAe;QAC9B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACzC,OAAO,YAAY;aAChB,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmB,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAe;QAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;CACF"}
@@ -0,0 +1,28 @@
1
+ #!/usr/bin/env node
2
+ import type { CanonicalEvent } from "../scraper/normalize.js";
3
+ import { ChunkArchive } from "./archive.js";
4
+ import { Manifest } from "@saga-sync/core";
5
+ import type { ChunkMeta } from "@saga-sync/core";
6
+ export type ProcessArgs = {
7
+ protocolId: string;
8
+ fromBlock: bigint;
9
+ toBlock: bigint;
10
+ sizeLimit: number;
11
+ archive: ChunkArchive;
12
+ manifest: Manifest;
13
+ seed?: {
14
+ events: CanonicalEvent[];
15
+ chunkFrom: bigint;
16
+ };
17
+ trailingMode?: "seal" | "suspend";
18
+ };
19
+ export type ProcessResult = {
20
+ sealed: ChunkMeta[];
21
+ trailing?: {
22
+ events: CanonicalEvent[];
23
+ fromBlock: bigint;
24
+ toBlock: bigint;
25
+ };
26
+ };
27
+ export declare function processStream(lines: AsyncIterable<string>, args: ProcessArgs): Promise<ProcessResult>;
28
+ //# sourceMappingURL=cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/chunk-builder/cli.ts"],"names":[],"mappings":";AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAE9D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAuEjD,MAAM,MAAM,WAAW,GAAG;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,YAAY,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;IAKnB,IAAI,CAAC,EAAE;QAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAGvD,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,SAAS,EAAE,CAAC;IAEpB,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7E,CAAC;AAMF,wBAAsB,aAAa,CACjC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,EAC5B,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CA+DxB"}
@@ -0,0 +1,167 @@
1
+ #!/usr/bin/env node
2
+ import { parseArgs } from "node:util";
3
+ import { realpathSync } from "node:fs";
4
+ import { fileURLToPath } from "node:url";
5
+ import { createInterface } from "node:readline";
6
+ import { numberToHex } from "viem";
7
+ import { createStore, parseStoreTarget } from "../storage/index.js";
8
+ import { ChunkArchive } from "./archive.js";
9
+ import { ChunkAccumulator } from "./accumulator.js";
10
+ import { Manifest } from "@saga-sync/core";
11
+ import { signerFromEnv } from "@saga-sync/core";
12
+ const DEFAULT_SIZE_LIMIT = 10 * 1024 * 1024; // 10 MiB
13
+ const USAGE = `chunk-builder — partition scraper NDJSON into immutable .jsonl.gz chunks
14
+
15
+ Usage:
16
+ chunk-builder --protocol-id <id> --from-block <hex> --to-block <hex> --output-dir <path> [options]
17
+
18
+ Required:
19
+ --protocol-id <id> manifest key + filename prefix
20
+ --from-block <hex> inclusive start of the scanned range
21
+ --to-block <hex> exclusive end of the scanned range
22
+ --output-dir <path> where to write chunks + index.json (created if missing)
23
+
24
+ Options:
25
+ --size-limit <n> max uncompressed bytes per chunk, default ${DEFAULT_SIZE_LIMIT} (10 MiB)
26
+ --dry-run compute chunk metadata but do not write files or manifest
27
+ --help show this message
28
+ `;
29
+ function fail(msg) {
30
+ process.stderr.write(`chunk-builder: ${msg}\n`);
31
+ process.exit(1);
32
+ }
33
+ function parseCliArgs() {
34
+ const { values } = parseArgs({
35
+ options: {
36
+ "protocol-id": { type: "string" },
37
+ "from-block": { type: "string" },
38
+ "to-block": { type: "string" },
39
+ "output-dir": { type: "string" },
40
+ "size-limit": { type: "string" },
41
+ "dry-run": { type: "boolean", default: false },
42
+ help: { type: "boolean", default: false },
43
+ },
44
+ });
45
+ if (values.help) {
46
+ process.stdout.write(USAGE);
47
+ process.exit(0);
48
+ }
49
+ const need = (name) => {
50
+ const v = values[name];
51
+ if (!v)
52
+ fail(`missing required --${name}\n\n${USAGE}`);
53
+ return v;
54
+ };
55
+ const fromBlock = BigInt(need("from-block"));
56
+ const toBlock = BigInt(need("to-block"));
57
+ if (fromBlock >= toBlock) {
58
+ fail(`--from-block (${numberToHex(fromBlock)}) must be < --to-block (${numberToHex(toBlock)})`);
59
+ }
60
+ const sizeLimit = values["size-limit"] ? Number(values["size-limit"]) : DEFAULT_SIZE_LIMIT;
61
+ if (!Number.isFinite(sizeLimit) || sizeLimit <= 0) {
62
+ fail(`--size-limit must be a positive integer; got ${values["size-limit"]}`);
63
+ }
64
+ return {
65
+ protocolId: need("protocol-id"),
66
+ fromBlock,
67
+ toBlock,
68
+ output: need("output-dir"), // raw — may be a local dir or a gs:// target
69
+ sizeLimit,
70
+ dryRun: values["dry-run"] ?? false,
71
+ };
72
+ }
73
+ // Reads CanonicalEvent NDJSON, partitions it into chunks via ChunkAccumulator,
74
+ // and seals each completed chunk through ChunkArchive + Manifest. The trailing
75
+ // accumulator is either sealed (seal mode) or returned (suspend mode — the
76
+ // hot-head carry-over path).
77
+ export async function processStream(lines, args) {
78
+ const sealed = [];
79
+ const chunkFrom = args.seed?.chunkFrom ?? args.fromBlock;
80
+ const accumulator = new ChunkAccumulator(args.sizeLimit, chunkFrom);
81
+ const sealCompleted = async (c) => {
82
+ const meta = await args.archive.seal(args.protocolId, c.events, { from: c.from, to: c.to });
83
+ await args.manifest.appendChunk(args.protocolId, meta);
84
+ sealed.push(meta);
85
+ };
86
+ // Seed events (from a prior hot head) lead the stream and skip the range check.
87
+ if (args.seed) {
88
+ for (const event of args.seed.events) {
89
+ const completed = accumulator.add(event);
90
+ if (completed)
91
+ await sealCompleted(completed);
92
+ }
93
+ }
94
+ for await (const rawLine of lines) {
95
+ const line = rawLine.trim();
96
+ if (line === "")
97
+ continue;
98
+ let event;
99
+ try {
100
+ event = JSON.parse(line);
101
+ }
102
+ catch (err) {
103
+ throw new Error(`malformed NDJSON line: ${err.message}`);
104
+ }
105
+ if (typeof event.blockNumber !== "string") {
106
+ throw new Error(`event missing blockNumber: ${line.slice(0, 120)}`);
107
+ }
108
+ const eventBlock = BigInt(event.blockNumber);
109
+ if (eventBlock < args.fromBlock || eventBlock >= args.toBlock) {
110
+ throw new Error(`event at block ${event.blockNumber} outside scanned range ` +
111
+ `[${numberToHex(args.fromBlock)}, ${numberToHex(args.toBlock)})`);
112
+ }
113
+ const completed = accumulator.add(event);
114
+ if (completed)
115
+ await sealCompleted(completed);
116
+ }
117
+ const { completed, trailing } = accumulator.finish();
118
+ if (completed)
119
+ await sealCompleted(completed);
120
+ if ((args.trailingMode ?? "seal") === "seal") {
121
+ // Seal the trailing accumulator as a final chunk up to args.toBlock — even
122
+ // with zero events, so the manifest asserts full range coverage.
123
+ const meta = await args.archive.seal(args.protocolId, trailing.events, {
124
+ from: trailing.fromBlock,
125
+ to: args.toBlock,
126
+ });
127
+ await args.manifest.appendChunk(args.protocolId, meta);
128
+ sealed.push(meta);
129
+ return { sealed };
130
+ }
131
+ return {
132
+ sealed,
133
+ trailing: { events: trailing.events, fromBlock: trailing.fromBlock, toBlock: args.toBlock },
134
+ };
135
+ }
136
+ async function main() {
137
+ const args = parseCliArgs();
138
+ const store = createStore({ ...parseStoreTarget(args.output), dryRun: args.dryRun });
139
+ const archive = new ChunkArchive(store);
140
+ const manifest = await Manifest.load(store, undefined, { signer: signerFromEnv() });
141
+ const rl = createInterface({ input: process.stdin, crlfDelay: Infinity });
142
+ const { sealed } = await processStream(rl, {
143
+ protocolId: args.protocolId,
144
+ fromBlock: args.fromBlock,
145
+ toBlock: args.toBlock,
146
+ sizeLimit: args.sizeLimit,
147
+ archive,
148
+ manifest,
149
+ });
150
+ process.stderr.write(`chunk-builder: ${sealed.length} chunk(s) for ${args.protocolId} ` +
151
+ `[${numberToHex(args.fromBlock)}, ${numberToHex(args.toBlock)})` +
152
+ (args.dryRun ? " (dry-run: no files or manifest written)\n" : "\n"));
153
+ }
154
+ function isMainModule() {
155
+ try {
156
+ return realpathSync(process.argv[1] ?? "") === realpathSync(fileURLToPath(import.meta.url));
157
+ }
158
+ catch {
159
+ return false;
160
+ }
161
+ }
162
+ if (isMainModule()) {
163
+ main().catch((err) => {
164
+ fail(err instanceof Error ? err.message : String(err));
165
+ });
166
+ }
167
+ //# sourceMappingURL=cli.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/chunk-builder/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,SAAS;AAEtD,MAAM,KAAK,GAAG;;;;;;;;;;;;qEAYuD,kBAAkB;;;CAGtF,CAAC;AAEF,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC3B,OAAO,EAAE;YACP,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC9C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,IAA8D,EAAU,EAAE;QACtF,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,IAAI,CAAC,sBAAsB,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IACzC,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,iBAAiB,WAAW,CAAC,SAAS,CAAC,2BAA2B,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAClG,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC3F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,gDAAgD,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,OAAO;QACL,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;QAC/B,SAAS;QACT,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,6CAA6C;QACzE,SAAS;QACT,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK;KACnC,CAAC;AACJ,CAAC;AAyBD,+EAA+E;AAC/E,+EAA+E;AAC/E,2EAA2E;AAC3E,6BAA6B;AAC7B,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA4B,EAC5B,IAAiB;IAEjB,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAEpE,MAAM,aAAa,GAAG,KAAK,EAAE,CAAiB,EAAiB,EAAE;QAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,gFAAgF;IAChF,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACrC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,SAAS;gBAAE,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,EAAE;YAAE,SAAS;QAE1B,IAAI,KAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmB,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA2B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC7C,IAAI,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CACb,kBAAkB,KAAK,CAAC,WAAW,yBAAyB;gBAC1D,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CACnE,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS;YAAE,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;IACrD,IAAI,SAAS;QAAE,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAE9C,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,KAAK,MAAM,EAAE,CAAC;QAC7C,2EAA2E;QAC3E,iEAAiE;QACjE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE;YACrE,IAAI,EAAE,QAAQ,CAAC,SAAS;YACxB,EAAE,EAAE,IAAI,CAAC,OAAO;SACjB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC;IAED,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;KAC5F,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;IAEpF,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1E,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,EAAE,EAAE;QACzC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,OAAO;QACP,QAAQ;KACT,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,kBAAkB,MAAM,CAAC,MAAM,iBAAiB,IAAI,CAAC,UAAU,GAAG;QAChE,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG;QAChE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,4CAA4C,CAAC,CAAC,CAAC,IAAI,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,YAAY,EAAE,EAAE,CAAC;IACnB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QAC5B,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,3 @@
1
+ export { ChunkArchive, buildJsonl } from "./chunk-builder/archive.js";
2
+ export type { Range } from "./chunk-builder/archive.js";
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACtE,YAAY,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,6 @@
1
+ // @saga-sync/producer — producer-side API. Day-to-day use is via the CLIs
2
+ // (scraper, chunk-builder, orchestrator); this entry exposes the pieces other
3
+ // packages compose against — e.g. the client's integration tests build fixtures
4
+ // with ChunkArchive, exactly as a real publisher would.
5
+ export { ChunkArchive, buildJsonl } from "./chunk-builder/archive.js";
6
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,8EAA8E;AAC9E,gFAAgF;AAChF,wDAAwD;AACxD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC"}
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env node
2
+ export {};
3
+ //# sourceMappingURL=keygen.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"keygen.d.ts","sourceRoot":"","sources":["../src/keygen.ts"],"names":[],"mappings":""}
package/dist/keygen.js ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env node
2
+ import { generateKeyPair } from "@saga-sync/core";
3
+ // Print a fresh Ed25519 manifest-signing keypair. The secret goes to the
4
+ // producer (MANIFEST_SIGNING_KEY env / Secret Manager); the public key is what
5
+ // consumers pin via --public-key (and, later, what a registry contract serves).
6
+ const { secretKey, publicKey } = generateKeyPair();
7
+ process.stdout.write(`# Ed25519 manifest-signing keypair — keep the secret secret\n` +
8
+ `MANIFEST_SIGNING_KEY=${secretKey}\n` +
9
+ `PUBLIC_KEY=${publicKey}\n`);
10
+ //# sourceMappingURL=keygen.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"keygen.js","sourceRoot":"","sources":["../src/keygen.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,yEAAyE;AACzE,+EAA+E;AAC/E,gFAAgF;AAChF,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,eAAe,EAAE,CAAC;AACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,+DAA+D;IAC7D,wBAAwB,SAAS,IAAI;IACrC,cAAc,SAAS,IAAI,CAC9B,CAAC"}
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env node
2
+ import type { PublicClient } from "viem";
3
+ import type { ScraperTarget } from "../scraper/config.js";
4
+ import { Manifest } from "@saga-sync/core";
5
+ import type { ChunkMeta } from "@saga-sync/core";
6
+ import { ChunkArchive } from "../chunk-builder/archive.js";
7
+ declare function parseCliArgs(): {
8
+ configPath: string;
9
+ rpc: string;
10
+ output: string;
11
+ lockDir: string | undefined;
12
+ protocolId: string | undefined;
13
+ batchSize: bigint;
14
+ confirmations: bigint;
15
+ window: number;
16
+ sizeLimit: number;
17
+ dryRun: boolean;
18
+ };
19
+ export declare function acquireLock(path: string): boolean;
20
+ declare function processProtocol(args: {
21
+ client: PublicClient;
22
+ protocolId: string;
23
+ protocol: ScraperTarget;
24
+ archive: ChunkArchive;
25
+ manifest: Manifest;
26
+ tip: bigint;
27
+ batchSize: bigint;
28
+ window: number;
29
+ sizeLimit: number;
30
+ }): Promise<{
31
+ ranBatches: number;
32
+ sealedChunks: number;
33
+ finalHotHead: ChunkMeta | null;
34
+ }>;
35
+ declare function main(): Promise<void>;
36
+ export { main, parseCliArgs, processProtocol };
37
+ //# sourceMappingURL=cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/orchestrator/cli.ts"],"names":[],"mappings":";AAcA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AAGzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAsC3D,iBAAS,YAAY;;;;;;;;;;;EA2CpB;AAKD,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CA2DjD;AAID,iBAAe,eAAe,CAAC,IAAI,EAAE;IACnC,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,aAAa,CAAC;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,SAAS,GAAG,IAAI,CAAA;CAAE,CAAC,CA6FxF;AAED,iBAAe,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAkGnC;AAgBD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,eAAe,EAAE,CAAC"}