@saga-sync/producer 0.0.0 → 0.1.2

package/dist/orchestrator/cli.js

@@ -0,0 +1,336 @@
+#!/usr/bin/env node
+import { parseArgs } from "node:util";
+import { mkdirSync, openSync, writeFileSync, closeSync, unlinkSync, readFileSync, realpathSync, } from "node:fs";
+import { resolve, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createPublicClient, http, numberToHex } from "viem";
+import { finalizedBlock, assertChainId } from "../scraper/cli.js";
+import { loadAllProtocols } from "../scraper/config.js";
+import { createStore, parseStoreTarget } from "../storage/index.js";
+import { Manifest } from "@saga-sync/core";
+import { signerFromEnv } from "@saga-sync/core";
+import { ChunkArchive } from "../chunk-builder/archive.js";
+import { runProtocolOnce } from "./pipeline.js";
+const DEFAULT_WINDOW = 2000;
+const DEFAULT_CONFIRMATIONS = 12n;
+const DEFAULT_SIZE_LIMIT = 10 * 1024 * 1024;
+const DEFAULT_BATCH_SIZE = 100000n;
+const USAGE = `orchestrator — run scrape→chunk in batches for every protocol
+
+Usage:
+  orchestrator --config <path> --rpc <url> --output-dir <path> [options]
+
+Required:
+  --config <path>        scraper config JSON (all protocols read)
+  --rpc <url>            Ethereum JSON-RPC URL (single chain per run)
+  --output-dir <path>    chunks + index.json directory
+
+Options:
+  --lock-dir <path>      directory for .orchestrator.lock; default = output-dir
+  --protocol-id <id>     restrict to one protocol (backfills / ad-hoc runs)
+  --batch-size <n>       blocks per batch (default 100000). Smaller batches
+                         bound the crash blast radius; larger batches reduce
+                         hot-head rewrite overhead.
+  --confirmations <n>    fallback reorg buffer when "finalized" tag is unsupported
+                         (default 12)
+  --window <n>           scraper window size in blocks (default 2000)
+  --size-limit <n>       chunk size cap in bytes if config does not specify it
+                         (default 10485760 = 10 MiB)
+  --dry-run              compute what would run, don't touch disk
+  --help
+`;
+function fail(msg) {
+    process.stderr.write(`orchestrator: ${msg}\n`);
+    process.exit(1);
+}
+function parseCliArgs() {
+    const { values } = parseArgs({
+        options: {
+            config: { type: "string" },
+            rpc: { type: "string" },
+            "output-dir": { type: "string" },
+            "lock-dir": { type: "string" },
+            "protocol-id": { type: "string" },
+            "batch-size": { type: "string" },
+            confirmations: { type: "string" },
+            window: { type: "string" },
+            "size-limit": { type: "string" },
+            "dry-run": { type: "boolean", default: false },
+            help: { type: "boolean", default: false },
+        },
+    });
+    if (values.help) {
+        process.stdout.write(USAGE);
+        process.exit(0);
+    }
+    const need = (name) => {
+        const v = values[name];
+        if (!v)
+            fail(`missing required --${name}\n\n${USAGE}`);
+        return v;
+    };
+    const batchSize = values["batch-size"] ? BigInt(values["batch-size"]) : DEFAULT_BATCH_SIZE;
+    if (batchSize <= 0n)
+        fail(`--batch-size must be positive; got ${values["batch-size"]}`);
+    return {
+        configPath: resolve(need("config")),
+        rpc: need("rpc"),
+        output: need("output-dir"), // raw — may be a local dir or a gs:// target
+        lockDir: values["lock-dir"] ? resolve(values["lock-dir"]) : undefined,
+        protocolId: values["protocol-id"],
+        batchSize,
+        confirmations: values.confirmations ? BigInt(values.confirmations) : DEFAULT_CONFIRMATIONS,
+        window: values.window ? Number(values.window) : DEFAULT_WINDOW,
+        sizeLimit: values["size-limit"] ? Number(values["size-limit"]) : DEFAULT_SIZE_LIMIT,
+        dryRun: values["dry-run"] ?? false,
+    };
+}
+// Atomic lock acquisition with stale-pid recovery. Returns true if we got the
+// lock, false if another live process already holds it. Disk-only by design —
+// the lock is a local single-machine coordination primitive, not object storage.
+export function acquireLock(path) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+        try {
+            const fd = openSync(path, "wx");
+            writeFileSync(fd, String(process.pid), "utf8");
+            closeSync(fd);
+            const cleanup = () => {
+                try {
+                    unlinkSync(path);
+                }
+                catch {
+                    /* already gone */
+                }
+            };
+            process.on("exit", cleanup);
+            process.on("SIGINT", () => {
+                cleanup();
+                process.exit(130);
+            });
+            process.on("SIGTERM", () => {
+                cleanup();
+                process.exit(143);
+            });
+            return true;
+        }
+        catch (err) {
+            const code = err.code;
+            if (code !== "EEXIST")
+                throw err;
+            let stalePid;
+            try {
+                stalePid = parseInt(readFileSync(path, "utf8").trim(), 10);
+            }
+            catch {
+                continue;
+            }
+            if (!Number.isFinite(stalePid) || stalePid <= 0) {
+                try {
+                    unlinkSync(path);
+                }
+                catch {
+                    /* race with another recovery */
+                }
+                continue;
+            }
+            try {
+                process.kill(stalePid, 0);
+                return false; // owner alive
+            }
+            catch (e) {
+                const ec = e.code;
+                if (ec === "ESRCH") {
+                    try {
+                        unlinkSync(path);
+                    }
+                    catch {
+                        /* race */
+                    }
+                    continue;
+                }
+                return false; // EPERM: exists but unsignalable — treat as alive
+            }
+        }
+    }
+    return false;
+}
+// Process one protocol: clean any stale hot head, loop in batches, persist the
+// final trailing accumulator as the new hot head.
+async function processProtocol(args) {
+    const { manifest, archive, protocolId, protocol, tip, batchSize } = args;
+    // 1. Resolve where to start. Take the max() of the hot head's toBlock and the
+    // last sealed chunk's toBlock — robust to a crash-post-seal-pre-clear state.
+    const sealedChunks = manifest.sealedChunks(protocolId);
+    const lastSealedChunk = sealedChunks.length > 0 ? sealedChunks[sealedChunks.length - 1] : undefined;
+    const lastSealed = lastSealedChunk ? BigInt(lastSealedChunk.toBlock) : null;
+    const recordedHot = manifest.hotHead(protocolId);
+    const recordedHotTo = recordedHot ? BigInt(recordedHot.toBlock) : null;
+    // Stale hot head: its toBlock is no further forward than what's already
+    // sealed. Clear it from the manifest and delete the file (best-effort).
+    let validHot = recordedHot;
+    if (recordedHot && lastSealed !== null && recordedHotTo !== null && recordedHotTo <= lastSealed) {
+        process.stderr.write(`orchestrator: ${protocolId} — stale hot head (toBlock ${recordedHot.toBlock} <= sealed ${numberToHex(lastSealed)}); cleaning up\n`);
+        await manifest.clearHotHead(protocolId);
+        await archive.delete(recordedHot);
+        validHot = undefined;
+    }
+    const startBlock = validHot && recordedHotTo !== null
+        ? recordedHotTo
+        : lastSealed !== null
+            ? lastSealed
+            : BigInt(protocol.fromBlock);
+    if (startBlock > tip) {
+        return { ranBatches: 0, sealedChunks: 0, finalHotHead: null };
+    }
+    // 2. Load the prior hot head's events — the seed for the first batch.
+    let trailing;
+    if (validHot) {
+        trailing = {
+            events: await archive.readEvents(validHot),
+            fromBlock: BigInt(validHot.fromBlock),
+        };
+    }
+    const sizeLimit = protocol.maxSizeBytes ?? args.sizeLimit;
+    // 3. Batch loop. Each batch's trailing carries into the next as the seed.
+    let batchStart = startBlock;
+    let totalSealed = 0;
+    let ranBatches = 0;
+    while (batchStart <= tip) {
+        const candidate = batchStart + batchSize - 1n;
+        const batchEnd = candidate > tip ? tip : candidate;
+        const result = await runProtocolOnce({
+            client: args.client,
+            protocolId,
+            fromBlock: batchStart,
+            toBlock: batchEnd,
+            events: protocol.events,
+            sizeLimit,
+            window: args.window,
+            archive,
+            manifest,
+            ...(trailing && {
+                hotHead: { events: trailing.events, fromBlock: trailing.fromBlock },
+            }),
+            trailingMode: "suspend",
+        });
+        totalSealed += result.sealed.length;
+        trailing = result.trailing;
+        batchStart = batchEnd + 1n;
+        ranBatches += 1;
+    }
+    // 4. Persist the final trailing accumulator as the new hot head. Even when
+    // events is empty the range advances, so the manifest must reflect it.
+    let finalHotHead = null;
+    if (trailing) {
+        const trailingState = trailing;
+        finalHotHead = await archive.writeHotHead(protocolId, trailingState.events, {
+            from: trailingState.fromBlock,
+            to: tip + 1n,
+        });
+        await manifest.setHotHead(protocolId, finalHotHead);
+        // Delete the prior hot-head file if the new file has a different name
+        // (it does whenever the range advanced — i.e., always, unless a no-op tick).
+        if (validHot && validHot.file !== finalHotHead.file) {
+            await archive.delete(validHot);
+        }
+    }
+    return { ranBatches, sealedChunks: totalSealed, finalHotHead };
+}
+async function main() {
+    const args = parseCliArgs();
+    const storeConfig = parseStoreTarget(args.output);
+    // The lockfile is filesystem-only. For a local output dir it lives there; for a
+    // gs:// target there is no local dir, so default to the cwd (override with
+    // --lock-dir). (On Cloud Run, single-execution scheduling is the real guard.)
+    const lockDir = args.lockDir ?? (storeConfig.protocol === "disk" ? storeConfig.baseDir : resolve("."));
+    // Dry-run is read-only: don't create dirs or hold the lock.
+    if (!args.dryRun) {
+        mkdirSync(lockDir, { recursive: true });
+        const lockPath = join(lockDir, ".orchestrator.lock");
+        if (!acquireLock(lockPath)) {
+            process.stderr.write(`orchestrator: another instance is running (lock ${lockPath}); exiting\n`);
+            return;
+        }
+    }
+    const protocols = loadAllProtocols(args.configPath);
+    const store = createStore({ ...storeConfig, dryRun: args.dryRun });
+    const archive = new ChunkArchive(store);
+    const manifest = await Manifest.load(store, undefined, { signer: signerFromEnv() });
+    const client = createPublicClient({ transport: http(args.rpc) });
+    const tip = (await finalizedBlock(client)) ?? (await client.getBlockNumber()) - args.confirmations;
+    const ids = args.protocolId
+        ? args.protocolId in protocols
+            ? [args.protocolId]
+            : fail(`unknown --protocol-id "${args.protocolId}". Known: ${Object.keys(protocols).join(", ")}`)
+        : Object.keys(protocols);
+    let ran = 0;
+    let skipped = 0;
+    let failed = 0;
+    for (const protocolId of ids) {
+        const protocol = protocols[protocolId];
+        if (!protocol)
+            continue;
+        try {
+            await assertChainId(client, protocol.chainId);
+        }
+        catch (err) {
+            process.stderr.write(`orchestrator: skipping ${protocolId} — ${err.message}\n`);
+            skipped += 1;
+            continue;
+        }
+        const startFrom = manifest.lastCoveredBlock(protocolId) ?? BigInt(protocol.fromBlock);
+        if (startFrom > tip) {
+            skipped += 1;
+            continue;
+        }
+        if (args.dryRun) {
+            const batches = Math.ceil(Number((tip - startFrom + 1n) / args.batchSize)) || 1;
+            process.stderr.write(`orchestrator: [dry-run] ${protocolId} would scan ` +
+                `[${numberToHex(startFrom)}, ${numberToHex(tip)}] in ${batches} batch(es)\n`);
+            ran += 1;
+            continue;
+        }
+        try {
+            const result = await processProtocol({
+                client,
+                protocolId,
+                protocol,
+                archive,
+                manifest,
+                tip,
+                batchSize: args.batchSize,
+                window: args.window,
+                sizeLimit: args.sizeLimit,
+            });
+            const hot = result.finalHotHead
+                ? ` + hot head [${result.finalHotHead.fromBlock}, ${result.finalHotHead.toBlock})`
+                : "";
+            process.stderr.write(`orchestrator: ${protocolId} ran ${result.ranBatches} batch(es), ` +
+                `sealed ${result.sealedChunks} chunk(s)${hot}\n`);
+            ran += 1;
+        }
+        catch (err) {
+            process.stderr.write(`orchestrator: ${protocolId} failed — ${err.message}\n`);
+            failed += 1;
+        }
+    }
+    process.stderr.write(`orchestrator: ${ran} ran, ${skipped} skipped, ${failed} failed [tip ${numberToHex(tip)}]` +
+        (args.dryRun ? " (dry-run)\n" : "\n"));
+    if (failed > 0)
+        process.exit(2);
+}
+function isMainModule() {
+    try {
+        return realpathSync(process.argv[1] ?? "") === realpathSync(fileURLToPath(import.meta.url));
+    }
+    catch {
+        return false;
+    }
+}
+if (isMainModule()) {
+    main().catch((err) => {
+        fail(err instanceof Error ? err.message : String(err));
+    });
+}
+export { main, parseCliArgs, processProtocol };
+//# sourceMappingURL=cli.js.map

package/dist/orchestrator/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/orchestrator/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EACL,SAAS,EACT,QAAQ,EACR,aAAa,EACb,SAAS,EACT,UAAU,EACV,YAAY,EACZ,YAAY,GACb,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AAE7D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAC5C,MAAM,kBAAkB,GAAG,OAAQ,CAAC;AAEpC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;;CAuBb,CAAC;AAEF,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACvB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC9C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,IAAqC,EAAU,EAAE;QAC7D,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,IAAI,CAAC,sBAAsB,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC3F,IAAI,SAAS,IAAI,EAAE;QAAE,IAAI,CAAC,sCAAsC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAExF,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;QAChB,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,6CAA6C;QACzE,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACrE,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC;QACjC,SAAS;QACT,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,qBAAqB;QAC1F,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc;QAC9D,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;QACnF,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK;KACnC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAChC,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;YAC/C,SAAS,CAAC,EAAE,CAAC,CAAC;YACd,MAAM,OAAO,GAAG,GAAS,EAAE;gBACzB,IAAI,CAAC;oBACH,UAAU,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;gBAAC,MAAM,CAAC;oBACP,kBAAkB;gBACpB,CAAC;YACH,CAAC,CAAC;YACF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC5B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;gBACxB,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;gBACzB,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;YACjD,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAEjC,IAAI,QAAgB,CAAC;YACrB,IAAI,CAAC;gBACH,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7D,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,UAAU,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;gBAAC,MAAM,CAAC;oBACP,gCAAgC;gBAClC,CAAC;gBACD,SAAS;YACX,CAAC;YACD,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAC1B,OAAO,KAAK,CAAC,CAAC,cAAc;YAC9B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,EAAE,GAAI,CAA2B,CAAC,IAAI,CAAC;gBAC7C,IAAI,EAAE,KAAK,OAAO,EAAE,CAAC;oBACnB,IAAI,CAAC;wBACH,UAAU,CAAC,IAAI,CAAC,CAAC;oBACnB,CAAC;oBAAC,MAAM,CAAC;wBACP,UAAU;oBACZ,CAAC;oBACD,SAAS;gBACX,CAAC;gBACD,OAAO,KAAK,CAAC,CAAC,kDAAkD;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,kDAAkD;AAClD,KAAK,UAAU,eAAe,CAAC,IAU9B;IACC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAEzE,8EAA8E;IAC9E,6EAA6E;IAC7E,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IACvD,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpG,MAAM,UAAU,GAAG,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE5E,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,wEAAwE;IACxE,wEAAwE;IACxE,IAAI,QAAQ,GAAG,WAAW,CAAC;IAC3B,IAAI,WAAW,IAAI,UAAU,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,IAAI,UAAU,EAAE,CAAC;QAChG,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iBAAiB,UAAU,8BAA8B,WAAW,CAAC,OAAO,cAAc,WAAW,CAAC,UAAU,CAAC,kBAAkB,CACpI,CAAC;QACF,MAAM,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAClC,QAAQ,GAAG,SAAS,CAAC;IACvB,CAAC;IAED,MAAM,UAAU,GACd,QAAQ,IAAI,aAAa,KAAK,IAAI;QAChC,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,UAAU,KAAK,IAAI;YACnB,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAEnC,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QACrB,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAChE,CAAC;IAED,sEAAsE;IACtE,IAAI,QAAoG,CAAC;IACzG,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,GAAG;YACT,MAAM,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC1C,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;SACtC,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,CAAC;IAE1D,0EAA0E;IAC1E,IAAI,UAAU,GAAG,UAAU,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,UAAU,IAAI,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,UAAU,GAAG,SAAS,GAAG,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAEnD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;YACnC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU;YACV,SAAS,EAAE,UAAU;YACrB,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,SAAS;YACT,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;YACP,QAAQ;YACR,GAAG,CAAC,QAAQ,IAAI;gBACd,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE;aACpE,CAAC;YACF,YAAY,EAAE,SAAS;SACxB,CAAC,CAAC;QACH,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;QACpC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC3B,UAAU,GAAG,QAAQ,GAAG,EAAE,CAAC;QAC3B,UAAU,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,2EAA2E;IAC3E,uEAAuE;IACvE,IAAI,YAAY,GAAqB,IAAI,CAAC;IAC1C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,aAAa,GAAG,QAAQ,CAAC;QAC/B,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,aAAa,CAAC,MAAM,EAAE;YAC1E,IAAI,EAAE,aAAa,CAAC,SAAS;YAC7B,EAAE,EAAE,GAAG,GAAG,EAAE;SACb,CAAC,CAAC;QACH,MAAM,QAAQ,CAAC,UAAU,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEpD,sEAAsE;QACtE,6EAA6E;QAC7E,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AACjE,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,gFAAgF;IAChF,2EAA2E;IAC3E,8EAA8E;IAC9E,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IAE1F,4DAA4D;IAC5D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,mDAAmD,QAAQ,cAAc,CAC1E,CAAC;YACF,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,GAAiB,kBAAkB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/E,MAAM,GAAG,GACP,CAAC,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC;IAEzF,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU;QACzB,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,SAAS;YAC5B,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;YACnB,CAAC,CAAC,IAAI,CAAC,0BAA0B,IAAI,CAAC,UAAU,aAAa,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAE3B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,UAAU,IAAI,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,IAAI,CAAC;YACH,MAAM,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,MAAO,GAAa,CAAC,OAAO,IAAI,CAAC,CAAC;YAC3F,OAAO,IAAI,CAAC,CAAC;YACb,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACtF,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,GAAG,SAAS,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC;YAChF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,2BAA2B,UAAU,cAAc;gBACjD,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,WAAW,CAAC,GAAG,CAAC,QAAQ,OAAO,cAAc,CAC/E,CAAC;YACF,GAAG,IAAI,CAAC,CAAC;YACT,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;gBACnC,MAAM;gBACN,UAAU;gBACV,QAAQ;gBACR,OAAO;gBACP,QAAQ;gBACR,GAAG;gBACH,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,YAAY;gBAC7B,CAAC,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,SAAS,KAAK,MAAM,CAAC,YAAY,CAAC,OAAO,GAAG;gBAClF,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iBAAiB,UAAU,QAAQ,MAAM,CAAC,UAAU,cAAc;gBAChE,UAAU,MAAM,CAAC,YAAY,YAAY,GAAG,IAAI,CACnD,CAAC;YACF,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,UAAU,aAAc,GAAa,CAAC,OAAO,IAAI,CAAC,CAAC;YACzF,MAAM,IAAI,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iBAAiB,GAAG,SAAS,OAAO,aAAa,MAAM,gBAAgB,WAAW,CAAC,GAAG,CAAC,GAAG;QACxF,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CACxC,CAAC;IACF,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,YAAY,EAAE,EAAE,CAAC;IACnB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QAC5B,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,eAAe,EAAE,CAAC"}

package/dist/orchestrator/pipeline.d.ts

@@ -0,0 +1,32 @@
+import type { PublicClient } from "viem";
+import type { CanonicalEvent } from "../scraper/normalize.js";
+import type { EventFilter } from "../scraper/config.js";
+import { ChunkArchive } from "../chunk-builder/archive.js";
+import { Manifest } from "@saga-sync/core";
+import type { ChunkMeta } from "@saga-sync/core";
+export type RunProtocolOptions = {
+    client: PublicClient;
+    protocolId: string;
+    fromBlock: bigint;
+    toBlock: bigint;
+    events: EventFilter[];
+    sizeLimit: number;
+    window: number;
+    archive: ChunkArchive;
+    manifest: Manifest;
+    hotHead?: {
+        events: CanonicalEvent[];
+        fromBlock: bigint;
+    };
+    trailingMode?: "seal" | "suspend";
+};
+export type RunProtocolResult = {
+    sealed: ChunkMeta[];
+    trailing?: {
+        events: CanonicalEvent[];
+        fromBlock: bigint;
+        toBlock: bigint;
+    };
+};
+export declare function runProtocolOnce(opts: RunProtocolOptions): Promise<RunProtocolResult>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/orchestrator/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/orchestrator/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AAGzC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;IAInB,OAAO,CAAC,EAAE;QAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAG1D,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7E,CAAC;AAKF,wBAAsB,eAAe,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAyB1F"}

package/dist/orchestrator/pipeline.js

@@ -0,0 +1,32 @@
+import { scrape } from "../scraper/scrape.js";
+import { normalize } from "../scraper/normalize.js";
+import { processStream } from "../chunk-builder/cli.js";
+// Compose scraper + chunk-builder in-process: scrape() yields raw logs, we
+// normalize and stringify into NDJSON lines, and processStream consumes them.
+// No subprocess, no stdio piping — errors propagate as exceptions.
+export async function runProtocolOnce(opts) {
+    async function* lines() {
+        for await (const log of scrape(opts.client, {
+            fromBlock: opts.fromBlock,
+            toBlock: opts.toBlock,
+            events: opts.events,
+            window: opts.window,
+        })) {
+            yield JSON.stringify(normalize(log));
+        }
+    }
+    return processStream(lines(), {
+        protocolId: opts.protocolId,
+        fromBlock: opts.fromBlock,
+        // scraper's inclusive [from, to] → chunk-builder's half-open [from, to+1)
+        toBlock: opts.toBlock + 1n,
+        sizeLimit: opts.sizeLimit,
+        archive: opts.archive,
+        manifest: opts.manifest,
+        ...(opts.hotHead && {
+            seed: { events: opts.hotHead.events, chunkFrom: opts.hotHead.fromBlock },
+        }),
+        ...(opts.trailingMode && { trailingMode: opts.trailingMode }),
+    });
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/orchestrator/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/orchestrator/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGpD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AA6BxD,2EAA2E;AAC3E,8EAA8E;AAC9E,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAwB;IAC5D,KAAK,SAAS,CAAC,CAAC,KAAK;QACnB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE;YAC1C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,EAAE,CAAC;YACH,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC,KAAK,EAAE,EAAE;QAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,0EAA0E;QAC1E,OAAO,EAAE,IAAI,CAAC,OAAO,GAAG,EAAE;QAC1B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI;YAClB,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;SACzE,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;KAC9D,CAAC,CAAC;AACL,CAAC"}

package/dist/scraper/cli.d.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+import type { Hex, PublicClient } from "viem";
+export declare function finalizedBlock(client: PublicClient): Promise<bigint | null>;
+export declare function assertChainId(client: PublicClient, expected: Hex): Promise<void>;
+//# sourceMappingURL=cli.d.ts.map

package/dist/scraper/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/scraper/cli.ts"],"names":[],"mappings":";AAMA,OAAO,KAAK,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AAiF9C,wBAAsB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOjF;AAID,wBAAsB,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAOtF"}

package/dist/scraper/cli.js

@@ -0,0 +1,153 @@
+#!/usr/bin/env node
+import { parseArgs } from "node:util";
+import { dirname, resolve } from "node:path";
+import { realpathSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { createPublicClient, http, numberToHex } from "viem";
+import { loadConfig } from "./config.js";
+import { Cursor } from "./cursor.js";
+import { scrape } from "./scrape.js";
+import { normalize } from "./normalize.js";
+import { createStore } from "../storage/index.js";
+const DEFAULT_WINDOW = 2000;
+const DEFAULT_CONFIRMATIONS = 12n;
+const USAGE = `scraper — fetch and normalize privacy-protocol events to NDJSON on stdout
+
+Usage:
+  scraper --config <path> --protocol-id <id> --rpc <url> [options]
+
+Required:
+  --config <path>        scraper config JSON
+  --protocol-id <id>     key in config.protocols to scrape
+  --rpc <url>            Ethereum JSON-RPC URL
+
+Options:
+  --from-block <hex>     override cursor / config fromBlock (e.g. for backfills)
+  --to-block <hex>       override the resolved finalized/head block
+  --confirmations <n>    fallback reorg buffer, default 12 (used only when the RPC
+                         does not support the "finalized" block tag)
+  --window <n>           blocks per eth_getLogs call, default 2000
+  --cursor-dir <path>    directory for cursor.json, default = config file's directory
+  --dry-run              do not persist the cursor update
+  --help                 show this message
+`;
+function fail(msg) {
+    process.stderr.write(`scraper: ${msg}\n`);
+    process.exit(1);
+}
+function parseCliArgs() {
+    const { values } = parseArgs({
+        options: {
+            config: { type: "string" },
+            "protocol-id": { type: "string" },
+            rpc: { type: "string" },
+            "from-block": { type: "string" },
+            "to-block": { type: "string" },
+            confirmations: { type: "string" },
+            window: { type: "string" },
+            "cursor-dir": { type: "string" },
+            "dry-run": { type: "boolean", default: false },
+            help: { type: "boolean", default: false },
+        },
+    });
+    if (values.help) {
+        process.stdout.write(USAGE);
+        process.exit(0);
+    }
+    const need = (name) => {
+        const v = values[name];
+        if (!v)
+            fail(`missing required --${name}\n\n${USAGE}`);
+        return v;
+    };
+    const configPath = resolve(need("config"));
+    return {
+        configPath,
+        protocolId: need("protocol-id"),
+        rpc: need("rpc"),
+        fromBlock: values["from-block"] ? BigInt(values["from-block"]) : undefined,
+        toBlock: values["to-block"] ? BigInt(values["to-block"]) : undefined,
+        confirmations: values.confirmations ? BigInt(values.confirmations) : DEFAULT_CONFIRMATIONS,
+        window: values.window ? Number(values.window) : DEFAULT_WINDOW,
+        cursorDir: resolve(values["cursor-dir"] ?? dirname(configPath)),
+        dryRun: values["dry-run"] ?? false,
+    };
+}
+// The chain's own finalized block — reorg-proof, no per-chain table needed.
+// Returns null if the RPC/chain does not support the "finalized" tag, which
+// triggers the --confirmations fallback in step 2.
+export async function finalizedBlock(client) {
+    try {
+        const block = await client.getBlock({ blockTag: "finalized" });
+        return block.number;
+    }
+    catch {
+        return null;
+    }
+}
+// Verify the RPC is actually on the chain the config declares — catches a
+// misconfigured --rpc pointed at the wrong network before any data is emitted.
+export async function assertChainId(client, expected) {
+    const actual = await client.getChainId();
+    if (BigInt(actual) !== BigInt(expected)) {
+        throw new Error(`chain mismatch: config declares chainId ${expected}, but the RPC reports ${numberToHex(actual)}`);
+    }
+}
+async function main() {
+    const args = parseCliArgs();
+    const config = loadConfig(args.configPath, args.protocolId);
+    const cursorStore = createStore({
+        protocol: "disk",
+        baseDir: args.cursorDir,
+        dryRun: args.dryRun,
+    });
+    const cursor = await Cursor.load(cursorStore);
+    // 1. Connect to the RPC (and verify it is on the chain the config declares)
+    const client = createPublicClient({ transport: http(args.rpc) });
+    await assertChainId(client, config.chainId);
+    // 2. Resolve the block range to scan
+    const toBlock = args.toBlock ??
+        (await finalizedBlock(client)) ??
+        (await client.getBlockNumber()) - args.confirmations;
+    const lastScraped = cursor.lastScrapedBlock(args.protocolId);
+    const fromBlock = args.fromBlock ??
+        (lastScraped ? BigInt(lastScraped) + 1n : undefined) ??
+        BigInt(config.fromBlock);
+    if (fromBlock > toBlock) {
+        process.stderr.write(`scraper: nothing to do — fromBlock ${numberToHex(fromBlock)} > toBlock ${numberToHex(toBlock)}\n`);
+        return;
+    }
+    // 3. Fetch logs (windowed, streaming)  +  4. Normalize  +  5. Emit
+    let count = 0;
+    for await (const log of scrape(client, {
+        fromBlock,
+        toBlock,
+        events: config.events,
+        window: args.window,
+    })) {
+        process.stdout.write(JSON.stringify(normalize(log)) + "\n");
+        count += 1;
+    }
+    // persist progress — one cursor update per successful run. In dry-run the
+    // cursor store is DryRunStore-wrapped, so this set() no-ops.
+    await cursor.set(args.protocolId, numberToHex(toBlock));
+    process.stderr.write(`scraper: ${count} event(s) for ${args.protocolId} ` +
+        `[${numberToHex(fromBlock)}, ${numberToHex(toBlock)}]` +
+        (args.dryRun ? " (dry-run: cursor not updated)\n" : "\n"));
+}
+// Only run the pipeline when invoked as the entry point — importing this module
+// (e.g. from tests) must not kick off a scrape.
+function isMainModule() {
+    try {
+        return realpathSync(process.argv[1] ?? "") === realpathSync(fileURLToPath(import.meta.url));
+    }
+    catch {
+        return false;
+    }
+}
+if (isMainModule()) {
+    main().catch((err) => {
+        fail(err instanceof Error ? err.message : String(err));
+    });
+}
+//# sourceMappingURL=cli.js.map

package/dist/scraper/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/scraper/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AAE7D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;CAmBb,CAAC;AAEF,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACvB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC9C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,IAAsC,EAAU,EAAE;QAC9D,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,IAAI,CAAC,sBAAsB,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE3C,OAAO;QACL,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;QAC/B,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1E,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACpE,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,qBAAqB;QAC1F,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc;QAC9D,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK;KACnC,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,4EAA4E;AAC5E,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAoB;IACvD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/D,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAoB,EAAE,QAAa;IACrE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,yBAAyB,WAAW,CAAC,MAAM,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAG,WAAW,CAAC;QAC9B,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,IAAI,CAAC,SAAS;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAE9C,4EAA4E;IAC5E,MAAM,MAAM,GAAG,kBAAkB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACjE,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,qCAAqC;IACrC,MAAM,OAAO,GACX,IAAI,CAAC,OAAO;QACZ,CAAC,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC;IAEvD,MAAM,WAAW,GAAG,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7D,MAAM,SAAS,GACb,IAAI,CAAC,SAAS;QACd,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAE3B,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sCAAsC,WAAW,CAAC,SAAS,CAAC,cAAc,WAAW,CAAC,OAAO,CAAC,IAAI,CACnG,CAAC;QACF,OAAO;IACT,CAAC;IAED,mEAAmE;IACnE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE;QACrC,SAAS;QACT,OAAO;QACP,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC,EAAE,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC5D,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,0EAA0E;IAC1E,6DAA6D;IAC7D,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,YAAY,KAAK,iBAAiB,IAAI,CAAC,UAAU,GAAG;QAClD,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,WAAW,CAAC,OAAO,CAAC,GAAG;QACtD,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5D,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,gDAAgD;AAChD,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,YAAY,EAAE,EAAE,CAAC;IACnB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QAC5B,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/scraper/config.d.ts

@@ -0,0 +1,15 @@
+import type { Hex } from "viem";
+export type EventFilter = {
+    contractAddress: Hex;
+    eventTopic: Hex;
+    filter?: Hex[];
+};
+export type ScraperTarget = {
+    chainId: Hex;
+    fromBlock: Hex;
+    events: EventFilter[];
+    maxSizeBytes?: number;
+};
+export declare function loadConfig(path: string, protocolId: string): ScraperTarget;
+export declare function loadAllProtocols(path: string): Record<string, ScraperTarget>;
+//# sourceMappingURL=config.d.ts.map

package/dist/scraper/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/scraper/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAmChC,MAAM,MAAM,WAAW,GAAG;IACxB,eAAe,EAAE,GAAG,CAAC;IACrB,UAAU,EAAE,GAAG,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,GAAG,CAAC;IACb,SAAS,EAAE,GAAG,CAAC;IACf,MAAM,EAAE,WAAW,EAAE,CAAC;IAGtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAwCF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,CAO1E;AAID,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAO5E"}