@saga-sync/producer 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/README.md +30 -5
- package/dist/chunk-builder/accumulator.d.ts +4 -4
- package/dist/chunk-builder/accumulator.d.ts.map +1 -1
- package/dist/chunk-builder/accumulator.js.map +1 -1
- package/dist/chunk-builder/archive.d.ts +5 -5
- package/dist/chunk-builder/archive.d.ts.map +1 -1
- package/dist/chunk-builder/archive.js +1 -1
- package/dist/chunk-builder/archive.js.map +1 -1
- package/dist/chunk-builder/cli.d.ts +4 -3
- package/dist/chunk-builder/cli.d.ts.map +1 -1
- package/dist/chunk-builder/cli.js +5 -3
- package/dist/chunk-builder/cli.js.map +1 -1
- package/dist/keygen.d.ts +1 -1
- package/dist/keygen.d.ts.map +1 -1
- package/dist/keygen.js +38 -6
- package/dist/keygen.js.map +1 -1
- package/dist/orchestrator/cli.d.ts +3 -3
- package/dist/orchestrator/cli.d.ts.map +1 -1
- package/dist/orchestrator/cli.js +33 -12
- package/dist/orchestrator/cli.js.map +1 -1
- package/dist/orchestrator/pipeline.d.ts +5 -8
- package/dist/orchestrator/pipeline.d.ts.map +1 -1
- package/dist/orchestrator/pipeline.js +5 -12
- package/dist/orchestrator/pipeline.js.map +1 -1
- package/dist/scraper/cli.js +10 -1
- package/dist/scraper/cli.js.map +1 -1
- package/dist/scraper/config.d.ts +11 -3
- package/dist/scraper/config.d.ts.map +1 -1
- package/dist/scraper/config.js +46 -7
- package/dist/scraper/config.js.map +1 -1
- package/dist/sources/index.d.ts +14 -0
- package/dist/sources/index.d.ts.map +1 -0
- package/dist/sources/index.js +25 -0
- package/dist/sources/index.js.map +1 -0
- package/dist/sources/rpc-log-source.d.ts +17 -0
- package/dist/sources/rpc-log-source.d.ts.map +1 -0
- package/dist/sources/rpc-log-source.js +43 -0
- package/dist/sources/rpc-log-source.js.map +1 -0
- package/dist/sources/subsquid-source.d.ts +38 -0
- package/dist/sources/subsquid-source.d.ts.map +1 -0
- package/dist/sources/subsquid-source.js +191 -0
- package/dist/sources/subsquid-source.js.map +1 -0
- package/dist/sources/types.d.ts +7 -0
- package/dist/sources/types.d.ts.map +1 -0
- package/dist/sources/types.js +2 -0
- package/dist/sources/types.js.map +1 -0
- package/dist/storage/gcs-store.d.ts.map +1 -1
- package/dist/storage/gcs-store.js +4 -1
- package/dist/storage/gcs-store.js.map +1 -1
- package/package.json +3 -3
package/dist/scraper/cli.js
CHANGED
|
@@ -127,11 +127,20 @@ async function main() {
|
|
|
127
127
|
return;
|
|
128
128
|
}
|
|
129
129
|
// 3. Fetch logs (windowed, streaming) + 4. Normalize + 5. Emit
|
|
130
|
+
//
|
|
131
|
+
// This CLI is the raw eth_getLogs path and nothing else. A stream configured
|
|
132
|
+
// with another source has no logs to emit, so say so rather than producing an
|
|
133
|
+
// empty NDJSON stream that looks like "no activity in this range".
|
|
134
|
+
if (config.source.kind !== "rpc" || !config.events) {
|
|
135
|
+
fail(`protocol "${args.protocolId}" uses the "${config.source.kind}" source; ` +
|
|
136
|
+
`the standalone scraper only reads logs. Run it through the orchestrator.`);
|
|
137
|
+
}
|
|
138
|
+
const eventFilters = config.events;
|
|
130
139
|
let count = 0;
|
|
131
140
|
for await (const log of scrape(client, {
|
|
132
141
|
fromBlock,
|
|
133
142
|
toBlock,
|
|
134
|
-
events:
|
|
143
|
+
events: eventFilters,
|
|
135
144
|
window: args.window,
|
|
136
145
|
})) {
|
|
137
146
|
process.stdout.write(JSON.stringify(normalize(log)) + "\n");
|
package/dist/scraper/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/scraper/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AAE7D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;CAmBb,CAAC;AAEF,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACvB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC9C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,IAAsC,EAAU,EAAE;QAC9D,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,IAAI,CAAC,sBAAsB,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE3C,OAAO;QACL,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;QAC/B,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1E,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACpE,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,qBAAqB;QAC1F,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc;QAC9D,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK;KACnC,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,+EAA+E;AAC/E,iFAAiF;AACjF,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,kBAAkB,CAAC;QACxB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;KACtE,CAAC,CAAC;AACL,CAAC;AAED,4EAA4E;AAC5E,4EAA4E;AAC5E,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAoB;IACvD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/D,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAoB,EAAE,QAAa;IACrE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,yBAAyB,WAAW,CAAC,MAAM,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAG,WAAW,CAAC;QAC9B,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,IAAI,CAAC,SAAS;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAE9C,4EAA4E;IAC5E,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,qCAAqC;IACrC,MAAM,OAAO,GACX,IAAI,CAAC,OAAO;QACZ,CAAC,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC;IAEvD,MAAM,WAAW,GAAG,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7D,MAAM,SAAS,GACb,IAAI,CAAC,SAAS;QACd,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAE3B,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sCAAsC,WAAW,CAAC,SAAS,CAAC,cAAc,WAAW,CAAC,OAAO,CAAC,IAAI,CACnG,CAAC;QACF,OAAO;IACT,CAAC;IAED,mEAAmE;IACnE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE;QACrC,SAAS;QACT,OAAO;QACP,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/scraper/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,MAAM,CAAC;AAE7D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;CAmBb,CAAC;AAEF,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACvB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC9C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,IAAsC,EAAU,EAAE;QAC9D,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,IAAI,CAAC,sBAAsB,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE3C,OAAO;QACL,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;QAC/B,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1E,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACpE,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,qBAAqB;QAC1F,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc;QAC9D,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK;KACnC,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,+EAA+E;AAC/E,iFAAiF;AACjF,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,kBAAkB,CAAC;QACxB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;KACtE,CAAC,CAAC;AACL,CAAC;AAED,4EAA4E;AAC5E,4EAA4E;AAC5E,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAoB;IACvD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/D,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAoB,EAAE,QAAa;IACrE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CACb,2CAA2C,QAAQ,yBAAyB,WAAW,CAAC,MAAM,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAG,WAAW,CAAC;QAC9B,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,IAAI,CAAC,SAAS;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAE9C,4EAA4E;IAC5E,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,qCAAqC;IACrC,MAAM,OAAO,GACX,IAAI,CAAC,OAAO;QACZ,CAAC,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC;IAEvD,MAAM,WAAW,GAAG,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7D,MAAM,SAAS,GACb,IAAI,CAAC,SAAS;QACd,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAE3B,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,sCAAsC,WAAW,CAAC,SAAS,CAAC,cAAc,WAAW,CAAC,OAAO,CAAC,IAAI,CACnG,CAAC;QACF,OAAO;IACT,CAAC;IAED,mEAAmE;IACnE,EAAE;IACF,6EAA6E;IAC7E,8EAA8E;IAC9E,mEAAmE;IACnE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnD,IAAI,CACF,aAAa,IAAI,CAAC,UAAU,eAAe,MAAM,CAAC,MAAM,CAAC,IAAI,YAAY;YACvE,0EAA0E,CAC7E,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE;QACrC,SAAS;QACT,OAAO;QACP,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC,EAAE,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC5D,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,0EAA0E;IAC1E,6DAA6D;IAC7D,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,YAAY,KAAK,iBAAiB,IAAI,CAAC,UAAU,GAAG;QAClD,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,WAAW,CAAC,OAAO,CAAC,GAAG;QACtD,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5D,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,gDAAgD;AAChD,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,YAAY,EAAE,EAAE,CAAC;IACnB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QAC5B,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/scraper/config.d.ts
CHANGED
|
@@ -4,15 +4,23 @@ export type EventFilter = {
|
|
|
4
4
|
eventTopic: Hex;
|
|
5
5
|
filter?: Hex[];
|
|
6
6
|
};
|
|
7
|
+
export type SourceConfig = {
|
|
8
|
+
kind: "rpc";
|
|
9
|
+
} | {
|
|
10
|
+
kind: "subsquid";
|
|
11
|
+
endpoint: string;
|
|
12
|
+
entity: string;
|
|
13
|
+
};
|
|
7
14
|
export type ScraperTarget = {
|
|
8
15
|
chainId: Hex;
|
|
9
16
|
fromBlock: Hex;
|
|
10
|
-
|
|
17
|
+
source: SourceConfig;
|
|
18
|
+
events?: EventFilter[];
|
|
11
19
|
maxSizeBytes?: number;
|
|
12
20
|
protocol?: string;
|
|
13
21
|
protocolMetadata?: Record<string, unknown>;
|
|
14
|
-
trackedAddresses
|
|
15
|
-
trackedEventTopics
|
|
22
|
+
trackedAddresses?: Hex[];
|
|
23
|
+
trackedEventTopics?: Hex[];
|
|
16
24
|
};
|
|
17
25
|
export declare function loadConfig(path: string, protocolId: string): ScraperTarget;
|
|
18
26
|
export declare function loadAllProtocols(path: string): Record<string, ScraperTarget>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/scraper/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/scraper/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAgFhC,MAAM,MAAM,WAAW,GAAG;IACxB,eAAe,EAAE,GAAG,CAAC;IACrB,UAAU,EAAE,GAAG,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC;CAChB,CAAC;AAIF,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,KAAK,CAAA;CAAE,GACf;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3D,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,GAAG,CAAC;IACb,SAAS,EAAE,GAAG,CAAC;IACf,MAAM,EAAE,YAAY,CAAC;IAErB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAGvB,YAAY,CAAC,EAAE,MAAM,CAAC;IAKtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,gBAAgB,CAAC,EAAE,GAAG,EAAE,CAAC;IACzB,kBAAkB,CAAC,EAAE,GAAG,EAAE,CAAC;CAC5B,CAAC;AAmDF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,CAO1E;AAID,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAO5E"}
|
package/dist/scraper/config.js
CHANGED
|
@@ -23,10 +23,32 @@ const chunkSettingsSchema = z
|
|
|
23
23
|
})
|
|
24
24
|
.passthrough()
|
|
25
25
|
.optional();
|
|
26
|
+
// Where a stream's records come from. Defaults to "rpc" when absent, so every
|
|
27
|
+
// config written before sources existed stays valid and unchanged in meaning.
|
|
28
|
+
//
|
|
29
|
+
// "subsquid" mirrors a Squid GraphQL index. That is a deliberate downgrade in
|
|
30
|
+
// provenance — the bytes are reproducible only against that third-party index,
|
|
31
|
+
// not against the chain — so it is opt-in per stream and recorded in the
|
|
32
|
+
// manifest's protocolMetadata, never inferred.
|
|
33
|
+
const sourceSchema = z
|
|
34
|
+
.discriminatedUnion("kind", [
|
|
35
|
+
z.object({ kind: z.literal("rpc") }),
|
|
36
|
+
z.object({
|
|
37
|
+
kind: z.literal("subsquid"),
|
|
38
|
+
endpoint: z.string().url("expected an absolute http(s) URL"),
|
|
39
|
+
// Which entity to mirror. Only "railgun-operation" exists today; naming it
|
|
40
|
+
// in config keeps the source from hardcoding one protocol's shape.
|
|
41
|
+
entity: z.string().min(1),
|
|
42
|
+
}),
|
|
43
|
+
])
|
|
44
|
+
.optional();
|
|
26
45
|
const targetSchema = z.object({
|
|
27
46
|
chainId: quantity,
|
|
28
47
|
fromBlock: quantity,
|
|
29
|
-
|
|
48
|
+
// Required for an rpc source, forbidden for the others — enforced below, since
|
|
49
|
+
// zod cannot express "depends on a sibling field" in the shape alone.
|
|
50
|
+
events: z.array(eventFilterSchema).optional(),
|
|
51
|
+
source: sourceSchema,
|
|
30
52
|
chunkSettings: chunkSettingsSchema,
|
|
31
53
|
// Descriptive metadata surfaced in the manifest (all optional). `protocol` is
|
|
32
54
|
// the family name (e.g. "tornado-cash"); `protocolMetadata` is a free-form
|
|
@@ -34,6 +56,22 @@ const targetSchema = z.object({
|
|
|
34
56
|
// protocol-specific fields. See the manifest's write-once immutability note.
|
|
35
57
|
protocol: z.string().optional(),
|
|
36
58
|
protocolMetadata: z.record(z.unknown()).optional(),
|
|
59
|
+
}).superRefine((t, ctx) => {
|
|
60
|
+
const kind = t.source?.kind ?? "rpc";
|
|
61
|
+
if (kind === "rpc" && (t.events === undefined || t.events.length === 0)) {
|
|
62
|
+
ctx.addIssue({
|
|
63
|
+
code: z.ZodIssueCode.custom,
|
|
64
|
+
path: ["events"],
|
|
65
|
+
message: "at least one event filter is required for an rpc source",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
if (kind !== "rpc" && t.events !== undefined) {
|
|
69
|
+
ctx.addIssue({
|
|
70
|
+
code: z.ZodIssueCode.custom,
|
|
71
|
+
path: ["events"],
|
|
72
|
+
message: `events are only meaningful for an rpc source (this stream's source is "${kind}")`,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
37
75
|
});
|
|
38
76
|
function readAndParse(path) {
|
|
39
77
|
let raw;
|
|
@@ -64,16 +102,17 @@ function parseTarget(path, protocolId, raw) {
|
|
|
64
102
|
? maxSizeRaw
|
|
65
103
|
: Number(BigInt(maxSizeRaw));
|
|
66
104
|
const events = parsed.data.events;
|
|
67
|
-
// Unique tracked contract addresses / event topics, in config (first-seen) order.
|
|
68
|
-
const trackedAddresses = [...new Set(events.map((e) => e.contractAddress))];
|
|
69
|
-
const trackedEventTopics = [...new Set(events.map((e) => e.eventTopic))];
|
|
70
105
|
const target = {
|
|
71
106
|
chainId: parsed.data.chainId,
|
|
72
107
|
fromBlock: parsed.data.fromBlock,
|
|
73
|
-
|
|
74
|
-
trackedAddresses,
|
|
75
|
-
trackedEventTopics,
|
|
108
|
+
source: (parsed.data.source ?? { kind: "rpc" }),
|
|
76
109
|
};
|
|
110
|
+
if (events !== undefined) {
|
|
111
|
+
target.events = events;
|
|
112
|
+
// Unique tracked contract addresses / event topics, in config (first-seen) order.
|
|
113
|
+
target.trackedAddresses = [...new Set(events.map((e) => e.contractAddress))];
|
|
114
|
+
target.trackedEventTopics = [...new Set(events.map((e) => e.eventTopic))];
|
|
115
|
+
}
|
|
77
116
|
if (maxSizeBytes !== undefined)
|
|
78
117
|
target.maxSizeBytes = maxSizeBytes;
|
|
79
118
|
if (parsed.data.protocol !== undefined)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/scraper/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,OAAO,GAAG,CAAC;KACd,MAAM,EAAE;KACR,KAAK,CAAC,qBAAqB,EAAE,0CAA0C,CAAC,CAAC;AAC5E,MAAM,KAAK,GAAG,CAAC;KACZ,MAAM,EAAE;KACR,KAAK,CAAC,qBAAqB,EAAE,wCAAwC,CAAC,CAAC;AAC1E,MAAM,QAAQ,GAAG,CAAC;KACf,MAAM,EAAE;KACR,KAAK,CAAC,kBAAkB,EAAE,qCAAqC,CAAC,CAAC;AAEpE,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,eAAe,EAAE,OAAO;IACxB,UAAU,EAAE,KAAK;IACjB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAEH,sEAAsE;AACtE,4EAA4E;AAC5E,uEAAuE;AACvE,MAAM,mBAAmB,GAAG,CAAC;KAC1B,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC1E,CAAC;KACD,WAAW,EAAE;KACb,QAAQ,EAAE,CAAC;AAEd,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/scraper/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,OAAO,GAAG,CAAC;KACd,MAAM,EAAE;KACR,KAAK,CAAC,qBAAqB,EAAE,0CAA0C,CAAC,CAAC;AAC5E,MAAM,KAAK,GAAG,CAAC;KACZ,MAAM,EAAE;KACR,KAAK,CAAC,qBAAqB,EAAE,wCAAwC,CAAC,CAAC;AAC1E,MAAM,QAAQ,GAAG,CAAC;KACf,MAAM,EAAE;KACR,KAAK,CAAC,kBAAkB,EAAE,qCAAqC,CAAC,CAAC;AAEpE,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,eAAe,EAAE,OAAO;IACxB,UAAU,EAAE,KAAK;IACjB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAEH,sEAAsE;AACtE,4EAA4E;AAC5E,uEAAuE;AACvE,MAAM,mBAAmB,GAAG,CAAC;KAC1B,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC1E,CAAC;KACD,WAAW,EAAE;KACb,QAAQ,EAAE,CAAC;AAEd,8EAA8E;AAC9E,8EAA8E;AAC9E,EAAE;AACF,8EAA8E;AAC9E,+EAA+E;AAC/E,yEAAyE;AACzE,+CAA+C;AAC/C,MAAM,YAAY,GAAG,CAAC;KACnB,kBAAkB,CAAC,MAAM,EAAE;IAC1B,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;IACpC,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,kCAAkC,CAAC;QAC5D,2EAA2E;QAC3E,mEAAmE;QACnE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;KAC1B,CAAC;CACH,CAAC;KACD,QAAQ,EAAE,CAAC;AAEd,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,QAAQ;IACnB,+EAA+E;IAC/E,sEAAsE;IACtE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE;IAC7C,MAAM,EAAE,YAAY;IACpB,aAAa,EAAE,mBAAmB;IAClC,8EAA8E;IAC9E,2EAA2E;IAC3E,oEAAoE;IACpE,6EAA6E;IAC7E,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;IACxB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,KAAK,CAAC;IACrC,IAAI,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,IAAI,EAAE,CAAC,QAAQ,CAAC;YAChB,OAAO,EAAE,yDAAyD;SACnE,CAAC,CAAC;IACL,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7C,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,IAAI,EAAE,CAAC,QAAQ,CAAC;YAChB,OAAO,EAAE,0EAA0E,IAAI,IAAI;SAC5F,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC,CAAC;AAiCH,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,SAAS,GAAI,GAAsD,EAAE,SAAS,CAAC;IACrF,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,wCAAwC,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,UAAkB,EAAE,GAAY;IACjE,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;aACjF,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,iBAAiB,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC;IAC3D,MAAM,YAAY,GAChB,UAAU,KAAK,SAAS;QACtB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,UAAU,KAAK,QAAQ;YAC9B,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAmC,CAAC;IAC/D,MAAM,MAAM,GAAkB;QAC5B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAc;QACnC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAgB;QACvC,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAiB;KAChE,CAAC;IACF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,kFAAkF;QAClF,MAAM,CAAC,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAU,CAAC;QACtF,MAAM,CAAC,kBAAkB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAU,CAAC;IACrF,CAAC;IACD,IAAI,YAAY,KAAK,SAAS;QAAE,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC;IACnE,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;QAAE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;IAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC/C,MAAM,CAAC,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC;IACzD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,UAAkB;IACzD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,kBAAkB,UAAU,aAAa,KAAK,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,kFAAkF;AAClF,0DAA0D;AAC1D,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,GAAG,GAAkC,EAAE,CAAC;IAC9C,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { PublicClient } from "viem";
|
|
2
|
+
import type { ScraperTarget } from "../scraper/config.js";
|
|
3
|
+
import type { ScraperSource } from "./types.js";
|
|
4
|
+
export type { ScraperSource } from "./types.js";
|
|
5
|
+
export { RpcLogSource } from "./rpc-log-source.js";
|
|
6
|
+
export { SubsquidSource, decodeSquidId, toCanonicalOperation } from "./subsquid-source.js";
|
|
7
|
+
export type CreateSourceOptions = {
|
|
8
|
+
client: PublicClient;
|
|
9
|
+
window: number;
|
|
10
|
+
finalizedTip: bigint;
|
|
11
|
+
fetchImpl?: typeof fetch;
|
|
12
|
+
};
|
|
13
|
+
export declare function createSource(target: ScraperTarget, opts: CreateSourceOptions): ScraperSource;
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sources/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIhD,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAE3F,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IAIf,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B,CAAC;AAIF,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,mBAAmB,GAAG,aAAa,CAiB5F"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { RpcLogSource } from "./rpc-log-source.js";
|
|
2
|
+
import { SubsquidSource } from "./subsquid-source.js";
|
|
3
|
+
export { RpcLogSource } from "./rpc-log-source.js";
|
|
4
|
+
export { SubsquidSource, decodeSquidId, toCanonicalOperation } from "./subsquid-source.js";
|
|
5
|
+
// The single switch point from config to source — mirrors createStore() in
|
|
6
|
+
// ../storage/index.ts. Adding a source kind means one case here and one class.
|
|
7
|
+
export function createSource(target, opts) {
|
|
8
|
+
switch (target.source.kind) {
|
|
9
|
+
case "rpc":
|
|
10
|
+
if (!target.events) {
|
|
11
|
+
// Unreachable via config (the schema requires events for rpc), but a
|
|
12
|
+
// programmatic caller could get here.
|
|
13
|
+
throw new Error('an "rpc" source requires event filters');
|
|
14
|
+
}
|
|
15
|
+
return new RpcLogSource(opts.client, target.events, opts.window, opts.finalizedTip);
|
|
16
|
+
case "subsquid":
|
|
17
|
+
return new SubsquidSource({
|
|
18
|
+
endpoint: target.source.endpoint,
|
|
19
|
+
entity: target.source.entity,
|
|
20
|
+
finalizedTip: opts.finalizedTip,
|
|
21
|
+
...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sources/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAY3F,2EAA2E;AAC3E,+EAA+E;AAC/E,MAAM,UAAU,YAAY,CAAC,MAAqB,EAAE,IAAyB;IAC3E,QAAQ,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3B,KAAK,KAAK;YACR,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,qEAAqE;gBACrE,sCAAsC;gBACtC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACtF,KAAK,UAAU;YACb,OAAO,IAAI,cAAc,CAAC;gBACxB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;gBAChC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;gBAC5B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzD,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { PublicClient } from "viem";
|
|
2
|
+
import type { CanonicalRecord } from "@saga-sync/core";
|
|
3
|
+
import type { EventFilter } from "../scraper/config.js";
|
|
4
|
+
import { finalizedBlock } from "../scraper/cli.js";
|
|
5
|
+
import type { ScraperSource } from "./types.js";
|
|
6
|
+
export declare class RpcLogSource implements ScraperSource {
|
|
7
|
+
private readonly client;
|
|
8
|
+
private readonly events;
|
|
9
|
+
private readonly window;
|
|
10
|
+
private readonly finalizedTip;
|
|
11
|
+
readonly kind = "rpc";
|
|
12
|
+
constructor(client: PublicClient, events: EventFilter[], window: number, finalizedTip: bigint);
|
|
13
|
+
fetch(fromBlock: bigint, toBlock: bigint): AsyncGenerator<CanonicalRecord>;
|
|
14
|
+
latestCoveredBlock(): Promise<bigint>;
|
|
15
|
+
}
|
|
16
|
+
export { finalizedBlock };
|
|
17
|
+
//# sourceMappingURL=rpc-log-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rpc-log-source.d.ts","sourceRoot":"","sources":["../../src/sources/rpc-log-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAQhD,qBAAa,YAAa,YAAW,aAAa;IAI9C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAIvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAT/B,QAAQ,CAAC,IAAI,SAAS;gBAGH,MAAM,EAAE,YAAY,EACpB,MAAM,EAAE,WAAW,EAAE,EACrB,MAAM,EAAE,MAAM,EAId,YAAY,EAAE,MAAM;IAGhC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,eAAe,CAAC;IAWjF,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC;CAGtC;AAID,OAAO,EAAE,cAAc,EAAE,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { scrape } from "../scraper/scrape.js";
|
|
2
|
+
import { normalize } from "../scraper/normalize.js";
|
|
3
|
+
import { finalizedBlock } from "../scraper/cli.js";
|
|
4
|
+
// The original source: windowed eth_getLogs against an Ethereum JSON-RPC.
|
|
5
|
+
//
|
|
6
|
+
// Behaviour is unchanged from when this lived inline in the pipeline — the same
|
|
7
|
+
// adaptive window halving on range errors and exponential backoff on rate limits
|
|
8
|
+
// (both in scrape()), and the same normalize() to canonical form. Extracting it
|
|
9
|
+
// is purely to give the subsquid source something to sit beside.
|
|
10
|
+
export class RpcLogSource {
|
|
11
|
+
client;
|
|
12
|
+
events;
|
|
13
|
+
window;
|
|
14
|
+
finalizedTip;
|
|
15
|
+
kind = "rpc";
|
|
16
|
+
constructor(client, events, window,
|
|
17
|
+
// The finalized tip, resolved once per orchestrator run and shared by every
|
|
18
|
+
// rpc-sourced stream — one eth_getBlockByNumber("finalized") for the run
|
|
19
|
+
// rather than one per protocol.
|
|
20
|
+
finalizedTip) {
|
|
21
|
+
this.client = client;
|
|
22
|
+
this.events = events;
|
|
23
|
+
this.window = window;
|
|
24
|
+
this.finalizedTip = finalizedTip;
|
|
25
|
+
}
|
|
26
|
+
async *fetch(fromBlock, toBlock) {
|
|
27
|
+
for await (const log of scrape(this.client, {
|
|
28
|
+
fromBlock,
|
|
29
|
+
toBlock,
|
|
30
|
+
events: this.events,
|
|
31
|
+
window: this.window,
|
|
32
|
+
})) {
|
|
33
|
+
yield normalize(log);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
latestCoveredBlock() {
|
|
37
|
+
return Promise.resolve(this.finalizedTip);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
// Re-exported so the orchestrator resolves the finalized tip through the same
|
|
41
|
+
// module that defines what "final" means for this source.
|
|
42
|
+
export { finalizedBlock };
|
|
43
|
+
//# sourceMappingURL=rpc-log-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rpc-log-source.js","sourceRoot":"","sources":["../../src/sources/rpc-log-source.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,0EAA0E;AAC1E,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,gFAAgF;AAChF,iEAAiE;AACjE,MAAM,OAAO,YAAY;IAIJ;IACA;IACA;IAIA;IATV,IAAI,GAAG,KAAK,CAAC;IAEtB,YACmB,MAAoB,EACpB,MAAqB,EACrB,MAAc;IAC/B,4EAA4E;IAC5E,yEAAyE;IACzE,gCAAgC;IACf,YAAoB;QANpB,WAAM,GAAN,MAAM,CAAc;QACpB,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAQ;QAId,iBAAY,GAAZ,YAAY,CAAQ;IACpC,CAAC;IAEJ,KAAK,CAAC,CAAC,KAAK,CAAC,SAAiB,EAAE,OAAe;QAC7C,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE;YAC1C,SAAS;YACT,OAAO;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,EAAE,CAAC;YACH,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,kBAAkB;QAChB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC;CACF;AAED,8EAA8E;AAC9E,0DAA0D;AAC1D,OAAO,EAAE,cAAc,EAAE,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { CanonicalEntity, CanonicalRecord } from "@saga-sync/core";
|
|
2
|
+
import type { ScraperSource } from "./types.js";
|
|
3
|
+
export type SquidId = {
|
|
4
|
+
blockNumber: bigint;
|
|
5
|
+
transactionIndex: bigint;
|
|
6
|
+
opIndex: bigint;
|
|
7
|
+
};
|
|
8
|
+
export declare function decodeSquidId(id: string): SquidId;
|
|
9
|
+
type RawOperation = {
|
|
10
|
+
id: string;
|
|
11
|
+
blockNumber: string;
|
|
12
|
+
nullifiers: string[];
|
|
13
|
+
commitments: string[];
|
|
14
|
+
boundParamsHash: string;
|
|
15
|
+
utxoTreeIn: string;
|
|
16
|
+
utxoTreeOut: string;
|
|
17
|
+
utxoBatchStartPositionOut: string;
|
|
18
|
+
};
|
|
19
|
+
export declare function toCanonicalOperation(raw: RawOperation, entity: string): CanonicalEntity;
|
|
20
|
+
export type SubsquidSourceOptions = {
|
|
21
|
+
endpoint: string;
|
|
22
|
+
entity: string;
|
|
23
|
+
finalizedTip: bigint;
|
|
24
|
+
fetchImpl?: typeof fetch;
|
|
25
|
+
retryDelayMs?: number;
|
|
26
|
+
};
|
|
27
|
+
export declare class SubsquidSource implements ScraperSource {
|
|
28
|
+
private readonly opts;
|
|
29
|
+
readonly kind = "subsquid";
|
|
30
|
+
private readonly fetchImpl;
|
|
31
|
+
private readonly retryDelayMs;
|
|
32
|
+
constructor(opts: SubsquidSourceOptions);
|
|
33
|
+
private query;
|
|
34
|
+
fetch(fromBlock: bigint, toBlock: bigint): AsyncGenerator<CanonicalRecord>;
|
|
35
|
+
latestCoveredBlock(): Promise<bigint>;
|
|
36
|
+
}
|
|
37
|
+
export {};
|
|
38
|
+
//# sourceMappingURL=subsquid-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"subsquid-source.d.ts","sourceRoot":"","sources":["../../src/sources/subsquid-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAO,MAAM,iBAAiB,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AA0BhD,MAAM,MAAM,OAAO,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzF,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CASjD;AAyCD,KAAK,YAAY,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,yBAAyB,EAAE,MAAM,CAAC;CACnC,CAAC;AAwBF,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,GAAG,eAAe,CAyBvF;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IAIf,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IAGzB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,qBAAa,cAAe,YAAW,aAAa;IAKtC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAJjC,QAAQ,CAAC,IAAI,cAAc;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAET,IAAI,EAAE,qBAAqB;YAK1C,KAAK;IA2CZ,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,eAAe,CAAC;IAkB3E,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC;CAY5C"}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// Mirror a RAILGUN Squid GraphQL index.
|
|
2
|
+
//
|
|
3
|
+
// Only for data no log carries. The squid's `transaction` entity holds
|
|
4
|
+
// boundParamsHash and utxoTreeIn, which exist solely in transact() calldata, plus
|
|
5
|
+
// the per-transaction split of nullifiers/commitments that needs the same
|
|
6
|
+
// calldata to reconstruct. kohaku consumes it for TXID/POI.
|
|
7
|
+
//
|
|
8
|
+
// Transport deliberately mirrors kohaku's own subsquid.rs (keyset pagination on
|
|
9
|
+
// id_gt, terminate on an empty page, 3 retries at a fixed delay) so that when it
|
|
10
|
+
// misbehaves, both clients misbehave the same way and a bug reproduces on either.
|
|
11
|
+
const PAGE_LIMIT = 20_000;
|
|
12
|
+
const MAX_RETRIES = 3;
|
|
13
|
+
const RETRY_DELAY_MS = 1_000;
|
|
14
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
15
|
+
// A squid `transaction.id` is three big-endian 32-byte words:
|
|
16
|
+
// blockNumber ‖ transactionIndex ‖ opIndex
|
|
17
|
+
// It is a chain-native coordinate, not a database row id — which is why the
|
|
18
|
+
// records we publish can carry the same triple as first-class fields and a
|
|
19
|
+
// future calldata-derived source can emit byte-identical output.
|
|
20
|
+
const ID_WORD_HEX = 64;
|
|
21
|
+
export function decodeSquidId(id) {
|
|
22
|
+
const hex = id.startsWith("0x") ? id.slice(2) : id;
|
|
23
|
+
if (hex.length !== ID_WORD_HEX * 3 || !/^[0-9a-fA-F]+$/.test(hex)) {
|
|
24
|
+
throw new Error(`subsquid: expected a 96-byte hex id (blockNumber‖transactionIndex‖opIndex), got "${id}"`);
|
|
25
|
+
}
|
|
26
|
+
const word = (i) => BigInt(`0x${hex.slice(i * ID_WORD_HEX, (i + 1) * ID_WORD_HEX)}`);
|
|
27
|
+
return { blockNumber: word(0), transactionIndex: word(1), opIndex: word(2) };
|
|
28
|
+
}
|
|
29
|
+
// Minimal 0x-hex, lowercase — the same quantity encoding the rest of the format
|
|
30
|
+
// uses. The squid returns BigInt fields as DECIMAL strings and Bytes as hex, so
|
|
31
|
+
// everything funnels through here; a non-deterministic conversion would make
|
|
32
|
+
// chunk digests irreproducible.
|
|
33
|
+
function quantity(value, field) {
|
|
34
|
+
let n;
|
|
35
|
+
try {
|
|
36
|
+
n = BigInt(value);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
throw new Error(`subsquid: ${field} is not an integer: ${JSON.stringify(value)}`);
|
|
40
|
+
}
|
|
41
|
+
if (n < 0n)
|
|
42
|
+
throw new Error(`subsquid: ${field} is negative: ${value}`);
|
|
43
|
+
return `0x${n.toString(16)}`;
|
|
44
|
+
}
|
|
45
|
+
// Nullifiers, commitments and boundParamsHash are `bytes32` in the contract ABI,
|
|
46
|
+
// so they are emitted as a full 32-byte word — always 0x + 64 hex digits.
|
|
47
|
+
//
|
|
48
|
+
// The squid does NOT do this: it strips leading zero bytes, so the same value can
|
|
49
|
+
// come back 30, 31 or 32 bytes wide (measured over the full history: 2,879
|
|
50
|
+
// boundParamsHash at 31 bytes, 14 at 30, 22 nullifiers and 42 commitments short).
|
|
51
|
+
// Passing that through would make our bytes depend on an indexer's formatting —
|
|
52
|
+
// a calldata-derived source would naturally emit the padded form and the two
|
|
53
|
+
// would disagree, breaking the point of a source-independent record. It would
|
|
54
|
+
// also render the same nullifier differently here than when decoded from a log.
|
|
55
|
+
//
|
|
56
|
+
// Re-rendering from the integer value is lossless for a fixed-width type and
|
|
57
|
+
// makes the output depend only on the value.
|
|
58
|
+
function bytes32(value, field) {
|
|
59
|
+
if (typeof value !== "string" || !/^0x[0-9a-fA-F]*$/.test(value)) {
|
|
60
|
+
throw new Error(`subsquid: ${field} is not 0x-hex: ${JSON.stringify(value)}`);
|
|
61
|
+
}
|
|
62
|
+
const hex = BigInt(value).toString(16);
|
|
63
|
+
if (hex.length > 64) {
|
|
64
|
+
throw new Error(`subsquid: ${field} exceeds 32 bytes: ${JSON.stringify(value)}`);
|
|
65
|
+
}
|
|
66
|
+
return `0x${hex.padStart(64, "0")}`;
|
|
67
|
+
}
|
|
68
|
+
// The query kohaku issues, field for field.
|
|
69
|
+
const OPERATIONS_QUERY = `query Operations($id_gt: String, $gte: BigInt, $lte: BigInt, $limit: Int) {
|
|
70
|
+
transactions(
|
|
71
|
+
orderBy: id_ASC
|
|
72
|
+
where: {id_gt: $id_gt, blockNumber_gte: $gte, blockNumber_lte: $lte}
|
|
73
|
+
limit: $limit
|
|
74
|
+
) {
|
|
75
|
+
id
|
|
76
|
+
blockNumber
|
|
77
|
+
nullifiers
|
|
78
|
+
commitments
|
|
79
|
+
boundParamsHash
|
|
80
|
+
utxoTreeIn
|
|
81
|
+
utxoTreeOut
|
|
82
|
+
utxoBatchStartPositionOut
|
|
83
|
+
}
|
|
84
|
+
}`;
|
|
85
|
+
const HEIGHT_QUERY = `query Height { squidStatus { height } }`;
|
|
86
|
+
// Field order here IS the wire order — JSON.stringify preserves insertion order,
|
|
87
|
+
// and the chunk digest is taken over those exact bytes. Do not reorder.
|
|
88
|
+
export function toCanonicalOperation(raw, entity) {
|
|
89
|
+
const id = decodeSquidId(raw.id);
|
|
90
|
+
const fromBlock = quantity(raw.blockNumber, "blockNumber");
|
|
91
|
+
if (BigInt(fromBlock) !== id.blockNumber) {
|
|
92
|
+
// The id encodes the block; if the column disagrees, one of our assumptions
|
|
93
|
+
// about this index is wrong and silently trusting either would be worse.
|
|
94
|
+
throw new Error(`subsquid: id block ${id.blockNumber} disagrees with blockNumber ${raw.blockNumber}`);
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
entity,
|
|
98
|
+
blockNumber: fromBlock,
|
|
99
|
+
transactionIndex: quantity(id.transactionIndex, "transactionIndex"),
|
|
100
|
+
opIndex: quantity(id.opIndex, "opIndex"),
|
|
101
|
+
nullifiers: raw.nullifiers.map((n, i) => bytes32(n, `nullifiers[${i}]`)),
|
|
102
|
+
commitments: raw.commitments.map((c, i) => bytes32(c, `commitments[${i}]`)),
|
|
103
|
+
boundParamsHash: bytes32(raw.boundParamsHash, "boundParamsHash"),
|
|
104
|
+
utxoTreeIn: quantity(raw.utxoTreeIn, "utxoTreeIn"),
|
|
105
|
+
utxoTreeOut: quantity(raw.utxoTreeOut, "utxoTreeOut"),
|
|
106
|
+
utxoBatchStartPositionOut: quantity(raw.utxoBatchStartPositionOut, "utxoBatchStartPositionOut"),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
export class SubsquidSource {
|
|
110
|
+
opts;
|
|
111
|
+
kind = "subsquid";
|
|
112
|
+
fetchImpl;
|
|
113
|
+
retryDelayMs;
|
|
114
|
+
constructor(opts) {
|
|
115
|
+
this.opts = opts;
|
|
116
|
+
this.fetchImpl = opts.fetchImpl ?? fetch;
|
|
117
|
+
this.retryDelayMs = opts.retryDelayMs ?? RETRY_DELAY_MS;
|
|
118
|
+
}
|
|
119
|
+
async query(query, variables) {
|
|
120
|
+
let lastError = "";
|
|
121
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
122
|
+
if (attempt > 0)
|
|
123
|
+
await sleep(this.retryDelayMs);
|
|
124
|
+
let res;
|
|
125
|
+
try {
|
|
126
|
+
res = await this.fetchImpl(this.opts.endpoint, {
|
|
127
|
+
method: "POST",
|
|
128
|
+
headers: { "content-type": "application/json" },
|
|
129
|
+
body: JSON.stringify({ query, variables }),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
lastError = err.message;
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (!res.ok) {
|
|
137
|
+
lastError = `HTTP ${res.status}`;
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
let body;
|
|
141
|
+
try {
|
|
142
|
+
body = (await res.json());
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
lastError = `malformed JSON: ${err.message}`;
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
// A GraphQL 200 can still carry errors; treating it as success would
|
|
149
|
+
// silently publish a short page as if the range were empty.
|
|
150
|
+
if (body.errors && body.errors.length > 0) {
|
|
151
|
+
lastError = `GraphQL errors: ${JSON.stringify(body.errors)}`;
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
if (!body.data) {
|
|
155
|
+
lastError = "response had no data";
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
return body.data;
|
|
159
|
+
}
|
|
160
|
+
throw new Error(`subsquid: ${this.opts.endpoint} failed after ${MAX_RETRIES} attempts — ${lastError}`);
|
|
161
|
+
}
|
|
162
|
+
async *fetch(fromBlock, toBlock) {
|
|
163
|
+
let cursor = "";
|
|
164
|
+
for (;;) {
|
|
165
|
+
const data = await this.query(OPERATIONS_QUERY, {
|
|
166
|
+
id_gt: cursor,
|
|
167
|
+
gte: fromBlock.toString(),
|
|
168
|
+
lte: toBlock.toString(),
|
|
169
|
+
limit: PAGE_LIMIT,
|
|
170
|
+
});
|
|
171
|
+
const page = data.transactions;
|
|
172
|
+
for (const raw of page)
|
|
173
|
+
yield toCanonicalOperation(raw, this.opts.entity);
|
|
174
|
+
// Terminate on an empty page rather than a short one — same as kohaku, so a
|
|
175
|
+
// paging bug in the index shows up identically in both.
|
|
176
|
+
if (page.length === 0)
|
|
177
|
+
return;
|
|
178
|
+
cursor = page[page.length - 1].id;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
async latestCoveredBlock() {
|
|
182
|
+
const data = await this.query(HEIGHT_QUERY, {});
|
|
183
|
+
const height = data.squidStatus?.height;
|
|
184
|
+
if (height === undefined || height === null) {
|
|
185
|
+
throw new Error("subsquid: squidStatus.height missing — cannot establish a safe tip");
|
|
186
|
+
}
|
|
187
|
+
const indexed = BigInt(height);
|
|
188
|
+
return indexed < this.opts.finalizedTip ? indexed : this.opts.finalizedTip;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
//# sourceMappingURL=subsquid-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"subsquid-source.js","sourceRoot":"","sources":["../../src/sources/subsquid-source.ts"],"names":[],"mappings":"AAGA,wCAAwC;AACxC,EAAE;AACF,uEAAuE;AACvE,kFAAkF;AAClF,0EAA0E;AAC1E,4DAA4D;AAC5D,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,kFAAkF;AAElF,MAAM,UAAU,GAAG,MAAM,CAAC;AAC1B,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAEnF,8DAA8D;AAC9D,6CAA6C;AAC7C,4EAA4E;AAC5E,2EAA2E;AAC3E,iEAAiE;AACjE,MAAM,WAAW,GAAG,EAAE,CAAC;AAIvB,MAAM,UAAU,aAAa,CAAC,EAAU;IACtC,MAAM,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,oFAAoF,EAAE,GAAG,CAC1F,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;IACrG,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/E,CAAC;AAED,gFAAgF;AAChF,gFAAgF;AAChF,6EAA6E;AAC7E,gCAAgC;AAChC,SAAS,QAAQ,CAAC,KAA+B,EAAE,KAAa;IAC9D,IAAI,CAAS,CAAC;IACd,IAAI,CAAC;QACH,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,uBAAuB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,CAAC,GAAG,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,iBAAiB,KAAK,EAAE,CAAC,CAAC;IACxE,OAAO,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAS,CAAC;AACtC,CAAC;AAED,iFAAiF;AACjF,0EAA0E;AAC1E,EAAE;AACF,kFAAkF;AAClF,2EAA2E;AAC3E,kFAAkF;AAClF,gFAAgF;AAChF,6EAA6E;AAC7E,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,6EAA6E;AAC7E,6CAA6C;AAC7C,SAAS,OAAO,CAAC,KAAa,EAAE,KAAa;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,mBAAmB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvC,IAAI,GAAG,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,sBAAsB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,KAAK,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,EAAS,CAAC;AAC7C,CAAC;AAaD,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;EAevB,CAAC;AAEH,MAAM,YAAY,GAAG,yCAAyC,CAAC;AAE/D,iFAAiF;AACjF,wEAAwE;AACxE,MAAM,UAAU,oBAAoB,CAAC,GAAiB,EAAE,MAAc;IACpE,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACjC,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC;QACzC,4EAA4E;QAC5E,yEAAyE;QACzE,MAAM,IAAI,KAAK,CACb,sBAAsB,EAAE,CAAC,WAAW,+BAA+B,GAAG,CAAC,WAAW,EAAE,CACrF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM;QACN,WAAW,EAAE,SAAS;QACtB,gBAAgB,EAAE,QAAQ,CAAC,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,CAAC;QACnE,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;QACxC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;QACxE,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC;QAC3E,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC;QAChE,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC;QAClD,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,aAAa,CAAC;QACrD,yBAAyB,EAAE,QAAQ,CACjC,GAAG,CAAC,yBAAyB,EAC7B,2BAA2B,CAC5B;KACF,CAAC;AACJ,CAAC;AAeD,MAAM,OAAO,cAAc;IAKI;IAJpB,IAAI,GAAG,UAAU,CAAC;IACV,SAAS,CAAe;IACxB,YAAY,CAAS;IAEtC,YAA6B,IAA2B;QAA3B,SAAI,GAAJ,IAAI,CAAuB;QACtD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,KAAK,CAAI,KAAa,EAAE,SAAkC;QACtE,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;oBAC7C,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;iBAC3C,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAI,GAAa,CAAC,OAAO,CAAC;gBACnC,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,SAAS,GAAG,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YACD,IAAI,IAAsC,CAAC;YAC3C,IAAI,CAAC;gBACH,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAqC,CAAC;YAChE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC;gBACxD,SAAS;YACX,CAAC;YACD,qEAAqE;YACrE,4DAA4D;YAC5D,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,SAAS,GAAG,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,SAAS;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACf,SAAS,GAAG,sBAAsB,CAAC;gBACnC,SAAS;YACX,CAAC;YACD,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,CAAC,IAAI,CAAC,QAAQ,iBAAiB,WAAW,eAAe,SAAS,EAAE,CACtF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,SAAiB,EAAE,OAAe;QAC7C,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAmC,gBAAgB,EAAE;gBAChF,KAAK,EAAE,MAAM;gBACb,GAAG,EAAE,SAAS,CAAC,QAAQ,EAAE;gBACzB,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;gBACvB,KAAK,EAAE,UAAU;aAClB,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC;YAC/B,KAAK,MAAM,GAAG,IAAI,IAAI;gBAAE,MAAM,oBAAoB,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1E,4EAA4E;YAC5E,wDAAwD;YACxD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC9B,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;QACrC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,YAAY,EACZ,EAAE,CACH,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;QACxC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACxF,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;IAC7E,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { CanonicalRecord } from "@saga-sync/core";
|
|
2
|
+
export type ScraperSource = {
|
|
3
|
+
fetch(fromBlock: bigint, toBlock: bigint): AsyncGenerator<CanonicalRecord>;
|
|
4
|
+
latestCoveredBlock(): Promise<bigint>;
|
|
5
|
+
readonly kind: string;
|
|
6
|
+
};
|
|
7
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/sources/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAUvD,MAAM,MAAM,aAAa,GAAG;IAI1B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,eAAe,CAAC,CAAC;IAQ3E,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAGtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/sources/types.ts"],"names":[],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gcs-store.d.ts","sourceRoot":"","sources":["../../src/storage/gcs-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAO7C,MAAM,MAAM,WAAW,GAAG;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AACF,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,QAAQ,IAAI,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAClC,MAAM,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7D;AACD,MAAM,WAAW,SAAS;IACxB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC,CAAC;CACxE;AACD,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,CAAC,CAAC;AAUpE,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"gcs-store.d.ts","sourceRoot":"","sources":["../../src/storage/gcs-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAO7C,MAAM,MAAM,WAAW,GAAG;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AACF,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,QAAQ,IAAI,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAClC,MAAM,CAAC,IAAI,EAAE;QAAE,cAAc,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7D;AACD,MAAM,WAAW,SAAS;IACxB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC,CAAC;CACxE;AACD,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,CAAC,CAAC;AAUpE,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOlD;AAkBD,MAAM,MAAM,eAAe,GAAG;IAG5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC,CAAC;AAKF,qBAAa,QAAS,YAAW,KAAK;IAMlC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAL7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,YAAY,CAAC,CAAqB;gBAGvB,UAAU,EAAE,MAAM,EACnC,IAAI,GAAE,eAAoB;IAM5B,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,GAAG;IAIL,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAW5C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAI9C;AAkBD,wBAAsB,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAUnE"}
|
|
@@ -13,7 +13,10 @@ export function cacheControlFor(key) {
|
|
|
13
13
|
export function contentTypeFor(key) {
|
|
14
14
|
if (key.endsWith(".gz"))
|
|
15
15
|
return "application/gzip";
|
|
16
|
-
|
|
16
|
+
// `.sigs` is the JSON signature envelope; `.sig` stays opaque — it is a bare
|
|
17
|
+
// hex string, not JSON. (Neither is cached long: only *.jsonl.gz is immutable,
|
|
18
|
+
// so both re-validate on the short TTL above.)
|
|
19
|
+
if (key.endsWith(".json") || key.endsWith(".sigs"))
|
|
17
20
|
return "application/json";
|
|
18
21
|
return "application/octet-stream";
|
|
19
22
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gcs-store.js","sourceRoot":"","sources":["../../src/storage/gcs-store.ts"],"names":[],"mappings":"AAuBA,MAAM,MAAM,GAAG,cAAc,CAAC;AAC9B,MAAM,GAAG,GAAG,mBAAmB,CAAC;AAEhC,iFAAiF;AACjF,iFAAiF;AACjF,kFAAkF;AAClF,kFAAkF;AAClF,sEAAsE;AACtE,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,qCAAqC,CAAC;IACrF,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACnD,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"gcs-store.js","sourceRoot":"","sources":["../../src/storage/gcs-store.ts"],"names":[],"mappings":"AAuBA,MAAM,MAAM,GAAG,cAAc,CAAC;AAC9B,MAAM,GAAG,GAAG,mBAAmB,CAAC;AAEhC,iFAAiF;AACjF,iFAAiF;AACjF,kFAAkF;AAClF,kFAAkF;AAClF,sEAAsE;AACtE,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,qCAAqC,CAAC;IACrF,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACnD,6EAA6E;IAC7E,+EAA+E;IAC/E,+CAA+C;IAC/C,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC9E,OAAO,0BAA0B,CAAC;AACpC,CAAC;AAED,kFAAkF;AAClF,+EAA+E;AAC/E,yCAAyC;AACzC,MAAM,qBAAqB,GAAmB,KAAK,EAAE,MAAM,EAAE,EAAE;IAC7D,MAAM,IAAI,GAAG,uBAAuB,CAAC;IACrC,IAAI,GAA+D,CAAC;IACpE,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAA0B,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC,CAAC;AAUF,gFAAgF;AAChF,8EAA8E;AAC9E,0EAA0E;AAC1E,MAAM,OAAO,QAAQ;IAMA;IALF,MAAM,CAAS;IACf,QAAQ,CAAiB;IAClC,YAAY,CAAsB;IAE1C,YACmB,UAAkB,EACnC,OAAwB,EAAE;QADT,eAAU,GAAV,UAAU,CAAQ;QAGnC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,qBAAqB,CAAC;IAC/D,CAAC;IAEO,MAAM;QACZ,OAAO,CAAC,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,GAAG,CAAC,GAAW;QACrB,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,IAAgB;QACrC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,0EAA0E;QAC1E,4EAA4E;QAC5E,qCAAqC;QACrC,MAAM,SAAS,CAAC,GAAG,EAAE,CACnB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACd,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC;YAChC,QAAQ,EAAE,EAAE,YAAY,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE;YAChD,SAAS,EAAE,KAAK;SACjB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,UAAU,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YACjC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACnF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,CAAC;CACF;AAED,SAAS,UAAU,CAAC,GAAY;IAC9B,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAK,GAAyB,CAAC,IAAI,KAAK,GAAG,CAAC;AAC5F,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,IAAK,GAAyB,CAAC,IAAI,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IACzD,MAAM,GAAG,GAAG,MAAM,CAAE,GAA6B,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/E,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,iFAAiF;AACjF,uEAAuE;AACvE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAI,EAAoB;IACrD,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,OAAO,IAAI,cAAc;gBAAE,MAAM,GAAG,CAAC;YAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;YAC3D,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;AACH,CAAC"}
|