@secondlayer/subgraphs 4.4.1 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +13 -21
  2. package/dist/{src/index.d.ts → index.d.ts} +59 -189
  3. package/dist/index.js +2151 -0
  4. package/dist/index.js.map +24 -0
  5. package/dist/{src/runtime → runtime}/emitter.d.ts +17 -11
  6. package/dist/{src/runtime → runtime}/emitter.js +58 -18
  7. package/dist/runtime/emitter.js.map +19 -0
  8. package/dist/{src/runtime → runtime}/replay.d.ts +15 -1
  9. package/dist/{src/runtime → runtime}/replay.js +418 -723
  10. package/dist/{src/runtime → runtime}/replay.js.map +6 -6
  11. package/dist/{src/schema → schema}/index.d.ts +10 -69
  12. package/dist/{src/schema → schema}/index.js +188 -281
  13. package/dist/schema/index.js.map +13 -0
  14. package/dist/{src/testing → testing}/index.d.ts +65 -115
  15. package/dist/{src/testing → testing}/index.js +554 -968
  16. package/dist/testing/index.js.map +21 -0
  17. package/dist/{src/types.d.ts → types.d.ts} +10 -66
  18. package/dist/{src/validate.d.ts → validate.d.ts} +11 -69
  19. package/dist/{src/validate.js → validate.js} +18 -98
  20. package/dist/validate.js.map +10 -0
  21. package/package.json +18 -26
  22. package/dist/src/index.js +0 -6081
  23. package/dist/src/index.js.map +0 -44
  24. package/dist/src/runtime/block-processor.d.ts +0 -559
  25. package/dist/src/runtime/block-processor.js +0 -3810
  26. package/dist/src/runtime/block-processor.js.map +0 -31
  27. package/dist/src/runtime/catchup.d.ts +0 -473
  28. package/dist/src/runtime/catchup.js +0 -4228
  29. package/dist/src/runtime/catchup.js.map +0 -35
  30. package/dist/src/runtime/clarity.d.ts +0 -15
  31. package/dist/src/runtime/clarity.js +0 -41
  32. package/dist/src/runtime/clarity.js.map +0 -10
  33. package/dist/src/runtime/context.d.ts +0 -325
  34. package/dist/src/runtime/context.js +0 -936
  35. package/dist/src/runtime/context.js.map +0 -13
  36. package/dist/src/runtime/emitter.js.map +0 -19
  37. package/dist/src/runtime/processor.d.ts +0 -11
  38. package/dist/src/runtime/processor.js +0 -5372
  39. package/dist/src/runtime/processor.js.map +0 -40
  40. package/dist/src/runtime/reindex.d.ts +0 -539
  41. package/dist/src/runtime/reindex.js +0 -4802
  42. package/dist/src/runtime/reindex.js.map +0 -37
  43. package/dist/src/runtime/reorg.d.ts +0 -432
  44. package/dist/src/runtime/reorg.js +0 -4355
  45. package/dist/src/runtime/reorg.js.map +0 -36
  46. package/dist/src/runtime/runner.d.ts +0 -719
  47. package/dist/src/runtime/runner.js +0 -1332
  48. package/dist/src/runtime/runner.js.map +0 -16
  49. package/dist/src/runtime/source-matcher.d.ts +0 -304
  50. package/dist/src/runtime/source-matcher.js +0 -335
  51. package/dist/src/runtime/source-matcher.js.map +0 -10
  52. package/dist/src/runtime/stats.d.ts +0 -23
  53. package/dist/src/runtime/stats.js +0 -71
  54. package/dist/src/runtime/stats.js.map +0 -10
  55. package/dist/src/schema/index.js.map +0 -13
  56. package/dist/src/service.d.ts +0 -0
  57. package/dist/src/service.js +0 -5398
  58. package/dist/src/service.js.map +0 -41
  59. package/dist/src/testing/index.js.map +0 -22
  60. package/dist/src/triggers/index.d.ts +0 -381
  61. package/dist/src/triggers/index.js +0 -31
  62. package/dist/src/triggers/index.js.map +0 -10
  63. package/dist/src/validate.js.map +0 -10
  64. /package/dist/{src/types.js → types.js} +0 -0
  65. /package/dist/{src/types.js.map → types.js.map} +0 -0
@@ -1,4802 +0,0 @@
1
- import { createRequire } from "node:module";
2
- var __require = /* @__PURE__ */ createRequire(import.meta.url);
3
-
4
- // src/runtime/context.ts
5
- import { logger as logger2 } from "@secondlayer/shared/logger";
6
- import { formatUnits } from "@secondlayer/stacks/utils";
7
- import { sql } from "kysely";
8
-
9
- // src/schema/generator.ts
10
- import { createHash } from "node:crypto";
11
-
12
- // src/schema/utils.ts
13
- import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
14
- function quotePgIdent(name) {
15
- if (/^[a-z_][a-z0-9_]*$/i.test(name))
16
- return name;
17
- return `"${name.replace(/"/g, '""')}"`;
18
- }
19
- function snakeToCamel(name) {
20
- return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase());
21
- }
22
- function pascalCase(name) {
23
- const camel = snakeToCamel(name);
24
- return camel.charAt(0).toUpperCase() + camel.slice(1);
25
- }
26
-
27
- // src/schema/generator.ts
28
- var TYPE_MAP = {
29
- text: "TEXT",
30
- uint: "NUMERIC",
31
- int: "NUMERIC",
32
- principal: "TEXT",
33
- boolean: "BOOLEAN",
34
- timestamp: "TIMESTAMPTZ",
35
- jsonb: "JSONB"
36
- };
37
- var MAX_IDENT_BYTES = 63;
38
- function safeIndexName(name) {
39
- if (name.length <= MAX_IDENT_BYTES)
40
- return name;
41
- const hash = createHash("sha256").update(name).digest("hex").slice(0, 8);
42
- return `${name.slice(0, MAX_IDENT_BYTES - hash.length - 1)}_${hash}`;
43
- }
44
- function emitIndexedColumnIndexDDL(schemaName, tableName, colName, opts) {
45
- const concurrently = opts?.concurrently ? " CONCURRENTLY" : "";
46
- const qualified = `${quotePgIdent(schemaName)}.${quotePgIdent(tableName)}`;
47
- const plain = `idx_${schemaName}_${tableName}_${colName}`;
48
- const composite = safeIndexName(`idx_${schemaName}_${tableName}_${colName}_id`);
49
- const col = quotePgIdent(colName);
50
- const id = quotePgIdent("_id");
51
- return {
52
- create: [
53
- `CREATE INDEX${concurrently} IF NOT EXISTS ${quotePgIdent(plain)} ON ${qualified} (${col})`,
54
- `CREATE INDEX${concurrently} IF NOT EXISTS ${quotePgIdent(composite)} ON ${qualified} (${col}, ${id})`
55
- ],
56
- drop: [
57
- `DROP INDEX${concurrently} IF EXISTS ${quotePgIdent(schemaName)}.${quotePgIdent(plain)}`,
58
- `DROP INDEX${concurrently} IF EXISTS ${quotePgIdent(schemaName)}.${quotePgIdent(composite)}`
59
- ]
60
- };
61
- }
62
- function escapeLiteralDefault(value) {
63
- if (value === null || value === undefined)
64
- return "NULL";
65
- if (typeof value === "number" || typeof value === "bigint")
66
- return String(value);
67
- if (typeof value === "boolean")
68
- return value ? "TRUE" : "FALSE";
69
- return `'${String(value).replace(/'/g, "''")}'`;
70
- }
71
- function tableNeedsTrgm(tableDef) {
72
- return Object.values(tableDef.columns).some((col) => col.search);
73
- }
74
- function emitTableDDL(schemaName, tableName, tableDef) {
75
- const qualifiedName = `${quotePgIdent(schemaName)}.${quotePgIdent(tableName)}`;
76
- const statements = [];
77
- const columnDefs = [
78
- "_id BIGSERIAL PRIMARY KEY",
79
- "_block_height BIGINT NOT NULL",
80
- "_tx_id TEXT NOT NULL",
81
- "_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
82
- ];
83
- for (const [colName, col] of Object.entries(tableDef.columns)) {
84
- const sqlType = TYPE_MAP[col.type];
85
- const nullable = col.nullable ? "" : " NOT NULL";
86
- let colDef = `${quotePgIdent(colName)} ${sqlType}${nullable}`;
87
- if (col.default !== undefined) {
88
- colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
89
- }
90
- if (col.type === "uint") {
91
- colDef += ` CHECK (${quotePgIdent(colName)} >= 0)`;
92
- }
93
- columnDefs.push(colDef);
94
- }
95
- statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
96
- ${columnDefs.join(`,
97
- `)}
98
- )`);
99
- statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_block_height`)} ON ${qualifiedName} (_block_height)`);
100
- statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_tx_id`)} ON ${qualifiedName} (_tx_id)`);
101
- for (const [colName, col] of Object.entries(tableDef.columns)) {
102
- if (col.indexed) {
103
- statements.push(...emitIndexedColumnIndexDDL(schemaName, tableName, colName).create);
104
- }
105
- }
106
- for (const [colName, col] of Object.entries(tableDef.columns)) {
107
- if (col.search) {
108
- statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_${colName}_trgm`)} ON ${qualifiedName} USING gin (${quotePgIdent(colName)} gin_trgm_ops)`);
109
- }
110
- }
111
- if (tableDef.indexes) {
112
- for (let i = 0;i < tableDef.indexes.length; i++) {
113
- const cols = tableDef.indexes[i];
114
- const idxName = quotePgIdent(`idx_${schemaName}_${tableName}_composite_${i}`);
115
- statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.map(quotePgIdent).join(", ")})`);
116
- }
117
- }
118
- if (tableDef.uniqueKeys) {
119
- for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
120
- const cols = tableDef.uniqueKeys[i];
121
- const constraintName = quotePgIdent(`uq_${schemaName}_${tableName}_${cols.join("_")}`);
122
- statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.map(quotePgIdent).join(", ")})`);
123
- }
124
- }
125
- return statements;
126
- }
127
- function emitJournalDDL(schemaName) {
128
- return [
129
- `CREATE TABLE IF NOT EXISTS ${schemaName}._journal (
130
- _jid BIGSERIAL PRIMARY KEY,
131
- block_height BIGINT NOT NULL,
132
- table_name TEXT NOT NULL,
133
- row_key JSONB NOT NULL,
134
- prev_row JSONB,
135
- _created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
136
- )`,
137
- `CREATE INDEX IF NOT EXISTS idx_${schemaName}_journal_height ON ${schemaName}._journal (block_height)`
138
- ];
139
- }
140
- function emitFactoryDDL(schemaName) {
141
- return [
142
- `CREATE TABLE IF NOT EXISTS ${schemaName}._factory_addresses (
143
- source_name TEXT NOT NULL,
144
- address TEXT NOT NULL,
145
- block_height BIGINT NOT NULL,
146
- _created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
147
- PRIMARY KEY (source_name, address)
148
- )`,
149
- `CREATE INDEX IF NOT EXISTS idx_${schemaName}_factory_height ON ${schemaName}._factory_addresses (block_height)`
150
- ];
151
- }
152
- function emitForeignKeyDDL(schemaName, tableName, tableDef) {
153
- return (tableDef.relations ?? []).map((rel) => {
154
- const constraintName = quotePgIdent(`fk_${schemaName}_${tableName}_${rel.name}`);
155
- return `ALTER TABLE ${quotePgIdent(schemaName)}.${quotePgIdent(tableName)} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.map(quotePgIdent).join(", ")}) ` + `REFERENCES ${quotePgIdent(schemaName)}.${quotePgIdent(rel.references)} (${rel.referencedColumns.map(quotePgIdent).join(", ")})`;
156
- });
157
- }
158
- function generateSubgraphSQL(def, schemaNameOverride) {
159
- const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
160
- const statements = [];
161
- const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
162
- if (needsTrgm) {
163
- statements.push("CREATE EXTENSION IF NOT EXISTS pg_trgm");
164
- }
165
- statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
166
- for (const [tableName, tableDef] of Object.entries(def.schema)) {
167
- statements.push(...emitTableDDL(schemaName, tableName, tableDef));
168
- }
169
- statements.push(...emitJournalDDL(schemaName));
170
- if (Object.values(def.sources ?? {}).some((src) => src.factory !== undefined)) {
171
- statements.push(...emitFactoryDDL(schemaName));
172
- }
173
- for (const [tableName, tableDef] of Object.entries(def.schema)) {
174
- statements.push(...emitForeignKeyDDL(schemaName, tableName, tableDef));
175
- }
176
- const hashInput = JSON.stringify({
177
- name: def.name,
178
- schema: def.schema,
179
- sources: def.sources
180
- }, (_key, value) => typeof value === "bigint" ? value.toString() : value);
181
- const hash = createHash("sha256").update(hashInput).digest("hex");
182
- return { statements, hash };
183
- }
184
-
185
- // src/runtime/chain-read.ts
186
- import { getSourceDb } from "@secondlayer/shared/db";
187
- import { logger } from "@secondlayer/shared/logger";
188
- import { http, createPublicClient } from "@secondlayer/stacks";
189
- import {
190
- ContractResponseError,
191
- buildFunctionArgs,
192
- isResponseOutput,
193
- readContract
194
- } from "@secondlayer/stacks/actions";
195
- import {
196
- clarityValueToJSUntyped,
197
- deserializeCV,
198
- normalizeAbi,
199
- serializeCV,
200
- toCamelCase
201
- } from "@secondlayer/stacks/clarity";
202
- function readContractAt(ctx, contractId, abi, options) {
203
- return ctx.client.contract(contractId, abi, options);
204
- }
205
-
206
- class ChainReadError extends Error {
207
- name = "ChainReadError";
208
- }
209
- function nodeClient(rpcUrl) {
210
- return createPublicClient({
211
- transport: http(rpcUrl, { retryCount: 0, timeout: 1e4 })
212
- });
213
- }
214
-
215
- class ConcurrencyGate {
216
- limit;
217
- active = 0;
218
- queue = [];
219
- constructor(limit) {
220
- this.limit = limit;
221
- }
222
- async run(fn) {
223
- if (this.active >= this.limit) {
224
- await new Promise((resolve) => this.queue.push(resolve));
225
- }
226
- this.active++;
227
- try {
228
- return await fn();
229
- } finally {
230
- this.active--;
231
- this.queue.shift()?.();
232
- }
233
- }
234
- }
235
- var DEFAULT_MAX_CONCURRENT_READS = 4;
236
- function maxConcurrentReads() {
237
- const raw = process.env.SUBGRAPH_CHAIN_READ_CONCURRENCY;
238
- const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
239
- return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_MAX_CONCURRENT_READS;
240
- }
241
- var gate = new ConcurrencyGate(maxConcurrentReads());
242
- function hashArgs(args) {
243
- const serialized = args.map((arg) => serializeCV(arg)).join(",");
244
- return Bun.hash(serialized).toString(16);
245
- }
246
- function findFunction(abi, name) {
247
- const fn = abi.functions.find((f) => f.name === name);
248
- if (!fn) {
249
- throw new ChainReadError(`No function "${name}" in the supplied ABI.`);
250
- }
251
- if (fn.access !== "read-only") {
252
- throw new ChainReadError(`"${name}" is ${fn.access}, not read-only — a handler cannot call it.`);
253
- }
254
- return fn;
255
- }
256
- function createChainReadClient(ctx) {
257
- const readValue = async (contractId, abi, functionName, args, options) => {
258
- const rpcUrl = ctx.rpcUrl ?? process.env.STACKS_NODE_RPC_URL;
259
- if (!rpcUrl) {
260
- throw new ChainReadError("ctx.client needs a Stacks node: set STACKS_NODE_RPC_URL on the subgraph processor.");
261
- }
262
- const constant = options.cache === "contract-constant";
263
- if (!constant && !ctx.indexBlockHash) {
264
- throw new ChainReadError(`Block ${ctx.blockHeight} has no index_block_hash, so a read cannot be pinned to it. Reading at the node's tip would make this handler non-deterministic (the same reindex would produce different rows). Re-ingest the block, or pass cache: "contract-constant" if the value genuinely cannot change.`);
265
- }
266
- const fn = findFunction(abi, functionName);
267
- const clarityArgs = buildFunctionArgs(fn, args);
268
- const db = ctx.db ?? getSourceDb();
269
- const argsHash = hashArgs(clarityArgs);
270
- const pinnedTo = constant ? null : ctx.indexBlockHash;
271
- const cached = await db.selectFrom("chain_read_cache").select("result_hex").where("contract_id", "=", contractId).where("function_name", "=", fn.name).where("args_hash", "=", argsHash).where((eb) => pinnedTo === null ? eb("index_block_hash", "is", null) : eb("index_block_hash", "=", pinnedTo)).executeTakeFirst();
272
- const resultHex = cached?.result_hex ?? await gate.run(async () => {
273
- if (!ctx.indexBlockHash) {
274
- logger.warn("Chain read is not pinned to a block (no index_block_hash)", {
275
- contractId,
276
- functionName: fn.name,
277
- blockHeight: ctx.blockHeight,
278
- note: 'cache: "contract-constant" asserts this value cannot change; the node answered at its own tip'
279
- });
280
- }
281
- const value2 = await readContract(nodeClient(rpcUrl), {
282
- contract: contractId,
283
- functionName: fn.name,
284
- args: clarityArgs,
285
- tip: ctx.indexBlockHash ?? undefined
286
- });
287
- return serializeCV(value2);
288
- });
289
- if (!cached) {
290
- await db.insertInto("chain_read_cache").values({
291
- contract_id: contractId,
292
- function_name: fn.name,
293
- args_hash: argsHash,
294
- index_block_hash: pinnedTo,
295
- block_height: constant ? null : ctx.blockHeight,
296
- result_hex: resultHex
297
- }).onConflict((oc) => oc.doNothing()).execute();
298
- }
299
- const value = deserializeCV(resultHex);
300
- const js = clarityValueToJSUntyped(fn.outputs, value);
301
- if (isResponseOutput(fn.outputs)) {
302
- const response = js;
303
- if ("ok" in response)
304
- return response.ok;
305
- throw new ContractResponseError(`${contractId}::${fn.name} returned an err response.`, response.err);
306
- }
307
- return js;
308
- };
309
- return {
310
- contract(contractId, rawAbi, options = {}) {
311
- const abi = normalizeAbi(rawAbi);
312
- const camelToKebab = new Map;
313
- for (const fn of abi.functions) {
314
- camelToKebab.set(toCamelCase(fn.name), fn.name);
315
- }
316
- const read = new Proxy({}, {
317
- get(_target, prop) {
318
- const fnName = camelToKebab.get(prop) ?? prop;
319
- return (args = {}) => readValue(contractId, abi, fnName, args, options);
320
- }
321
- });
322
- return { read };
323
- }
324
- };
325
- }
326
-
327
- // src/runtime/context.ts
328
- var JOURNAL_RETENTION_BLOCKS = 300;
329
- var journalEnsured = new Set;
330
- function validateColumnName(name) {
331
- if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
332
- throw new Error(`Invalid column name: ${name}`);
333
- }
334
- }
335
-
336
- class SubgraphContext {
337
- block;
338
- _tx;
339
- db;
340
- pgSchemaName;
341
- subgraphSchema;
342
- ops = [];
343
- journal;
344
- _client;
345
- constructor(db, pgSchemaName2, subgraphSchema, block, tx, journal = false) {
346
- this.db = db;
347
- this.pgSchemaName = pgSchemaName2;
348
- this.subgraphSchema = subgraphSchema;
349
- this.block = block;
350
- this._tx = tx;
351
- this.journal = journal;
352
- }
353
- get client() {
354
- this._client ??= createChainReadClient({
355
- blockHeight: this.block.height,
356
- indexBlockHash: this.block.indexBlockHash
357
- });
358
- return this._client;
359
- }
360
- get tx() {
361
- return this._tx;
362
- }
363
- setTx(tx) {
364
- this._tx = tx;
365
- }
366
- insert(table, row) {
367
- this.validateTable(table);
368
- this.ops.push({
369
- kind: "insert",
370
- table,
371
- data: { ...row, _block_height: this.block.height, _tx_id: this._tx.txId }
372
- });
373
- }
374
- update(table, where, set) {
375
- this.validateTable(table);
376
- this.ops.push({ kind: "update", table, data: where, set });
377
- }
378
- upsert(table, key, row) {
379
- this.validateTable(table);
380
- const tableDef = this.subgraphSchema[table];
381
- if (!tableDef)
382
- return;
383
- const keyColumns = Object.keys(key);
384
- const hasUniqueConstraint = tableDef.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
385
- const meta = { _block_height: this.block.height, _tx_id: this._tx.txId };
386
- if (hasUniqueConstraint) {
387
- this.ops.push({
388
- kind: "insert",
389
- table,
390
- data: { ...key, ...row, ...meta, _upsert_keys: keyColumns }
391
- });
392
- } else {
393
- logger2.warn("upsert called without matching uniqueKeys constraint, using fallback", {
394
- table,
395
- keys: keyColumns
396
- });
397
- this.ops.push({
398
- kind: "insert",
399
- table,
400
- data: {
401
- ...key,
402
- ...row,
403
- ...meta,
404
- _upsert_fallback_keys: keyColumns,
405
- _upsert_fallback_set: row
406
- }
407
- });
408
- }
409
- }
410
- delete(table, where) {
411
- this.validateTable(table);
412
- this.ops.push({ kind: "delete", table, data: where });
413
- }
414
- increment(table, key, deltas) {
415
- this.validateTable(table);
416
- const tableDef = this.subgraphSchema[table];
417
- const keyColumns = Object.keys(key);
418
- const hasUniqueConstraint = tableDef?.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
419
- if (!hasUniqueConstraint) {
420
- throw new Error(`increment("${table}") requires a uniqueKeys constraint on [${keyColumns.join(", ")}]`);
421
- }
422
- for (const [col, v] of Object.entries(deltas)) {
423
- validateColumnName(col);
424
- if (keyColumns.includes(col)) {
425
- throw new Error(`increment("${table}"): "${col}" is a key column`);
426
- }
427
- if (typeof v !== "bigint" && typeof v !== "number") {
428
- throw new Error(`increment("${table}"): delta for "${col}" must be bigint or number`);
429
- }
430
- }
431
- this.ops.push({
432
- kind: "increment",
433
- table,
434
- data: {
435
- ...key,
436
- _block_height: this.block.height,
437
- _tx_id: this._tx.txId,
438
- _upsert_keys: keyColumns
439
- },
440
- set: { ...deltas }
441
- });
442
- }
443
- opsCheckpoint() {
444
- return this.ops.length;
445
- }
446
- rollbackTo(checkpoint) {
447
- if (checkpoint < 0 || checkpoint > this.ops.length)
448
- return;
449
- this.ops.length = checkpoint;
450
- }
451
- patch(table, where, set) {
452
- this.update(table, where, set);
453
- }
454
- async patchOrInsert(table, key, row) {
455
- const existing = await this.findOne(table, key);
456
- const resolved = {};
457
- for (const [k, v] of Object.entries(row)) {
458
- resolved[k] = typeof v === "function" ? v(existing) : v;
459
- }
460
- this.upsert(table, key, resolved);
461
- }
462
- formatUnits(value, decimals) {
463
- return formatUnits(value, decimals);
464
- }
465
- async readRows(table, where, limit) {
466
- const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
467
- const { clause } = buildWhereClause(where);
468
- const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause}${limit !== undefined ? ` LIMIT ${limit}` : ""}`;
469
- const { rows } = await sql.raw(query).execute(this.db);
470
- return rows.map((r) => this.coerceRow(table, r));
471
- }
472
- async findOne(table, where) {
473
- this.validateTable(table);
474
- const row = (await this.readRows(table, where, 1))[0] ?? null;
475
- return this.overlayOne(table, where, row);
476
- }
477
- async findMany(table, where) {
478
- this.validateTable(table);
479
- return this.overlayMany(table, where, await this.readRows(table, where));
480
- }
481
- overlayOne(table, where, dbRow) {
482
- if (this.ops.length === 0)
483
- return dbRow;
484
- let row = dbRow;
485
- for (const op of this.ops) {
486
- if (op.table !== table)
487
- continue;
488
- row = this.applyOpToRow(op, row, where);
489
- }
490
- return row;
491
- }
492
- overlayMany(table, where, dbRows) {
493
- if (this.ops.length === 0)
494
- return [...dbRows];
495
- let result = [...dbRows];
496
- for (const op of this.ops) {
497
- if (op.table !== table)
498
- continue;
499
- if (op.kind === "update") {
500
- for (let i = 0;i < result.length; i++) {
501
- if (rowMatches(result[i], op.data))
502
- result[i] = { ...result[i], ...op.set ?? {} };
503
- }
504
- } else if (op.kind === "delete") {
505
- result = result.filter((r) => !rowMatches(r, op.data));
506
- } else {
507
- const upsertKeys = op.data._upsert_keys;
508
- const clean = stripControlKeys(op.data);
509
- const idx = upsertKeys ? result.findIndex((r) => upsertKeys.every((k) => valEq(r[k], clean[k]))) : -1;
510
- if (idx >= 0) {
511
- const existing = result[idx];
512
- result[idx] = this.applyOpToRow(op, existing, where) ?? existing;
513
- } else {
514
- const created = this.applyOpToRow(op, null, where);
515
- if (created)
516
- result.push(created);
517
- }
518
- }
519
- }
520
- return result;
521
- }
522
- applyOpToRow(op, row, where) {
523
- const upsertKeys = op.data._upsert_keys;
524
- const clean = stripControlKeys(op.data);
525
- switch (op.kind) {
526
- case "insert": {
527
- if (row) {
528
- if (upsertKeys?.every((k) => valEq(row[k], clean[k]))) {
529
- const merged = { ...row };
530
- for (const [k, v] of Object.entries(clean)) {
531
- if (!upsertKeys.includes(k) && !k.startsWith("_"))
532
- merged[k] = v;
533
- }
534
- return merged;
535
- }
536
- return row;
537
- }
538
- return rowMatches(clean, where) ? { ...clean } : null;
539
- }
540
- case "increment": {
541
- const deltas = op.set ?? {};
542
- if (row) {
543
- if (upsertKeys.every((k) => valEq(row[k], clean[k]))) {
544
- const merged = { ...row };
545
- for (const [col, d] of Object.entries(deltas)) {
546
- merged[col] = toBigIntOr0(merged[col]) + toBigIntOr0(d);
547
- }
548
- return merged;
549
- }
550
- return row;
551
- }
552
- if (!rowMatches(clean, where))
553
- return null;
554
- const created = { ...clean };
555
- for (const [col, d] of Object.entries(deltas)) {
556
- created[col] = toBigIntOr0(d);
557
- }
558
- return created;
559
- }
560
- case "update":
561
- return row && rowMatches(row, op.data) ? { ...row, ...op.set ?? {} } : row;
562
- case "delete":
563
- return row && rowMatches(row, op.data) ? null : row;
564
- }
565
- }
566
- async count(table, where) {
567
- this.validateTable(table);
568
- const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
569
- const whereClause = where ? `WHERE ${buildWhereClause(where).clause}` : "";
570
- const { rows } = await sql.raw(`SELECT COUNT(*)::int AS count FROM ${qualifiedTable} ${whereClause}`).execute(this.db);
571
- return Number(rows[0]?.count ?? 0);
572
- }
573
- async sum(table, column, where) {
574
- this.validateTable(table);
575
- validateColumnName(column);
576
- const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
577
- const whereClause = where ? `WHERE ${buildWhereClause(where).clause}` : "";
578
- const { rows } = await sql.raw(`SELECT COALESCE(SUM("${column}"), 0) AS total FROM ${qualifiedTable} ${whereClause}`).execute(this.db);
579
- return BigInt(rows[0]?.total?.toString() ?? "0");
580
- }
581
- async min(table, column, where) {
582
- this.validateTable(table);
583
- validateColumnName(column);
584
- const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
585
- const whereClause = where ? `WHERE ${buildWhereClause(where).clause}` : "";
586
- const { rows } = await sql.raw(`SELECT MIN("${column}") AS val FROM ${qualifiedTable} ${whereClause}`).execute(this.db);
587
- const val = rows[0]?.val;
588
- return val != null ? BigInt(val.toString()) : null;
589
- }
590
- async max(table, column, where) {
591
- this.validateTable(table);
592
- validateColumnName(column);
593
- const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
594
- const whereClause = where ? `WHERE ${buildWhereClause(where).clause}` : "";
595
- const { rows } = await sql.raw(`SELECT MAX("${column}") AS val FROM ${qualifiedTable} ${whereClause}`).execute(this.db);
596
- const val = rows[0]?.val;
597
- return val != null ? BigInt(val.toString()) : null;
598
- }
599
- async countDistinct(table, column, where) {
600
- this.validateTable(table);
601
- validateColumnName(column);
602
- const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
603
- const whereClause = where ? `WHERE ${buildWhereClause(where).clause}` : "";
604
- const { rows } = await sql.raw(`SELECT COUNT(DISTINCT "${column}")::int AS count FROM ${qualifiedTable} ${whereClause}`).execute(this.db);
605
- return Number(rows[0]?.count ?? 0);
606
- }
607
- coerceRow(table, row) {
608
- const tableDef = this.subgraphSchema[table];
609
- if (!tableDef)
610
- return row;
611
- const result = { ...row };
612
- for (const [col, def] of Object.entries(tableDef.columns)) {
613
- if ((def.type === "uint" || def.type === "int") && typeof result[col] === "string") {
614
- result[col] = BigInt(result[col]);
615
- }
616
- }
617
- return result;
618
- }
619
- get pendingOps() {
620
- return this.ops.length;
621
- }
622
- async flush() {
623
- if (this.ops.length === 0)
624
- return { count: 0, writes: [] };
625
- await this.ensureJournalTable();
626
- const opsToFlush = [...this.ops];
627
- this.ops.length = 0;
628
- const statements = this.buildStatements(opsToFlush);
629
- if ("isTransaction" in this.db) {
630
- await this.execStatements(this.db, statements);
631
- } else {
632
- await this.db.transaction().execute(async (tx) => {
633
- await this.execStatements(tx, statements);
634
- });
635
- }
636
- const writes = opsToFlush.map((op, rowIndex) => {
637
- const blockHeight = op.data._block_height ?? this.block.height;
638
- const txId = op.data._tx_id ?? this._tx.txId;
639
- const baseRow = op.kind === "update" || op.kind === "increment" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
640
- baseRow._upsert_keys = undefined;
641
- baseRow._upsert_fallback_keys = undefined;
642
- baseRow._upsert_fallback_set = undefined;
643
- return {
644
- op: op.kind === "increment" ? "update" : op.kind,
645
- table: op.table,
646
- row: jsonSafe(baseRow),
647
- pk: { blockHeight, txId, rowIndex }
648
- };
649
- });
650
- return { count: opsToFlush.length, writes };
651
- }
652
- async execStatements(executor, statements) {
653
- const affected = (r) => r.numAffectedRows ?? 0n;
654
- for (const stmt of statements) {
655
- if (typeof stmt === "string") {
656
- await sql.raw(stmt).execute(executor);
657
- continue;
658
- }
659
- const updated = await sql.raw(stmt.update).execute(executor);
660
- if (affected(updated) > 0n)
661
- continue;
662
- const inserted = await sql.raw(stmt.insert).execute(executor);
663
- if (affected(inserted) > 0n)
664
- continue;
665
- const retried = await sql.raw(stmt.update).execute(executor);
666
- if (affected(retried) === 0n) {
667
- throw new Error(`increment could not be applied to ${this.pgSchemaName}.${stmt.table} where ${stmt.keyPred}: row neither updated nor inserted`);
668
- }
669
- }
670
- }
671
- prepareInsert(op) {
672
- const upsertKeys = op.data._upsert_keys;
673
- const data = { ...op.data };
674
- delete data._upsert_keys;
675
- delete data._upsert_fallback_keys;
676
- delete data._upsert_fallback_set;
677
- if (!data._block_height)
678
- data._block_height = this.block.height;
679
- if (!data._tx_id)
680
- data._tx_id = this._tx.txId;
681
- data._created_at = "NOW()";
682
- const cols = Object.keys(data);
683
- cols.forEach(validateColumnName);
684
- const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
685
- const batchKey = `${op.table}:${[...cols].sort().join(",")}:${upsertKeys ? [...upsertKeys].sort().join(",") : ""}`;
686
- return { data, cols, vals, upsertKeys, batchKey };
687
- }
688
- async ensureJournalTable() {
689
- if (!this.journal || journalEnsured.has(this.pgSchemaName))
690
- return;
691
- const { rows } = await sql.raw(`SELECT to_regclass('"${this.pgSchemaName}"."_journal"') AS r`).execute(this.db);
692
- if (rows[0]?.r) {
693
- journalEnsured.add(this.pgSchemaName);
694
- return;
695
- }
696
- for (const stmt of emitJournalDDL(this.pgSchemaName)) {
697
- await sql.raw(stmt).execute(this.db);
698
- }
699
- }
700
- columnSqlType(table, col) {
701
- const def = this.subgraphSchema[table]?.columns?.[col];
702
- return def ? TYPE_MAP[def.type] : undefined;
703
- }
704
- journalCaptureSQL(table, keyCols, keyLiteralRows) {
705
- const cast = (col, expr) => {
706
- const t = this.columnSqlType(table, col);
707
- return t ? `CAST(${expr} AS ${t})` : expr;
708
- };
709
- const keyObj = keyCols.map((k) => `'${k}', ${cast(k, `v."${k}"`)}`).join(", ");
710
- const joinCond = keyCols.map((k) => `t."${k}" = ${cast(k, `v."${k}"`)}`).join(" AND ");
711
- const valuesList = keyLiteralRows.map((r) => `(${r.join(", ")})`).join(", ");
712
- return `INSERT INTO "${this.pgSchemaName}"."_journal" ("block_height", "table_name", "row_key", "prev_row") ` + `SELECT ${this.block.height}, '${table}', jsonb_build_object(${keyObj}), to_jsonb(t.*) ` + `FROM (VALUES ${valuesList}) AS v(${keyCols.map((k) => `"${k}"`).join(", ")}) ` + `LEFT JOIN "${this.pgSchemaName}"."${table}" t ON ${joinCond}`;
713
- }
714
- journalCaptureByWhereSQL(table, clause) {
715
- return `INSERT INTO "${this.pgSchemaName}"."_journal" ("block_height", "table_name", "row_key", "prev_row") ` + `SELECT ${this.block.height}, '${table}', jsonb_build_object('_id', t."_id"), to_jsonb(t.*) ` + `FROM "${this.pgSchemaName}"."${table}" t WHERE ${clause}`;
716
- }
717
- buildStatements(ops) {
718
- const statements = [];
719
- let currentBatch = null;
720
- let currentBatchKey = "";
721
- let incBatch = null;
722
- let incBatchKey = "";
723
- const flushIncrementBatch = () => {
724
- if (!incBatch)
725
- return;
726
- const batch = incBatch;
727
- const qualifiedTable = `"${this.pgSchemaName}"."${batch.table}"`;
728
- const cols = [
729
- ...batch.keyCols,
730
- ...batch.deltaCols,
731
- "_block_height",
732
- "_tx_id",
733
- "_created_at"
734
- ];
735
- if (this.journal) {
736
- statements.push(this.journalCaptureSQL(batch.table, batch.keyCols, Array.from(batch.rows.values()).map((r) => batch.keyCols.map((k) => escapeLiteral(r.keys[k])))));
737
- }
738
- for (const r of batch.rows.values()) {
739
- const keyPred = batch.keyCols.map((k) => `"${k}" = ${escapeLiteral(r.keys[k])}`).join(" AND ");
740
- const updSet = batch.deltaCols.map((c) => `"${c}" = COALESCE("${c}", 0) + (${String(r.deltas[c] ?? 0n)})`).join(", ");
741
- const insertVals = [
742
- ...batch.keyCols.map((k) => escapeLiteral(r.keys[k])),
743
- ...batch.deltaCols.map((c) => String(r.deltas[c] ?? 0n)),
744
- escapeLiteral(r.meta.blockHeight),
745
- escapeLiteral(r.meta.txId),
746
- "NOW()"
747
- ];
748
- statements.push({
749
- kind: "increment",
750
- table: batch.table,
751
- keyPred,
752
- update: `UPDATE ${qualifiedTable} SET ${updSet} WHERE ${keyPred}`,
753
- insert: `INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) ` + `SELECT ${insertVals.join(", ")} ` + `WHERE NOT EXISTS (SELECT 1 FROM ${qualifiedTable} WHERE ${keyPred})`
754
- });
755
- }
756
- incBatch = null;
757
- incBatchKey = "";
758
- };
759
- const flushInsertBatch = () => {
760
- if (!currentBatch)
761
- return;
762
- const batch = currentBatch;
763
- const qualifiedTable = `"${this.pgSchemaName}"."${batch.table}"`;
764
- const colList = batch.cols.map((c) => `"${c}"`).join(", ");
765
- let rows = batch.rows;
766
- if (batch.upsertKeys && batch.upsertKeys.length > 0) {
767
- const uKeys = batch.upsertKeys;
768
- const keyIndices = uKeys.map((k) => batch.cols.indexOf(k));
769
- const seen = new Map;
770
- for (let i = 0;i < rows.length; i++) {
771
- const key = keyIndices.map((ki) => rows[i][ki]).join("\x00");
772
- seen.set(key, i);
773
- }
774
- if (seen.size < rows.length) {
775
- rows = Array.from(seen.values()).map((i) => rows[i]);
776
- }
777
- }
778
- const valuesList = rows.map((r) => `(${r.join(", ")})`).join(", ");
779
- let stmt = `INSERT INTO ${qualifiedTable} (${colList}) VALUES ${valuesList}`;
780
- if (this.journal && batch.upsertKeys && batch.upsertKeys.length > 0) {
781
- const uKeys = batch.upsertKeys;
782
- const keyIndices = uKeys.map((k) => batch.cols.indexOf(k));
783
- statements.push(this.journalCaptureSQL(batch.table, uKeys, rows.map((r) => keyIndices.map((ki) => r[ki]))));
784
- }
785
- if (batch.upsertKeys && batch.upsertKeys.length > 0) {
786
- const batchKeys = batch.upsertKeys;
787
- const updateCols = batch.cols.filter((c) => !batchKeys.includes(c) && !c.startsWith("_"));
788
- if (updateCols.length > 0) {
789
- const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
790
- stmt += ` ON CONFLICT (${batchKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
791
- } else {
792
- stmt += ` ON CONFLICT (${batchKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
793
- }
794
- }
795
- statements.push(stmt);
796
- currentBatch = null;
797
- currentBatchKey = "";
798
- };
799
- for (const op of ops) {
800
- const qualifiedTable = `"${this.pgSchemaName}"."${op.table}"`;
801
- if (op.kind === "insert") {
802
- flushIncrementBatch();
803
- const { cols, vals, upsertKeys, batchKey } = this.prepareInsert(op);
804
- if (batchKey === currentBatchKey && currentBatch) {
805
- currentBatch.rows.push(vals);
806
- } else {
807
- flushInsertBatch();
808
- currentBatch = { table: op.table, cols, rows: [vals], upsertKeys };
809
- currentBatchKey = batchKey;
810
- }
811
- } else if (op.kind === "increment") {
812
- flushInsertBatch();
813
- const keyCols = [...op.data._upsert_keys].sort();
814
- const deltaCols = Object.keys(op.set ?? {}).sort();
815
- const batchKey = `inc:${op.table}:${keyCols.join(",")}:${deltaCols.join(",")}`;
816
- if (batchKey !== incBatchKey || !incBatch) {
817
- flushIncrementBatch();
818
- incBatch = { table: op.table, keyCols, deltaCols, rows: new Map };
819
- incBatchKey = batchKey;
820
- }
821
- const clean = stripControlKeys(op.data);
822
- const keySig = keyCols.map((k) => escapeLiteral(clean[k])).join("\x00");
823
- const existing = incBatch.rows.get(keySig);
824
- if (existing) {
825
- for (const c of deltaCols) {
826
- existing.deltas[c] = (existing.deltas[c] ?? 0n) + toBigIntOr0(op.set?.[c]);
827
- }
828
- } else {
829
- const deltas = {};
830
- for (const c of deltaCols)
831
- deltas[c] = toBigIntOr0(op.set?.[c]);
832
- incBatch.rows.set(keySig, {
833
- keys: clean,
834
- deltas,
835
- meta: {
836
- blockHeight: op.data._block_height ?? this.block.height,
837
- txId: op.data._tx_id ?? this._tx.txId
838
- }
839
- });
840
- }
841
- } else {
842
- flushInsertBatch();
843
- flushIncrementBatch();
844
- if (op.kind === "update") {
845
- const setEntries = Object.entries(op.set ?? {});
846
- for (const [k] of setEntries)
847
- validateColumnName(k);
848
- const setClauses = setEntries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
849
- const { clause } = buildWhereClause(op.data);
850
- if (this.journal) {
851
- statements.push(this.journalCaptureByWhereSQL(op.table, clause));
852
- }
853
- statements.push(`UPDATE ${qualifiedTable} SET ${setClauses.join(", ")} WHERE ${clause}`);
854
- } else if (op.kind === "delete") {
855
- const { clause } = buildWhereClause(op.data);
856
- if (this.journal) {
857
- statements.push(this.journalCaptureByWhereSQL(op.table, clause));
858
- }
859
- statements.push(`DELETE FROM ${qualifiedTable} WHERE ${clause}`);
860
- }
861
- }
862
- }
863
- flushInsertBatch();
864
- flushIncrementBatch();
865
- return statements;
866
- }
867
- validateTable(table) {
868
- if (!this.subgraphSchema[table]) {
869
- throw new Error(`Table "${table}" not found in subgraph schema. Available: [${Object.keys(this.subgraphSchema).join(", ")}]`);
870
- }
871
- }
872
- }
873
- function stripControlKeys(data) {
874
- const {
875
- _upsert_keys: _a,
876
- _upsert_fallback_keys: _b,
877
- _upsert_fallback_set: _c,
878
- ...clean
879
- } = data;
880
- return clean;
881
- }
882
- function valEq(a, b) {
883
- if (a === b)
884
- return true;
885
- if (a == null || b == null)
886
- return false;
887
- return String(a) === String(b);
888
- }
889
- function rowMatches(row, where) {
890
- return Object.entries(where).every(([k, v]) => valEq(row[k], v));
891
- }
892
- function toBigIntOr0(v) {
893
- if (typeof v === "bigint")
894
- return v;
895
- if (v == null)
896
- return 0n;
897
- try {
898
- return BigInt(String(v));
899
- } catch {
900
- return 0n;
901
- }
902
- }
903
- function jsonSafe(row) {
904
- const out = {};
905
- for (const [k, v] of Object.entries(row)) {
906
- out[k] = typeof v === "bigint" ? v.toString() : v;
907
- }
908
- return out;
909
- }
910
- function escapeLiteral(value) {
911
- if (value === null || value === undefined)
912
- return "NULL";
913
- if (typeof value === "number" || typeof value === "bigint")
914
- return String(value);
915
- if (typeof value === "boolean")
916
- return value ? "TRUE" : "FALSE";
917
- if (typeof value === "object")
918
- return `'${JSON.stringify(value, (_k, v) => typeof v === "bigint" ? v.toString() : v).replace(/'/g, "''")}'::jsonb`;
919
- return `'${String(value).replace(/'/g, "''")}'`;
920
- }
921
- function buildWhereClause(where) {
922
- const entries = Object.entries(where);
923
- if (entries.length === 0)
924
- return { clause: "TRUE", values: [] };
925
- for (const [k] of entries)
926
- validateColumnName(k);
927
- const parts = entries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
928
- return { clause: parts.join(" AND "), values: [] };
929
- }
930
-
931
- // src/runtime/clarity.ts
932
- import { cvToValue, deserializeCV as deserializeCV2 } from "@secondlayer/stacks/clarity";
933
- function decodeClarityValue(hex) {
934
- try {
935
- const cleanHex = hex.startsWith("0x") ? hex.slice(2) : hex;
936
- const cv = deserializeCV2(cleanHex);
937
- return cvToValue(cv);
938
- } catch {
939
- return hex;
940
- }
941
- }
942
- function decodeEventData(data) {
943
- if (typeof data === "string" && data.startsWith("0x") && data.length > 10) {
944
- return decodeClarityValue(data);
945
- }
946
- if (Array.isArray(data)) {
947
- return data.map(decodeEventData);
948
- }
949
- if (typeof data === "object" && data !== null) {
950
- const decoded = {};
951
- for (const [key, value] of Object.entries(data)) {
952
- decoded[key] = decodeEventData(value);
953
- }
954
- return decoded;
955
- }
956
- return data;
957
- }
958
- function decodeFunctionArgs(args) {
959
- return args.map(decodeClarityValue);
960
- }
961
-
962
- // src/runtime/source-matcher.ts
963
- var patternCache = new Map;
964
- function matchAny(value, pattern) {
965
- return Array.isArray(pattern) ? pattern.some((p) => matchPattern(value, p)) : matchPattern(value, pattern);
966
- }
967
- function matchPattern(value, pattern) {
968
- if (!pattern.includes("*"))
969
- return value === pattern;
970
- let re = patternCache.get(pattern);
971
- if (!re) {
972
- const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
973
- re = new RegExp(`^${regex}$`);
974
- patternCache.set(pattern, re);
975
- }
976
- return re.test(value);
977
- }
978
- var EMPTY_SET = new Set;
979
- function factoryAllows(filter, contractId, factoryContracts) {
980
- if (!filter.factory)
981
- return true;
982
- if (!contractId)
983
- return false;
984
- return (factoryContracts.get(filter.factory.from) ?? EMPTY_SET).has(contractId);
985
- }
986
- function readPath(payload, path) {
987
- let cur = payload;
988
- for (const key of path.split(".")) {
989
- if (cur === null || typeof cur !== "object")
990
- return;
991
- cur = cur[key];
992
- }
993
- return cur;
994
- }
995
- function traitAllows(filter, contractId, traitContracts) {
996
- const trait = filter.trait;
997
- if (!trait)
998
- return true;
999
- if (!contractId)
1000
- return false;
1001
- return (traitContracts.get(trait) ?? EMPTY_SET).has(contractId);
1002
- }
1003
- function assetContract(assetId) {
1004
- return assetId?.split("::")[0];
1005
- }
1006
- function printContractId(data) {
1007
- if (!data)
1008
- return;
1009
- const id = (typeof data.contract_identifier === "string" ? data.contract_identifier : undefined) ?? (typeof data.contract_id === "string" ? data.contract_id : undefined);
1010
- return id || undefined;
1011
- }
1012
- function matchFilter(filter, transactions, eventsByTx, traitContracts, factoryContracts = new Map, vmEventsByTx = new Map) {
1013
- const results = [];
1014
- switch (filter.type) {
1015
- case "stx_transfer":
1016
- case "stx_mint":
1017
- case "stx_burn":
1018
- case "stx_lock": {
1019
- const eventType = `${filter.type}_event`;
1020
- for (const tx of transactions) {
1021
- const txEvents = eventsByTx.get(tx.tx_id) ?? [];
1022
- const matched = txEvents.filter((e) => e.type === eventType);
1023
- if (matched.length === 0)
1024
- continue;
1025
- const filtered = matched.filter((e) => {
1026
- const data = e.data;
1027
- if (!data)
1028
- return false;
1029
- if ("sender" in filter && filter.sender) {
1030
- if (!matchPattern(data.sender, filter.sender))
1031
- return false;
1032
- }
1033
- if ("recipient" in filter && filter.recipient) {
1034
- if (!matchPattern(data.recipient, filter.recipient))
1035
- return false;
1036
- }
1037
- if ("lockedAddress" in filter && filter.lockedAddress) {
1038
- if (!matchPattern(data.locked_address, filter.lockedAddress))
1039
- return false;
1040
- }
1041
- if ("minAmount" in filter && filter.minAmount !== undefined) {
1042
- const amount = BigInt(data.amount ?? data.locked_amount ?? "0");
1043
- if (amount < filter.minAmount)
1044
- return false;
1045
- }
1046
- if ("maxAmount" in filter && filter.maxAmount !== undefined) {
1047
- const amount = BigInt(data.amount ?? "0");
1048
- if (amount > filter.maxAmount)
1049
- return false;
1050
- }
1051
- return true;
1052
- });
1053
- if (filtered.length > 0) {
1054
- results.push({ tx, events: filtered });
1055
- }
1056
- }
1057
- break;
1058
- }
1059
- case "ft_transfer":
1060
- case "ft_mint":
1061
- case "ft_burn": {
1062
- const eventType = `${filter.type}_event`;
1063
- for (const tx of transactions) {
1064
- const txEvents = eventsByTx.get(tx.tx_id) ?? [];
1065
- const matched = txEvents.filter((e) => {
1066
- if (e.type !== eventType)
1067
- return false;
1068
- const data = e.data;
1069
- if (!data)
1070
- return false;
1071
- if (filter.assetIdentifier) {
1072
- const assetId = data.asset_identifier;
1073
- if (!assetId || !matchPattern(assetId, filter.assetIdentifier))
1074
- return false;
1075
- }
1076
- if (!traitAllows(filter, assetContract(data.asset_identifier), traitContracts))
1077
- return false;
1078
- if ("sender" in filter && filter.sender) {
1079
- if (!matchPattern(data.sender, filter.sender))
1080
- return false;
1081
- }
1082
- if ("recipient" in filter && filter.recipient) {
1083
- if (!matchPattern(data.recipient, filter.recipient))
1084
- return false;
1085
- }
1086
- if (filter.minAmount !== undefined) {
1087
- const amount = BigInt(data.amount ?? "0");
1088
- if (amount < filter.minAmount)
1089
- return false;
1090
- }
1091
- return true;
1092
- });
1093
- if (matched.length > 0) {
1094
- results.push({ tx, events: matched });
1095
- }
1096
- }
1097
- break;
1098
- }
1099
- case "nft_transfer":
1100
- case "nft_mint":
1101
- case "nft_burn": {
1102
- const eventType = `${filter.type}_event`;
1103
- for (const tx of transactions) {
1104
- const txEvents = eventsByTx.get(tx.tx_id) ?? [];
1105
- const matched = txEvents.filter((e) => {
1106
- if (e.type !== eventType)
1107
- return false;
1108
- const data = e.data;
1109
- if (!data)
1110
- return false;
1111
- if (filter.assetIdentifier) {
1112
- const assetId = data.asset_identifier;
1113
- if (!assetId || !matchPattern(assetId, filter.assetIdentifier))
1114
- return false;
1115
- }
1116
- if (!traitAllows(filter, assetContract(data.asset_identifier), traitContracts))
1117
- return false;
1118
- if ("sender" in filter && filter.sender) {
1119
- if (!matchPattern(data.sender, filter.sender))
1120
- return false;
1121
- }
1122
- if ("recipient" in filter && filter.recipient) {
1123
- if (!matchPattern(data.recipient, filter.recipient))
1124
- return false;
1125
- }
1126
- return true;
1127
- });
1128
- if (matched.length > 0) {
1129
- results.push({ tx, events: matched });
1130
- }
1131
- }
1132
- break;
1133
- }
1134
- case "contract_call": {
1135
- for (const tx of transactions) {
1136
- if (tx.type !== "contract_call")
1137
- continue;
1138
- if (filter.contractId) {
1139
- if (!tx.contract_id || !matchAny(tx.contract_id, filter.contractId))
1140
- continue;
1141
- }
1142
- if (!factoryAllows(filter, tx.contract_id, factoryContracts))
1143
- continue;
1144
- if (filter.functionName) {
1145
- if (!tx.function_name || !matchPattern(tx.function_name, filter.functionName))
1146
- continue;
1147
- }
1148
- if (filter.caller) {
1149
- if (!matchPattern(tx.sender, filter.caller))
1150
- continue;
1151
- }
1152
- if (!traitAllows(filter, tx.contract_id, traitContracts))
1153
- continue;
1154
- const txEvents = eventsByTx.get(tx.tx_id) ?? [];
1155
- results.push({ tx, events: txEvents });
1156
- }
1157
- break;
1158
- }
1159
- case "contract_deploy": {
1160
- for (const tx of transactions) {
1161
- if (tx.type !== "smart_contract")
1162
- continue;
1163
- if (filter.deployer) {
1164
- if (!matchPattern(tx.sender, filter.deployer))
1165
- continue;
1166
- }
1167
- if (filter.contractName) {
1168
- const name = tx.contract_id?.split(".")[1] ?? "";
1169
- if (!matchPattern(name, filter.contractName))
1170
- continue;
1171
- }
1172
- const txEvents = eventsByTx.get(tx.tx_id) ?? [];
1173
- results.push({ tx, events: txEvents });
1174
- }
1175
- break;
1176
- }
1177
- case "nested_contract_call":
1178
- case "var_set":
1179
- case "map_set":
1180
- case "map_insert":
1181
- case "map_delete": {
1182
- for (const tx of transactions) {
1183
- const txEvents = vmEventsByTx.get(tx.tx_id) ?? [];
1184
- const matched = txEvents.filter((e) => {
1185
- if (e.type !== filter.type)
1186
- return false;
1187
- const data = e.data;
1188
- if (!data)
1189
- return false;
1190
- const contractId = printContractId(data);
1191
- if ("contractId" in filter && filter.contractId) {
1192
- if (!contractId || !matchAny(contractId, filter.contractId))
1193
- return false;
1194
- }
1195
- if (!traitAllows(filter, contractId, traitContracts))
1196
- return false;
1197
- if (!factoryAllows(filter, contractId, factoryContracts))
1198
- return false;
1199
- if (filter.type === "nested_contract_call") {
1200
- if (filter.functionName) {
1201
- if (typeof data.function_name !== "string" || !matchPattern(data.function_name, filter.functionName))
1202
- return false;
1203
- }
1204
- if (filter.caller) {
1205
- if (typeof data.caller !== "string" || !matchPattern(data.caller, filter.caller))
1206
- return false;
1207
- }
1208
- if (filter.sender) {
1209
- if (typeof data.sender !== "string" || !matchPattern(data.sender, filter.sender))
1210
- return false;
1211
- }
1212
- }
1213
- if (filter.type === "var_set" && filter.varName) {
1214
- if (typeof data.var_name !== "string" || !matchPattern(data.var_name, filter.varName))
1215
- return false;
1216
- }
1217
- if ((filter.type === "map_set" || filter.type === "map_insert" || filter.type === "map_delete") && filter.map) {
1218
- if (typeof data.map_name !== "string" || !matchPattern(data.map_name, filter.map))
1219
- return false;
1220
- }
1221
- return true;
1222
- });
1223
- if (matched.length > 0)
1224
- results.push({ tx, events: matched });
1225
- }
1226
- break;
1227
- }
1228
- case "print_event": {
1229
- for (const tx of transactions) {
1230
- const txEvents = eventsByTx.get(tx.tx_id) ?? [];
1231
- const matched = txEvents.filter((e) => {
1232
- if (e.type !== "smart_contract_event" && e.type !== "contract_event")
1233
- return false;
1234
- const data = e.data;
1235
- if (!data)
1236
- return false;
1237
- if (data.topic !== "print")
1238
- return false;
1239
- const contractId = printContractId(data);
1240
- if (filter.contractId) {
1241
- if (!contractId || !matchAny(contractId, filter.contractId))
1242
- return false;
1243
- }
1244
- if (!traitAllows(filter, contractId, traitContracts))
1245
- return false;
1246
- if (!factoryAllows(filter, contractId, factoryContracts))
1247
- return false;
1248
- return true;
1249
- });
1250
- if (matched.length > 0) {
1251
- results.push({ tx, events: matched });
1252
- }
1253
- }
1254
- break;
1255
- }
1256
- }
1257
- return results;
1258
- }
1259
- function matchSources(sources, transactions, events, traitContracts = new Map, factoryContracts = new Map, vmEvents = []) {
1260
- const eventsByTx = new Map;
1261
- for (const event of events) {
1262
- const list = eventsByTx.get(event.tx_id) ?? [];
1263
- list.push(event);
1264
- eventsByTx.set(event.tx_id, list);
1265
- }
1266
- const vmEventsByTx = new Map;
1267
- for (const event of vmEvents) {
1268
- const list = vmEventsByTx.get(event.tx_id) ?? [];
1269
- list.push(event);
1270
- vmEventsByTx.set(event.tx_id, list);
1271
- }
1272
- const seen = new Set;
1273
- const results = [];
1274
- for (const [sourceName, filter] of Object.entries(sources)) {
1275
- const matches = matchFilter(filter, transactions, eventsByTx, traitContracts, factoryContracts, vmEventsByTx);
1276
- for (const match of matches) {
1277
- const dedupeKey = `${match.tx.tx_id}:${sourceName}`;
1278
- if (!seen.has(dedupeKey)) {
1279
- seen.add(dedupeKey);
1280
- results.push({ ...match, sourceName });
1281
- }
1282
- }
1283
- }
1284
- return results;
1285
- }
1286
-
1287
- // src/runtime/runner.ts
1288
- import { getErrorMessage } from "@secondlayer/shared";
1289
- import { logger as logger4 } from "@secondlayer/shared/logger";
1290
- import {
1291
- clarityValueToJS,
1292
- deserializeCV as deserializeCV4,
1293
- toCamelCase as toCamelCase2
1294
- } from "@secondlayer/stacks/clarity";
1295
-
1296
- // src/print-schema.ts
1297
- import {
1298
- deserializeCV as deserializeCV3
1299
- } from "@secondlayer/stacks/clarity";
1300
- function camelizeDataKey(str) {
1301
- return str.replace(/-([a-z0-9])/g, (_, c) => c.toUpperCase());
1302
- }
1303
- function camelizeKeys(obj) {
1304
- if (obj === null || obj === undefined)
1305
- return obj;
1306
- if (typeof obj !== "object")
1307
- return obj;
1308
- if (Array.isArray(obj))
1309
- return obj.map(camelizeKeys);
1310
- const result = {};
1311
- for (const [k, v] of Object.entries(obj)) {
1312
- result[camelizeDataKey(k)] = camelizeKeys(v);
1313
- }
1314
- return result;
1315
- }
1316
- function cvToTree(cv) {
1317
- switch (cv.type) {
1318
- case "uint":
1319
- case "int":
1320
- return { kind: cv.type };
1321
- case "true":
1322
- case "false":
1323
- return { kind: "bool" };
1324
- case "address":
1325
- case "contract":
1326
- return { kind: "principal" };
1327
- case "buffer":
1328
- return { kind: "buffer", len: cv.value.length / 2 };
1329
- case "ascii":
1330
- return { kind: "ascii", len: cv.value.length };
1331
- case "utf8":
1332
- return { kind: "utf8", len: new TextEncoder().encode(cv.value).length };
1333
- case "none":
1334
- return { kind: "optional", inner: null };
1335
- case "some":
1336
- return { kind: "optional", inner: cvToTree(cv.value) };
1337
- case "ok":
1338
- return { kind: "response", ok: cvToTree(cv.value), err: null };
1339
- case "err":
1340
- return { kind: "response", ok: null, err: cvToTree(cv.value) };
1341
- case "list": {
1342
- let inner = null;
1343
- for (const el of cv.value) {
1344
- const t = cvToTree(el);
1345
- inner = inner ? unify(inner, t) : t;
1346
- }
1347
- return { kind: "list", inner };
1348
- }
1349
- case "tuple": {
1350
- const entries = new Map;
1351
- for (const [k, v] of Object.entries(cv.value)) {
1352
- entries.set(k, { tree: cvToTree(v), present: 1 });
1353
- }
1354
- return { kind: "tuple", count: 1, entries };
1355
- }
1356
- }
1357
- }
1358
- function unifyNullable(a, b) {
1359
- if (!a)
1360
- return b;
1361
- if (!b)
1362
- return a;
1363
- return unify(a, b);
1364
- }
1365
- var UNION_KIND_ORDER = {
1366
- uint: 0,
1367
- int: 1,
1368
- bool: 2,
1369
- principal: 3,
1370
- buffer: 4,
1371
- ascii: 5,
1372
- utf8: 6,
1373
- list: 7,
1374
- tuple: 8,
1375
- response: 9
1376
- };
1377
- function makeUnion(members) {
1378
- const sorted = [...members].sort((a, b) => (UNION_KIND_ORDER[a.kind] ?? 99) - (UNION_KIND_ORDER[b.kind] ?? 99));
1379
- return { kind: "union", members: sorted };
1380
- }
1381
- function unify(a, b) {
1382
- if (a.kind === "optional" || b.kind === "optional") {
1383
- const ai = a.kind === "optional" ? a.inner : a;
1384
- const bi = b.kind === "optional" ? b.inner : b;
1385
- return { kind: "optional", inner: unifyNullable(ai, bi) };
1386
- }
1387
- if (a.kind === "union")
1388
- return unionAdd(a.members, b);
1389
- if (b.kind === "union")
1390
- return unionAdd(b.members, a);
1391
- if (a.kind !== b.kind)
1392
- return makeUnion([a, b]);
1393
- switch (a.kind) {
1394
- case "uint":
1395
- case "int":
1396
- case "bool":
1397
- case "principal":
1398
- return a;
1399
- case "buffer":
1400
- case "ascii":
1401
- case "utf8":
1402
- return { kind: a.kind, len: Math.max(a.len, b.len) };
1403
- case "list":
1404
- return {
1405
- kind: "list",
1406
- inner: unifyNullable(a.inner, b.inner)
1407
- };
1408
- case "response": {
1409
- const rb = b;
1410
- return {
1411
- kind: "response",
1412
- ok: unifyNullable(a.ok, rb.ok),
1413
- err: unifyNullable(a.err, rb.err)
1414
- };
1415
- }
1416
- case "tuple": {
1417
- const tb = b;
1418
- const entries = new Map([...a.entries].map(([k, e]) => [k, { ...e }]));
1419
- for (const [k, e] of tb.entries) {
1420
- const existing = entries.get(k);
1421
- entries.set(k, existing ? {
1422
- tree: unify(existing.tree, e.tree),
1423
- present: existing.present + e.present
1424
- } : { ...e });
1425
- }
1426
- return { kind: "tuple", count: a.count + tb.count, entries };
1427
- }
1428
- }
1429
- }
1430
- function unionAdd(members, t) {
1431
- if (t.kind === "union") {
1432
- let acc = { kind: "union", members };
1433
- for (const m of t.members)
1434
- acc = unify(acc, m);
1435
- return acc;
1436
- }
1437
- const next = [...members];
1438
- for (let i = 0;i < next.length; i++) {
1439
- const member = next[i];
1440
- if (!member)
1441
- continue;
1442
- const merged = unify(member, t);
1443
- if (merged.kind !== "union") {
1444
- next[i] = merged;
1445
- return makeUnion(next);
1446
- }
1447
- }
1448
- next.push(t);
1449
- return makeUnion(next);
1450
- }
1451
- function wrapOptional(t) {
1452
- return t.kind === "optional" ? t : { kind: "optional", inner: t };
1453
- }
1454
- function renderClarity(t) {
1455
- if (!t)
1456
- return "?";
1457
- switch (t.kind) {
1458
- case "uint":
1459
- case "int":
1460
- case "bool":
1461
- case "principal":
1462
- return t.kind;
1463
- case "buffer":
1464
- return `(buff ${t.len})`;
1465
- case "ascii":
1466
- return `(string-ascii ${t.len})`;
1467
- case "utf8":
1468
- return `(string-utf8 ${t.len})`;
1469
- case "optional":
1470
- return `(optional ${renderClarity(t.inner)})`;
1471
- case "list":
1472
- return `(list ${renderClarity(t.inner)})`;
1473
- case "response":
1474
- return `(response ${renderClarity(t.ok)} ${renderClarity(t.err)})`;
1475
- case "tuple": {
1476
- const parts = [...t.entries].map(([k, e]) => {
1477
- const tree = e.present < t.count ? wrapOptional(e.tree) : e.tree;
1478
- return `(${k} ${renderClarity(tree)})`;
1479
- });
1480
- return `(tuple ${parts.join(" ")})`;
1481
- }
1482
- case "union":
1483
- return t.members.map(renderClarity).join(" | ");
1484
- }
1485
- }
1486
- function renderTs(t) {
1487
- if (!t)
1488
- return "unknown";
1489
- switch (t.kind) {
1490
- case "uint":
1491
- case "int":
1492
- return "bigint";
1493
- case "bool":
1494
- return "boolean";
1495
- case "principal":
1496
- case "buffer":
1497
- case "ascii":
1498
- case "utf8":
1499
- return "string";
1500
- case "optional":
1501
- return t.inner ? `${renderTs(t.inner)} | null` : "unknown | null";
1502
- case "list": {
1503
- const inner = renderTs(t.inner);
1504
- return inner.includes(" | ") ? `(${inner})[]` : `${inner}[]`;
1505
- }
1506
- case "response": {
1507
- const sides = [...new Set([renderTs(t.ok), renderTs(t.err)])];
1508
- return sides.join(" | ");
1509
- }
1510
- case "tuple": {
1511
- const parts = [...t.entries].map(([k, e]) => {
1512
- const opt = e.present < t.count ? "?" : "";
1513
- return `${camelizeDataKey(k)}${opt}: ${renderTs(e.tree)}`;
1514
- });
1515
- return `{ ${parts.join("; ")} }`;
1516
- }
1517
- case "union":
1518
- return [...new Set(t.members.map((m) => renderTs(m)))].join(" | ");
1519
- }
1520
- }
1521
- function toColumnType(t) {
1522
- if (!t)
1523
- return "jsonb";
1524
- switch (t.kind) {
1525
- case "uint":
1526
- return "uint";
1527
- case "int":
1528
- return "int";
1529
- case "bool":
1530
- return "boolean";
1531
- case "principal":
1532
- return "principal";
1533
- case "buffer":
1534
- case "ascii":
1535
- case "utf8":
1536
- return "text";
1537
- case "list":
1538
- case "tuple":
1539
- return "jsonb";
1540
- case "optional":
1541
- return toColumnType(t.inner);
1542
- case "response":
1543
- return t.ok ? toColumnType(t.ok) : "jsonb";
1544
- case "union":
1545
- return "jsonb";
1546
- }
1547
- }
1548
- var MAX_DECODED_PER_TOPIC_NEWEST = 75;
1549
- var MAX_DECODED_PER_TOPIC_OLDEST = 25;
1550
- function inferPrintTopics(samples) {
1551
- const groups = new Map;
1552
- for (const s of samples) {
1553
- const group = groups.get(s.topic);
1554
- if (group)
1555
- group.push(s);
1556
- else
1557
- groups.set(s.topic, [s]);
1558
- }
1559
- const out = [];
1560
- for (const [topic, rows] of groups) {
1561
- let first = Number.POSITIVE_INFINITY;
1562
- let last = Number.NEGATIVE_INFINITY;
1563
- for (const r of rows) {
1564
- if (r.blockHeight < first)
1565
- first = r.blockHeight;
1566
- if (r.blockHeight > last)
1567
- last = r.blockHeight;
1568
- }
1569
- const withHex = [...rows].filter((r) => r.rawHex !== null).sort((a, b) => b.blockHeight - a.blockHeight);
1570
- const budget = MAX_DECODED_PER_TOPIC_NEWEST + MAX_DECODED_PER_TOPIC_OLDEST;
1571
- const picked = withHex.length <= budget ? withHex : [
1572
- ...withHex.slice(0, MAX_DECODED_PER_TOPIC_NEWEST),
1573
- ...withHex.slice(-MAX_DECODED_PER_TOPIC_OLDEST)
1574
- ];
1575
- const tuples = [];
1576
- let decoded = 0;
1577
- for (const p of picked) {
1578
- try {
1579
- const cv = deserializeCV3(p.rawHex);
1580
- decoded++;
1581
- if (cv.type === "tuple")
1582
- tuples.push(cv);
1583
- } catch {}
1584
- }
1585
- const nonTuple = decoded > 0 && tuples.length === 0;
1586
- const fields = [];
1587
- if (!nonTuple) {
1588
- const stats = new Map;
1589
- for (const t of tuples) {
1590
- for (const [key, value] of Object.entries(t.value)) {
1591
- if (key === "topic")
1592
- continue;
1593
- const tree = cvToTree(value);
1594
- const existing = stats.get(key);
1595
- if (existing) {
1596
- existing.present++;
1597
- existing.tree = unify(existing.tree, tree);
1598
- if (value.type === "none")
1599
- existing.noneCount++;
1600
- if (value.type === "none" || value.type === "some") {
1601
- existing.optionalSeen = true;
1602
- }
1603
- } else {
1604
- stats.set(key, {
1605
- tree,
1606
- present: 1,
1607
- noneCount: value.type === "none" ? 1 : 0,
1608
- optionalSeen: value.type === "none" || value.type === "some"
1609
- });
1610
- }
1611
- }
1612
- }
1613
- for (const [name, st] of [...stats].sort(([a], [b]) => a.localeCompare(b))) {
1614
- const field = {
1615
- name,
1616
- camel_name: camelizeDataKey(name),
1617
- clarity_type: renderClarity(st.tree),
1618
- ts_type: renderTs(st.tree),
1619
- column_type: toColumnType(st.tree),
1620
- always_present: st.present === tuples.length
1621
- };
1622
- if (st.optionalSeen) {
1623
- field.optional_some_rate = (st.present - st.noneCount) / st.present;
1624
- }
1625
- fields.push(field);
1626
- }
1627
- }
1628
- out.push({
1629
- topic,
1630
- count: rows.length,
1631
- first_height: first,
1632
- last_height: last,
1633
- non_tuple: nonTuple,
1634
- fields
1635
- });
1636
- }
1637
- return out.sort((a, b) => b.count - a.count);
1638
- }
1639
-
1640
- // src/runtime/materialize.ts
1641
- function resolveColumn(col, event, tx, block) {
1642
- if ("fromTx" in col) {
1643
- return { ok: true, value: tx[col.fromTx] };
1644
- }
1645
- if ("fromBlock" in col) {
1646
- return { ok: true, value: block[col.fromBlock] };
1647
- }
1648
- const data = event.data;
1649
- const dataObj = data !== null && typeof data === "object" && !Array.isArray(data) ? data : undefined;
1650
- if (dataObj && Object.prototype.hasOwnProperty.call(dataObj, col.from)) {
1651
- return { ok: true, value: dataObj[col.from] };
1652
- }
1653
- if (col.from === "topic" && "topic" in event) {
1654
- return { ok: true, value: event.topic };
1655
- }
1656
- if (Object.prototype.hasOwnProperty.call(event, col.from)) {
1657
- return { ok: true, value: event[col.from] };
1658
- }
1659
- return { ok: false, missing: col.from };
1660
- }
1661
- function buildMaterializeRow(spec, event, ctx, tableColumns) {
1662
- const row = {};
1663
- for (const [colName, mapping] of Object.entries(spec.columns)) {
1664
- const resolved = resolveColumn(mapping, event, ctx.tx, ctx.block);
1665
- if (!resolved.ok) {
1666
- const colDef = tableColumns[colName];
1667
- if (colDef?.nullable) {
1668
- row[colName] = null;
1669
- continue;
1670
- }
1671
- return {
1672
- ok: false,
1673
- reason: `required materialize field "${resolved.missing}" missing for column "${colName}"`
1674
- };
1675
- }
1676
- row[colName] = resolved.value ?? null;
1677
- }
1678
- return { ok: true, row };
1679
- }
1680
- function applyMaterializeInsert(spec, event, ctx, schema) {
1681
- const table = schema[spec.table];
1682
- if (!table) {
1683
- return { ok: false, reason: `materialize table "${spec.table}" missing` };
1684
- }
1685
- const built = buildMaterializeRow(spec, event, ctx, table.columns);
1686
- if (!built.ok)
1687
- return built;
1688
- ctx.insert(spec.table, built.row);
1689
- return { ok: true };
1690
- }
1691
-
1692
- // src/runtime/print-validate.ts
1693
- var COLUMN_TYPES = new Set([
1694
- "text",
1695
- "uint",
1696
- "int",
1697
- "principal",
1698
- "boolean",
1699
- "timestamp",
1700
- "jsonb"
1701
- ]);
1702
- function isColumnType(field) {
1703
- return typeof field === "string" && COLUMN_TYPES.has(field);
1704
- }
1705
- function matchesField(field, value) {
1706
- if (isColumnType(field)) {
1707
- switch (field) {
1708
- case "uint":
1709
- case "int":
1710
- return typeof value === "bigint" || typeof value === "number";
1711
- case "boolean":
1712
- return typeof value === "boolean";
1713
- case "text":
1714
- case "principal":
1715
- case "timestamp":
1716
- return typeof value === "string" || typeof value === "number" || typeof value === "bigint";
1717
- case "jsonb":
1718
- return value !== undefined;
1719
- }
1720
- }
1721
- if (typeof field !== "object" || field === null)
1722
- return false;
1723
- if ("optional" in field) {
1724
- return value === undefined || value === null || matchesField(field.type, value);
1725
- }
1726
- if ("tuple" in field) {
1727
- if (typeof value !== "object" || value === null || Array.isArray(value)) {
1728
- return false;
1729
- }
1730
- return matchesTopic(field.tuple, value).ok;
1731
- }
1732
- if ("list" in field) {
1733
- return Array.isArray(value) && value.every((item) => matchesField(field.list, item));
1734
- }
1735
- return false;
1736
- }
1737
- function matchesTopic(declared, data) {
1738
- for (const [name, field] of Object.entries(declared)) {
1739
- const present = name in data;
1740
- const optional = typeof field === "object" && field !== null && "optional" in field;
1741
- if (!present) {
1742
- if (optional)
1743
- continue;
1744
- return { ok: false, reason: `missing declared field "${name}"` };
1745
- }
1746
- if (!matchesField(field, data[name])) {
1747
- return {
1748
- ok: false,
1749
- reason: `field "${name}" does not match its declared type (got ${describe(data[name])})`
1750
- };
1751
- }
1752
- }
1753
- return { ok: true };
1754
- }
1755
- function describe(value) {
1756
- if (value === null)
1757
- return "null";
1758
- if (Array.isArray(value))
1759
- return "array";
1760
- return typeof value;
1761
- }
1762
- function validatePrintPayload(prints, topic, data) {
1763
- if (!prints)
1764
- return { ok: true };
1765
- const declared = prints[topic];
1766
- if (!declared)
1767
- return { ok: true };
1768
- return matchesTopic(declared, data);
1769
- }
1770
-
1771
- // src/runtime/violations.ts
1772
- import { getTargetDb } from "@secondlayer/shared/db";
1773
- import { logger as logger3 } from "@secondlayer/shared/logger";
1774
- import { sql as sql2 } from "kysely";
1775
- var VIOLATIONS_CAP_PER_SUBGRAPH = 100;
1776
- function jsonSafe2(value) {
1777
- if (typeof value === "bigint")
1778
- return value.toString();
1779
- if (value === null || value === undefined)
1780
- return value;
1781
- if (Array.isArray(value))
1782
- return value.map(jsonSafe2);
1783
- if (typeof value === "object") {
1784
- const out = {};
1785
- for (const [k, v] of Object.entries(value)) {
1786
- out[k] = jsonSafe2(v);
1787
- }
1788
- return out;
1789
- }
1790
- return value;
1791
- }
1792
- async function defaultRecordPrintViolation(input) {
1793
- try {
1794
- const db = getTargetDb();
1795
- const sample = jsonSafe2(input.samplePayload);
1796
- await db.insertInto("subgraph_violations").values({
1797
- subgraph_name: input.subgraphName,
1798
- source_name: input.sourceName,
1799
- block_height: input.blockHeight,
1800
- tx_id: input.txId,
1801
- reason: input.reason,
1802
- sample_payload: sample
1803
- }).execute();
1804
- await sql2`
1805
- DELETE FROM subgraph_violations
1806
- WHERE id IN (
1807
- SELECT id FROM subgraph_violations
1808
- WHERE subgraph_name = ${input.subgraphName}
1809
- ORDER BY seen_at DESC
1810
- OFFSET ${VIOLATIONS_CAP_PER_SUBGRAPH}
1811
- )
1812
- `.execute(db);
1813
- } catch (err) {
1814
- logger3.warn("Failed to record print violation", {
1815
- subgraph: input.subgraphName,
1816
- error: err instanceof Error ? err.message : String(err)
1817
- });
1818
- }
1819
- }
1820
- var recorder = defaultRecordPrintViolation;
1821
- async function recordPrintViolation(input) {
1822
- await recorder(input);
1823
- }
1824
-
1825
- // src/runtime/runner.ts
1826
- var DEFAULT_ERROR_THRESHOLD = 50;
1827
- function decodeFunctionArgs2(args) {
1828
- let parsed = args;
1829
- if (typeof parsed === "string") {
1830
- try {
1831
- parsed = JSON.parse(parsed);
1832
- } catch {
1833
- return [];
1834
- }
1835
- }
1836
- if (!Array.isArray(parsed))
1837
- return [];
1838
- return parsed.map((arg) => {
1839
- if (typeof arg === "string")
1840
- return decodeClarityValue(arg);
1841
- return arg;
1842
- });
1843
- }
1844
- function decodeRawResult(raw) {
1845
- if (typeof raw === "string" && raw.length > 2) {
1846
- return decodeClarityValue(raw);
1847
- }
1848
- return null;
1849
- }
1850
- function decodeRawHex(hex) {
1851
- if (hex.startsWith("0x") && hex.length > 2) {
1852
- return decodeClarityValue(hex);
1853
- }
1854
- return hex;
1855
- }
1856
- function safeBigInt(val) {
1857
- if (typeof val === "bigint")
1858
- return val;
1859
- if (typeof val === "number")
1860
- return BigInt(val);
1861
- if (typeof val === "string") {
1862
- try {
1863
- return BigInt(val);
1864
- } catch {
1865
- return 0n;
1866
- }
1867
- }
1868
- return 0n;
1869
- }
1870
- function buildContractCallInput(filter, tx) {
1871
- const abi = filter.abi;
1872
- const fnName = tx.function_name;
1873
- if (!abi || !fnName)
1874
- return;
1875
- const fn = abi.functions?.find((f) => f.name === fnName);
1876
- if (!fn || !Array.isArray(fn.args))
1877
- return;
1878
- let rawArgs = tx.function_args;
1879
- if (typeof rawArgs === "string") {
1880
- try {
1881
- rawArgs = JSON.parse(rawArgs);
1882
- } catch {
1883
- return;
1884
- }
1885
- }
1886
- if (!Array.isArray(rawArgs))
1887
- return;
1888
- const decodeArg = clarityValueToJS;
1889
- const input = {};
1890
- fn.args.forEach((arg, i) => {
1891
- const hex = rawArgs[i];
1892
- if (typeof hex !== "string")
1893
- return;
1894
- try {
1895
- const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
1896
- input[toCamelCase2(arg.name)] = decodeArg(arg.type, deserializeCV4(clean));
1897
- } catch {}
1898
- });
1899
- return input;
1900
- }
1901
- function buildEventPayload(filter, tx, event) {
1902
- const txMeta = {
1903
- txId: tx.tx_id,
1904
- sender: tx.sender,
1905
- type: tx.type,
1906
- status: tx.status,
1907
- contractId: tx.contract_id ?? null,
1908
- functionName: tx.function_name ?? null
1909
- };
1910
- const decodedArgs = decodeFunctionArgs2(tx.function_args);
1911
- const decodedResult = decodeRawResult(tx.raw_result);
1912
- if (!event) {
1913
- switch (filter.type) {
1914
- case "contract_call": {
1915
- const input = buildContractCallInput(filter, tx);
1916
- return {
1917
- type: "contract_call",
1918
- contractId: tx.contract_id ?? "",
1919
- functionName: tx.function_name ?? "",
1920
- sender: tx.sender,
1921
- args: decodedArgs,
1922
- ...input !== undefined ? { input } : {},
1923
- result: decodedResult,
1924
- resultHex: tx.raw_result ?? null,
1925
- tx: txMeta
1926
- };
1927
- }
1928
- case "contract_deploy":
1929
- return {
1930
- contractId: tx.contract_id ?? "",
1931
- deployer: tx.sender,
1932
- tx: txMeta
1933
- };
1934
- default:
1935
- return { tx: txMeta };
1936
- }
1937
- }
1938
- const decoded = decodeEventData(event.data);
1939
- switch (filter.type) {
1940
- case "ft_transfer":
1941
- return {
1942
- sender: decoded.sender,
1943
- recipient: decoded.recipient,
1944
- amount: safeBigInt(decoded.amount),
1945
- assetIdentifier: decoded.asset_identifier,
1946
- tx: txMeta
1947
- };
1948
- case "ft_mint":
1949
- return {
1950
- recipient: decoded.recipient,
1951
- amount: safeBigInt(decoded.amount),
1952
- assetIdentifier: decoded.asset_identifier,
1953
- tx: txMeta
1954
- };
1955
- case "ft_burn":
1956
- return {
1957
- sender: decoded.sender,
1958
- amount: safeBigInt(decoded.amount),
1959
- assetIdentifier: decoded.asset_identifier,
1960
- tx: txMeta
1961
- };
1962
- case "nft_transfer":
1963
- return {
1964
- sender: decoded.sender,
1965
- recipient: decoded.recipient,
1966
- tokenId: decoded.raw_value ?? decoded.value,
1967
- assetIdentifier: decoded.asset_identifier,
1968
- tx: txMeta
1969
- };
1970
- case "nft_mint":
1971
- return {
1972
- recipient: decoded.recipient,
1973
- tokenId: decoded.raw_value ?? decoded.value,
1974
- assetIdentifier: decoded.asset_identifier,
1975
- tx: txMeta
1976
- };
1977
- case "nft_burn":
1978
- return {
1979
- sender: decoded.sender,
1980
- tokenId: decoded.raw_value ?? decoded.value,
1981
- assetIdentifier: decoded.asset_identifier,
1982
- tx: txMeta
1983
- };
1984
- case "stx_transfer":
1985
- return {
1986
- sender: decoded.sender,
1987
- recipient: decoded.recipient,
1988
- amount: safeBigInt(decoded.amount),
1989
- memo: decoded.memo ?? "",
1990
- tx: txMeta
1991
- };
1992
- case "stx_mint":
1993
- return {
1994
- recipient: decoded.recipient,
1995
- amount: safeBigInt(decoded.amount),
1996
- tx: txMeta
1997
- };
1998
- case "stx_burn":
1999
- return {
2000
- sender: decoded.sender,
2001
- amount: safeBigInt(decoded.amount),
2002
- tx: txMeta
2003
- };
2004
- case "stx_lock":
2005
- return {
2006
- lockedAddress: decoded.locked_address,
2007
- lockedAmount: safeBigInt(decoded.locked_amount),
2008
- unlockHeight: safeBigInt(decoded.unlock_height),
2009
- tx: txMeta
2010
- };
2011
- case "print_event": {
2012
- const rawHex = event.data?.raw_value;
2013
- const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
2014
- const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
2015
- const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
2016
- const { topic: _, ...rest } = clarityObj ?? {};
2017
- const data = Object.keys(rest).length > 0 ? camelizeKeys(rest) : rawValue && typeof rawValue !== "object" ? rawValue : {};
2018
- return {
2019
- contractId: printContractId(decoded) ?? tx.contract_id ?? "",
2020
- topic,
2021
- data: data ?? {},
2022
- tx: txMeta
2023
- };
2024
- }
2025
- case "contract_call": {
2026
- const ccRawHex = event.data?.raw_value;
2027
- const normalized = typeof ccRawHex === "string" && ccRawHex.startsWith("0x") ? { ...decoded, value: decodeClarityValue(ccRawHex) } : decoded;
2028
- const input = buildContractCallInput(filter, tx);
2029
- return {
2030
- ...normalized,
2031
- type: "contract_call",
2032
- _eventType: event.type,
2033
- contractId: tx.contract_id ?? "",
2034
- functionName: tx.function_name ?? "",
2035
- sender: tx.sender,
2036
- args: decodedArgs,
2037
- ...input !== undefined ? { input } : {},
2038
- result: decodedResult,
2039
- resultHex: tx.raw_result ?? null,
2040
- tx: txMeta
2041
- };
2042
- }
2043
- case "contract_deploy":
2044
- return {
2045
- contractId: tx.contract_id ?? "",
2046
- deployer: tx.sender,
2047
- tx: txMeta
2048
- };
2049
- case "nested_contract_call": {
2050
- const data = event.data ?? {};
2051
- const argsHex = Array.isArray(data.function_args) ? data.function_args.filter((a) => typeof a === "string") : [];
2052
- return {
2053
- contractId: printContractId(data) ?? "",
2054
- sender: typeof data.sender === "string" ? data.sender : null,
2055
- caller: typeof data.caller === "string" ? data.caller : "",
2056
- functionName: typeof data.function_name === "string" ? data.function_name : "",
2057
- args: decodeFunctionArgs2(argsHex),
2058
- arguments: argsHex,
2059
- rawResult: typeof data.raw_result === "string" ? data.raw_result : "",
2060
- result: decodeRawResult(data.raw_result),
2061
- tx: txMeta
2062
- };
2063
- }
2064
- case "var_set": {
2065
- const data = event.data ?? {};
2066
- const rawValue = typeof data.raw_value === "string" ? data.raw_value : "";
2067
- return {
2068
- contractId: printContractId(data) ?? "",
2069
- varName: typeof data.var_name === "string" ? data.var_name : "",
2070
- rawValue,
2071
- value: decodeRawHex(rawValue),
2072
- tx: txMeta
2073
- };
2074
- }
2075
- case "map_set":
2076
- case "map_insert": {
2077
- const data = event.data ?? {};
2078
- const rawKey = typeof data.raw_key === "string" ? data.raw_key : "";
2079
- const rawValue = typeof data.raw_value === "string" ? data.raw_value : "";
2080
- return {
2081
- contractId: printContractId(data) ?? "",
2082
- map: typeof data.map_name === "string" ? data.map_name : "",
2083
- rawKey,
2084
- key: decodeRawHex(rawKey),
2085
- rawValue,
2086
- value: decodeRawHex(rawValue),
2087
- tx: txMeta
2088
- };
2089
- }
2090
- case "map_delete": {
2091
- const data = event.data ?? {};
2092
- const rawKey = typeof data.raw_key === "string" ? data.raw_key : "";
2093
- return {
2094
- contractId: printContractId(data) ?? "",
2095
- map: typeof data.map_name === "string" ? data.map_name : "",
2096
- rawKey,
2097
- key: decodeRawHex(rawKey),
2098
- tx: txMeta
2099
- };
2100
- }
2101
- default:
2102
- return {
2103
- ...decoded,
2104
- _eventType: event.type,
2105
- tx: txMeta
2106
- };
2107
- }
2108
- }
2109
- async function runHandlers(subgraph, matched, ctx, opts) {
2110
- let processed = 0;
2111
- let errors = 0;
2112
- let skipped = 0;
2113
- const threshold = opts?.errorThreshold ?? DEFAULT_ERROR_THRESHOLD;
2114
- const filterLookup = new Map;
2115
- if (!Array.isArray(subgraph.sources)) {
2116
- for (const [name, filter] of Object.entries(subgraph.sources)) {
2117
- filterLookup.set(name, filter);
2118
- }
2119
- }
2120
- const units = [];
2121
- for (const { tx, events, sourceName } of matched) {
2122
- if (events.length === 0) {
2123
- units.push({ tx, sourceName, event: null });
2124
- } else {
2125
- for (const event of events)
2126
- units.push({ tx, sourceName, event });
2127
- }
2128
- }
2129
- const clockRank = (e) => e?.clock === "vm" ? 1 : 0;
2130
- units.sort((a, b) => (a.tx.tx_index ?? 0) - (b.tx.tx_index ?? 0) || clockRank(a.event) - clockRank(b.event) || (a.event?.event_index ?? -1) - (b.event?.event_index ?? -1));
2131
- for (const { tx, event, sourceName } of units) {
2132
- if (errors >= threshold) {
2133
- logger4.error("Subgraph error threshold reached, skipping remaining events", {
2134
- subgraph: subgraph.name,
2135
- errors,
2136
- threshold
2137
- });
2138
- return { processed, errors, ...skipped > 0 ? { skipped } : {} };
2139
- }
2140
- const filter = filterLookup.get(sourceName);
2141
- const materialize = filter && "materialize" in filter && filter.materialize !== undefined ? filter.materialize : undefined;
2142
- const handler = subgraph.handlers[sourceName] ?? subgraph.handlers["*"] ?? null;
2143
- if (!handler && !materialize) {
2144
- logger4.warn("No handler found for source", {
2145
- subgraph: subgraph.name,
2146
- sourceName,
2147
- txId: tx.tx_id
2148
- });
2149
- continue;
2150
- }
2151
- ctx.setTx({
2152
- txId: tx.tx_id,
2153
- sender: tx.sender,
2154
- type: tx.type,
2155
- status: tx.status,
2156
- contractId: tx.contract_id ?? null,
2157
- functionName: tx.function_name ?? null
2158
- });
2159
- const checkpoint = ctx.opsCheckpoint();
2160
- try {
2161
- let payload;
2162
- if (event === null) {
2163
- payload = filter ? buildEventPayload(filter, tx, null) : {
2164
- tx: {
2165
- txId: tx.tx_id,
2166
- sender: tx.sender,
2167
- type: tx.type,
2168
- status: tx.status,
2169
- contractId: tx.contract_id,
2170
- functionName: tx.function_name
2171
- }
2172
- };
2173
- } else if (filter) {
2174
- payload = buildEventPayload(filter, tx, event);
2175
- } else {
2176
- const decoded = decodeEventData(event.data);
2177
- payload = {
2178
- ...decoded,
2179
- _eventId: event.id,
2180
- _eventType: event.type,
2181
- _eventIndex: event.event_index,
2182
- tx: {
2183
- txId: tx.tx_id,
2184
- sender: tx.sender,
2185
- type: tx.type,
2186
- status: tx.status,
2187
- contractId: tx.contract_id,
2188
- functionName: tx.function_name
2189
- }
2190
- };
2191
- }
2192
- if (event !== null && filter?.type === "print_event" && filter.topic && payload.topic !== filter.topic) {
2193
- continue;
2194
- }
2195
- if (event !== null && filter?.type === "print_event" && filter.prints) {
2196
- const printPayload = payload;
2197
- const verdict = validatePrintPayload(filter.prints, printPayload.topic ?? "", printPayload.data ?? {});
2198
- if (!verdict.ok) {
2199
- skipped++;
2200
- logger4.warn("Print payload does not match declared prints schema", {
2201
- subgraph: subgraph.name,
2202
- sourceName,
2203
- topic: printPayload.topic,
2204
- txId: tx.tx_id,
2205
- reason: verdict.reason
2206
- });
2207
- recordPrintViolation({
2208
- subgraphName: subgraph.name,
2209
- sourceName,
2210
- blockHeight: ctx.block.height,
2211
- txId: tx.tx_id,
2212
- reason: verdict.reason,
2213
- samplePayload: {
2214
- topic: printPayload.topic ?? "",
2215
- data: printPayload.data ?? {}
2216
- }
2217
- });
2218
- continue;
2219
- }
2220
- }
2221
- if (materialize) {
2222
- const result = applyMaterializeInsert(materialize, payload, ctx, subgraph.schema);
2223
- if (!result.ok) {
2224
- skipped++;
2225
- logger4.warn("Materialize skipped: required field missing", {
2226
- subgraph: subgraph.name,
2227
- sourceName,
2228
- txId: tx.tx_id,
2229
- reason: result.reason
2230
- });
2231
- continue;
2232
- }
2233
- processed++;
2234
- } else if (handler) {
2235
- await handler(payload, ctx);
2236
- processed++;
2237
- }
2238
- } catch (err) {
2239
- ctx.rollbackTo(checkpoint);
2240
- errors++;
2241
- logger4.error("Subgraph handler error", {
2242
- subgraph: subgraph.name,
2243
- sourceName,
2244
- txId: tx.tx_id,
2245
- ...event !== null ? { eventId: event.id, eventType: event.type } : {},
2246
- error: getErrorMessage(err)
2247
- });
2248
- }
2249
- }
2250
- return { processed, errors, ...skipped > 0 ? { skipped } : {} };
2251
- }
2252
-
2253
- // src/runtime/block-processor.ts
2254
- import { getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
2255
- import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
2256
- import { recordGapBatch } from "@secondlayer/shared/db/queries/subgraph-gaps";
2257
- import { advanceOperationCursor } from "@secondlayer/shared/db/queries/subgraph-operations";
2258
- import {
2259
- recordLiveError,
2260
- recordLiveProgress,
2261
- recordSubgraphProcessed,
2262
- rewindLiveProgress,
2263
- updateSubgraphStatus
2264
- } from "@secondlayer/shared/db/queries/subgraphs";
2265
- import { logger as logger8 } from "@secondlayer/shared/logger";
2266
- import { sql as sql4 } from "kysely";
2267
-
2268
- // src/runtime/block-source.ts
2269
- import { getSourceDb as getSourceDb2 } from "@secondlayer/shared/db";
2270
- import {
2271
- createInternalIndexHttpClient
2272
- } from "@secondlayer/shared/index-http";
2273
- import { logger as logger5 } from "@secondlayer/shared/logger";
2274
-
2275
- // src/runtime/batch-loader.ts
2276
- function vmEventId(txId, vmEventIndex) {
2277
- return `${txId}#vm:${vmEventIndex}`;
2278
- }
2279
- async function loadBlockRange(db, fromHeight, toHeight) {
2280
- const [blocks, txs, events, vmRows] = await Promise.all([
2281
- db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
2282
- db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
2283
- db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
2284
- db.selectFrom("vm_events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).orderBy("ordinal", "asc").execute()
2285
- ]);
2286
- const txsByHeight = new Map;
2287
- for (const tx of txs) {
2288
- const h = Number(tx.block_height);
2289
- const list = txsByHeight.get(h) ?? [];
2290
- list.push(tx);
2291
- txsByHeight.set(h, list);
2292
- }
2293
- const eventsByHeight = new Map;
2294
- for (const evt of events) {
2295
- const h = Number(evt.block_height);
2296
- const list = eventsByHeight.get(h) ?? [];
2297
- list.push(evt);
2298
- eventsByHeight.set(h, list);
2299
- }
2300
- const vmByHeight = new Map;
2301
- for (const row of vmRows) {
2302
- const h = Number(row.block_height);
2303
- const list = vmByHeight.get(h) ?? [];
2304
- list.push({
2305
- id: vmEventId(row.tx_id, Number(row.ordinal)),
2306
- tx_id: row.tx_id,
2307
- block_height: h,
2308
- event_index: Number(row.ordinal),
2309
- type: row.type,
2310
- data: row.data,
2311
- created_at: row.created_at,
2312
- clock: "vm"
2313
- });
2314
- vmByHeight.set(h, list);
2315
- }
2316
- const result = new Map;
2317
- for (const block of blocks) {
2318
- const h = Number(block.height);
2319
- result.set(h, {
2320
- block,
2321
- txs: txsByHeight.get(h) ?? [],
2322
- events: eventsByHeight.get(h) ?? [],
2323
- vmEvents: vmByHeight.get(h) ?? []
2324
- });
2325
- }
2326
- return result;
2327
- }
2328
- function avgEventsPerBlock(batch) {
2329
- if (batch.size === 0)
2330
- return 0;
2331
- let totalEvents = 0;
2332
- for (const data of batch.values()) {
2333
- totalEvents += data.events.length;
2334
- }
2335
- return totalEvents / batch.size;
2336
- }
2337
-
2338
- // src/runtime/observer-http-source.ts
2339
- import {
2340
- VM_NODE_TO_STORED_TYPE
2341
- } from "@secondlayer/shared";
2342
- var OBSERVER_HTTP_EXPORT_PATH = "/internal/observer-events";
2343
- function stripTrailingSlash(url) {
2344
- return url.replace(/\/+$/, "");
2345
- }
2346
- function blockTimestamp(payload) {
2347
- for (const value of [
2348
- payload.timestamp,
2349
- payload.block_time,
2350
- payload.burn_block_time,
2351
- payload.burn_block_timestamp
2352
- ]) {
2353
- if (typeof value === "number" && Number.isFinite(value) && value > 0) {
2354
- return value;
2355
- }
2356
- }
2357
- return 0;
2358
- }
2359
- function eventData(event) {
2360
- if (typeof event.type === "string" && event.type in event) {
2361
- return event[event.type] ?? {};
2362
- }
2363
- return {};
2364
- }
2365
- function mapNewBlockPayloadToBlockData(payload) {
2366
- const p = payload;
2367
- if (typeof p.block_height !== "number" || !Number.isFinite(p.block_height) || typeof p.index_block_hash !== "string" || p.index_block_hash.length === 0) {
2368
- throw new Error("observer /new_block payload missing block_height or index_block_hash");
2369
- }
2370
- const height = p.block_height;
2371
- const block = {
2372
- height,
2373
- hash: p.block_hash ?? "",
2374
- parent_hash: p.parent_block_hash ?? "",
2375
- burn_block_height: p.burn_block_height ?? 0,
2376
- burn_block_hash: p.burn_block_hash ?? null,
2377
- index_block_hash: p.index_block_hash,
2378
- timestamp: blockTimestamp(p),
2379
- canonical: true,
2380
- created_at: new Date(0)
2381
- };
2382
- const txs = (p.transactions ?? []).map((tx) => {
2383
- let contractId = null;
2384
- let functionName = null;
2385
- if (tx.tx_type === "contract_call" && tx.contract_call) {
2386
- contractId = tx.contract_call.contract_id ?? null;
2387
- functionName = tx.contract_call.function_name ?? null;
2388
- } else if (tx.tx_type === "smart_contract" && tx.smart_contract) {
2389
- contractId = tx.smart_contract.contract_id ?? null;
2390
- }
2391
- return {
2392
- tx_id: tx.txid ?? "",
2393
- block_height: height,
2394
- tx_index: tx.tx_index ?? 0,
2395
- type: tx.tx_type ?? "",
2396
- sender: tx.sender_address ?? "",
2397
- status: tx.status ?? "success",
2398
- contract_id: contractId,
2399
- function_name: functionName,
2400
- function_args: [],
2401
- raw_result: tx.raw_result ?? null,
2402
- raw_tx: tx.raw_tx ?? "",
2403
- created_at: new Date(0)
2404
- };
2405
- });
2406
- const events = (p.events ?? []).map((event) => {
2407
- const txId = event.txid ?? "";
2408
- const eventIndex = event.event_index ?? 0;
2409
- return {
2410
- id: `${txId}#${eventIndex}`,
2411
- tx_id: txId,
2412
- block_height: height,
2413
- event_index: eventIndex,
2414
- type: event.type ?? "",
2415
- data: eventData(event),
2416
- created_at: new Date(0)
2417
- };
2418
- });
2419
- const vmEvents = [];
2420
- for (const [i, trace] of (p.vm_events ?? []).entries()) {
2421
- const nodeType = trace.type;
2422
- if (typeof nodeType !== "string" || !(nodeType in VM_NODE_TO_STORED_TYPE))
2423
- continue;
2424
- const stored = VM_NODE_TO_STORED_TYPE[nodeType];
2425
- const body = trace[nodeType];
2426
- if (!body || typeof body !== "object")
2427
- continue;
2428
- const txId = trace.txid ?? "";
2429
- const vmIndex = i;
2430
- vmEvents.push({
2431
- id: vmEventId(txId, vmIndex),
2432
- tx_id: txId,
2433
- block_height: height,
2434
- event_index: vmIndex,
2435
- type: stored,
2436
- data: body,
2437
- created_at: new Date(0),
2438
- clock: "vm"
2439
- });
2440
- }
2441
- return { block, txs, events, vmEvents };
2442
- }
2443
-
2444
- class ObserverHttpBlockSource {
2445
- baseUrl;
2446
- token;
2447
- fetchFn;
2448
- constructor(opts) {
2449
- this.baseUrl = stripTrailingSlash(opts.baseUrl);
2450
- this.token = opts.token ?? null;
2451
- this.fetchFn = opts.fetch ?? globalThis.fetch.bind(globalThis);
2452
- }
2453
- headers() {
2454
- if (!this.token)
2455
- return {};
2456
- return { Authorization: `Bearer ${this.token}` };
2457
- }
2458
- async getTip() {
2459
- const res = await this.fetchFn(`${this.baseUrl}${OBSERVER_HTTP_EXPORT_PATH}/tip`, { headers: this.headers() });
2460
- if (!res.ok) {
2461
- throw new Error(`observer tip HTTP ${res.status}: ${await res.text().catch(() => "")}`);
2462
- }
2463
- let body;
2464
- try {
2465
- body = await res.json();
2466
- } catch (err) {
2467
- throw new Error(`observer tip invalid JSON: ${err instanceof Error ? err.message : String(err)}`);
2468
- }
2469
- if (typeof body.block_height !== "number" || !Number.isFinite(body.block_height)) {
2470
- throw new Error("observer tip missing block_height");
2471
- }
2472
- return body.block_height;
2473
- }
2474
- async loadBlockRange(fromHeight, toHeight) {
2475
- const result = new Map;
2476
- let afterHeight = fromHeight > 0 ? fromHeight - 1 : undefined;
2477
- let afterIndexBlockHash;
2478
- for (;; ) {
2479
- const url = new URL(`${this.baseUrl}${OBSERVER_HTTP_EXPORT_PATH}`);
2480
- url.searchParams.set("path", "/new_block");
2481
- url.searchParams.set("limit", "100");
2482
- if (afterHeight != null) {
2483
- url.searchParams.set("after_height", String(afterHeight));
2484
- }
2485
- if (afterIndexBlockHash) {
2486
- url.searchParams.set("after_index_block_hash", afterIndexBlockHash);
2487
- }
2488
- const res = await this.fetchFn(url.toString(), {
2489
- headers: this.headers()
2490
- });
2491
- if (!res.ok) {
2492
- throw new Error(`observer events HTTP ${res.status}: ${await res.text().catch(() => "")}`);
2493
- }
2494
- let body;
2495
- try {
2496
- body = await res.json();
2497
- } catch (err) {
2498
- throw new Error(`observer events invalid JSON: ${err instanceof Error ? err.message : String(err)}`);
2499
- }
2500
- let pastTo = false;
2501
- for (const message of body.events ?? []) {
2502
- if (message.path === "/new_burn_block")
2503
- continue;
2504
- const height = message.payload?.block_height;
2505
- if (typeof height === "number" && height > toHeight) {
2506
- pastTo = true;
2507
- break;
2508
- }
2509
- const data = mapNewBlockPayloadToBlockData(message.payload);
2510
- const h = data.block.height;
2511
- if (h >= fromHeight && h <= toHeight) {
2512
- result.set(h, data);
2513
- }
2514
- }
2515
- if (pastTo || body.next == null)
2516
- break;
2517
- afterHeight = body.next.after_height;
2518
- afterIndexBlockHash = body.next.after_index_block_hash;
2519
- }
2520
- return result;
2521
- }
2522
- }
2523
-
2524
- // src/runtime/reconstruct.ts
2525
- function isoToUnixSeconds(iso) {
2526
- if (!iso)
2527
- return 0;
2528
- return Math.floor(new Date(iso).getTime() / 1000);
2529
- }
2530
- function reconstructBlock(b) {
2531
- return {
2532
- height: b.block_height,
2533
- hash: b.block_hash,
2534
- parent_hash: b.parent_hash,
2535
- burn_block_height: b.burn_block_height,
2536
- burn_block_hash: b.burn_block_hash,
2537
- index_block_hash: b.index_block_hash ?? null,
2538
- timestamp: isoToUnixSeconds(b.block_time),
2539
- canonical: true,
2540
- created_at: new Date(0)
2541
- };
2542
- }
2543
- function reconstructTransaction(t) {
2544
- return {
2545
- tx_id: t.tx_id,
2546
- block_height: t.block_height,
2547
- tx_index: t.tx_index,
2548
- type: t.tx_type,
2549
- sender: t.sender,
2550
- status: t.status,
2551
- contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
2552
- function_name: t.contract_call?.function_name ?? null,
2553
- function_args: t.contract_call?.function_args_hex ?? [],
2554
- raw_result: t.contract_call?.result_hex ?? null,
2555
- raw_tx: "",
2556
- created_at: new Date(0)
2557
- };
2558
- }
2559
- function reconstructTxFromEventRow(e) {
2560
- return {
2561
- tx_id: e.tx_id,
2562
- block_height: e.block_height,
2563
- tx_index: e.tx_index ?? 0,
2564
- type: e.tx_type ?? "",
2565
- sender: e.tx_sender ?? "",
2566
- status: e.tx_status ?? "success",
2567
- contract_id: e.tx_contract_id ?? null,
2568
- function_name: e.tx_function_name ?? null,
2569
- function_args: [],
2570
- raw_result: null,
2571
- raw_tx: "",
2572
- created_at: new Date(0)
2573
- };
2574
- }
2575
- function reconstructEvent(e) {
2576
- const base = {
2577
- id: `${e.tx_id}#${e.event_index}`,
2578
- tx_id: e.tx_id,
2579
- block_height: e.block_height,
2580
- event_index: e.event_index,
2581
- created_at: new Date(0)
2582
- };
2583
- switch (e.event_type) {
2584
- case "ft_transfer":
2585
- case "ft_mint":
2586
- case "ft_burn":
2587
- return {
2588
- ...base,
2589
- type: `${e.event_type}_event`,
2590
- data: {
2591
- asset_identifier: e.asset_identifier,
2592
- sender: e.sender,
2593
- recipient: e.recipient,
2594
- amount: e.amount
2595
- }
2596
- };
2597
- case "nft_transfer":
2598
- case "nft_mint":
2599
- case "nft_burn":
2600
- return {
2601
- ...base,
2602
- type: `${e.event_type}_event`,
2603
- data: {
2604
- asset_identifier: e.asset_identifier,
2605
- sender: e.sender,
2606
- recipient: e.recipient,
2607
- raw_value: e.value
2608
- }
2609
- };
2610
- case "stx_transfer":
2611
- case "stx_mint":
2612
- case "stx_burn":
2613
- return {
2614
- ...base,
2615
- type: `${e.event_type}_event`,
2616
- data: {
2617
- sender: e.sender,
2618
- recipient: e.recipient,
2619
- amount: e.amount,
2620
- ...e.event_type === "stx_transfer" ? { memo: e.memo ?? "" } : {}
2621
- }
2622
- };
2623
- case "stx_lock":
2624
- return {
2625
- ...base,
2626
- type: "stx_lock_event",
2627
- data: {
2628
- locked_address: e.sender,
2629
- locked_amount: e.amount,
2630
- unlock_height: e.payload.unlock_height
2631
- }
2632
- };
2633
- case "print":
2634
- return {
2635
- ...base,
2636
- type: "contract_event",
2637
- data: {
2638
- topic: e.payload.topic,
2639
- contract_id: e.contract_id,
2640
- value: e.payload.value,
2641
- raw_value: e.payload.raw_value
2642
- }
2643
- };
2644
- case "nested_contract_call":
2645
- case "var_set":
2646
- case "map_set":
2647
- case "map_insert":
2648
- case "map_delete":
2649
- return {
2650
- ...base,
2651
- id: vmEventId(e.tx_id, e.event_index),
2652
- clock: "vm",
2653
- type: e.event_type,
2654
- data: {
2655
- contract_identifier: e.contract_id,
2656
- ..."payload" in e && e.payload && typeof e.payload === "object" ? e.payload : {},
2657
- sender: "sender" in e ? e.sender : undefined,
2658
- caller: "caller" in e ? e.caller : undefined,
2659
- function_name: "function_name" in e ? e.function_name : undefined,
2660
- function_args: "function_args" in e ? e.function_args : undefined,
2661
- raw_result: "raw_result" in e ? e.raw_result : undefined,
2662
- map_name: "map" in e ? e.map : undefined,
2663
- var_name: "var_name" in e ? e.var_name : undefined,
2664
- raw_key: "raw_key" in e ? e.raw_key : undefined,
2665
- raw_value: "raw_value" in e ? e.raw_value : undefined
2666
- }
2667
- };
2668
- }
2669
- }
2670
-
2671
- // src/runtime/block-source.ts
2672
- function canSparseScan(subgraph) {
2673
- if (Array.isArray(subgraph.sources))
2674
- return false;
2675
- const filters = sourceFilters(subgraph);
2676
- if (filters.length === 0)
2677
- return false;
2678
- return filters.every((f) => Boolean(EVENT_FILTER_TO_INDEX_TYPE[f.type]));
2679
- }
2680
- function sparseProbeTargets(subgraph) {
2681
- const targets = new Map;
2682
- for (const f of sourceFilters(subgraph)) {
2683
- const eventType = EVENT_FILTER_TO_INDEX_TYPE[f.type];
2684
- if (!eventType)
2685
- continue;
2686
- const scoped = f;
2687
- const contractId = scoped.contractId ?? scoped.assetIdentifier?.split("::")[0];
2688
- const key = `${eventType}|${contractId ?? ""}`;
2689
- targets.set(key, { eventType, ...contractId ? { contractId } : {} });
2690
- }
2691
- return [...targets.values()];
2692
- }
2693
-
2694
- class PostgresBlockSource {
2695
- async getTip() {
2696
- const progress = await getSourceDb2().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
2697
- return progress ? Number(progress.highest_seen_block) : 0;
2698
- }
2699
- loadBlockRange(fromHeight, toHeight) {
2700
- return loadBlockRange(getSourceDb2(), fromHeight, toHeight);
2701
- }
2702
- }
2703
- var EVENT_FILTER_TO_INDEX_TYPE = {
2704
- stx_transfer: "stx_transfer",
2705
- stx_mint: "stx_mint",
2706
- stx_burn: "stx_burn",
2707
- stx_lock: "stx_lock",
2708
- ft_transfer: "ft_transfer",
2709
- ft_mint: "ft_mint",
2710
- ft_burn: "ft_burn",
2711
- nft_transfer: "nft_transfer",
2712
- nft_mint: "nft_mint",
2713
- nft_burn: "nft_burn",
2714
- print_event: "print",
2715
- nested_contract_call: "nested_contract_call",
2716
- var_set: "var_set",
2717
- map_set: "map_set",
2718
- map_insert: "map_insert",
2719
- map_delete: "map_delete"
2720
- };
2721
- var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
2722
- var VM_INDEX_EVENT_TYPES = new Set([
2723
- "nested_contract_call",
2724
- "var_set",
2725
- "map_set",
2726
- "map_insert",
2727
- "map_delete"
2728
- ]);
2729
- var CLASSIC_INDEX_EVENT_TYPES = [
2730
- ...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
2731
- ].filter((t) => !VM_INDEX_EVENT_TYPES.has(t));
2732
- function sourceFilters(subgraph) {
2733
- const sources = subgraph.sources;
2734
- return Array.isArray(sources) ? sources : Object.values(sources);
2735
- }
2736
- function needsTransactionData(subgraph) {
2737
- return sourceFilters(subgraph).some((f) => TX_SOURCE_TYPES.has(f.type));
2738
- }
2739
- function synthesizeTxsFromEvents(events) {
2740
- const byId = new Map;
2741
- for (const e of events) {
2742
- if (!byId.has(e.tx_id))
2743
- byId.set(e.tx_id, reconstructTxFromEventRow(e));
2744
- }
2745
- return [...byId.values()];
2746
- }
2747
- function indexEventTypesForFilterTypes(filterTypes) {
2748
- const types = new Set;
2749
- if (filterTypes.some((t) => TX_SOURCE_TYPES.has(t))) {
2750
- for (const t of CLASSIC_INDEX_EVENT_TYPES)
2751
- types.add(t);
2752
- }
2753
- for (const t of filterTypes) {
2754
- const indexType = EVENT_FILTER_TO_INDEX_TYPE[t];
2755
- if (indexType)
2756
- types.add(indexType);
2757
- }
2758
- return [...types];
2759
- }
2760
- function referencedIndexEventTypes(subgraph) {
2761
- if (!subgraph.sources)
2762
- return [];
2763
- return indexEventTypesForFilterTypes(sourceFilters(subgraph).map((f) => f.type));
2764
- }
2765
- function isStreamsIndexEligible(subgraph) {
2766
- if (Array.isArray(subgraph.sources))
2767
- return false;
2768
- const filters = sourceFilters(subgraph);
2769
- if (filters.length === 0)
2770
- return false;
2771
- for (const f of filters) {
2772
- const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
2773
- if (!known)
2774
- return false;
2775
- }
2776
- return true;
2777
- }
2778
-
2779
- class PublicApiBlockSource {
2780
- http;
2781
- eventTypes;
2782
- probeTargets;
2783
- needsTransactions;
2784
- constructor(http2, eventTypes, probeTargets, needsTransactions = true) {
2785
- this.http = http2;
2786
- this.eventTypes = eventTypes;
2787
- this.probeTargets = probeTargets;
2788
- this.needsTransactions = needsTransactions;
2789
- }
2790
- async nextDataHeight(afterHeight, untilHeight) {
2791
- if (!this.probeTargets?.length)
2792
- return afterHeight + 1;
2793
- const hits = await Promise.all(this.probeTargets.map((t) => this.http.firstEventHeight(t.eventType, afterHeight + 1, untilHeight, t.contractId)));
2794
- const found = hits.filter((h) => h !== null);
2795
- return found.length ? Math.min(...found) : null;
2796
- }
2797
- getTip() {
2798
- if (this.eventTypes.some((t) => VM_INDEX_EVENT_TYPES.has(t))) {
2799
- return this.http.getIndexSourceTip();
2800
- }
2801
- return this.http.getIndexTip();
2802
- }
2803
- async loadBlockRange(fromHeight, toHeight) {
2804
- const withTx = !this.needsTransactions;
2805
- const [blocks, txRows, eventLists] = await Promise.all([
2806
- this.http.walkBlocks(fromHeight, toHeight),
2807
- this.needsTransactions ? this.http.walkTransactions(fromHeight, toHeight) : Promise.resolve([]),
2808
- Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight, withTx)))
2809
- ]);
2810
- const map = new Map;
2811
- for (const b of blocks) {
2812
- map.set(b.block_height, {
2813
- block: reconstructBlock(b),
2814
- txs: [],
2815
- events: []
2816
- });
2817
- }
2818
- const txs = this.needsTransactions ? txRows.map(reconstructTransaction) : synthesizeTxsFromEvents(eventLists.flat());
2819
- for (const t of txs) {
2820
- map.get(t.block_height)?.txs.push(t);
2821
- }
2822
- for (const list of eventLists) {
2823
- for (const e of list) {
2824
- const bd = map.get(e.block_height);
2825
- if (!bd)
2826
- continue;
2827
- if (VM_INDEX_EVENT_TYPES.has(e.event_type)) {
2828
- bd.vmEvents ??= [];
2829
- bd.vmEvents.push(reconstructEvent(e));
2830
- } else {
2831
- bd.events.push(reconstructEvent(e));
2832
- }
2833
- }
2834
- }
2835
- for (const bd of map.values()) {
2836
- bd.txs.sort((a, b) => a.tx_index - b.tx_index);
2837
- bd.events.sort((a, b) => a.event_index - b.event_index);
2838
- bd.vmEvents?.sort((a, b) => a.event_index - b.event_index);
2839
- }
2840
- return map;
2841
- }
2842
- }
2843
-
2844
- class FallbackBlockSource {
2845
- primary;
2846
- fallback;
2847
- constructor(primary, fallback) {
2848
- this.primary = primary;
2849
- this.fallback = fallback;
2850
- }
2851
- async getTip() {
2852
- try {
2853
- return await this.primary.getTip();
2854
- } catch (err) {
2855
- logger5.warn("block source primary getTip failed — using DB tap", {
2856
- error: err instanceof Error ? err.message : String(err)
2857
- });
2858
- return this.fallback.getTip();
2859
- }
2860
- }
2861
- async loadBlockRange(fromHeight, toHeight) {
2862
- try {
2863
- return await this.primary.loadBlockRange(fromHeight, toHeight);
2864
- } catch (err) {
2865
- logger5.warn("block source primary loadBlockRange failed — using DB tap", {
2866
- from: fromHeight,
2867
- to: toHeight,
2868
- error: err instanceof Error ? err.message : String(err)
2869
- });
2870
- return this.fallback.loadBlockRange(fromHeight, toHeight);
2871
- }
2872
- }
2873
- async nextDataHeight(afterHeight, untilHeight) {
2874
- if (!this.primary.nextDataHeight)
2875
- return afterHeight + 1;
2876
- try {
2877
- return await this.primary.nextDataHeight(afterHeight, untilHeight);
2878
- } catch {
2879
- return afterHeight + 1;
2880
- }
2881
- }
2882
- }
2883
- var postgresBlockSource = new PostgresBlockSource;
2884
- function buildHttpClient() {
2885
- return createInternalIndexHttpClient();
2886
- }
2887
- function resolveBlockSource(subgraph) {
2888
- if (process.env.SUBGRAPH_SOURCE === "observer-http") {
2889
- const baseUrl = process.env.OBSERVER_HTTP_URL;
2890
- if (!baseUrl) {
2891
- throw new Error("SUBGRAPH_SOURCE=observer-http requires OBSERVER_HTTP_URL");
2892
- }
2893
- return new ObserverHttpBlockSource({
2894
- baseUrl,
2895
- token: process.env.OBSERVER_HTTP_EXPORT_TOKEN || null
2896
- });
2897
- }
2898
- if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
2899
- return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph), canSparseScan(subgraph) ? sparseProbeTargets(subgraph) : undefined, needsTransactionData(subgraph)), postgresBlockSource);
2900
- }
2901
- if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
2902
- logger5.debug("Subgraph not streams-index eligible, using DB tap", {
2903
- subgraph: subgraph.name
2904
- });
2905
- }
2906
- return postgresBlockSource;
2907
- }
2908
-
2909
- // src/runtime/outbox-emit.ts
2910
- import { createHash as createHash2 } from "node:crypto";
2911
- import { logger as logger6 } from "@secondlayer/shared/logger";
2912
- var loggedKillSwitch = false;
2913
- var OP_VERB = {
2914
- insert: "created",
2915
- update: "updated",
2916
- delete: "deleted"
2917
- };
2918
- function isEmitOutboxEnabled() {
2919
- return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
2920
- }
2921
- function dedupKey(subgraphName, tableName, blockHeight, txId, rowIndex, row) {
2922
- const canonical = `${subgraphName}:${tableName}:${blockHeight}:${txId}:${rowIndex}:${stableStringify(row)}`;
2923
- return createHash2("sha256").update(canonical).digest("hex").slice(0, 32);
2924
- }
2925
- function stableStringify(obj) {
2926
- const keys = Object.keys(obj).sort();
2927
- return JSON.stringify(keys.reduce((acc, k) => {
2928
- acc[k] = obj[k];
2929
- return acc;
2930
- }, {}));
2931
- }
2932
- async function emitWebhookOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
2933
- if (!isEmitOutboxEnabled()) {
2934
- if (!loggedKillSwitch) {
2935
- logger6.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
2936
- loggedKillSwitch = true;
2937
- }
2938
- return 0;
2939
- }
2940
- loggedKillSwitch = false;
2941
- if (manifest.count === 0 || matcher.size() === 0)
2942
- return 0;
2943
- const rows = [];
2944
- for (const write of manifest.writes) {
2945
- const subs = matcher.match(subgraphName, write.table, write.row);
2946
- if (subs.length === 0)
2947
- continue;
2948
- const eventType = `${subgraphName}.${write.table}.${OP_VERB[write.op]}`;
2949
- for (const s of subs) {
2950
- rows.push({
2951
- webhook_id: s.id,
2952
- subgraph_name: subgraphName,
2953
- table_name: write.table,
2954
- block_height: blockHeight,
2955
- tx_id: write.pk.txId ?? null,
2956
- row_pk: write.pk,
2957
- event_type: eventType,
2958
- payload: write.row,
2959
- dedup_key: dedupKey(subgraphName, write.table, write.pk.blockHeight, write.pk.txId, write.pk.rowIndex, write.row)
2960
- });
2961
- }
2962
- }
2963
- if (rows.length === 0)
2964
- return 0;
2965
- await tx.insertInto("webhook_outbox").values(rows).onConflict((oc) => oc.columns(["webhook_id", "dedup_key"]).doNothing()).execute();
2966
- return rows.length;
2967
- }
2968
-
2969
- // src/runtime/sandbox/flag.ts
2970
- function sandboxWorkersGloballyEnabled() {
2971
- return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
2972
- }
2973
- function sandboxEnabled(subgraph) {
2974
- return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
2975
- }
2976
-
2977
- // src/runtime/sandbox/host.ts
2978
- import { tmpdir } from "node:os";
2979
- import { fileURLToPath } from "node:url";
2980
- import { logger as logger7 } from "@secondlayer/shared/logger";
2981
-
2982
- // src/runtime/sandbox/bundle.ts
2983
- import esbuild from "esbuild";
2984
- var ALLOWED_BARE_SPECIFIERS = new Set;
2985
- var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
2986
- var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
2987
- `;
2988
- function resolverLockdownPlugin() {
2989
- return {
2990
- name: "f071-sandbox-resolver-lockdown",
2991
- setup(build) {
2992
- build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
2993
- path: args.path,
2994
- namespace: "f071-sandbox-subgraphs-stub"
2995
- }));
2996
- build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
2997
- build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
2998
- path: args.path,
2999
- namespace: "f071-sandbox-blocked"
3000
- }));
3001
- build.onResolve({ filter: /^[^./]/ }, (args) => {
3002
- if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
3003
- return;
3004
- if (ALLOWED_BARE_SPECIFIERS.has(args.path))
3005
- return;
3006
- return { path: args.path, namespace: "f071-sandbox-blocked" };
3007
- });
3008
- build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
3009
- contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
3010
- loader: "js"
3011
- }));
3012
- }
3013
- };
3014
- }
3015
- async function bundleHandlerCode(handlerCode) {
3016
- const result = await esbuild.build({
3017
- stdin: {
3018
- contents: handlerCode,
3019
- loader: "ts",
3020
- resolveDir: process.cwd()
3021
- },
3022
- bundle: true,
3023
- platform: "neutral",
3024
- format: "esm",
3025
- write: false,
3026
- logLevel: "silent",
3027
- plugins: [resolverLockdownPlugin()]
3028
- });
3029
- const out = result.outputFiles?.[0];
3030
- if (!out)
3031
- throw new Error("esbuild produced no output for handler_code bundle");
3032
- return new TextDecoder().decode(out.contents);
3033
- }
3034
-
3035
- // src/runtime/sandbox/host.ts
3036
- var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
3037
- var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
3038
- function spawnSandboxProcess() {
3039
- let onMessage = null;
3040
- const proc = Bun.spawn({
3041
- cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
3042
- env: { PATH: process.env.PATH ?? "" },
3043
- cwd: tmpdir(),
3044
- stdio: ["ignore", "ignore", "pipe"],
3045
- serialization: "advanced",
3046
- ipc(message) {
3047
- onMessage?.(message);
3048
- }
3049
- });
3050
- (async () => {
3051
- const reader = proc.stderr.getReader();
3052
- const decoder = new TextDecoder;
3053
- try {
3054
- while (true) {
3055
- const { done, value } = await reader.read();
3056
- if (done)
3057
- break;
3058
- const text = decoder.decode(value, { stream: true });
3059
- if (text.trim()) {
3060
- logger7.warn("Sandbox subprocess stderr", { stderr: text.trim() });
3061
- }
3062
- }
3063
- } catch {}
3064
- })();
3065
- return {
3066
- send: (msg) => proc.send(msg),
3067
- kill: () => proc.kill(),
3068
- exited: proc.exited,
3069
- setOnMessage: (handler) => {
3070
- onMessage = handler;
3071
- }
3072
- };
3073
- }
3074
- var pool = new Map;
3075
-
3076
- class SandboxWorkerCrash extends Error {
3077
- constructor(subgraphName, cause) {
3078
- super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
3079
- this.name = "SandboxWorkerCrash";
3080
- }
3081
- }
3082
- async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
3083
- const bundledCode = await bundleHandlerCode(handlerCode);
3084
- let settled = false;
3085
- const ready = new Promise((resolve, reject) => {
3086
- proc.setOnMessage((msg) => {
3087
- if (msg.type === "ready" && msg.version === version) {
3088
- settled = true;
3089
- proc.setOnMessage(null);
3090
- resolve();
3091
- } else if (msg.type === "error") {
3092
- settled = true;
3093
- proc.setOnMessage(null);
3094
- reject(new SandboxWorkerCrash(subgraphName, msg.message));
3095
- }
3096
- });
3097
- const init = { type: "init", bundledCode, version };
3098
- proc.send(init);
3099
- });
3100
- const crashed = proc.exited.then((code) => {
3101
- if (settled)
3102
- return;
3103
- throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
3104
- });
3105
- await Promise.race([ready, crashed]);
3106
- }
3107
- async function ensureSandboxProcess(subgraphName, version, handlerCode) {
3108
- if (handlerCode == null) {
3109
- throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
3110
- }
3111
- const existing = pool.get(subgraphName);
3112
- if (existing && existing.version === version)
3113
- return existing;
3114
- if (existing) {
3115
- try {
3116
- await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
3117
- existing.version = version;
3118
- logger7.info("Sandbox subprocess handler reloaded", {
3119
- subgraph: subgraphName,
3120
- version
3121
- });
3122
- return existing;
3123
- } catch (err) {
3124
- existing.proc.kill();
3125
- pool.delete(subgraphName);
3126
- logger7.warn("Sandbox subprocess re-init failed; respawning", {
3127
- subgraph: subgraphName,
3128
- error: err instanceof Error ? err.message : String(err)
3129
- });
3130
- }
3131
- }
3132
- const proc = spawnSandboxProcess();
3133
- try {
3134
- await initSandboxProcess(proc, subgraphName, version, handlerCode);
3135
- } catch (err) {
3136
- proc.kill();
3137
- throw err;
3138
- }
3139
- const entry = { proc, version, chain: Promise.resolve() };
3140
- pool.set(subgraphName, entry);
3141
- return entry;
3142
- }
3143
- function evictSandboxWorker(subgraphName) {
3144
- const entry = pool.get(subgraphName);
3145
- if (!entry)
3146
- return;
3147
- entry.proc.kill();
3148
- pool.delete(subgraphName);
3149
- }
3150
- async function runHandlersSandboxed(params) {
3151
- const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
3152
- const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
3153
- const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
3154
- entry.chain = run.catch(() => {
3155
- return;
3156
- });
3157
- return run;
3158
- }
3159
- async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
3160
- const { proc } = entry;
3161
- let timeoutHandle;
3162
- let settled = false;
3163
- try {
3164
- const done = new Promise((resolve, reject) => {
3165
- proc.setOnMessage((msg) => {
3166
- if (msg.type === "readRequest") {
3167
- answerRead(hostCtx, msg).then((reply) => {
3168
- proc.send({ type: "readResponse", id: msg.id, reply });
3169
- }).catch((err) => {
3170
- settled = true;
3171
- reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
3172
- });
3173
- return;
3174
- }
3175
- if (msg.type === "blockDone") {
3176
- settled = true;
3177
- resolve({
3178
- ops: msg.ops,
3179
- result: { processed: msg.processed, errors: msg.errors }
3180
- });
3181
- return;
3182
- }
3183
- if (msg.type === "error") {
3184
- settled = true;
3185
- reject(new SandboxWorkerCrash(subgraphName, msg.message));
3186
- }
3187
- });
3188
- proc.send({ type: "runBlock", block, matched });
3189
- });
3190
- const crashed = proc.exited.then((code) => {
3191
- if (settled)
3192
- return;
3193
- throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
3194
- });
3195
- const deadline = new Promise((_, reject) => {
3196
- timeoutHandle = setTimeout(() => {
3197
- settled = true;
3198
- reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
3199
- }, BLOCK_TIMEOUT_MS);
3200
- });
3201
- const { ops, result } = await Promise.race([done, crashed, deadline]);
3202
- for (const op of ops) {
3203
- hostCtx.setTx(op.tx);
3204
- replayOp(hostCtx, op);
3205
- }
3206
- return result;
3207
- } catch (err) {
3208
- evictSandboxWorker(subgraphName);
3209
- logger7.warn("Sandbox subprocess run failed; subprocess evicted", {
3210
- subgraph: subgraphName,
3211
- blockHeight: block.height,
3212
- error: err instanceof Error ? err.message : String(err)
3213
- });
3214
- throw err;
3215
- } finally {
3216
- if (timeoutHandle !== undefined)
3217
- clearTimeout(timeoutHandle);
3218
- proc.setOnMessage(null);
3219
- }
3220
- }
3221
- function replayOp(hostCtx, op) {
3222
- switch (op.method) {
3223
- case "insert":
3224
- hostCtx.insert(op.table, op.args[0]);
3225
- return;
3226
- case "upsert":
3227
- hostCtx.upsert(op.table, op.args[0], op.args[1]);
3228
- return;
3229
- case "increment":
3230
- hostCtx.increment(op.table, op.args[0], op.args[1]);
3231
- return;
3232
- case "update":
3233
- hostCtx.update(op.table, op.args[0], op.args[1]);
3234
- return;
3235
- case "delete":
3236
- hostCtx.delete(op.table, op.args[0]);
3237
- return;
3238
- }
3239
- }
3240
- async function answerRead(hostCtx, msg) {
3241
- const { method, table, where, column } = msg;
3242
- switch (method) {
3243
- case "findOne":
3244
- return { kind: "row", row: await hostCtx.findOne(table, where) };
3245
- case "findMany":
3246
- return { kind: "rows", rows: await hostCtx.findMany(table, where) };
3247
- case "count":
3248
- return { kind: "count", count: await hostCtx.count(table, where) };
3249
- case "countDistinct": {
3250
- if (!column)
3251
- throw new Error("countDistinct read missing column");
3252
- return {
3253
- kind: "count",
3254
- count: await hostCtx.countDistinct(table, column, where)
3255
- };
3256
- }
3257
- case "sum": {
3258
- if (!column)
3259
- throw new Error("sum read missing column");
3260
- const v = await hostCtx.sum(table, column, where);
3261
- return { kind: "amount", amount: v.toString() };
3262
- }
3263
- case "min": {
3264
- if (!column)
3265
- throw new Error("min read missing column");
3266
- const v = await hostCtx.min(table, column, where);
3267
- return { kind: "amount", amount: v == null ? null : v.toString() };
3268
- }
3269
- case "max": {
3270
- if (!column)
3271
- throw new Error("max read missing column");
3272
- const v = await hostCtx.max(table, column, where);
3273
- return { kind: "amount", amount: v == null ? null : v.toString() };
3274
- }
3275
- }
3276
- }
3277
-
3278
- // src/runtime/webhook-state.ts
3279
- import { listWebhooks } from "@secondlayer/shared/db/queries/webhooks";
3280
- import { sql as sql3 } from "kysely";
3281
-
3282
- // src/runtime/emitter-matcher.ts
3283
- function isPrimitive(v) {
3284
- const t = typeof v;
3285
- return t === "string" || t === "number" || t === "boolean";
3286
- }
3287
- function coerceBigInt(v) {
3288
- if (typeof v === "bigint")
3289
- return v;
3290
- if (typeof v === "number") {
3291
- if (!Number.isFinite(v))
3292
- return null;
3293
- if (!Number.isInteger(v))
3294
- return null;
3295
- return BigInt(v);
3296
- }
3297
- if (typeof v === "string" && /^-?\d+$/.test(v)) {
3298
- try {
3299
- return BigInt(v);
3300
- } catch {
3301
- return null;
3302
- }
3303
- }
3304
- return null;
3305
- }
3306
- function coerceFloat(v) {
3307
- if (typeof v === "number" && Number.isFinite(v))
3308
- return v;
3309
- if (typeof v === "bigint")
3310
- return Number(v);
3311
- if (typeof v === "string" && v !== "" && !Number.isNaN(Number(v))) {
3312
- return Number(v);
3313
- }
3314
- return null;
3315
- }
3316
- function compareNumeric(a, b) {
3317
- const ba = coerceBigInt(a);
3318
- const bb = coerceBigInt(b);
3319
- if (ba !== null && bb !== null) {
3320
- if (ba === bb)
3321
- return 0;
3322
- return ba > bb ? 1 : -1;
3323
- }
3324
- const fa = coerceFloat(a);
3325
- const fb = coerceFloat(b);
3326
- if (fa === null || fb === null)
3327
- return null;
3328
- if (fa === fb)
3329
- return 0;
3330
- return fa > fb ? 1 : -1;
3331
- }
3332
- function matchClause(rowValue, clause) {
3333
- const rowIsPrimitive = isPrimitive(rowValue) || typeof rowValue === "bigint";
3334
- if (isPrimitive(clause)) {
3335
- if (!rowIsPrimitive)
3336
- return false;
3337
- if (typeof clause === "number" || typeof rowValue === "number" || typeof rowValue === "bigint") {
3338
- const cmp = compareNumeric(rowValue, clause);
3339
- if (cmp !== null)
3340
- return cmp === 0;
3341
- }
3342
- return rowValue === clause || String(rowValue) === String(clause);
3343
- }
3344
- if (clause === null || typeof clause !== "object" || Array.isArray(clause)) {
3345
- return false;
3346
- }
3347
- const keys = Object.keys(clause);
3348
- if (keys.length !== 1)
3349
- return false;
3350
- const op = keys[0];
3351
- const c = clause;
3352
- switch (op) {
3353
- case "eq":
3354
- return matchClause(rowValue, c.eq);
3355
- case "neq":
3356
- return !matchClause(rowValue, c.neq);
3357
- case "gt":
3358
- case "gte":
3359
- case "lt":
3360
- case "lte": {
3361
- if (!rowIsPrimitive)
3362
- return false;
3363
- const cmp = compareNumeric(rowValue, c[op]);
3364
- if (cmp === null)
3365
- return false;
3366
- if (op === "gt")
3367
- return cmp > 0;
3368
- if (op === "gte")
3369
- return cmp >= 0;
3370
- if (op === "lt")
3371
- return cmp < 0;
3372
- return cmp <= 0;
3373
- }
3374
- case "in": {
3375
- const list = c.in;
3376
- if (!Array.isArray(list))
3377
- return false;
3378
- if (!rowIsPrimitive)
3379
- return false;
3380
- return list.some((item) => isPrimitive(item) && matchClause(rowValue, item));
3381
- }
3382
- default:
3383
- return false;
3384
- }
3385
- }
3386
- function matchesFilter(filter, row) {
3387
- if (!filter || Object.keys(filter).length === 0)
3388
- return true;
3389
- for (const [col, clause] of Object.entries(filter)) {
3390
- if (!matchClause(row[col], clause))
3391
- return false;
3392
- }
3393
- return true;
3394
- }
3395
- function key(subgraphName, tableName) {
3396
- return `${subgraphName}\x00${tableName}`;
3397
- }
3398
-
3399
- class WebhookMatcher {
3400
- byKey = new Map;
3401
- byId = new Map;
3402
- setAll(subs) {
3403
- this.byKey.clear();
3404
- this.byId.clear();
3405
- for (const sub of subs) {
3406
- if (sub.status !== "active")
3407
- continue;
3408
- if (sub.kind !== "subgraph" || !sub.subgraph_name || !sub.table_name)
3409
- continue;
3410
- this.byId.set(sub.id, sub);
3411
- const k = key(sub.subgraph_name, sub.table_name);
3412
- const arr = this.byKey.get(k);
3413
- if (arr)
3414
- arr.push(sub);
3415
- else
3416
- this.byKey.set(k, [sub]);
3417
- }
3418
- }
3419
- match(subgraphName, tableName, row) {
3420
- const bucket = this.byKey.get(key(subgraphName, tableName));
3421
- if (!bucket)
3422
- return [];
3423
- const hits = [];
3424
- for (const sub of bucket) {
3425
- if (matchesFilter(sub.filter, row))
3426
- hits.push(sub);
3427
- }
3428
- return hits;
3429
- }
3430
- has(subgraphName, tableName) {
3431
- return this.byKey.has(key(subgraphName, tableName));
3432
- }
3433
- size() {
3434
- return this.byId.size;
3435
- }
3436
- get(id) {
3437
- return this.byId.get(id);
3438
- }
3439
- }
3440
-
3441
- // src/runtime/webhook-state.ts
3442
- var matcher = new WebhookMatcher;
3443
- async function refreshMatcher(db) {
3444
- const rows = await sql3`
3445
- SELECT * FROM webhooks WHERE status = 'active'
3446
- `.execute(db);
3447
- matcher.setAll(rows.rows);
3448
- return matcher.size();
3449
- }
3450
-
3451
- // src/runtime/block-processor.ts
3452
- var routeCache = new Map;
3453
- async function resolveRoute(subgraphName, targetDb) {
3454
- const cached = routeCache.get(subgraphName);
3455
- if (cached)
3456
- return cached;
3457
- const row = await targetDb.selectFrom("subgraphs").selectAll().where("name", "=", subgraphName).executeTakeFirst();
3458
- const route = {
3459
- schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
3460
- dataDb: targetDb,
3461
- sandboxWorkers: row?.sandbox_workers === true,
3462
- handlerCode: row?.handler_code ?? null,
3463
- version: row?.version ?? ""
3464
- };
3465
- routeCache.set(subgraphName, route);
3466
- logger8.info("subgraph handler dispatch path resolved", {
3467
- subgraph: subgraphName,
3468
- path: sandboxEnabled({ sandbox_workers: route.sandboxWorkers }) ? "sandbox" : "in-process"
3469
- });
3470
- return route;
3471
- }
3472
- function invalidateSubgraphRoute(subgraphName) {
3473
- routeCache.delete(subgraphName);
3474
- evictSandboxWorker(subgraphName);
3475
- }
3476
- async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts, vmEvts = []) {
3477
- const factories = new Map;
3478
- for (const source of Object.values(subgraph.sources)) {
3479
- const factory = source.factory;
3480
- if (factory)
3481
- factories.set(factory.from, factory);
3482
- }
3483
- const resolved = new Map;
3484
- const discovered = [];
3485
- if (factories.size === 0)
3486
- return { resolved, discovered };
3487
- for (const [discoveringSource, factory] of factories) {
3488
- const known = new Set;
3489
- try {
3490
- const rows = await sql4`
3491
- SELECT address FROM ${sql4.raw(`"${schemaName}"."_factory_addresses"`)}
3492
- WHERE source_name = ${discoveringSource} AND block_height <= ${blockHeight}
3493
- `.execute(db);
3494
- for (const row of rows.rows)
3495
- known.add(row.address);
3496
- } catch {}
3497
- const discovering = subgraph.sources[factory.from];
3498
- if (discovering) {
3499
- const matches = matchSources({ [factory.from]: discovering }, txs, evts, new Map, new Map, vmEvts);
3500
- for (const match of matches) {
3501
- for (const event of match.events ?? []) {
3502
- const payload = buildEventPayload(discovering, match.tx, event);
3503
- const value = readPath(payload, factory.field);
3504
- if (typeof value === "string" && value.length > 0 && !known.has(value)) {
3505
- known.add(value);
3506
- discovered.push({ sourceName: discoveringSource, address: value });
3507
- }
3508
- }
3509
- }
3510
- }
3511
- resolved.set(discoveringSource, known);
3512
- }
3513
- return { resolved, discovered };
3514
- }
3515
- async function persistFactoryDiscoveries(schemaName, db, blockHeight, discovered) {
3516
- if (discovered.length === 0)
3517
- return;
3518
- for (const { sourceName, address } of discovered) {
3519
- await sql4`
3520
- INSERT INTO ${sql4.raw(`"${schemaName}"."_factory_addresses"`)}
3521
- (source_name, address, block_height)
3522
- VALUES (${sourceName}, ${address}, ${blockHeight})
3523
- ON CONFLICT (source_name, address) DO NOTHING
3524
- `.execute(db);
3525
- }
3526
- }
3527
- async function resolveTraitContracts(subgraph, blockHeight, db) {
3528
- const traits = new Set;
3529
- for (const source of Object.values(subgraph.sources)) {
3530
- const trait = source.trait;
3531
- if (trait)
3532
- traits.add(trait);
3533
- }
3534
- const resolved = new Map;
3535
- for (const trait of traits) {
3536
- const ids = await resolveTraitContractIds(db, trait, blockHeight);
3537
- resolved.set(trait, new Set(ids));
3538
- }
3539
- return resolved;
3540
- }
3541
-
3542
- class CursorRaceLostError extends Error {
3543
- constructor(operationId, height) {
3544
- super(`op ${operationId} lost cursor race at block ${height}`);
3545
- this.name = "CursorRaceLostError";
3546
- }
3547
- }
3548
-
3549
- class LiveCursorRaceLostError extends Error {
3550
- constructor(subgraphName, height) {
3551
- super(`live cursor race lost for ${subgraphName} at block ${height}`);
3552
- this.name = "LiveCursorRaceLostError";
3553
- }
3554
- }
3555
- function opCursorMode(opts) {
3556
- const ap = opts?.atomicProgress;
3557
- return ap && "operationId" in ap ? ap : undefined;
3558
- }
3559
- function isReorgRewind(opts) {
3560
- return opts?.reorgRewind === true;
3561
- }
3562
- function statusMode(opts) {
3563
- const ap = opts?.atomicProgress;
3564
- return ap && "status" in ap ? ap : undefined;
3565
- }
3566
- var BLOCK_RETRY_DELAYS_MS = [500, 2000, 5000];
3567
- function journalEnabled(opts) {
3568
- return !opts?.skipProgressUpdate;
3569
- }
3570
- async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts, retryDelaysMs = BLOCK_RETRY_DELAYS_MS) {
3571
- let lastError;
3572
- for (let attempt = 0;attempt <= retryDelaysMs.length; attempt++) {
3573
- try {
3574
- return await processBlock(subgraph, subgraphName, blockHeight, opts);
3575
- } catch (err) {
3576
- lastError = err;
3577
- const delay = retryDelaysMs[attempt];
3578
- if (delay === undefined)
3579
- break;
3580
- logger8.warn("Block processing failed, retrying", {
3581
- subgraph: subgraphName,
3582
- blockHeight,
3583
- attempt: attempt + 1,
3584
- retryInMs: delay,
3585
- error: err instanceof Error ? err.message : String(err)
3586
- });
3587
- await new Promise((r) => setTimeout(r, delay));
3588
- }
3589
- }
3590
- throw lastError;
3591
- }
3592
- async function processBlock(subgraph, subgraphName, blockHeight, opts) {
3593
- const targetDb = getTargetDb2();
3594
- const blockStart = performance.now();
3595
- const result = {
3596
- blockHeight,
3597
- matched: 0,
3598
- processed: 0,
3599
- errors: 0,
3600
- skipped: false
3601
- };
3602
- let block;
3603
- let txs;
3604
- let evts;
3605
- let vmEvts;
3606
- if (opts?.preloaded) {
3607
- block = opts.preloaded.block;
3608
- txs = opts.preloaded.txs;
3609
- evts = opts.preloaded.events;
3610
- vmEvts = opts.preloaded.vmEvents ?? [];
3611
- } else {
3612
- const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
3613
- if (!data) {
3614
- logger8.debug("Block not found or non-canonical for subgraph processing", {
3615
- subgraph: subgraphName,
3616
- blockHeight
3617
- });
3618
- result.skipped = true;
3619
- return result;
3620
- }
3621
- block = data.block;
3622
- txs = data.txs;
3623
- evts = data.events;
3624
- vmEvts = data.vmEvents ?? [];
3625
- }
3626
- const route = await resolveRoute(subgraphName, targetDb);
3627
- const schemaName = route.schemaName;
3628
- const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
3629
- const { resolved: factoryContracts, discovered } = await resolveFactoryContracts(subgraph, blockHeight, schemaName, targetDb, txs, evts, vmEvts);
3630
- const matched = matchSources(subgraph.sources, txs, evts, traitContracts, factoryContracts, vmEvts);
3631
- result.matched = matched.length;
3632
- if (matched.length === 0) {
3633
- if (!opts?.skipProgressUpdate) {
3634
- if (isReorgRewind(opts)) {
3635
- await rewindLiveProgress(targetDb, subgraphName, blockHeight);
3636
- } else {
3637
- await recordLiveProgress(targetDb, subgraphName, blockHeight);
3638
- }
3639
- }
3640
- return result;
3641
- }
3642
- const blockMeta = {
3643
- height: block.height,
3644
- hash: block.hash,
3645
- timestamp: block.timestamp,
3646
- burnBlockHeight: block.burn_block_height,
3647
- indexBlockHash: block.index_block_hash
3648
- };
3649
- const initialTx = {
3650
- txId: "",
3651
- sender: "",
3652
- type: "",
3653
- status: ""
3654
- };
3655
- let handlerMs = 0;
3656
- let flushMs = 0;
3657
- const applyProgress = async (tx, rr, flushedWrites) => {
3658
- if (opts?.skipProgressUpdate)
3659
- return;
3660
- const rewind = isReorgRewind(opts);
3661
- if (rr.errors > 0 && rr.processed === 0) {
3662
- if (rewind) {
3663
- await updateSubgraphStatus(tx, subgraphName, "error", blockHeight);
3664
- } else {
3665
- await recordLiveError(tx, subgraphName, blockHeight);
3666
- }
3667
- } else if (rewind) {
3668
- await rewindLiveProgress(tx, subgraphName, blockHeight);
3669
- } else {
3670
- const advanced = await recordLiveProgress(tx, subgraphName, blockHeight);
3671
- if (!advanced && flushedWrites) {
3672
- throw new LiveCursorRaceLostError(subgraphName, blockHeight);
3673
- }
3674
- }
3675
- if (rr.errors > 0 && !rewind) {
3676
- if (opts?.subgraphId) {
3677
- await recordGapBatch(tx, opts.subgraphId, subgraphName, [
3678
- { start: blockHeight, end: blockHeight, reason: "processing_error" }
3679
- ]);
3680
- } else {
3681
- logger8.warn("Block errors not recorded as a gap — no subgraphId", {
3682
- subgraph: subgraphName,
3683
- blockHeight,
3684
- errors: rr.errors
3685
- });
3686
- }
3687
- }
3688
- if (rr.processed > 0 || rr.errors > 0) {
3689
- const lastError = rr.errors > 0 ? `${rr.errors} error(s) at block ${blockHeight}` : undefined;
3690
- await recordSubgraphProcessed(tx, subgraphName, rr.processed, rr.errors, lastError);
3691
- }
3692
- };
3693
- try {
3694
- await targetDb.transaction().execute(async (tx) => {
3695
- const opMode = opCursorMode(opts);
3696
- const sm = statusMode(opts);
3697
- const rewind = isReorgRewind(opts);
3698
- if (sm) {
3699
- const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
3700
- if (row && Number(row.last_processed_block) >= blockHeight) {
3701
- result.skipped = true;
3702
- return;
3703
- }
3704
- } else if (opMode) {
3705
- const row = await tx.selectFrom("subgraph_operations").select("cursor_block").where("id", "=", opMode.operationId).executeTakeFirst();
3706
- if (row?.cursor_block != null && Number(row.cursor_block) >= blockHeight) {
3707
- result.skipped = true;
3708
- return;
3709
- }
3710
- } else if (!rewind) {
3711
- const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
3712
- if (row && Number(row.last_processed_block) >= blockHeight) {
3713
- result.skipped = true;
3714
- return;
3715
- }
3716
- }
3717
- const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, journalEnabled(opts));
3718
- const handlerStart = performance.now();
3719
- const runResult = sandboxEnabled({
3720
- sandbox_workers: route.sandboxWorkers
3721
- }) ? await runHandlersSandboxed({
3722
- subgraphName,
3723
- version: route.version,
3724
- handlerCode: route.handlerCode,
3725
- hostCtx: ctx,
3726
- block: blockMeta,
3727
- matched
3728
- }) : await runHandlers(subgraph, matched, ctx);
3729
- handlerMs = performance.now() - handlerStart;
3730
- result.processed = runResult.processed;
3731
- result.errors = runResult.errors;
3732
- let flushedWrites = false;
3733
- if (ctx.pendingOps > 0) {
3734
- const flushStart = performance.now();
3735
- const manifest = await ctx.flush();
3736
- flushedWrites = manifest.count > 0;
3737
- if (manifest.count > 0) {
3738
- await emitWebhookOutbox(tx, subgraphName, manifest, matcher, block.height);
3739
- }
3740
- flushMs = performance.now() - flushStart;
3741
- }
3742
- if (sm && flushedWrites) {
3743
- await updateSubgraphStatus(tx, subgraphName, sm.status, blockHeight);
3744
- } else if (opMode && flushedWrites) {
3745
- const advanced = await advanceOperationCursor(tx, opMode.operationId, blockHeight);
3746
- if (!advanced) {
3747
- throw new CursorRaceLostError(opMode.operationId, blockHeight);
3748
- }
3749
- }
3750
- await applyProgress(tx, runResult, flushedWrites);
3751
- });
3752
- } catch (err) {
3753
- if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
3754
- logger8.warn("cursor race lost — block already covered", {
3755
- subgraph: subgraphName,
3756
- blockHeight,
3757
- error: err.message
3758
- });
3759
- result.skipped = true;
3760
- return result;
3761
- }
3762
- throw err;
3763
- }
3764
- if (!result.skipped) {
3765
- await persistFactoryDiscoveries(schemaName, route.dataDb, blockHeight, discovered);
3766
- }
3767
- const totalMs = performance.now() - blockStart;
3768
- if (!result.skipped) {
3769
- result.timing = {
3770
- totalMs: Math.round(totalMs),
3771
- handlerMs: Math.round(handlerMs),
3772
- flushMs: Math.round(flushMs)
3773
- };
3774
- }
3775
- if (blockHeight % 1000 === 0) {
3776
- try {
3777
- const tables = Object.keys(subgraph.schema);
3778
- for (const table of tables) {
3779
- const { rows } = await sql4.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
3780
- const count = Number(rows[0]?.count ?? 0);
3781
- if (count >= 1e7) {
3782
- logger8.warn("Subgraph table exceeds 10M rows (estimate)", {
3783
- subgraph: subgraphName,
3784
- table,
3785
- count
3786
- });
3787
- }
3788
- }
3789
- } catch (err) {
3790
- logger8.debug("Row count sample failed", {
3791
- subgraph: subgraphName,
3792
- error: err instanceof Error ? err.message : String(err)
3793
- });
3794
- }
3795
- if (journalEnabled(opts)) {
3796
- await sql4.raw(`DELETE FROM "${schemaName}"."_journal" WHERE "block_height" < ${blockHeight - JOURNAL_RETENTION_BLOCKS}`).execute(route.dataDb).catch(() => {});
3797
- }
3798
- }
3799
- return result;
3800
- }
3801
-
3802
- // src/runtime/stats.ts
3803
- var FLUSH_INTERVAL_BLOCKS = 100;
3804
- var FLUSH_INTERVAL_MS = 60000;
3805
-
3806
- class StatsAccumulator {
3807
- subgraphName;
3808
- isCatchup;
3809
- current;
3810
- lastFlush = Date.now();
3811
- constructor(subgraphName, isCatchup) {
3812
- this.subgraphName = subgraphName;
3813
- this.isCatchup = isCatchup;
3814
- this.current = this.newEntry();
3815
- }
3816
- record(timing, opsCount) {
3817
- this.current.blocksProcessed++;
3818
- this.current.totalTimeMs += timing.totalMs;
3819
- this.current.handlerTimeMs += timing.handlerMs;
3820
- this.current.flushTimeMs += timing.flushMs;
3821
- this.current.maxBlockTimeMs = Math.max(this.current.maxBlockTimeMs, timing.totalMs);
3822
- this.current.maxHandlerTimeMs = Math.max(this.current.maxHandlerTimeMs, timing.handlerMs);
3823
- this.current.totalOps += opsCount;
3824
- }
3825
- shouldFlush() {
3826
- return this.current.blocksProcessed >= FLUSH_INTERVAL_BLOCKS || Date.now() - this.lastFlush >= FLUSH_INTERVAL_MS;
3827
- }
3828
- async flush(db) {
3829
- if (this.current.blocksProcessed === 0)
3830
- return;
3831
- const entry = this.current;
3832
- this.current = this.newEntry();
3833
- this.lastFlush = Date.now();
3834
- const avgOpsPerBlock = entry.blocksProcessed > 0 ? entry.totalOps / entry.blocksProcessed : 0;
3835
- await db.insertInto("subgraph_processing_stats").values({
3836
- subgraph_name: entry.subgraphName,
3837
- bucket_start: entry.bucketStart,
3838
- bucket_end: new Date,
3839
- blocks_processed: entry.blocksProcessed,
3840
- total_time_ms: Math.round(entry.totalTimeMs),
3841
- handler_time_ms: Math.round(entry.handlerTimeMs),
3842
- flush_time_ms: Math.round(entry.flushTimeMs),
3843
- max_block_time_ms: Math.round(entry.maxBlockTimeMs),
3844
- max_handler_time_ms: Math.round(entry.maxHandlerTimeMs),
3845
- avg_ops_per_block: Number.parseFloat(avgOpsPerBlock.toFixed(2)),
3846
- is_catchup: entry.isCatchup
3847
- }).execute();
3848
- }
3849
- newEntry() {
3850
- return {
3851
- subgraphName: this.subgraphName,
3852
- bucketStart: new Date,
3853
- blocksProcessed: 0,
3854
- totalTimeMs: 0,
3855
- handlerTimeMs: 0,
3856
- flushTimeMs: 0,
3857
- maxBlockTimeMs: 0,
3858
- maxHandlerTimeMs: 0,
3859
- totalOps: 0,
3860
- isCatchup: this.isCatchup
3861
- };
3862
- }
3863
- }
3864
-
3865
- // src/runtime/catchup.ts
3866
- import { getTargetDb as getTargetDb3, sql as sql5 } from "@secondlayer/shared/db";
3867
- import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
3868
- import { logger as logger9 } from "@secondlayer/shared/logger";
3869
-
3870
- // src/runtime/catchup-leader.ts
3871
- import {
3872
- SUBGRAPH_CATCHUP_LOCK_KEY,
3873
- createPostgresLeaderBackend,
3874
- withLeaderLock
3875
- } from "@secondlayer/shared/leader";
3876
- import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
3877
- var catchUpLeader = false;
3878
- function isCatchUpLeader() {
3879
- return catchUpLeader;
3880
- }
3881
- function startCatchUpLeader(opts = {}) {
3882
- return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
3883
- catchUpLeader = true;
3884
- await opts.onAcquire?.();
3885
- return () => {
3886
- catchUpLeader = false;
3887
- };
3888
- }, {
3889
- pollMs: opts.pollMs,
3890
- heartbeatMs: opts.heartbeatMs,
3891
- createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
3892
- });
3893
- }
3894
-
3895
- // src/runtime/decoder-bound.ts
3896
- import { decodeStreamsCursor, isEmptyRangeCursor } from "@secondlayer/shared";
3897
- import { getSourceDb as getSourceDb3 } from "@secondlayer/shared/db";
3898
- function decoderNameForEventType(indexEventType) {
3899
- return `decode.${indexEventType}.v1`;
3900
- }
3901
- function decoderNamesForIndexEventTypes(eventTypes) {
3902
- return eventTypes.filter((t) => !VM_INDEX_EVENT_TYPES.has(t)).map(decoderNameForEventType);
3903
- }
3904
- function decoderNamesForSubgraph(subgraph) {
3905
- return decoderNamesForIndexEventTypes(referencedIndexEventTypes(subgraph));
3906
- }
3907
- function usesDecodedIndexPlane(subgraph) {
3908
- return process.env.SUBGRAPH_SOURCE === "streams-index" && isStreamsIndexEligible(subgraph);
3909
- }
3910
- function committedHeight(cursor) {
3911
- if (!cursor)
3912
- return null;
3913
- try {
3914
- const decoded = decodeStreamsCursor(cursor);
3915
- if (isEmptyRangeCursor(decoded))
3916
- return decoded.block_height;
3917
- return Math.max(0, decoded.block_height - 1);
3918
- } catch {
3919
- return null;
3920
- }
3921
- }
3922
- async function decoderBoundTip(decoderNames, opts) {
3923
- if (decoderNames.length === 0)
3924
- return { kind: "unbounded" };
3925
- const sourceDb = opts?.sourceDb ?? getSourceDb3();
3926
- const rows = await sourceDb.selectFrom("decoder_checkpoints").select(["decoder_name", "last_cursor"]).where("decoder_name", "in", decoderNames).execute();
3927
- const byName = new Map(rows.map((r) => [r.decoder_name, r.last_cursor]));
3928
- const missing = [];
3929
- const heights = [];
3930
- for (const name of decoderNames) {
3931
- const height = committedHeight(byName.get(name));
3932
- if (height === null)
3933
- missing.push(name);
3934
- else
3935
- heights.push(height);
3936
- }
3937
- if (missing.length > 0)
3938
- return { kind: "stall", missing };
3939
- return { kind: "height", height: Math.min(...heights) };
3940
- }
3941
- async function boundSourceTip(rawTip, decoderNames, opts) {
3942
- const bound = await decoderBoundTip(decoderNames, opts);
3943
- if (bound.kind === "stall")
3944
- return { ok: false, missing: bound.missing };
3945
- if (bound.kind === "unbounded") {
3946
- return { ok: true, tip: rawTip, floor: null };
3947
- }
3948
- return {
3949
- ok: true,
3950
- tip: Math.min(rawTip, bound.height),
3951
- floor: bound.height
3952
- };
3953
- }
3954
-
3955
- // src/runtime/catchup.ts
3956
- var LOG_INTERVAL = 1000;
3957
- var STANDARD_CATCHUP_BATCH_CONFIG = {
3958
- defaultBatchSize: 500,
3959
- minBatchSize: 100,
3960
- maxBatchSize: 1000,
3961
- prefetch: true
3962
- };
3963
- var catchingUp = new Set;
3964
- var subgraphBlockLockHeld = new Set;
3965
- var subgraphBlockLockWaiters = new Map;
3966
- var reorgEpoch = new Map;
3967
- async function acquireSubgraphBlockLock(name) {
3968
- if (!subgraphBlockLockHeld.has(name)) {
3969
- subgraphBlockLockHeld.add(name);
3970
- return () => releaseSubgraphBlockLock(name);
3971
- }
3972
- return new Promise((resolve) => {
3973
- const waiters = subgraphBlockLockWaiters.get(name) ?? [];
3974
- waiters.push(() => resolve(() => releaseSubgraphBlockLock(name)));
3975
- subgraphBlockLockWaiters.set(name, waiters);
3976
- });
3977
- }
3978
- function releaseSubgraphBlockLock(name) {
3979
- const waiters = subgraphBlockLockWaiters.get(name);
3980
- if (waiters && waiters.length > 0) {
3981
- const next = waiters.shift();
3982
- if (waiters.length === 0)
3983
- subgraphBlockLockWaiters.delete(name);
3984
- next();
3985
- return;
3986
- }
3987
- subgraphBlockLockHeld.delete(name);
3988
- }
3989
- async function withSubgraphBlockLock(name, fn) {
3990
- const release = await acquireSubgraphBlockLock(name);
3991
- try {
3992
- return await fn();
3993
- } finally {
3994
- release();
3995
- }
3996
- }
3997
- async function withSubgraphBlockAdvisoryLock(name, fn) {
3998
- return getTargetDb3().transaction().execute(async (lockTx) => {
3999
- await sql5`SELECT pg_advisory_xact_lock(hashtext(${`subgraph-block:${name}`}))`.execute(lockTx);
4000
- return fn();
4001
- });
4002
- }
4003
- async function withSubgraphBlockWriteLock(name, fn) {
4004
- return withSubgraphBlockLock(name, () => withSubgraphBlockAdvisoryLock(name, fn));
4005
- }
4006
- function bumpReorgEpoch(name) {
4007
- reorgEpoch.set(name, (reorgEpoch.get(name) ?? 0) + 1);
4008
- }
4009
- function getReorgEpoch(name) {
4010
- return reorgEpoch.get(name) ?? 0;
4011
- }
4012
-
4013
- class ReorgEpochChangedError extends Error {
4014
- constructor(name) {
4015
- super(`reorg epoch changed for ${name} mid catch-up`);
4016
- this.name = "ReorgEpochChangedError";
4017
- }
4018
- }
4019
- function parsePositiveInt(value) {
4020
- if (value == null || value.trim() === "")
4021
- return;
4022
- const parsed = Number.parseInt(value, 10);
4023
- return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
4024
- }
4025
- function parseBoolean(value) {
4026
- if (value == null || value.trim() === "")
4027
- return;
4028
- const normalized = value.trim().toLowerCase();
4029
- if (normalized === "true")
4030
- return true;
4031
- if (normalized === "false")
4032
- return false;
4033
- return;
4034
- }
4035
- function resolveCatchupBatchConfig(env = process.env) {
4036
- const base = STANDARD_CATCHUP_BATCH_CONFIG;
4037
- const minBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MIN_BATCH_SIZE) ?? base.minBatchSize;
4038
- const maxBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MAX_BATCH_SIZE) ?? base.maxBatchSize;
4039
- const defaultBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_BATCH_SIZE) ?? base.defaultBatchSize;
4040
- return {
4041
- minBatchSize,
4042
- maxBatchSize,
4043
- defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize),
4044
- prefetch: parseBoolean(env.SUBGRAPH_CATCHUP_PREFETCH) ?? base.prefetch
4045
- };
4046
- }
4047
- function adjustBatchSize(current, avgEvents, config) {
4048
- if (avgEvents > 50)
4049
- return Math.max(Math.round(current * 0.5), config.minBatchSize);
4050
- if (avgEvents < 10)
4051
- return Math.min(Math.round(current * 1.5), config.maxBatchSize);
4052
- return current;
4053
- }
4054
- async function catchUpSubgraph(subgraph, subgraphName) {
4055
- if (catchingUp.has(subgraphName))
4056
- return 0;
4057
- catchingUp.add(subgraphName);
4058
- try {
4059
- const source = resolveBlockSource(subgraph);
4060
- const targetDb = getTargetDb3();
4061
- const subgraphRow = await getSubgraph(targetDb, subgraphName);
4062
- if (!subgraphRow)
4063
- return 0;
4064
- const lastProcessedBlock = Number(subgraphRow.last_processed_block);
4065
- const rawTip = await source.getTip();
4066
- if (rawTip <= 0)
4067
- return 0;
4068
- let chainTip = rawTip;
4069
- if (usesDecodedIndexPlane(subgraph)) {
4070
- const bound = await boundSourceTip(rawTip, decoderNamesForSubgraph(subgraph));
4071
- if (!bound.ok) {
4072
- logger9.warn("Subgraph catch-up stalled: missing decoder checkpoint", {
4073
- event: "subgraph_catchup_decoder_stall",
4074
- subgraph: subgraphName,
4075
- missing: bound.missing
4076
- });
4077
- return 0;
4078
- }
4079
- if (bound.floor !== null && bound.floor < rawTip) {
4080
- logger9.debug("Subgraph catch-up tip bounded by decoder progress", {
4081
- event: "subgraph_catchup_decoder_floor",
4082
- subgraph: subgraphName,
4083
- rawTip,
4084
- floor: bound.floor,
4085
- tip: bound.tip
4086
- });
4087
- }
4088
- chainTip = bound.tip;
4089
- }
4090
- if (chainTip <= 0 || lastProcessedBlock >= chainTip)
4091
- return 0;
4092
- const subgraphStart = Number(subgraphRow.start_block) || 1;
4093
- const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
4094
- const totalBlocks = chainTip - lastProcessedBlock;
4095
- logger9.info("Subgraph catch-up starting", {
4096
- subgraph: subgraphName,
4097
- from: startBlock,
4098
- to: chainTip,
4099
- blocks: totalBlocks
4100
- });
4101
- const stats = new StatsAccumulator(subgraphName, true);
4102
- let processed = 0;
4103
- const batchConfig = resolveCatchupBatchConfig();
4104
- let batchSize = batchConfig.defaultBatchSize;
4105
- let currentHeight = startBlock;
4106
- const walkEpoch = getReorgEpoch(subgraphName);
4107
- let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
4108
- let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
4109
- while (currentHeight <= chainTip) {
4110
- if (!isCatchUpLeader()) {
4111
- logger9.info("Lost catch-up leadership, stopping catch-up", {
4112
- subgraph: subgraphName,
4113
- processed
4114
- });
4115
- break;
4116
- }
4117
- const currentRow = await getSubgraph(targetDb, subgraphName);
4118
- if (!currentRow || currentRow.status !== "active") {
4119
- logger9.info("Subgraph status changed, stopping catch-up", {
4120
- subgraph: subgraphName,
4121
- status: currentRow?.status ?? "deleted"
4122
- });
4123
- break;
4124
- }
4125
- let batchEnd;
4126
- let batch;
4127
- if (nextBatchPromise) {
4128
- batch = await nextBatchPromise;
4129
- batchEnd = prefetchedBatchEnd;
4130
- const nextStart = batchEnd + 1;
4131
- if (nextStart <= chainTip) {
4132
- prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
4133
- nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
4134
- } else {
4135
- nextBatchPromise = undefined;
4136
- }
4137
- } else {
4138
- batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
4139
- batch = await source.loadBlockRange(currentHeight, batchEnd);
4140
- }
4141
- let stopCatchup = false;
4142
- for (let height = currentHeight;height <= batchEnd; height++) {
4143
- let blockData = batch.get(height);
4144
- if (!blockData) {
4145
- blockData = (await source.loadBlockRange(height, height)).get(height);
4146
- }
4147
- if (!blockData) {
4148
- logger9.warn("Block missing during catch-up, deferring to next tick", {
4149
- subgraph: subgraphName,
4150
- blockHeight: height
4151
- });
4152
- stopCatchup = true;
4153
- break;
4154
- }
4155
- const preloaded = blockData;
4156
- let result;
4157
- try {
4158
- result = await withSubgraphBlockWriteLock(subgraphName, async () => {
4159
- if (getReorgEpoch(subgraphName) !== walkEpoch) {
4160
- throw new ReorgEpochChangedError(subgraphName);
4161
- }
4162
- return processBlockWithRetry(subgraph, subgraphName, height, {
4163
- preloaded,
4164
- subgraphId: subgraphRow.id
4165
- });
4166
- });
4167
- } catch (err) {
4168
- if (err instanceof ReorgEpochChangedError) {
4169
- logger9.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
4170
- stopCatchup = true;
4171
- break;
4172
- }
4173
- const errorMsg = err instanceof Error ? err.message : String(err);
4174
- logger9.error("Block processing failed persistently during catch-up", {
4175
- subgraph: subgraphName,
4176
- blockHeight: height,
4177
- error: errorMsg
4178
- });
4179
- const { updateSubgraphStatus: updateSubgraphStatus2, recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
4180
- await recordSubgraphProcessed2(targetDb, subgraphName, 0, 1, `catch-up halted at block ${height}: ${errorMsg}`).catch(() => {});
4181
- await updateSubgraphStatus2(targetDb, subgraphName, "error").catch(() => {});
4182
- stopCatchup = true;
4183
- break;
4184
- }
4185
- processed++;
4186
- if (result.timing) {
4187
- stats.record(result.timing, result.processed);
4188
- if (stats.shouldFlush()) {
4189
- await stats.flush(targetDb);
4190
- }
4191
- }
4192
- if (processed % LOG_INTERVAL === 0) {
4193
- logger9.info("Subgraph catch-up progress", {
4194
- subgraph: subgraphName,
4195
- processed,
4196
- total: totalBlocks,
4197
- currentBlock: height,
4198
- pct: Math.round(processed / totalBlocks * 100)
4199
- });
4200
- }
4201
- }
4202
- if (stopCatchup)
4203
- break;
4204
- const avg = avgEventsPerBlock(batch);
4205
- batchSize = adjustBatchSize(batchSize, avg, batchConfig);
4206
- currentHeight = batchEnd + 1;
4207
- }
4208
- await stats.flush(targetDb);
4209
- logger9.info("Subgraph catch-up complete", {
4210
- subgraph: subgraphName,
4211
- processed
4212
- });
4213
- return processed;
4214
- } finally {
4215
- catchingUp.delete(subgraphName);
4216
- }
4217
- }
4218
-
4219
- // src/runtime/reindex.ts
4220
- import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
4221
- import { getRawClient, getTargetDb as getTargetDb4 } from "@secondlayer/shared/db";
4222
- import {
4223
- recordGapBatch as recordGapBatch2,
4224
- resolveGaps
4225
- } from "@secondlayer/shared/db/queries/subgraph-gaps";
4226
- import {
4227
- advanceOperationCursor as advanceOperationCursor2,
4228
- updateOperationProcessedEvents
4229
- } from "@secondlayer/shared/db/queries/subgraph-operations";
4230
- import {
4231
- recordSubgraphProcessed as recordSubgraphProcessed2,
4232
- updateSubgraphStatus as updateSubgraphStatus2
4233
- } from "@secondlayer/shared/db/queries/subgraphs";
4234
- import { logger as logger11 } from "@secondlayer/shared/logger";
4235
-
4236
- // src/runtime/reindex-notify.ts
4237
- import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
4238
- import { logger as logger10 } from "@secondlayer/shared/logger";
4239
- async function notifyReindexComplete(db, subgraphName, stats) {
4240
- try {
4241
- const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
4242
- if (!subgraph)
4243
- return;
4244
- const account = await db.selectFrom("accounts").select(["email", "notify_reindex_complete"]).where("id", "=", subgraph.account_id).executeTakeFirst();
4245
- if (!account?.email || !account.notify_reindex_complete)
4246
- return;
4247
- const resendKey = process.env.RESEND_API_KEY;
4248
- if (!resendKey) {
4249
- logger10.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
4250
- subgraph: subgraphName
4251
- });
4252
- return;
4253
- }
4254
- const from = process.env.EMAIL_FROM ?? "Secondlayer <noreply@secondlayer.tools>";
4255
- const body = stats.errors > 0 ? `Your subgraph "${subgraphName}" finished reindexing — ${stats.blocks.toLocaleString()} blocks, ${stats.events.toLocaleString()} events, ${stats.errors.toLocaleString()} errors. Check the dashboard for details.` : `Your subgraph "${subgraphName}" finished reindexing — ${stats.blocks.toLocaleString()} blocks, ${stats.events.toLocaleString()} events processed, no errors. It's live now.`;
4256
- const res = await fetch("https://api.resend.com/emails", {
4257
- method: "POST",
4258
- headers: {
4259
- Authorization: `Bearer ${resendKey}`,
4260
- "Content-Type": "application/json"
4261
- },
4262
- body: JSON.stringify({
4263
- from,
4264
- to: [account.email],
4265
- subject: `Reindex complete: ${subgraphName}`,
4266
- text: body
4267
- })
4268
- });
4269
- if (!res.ok) {
4270
- const text = await res.text().catch(() => "");
4271
- logger10.warn("reindex-complete email failed", {
4272
- subgraph: subgraphName,
4273
- status: res.status,
4274
- body: text.slice(0, 200)
4275
- });
4276
- }
4277
- } catch (err) {
4278
- logger10.warn("reindex-complete email threw", {
4279
- subgraph: subgraphName,
4280
- error: getErrorMessage2(err)
4281
- });
4282
- }
4283
- }
4284
-
4285
- // src/runtime/reindex.ts
4286
- var LOG_INTERVAL2 = 1000;
4287
- var HEALTH_FLUSH_INTERVAL = 1000;
4288
- var PROGRESS_FLUSH_INTERVAL_MS = 5000;
4289
- var EMPTY_BATCH_HALT_THRESHOLD = 3;
4290
- var STANDARD_REINDEX_BATCH_CONFIG = {
4291
- defaultBatchSize: 500,
4292
- minBatchSize: 100,
4293
- maxBatchSize: 1000
4294
- };
4295
- function initialReindexProgressBlock(fromBlock) {
4296
- return Math.max(0, fromBlock - 1);
4297
- }
4298
- function resolveReindexResumeBlock(row) {
4299
- if (row.reindex_from_block == null || row.reindex_to_block == null) {
4300
- return null;
4301
- }
4302
- const lastProcessedBlock = Number(row.last_processed_block);
4303
- const reindexFromBlock = Number(row.reindex_from_block);
4304
- return Math.max(lastProcessedBlock + 1, reindexFromBlock);
4305
- }
4306
- function parsePositiveInt2(value) {
4307
- if (value == null || value.trim() === "")
4308
- return;
4309
- const parsed = Number.parseInt(value, 10);
4310
- return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
4311
- }
4312
- function resolveReindexBatchConfig(env = process.env) {
4313
- const base = STANDARD_REINDEX_BATCH_CONFIG;
4314
- const minBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_MIN_BATCH_SIZE) ?? base.minBatchSize;
4315
- const maxBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_MAX_BATCH_SIZE) ?? base.maxBatchSize;
4316
- const defaultBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_BATCH_SIZE) ?? base.defaultBatchSize;
4317
- return {
4318
- minBatchSize,
4319
- maxBatchSize,
4320
- defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize)
4321
- };
4322
- }
4323
- function coalesceFailedBlocks(blocks) {
4324
- if (blocks.length === 0)
4325
- return [];
4326
- blocks.sort((a, b) => a.height - b.height);
4327
- const ranges = [];
4328
- let start = blocks[0].height;
4329
- let end = blocks[0].height;
4330
- let reason = blocks[0].reason;
4331
- for (let i = 1;i < blocks.length; i++) {
4332
- const b = blocks[i];
4333
- if (b.height === end + 1 && b.reason === reason) {
4334
- end = b.height;
4335
- } else {
4336
- ranges.push({ start, end, reason });
4337
- start = b.height;
4338
- end = b.height;
4339
- reason = b.reason;
4340
- }
4341
- }
4342
- ranges.push({ start, end, reason });
4343
- return ranges;
4344
- }
4345
- async function processBlockRange(def, opts) {
4346
- const source = resolveBlockSource(def);
4347
- const targetDb = getTargetDb4();
4348
- const subgraphName = def.name;
4349
- const { fromBlock, toBlock, status } = opts;
4350
- const totalBlocks = toBlock - fromBlock + 1;
4351
- const stats = new StatsAccumulator(subgraphName, opts.isCatchup);
4352
- let blocksProcessed = 0;
4353
- let blocksSkippedByCursor = 0;
4354
- let totalEventsProcessed = 0;
4355
- let totalErrors = 0;
4356
- let pendingEventsProcessed = 0;
4357
- let pendingErrors = 0;
4358
- let pendingLastError;
4359
- let lastHealthFlushBlock = 0;
4360
- let lastHealthFlushAt = Date.now();
4361
- let lastProgressFlushAt = Date.now();
4362
- const batchConfig = resolveReindexBatchConfig();
4363
- let batchSize = batchConfig.defaultBatchSize;
4364
- let currentHeight = fromBlock;
4365
- let aborted = false;
4366
- let consecutiveEmptyBatches = 0;
4367
- const sparse = Boolean(source.nextDataHeight && canSparseScan(def));
4368
- const flushHealth = async () => {
4369
- if (pendingEventsProcessed === 0 && pendingErrors === 0)
4370
- return;
4371
- await recordSubgraphProcessed2(targetDb, subgraphName, pendingEventsProcessed, pendingErrors, pendingLastError);
4372
- pendingEventsProcessed = 0;
4373
- pendingErrors = 0;
4374
- pendingLastError = undefined;
4375
- lastHealthFlushBlock = blocksProcessed;
4376
- lastHealthFlushAt = Date.now();
4377
- };
4378
- const haltRange = async (errorMsg, height) => {
4379
- pendingErrors++;
4380
- pendingLastError = errorMsg;
4381
- await flushHealth().catch(() => {});
4382
- await updateSubgraphStatus2(targetDb, subgraphName, "error").catch(() => {});
4383
- throw new Error(`${subgraphName}: halted at block ${height}: ${errorMsg}`);
4384
- };
4385
- let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
4386
- let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
4387
- while (currentHeight <= toBlock) {
4388
- if (opts.signal?.aborted) {
4389
- aborted = true;
4390
- logger11.info("Block processing aborted", {
4391
- subgraph: subgraphName,
4392
- currentBlock: currentHeight,
4393
- reason: String(opts.signal.reason ?? "unknown")
4394
- });
4395
- break;
4396
- }
4397
- const batch = await nextBatchPromise;
4398
- const batchEnd = nextBatchEnd;
4399
- if (batch.size === 0 && batchEnd >= currentHeight) {
4400
- consecutiveEmptyBatches++;
4401
- if (consecutiveEmptyBatches >= EMPTY_BATCH_HALT_THRESHOLD) {
4402
- await haltRange(`block source returned ${consecutiveEmptyBatches} consecutive empty batches (ending ${currentHeight}..${batchEnd}) — source degraded`, currentHeight);
4403
- }
4404
- } else {
4405
- consecutiveEmptyBatches = 0;
4406
- }
4407
- const nextStart = batchEnd + 1;
4408
- if (nextStart <= toBlock) {
4409
- nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
4410
- nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
4411
- }
4412
- const batchFailedBlocks = [];
4413
- let batchMatched = 0;
4414
- const opCursor = status === "active" && opts.operationId ? { operationId: opts.operationId } : undefined;
4415
- const atomicProgress = status === "reindexing" ? { status } : opCursor;
4416
- for (let height = currentHeight;height <= batchEnd; height++) {
4417
- let blockData = batch.get(height);
4418
- if (!blockData) {
4419
- blockData = (await source.loadBlockRange(height, height)).get(height);
4420
- }
4421
- if (!blockData) {
4422
- if (status === "reindexing") {
4423
- const errorMsg = `block ${height} missing from source — halting reindex (cursor stays at ${height - 1})`;
4424
- await haltRange(errorMsg, height);
4425
- }
4426
- batchFailedBlocks.push({ height, reason: "block_missing" });
4427
- blocksProcessed++;
4428
- continue;
4429
- }
4430
- let result;
4431
- try {
4432
- const mayWrite = blockData.txs.length > 0 || blockData.events.length > 0 || (blockData.vmEvents?.length ?? 0) > 0;
4433
- const runBlock = () => processBlockWithRetry(def, subgraphName, height, {
4434
- skipProgressUpdate: true,
4435
- atomicProgress,
4436
- preloaded: blockData
4437
- });
4438
- result = mayWrite ? await withSubgraphBlockWriteLock(subgraphName, runBlock) : await runBlock();
4439
- } catch (err) {
4440
- const errorMsg = getErrorMessage3(err);
4441
- logger11.error("Block processing failed persistently", {
4442
- subgraph: subgraphName,
4443
- blockHeight: height,
4444
- error: errorMsg
4445
- });
4446
- if (status === "reindexing") {
4447
- await haltRange(`block ${height} failed persistently: ${errorMsg}`, height);
4448
- }
4449
- batchFailedBlocks.push({ height, reason: "processing_error" });
4450
- blocksProcessed++;
4451
- totalErrors++;
4452
- pendingErrors++;
4453
- pendingLastError = errorMsg;
4454
- continue;
4455
- }
4456
- blocksProcessed++;
4457
- if (result.skipped)
4458
- blocksSkippedByCursor++;
4459
- batchMatched += result.matched;
4460
- totalEventsProcessed += result.processed;
4461
- totalErrors += result.errors;
4462
- pendingEventsProcessed += result.processed;
4463
- pendingErrors += result.errors;
4464
- if (result.errors > 0) {
4465
- pendingLastError = `${result.errors} error(s) while processing block ${height}`;
4466
- }
4467
- if (result.timing) {
4468
- stats.record(result.timing, result.processed);
4469
- if (stats.shouldFlush()) {
4470
- await stats.flush(targetDb);
4471
- }
4472
- }
4473
- const now = Date.now();
4474
- const shouldFlushProgress = blocksProcessed % 100 === 0 || now - lastProgressFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
4475
- if (shouldFlushProgress) {
4476
- if (opCursor) {
4477
- await advanceOperationCursor2(targetDb, opCursor.operationId, height);
4478
- } else {
4479
- await updateSubgraphStatus2(targetDb, subgraphName, status, height);
4480
- }
4481
- if (opts.operationId) {
4482
- await updateOperationProcessedEvents(targetDb, opts.operationId, totalEventsProcessed).catch(() => {});
4483
- }
4484
- lastProgressFlushAt = now;
4485
- }
4486
- if (blocksProcessed % LOG_INTERVAL2 === 0) {
4487
- logger11.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
4488
- subgraph: subgraphName,
4489
- processed: blocksProcessed,
4490
- total: totalBlocks,
4491
- currentBlock: height,
4492
- pct: Math.round(blocksProcessed / totalBlocks * 100),
4493
- ...blocksSkippedByCursor > 0 ? { skippedByCursor: blocksSkippedByCursor } : {}
4494
- });
4495
- }
4496
- }
4497
- if (status === "reindexing" && blocksProcessed > 0) {
4498
- await updateSubgraphStatus2(targetDb, subgraphName, status, batchEnd);
4499
- }
4500
- const shouldFlushHealth = blocksProcessed - lastHealthFlushBlock >= HEALTH_FLUSH_INTERVAL || Date.now() - lastHealthFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
4501
- if (shouldFlushHealth) {
4502
- await flushHealth();
4503
- }
4504
- if (batchFailedBlocks.length > 0 && opts.subgraphId) {
4505
- const gaps = coalesceFailedBlocks(batchFailedBlocks);
4506
- await recordGapBatch2(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
4507
- logger11.warn("Failed to record subgraph gaps", {
4508
- subgraph: subgraphName,
4509
- error: err instanceof Error ? err.message : String(err)
4510
- });
4511
- });
4512
- }
4513
- if (sparse && batchMatched === 0 && batchEnd < toBlock && source.nextDataHeight) {
4514
- const next = await source.nextDataHeight(batchEnd, toBlock);
4515
- const jumpTo = next === null ? toBlock + 1 : Math.max(next, batchEnd + 1);
4516
- if (jumpTo > batchEnd + 1) {
4517
- const skipped = Math.min(jumpTo, toBlock + 1) - (batchEnd + 1);
4518
- blocksProcessed += skipped;
4519
- if (opCursor) {
4520
- await advanceOperationCursor2(targetDb, opCursor.operationId, jumpTo - 1);
4521
- } else {
4522
- await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
4523
- }
4524
- logger11.info("Sparse skip", {
4525
- subgraph: subgraphName,
4526
- from: batchEnd + 1,
4527
- to: jumpTo - 1,
4528
- skipped
4529
- });
4530
- currentHeight = jumpTo;
4531
- if (currentHeight <= toBlock) {
4532
- nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
4533
- nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
4534
- }
4535
- continue;
4536
- }
4537
- }
4538
- const avg = avgEventsPerBlock(batch);
4539
- if (avg > 50)
4540
- batchSize = Math.max(Math.round(batchSize * 0.5), batchConfig.minBatchSize);
4541
- else if (avg < 10)
4542
- batchSize = Math.min(Math.round(batchSize * 1.5), batchConfig.maxBatchSize);
4543
- currentHeight = batchEnd + 1;
4544
- }
4545
- await stats.flush(targetDb);
4546
- await flushHealth();
4547
- return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
4548
- }
4549
- async function resolveBlockRange(source, def, opts) {
4550
- const definitionStart = def.startBlock ?? 1;
4551
- const fromBlock = opts?.startBlockFloor != null ? Math.max(definitionStart, opts.startBlockFloor) : definitionStart;
4552
- const rawTip = await source.getTip();
4553
- if (!usesDecodedIndexPlane(def)) {
4554
- return { fromBlock, toBlock: rawTip };
4555
- }
4556
- const bound = await boundSourceTip(rawTip, decoderNamesForSubgraph(def));
4557
- if (!bound.ok) {
4558
- logger11.warn("Reindex stalled: missing decoder checkpoint", {
4559
- event: "subgraph_reindex_decoder_stall",
4560
- subgraph: def.name,
4561
- missing: bound.missing
4562
- });
4563
- throw new Error(`Reindex stalled: missing decoder checkpoint (${bound.missing.join(", ")})`);
4564
- }
4565
- if (bound.floor !== null && bound.floor < rawTip) {
4566
- logger11.debug("Reindex tip bounded by decoder progress", {
4567
- event: "subgraph_reindex_decoder_floor",
4568
- subgraph: def.name,
4569
- rawTip,
4570
- floor: bound.floor,
4571
- tip: bound.tip
4572
- });
4573
- }
4574
- return { fromBlock, toBlock: bound.tip };
4575
- }
4576
- async function clearReindexMetadata(db, subgraphName) {
4577
- await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
4578
- }
4579
- async function reindexSubgraph(def, opts) {
4580
- const source = resolveBlockSource(def);
4581
- const targetDb = getTargetDb4();
4582
- const client = getRawClient("target");
4583
- const subgraphName = def.name;
4584
- const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
4585
- await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
4586
- logger11.info("Reindex starting", { subgraph: subgraphName });
4587
- try {
4588
- const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
4589
- if (fromBlock > toBlock) {
4590
- logger11.info("No blocks to reindex", {
4591
- subgraph: subgraphName,
4592
- fromBlock,
4593
- toBlock
4594
- });
4595
- await updateSubgraphStatus2(targetDb, subgraphName, "active", 0);
4596
- return { processed: 0 };
4597
- }
4598
- await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
4599
- const { statements } = generateSubgraphSQL(def, schemaName);
4600
- for (const stmt of statements) {
4601
- await client.unsafe(stmt);
4602
- }
4603
- await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
4604
- logger11.info("Schema recreated for reindex", {
4605
- subgraph: subgraphName,
4606
- cursorResetTo: Math.max(0, fromBlock - 1)
4607
- });
4608
- await targetDb.updateTable("subgraphs").set({
4609
- last_processed_block: initialReindexProgressBlock(fromBlock),
4610
- reindex_from_block: fromBlock,
4611
- reindex_to_block: toBlock,
4612
- total_processed: 0,
4613
- total_errors: 0,
4614
- last_error: null,
4615
- last_error_at: null,
4616
- updated_at: new Date
4617
- }).where("name", "=", subgraphName).execute();
4618
- logger11.info("Reindexing blocks", {
4619
- subgraph: subgraphName,
4620
- fromBlock,
4621
- toBlock,
4622
- totalBlocks: toBlock - fromBlock + 1
4623
- });
4624
- const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
4625
- const result = await processBlockRange(def, {
4626
- fromBlock,
4627
- toBlock,
4628
- status: "reindexing",
4629
- isCatchup: false,
4630
- apiKeyId: null,
4631
- subgraphId: subgraphRow?.id,
4632
- operationId: opts?.operationId,
4633
- signal: opts?.signal
4634
- });
4635
- if (result.aborted) {
4636
- const reason = String(opts?.signal?.reason ?? "unknown");
4637
- if (reason === "user-cancelled") {
4638
- await updateSubgraphStatus2(targetDb, subgraphName, "active");
4639
- await clearReindexMetadata(targetDb, subgraphName);
4640
- logger11.info("Reindex cancelled by user", { subgraph: subgraphName });
4641
- } else {
4642
- logger11.info("Reindex interrupted by shutdown, will resume", {
4643
- subgraph: subgraphName
4644
- });
4645
- }
4646
- return { processed: result.blocksProcessed };
4647
- }
4648
- await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
4649
- await clearReindexMetadata(targetDb, subgraphName);
4650
- logger11.info("Reindex complete", {
4651
- subgraph: subgraphName,
4652
- blocks: result.blocksProcessed,
4653
- events: result.totalEventsProcessed,
4654
- errors: result.totalErrors
4655
- });
4656
- notifyReindexComplete(targetDb, subgraphName, {
4657
- blocks: result.blocksProcessed,
4658
- events: result.totalEventsProcessed,
4659
- errors: result.totalErrors
4660
- });
4661
- return { processed: result.blocksProcessed };
4662
- } catch (err) {
4663
- logger11.error("Reindex failed", {
4664
- subgraph: subgraphName,
4665
- error: getErrorMessage3(err)
4666
- });
4667
- await updateSubgraphStatus2(targetDb, subgraphName, "error");
4668
- throw err;
4669
- }
4670
- }
4671
- async function resumeReindex(def, opts) {
4672
- const targetDb = getTargetDb4();
4673
- const subgraphName = def.name;
4674
- const row = await targetDb.selectFrom("subgraphs").select([
4675
- "id",
4676
- "last_processed_block",
4677
- "reindex_from_block",
4678
- "reindex_to_block"
4679
- ]).where("name", "=", subgraphName).executeTakeFirst();
4680
- if (!row)
4681
- throw new Error(`Subgraph "${subgraphName}" not found`);
4682
- const fromBlock = resolveReindexResumeBlock(row);
4683
- const toBlock = Number(row.reindex_to_block);
4684
- if (fromBlock == null) {
4685
- logger11.info("No reindex metadata, starting fresh reindex", {
4686
- subgraph: subgraphName
4687
- });
4688
- return reindexSubgraph(def, {
4689
- schemaName: opts.schemaName,
4690
- signal: opts.signal
4691
- });
4692
- }
4693
- if (fromBlock > toBlock) {
4694
- logger11.info("Resume: no remaining blocks", { subgraph: subgraphName });
4695
- await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
4696
- await clearReindexMetadata(targetDb, subgraphName);
4697
- return { processed: 0 };
4698
- }
4699
- logger11.info("Resuming reindex", {
4700
- subgraph: subgraphName,
4701
- fromBlock,
4702
- toBlock,
4703
- remaining: toBlock - fromBlock + 1
4704
- });
4705
- try {
4706
- const result = await processBlockRange(def, {
4707
- fromBlock,
4708
- toBlock,
4709
- status: "reindexing",
4710
- isCatchup: false,
4711
- apiKeyId: null,
4712
- subgraphId: row.id,
4713
- operationId: opts.operationId,
4714
- signal: opts.signal
4715
- });
4716
- if (result.aborted) {
4717
- const reason = String(opts.signal?.reason ?? "unknown");
4718
- if (reason === "user-cancelled") {
4719
- await updateSubgraphStatus2(targetDb, subgraphName, "active");
4720
- await clearReindexMetadata(targetDb, subgraphName);
4721
- logger11.info("Resume cancelled by user", { subgraph: subgraphName });
4722
- } else {
4723
- logger11.info("Resume interrupted by shutdown, will resume again", {
4724
- subgraph: subgraphName
4725
- });
4726
- }
4727
- return { processed: result.blocksProcessed };
4728
- }
4729
- await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
4730
- await clearReindexMetadata(targetDb, subgraphName);
4731
- logger11.info("Resumed reindex complete", {
4732
- subgraph: subgraphName,
4733
- blocks: result.blocksProcessed
4734
- });
4735
- return { processed: result.blocksProcessed };
4736
- } catch (err) {
4737
- logger11.error("Resumed reindex failed", {
4738
- subgraph: subgraphName,
4739
- error: getErrorMessage3(err)
4740
- });
4741
- await updateSubgraphStatus2(targetDb, subgraphName, "error");
4742
- throw err;
4743
- }
4744
- }
4745
- async function backfillSubgraph(def, opts) {
4746
- const targetDb = getTargetDb4();
4747
- const subgraphName = def.name;
4748
- logger11.info("Backfill starting", {
4749
- subgraph: subgraphName,
4750
- from: opts.fromBlock,
4751
- to: opts.toBlock
4752
- });
4753
- try {
4754
- const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
4755
- const result = await processBlockRange(def, {
4756
- fromBlock: opts.fromBlock,
4757
- toBlock: opts.toBlock,
4758
- status: "active",
4759
- isCatchup: false,
4760
- apiKeyId: null,
4761
- subgraphId: subgraphRow?.id,
4762
- operationId: opts.operationId,
4763
- signal: opts.signal
4764
- });
4765
- if (result.aborted) {
4766
- logger11.info("Backfill aborted", { subgraph: subgraphName });
4767
- return { processed: result.blocksProcessed };
4768
- }
4769
- const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
4770
- if (resolved > 0) {
4771
- logger11.info("Resolved subgraph gaps via backfill", {
4772
- subgraph: subgraphName,
4773
- resolved
4774
- });
4775
- }
4776
- logger11.info("Backfill complete", {
4777
- subgraph: subgraphName,
4778
- blocks: result.blocksProcessed,
4779
- events: result.totalEventsProcessed,
4780
- errors: result.totalErrors
4781
- });
4782
- return { processed: result.blocksProcessed };
4783
- } catch (err) {
4784
- logger11.error("Backfill failed", {
4785
- subgraph: subgraphName,
4786
- error: getErrorMessage3(err)
4787
- });
4788
- throw err;
4789
- }
4790
- }
4791
- export {
4792
- resumeReindex,
4793
- resolveReindexResumeBlock,
4794
- resolveReindexBatchConfig,
4795
- resolveBlockRange,
4796
- reindexSubgraph,
4797
- initialReindexProgressBlock,
4798
- backfillSubgraph
4799
- };
4800
-
4801
- //# debugId=8D0663264E89460C64756E2164756E21
4802
- //# sourceMappingURL=reindex.js.map