@secondlayer/subgraphs 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +3 -1
- package/dist/src/index.js +233 -119
- package/dist/src/index.js.map +16 -16
- package/dist/src/runtime/block-processor.d.ts +2 -0
- package/dist/src/runtime/block-processor.js +130 -100
- package/dist/src/runtime/block-processor.js.map +9 -9
- package/dist/src/runtime/catchup.d.ts +2 -0
- package/dist/src/runtime/catchup.js +180 -104
- package/dist/src/runtime/catchup.js.map +12 -12
- package/dist/src/runtime/clarity.js.map +2 -2
- package/dist/src/runtime/context.d.ts +1 -1
- package/dist/src/runtime/context.js +17 -4
- package/dist/src/runtime/context.js.map +3 -3
- package/dist/src/runtime/processor.js +198 -110
- package/dist/src/runtime/processor.js.map +14 -14
- package/dist/src/runtime/reindex.d.ts +2 -0
- package/dist/src/runtime/reindex.js +223 -117
- package/dist/src/runtime/reindex.js.map +13 -13
- package/dist/src/runtime/reorg.d.ts +2 -0
- package/dist/src/runtime/reorg.js +143 -104
- package/dist/src/runtime/reorg.js.map +10 -10
- package/dist/src/runtime/runner.d.ts +3 -1
- package/dist/src/runtime/runner.js +7 -3
- package/dist/src/runtime/runner.js.map +4 -4
- package/dist/src/runtime/source-matcher.js.map +3 -3
- package/dist/src/runtime/stats.d.ts +1 -1
- package/dist/src/runtime/stats.js +2 -2
- package/dist/src/runtime/stats.js.map +3 -3
- package/dist/src/schema/index.d.ts +3 -1
- package/dist/src/schema/index.js +11 -3
- package/dist/src/schema/index.js.map +5 -5
- package/dist/src/service.js +199 -111
- package/dist/src/service.js.map +15 -15
- package/dist/src/templates.js.map +2 -2
- package/dist/src/types.d.ts +2 -0
- package/dist/src/types.js.map +2 -2
- package/dist/src/validate.d.ts +2 -0
- package/dist/src/validate.js.map +2 -2
- package/package.json +51 -51
|
@@ -16,8 +16,8 @@ function sourceKey(source) {
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
// src/runtime/context.ts
|
|
19
|
-
import { sql } from "kysely";
|
|
20
19
|
import { logger } from "@secondlayer/shared/logger";
|
|
20
|
+
import { sql } from "kysely";
|
|
21
21
|
function validateColumnName(name) {
|
|
22
22
|
if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
|
|
23
23
|
throw new Error(`Invalid column name: ${name}`);
|
|
@@ -58,13 +58,26 @@ class SubgraphContext {
|
|
|
58
58
|
const keyColumns = Object.keys(key);
|
|
59
59
|
const hasUniqueConstraint = tableDef.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
|
|
60
60
|
if (hasUniqueConstraint) {
|
|
61
|
-
this.ops.push({
|
|
61
|
+
this.ops.push({
|
|
62
|
+
kind: "insert",
|
|
63
|
+
table,
|
|
64
|
+
data: { ...key, ...row, _upsert_keys: keyColumns }
|
|
65
|
+
});
|
|
62
66
|
} else {
|
|
63
67
|
logger.warn("upsert called without matching uniqueKeys constraint, using fallback", {
|
|
64
68
|
table,
|
|
65
69
|
keys: keyColumns
|
|
66
70
|
});
|
|
67
|
-
this.ops.push({
|
|
71
|
+
this.ops.push({
|
|
72
|
+
kind: "insert",
|
|
73
|
+
table,
|
|
74
|
+
data: {
|
|
75
|
+
...key,
|
|
76
|
+
...row,
|
|
77
|
+
_upsert_fallback_keys: keyColumns,
|
|
78
|
+
_upsert_fallback_set: row
|
|
79
|
+
}
|
|
80
|
+
});
|
|
68
81
|
}
|
|
69
82
|
}
|
|
70
83
|
delete(table, where) {
|
|
@@ -227,95 +240,6 @@ function buildWhereClause(where) {
|
|
|
227
240
|
return { clause: parts.join(" AND "), values: [] };
|
|
228
241
|
}
|
|
229
242
|
|
|
230
|
-
// src/runtime/source-matcher.ts
|
|
231
|
-
function matchPattern(value, pattern) {
|
|
232
|
-
if (!pattern.includes("*"))
|
|
233
|
-
return value === pattern;
|
|
234
|
-
const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
235
|
-
return new RegExp(`^${regex}$`).test(value);
|
|
236
|
-
}
|
|
237
|
-
function matchSource(source, transactions, eventsByTx) {
|
|
238
|
-
const results = [];
|
|
239
|
-
const key = sourceKey(source);
|
|
240
|
-
for (const tx of transactions) {
|
|
241
|
-
if (source.type) {
|
|
242
|
-
if (!matchPattern(tx.type, source.type))
|
|
243
|
-
continue;
|
|
244
|
-
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
245
|
-
let matchedEvents = txEvents;
|
|
246
|
-
if (source.minAmount !== undefined) {
|
|
247
|
-
const amountEvents = matchedEvents.filter((e) => {
|
|
248
|
-
const data = e.data;
|
|
249
|
-
const rawAmount = data?.amount;
|
|
250
|
-
if (rawAmount === undefined)
|
|
251
|
-
return false;
|
|
252
|
-
const amount = BigInt(rawAmount);
|
|
253
|
-
return amount >= source.minAmount;
|
|
254
|
-
});
|
|
255
|
-
if (amountEvents.length === 0)
|
|
256
|
-
continue;
|
|
257
|
-
matchedEvents = amountEvents;
|
|
258
|
-
}
|
|
259
|
-
results.push({ tx, events: matchedEvents, sourceKey: key });
|
|
260
|
-
continue;
|
|
261
|
-
}
|
|
262
|
-
if (source.contract) {
|
|
263
|
-
const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
|
|
264
|
-
if (source.function && tx.function_name) {
|
|
265
|
-
if (!matchPattern(tx.function_name, source.function))
|
|
266
|
-
continue;
|
|
267
|
-
} else if (source.function && !tx.function_name) {
|
|
268
|
-
continue;
|
|
269
|
-
}
|
|
270
|
-
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
271
|
-
let matchedEvents = txEvents;
|
|
272
|
-
if (!txContractMatch) {
|
|
273
|
-
matchedEvents = txEvents.filter((e) => {
|
|
274
|
-
const data = e.data;
|
|
275
|
-
const contractIdentifier = data?.contract_identifier;
|
|
276
|
-
return contractIdentifier && matchPattern(contractIdentifier, source.contract);
|
|
277
|
-
});
|
|
278
|
-
if (matchedEvents.length === 0)
|
|
279
|
-
continue;
|
|
280
|
-
}
|
|
281
|
-
if (source.event) {
|
|
282
|
-
matchedEvents = matchedEvents.filter((e) => {
|
|
283
|
-
if (matchPattern(e.type, source.event))
|
|
284
|
-
return true;
|
|
285
|
-
const data = e.data;
|
|
286
|
-
const topic = data?.topic;
|
|
287
|
-
return topic ? matchPattern(topic, source.event) : false;
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
|
-
if (txContractMatch || matchedEvents.length > 0) {
|
|
291
|
-
results.push({ tx, events: matchedEvents, sourceKey: key });
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
return results;
|
|
296
|
-
}
|
|
297
|
-
function matchSources(sources, transactions, events) {
|
|
298
|
-
const eventsByTx = new Map;
|
|
299
|
-
for (const event of events) {
|
|
300
|
-
const list = eventsByTx.get(event.tx_id) ?? [];
|
|
301
|
-
list.push(event);
|
|
302
|
-
eventsByTx.set(event.tx_id, list);
|
|
303
|
-
}
|
|
304
|
-
const seen = new Set;
|
|
305
|
-
const results = [];
|
|
306
|
-
for (const source of sources) {
|
|
307
|
-
const matches = matchSource(source, transactions, eventsByTx);
|
|
308
|
-
for (const match of matches) {
|
|
309
|
-
const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
|
|
310
|
-
if (!seen.has(dedupeKey)) {
|
|
311
|
-
seen.add(dedupeKey);
|
|
312
|
-
results.push(match);
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
return results;
|
|
317
|
-
}
|
|
318
|
-
|
|
319
243
|
// src/runtime/clarity.ts
|
|
320
244
|
import { cvToJSON, deserializeCV } from "@secondlayer/stacks/clarity";
|
|
321
245
|
function decodeClarityValue(hex) {
|
|
@@ -348,8 +272,8 @@ function decodeFunctionArgs(args) {
|
|
|
348
272
|
}
|
|
349
273
|
|
|
350
274
|
// src/runtime/runner.ts
|
|
351
|
-
import { logger as logger2 } from "@secondlayer/shared/logger";
|
|
352
275
|
import { getErrorMessage } from "@secondlayer/shared";
|
|
276
|
+
import { logger as logger2 } from "@secondlayer/shared/logger";
|
|
353
277
|
var DEFAULT_ERROR_THRESHOLD = 50;
|
|
354
278
|
function resolveHandler(handlers, key) {
|
|
355
279
|
return handlers[key] ?? handlers["*"] ?? null;
|
|
@@ -361,7 +285,11 @@ async function runHandlers(subgraph, matched, ctx, opts) {
|
|
|
361
285
|
for (const { tx, events, sourceKey: sourceKey2 } of matched) {
|
|
362
286
|
const handler = resolveHandler(subgraph.handlers, sourceKey2);
|
|
363
287
|
if (!handler) {
|
|
364
|
-
logger2.warn("No handler found for source key", {
|
|
288
|
+
logger2.warn("No handler found for source key", {
|
|
289
|
+
subgraph: subgraph.name,
|
|
290
|
+
sourceKey: sourceKey2,
|
|
291
|
+
txId: tx.tx_id
|
|
292
|
+
});
|
|
365
293
|
continue;
|
|
366
294
|
}
|
|
367
295
|
ctx.setTx({
|
|
@@ -438,11 +366,103 @@ async function runHandlers(subgraph, matched, ctx, opts) {
|
|
|
438
366
|
return { processed, errors };
|
|
439
367
|
}
|
|
440
368
|
|
|
369
|
+
// src/runtime/source-matcher.ts
|
|
370
|
+
function matchPattern(value, pattern) {
|
|
371
|
+
if (!pattern.includes("*"))
|
|
372
|
+
return value === pattern;
|
|
373
|
+
const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
374
|
+
return new RegExp(`^${regex}$`).test(value);
|
|
375
|
+
}
|
|
376
|
+
function matchSource(source, transactions, eventsByTx) {
|
|
377
|
+
const results = [];
|
|
378
|
+
const key = sourceKey(source);
|
|
379
|
+
for (const tx of transactions) {
|
|
380
|
+
if (source.type) {
|
|
381
|
+
if (!matchPattern(tx.type, source.type))
|
|
382
|
+
continue;
|
|
383
|
+
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
384
|
+
let matchedEvents = txEvents;
|
|
385
|
+
if (source.minAmount !== undefined) {
|
|
386
|
+
const amountEvents = matchedEvents.filter((e) => {
|
|
387
|
+
const data = e.data;
|
|
388
|
+
const rawAmount = data?.amount;
|
|
389
|
+
if (rawAmount === undefined)
|
|
390
|
+
return false;
|
|
391
|
+
const amount = BigInt(rawAmount);
|
|
392
|
+
return amount >= source.minAmount;
|
|
393
|
+
});
|
|
394
|
+
if (amountEvents.length === 0)
|
|
395
|
+
continue;
|
|
396
|
+
matchedEvents = amountEvents;
|
|
397
|
+
}
|
|
398
|
+
results.push({ tx, events: matchedEvents, sourceKey: key });
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
if (source.contract) {
|
|
402
|
+
const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
|
|
403
|
+
if (source.function && tx.function_name) {
|
|
404
|
+
if (!matchPattern(tx.function_name, source.function))
|
|
405
|
+
continue;
|
|
406
|
+
} else if (source.function && !tx.function_name) {
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
410
|
+
let matchedEvents = txEvents;
|
|
411
|
+
if (!txContractMatch) {
|
|
412
|
+
matchedEvents = txEvents.filter((e) => {
|
|
413
|
+
const data = e.data;
|
|
414
|
+
const contractIdentifier = data?.contract_identifier;
|
|
415
|
+
return contractIdentifier && matchPattern(contractIdentifier, source.contract);
|
|
416
|
+
});
|
|
417
|
+
if (matchedEvents.length === 0)
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
if (source.event) {
|
|
421
|
+
matchedEvents = matchedEvents.filter((e) => {
|
|
422
|
+
if (matchPattern(e.type, source.event))
|
|
423
|
+
return true;
|
|
424
|
+
const data = e.data;
|
|
425
|
+
const topic = data?.topic;
|
|
426
|
+
return topic ? matchPattern(topic, source.event) : false;
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
if (txContractMatch || matchedEvents.length > 0) {
|
|
430
|
+
results.push({ tx, events: matchedEvents, sourceKey: key });
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
return results;
|
|
435
|
+
}
|
|
436
|
+
function matchSources(sources, transactions, events) {
|
|
437
|
+
const eventsByTx = new Map;
|
|
438
|
+
for (const event of events) {
|
|
439
|
+
const list = eventsByTx.get(event.tx_id) ?? [];
|
|
440
|
+
list.push(event);
|
|
441
|
+
eventsByTx.set(event.tx_id, list);
|
|
442
|
+
}
|
|
443
|
+
const seen = new Set;
|
|
444
|
+
const results = [];
|
|
445
|
+
for (const source of sources) {
|
|
446
|
+
const matches = matchSource(source, transactions, eventsByTx);
|
|
447
|
+
for (const match of matches) {
|
|
448
|
+
const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
|
|
449
|
+
if (!seen.has(dedupeKey)) {
|
|
450
|
+
seen.add(dedupeKey);
|
|
451
|
+
results.push(match);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return results;
|
|
456
|
+
}
|
|
457
|
+
|
|
441
458
|
// src/runtime/block-processor.ts
|
|
442
|
-
import { sql as sql2 } from "kysely";
|
|
443
459
|
import { getDb } from "@secondlayer/shared/db";
|
|
460
|
+
import {
|
|
461
|
+
recordSubgraphProcessed,
|
|
462
|
+
updateSubgraphStatus
|
|
463
|
+
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
444
464
|
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
445
|
-
import {
|
|
465
|
+
import { sql as sql2 } from "kysely";
|
|
446
466
|
|
|
447
467
|
// src/schema/utils.ts
|
|
448
468
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
@@ -467,12 +487,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
467
487
|
} else {
|
|
468
488
|
block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
|
|
469
489
|
if (!block) {
|
|
470
|
-
logger3.warn("Block not found for subgraph processing", {
|
|
490
|
+
logger3.warn("Block not found for subgraph processing", {
|
|
491
|
+
subgraph: subgraphName,
|
|
492
|
+
blockHeight
|
|
493
|
+
});
|
|
471
494
|
result.skipped = true;
|
|
472
495
|
return result;
|
|
473
496
|
}
|
|
474
497
|
if (!block.canonical) {
|
|
475
|
-
logger3.debug("Skipping non-canonical block", {
|
|
498
|
+
logger3.debug("Skipping non-canonical block", {
|
|
499
|
+
subgraph: subgraphName,
|
|
500
|
+
blockHeight
|
|
501
|
+
});
|
|
476
502
|
result.skipped = true;
|
|
477
503
|
return result;
|
|
478
504
|
}
|
|
@@ -541,7 +567,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
541
567
|
const { rows } = await sql2.raw(`SELECT COUNT(*) as count FROM "${schemaName}"."${table}"`).execute(db);
|
|
542
568
|
const count = Number(rows[0].count);
|
|
543
569
|
if (count >= 1e7) {
|
|
544
|
-
logger3.warn("Subgraph table exceeds 10M rows", {
|
|
570
|
+
logger3.warn("Subgraph table exceeds 10M rows", {
|
|
571
|
+
subgraph: subgraphName,
|
|
572
|
+
table,
|
|
573
|
+
count
|
|
574
|
+
});
|
|
545
575
|
}
|
|
546
576
|
}
|
|
547
577
|
} catch {}
|
|
@@ -550,17 +580,20 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
550
580
|
}
|
|
551
581
|
|
|
552
582
|
// src/runtime/reorg.ts
|
|
583
|
+
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
553
584
|
import { getDb as getDb2, getRawClient } from "@secondlayer/shared/db";
|
|
554
585
|
import { listSubgraphs } from "@secondlayer/shared/db/queries/subgraphs";
|
|
555
586
|
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
556
|
-
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
557
587
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
558
588
|
const db = getDb2();
|
|
559
589
|
const client = getRawClient();
|
|
560
590
|
const activeSubgraphs = (await listSubgraphs(db)).filter((v) => v.status === "active");
|
|
561
591
|
if (activeSubgraphs.length === 0)
|
|
562
592
|
return;
|
|
563
|
-
logger4.info("Propagating reorg to subgraphs", {
|
|
593
|
+
logger4.info("Propagating reorg to subgraphs", {
|
|
594
|
+
blockHeight,
|
|
595
|
+
subgraphCount: activeSubgraphs.length
|
|
596
|
+
});
|
|
564
597
|
for (const sg of activeSubgraphs) {
|
|
565
598
|
try {
|
|
566
599
|
const schema = sg.definition.schema;
|
|
@@ -570,10 +603,16 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
570
603
|
for (const tableName of Object.keys(schema)) {
|
|
571
604
|
await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" = $1`, [blockHeight]);
|
|
572
605
|
}
|
|
573
|
-
logger4.info("Subgraph reorg cleanup done", {
|
|
606
|
+
logger4.info("Subgraph reorg cleanup done", {
|
|
607
|
+
subgraph: sg.name,
|
|
608
|
+
blockHeight
|
|
609
|
+
});
|
|
574
610
|
const def = await loadSubgraphDef(sg.handler_path);
|
|
575
611
|
await processBlock(def, sg.name, blockHeight);
|
|
576
|
-
logger4.info("Subgraph reorg reprocessed", {
|
|
612
|
+
logger4.info("Subgraph reorg reprocessed", {
|
|
613
|
+
subgraph: sg.name,
|
|
614
|
+
blockHeight
|
|
615
|
+
});
|
|
577
616
|
} catch (err) {
|
|
578
617
|
logger4.error("Subgraph reorg handling failed", {
|
|
579
618
|
subgraph: sg.name,
|
|
@@ -587,5 +626,5 @@ export {
|
|
|
587
626
|
handleSubgraphReorg
|
|
588
627
|
};
|
|
589
628
|
|
|
590
|
-
//# debugId=
|
|
629
|
+
//# debugId=9A297975E56BD47664756E2164756E21
|
|
591
630
|
//# sourceMappingURL=reorg.js.map
|