@secondlayer/subgraphs 3.26.3 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -63
- package/dist/src/index.d.ts +8 -38
- package/dist/src/index.js +70 -174
- package/dist/src/index.js.map +8 -8
- package/dist/src/runtime/block-processor.d.ts +2 -13
- package/dist/src/runtime/block-processor.js +66 -136
- package/dist/src/runtime/block-processor.js.map +5 -5
- package/dist/src/runtime/catchup.d.ts +4 -4
- package/dist/src/runtime/catchup.js +66 -135
- package/dist/src/runtime/catchup.js.map +6 -6
- package/dist/src/runtime/context.d.ts +1 -9
- package/dist/src/runtime/context.js +2 -13
- package/dist/src/runtime/context.js.map +3 -3
- package/dist/src/runtime/processor.js +69 -145
- package/dist/src/runtime/processor.js.map +9 -9
- package/dist/src/runtime/reindex.d.ts +3 -4
- package/dist/src/runtime/reindex.js +66 -135
- package/dist/src/runtime/reindex.js.map +7 -7
- package/dist/src/runtime/reorg.js +69 -141
- package/dist/src/runtime/reorg.js.map +7 -7
- package/dist/src/runtime/runner.d.ts +1 -9
- package/dist/src/runtime/runner.js.map +2 -2
- package/dist/src/schema/index.d.ts +0 -8
- package/dist/src/schema/index.js +4 -38
- package/dist/src/schema/index.js.map +3 -3
- package/dist/src/service.js +69 -145
- package/dist/src/service.js.map +9 -9
- package/dist/src/testing/index.js +3 -14
- package/dist/src/testing/index.js.map +5 -5
- package/package.json +2 -2
|
@@ -323,16 +323,14 @@ class SubgraphContext {
|
|
|
323
323
|
pgSchemaName;
|
|
324
324
|
subgraphSchema;
|
|
325
325
|
ops = [];
|
|
326
|
-
byo;
|
|
327
326
|
journal;
|
|
328
327
|
_client;
|
|
329
|
-
constructor(db, pgSchemaName2, subgraphSchema, block, tx,
|
|
328
|
+
constructor(db, pgSchemaName2, subgraphSchema, block, tx, journal = false) {
|
|
330
329
|
this.db = db;
|
|
331
330
|
this.pgSchemaName = pgSchemaName2;
|
|
332
331
|
this.subgraphSchema = subgraphSchema;
|
|
333
332
|
this.block = block;
|
|
334
333
|
this._tx = tx;
|
|
335
|
-
this.byo = byo;
|
|
336
334
|
this.journal = journal;
|
|
337
335
|
}
|
|
338
336
|
get client() {
|
|
@@ -701,15 +699,6 @@ class SubgraphContext {
|
|
|
701
699
|
}
|
|
702
700
|
buildStatements(ops) {
|
|
703
701
|
const statements = [];
|
|
704
|
-
if (this.byo) {
|
|
705
|
-
const insertTables = new Set;
|
|
706
|
-
for (const op of ops)
|
|
707
|
-
if (op.kind === "insert")
|
|
708
|
-
insertTables.add(op.table);
|
|
709
|
-
for (const table of insertTables) {
|
|
710
|
-
statements.push(`DELETE FROM "${this.pgSchemaName}"."${table}" WHERE "_block_height" = ${this.block.height}`);
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
702
|
let currentBatch = null;
|
|
714
703
|
let currentBatchKey = "";
|
|
715
704
|
let incBatch = null;
|
|
@@ -1662,16 +1651,13 @@ function matchSources(sources, transactions, events, traitContracts = new Map, f
|
|
|
1662
1651
|
}
|
|
1663
1652
|
|
|
1664
1653
|
// src/runtime/block-processor.ts
|
|
1665
|
-
import { secretsKeyAvailable } from "@secondlayer/shared/crypto/secrets";
|
|
1666
1654
|
import { getTargetDb } from "@secondlayer/shared/db";
|
|
1667
1655
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
1668
1656
|
import { advanceOperationCursor } from "@secondlayer/shared/db/queries/subgraph-operations";
|
|
1669
1657
|
import {
|
|
1670
|
-
isByoSubgraph,
|
|
1671
1658
|
recordLiveError,
|
|
1672
1659
|
recordLiveProgress,
|
|
1673
1660
|
recordSubgraphProcessed,
|
|
1674
|
-
resolveSubgraphDb,
|
|
1675
1661
|
rewindLiveProgress,
|
|
1676
1662
|
updateSubgraphStatus
|
|
1677
1663
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
@@ -2600,26 +2586,14 @@ async function refreshMatcher(db) {
|
|
|
2600
2586
|
|
|
2601
2587
|
// src/runtime/block-processor.ts
|
|
2602
2588
|
var routeCache = new Map;
|
|
2603
|
-
|
|
2604
|
-
class ByoKeyUnavailableError extends Error {
|
|
2605
|
-
constructor(subgraphName) {
|
|
2606
|
-
super(`subgraph ${subgraphName} is BYO (database_url_enc set) but this process has no SECONDLAYER_SECRETS_KEY — the subgraph processor deliberately does not hold it (f072). Route BYO subgraphs to a process that does, or unset BYO.`);
|
|
2607
|
-
this.name = "ByoKeyUnavailableError";
|
|
2608
|
-
}
|
|
2609
|
-
}
|
|
2610
2589
|
async function resolveRoute(subgraphName, targetDb) {
|
|
2611
2590
|
const cached = routeCache.get(subgraphName);
|
|
2612
2591
|
if (cached)
|
|
2613
2592
|
return cached;
|
|
2614
2593
|
const row = await targetDb.selectFrom("subgraphs").selectAll().where("name", "=", subgraphName).executeTakeFirst();
|
|
2615
|
-
const byo = row ? isByoSubgraph(row) : false;
|
|
2616
|
-
if (row && byo && !secretsKeyAvailable()) {
|
|
2617
|
-
throw new ByoKeyUnavailableError(subgraphName);
|
|
2618
|
-
}
|
|
2619
2594
|
const route = {
|
|
2620
2595
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2621
|
-
dataDb:
|
|
2622
|
-
byo,
|
|
2596
|
+
dataDb: targetDb,
|
|
2623
2597
|
sandboxWorkers: row?.sandbox_workers === true,
|
|
2624
2598
|
handlerCode: row?.handler_code ?? null,
|
|
2625
2599
|
version: row?.version ?? ""
|
|
@@ -2827,7 +2801,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2827
2801
|
await rewindLiveProgress(tx, subgraphName, blockHeight);
|
|
2828
2802
|
} else {
|
|
2829
2803
|
const advanced = await recordLiveProgress(tx, subgraphName, blockHeight);
|
|
2830
|
-
if (!advanced && flushedWrites
|
|
2804
|
+
if (!advanced && flushedWrites) {
|
|
2831
2805
|
throw new LiveCursorRaceLostError(subgraphName, blockHeight);
|
|
2832
2806
|
}
|
|
2833
2807
|
}
|
|
@@ -2836,122 +2810,79 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2836
2810
|
await recordSubgraphProcessed(tx, subgraphName, rr.processed, rr.errors, lastError);
|
|
2837
2811
|
}
|
|
2838
2812
|
};
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
const
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2813
|
+
try {
|
|
2814
|
+
await targetDb.transaction().execute(async (tx) => {
|
|
2815
|
+
const opMode = opCursorMode(opts);
|
|
2816
|
+
const sm = statusMode(opts);
|
|
2817
|
+
const rewind = isReorgRewind(opts);
|
|
2818
|
+
if (sm) {
|
|
2819
|
+
const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
|
|
2820
|
+
if (row && Number(row.last_processed_block) >= blockHeight) {
|
|
2821
|
+
result.skipped = true;
|
|
2822
|
+
return;
|
|
2823
|
+
}
|
|
2824
|
+
} else if (opMode) {
|
|
2825
|
+
const row = await tx.selectFrom("subgraph_operations").select("cursor_block").where("id", "=", opMode.operationId).executeTakeFirst();
|
|
2826
|
+
if (row?.cursor_block != null && Number(row.cursor_block) >= blockHeight) {
|
|
2827
|
+
result.skipped = true;
|
|
2828
|
+
return;
|
|
2829
|
+
}
|
|
2830
|
+
} else if (!rewind) {
|
|
2831
|
+
const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
|
|
2832
|
+
if (row && Number(row.last_processed_block) >= blockHeight) {
|
|
2833
|
+
result.skipped = true;
|
|
2834
|
+
return;
|
|
2835
|
+
}
|
|
2852
2836
|
}
|
|
2853
|
-
|
|
2854
|
-
let runResult = { processed: 0, errors: 0 };
|
|
2855
|
-
let manifest;
|
|
2856
|
-
await route.dataDb.transaction().execute(async (tx) => {
|
|
2857
|
-
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, true, journalEnabled(opts));
|
|
2837
|
+
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, journalEnabled(opts));
|
|
2858
2838
|
const handlerStart = performance.now();
|
|
2859
|
-
runResult =
|
|
2839
|
+
const runResult = sandboxEnabled({
|
|
2840
|
+
sandbox_workers: route.sandboxWorkers
|
|
2841
|
+
}) ? await runHandlersSandboxed({
|
|
2842
|
+
subgraphName,
|
|
2843
|
+
version: route.version,
|
|
2844
|
+
handlerCode: route.handlerCode,
|
|
2845
|
+
hostCtx: ctx,
|
|
2846
|
+
block: blockMeta,
|
|
2847
|
+
matched
|
|
2848
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2860
2849
|
handlerMs = performance.now() - handlerStart;
|
|
2850
|
+
result.processed = runResult.processed;
|
|
2851
|
+
result.errors = runResult.errors;
|
|
2852
|
+
let flushedWrites = false;
|
|
2861
2853
|
if (ctx.pendingOps > 0) {
|
|
2862
2854
|
const flushStart = performance.now();
|
|
2863
|
-
manifest = await ctx.flush();
|
|
2855
|
+
const manifest = await ctx.flush();
|
|
2856
|
+
flushedWrites = manifest.count > 0;
|
|
2857
|
+
if (manifest.count > 0) {
|
|
2858
|
+
await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
|
|
2859
|
+
}
|
|
2864
2860
|
flushMs = performance.now() - flushStart;
|
|
2865
2861
|
}
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
}
|
|
2874
|
-
const byoSm = statusMode(opts);
|
|
2875
|
-
const byoOm = opCursorMode(opts);
|
|
2876
|
-
if (byoSm && manifest && manifest.count > 0) {
|
|
2877
|
-
await updateSubgraphStatus(tx, subgraphName, byoSm.status, blockHeight);
|
|
2878
|
-
} else if (byoOm && manifest && manifest.count > 0) {
|
|
2879
|
-
await advanceOperationCursor(tx, byoOm.operationId, blockHeight);
|
|
2862
|
+
if (sm && flushedWrites) {
|
|
2863
|
+
await updateSubgraphStatus(tx, subgraphName, sm.status, blockHeight);
|
|
2864
|
+
} else if (opMode && flushedWrites) {
|
|
2865
|
+
const advanced = await advanceOperationCursor(tx, opMode.operationId, blockHeight);
|
|
2866
|
+
if (!advanced) {
|
|
2867
|
+
throw new CursorRaceLostError(opMode.operationId, blockHeight);
|
|
2868
|
+
}
|
|
2880
2869
|
}
|
|
2881
|
-
await applyProgress(tx, runResult,
|
|
2870
|
+
await applyProgress(tx, runResult, flushedWrites);
|
|
2882
2871
|
});
|
|
2883
|
-
}
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
if (sm) {
|
|
2890
|
-
const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
|
|
2891
|
-
if (row && Number(row.last_processed_block) >= blockHeight) {
|
|
2892
|
-
result.skipped = true;
|
|
2893
|
-
return;
|
|
2894
|
-
}
|
|
2895
|
-
} else if (opMode) {
|
|
2896
|
-
const row = await tx.selectFrom("subgraph_operations").select("cursor_block").where("id", "=", opMode.operationId).executeTakeFirst();
|
|
2897
|
-
if (row?.cursor_block != null && Number(row.cursor_block) >= blockHeight) {
|
|
2898
|
-
result.skipped = true;
|
|
2899
|
-
return;
|
|
2900
|
-
}
|
|
2901
|
-
} else if (!rewind) {
|
|
2902
|
-
const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
|
|
2903
|
-
if (row && Number(row.last_processed_block) >= blockHeight) {
|
|
2904
|
-
result.skipped = true;
|
|
2905
|
-
return;
|
|
2906
|
-
}
|
|
2907
|
-
}
|
|
2908
|
-
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2909
|
-
const handlerStart = performance.now();
|
|
2910
|
-
const runResult = sandboxEnabled({
|
|
2911
|
-
sandbox_workers: route.sandboxWorkers
|
|
2912
|
-
}) ? await runHandlersSandboxed({
|
|
2913
|
-
subgraphName,
|
|
2914
|
-
version: route.version,
|
|
2915
|
-
handlerCode: route.handlerCode,
|
|
2916
|
-
hostCtx: ctx,
|
|
2917
|
-
block: blockMeta,
|
|
2918
|
-
matched
|
|
2919
|
-
}) : await runHandlers(subgraph, matched, ctx);
|
|
2920
|
-
handlerMs = performance.now() - handlerStart;
|
|
2921
|
-
result.processed = runResult.processed;
|
|
2922
|
-
result.errors = runResult.errors;
|
|
2923
|
-
let flushedWrites = false;
|
|
2924
|
-
if (ctx.pendingOps > 0) {
|
|
2925
|
-
const flushStart = performance.now();
|
|
2926
|
-
const manifest = await ctx.flush();
|
|
2927
|
-
flushedWrites = manifest.count > 0;
|
|
2928
|
-
if (manifest.count > 0) {
|
|
2929
|
-
await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
|
|
2930
|
-
}
|
|
2931
|
-
flushMs = performance.now() - flushStart;
|
|
2932
|
-
}
|
|
2933
|
-
if (sm && flushedWrites) {
|
|
2934
|
-
await updateSubgraphStatus(tx, subgraphName, sm.status, blockHeight);
|
|
2935
|
-
} else if (opMode && flushedWrites) {
|
|
2936
|
-
const advanced = await advanceOperationCursor(tx, opMode.operationId, blockHeight);
|
|
2937
|
-
if (!advanced) {
|
|
2938
|
-
throw new CursorRaceLostError(opMode.operationId, blockHeight);
|
|
2939
|
-
}
|
|
2940
|
-
}
|
|
2941
|
-
await applyProgress(tx, runResult, flushedWrites);
|
|
2872
|
+
} catch (err) {
|
|
2873
|
+
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2874
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2875
|
+
subgraph: subgraphName,
|
|
2876
|
+
blockHeight,
|
|
2877
|
+
error: err.message
|
|
2942
2878
|
});
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
logger7.warn("cursor race lost — block already covered", {
|
|
2946
|
-
subgraph: subgraphName,
|
|
2947
|
-
blockHeight,
|
|
2948
|
-
error: err.message
|
|
2949
|
-
});
|
|
2950
|
-
result.skipped = true;
|
|
2951
|
-
return result;
|
|
2952
|
-
}
|
|
2953
|
-
throw err;
|
|
2879
|
+
result.skipped = true;
|
|
2880
|
+
return result;
|
|
2954
2881
|
}
|
|
2882
|
+
throw err;
|
|
2883
|
+
}
|
|
2884
|
+
if (!result.skipped) {
|
|
2885
|
+
await persistFactoryDiscoveries(schemaName, route.dataDb, blockHeight, discovered);
|
|
2955
2886
|
}
|
|
2956
2887
|
const totalMs = performance.now() - blockStart;
|
|
2957
2888
|
if (!result.skipped) {
|
|
@@ -3322,11 +3253,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3322
3253
|
|
|
3323
3254
|
// src/runtime/reorg.ts
|
|
3324
3255
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
3325
|
-
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3326
|
-
import {
|
|
3327
|
-
listSubgraphs,
|
|
3328
|
-
resolveSubgraphRawClient
|
|
3329
|
-
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3256
|
+
import { getRawClient, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3257
|
+
import { listSubgraphs } from "@secondlayer/shared/db/queries/subgraphs";
|
|
3330
3258
|
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
3331
3259
|
import { sql as sql5 } from "kysely";
|
|
3332
3260
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
@@ -3362,7 +3290,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
3362
3290
|
async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
3363
3291
|
const targetDb = getTargetDb3();
|
|
3364
3292
|
const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
|
|
3365
|
-
const client =
|
|
3293
|
+
const client = getRawClient("target");
|
|
3366
3294
|
const tableNames = Object.keys(schema);
|
|
3367
3295
|
const revertedByTable = {};
|
|
3368
3296
|
for (const tableName of tableNames) {
|
|
@@ -3457,5 +3385,5 @@ export {
|
|
|
3457
3385
|
handleSubgraphReorg
|
|
3458
3386
|
};
|
|
3459
3387
|
|
|
3460
|
-
//# debugId=
|
|
3388
|
+
//# debugId=4BA27FCD8C456A6C64756E2164756E21
|
|
3461
3389
|
//# sourceMappingURL=reorg.js.map
|