@secondlayer/subgraphs 3.17.0 → 3.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +87 -31
- package/dist/src/index.js.map +5 -4
- package/dist/src/runtime/processor.js +137 -81
- package/dist/src/runtime/processor.js.map +5 -4
- package/dist/src/runtime/reindex.js +87 -31
- package/dist/src/runtime/reindex.js.map +5 -4
- package/dist/src/runtime/replay.d.ts +40 -1
- package/dist/src/runtime/replay.js +143 -7
- package/dist/src/runtime/replay.js.map +3 -3
- package/dist/src/service.js +140 -84
- package/dist/src/service.js.map +5 -4
- package/package.json +5 -3
|
@@ -2302,7 +2302,7 @@ class StatsAccumulator {
|
|
|
2302
2302
|
}
|
|
2303
2303
|
|
|
2304
2304
|
// src/runtime/reindex.ts
|
|
2305
|
-
import { getErrorMessage as
|
|
2305
|
+
import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
|
|
2306
2306
|
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
2307
2307
|
import {
|
|
2308
2308
|
recordGapBatch,
|
|
@@ -2316,7 +2316,58 @@ import {
|
|
|
2316
2316
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2317
2317
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2318
2318
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2319
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
2320
|
+
|
|
2321
|
+
// src/runtime/reindex-notify.ts
|
|
2322
|
+
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2319
2323
|
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2324
|
+
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2325
|
+
try {
|
|
2326
|
+
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
2327
|
+
if (!subgraph)
|
|
2328
|
+
return;
|
|
2329
|
+
const account = await db.selectFrom("accounts").select(["email", "notify_reindex_complete"]).where("id", "=", subgraph.account_id).executeTakeFirst();
|
|
2330
|
+
if (!account?.email || !account.notify_reindex_complete)
|
|
2331
|
+
return;
|
|
2332
|
+
const resendKey = process.env.RESEND_API_KEY;
|
|
2333
|
+
if (!resendKey) {
|
|
2334
|
+
logger6.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2335
|
+
subgraph: subgraphName
|
|
2336
|
+
});
|
|
2337
|
+
return;
|
|
2338
|
+
}
|
|
2339
|
+
const from = process.env.EMAIL_FROM ?? "Secondlayer <noreply@secondlayer.tools>";
|
|
2340
|
+
const body = stats.errors > 0 ? `Your subgraph "${subgraphName}" finished reindexing — ${stats.blocks.toLocaleString()} blocks, ${stats.events.toLocaleString()} events, ${stats.errors.toLocaleString()} errors. Check the dashboard for details.` : `Your subgraph "${subgraphName}" finished reindexing — ${stats.blocks.toLocaleString()} blocks, ${stats.events.toLocaleString()} events processed, no errors. It's live now.`;
|
|
2341
|
+
const res = await fetch("https://api.resend.com/emails", {
|
|
2342
|
+
method: "POST",
|
|
2343
|
+
headers: {
|
|
2344
|
+
Authorization: `Bearer ${resendKey}`,
|
|
2345
|
+
"Content-Type": "application/json"
|
|
2346
|
+
},
|
|
2347
|
+
body: JSON.stringify({
|
|
2348
|
+
from,
|
|
2349
|
+
to: [account.email],
|
|
2350
|
+
subject: `Reindex complete: ${subgraphName}`,
|
|
2351
|
+
text: body
|
|
2352
|
+
})
|
|
2353
|
+
});
|
|
2354
|
+
if (!res.ok) {
|
|
2355
|
+
const text = await res.text().catch(() => "");
|
|
2356
|
+
logger6.warn("reindex-complete email failed", {
|
|
2357
|
+
subgraph: subgraphName,
|
|
2358
|
+
status: res.status,
|
|
2359
|
+
body: text.slice(0, 200)
|
|
2360
|
+
});
|
|
2361
|
+
}
|
|
2362
|
+
} catch (err) {
|
|
2363
|
+
logger6.warn("reindex-complete email threw", {
|
|
2364
|
+
subgraph: subgraphName,
|
|
2365
|
+
error: getErrorMessage2(err)
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
// src/runtime/reindex.ts
|
|
2320
2371
|
var LOG_INTERVAL = 1000;
|
|
2321
2372
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
2322
2373
|
var PROGRESS_FLUSH_INTERVAL_MS = 5000;
|
|
@@ -2421,7 +2472,7 @@ async function processBlockRange(def, opts) {
|
|
|
2421
2472
|
while (currentHeight <= toBlock) {
|
|
2422
2473
|
if (opts.signal?.aborted) {
|
|
2423
2474
|
aborted = true;
|
|
2424
|
-
|
|
2475
|
+
logger7.info("Block processing aborted", {
|
|
2425
2476
|
subgraph: subgraphName,
|
|
2426
2477
|
currentBlock: currentHeight,
|
|
2427
2478
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2469,8 +2520,8 @@ async function processBlockRange(def, opts) {
|
|
|
2469
2520
|
preloaded: blockData
|
|
2470
2521
|
});
|
|
2471
2522
|
} catch (err) {
|
|
2472
|
-
const errorMsg =
|
|
2473
|
-
|
|
2523
|
+
const errorMsg = getErrorMessage3(err);
|
|
2524
|
+
logger7.error("Block processing failed persistently", {
|
|
2474
2525
|
subgraph: subgraphName,
|
|
2475
2526
|
blockHeight: height,
|
|
2476
2527
|
error: errorMsg
|
|
@@ -2516,7 +2567,7 @@ async function processBlockRange(def, opts) {
|
|
|
2516
2567
|
lastProgressFlushAt = now;
|
|
2517
2568
|
}
|
|
2518
2569
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2519
|
-
|
|
2570
|
+
logger7.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2520
2571
|
subgraph: subgraphName,
|
|
2521
2572
|
processed: blocksProcessed,
|
|
2522
2573
|
total: totalBlocks,
|
|
@@ -2536,7 +2587,7 @@ async function processBlockRange(def, opts) {
|
|
|
2536
2587
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2537
2588
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2538
2589
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2539
|
-
|
|
2590
|
+
logger7.warn("Failed to record subgraph gaps", {
|
|
2540
2591
|
subgraph: subgraphName,
|
|
2541
2592
|
error: err instanceof Error ? err.message : String(err)
|
|
2542
2593
|
});
|
|
@@ -2553,7 +2604,7 @@ async function processBlockRange(def, opts) {
|
|
|
2553
2604
|
} else {
|
|
2554
2605
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2555
2606
|
}
|
|
2556
|
-
|
|
2607
|
+
logger7.info("Sparse skip", {
|
|
2557
2608
|
subgraph: subgraphName,
|
|
2558
2609
|
from: batchEnd + 1,
|
|
2559
2610
|
to: jumpTo - 1,
|
|
@@ -2593,11 +2644,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
2593
2644
|
const subgraphName = def.name;
|
|
2594
2645
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
2595
2646
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
2596
|
-
|
|
2647
|
+
logger7.info("Reindex starting", { subgraph: subgraphName });
|
|
2597
2648
|
try {
|
|
2598
2649
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
2599
2650
|
if (fromBlock > toBlock) {
|
|
2600
|
-
|
|
2651
|
+
logger7.info("No blocks to reindex", {
|
|
2601
2652
|
subgraph: subgraphName,
|
|
2602
2653
|
fromBlock,
|
|
2603
2654
|
toBlock
|
|
@@ -2611,7 +2662,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
2611
2662
|
await client.unsafe(stmt);
|
|
2612
2663
|
}
|
|
2613
2664
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
2614
|
-
|
|
2665
|
+
logger7.info("Schema recreated for reindex", {
|
|
2615
2666
|
subgraph: subgraphName,
|
|
2616
2667
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
2617
2668
|
});
|
|
@@ -2625,7 +2676,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
2625
2676
|
last_error_at: null,
|
|
2626
2677
|
updated_at: new Date
|
|
2627
2678
|
}).where("name", "=", subgraphName).execute();
|
|
2628
|
-
|
|
2679
|
+
logger7.info("Reindexing blocks", {
|
|
2629
2680
|
subgraph: subgraphName,
|
|
2630
2681
|
fromBlock,
|
|
2631
2682
|
toBlock,
|
|
@@ -2647,9 +2698,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
2647
2698
|
if (reason === "user-cancelled") {
|
|
2648
2699
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
2649
2700
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2650
|
-
|
|
2701
|
+
logger7.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
2651
2702
|
} else {
|
|
2652
|
-
|
|
2703
|
+
logger7.info("Reindex interrupted by shutdown, will resume", {
|
|
2653
2704
|
subgraph: subgraphName
|
|
2654
2705
|
});
|
|
2655
2706
|
}
|
|
@@ -2657,17 +2708,22 @@ async function reindexSubgraph(def, opts) {
|
|
|
2657
2708
|
}
|
|
2658
2709
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2659
2710
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2660
|
-
|
|
2711
|
+
logger7.info("Reindex complete", {
|
|
2661
2712
|
subgraph: subgraphName,
|
|
2662
2713
|
blocks: result.blocksProcessed,
|
|
2663
2714
|
events: result.totalEventsProcessed,
|
|
2664
2715
|
errors: result.totalErrors
|
|
2665
2716
|
});
|
|
2717
|
+
notifyReindexComplete(targetDb, subgraphName, {
|
|
2718
|
+
blocks: result.blocksProcessed,
|
|
2719
|
+
events: result.totalEventsProcessed,
|
|
2720
|
+
errors: result.totalErrors
|
|
2721
|
+
});
|
|
2666
2722
|
return { processed: result.blocksProcessed };
|
|
2667
2723
|
} catch (err) {
|
|
2668
|
-
|
|
2724
|
+
logger7.error("Reindex failed", {
|
|
2669
2725
|
subgraph: subgraphName,
|
|
2670
|
-
error:
|
|
2726
|
+
error: getErrorMessage3(err)
|
|
2671
2727
|
});
|
|
2672
2728
|
await updateSubgraphStatus2(targetDb, subgraphName, "error");
|
|
2673
2729
|
throw err;
|
|
@@ -2687,7 +2743,7 @@ async function resumeReindex(def, opts) {
|
|
|
2687
2743
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
2688
2744
|
const toBlock = Number(row.reindex_to_block);
|
|
2689
2745
|
if (fromBlock == null) {
|
|
2690
|
-
|
|
2746
|
+
logger7.info("No reindex metadata, starting fresh reindex", {
|
|
2691
2747
|
subgraph: subgraphName
|
|
2692
2748
|
});
|
|
2693
2749
|
return reindexSubgraph(def, {
|
|
@@ -2696,12 +2752,12 @@ async function resumeReindex(def, opts) {
|
|
|
2696
2752
|
});
|
|
2697
2753
|
}
|
|
2698
2754
|
if (fromBlock > toBlock) {
|
|
2699
|
-
|
|
2755
|
+
logger7.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
2700
2756
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2701
2757
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2702
2758
|
return { processed: 0 };
|
|
2703
2759
|
}
|
|
2704
|
-
|
|
2760
|
+
logger7.info("Resuming reindex", {
|
|
2705
2761
|
subgraph: subgraphName,
|
|
2706
2762
|
fromBlock,
|
|
2707
2763
|
toBlock,
|
|
@@ -2723,9 +2779,9 @@ async function resumeReindex(def, opts) {
|
|
|
2723
2779
|
if (reason === "user-cancelled") {
|
|
2724
2780
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
2725
2781
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2726
|
-
|
|
2782
|
+
logger7.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
2727
2783
|
} else {
|
|
2728
|
-
|
|
2784
|
+
logger7.info("Resume interrupted by shutdown, will resume again", {
|
|
2729
2785
|
subgraph: subgraphName
|
|
2730
2786
|
});
|
|
2731
2787
|
}
|
|
@@ -2733,15 +2789,15 @@ async function resumeReindex(def, opts) {
|
|
|
2733
2789
|
}
|
|
2734
2790
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2735
2791
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2736
|
-
|
|
2792
|
+
logger7.info("Resumed reindex complete", {
|
|
2737
2793
|
subgraph: subgraphName,
|
|
2738
2794
|
blocks: result.blocksProcessed
|
|
2739
2795
|
});
|
|
2740
2796
|
return { processed: result.blocksProcessed };
|
|
2741
2797
|
} catch (err) {
|
|
2742
|
-
|
|
2798
|
+
logger7.error("Resumed reindex failed", {
|
|
2743
2799
|
subgraph: subgraphName,
|
|
2744
|
-
error:
|
|
2800
|
+
error: getErrorMessage3(err)
|
|
2745
2801
|
});
|
|
2746
2802
|
await updateSubgraphStatus2(targetDb, subgraphName, "error");
|
|
2747
2803
|
throw err;
|
|
@@ -2750,7 +2806,7 @@ async function resumeReindex(def, opts) {
|
|
|
2750
2806
|
async function backfillSubgraph(def, opts) {
|
|
2751
2807
|
const targetDb = getTargetDb2();
|
|
2752
2808
|
const subgraphName = def.name;
|
|
2753
|
-
|
|
2809
|
+
logger7.info("Backfill starting", {
|
|
2754
2810
|
subgraph: subgraphName,
|
|
2755
2811
|
from: opts.fromBlock,
|
|
2756
2812
|
to: opts.toBlock
|
|
@@ -2768,17 +2824,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
2768
2824
|
signal: opts.signal
|
|
2769
2825
|
});
|
|
2770
2826
|
if (result.aborted) {
|
|
2771
|
-
|
|
2827
|
+
logger7.info("Backfill aborted", { subgraph: subgraphName });
|
|
2772
2828
|
return { processed: result.blocksProcessed };
|
|
2773
2829
|
}
|
|
2774
2830
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
2775
2831
|
if (resolved > 0) {
|
|
2776
|
-
|
|
2832
|
+
logger7.info("Resolved subgraph gaps via backfill", {
|
|
2777
2833
|
subgraph: subgraphName,
|
|
2778
2834
|
resolved
|
|
2779
2835
|
});
|
|
2780
2836
|
}
|
|
2781
|
-
|
|
2837
|
+
logger7.info("Backfill complete", {
|
|
2782
2838
|
subgraph: subgraphName,
|
|
2783
2839
|
blocks: result.blocksProcessed,
|
|
2784
2840
|
events: result.totalEventsProcessed,
|
|
@@ -2786,9 +2842,9 @@ async function backfillSubgraph(def, opts) {
|
|
|
2786
2842
|
});
|
|
2787
2843
|
return { processed: result.blocksProcessed };
|
|
2788
2844
|
} catch (err) {
|
|
2789
|
-
|
|
2845
|
+
logger7.error("Backfill failed", {
|
|
2790
2846
|
subgraph: subgraphName,
|
|
2791
|
-
error:
|
|
2847
|
+
error: getErrorMessage3(err)
|
|
2792
2848
|
});
|
|
2793
2849
|
throw err;
|
|
2794
2850
|
}
|
|
@@ -2797,7 +2853,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
2797
2853
|
// src/runtime/catchup.ts
|
|
2798
2854
|
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
2799
2855
|
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
2800
|
-
import { logger as
|
|
2856
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2801
2857
|
var LOG_INTERVAL2 = 1000;
|
|
2802
2858
|
var STANDARD_CATCHUP_BATCH_CONFIG = {
|
|
2803
2859
|
defaultBatchSize: 500,
|
|
@@ -2858,7 +2914,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2858
2914
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
2859
2915
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
2860
2916
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
2861
|
-
|
|
2917
|
+
logger8.info("Subgraph catch-up starting", {
|
|
2862
2918
|
subgraph: subgraphName,
|
|
2863
2919
|
from: startBlock,
|
|
2864
2920
|
to: chainTip,
|
|
@@ -2874,7 +2930,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2874
2930
|
while (currentHeight <= chainTip) {
|
|
2875
2931
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
2876
2932
|
if (!currentRow || currentRow.status !== "active") {
|
|
2877
|
-
|
|
2933
|
+
logger8.info("Subgraph status changed, stopping catch-up", {
|
|
2878
2934
|
subgraph: subgraphName,
|
|
2879
2935
|
status: currentRow?.status ?? "deleted"
|
|
2880
2936
|
});
|
|
@@ -2903,7 +2959,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2903
2959
|
blockData = (await source.loadBlockRange(height, height)).get(height);
|
|
2904
2960
|
}
|
|
2905
2961
|
if (!blockData) {
|
|
2906
|
-
|
|
2962
|
+
logger8.warn("Block missing during catch-up, deferring to next tick", {
|
|
2907
2963
|
subgraph: subgraphName,
|
|
2908
2964
|
blockHeight: height
|
|
2909
2965
|
});
|
|
@@ -2917,7 +2973,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2917
2973
|
});
|
|
2918
2974
|
} catch (err) {
|
|
2919
2975
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
2920
|
-
|
|
2976
|
+
logger8.error("Block processing failed persistently during catch-up", {
|
|
2921
2977
|
subgraph: subgraphName,
|
|
2922
2978
|
blockHeight: height,
|
|
2923
2979
|
error: errorMsg
|
|
@@ -2936,7 +2992,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2936
2992
|
}
|
|
2937
2993
|
}
|
|
2938
2994
|
if (processed % LOG_INTERVAL2 === 0) {
|
|
2939
|
-
|
|
2995
|
+
logger8.info("Subgraph catch-up progress", {
|
|
2940
2996
|
subgraph: subgraphName,
|
|
2941
2997
|
processed,
|
|
2942
2998
|
total: totalBlocks,
|
|
@@ -2952,7 +3008,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2952
3008
|
currentHeight = batchEnd + 1;
|
|
2953
3009
|
}
|
|
2954
3010
|
await stats.flush(targetDb);
|
|
2955
|
-
|
|
3011
|
+
logger8.info("Subgraph catch-up complete", {
|
|
2956
3012
|
subgraph: subgraphName,
|
|
2957
3013
|
processed
|
|
2958
3014
|
});
|
|
@@ -2963,19 +3019,19 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2963
3019
|
}
|
|
2964
3020
|
|
|
2965
3021
|
// src/runtime/reorg.ts
|
|
2966
|
-
import { getErrorMessage as
|
|
3022
|
+
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
2967
3023
|
import { getTargetDb as getTargetDb4 } from "@secondlayer/shared/db";
|
|
2968
3024
|
import {
|
|
2969
3025
|
listSubgraphs,
|
|
2970
3026
|
resolveSubgraphRawClient
|
|
2971
3027
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2972
|
-
import { logger as
|
|
3028
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2973
3029
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
2974
3030
|
const targetDb = getTargetDb4();
|
|
2975
3031
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
2976
3032
|
if (activeSubgraphs.length === 0)
|
|
2977
3033
|
return;
|
|
2978
|
-
|
|
3034
|
+
logger9.info("Propagating reorg to subgraphs", {
|
|
2979
3035
|
blockHeight,
|
|
2980
3036
|
subgraphCount: activeSubgraphs.length
|
|
2981
3037
|
});
|
|
@@ -3012,7 +3068,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
3012
3068
|
await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
|
|
3013
3069
|
});
|
|
3014
3070
|
} else {
|
|
3015
|
-
|
|
3071
|
+
logger9.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
|
|
3016
3072
|
for (const tableName of tableNames) {
|
|
3017
3073
|
await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
|
|
3018
3074
|
}
|
|
@@ -3044,31 +3100,31 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
3044
3100
|
}).as("row_pk"),
|
|
3045
3101
|
eb2.val("pending").as("status")
|
|
3046
3102
|
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.column("dedup_key").doNothing()).execute().catch((err) => {
|
|
3047
|
-
|
|
3103
|
+
logger9.error("Failed to emit revert event for subscriptions", {
|
|
3048
3104
|
event: "reorg_revert_emit_dropped",
|
|
3049
3105
|
subgraph: sg.name,
|
|
3050
3106
|
table: tableName,
|
|
3051
3107
|
blockHeight,
|
|
3052
|
-
error:
|
|
3108
|
+
error: getErrorMessage4(err)
|
|
3053
3109
|
});
|
|
3054
3110
|
});
|
|
3055
3111
|
}
|
|
3056
|
-
|
|
3112
|
+
logger9.info("Subgraph reorg cleanup done", {
|
|
3057
3113
|
subgraph: sg.name,
|
|
3058
3114
|
blockHeight,
|
|
3059
3115
|
tablesAffected: Object.keys(revertedByTable).length
|
|
3060
3116
|
});
|
|
3061
3117
|
const def = await loadSubgraphDef(sg);
|
|
3062
3118
|
await processBlock(def, sg.name, blockHeight);
|
|
3063
|
-
|
|
3119
|
+
logger9.info("Subgraph reorg reprocessed", {
|
|
3064
3120
|
subgraph: sg.name,
|
|
3065
3121
|
blockHeight
|
|
3066
3122
|
});
|
|
3067
3123
|
} catch (err) {
|
|
3068
|
-
|
|
3124
|
+
logger9.error("Subgraph reorg handling failed", {
|
|
3069
3125
|
subgraph: sg.name,
|
|
3070
3126
|
blockHeight,
|
|
3071
|
-
error:
|
|
3127
|
+
error: getErrorMessage4(err)
|
|
3072
3128
|
});
|
|
3073
3129
|
}
|
|
3074
3130
|
}
|
|
@@ -3079,7 +3135,7 @@ import { randomUUID } from "node:crypto";
|
|
|
3079
3135
|
import { hostname } from "node:os";
|
|
3080
3136
|
import { resolve } from "node:path";
|
|
3081
3137
|
import { pathToFileURL } from "node:url";
|
|
3082
|
-
import { getErrorMessage as
|
|
3138
|
+
import { getErrorMessage as getErrorMessage6 } from "@secondlayer/shared";
|
|
3083
3139
|
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
3084
3140
|
import {
|
|
3085
3141
|
cancelSubgraphOperation,
|
|
@@ -3098,7 +3154,7 @@ import {
|
|
|
3098
3154
|
pgSchemaName as pgSchemaName2,
|
|
3099
3155
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
3100
3156
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3101
|
-
import { logger as
|
|
3157
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
3102
3158
|
import {
|
|
3103
3159
|
listen,
|
|
3104
3160
|
sourceListenerUrl,
|
|
@@ -3131,16 +3187,16 @@ function startCatchUpLeader(opts = {}) {
|
|
|
3131
3187
|
}
|
|
3132
3188
|
|
|
3133
3189
|
// src/runtime/streams-reorg-poll.ts
|
|
3134
|
-
import { getErrorMessage as
|
|
3190
|
+
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
3135
3191
|
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3136
|
-
import { logger as
|
|
3192
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
3137
3193
|
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3138
3194
|
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3139
3195
|
async function pollReorgsOnce(http, cursor, onReorg) {
|
|
3140
3196
|
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
3141
3197
|
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3142
3198
|
for (const r of sorted) {
|
|
3143
|
-
|
|
3199
|
+
logger10.info("Streams reorg — rewinding", {
|
|
3144
3200
|
forkPointHeight: r.fork_point_height
|
|
3145
3201
|
});
|
|
3146
3202
|
await onReorg(r.fork_point_height);
|
|
@@ -3163,15 +3219,15 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3163
3219
|
try {
|
|
3164
3220
|
since = await pollReorgsOnce(http, since, onReorg);
|
|
3165
3221
|
} catch (err) {
|
|
3166
|
-
|
|
3167
|
-
error:
|
|
3222
|
+
logger10.error("Streams reorg poll failed", {
|
|
3223
|
+
error: getErrorMessage5(err)
|
|
3168
3224
|
});
|
|
3169
3225
|
}
|
|
3170
3226
|
if (running)
|
|
3171
3227
|
timer = setTimeout(tick, POLL_MS);
|
|
3172
3228
|
};
|
|
3173
3229
|
timer = setTimeout(tick, POLL_MS);
|
|
3174
|
-
|
|
3230
|
+
logger10.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3175
3231
|
return () => {
|
|
3176
3232
|
running = false;
|
|
3177
3233
|
if (timer)
|
|
@@ -3198,11 +3254,11 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3198
3254
|
const def = await loadSubgraphDefinition(sg);
|
|
3199
3255
|
await catchUpSubgraph(def, sg.name);
|
|
3200
3256
|
} catch (err) {
|
|
3201
|
-
const msg =
|
|
3257
|
+
const msg = getErrorMessage6(err);
|
|
3202
3258
|
if (isHandlerNotFoundError(err)) {
|
|
3203
3259
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3204
3260
|
}
|
|
3205
|
-
|
|
3261
|
+
logger11.error("Subgraph catch-up failed", {
|
|
3206
3262
|
subgraph: sg.name,
|
|
3207
3263
|
error: msg
|
|
3208
3264
|
});
|
|
@@ -3242,7 +3298,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3242
3298
|
definitionCache.set(sg.name, def);
|
|
3243
3299
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3244
3300
|
invalidateSubgraphRoute(sg.name);
|
|
3245
|
-
|
|
3301
|
+
logger11.info("Subgraph handler reloaded", {
|
|
3246
3302
|
subgraph: sg.name,
|
|
3247
3303
|
from: prevVersion,
|
|
3248
3304
|
to: sg.version
|
|
@@ -3276,7 +3332,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3276
3332
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3277
3333
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3278
3334
|
});
|
|
3279
|
-
|
|
3335
|
+
logger11.info("Queued legacy reindex resume operation", {
|
|
3280
3336
|
subgraph: sg.name
|
|
3281
3337
|
});
|
|
3282
3338
|
} catch (err) {
|
|
@@ -3339,7 +3395,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3339
3395
|
const activeRuns = new Map;
|
|
3340
3396
|
let running = true;
|
|
3341
3397
|
let draining = false;
|
|
3342
|
-
|
|
3398
|
+
logger11.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3343
3399
|
try {
|
|
3344
3400
|
const stranded = await db.selectFrom("subgraphs").select(["id", "name", "account_id"]).where("status", "=", "reindexing").where("reindex_from_block", "is not", null).where("reindex_to_block", "is not", null).where(({ not, exists, selectFrom }) => not(exists(selectFrom("subgraph_operations").select("id").whereRef("subgraph_id", "=", "subgraphs.id").where("status", "in", ["queued", "running"])))).execute();
|
|
3345
3401
|
for (const row of stranded) {
|
|
@@ -3350,15 +3406,15 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3350
3406
|
accountId: row.account_id,
|
|
3351
3407
|
kind: "reindex"
|
|
3352
3408
|
});
|
|
3353
|
-
|
|
3409
|
+
logger11.info("Re-enqueued stranded reindex", { subgraph: row.name });
|
|
3354
3410
|
} catch (err) {
|
|
3355
3411
|
if (!isActiveSubgraphOperationConflict(err))
|
|
3356
3412
|
throw err;
|
|
3357
3413
|
}
|
|
3358
3414
|
}
|
|
3359
3415
|
} catch (err) {
|
|
3360
|
-
|
|
3361
|
-
error:
|
|
3416
|
+
logger11.warn("Stranded-reindex sweep failed", {
|
|
3417
|
+
error: getErrorMessage6(err)
|
|
3362
3418
|
});
|
|
3363
3419
|
}
|
|
3364
3420
|
const startOperation = (operation) => {
|
|
@@ -3368,9 +3424,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3368
3424
|
if (!running)
|
|
3369
3425
|
return;
|
|
3370
3426
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3371
|
-
|
|
3427
|
+
logger11.warn("Subgraph operation heartbeat failed", {
|
|
3372
3428
|
operationId: operation.id,
|
|
3373
|
-
error:
|
|
3429
|
+
error: getErrorMessage6(err)
|
|
3374
3430
|
});
|
|
3375
3431
|
});
|
|
3376
3432
|
}, HEARTBEAT_INTERVAL_MS);
|
|
@@ -3380,9 +3436,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3380
3436
|
controller.abort("user-cancelled");
|
|
3381
3437
|
}
|
|
3382
3438
|
}).catch((err) => {
|
|
3383
|
-
|
|
3439
|
+
logger11.warn("Subgraph operation cancel poll failed", {
|
|
3384
3440
|
operationId: operation.id,
|
|
3385
|
-
error:
|
|
3441
|
+
error: getErrorMessage6(err)
|
|
3386
3442
|
});
|
|
3387
3443
|
});
|
|
3388
3444
|
}, CANCEL_POLL_INTERVAL_MS);
|
|
@@ -3397,14 +3453,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3397
3453
|
const reason = String(controller.signal.reason ?? "");
|
|
3398
3454
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3399
3455
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3400
|
-
|
|
3456
|
+
logger11.info("Subgraph operation cancelled", {
|
|
3401
3457
|
operationId: operation.id,
|
|
3402
3458
|
subgraph: operation.subgraph_name
|
|
3403
3459
|
});
|
|
3404
3460
|
return;
|
|
3405
3461
|
}
|
|
3406
3462
|
if (controller.signal.aborted) {
|
|
3407
|
-
|
|
3463
|
+
logger11.info("Subgraph operation interrupted", {
|
|
3408
3464
|
operationId: operation.id,
|
|
3409
3465
|
subgraph: operation.subgraph_name,
|
|
3410
3466
|
reason
|
|
@@ -3412,7 +3468,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3412
3468
|
return;
|
|
3413
3469
|
}
|
|
3414
3470
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3415
|
-
|
|
3471
|
+
logger11.info("Subgraph operation completed", {
|
|
3416
3472
|
operationId: operation.id,
|
|
3417
3473
|
subgraph: operation.subgraph_name,
|
|
3418
3474
|
processed
|
|
@@ -3420,7 +3476,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3420
3476
|
} catch (err) {
|
|
3421
3477
|
const reason = String(controller.signal.reason ?? "");
|
|
3422
3478
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3423
|
-
|
|
3479
|
+
logger11.info("Subgraph operation interrupted by shutdown", {
|
|
3424
3480
|
operationId: operation.id,
|
|
3425
3481
|
subgraph: operation.subgraph_name
|
|
3426
3482
|
});
|
|
@@ -3430,11 +3486,11 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3430
3486
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3431
3487
|
return;
|
|
3432
3488
|
}
|
|
3433
|
-
await failSubgraphOperation(db, operation.id, lockedBy,
|
|
3434
|
-
|
|
3489
|
+
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
|
|
3490
|
+
logger11.error("Subgraph operation failed", {
|
|
3435
3491
|
operationId: operation.id,
|
|
3436
3492
|
subgraph: operation.subgraph_name,
|
|
3437
|
-
error:
|
|
3493
|
+
error: getErrorMessage6(err)
|
|
3438
3494
|
});
|
|
3439
3495
|
} finally {
|
|
3440
3496
|
clearInterval(heartbeat);
|
|
@@ -3478,13 +3534,13 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3478
3534
|
controller.abort("shutdown");
|
|
3479
3535
|
}
|
|
3480
3536
|
await Promise.allSettled(activeRuns.values());
|
|
3481
|
-
|
|
3537
|
+
logger11.info("Subgraph operation runner stopped");
|
|
3482
3538
|
};
|
|
3483
3539
|
}
|
|
3484
3540
|
async function startSubgraphProcessor(opts) {
|
|
3485
3541
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3486
3542
|
let running = true;
|
|
3487
|
-
|
|
3543
|
+
logger11.info("Starting subgraph processor", { concurrency });
|
|
3488
3544
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3489
3545
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3490
3546
|
});
|
|
@@ -3510,8 +3566,8 @@ async function startSubgraphProcessor(opts) {
|
|
|
3510
3566
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3511
3567
|
}
|
|
3512
3568
|
} catch (err) {
|
|
3513
|
-
|
|
3514
|
-
error:
|
|
3569
|
+
logger11.error("Subgraph reorg handling failed", {
|
|
3570
|
+
error: getErrorMessage6(err)
|
|
3515
3571
|
});
|
|
3516
3572
|
}
|
|
3517
3573
|
}, { connectionString: sourceListenerUrl() });
|
|
@@ -3519,7 +3575,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
3519
3575
|
runCatchUp();
|
|
3520
3576
|
}, POLL_INTERVAL_MS);
|
|
3521
3577
|
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
3522
|
-
|
|
3578
|
+
logger11.info("Subgraph processor ready");
|
|
3523
3579
|
return async () => {
|
|
3524
3580
|
running = false;
|
|
3525
3581
|
clearInterval(pollInterval);
|
|
@@ -3528,7 +3584,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
3528
3584
|
await stopReorgListening();
|
|
3529
3585
|
stopStreamsReorgPoll?.();
|
|
3530
3586
|
await stopOperations();
|
|
3531
|
-
|
|
3587
|
+
logger11.info("Subgraph processor stopped");
|
|
3532
3588
|
};
|
|
3533
3589
|
}
|
|
3534
3590
|
export {
|
|
@@ -3536,5 +3592,5 @@ export {
|
|
|
3536
3592
|
startSubgraphOperationRunner
|
|
3537
3593
|
};
|
|
3538
3594
|
|
|
3539
|
-
//# debugId=
|
|
3595
|
+
//# debugId=55EF8F5DD7627C2A64756E2164756E21
|
|
3540
3596
|
//# sourceMappingURL=processor.js.map
|