@secondlayer/subgraphs 3.18.0 → 3.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +88 -32
- package/dist/src/index.js.map +6 -5
- package/dist/src/runtime/processor.js +137 -81
- package/dist/src/runtime/processor.js.map +5 -4
- package/dist/src/runtime/reindex.js +87 -31
- package/dist/src/runtime/reindex.js.map +5 -4
- package/dist/src/schema/index.js +2 -2
- package/dist/src/schema/index.js.map +3 -3
- package/dist/src/service.js +140 -84
- package/dist/src/service.js.map +5 -4
- package/dist/src/validate.js +2 -2
- package/dist/src/validate.js.map +3 -3
- package/package.json +2 -2
package/dist/src/index.js
CHANGED
|
@@ -26,7 +26,7 @@ var SubgraphTableSchema = z.object({
|
|
|
26
26
|
indexes: z.array(z.array(SqlIdentifierSchema)).optional(),
|
|
27
27
|
uniqueKeys: z.array(z.array(SqlIdentifierSchema)).optional(),
|
|
28
28
|
relations: z.array(z.object({
|
|
29
|
-
name:
|
|
29
|
+
name: SqlIdentifierSchema,
|
|
30
30
|
references: SqlIdentifierSchema,
|
|
31
31
|
fields: z.array(SqlIdentifierSchema).min(1),
|
|
32
32
|
referencedColumns: z.array(SqlIdentifierSchema).min(1)
|
|
@@ -2381,7 +2381,7 @@ class StatsAccumulator {
|
|
|
2381
2381
|
}
|
|
2382
2382
|
|
|
2383
2383
|
// src/runtime/reindex.ts
|
|
2384
|
-
import { getErrorMessage as
|
|
2384
|
+
import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
|
|
2385
2385
|
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
2386
2386
|
import {
|
|
2387
2387
|
recordGapBatch,
|
|
@@ -2395,7 +2395,58 @@ import {
|
|
|
2395
2395
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2396
2396
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2397
2397
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2398
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
2399
|
+
|
|
2400
|
+
// src/runtime/reindex-notify.ts
|
|
2401
|
+
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2398
2402
|
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2403
|
+
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2404
|
+
try {
|
|
2405
|
+
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
2406
|
+
if (!subgraph)
|
|
2407
|
+
return;
|
|
2408
|
+
const account = await db.selectFrom("accounts").select(["email", "notify_reindex_complete"]).where("id", "=", subgraph.account_id).executeTakeFirst();
|
|
2409
|
+
if (!account?.email || !account.notify_reindex_complete)
|
|
2410
|
+
return;
|
|
2411
|
+
const resendKey = process.env.RESEND_API_KEY;
|
|
2412
|
+
if (!resendKey) {
|
|
2413
|
+
logger6.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2414
|
+
subgraph: subgraphName
|
|
2415
|
+
});
|
|
2416
|
+
return;
|
|
2417
|
+
}
|
|
2418
|
+
const from = process.env.EMAIL_FROM ?? "Secondlayer <noreply@secondlayer.tools>";
|
|
2419
|
+
const body = stats.errors > 0 ? `Your subgraph "${subgraphName}" finished reindexing — ${stats.blocks.toLocaleString()} blocks, ${stats.events.toLocaleString()} events, ${stats.errors.toLocaleString()} errors. Check the dashboard for details.` : `Your subgraph "${subgraphName}" finished reindexing — ${stats.blocks.toLocaleString()} blocks, ${stats.events.toLocaleString()} events processed, no errors. It's live now.`;
|
|
2420
|
+
const res = await fetch("https://api.resend.com/emails", {
|
|
2421
|
+
method: "POST",
|
|
2422
|
+
headers: {
|
|
2423
|
+
Authorization: `Bearer ${resendKey}`,
|
|
2424
|
+
"Content-Type": "application/json"
|
|
2425
|
+
},
|
|
2426
|
+
body: JSON.stringify({
|
|
2427
|
+
from,
|
|
2428
|
+
to: [account.email],
|
|
2429
|
+
subject: `Reindex complete: ${subgraphName}`,
|
|
2430
|
+
text: body
|
|
2431
|
+
})
|
|
2432
|
+
});
|
|
2433
|
+
if (!res.ok) {
|
|
2434
|
+
const text = await res.text().catch(() => "");
|
|
2435
|
+
logger6.warn("reindex-complete email failed", {
|
|
2436
|
+
subgraph: subgraphName,
|
|
2437
|
+
status: res.status,
|
|
2438
|
+
body: text.slice(0, 200)
|
|
2439
|
+
});
|
|
2440
|
+
}
|
|
2441
|
+
} catch (err) {
|
|
2442
|
+
logger6.warn("reindex-complete email threw", {
|
|
2443
|
+
subgraph: subgraphName,
|
|
2444
|
+
error: getErrorMessage2(err)
|
|
2445
|
+
});
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
// src/runtime/reindex.ts
|
|
2399
2450
|
var LOG_INTERVAL = 1000;
|
|
2400
2451
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
2401
2452
|
var PROGRESS_FLUSH_INTERVAL_MS = 5000;
|
|
@@ -2500,7 +2551,7 @@ async function processBlockRange(def, opts) {
|
|
|
2500
2551
|
while (currentHeight <= toBlock) {
|
|
2501
2552
|
if (opts.signal?.aborted) {
|
|
2502
2553
|
aborted = true;
|
|
2503
|
-
|
|
2554
|
+
logger7.info("Block processing aborted", {
|
|
2504
2555
|
subgraph: subgraphName,
|
|
2505
2556
|
currentBlock: currentHeight,
|
|
2506
2557
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2548,8 +2599,8 @@ async function processBlockRange(def, opts) {
|
|
|
2548
2599
|
preloaded: blockData
|
|
2549
2600
|
});
|
|
2550
2601
|
} catch (err) {
|
|
2551
|
-
const errorMsg =
|
|
2552
|
-
|
|
2602
|
+
const errorMsg = getErrorMessage3(err);
|
|
2603
|
+
logger7.error("Block processing failed persistently", {
|
|
2553
2604
|
subgraph: subgraphName,
|
|
2554
2605
|
blockHeight: height,
|
|
2555
2606
|
error: errorMsg
|
|
@@ -2595,7 +2646,7 @@ async function processBlockRange(def, opts) {
|
|
|
2595
2646
|
lastProgressFlushAt = now;
|
|
2596
2647
|
}
|
|
2597
2648
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2598
|
-
|
|
2649
|
+
logger7.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2599
2650
|
subgraph: subgraphName,
|
|
2600
2651
|
processed: blocksProcessed,
|
|
2601
2652
|
total: totalBlocks,
|
|
@@ -2615,7 +2666,7 @@ async function processBlockRange(def, opts) {
|
|
|
2615
2666
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2616
2667
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2617
2668
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2618
|
-
|
|
2669
|
+
logger7.warn("Failed to record subgraph gaps", {
|
|
2619
2670
|
subgraph: subgraphName,
|
|
2620
2671
|
error: err instanceof Error ? err.message : String(err)
|
|
2621
2672
|
});
|
|
@@ -2632,7 +2683,7 @@ async function processBlockRange(def, opts) {
|
|
|
2632
2683
|
} else {
|
|
2633
2684
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2634
2685
|
}
|
|
2635
|
-
|
|
2686
|
+
logger7.info("Sparse skip", {
|
|
2636
2687
|
subgraph: subgraphName,
|
|
2637
2688
|
from: batchEnd + 1,
|
|
2638
2689
|
to: jumpTo - 1,
|
|
@@ -2672,11 +2723,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
2672
2723
|
const subgraphName = def.name;
|
|
2673
2724
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
2674
2725
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
2675
|
-
|
|
2726
|
+
logger7.info("Reindex starting", { subgraph: subgraphName });
|
|
2676
2727
|
try {
|
|
2677
2728
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
2678
2729
|
if (fromBlock > toBlock) {
|
|
2679
|
-
|
|
2730
|
+
logger7.info("No blocks to reindex", {
|
|
2680
2731
|
subgraph: subgraphName,
|
|
2681
2732
|
fromBlock,
|
|
2682
2733
|
toBlock
|
|
@@ -2690,7 +2741,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
2690
2741
|
await client.unsafe(stmt);
|
|
2691
2742
|
}
|
|
2692
2743
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
2693
|
-
|
|
2744
|
+
logger7.info("Schema recreated for reindex", {
|
|
2694
2745
|
subgraph: subgraphName,
|
|
2695
2746
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
2696
2747
|
});
|
|
@@ -2704,7 +2755,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
2704
2755
|
last_error_at: null,
|
|
2705
2756
|
updated_at: new Date
|
|
2706
2757
|
}).where("name", "=", subgraphName).execute();
|
|
2707
|
-
|
|
2758
|
+
logger7.info("Reindexing blocks", {
|
|
2708
2759
|
subgraph: subgraphName,
|
|
2709
2760
|
fromBlock,
|
|
2710
2761
|
toBlock,
|
|
@@ -2726,9 +2777,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
2726
2777
|
if (reason === "user-cancelled") {
|
|
2727
2778
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
2728
2779
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2729
|
-
|
|
2780
|
+
logger7.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
2730
2781
|
} else {
|
|
2731
|
-
|
|
2782
|
+
logger7.info("Reindex interrupted by shutdown, will resume", {
|
|
2732
2783
|
subgraph: subgraphName
|
|
2733
2784
|
});
|
|
2734
2785
|
}
|
|
@@ -2736,17 +2787,22 @@ async function reindexSubgraph(def, opts) {
|
|
|
2736
2787
|
}
|
|
2737
2788
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2738
2789
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2739
|
-
|
|
2790
|
+
logger7.info("Reindex complete", {
|
|
2740
2791
|
subgraph: subgraphName,
|
|
2741
2792
|
blocks: result.blocksProcessed,
|
|
2742
2793
|
events: result.totalEventsProcessed,
|
|
2743
2794
|
errors: result.totalErrors
|
|
2744
2795
|
});
|
|
2796
|
+
notifyReindexComplete(targetDb, subgraphName, {
|
|
2797
|
+
blocks: result.blocksProcessed,
|
|
2798
|
+
events: result.totalEventsProcessed,
|
|
2799
|
+
errors: result.totalErrors
|
|
2800
|
+
});
|
|
2745
2801
|
return { processed: result.blocksProcessed };
|
|
2746
2802
|
} catch (err) {
|
|
2747
|
-
|
|
2803
|
+
logger7.error("Reindex failed", {
|
|
2748
2804
|
subgraph: subgraphName,
|
|
2749
|
-
error:
|
|
2805
|
+
error: getErrorMessage3(err)
|
|
2750
2806
|
});
|
|
2751
2807
|
await updateSubgraphStatus2(targetDb, subgraphName, "error");
|
|
2752
2808
|
throw err;
|
|
@@ -2766,7 +2822,7 @@ async function resumeReindex(def, opts) {
|
|
|
2766
2822
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
2767
2823
|
const toBlock = Number(row.reindex_to_block);
|
|
2768
2824
|
if (fromBlock == null) {
|
|
2769
|
-
|
|
2825
|
+
logger7.info("No reindex metadata, starting fresh reindex", {
|
|
2770
2826
|
subgraph: subgraphName
|
|
2771
2827
|
});
|
|
2772
2828
|
return reindexSubgraph(def, {
|
|
@@ -2775,12 +2831,12 @@ async function resumeReindex(def, opts) {
|
|
|
2775
2831
|
});
|
|
2776
2832
|
}
|
|
2777
2833
|
if (fromBlock > toBlock) {
|
|
2778
|
-
|
|
2834
|
+
logger7.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
2779
2835
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2780
2836
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2781
2837
|
return { processed: 0 };
|
|
2782
2838
|
}
|
|
2783
|
-
|
|
2839
|
+
logger7.info("Resuming reindex", {
|
|
2784
2840
|
subgraph: subgraphName,
|
|
2785
2841
|
fromBlock,
|
|
2786
2842
|
toBlock,
|
|
@@ -2802,9 +2858,9 @@ async function resumeReindex(def, opts) {
|
|
|
2802
2858
|
if (reason === "user-cancelled") {
|
|
2803
2859
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
2804
2860
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2805
|
-
|
|
2861
|
+
logger7.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
2806
2862
|
} else {
|
|
2807
|
-
|
|
2863
|
+
logger7.info("Resume interrupted by shutdown, will resume again", {
|
|
2808
2864
|
subgraph: subgraphName
|
|
2809
2865
|
});
|
|
2810
2866
|
}
|
|
@@ -2812,15 +2868,15 @@ async function resumeReindex(def, opts) {
|
|
|
2812
2868
|
}
|
|
2813
2869
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2814
2870
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2815
|
-
|
|
2871
|
+
logger7.info("Resumed reindex complete", {
|
|
2816
2872
|
subgraph: subgraphName,
|
|
2817
2873
|
blocks: result.blocksProcessed
|
|
2818
2874
|
});
|
|
2819
2875
|
return { processed: result.blocksProcessed };
|
|
2820
2876
|
} catch (err) {
|
|
2821
|
-
|
|
2877
|
+
logger7.error("Resumed reindex failed", {
|
|
2822
2878
|
subgraph: subgraphName,
|
|
2823
|
-
error:
|
|
2879
|
+
error: getErrorMessage3(err)
|
|
2824
2880
|
});
|
|
2825
2881
|
await updateSubgraphStatus2(targetDb, subgraphName, "error");
|
|
2826
2882
|
throw err;
|
|
@@ -2829,7 +2885,7 @@ async function resumeReindex(def, opts) {
|
|
|
2829
2885
|
async function backfillSubgraph(def, opts) {
|
|
2830
2886
|
const targetDb = getTargetDb2();
|
|
2831
2887
|
const subgraphName = def.name;
|
|
2832
|
-
|
|
2888
|
+
logger7.info("Backfill starting", {
|
|
2833
2889
|
subgraph: subgraphName,
|
|
2834
2890
|
from: opts.fromBlock,
|
|
2835
2891
|
to: opts.toBlock
|
|
@@ -2847,17 +2903,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
2847
2903
|
signal: opts.signal
|
|
2848
2904
|
});
|
|
2849
2905
|
if (result.aborted) {
|
|
2850
|
-
|
|
2906
|
+
logger7.info("Backfill aborted", { subgraph: subgraphName });
|
|
2851
2907
|
return { processed: result.blocksProcessed };
|
|
2852
2908
|
}
|
|
2853
2909
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
2854
2910
|
if (resolved > 0) {
|
|
2855
|
-
|
|
2911
|
+
logger7.info("Resolved subgraph gaps via backfill", {
|
|
2856
2912
|
subgraph: subgraphName,
|
|
2857
2913
|
resolved
|
|
2858
2914
|
});
|
|
2859
2915
|
}
|
|
2860
|
-
|
|
2916
|
+
logger7.info("Backfill complete", {
|
|
2861
2917
|
subgraph: subgraphName,
|
|
2862
2918
|
blocks: result.blocksProcessed,
|
|
2863
2919
|
events: result.totalEventsProcessed,
|
|
@@ -2865,9 +2921,9 @@ async function backfillSubgraph(def, opts) {
|
|
|
2865
2921
|
});
|
|
2866
2922
|
return { processed: result.blocksProcessed };
|
|
2867
2923
|
} catch (err) {
|
|
2868
|
-
|
|
2924
|
+
logger7.error("Backfill failed", {
|
|
2869
2925
|
subgraph: subgraphName,
|
|
2870
|
-
error:
|
|
2926
|
+
error: getErrorMessage3(err)
|
|
2871
2927
|
});
|
|
2872
2928
|
throw err;
|
|
2873
2929
|
}
|
|
@@ -4070,5 +4126,5 @@ export {
|
|
|
4070
4126
|
ByoBreakingChangeError
|
|
4071
4127
|
};
|
|
4072
4128
|
|
|
4073
|
-
//# debugId=
|
|
4129
|
+
//# debugId=3B1E439BDA4C0FCE64756E2164756E21
|
|
4074
4130
|
//# sourceMappingURL=index.js.map
|