@secondlayer/subgraphs 3.7.1 → 3.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +7 -4
- package/dist/src/index.js.map +3 -3
- package/dist/src/runtime/block-processor.js +7 -4
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.js +7 -4
- package/dist/src/runtime/catchup.js.map +3 -3
- package/dist/src/runtime/processor.js +208 -123
- package/dist/src/runtime/processor.js.map +15 -12
- package/dist/src/runtime/reindex.js +7 -4
- package/dist/src/runtime/reindex.js.map +3 -3
- package/dist/src/runtime/reorg.js +7 -4
- package/dist/src/runtime/reorg.js.map +3 -3
- package/dist/src/runtime/replay.js +686 -4
- package/dist/src/runtime/replay.js.map +10 -5
- package/dist/src/service.js +211 -126
- package/dist/src/service.js.map +15 -12
- package/package.json +2 -2
|
@@ -1242,6 +1242,11 @@ function resolveBlockSource(subgraph) {
|
|
|
1242
1242
|
import { createHash } from "node:crypto";
|
|
1243
1243
|
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
1244
1244
|
var loggedKillSwitch = false;
|
|
1245
|
+
var OP_VERB = {
|
|
1246
|
+
insert: "created",
|
|
1247
|
+
update: "updated",
|
|
1248
|
+
delete: "deleted"
|
|
1249
|
+
};
|
|
1245
1250
|
function isEmitOutboxEnabled() {
|
|
1246
1251
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
1247
1252
|
}
|
|
@@ -1269,12 +1274,10 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
1269
1274
|
return 0;
|
|
1270
1275
|
const rows = [];
|
|
1271
1276
|
for (const write of manifest.writes) {
|
|
1272
|
-
if (write.op !== "insert")
|
|
1273
|
-
continue;
|
|
1274
1277
|
const subs = matcher.match(subgraphName, write.table, write.row);
|
|
1275
1278
|
if (subs.length === 0)
|
|
1276
1279
|
continue;
|
|
1277
|
-
const eventType = `${subgraphName}.${write.table}.
|
|
1280
|
+
const eventType = `${subgraphName}.${write.table}.${OP_VERB[write.op]}`;
|
|
1278
1281
|
for (const s of subs) {
|
|
1279
1282
|
rows.push({
|
|
1280
1283
|
subscription_id: s.id,
|
|
@@ -2515,12 +2518,93 @@ import {
|
|
|
2515
2518
|
pgSchemaName as pgSchemaName2,
|
|
2516
2519
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
2517
2520
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2521
|
+
import { logger as logger15 } from "@secondlayer/shared/logger";
|
|
2522
|
+
import {
|
|
2523
|
+
listen as listen2,
|
|
2524
|
+
sourceListenerUrl,
|
|
2525
|
+
targetListenerUrl as targetListenerUrl4
|
|
2526
|
+
} from "@secondlayer/shared/queue/listener";
|
|
2527
|
+
|
|
2528
|
+
// src/runtime/catchup-leader.ts
|
|
2529
|
+
import {
|
|
2530
|
+
SUBGRAPH_CATCHUP_LOCK_KEY,
|
|
2531
|
+
createPostgresLeaderBackend,
|
|
2532
|
+
withLeaderLock
|
|
2533
|
+
} from "@secondlayer/shared/leader";
|
|
2534
|
+
import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
2535
|
+
var catchUpLeader = false;
|
|
2536
|
+
function isCatchUpLeader() {
|
|
2537
|
+
return catchUpLeader;
|
|
2538
|
+
}
|
|
2539
|
+
function startCatchUpLeader(opts = {}) {
|
|
2540
|
+
return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
|
|
2541
|
+
catchUpLeader = true;
|
|
2542
|
+
await opts.onAcquire?.();
|
|
2543
|
+
return () => {
|
|
2544
|
+
catchUpLeader = false;
|
|
2545
|
+
};
|
|
2546
|
+
}, {
|
|
2547
|
+
pollMs: opts.pollMs,
|
|
2548
|
+
heartbeatMs: opts.heartbeatMs,
|
|
2549
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
|
|
2550
|
+
});
|
|
2551
|
+
}
|
|
2552
|
+
|
|
2553
|
+
// src/runtime/streams-reorg-poll.ts
|
|
2554
|
+
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
2555
|
+
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
2556
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2557
|
+
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
2558
|
+
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
2559
|
+
async function pollReorgsOnce(http, cursor, onReorg) {
|
|
2560
|
+
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
2561
|
+
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
2562
|
+
for (const r of sorted) {
|
|
2563
|
+
logger9.info("Streams reorg — rewinding", {
|
|
2564
|
+
forkPointHeight: r.fork_point_height
|
|
2565
|
+
});
|
|
2566
|
+
await onReorg(r.fork_point_height);
|
|
2567
|
+
}
|
|
2568
|
+
return next_since ?? cursor;
|
|
2569
|
+
}
|
|
2570
|
+
function startStreamsReorgPoll(onReorg) {
|
|
2571
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
2572
|
+
const http = new IndexHttpClient2({
|
|
2573
|
+
indexBaseUrl: baseUrl,
|
|
2574
|
+
streamsBaseUrl: baseUrl,
|
|
2575
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
2576
|
+
});
|
|
2577
|
+
let since = new Date(Date.now() - STARTUP_MARGIN_MS).toISOString();
|
|
2578
|
+
let running = true;
|
|
2579
|
+
let timer;
|
|
2580
|
+
const tick = async () => {
|
|
2581
|
+
if (!running)
|
|
2582
|
+
return;
|
|
2583
|
+
try {
|
|
2584
|
+
since = await pollReorgsOnce(http, since, onReorg);
|
|
2585
|
+
} catch (err) {
|
|
2586
|
+
logger9.error("Streams reorg poll failed", {
|
|
2587
|
+
error: getErrorMessage4(err)
|
|
2588
|
+
});
|
|
2589
|
+
}
|
|
2590
|
+
if (running)
|
|
2591
|
+
timer = setTimeout(tick, POLL_MS);
|
|
2592
|
+
};
|
|
2593
|
+
timer = setTimeout(tick, POLL_MS);
|
|
2594
|
+
logger9.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
2595
|
+
return () => {
|
|
2596
|
+
running = false;
|
|
2597
|
+
if (timer)
|
|
2598
|
+
clearTimeout(timer);
|
|
2599
|
+
};
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
// src/runtime/subscription-plane.ts
|
|
2518
2603
|
import { logger as logger14 } from "@secondlayer/shared/logger";
|
|
2519
|
-
import { listen as listen2 } from "@secondlayer/shared/queue/listener";
|
|
2520
2604
|
|
|
2521
2605
|
// src/runtime/chain-reorg.ts
|
|
2522
2606
|
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
2523
|
-
import { logger as
|
|
2607
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2524
2608
|
var MAX_ORPHANED_PER_SUB = 500;
|
|
2525
2609
|
async function handleChainReorg(forkHeight, db = getTargetDb5()) {
|
|
2526
2610
|
await db.deleteFrom("subscription_outbox").where("kind", "=", "chain").where("block_height", ">=", forkHeight).where("status", "=", "pending").where("event_type", "like", "chain.%.apply").execute();
|
|
@@ -2556,7 +2640,7 @@ async function handleChainReorg(forkHeight, db = getTargetDb5()) {
|
|
|
2556
2640
|
});
|
|
2557
2641
|
}
|
|
2558
2642
|
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
|
|
2559
|
-
|
|
2643
|
+
logger10.info("Chain reorg — emitted rollbacks", {
|
|
2560
2644
|
forkPointHeight: forkHeight,
|
|
2561
2645
|
subscriptions: bySub.size
|
|
2562
2646
|
});
|
|
@@ -2574,12 +2658,13 @@ import {
|
|
|
2574
2658
|
getTargetDb as getTargetDb6
|
|
2575
2659
|
} from "@secondlayer/shared/db";
|
|
2576
2660
|
import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2577
|
-
import { logger as
|
|
2578
|
-
import { listen } from "@secondlayer/shared/queue/listener";
|
|
2661
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
2662
|
+
import { listen, targetListenerUrl as targetListenerUrl2 } from "@secondlayer/shared/queue/listener";
|
|
2579
2663
|
import { sql as sql4 } from "kysely";
|
|
2580
2664
|
|
|
2581
2665
|
// src/runtime/formats/index.ts
|
|
2582
|
-
import {
|
|
2666
|
+
import { signSecondlayerWebhook } from "@secondlayer/shared/crypto/secondlayer-webhook";
|
|
2667
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
2583
2668
|
|
|
2584
2669
|
// src/runtime/formats/cloudevents.ts
|
|
2585
2670
|
function buildCloudEvents(outboxRow, _sub) {
|
|
@@ -2711,7 +2796,7 @@ function buildTrigger(outboxRow, sub) {
|
|
|
2711
2796
|
}
|
|
2712
2797
|
|
|
2713
2798
|
// src/runtime/formats/index.ts
|
|
2714
|
-
function
|
|
2799
|
+
function buildBody(outboxRow, sub, signingSecret) {
|
|
2715
2800
|
switch (sub.format) {
|
|
2716
2801
|
case "inngest":
|
|
2717
2802
|
return buildInngest(outboxRow);
|
|
@@ -2726,13 +2811,21 @@ function buildForFormat(outboxRow, sub, signingSecret) {
|
|
|
2726
2811
|
case "standard-webhooks":
|
|
2727
2812
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2728
2813
|
default:
|
|
2729
|
-
|
|
2814
|
+
logger11.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
2730
2815
|
format: sub.format,
|
|
2731
2816
|
subscriptionId: sub.id
|
|
2732
2817
|
});
|
|
2733
2818
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2734
2819
|
}
|
|
2735
2820
|
}
|
|
2821
|
+
function buildForFormat(outboxRow, sub, signingSecret) {
|
|
2822
|
+
const result = buildBody(outboxRow, sub, signingSecret);
|
|
2823
|
+
const sigHeaders = signSecondlayerWebhook(outboxRow.id, result.body);
|
|
2824
|
+
if (sigHeaders) {
|
|
2825
|
+
result.headers = { ...result.headers, ...sigHeaders };
|
|
2826
|
+
}
|
|
2827
|
+
return result;
|
|
2828
|
+
}
|
|
2736
2829
|
|
|
2737
2830
|
// src/runtime/emitter.ts
|
|
2738
2831
|
var BATCH_SIZE = 50;
|
|
@@ -2809,7 +2902,7 @@ async function dispatchOne(db, outboxRow, sub) {
|
|
|
2809
2902
|
let responseHeaders = {};
|
|
2810
2903
|
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
2811
2904
|
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
2812
|
-
|
|
2905
|
+
logger12.warn("[emitter] refused private egress", {
|
|
2813
2906
|
subscription: sub.name,
|
|
2814
2907
|
url: sub.url
|
|
2815
2908
|
});
|
|
@@ -2905,7 +2998,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
2905
2998
|
circuit_opened_at: new Date,
|
|
2906
2999
|
updated_at: new Date
|
|
2907
3000
|
}).where("id", "=", sub.id).execute();
|
|
2908
|
-
|
|
3001
|
+
logger12.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
2909
3002
|
subscription: sub.name,
|
|
2910
3003
|
failures: newFailures
|
|
2911
3004
|
});
|
|
@@ -2993,7 +3086,7 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
2993
3086
|
await settleFailed(db, row, sub, err);
|
|
2994
3087
|
}
|
|
2995
3088
|
} catch (err) {
|
|
2996
|
-
|
|
3089
|
+
logger12.error("Emitter dispatch crashed", {
|
|
2997
3090
|
outboxId: row.id,
|
|
2998
3091
|
error: err instanceof Error ? err.message : String(err)
|
|
2999
3092
|
});
|
|
@@ -3030,7 +3123,7 @@ async function startEmitter(opts) {
|
|
|
3030
3123
|
};
|
|
3031
3124
|
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
3032
3125
|
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
3033
|
-
|
|
3126
|
+
logger12.info("[emitter] started", { id: emitterId });
|
|
3034
3127
|
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
3035
3128
|
let lastErr = null;
|
|
3036
3129
|
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
@@ -3041,7 +3134,7 @@ async function startEmitter(opts) {
|
|
|
3041
3134
|
} catch (err) {
|
|
3042
3135
|
lastErr = err;
|
|
3043
3136
|
const delayMs = 500 * 2 ** i;
|
|
3044
|
-
|
|
3137
|
+
logger12.warn("[emitter] matcher refresh failed, retrying", {
|
|
3045
3138
|
attempt: i + 1,
|
|
3046
3139
|
delayMs,
|
|
3047
3140
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -3052,24 +3145,25 @@ async function startEmitter(opts) {
|
|
|
3052
3145
|
if (lastErr) {
|
|
3053
3146
|
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
3054
3147
|
}
|
|
3148
|
+
const listenUrl = targetListenerUrl2();
|
|
3055
3149
|
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
3056
3150
|
if (!state.running)
|
|
3057
3151
|
return;
|
|
3058
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3152
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger12.error("[emitter] claim failed", {
|
|
3059
3153
|
error: err instanceof Error ? err.message : String(err)
|
|
3060
3154
|
}));
|
|
3061
|
-
});
|
|
3155
|
+
}, { connectionString: listenUrl });
|
|
3062
3156
|
const stopChanged = await listen("subscriptions:changed", () => {
|
|
3063
3157
|
if (!state.running)
|
|
3064
3158
|
return;
|
|
3065
|
-
refreshMatcher(db).catch((err) =>
|
|
3159
|
+
refreshMatcher(db).catch((err) => logger12.error("[emitter] matcher refresh failed", {
|
|
3066
3160
|
error: err instanceof Error ? err.message : String(err)
|
|
3067
3161
|
}));
|
|
3068
|
-
});
|
|
3162
|
+
}, { connectionString: listenUrl });
|
|
3069
3163
|
const poll = setInterval(() => {
|
|
3070
3164
|
if (!state.running)
|
|
3071
3165
|
return;
|
|
3072
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3166
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger12.error("[emitter] poll claim failed", {
|
|
3073
3167
|
error: err instanceof Error ? err.message : String(err)
|
|
3074
3168
|
}));
|
|
3075
3169
|
}, pollIntervalMs);
|
|
@@ -3077,7 +3171,7 @@ async function startEmitter(opts) {
|
|
|
3077
3171
|
const retention = setInterval(() => {
|
|
3078
3172
|
if (!state.running)
|
|
3079
3173
|
return;
|
|
3080
|
-
runRetention(db).catch((err) =>
|
|
3174
|
+
runRetention(db).catch((err) => logger12.error("[emitter] retention failed", {
|
|
3081
3175
|
error: err instanceof Error ? err.message : String(err)
|
|
3082
3176
|
}));
|
|
3083
3177
|
}, retentionIntervalMs);
|
|
@@ -3087,60 +3181,17 @@ async function startEmitter(opts) {
|
|
|
3087
3181
|
clearInterval(retention);
|
|
3088
3182
|
await stopNew();
|
|
3089
3183
|
await stopChanged();
|
|
3090
|
-
|
|
3184
|
+
logger12.info("[emitter] stopped", { id: emitterId });
|
|
3091
3185
|
};
|
|
3092
3186
|
}
|
|
3093
3187
|
|
|
3094
|
-
// src/runtime/
|
|
3095
|
-
import {
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
3102
|
-
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3103
|
-
for (const r of sorted) {
|
|
3104
|
-
logger12.info("Streams reorg — rewinding subgraphs", {
|
|
3105
|
-
forkPointHeight: r.fork_point_height
|
|
3106
|
-
});
|
|
3107
|
-
await handleReorg(r.fork_point_height, loadDef);
|
|
3108
|
-
if (handleChainReorg2)
|
|
3109
|
-
await handleChainReorg2(r.fork_point_height);
|
|
3110
|
-
}
|
|
3111
|
-
return next_since ?? cursor;
|
|
3112
|
-
}
|
|
3113
|
-
function startStreamsReorgPoll(handleReorg, loadDef, handleChainReorg2) {
|
|
3114
|
-
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
3115
|
-
const http = new IndexHttpClient2({
|
|
3116
|
-
indexBaseUrl: baseUrl,
|
|
3117
|
-
streamsBaseUrl: baseUrl,
|
|
3118
|
-
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
3119
|
-
});
|
|
3120
|
-
let since = new Date(Date.now() - STARTUP_MARGIN_MS).toISOString();
|
|
3121
|
-
let running = true;
|
|
3122
|
-
let timer;
|
|
3123
|
-
const tick = async () => {
|
|
3124
|
-
if (!running)
|
|
3125
|
-
return;
|
|
3126
|
-
try {
|
|
3127
|
-
since = await pollReorgsOnce(http, since, handleReorg, loadDef, handleChainReorg2);
|
|
3128
|
-
} catch (err) {
|
|
3129
|
-
logger12.error("Streams reorg poll failed", {
|
|
3130
|
-
error: getErrorMessage4(err)
|
|
3131
|
-
});
|
|
3132
|
-
}
|
|
3133
|
-
if (running)
|
|
3134
|
-
timer = setTimeout(tick, POLL_MS);
|
|
3135
|
-
};
|
|
3136
|
-
timer = setTimeout(tick, POLL_MS);
|
|
3137
|
-
logger12.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3138
|
-
return () => {
|
|
3139
|
-
running = false;
|
|
3140
|
-
if (timer)
|
|
3141
|
-
clearTimeout(timer);
|
|
3142
|
-
};
|
|
3143
|
-
}
|
|
3188
|
+
// src/runtime/subscription-leader.ts
|
|
3189
|
+
import {
|
|
3190
|
+
SUBSCRIPTION_EVALUATOR_LOCK_KEY,
|
|
3191
|
+
createPostgresLeaderBackend as createPostgresLeaderBackend2,
|
|
3192
|
+
withLeaderLock as withLeaderLock2
|
|
3193
|
+
} from "@secondlayer/shared/leader";
|
|
3194
|
+
import { targetListenerUrl as targetListenerUrl3 } from "@secondlayer/shared/queue/listener";
|
|
3144
3195
|
|
|
3145
3196
|
// src/runtime/trigger-evaluator-loop.ts
|
|
3146
3197
|
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
@@ -3217,10 +3268,11 @@ async function buildTraitContracts(db, chainSubs, asOfBlock) {
|
|
|
3217
3268
|
function evaluateBlock(block, sources, traitContracts) {
|
|
3218
3269
|
return matchSources(sources, block.txs, block.events, traitContracts);
|
|
3219
3270
|
}
|
|
3220
|
-
function chainDedupKey(subscriptionId, txId, eventIndex, blockHash) {
|
|
3221
|
-
|
|
3271
|
+
function chainDedupKey(subscriptionId, txId, eventIndex, blockHash, replayId) {
|
|
3272
|
+
const base = `chain:${subscriptionId}:${txId}:${eventIndex}:${blockHash}`;
|
|
3273
|
+
return replayId ? `replay:${replayId}:${base}` : base;
|
|
3222
3274
|
}
|
|
3223
|
-
function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event) {
|
|
3275
|
+
function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event, replayId) {
|
|
3224
3276
|
const payload = {
|
|
3225
3277
|
action: "apply",
|
|
3226
3278
|
block_hash: blockHash,
|
|
@@ -3240,10 +3292,12 @@ function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event) {
|
|
|
3240
3292
|
row_pk: { tx_id: txId, event_index: eventIndex },
|
|
3241
3293
|
event_type: `chain.${meta.triggerType}.apply`,
|
|
3242
3294
|
payload,
|
|
3243
|
-
dedup_key: chainDedupKey(meta.subscriptionId, txId, eventIndex, blockHash)
|
|
3295
|
+
dedup_key: chainDedupKey(meta.subscriptionId, txId, eventIndex, blockHash, replayId),
|
|
3296
|
+
...replayId ? { is_replay: true } : {}
|
|
3244
3297
|
};
|
|
3245
3298
|
}
|
|
3246
|
-
async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
3299
|
+
async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash, opts) {
|
|
3300
|
+
const replayId = opts?.replayId;
|
|
3247
3301
|
const rows = [];
|
|
3248
3302
|
for (const match of matches) {
|
|
3249
3303
|
const meta = keyMeta.get(match.sourceName);
|
|
@@ -3260,7 +3314,7 @@ async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
|
3260
3314
|
function_name: match.tx.function_name ?? null,
|
|
3261
3315
|
function_args: match.tx.function_args ?? null,
|
|
3262
3316
|
result_hex: match.tx.raw_result ?? null
|
|
3263
|
-
}));
|
|
3317
|
+
}, replayId));
|
|
3264
3318
|
} else {
|
|
3265
3319
|
for (const event of match.events) {
|
|
3266
3320
|
rows.push(applyRow(meta, blockHeight, blockHash, txId, event.event_index, {
|
|
@@ -3268,14 +3322,14 @@ async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
|
3268
3322
|
type: event.type,
|
|
3269
3323
|
event_index: event.event_index,
|
|
3270
3324
|
data: event.data
|
|
3271
|
-
}));
|
|
3325
|
+
}, replayId));
|
|
3272
3326
|
}
|
|
3273
3327
|
}
|
|
3274
3328
|
}
|
|
3275
3329
|
if (rows.length === 0)
|
|
3276
3330
|
return 0;
|
|
3277
|
-
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).
|
|
3278
|
-
return
|
|
3331
|
+
const result = await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).executeTakeFirst();
|
|
3332
|
+
return Number(result.numInsertedOrUpdatedRows ?? 0);
|
|
3279
3333
|
}
|
|
3280
3334
|
|
|
3281
3335
|
// src/runtime/trigger-evaluator-loop.ts
|
|
@@ -3357,6 +3411,48 @@ function startTriggerEvaluator() {
|
|
|
3357
3411
|
};
|
|
3358
3412
|
}
|
|
3359
3413
|
|
|
3414
|
+
// src/runtime/subscription-leader.ts
|
|
3415
|
+
var evaluatorLeader = false;
|
|
3416
|
+
function isEvaluatorLeader() {
|
|
3417
|
+
return evaluatorLeader;
|
|
3418
|
+
}
|
|
3419
|
+
function gateChainReorgOnLeader(handler, isLeader = isEvaluatorLeader) {
|
|
3420
|
+
return async (forkHeight) => {
|
|
3421
|
+
if (!isLeader())
|
|
3422
|
+
return;
|
|
3423
|
+
await handler(forkHeight);
|
|
3424
|
+
};
|
|
3425
|
+
}
|
|
3426
|
+
function startTriggerEvaluatorLeader(opts = {}) {
|
|
3427
|
+
const startWork = opts.startWork ?? startTriggerEvaluator;
|
|
3428
|
+
return withLeaderLock2(SUBSCRIPTION_EVALUATOR_LOCK_KEY, async () => {
|
|
3429
|
+
evaluatorLeader = true;
|
|
3430
|
+
const stop = await startWork();
|
|
3431
|
+
return () => {
|
|
3432
|
+
evaluatorLeader = false;
|
|
3433
|
+
stop();
|
|
3434
|
+
};
|
|
3435
|
+
}, {
|
|
3436
|
+
pollMs: opts.pollMs,
|
|
3437
|
+
heartbeatMs: opts.heartbeatMs,
|
|
3438
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend2(targetListenerUrl3()))
|
|
3439
|
+
});
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
// src/runtime/subscription-plane.ts
|
|
3443
|
+
async function startSubscriptionPlane() {
|
|
3444
|
+
const streamsIndex = process.env.SUBGRAPH_SOURCE === "streams-index";
|
|
3445
|
+
const stopChainReorgPoll = streamsIndex ? startStreamsReorgPoll(gateChainReorgOnLeader((forkHeight) => handleChainReorg(forkHeight))) : undefined;
|
|
3446
|
+
const stopTriggerEvaluator = streamsIndex ? startTriggerEvaluatorLeader() : undefined;
|
|
3447
|
+
const stopEmitter = await startEmitter();
|
|
3448
|
+
logger14.info("Subscription plane ready", { streamsIndex });
|
|
3449
|
+
return async () => {
|
|
3450
|
+
stopChainReorgPoll?.();
|
|
3451
|
+
await stopTriggerEvaluator?.();
|
|
3452
|
+
await stopEmitter();
|
|
3453
|
+
};
|
|
3454
|
+
}
|
|
3455
|
+
|
|
3360
3456
|
// src/runtime/processor.ts
|
|
3361
3457
|
var CHANNEL_NEW_BLOCK = "indexer:new_block";
|
|
3362
3458
|
var CHANNEL_SUBGRAPH_OPERATIONS = "subgraph_operations:new";
|
|
@@ -3380,7 +3476,7 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3380
3476
|
if (isHandlerNotFoundError(err)) {
|
|
3381
3477
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3382
3478
|
}
|
|
3383
|
-
|
|
3479
|
+
logger15.error("Subgraph catch-up failed", {
|
|
3384
3480
|
subgraph: sg.name,
|
|
3385
3481
|
error: msg
|
|
3386
3482
|
});
|
|
@@ -3392,12 +3488,6 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3392
3488
|
function handlerImportUrl(handlerPath, cacheBust = Date.now()) {
|
|
3393
3489
|
return `${pathToFileURL(resolve(handlerPath)).href}?t=${cacheBust}`;
|
|
3394
3490
|
}
|
|
3395
|
-
function sourceListenerUrl() {
|
|
3396
|
-
return process.env.SOURCE_DATABASE_URL || process.env.DATABASE_URL;
|
|
3397
|
-
}
|
|
3398
|
-
function targetListenerUrl() {
|
|
3399
|
-
return process.env.TARGET_DATABASE_URL || process.env.DATABASE_URL;
|
|
3400
|
-
}
|
|
3401
3491
|
function isHandlerNotFoundError(err) {
|
|
3402
3492
|
if (!(err instanceof Error))
|
|
3403
3493
|
return false;
|
|
@@ -3426,7 +3516,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3426
3516
|
definitionCache.set(sg.name, def);
|
|
3427
3517
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3428
3518
|
invalidateSubgraphRoute(sg.name);
|
|
3429
|
-
|
|
3519
|
+
logger15.info("Subgraph handler reloaded", {
|
|
3430
3520
|
subgraph: sg.name,
|
|
3431
3521
|
from: prevVersion,
|
|
3432
3522
|
to: sg.version
|
|
@@ -3460,7 +3550,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3460
3550
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3461
3551
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3462
3552
|
});
|
|
3463
|
-
|
|
3553
|
+
logger15.info("Queued legacy reindex resume operation", {
|
|
3464
3554
|
subgraph: sg.name
|
|
3465
3555
|
});
|
|
3466
3556
|
} catch (err) {
|
|
@@ -3516,7 +3606,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3516
3606
|
const activeRuns = new Map;
|
|
3517
3607
|
let running = true;
|
|
3518
3608
|
let draining = false;
|
|
3519
|
-
|
|
3609
|
+
logger15.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3520
3610
|
const startOperation = (operation) => {
|
|
3521
3611
|
const controller = new AbortController;
|
|
3522
3612
|
active.set(operation.id, controller);
|
|
@@ -3524,7 +3614,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3524
3614
|
if (!running)
|
|
3525
3615
|
return;
|
|
3526
3616
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3527
|
-
|
|
3617
|
+
logger15.warn("Subgraph operation heartbeat failed", {
|
|
3528
3618
|
operationId: operation.id,
|
|
3529
3619
|
error: getErrorMessage6(err)
|
|
3530
3620
|
});
|
|
@@ -3536,7 +3626,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3536
3626
|
controller.abort("user-cancelled");
|
|
3537
3627
|
}
|
|
3538
3628
|
}).catch((err) => {
|
|
3539
|
-
|
|
3629
|
+
logger15.warn("Subgraph operation cancel poll failed", {
|
|
3540
3630
|
operationId: operation.id,
|
|
3541
3631
|
error: getErrorMessage6(err)
|
|
3542
3632
|
});
|
|
@@ -3553,14 +3643,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3553
3643
|
const reason = String(controller.signal.reason ?? "");
|
|
3554
3644
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3555
3645
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3556
|
-
|
|
3646
|
+
logger15.info("Subgraph operation cancelled", {
|
|
3557
3647
|
operationId: operation.id,
|
|
3558
3648
|
subgraph: operation.subgraph_name
|
|
3559
3649
|
});
|
|
3560
3650
|
return;
|
|
3561
3651
|
}
|
|
3562
3652
|
if (controller.signal.aborted) {
|
|
3563
|
-
|
|
3653
|
+
logger15.info("Subgraph operation interrupted", {
|
|
3564
3654
|
operationId: operation.id,
|
|
3565
3655
|
subgraph: operation.subgraph_name,
|
|
3566
3656
|
reason
|
|
@@ -3568,7 +3658,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3568
3658
|
return;
|
|
3569
3659
|
}
|
|
3570
3660
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3571
|
-
|
|
3661
|
+
logger15.info("Subgraph operation completed", {
|
|
3572
3662
|
operationId: operation.id,
|
|
3573
3663
|
subgraph: operation.subgraph_name,
|
|
3574
3664
|
processed
|
|
@@ -3576,7 +3666,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3576
3666
|
} catch (err) {
|
|
3577
3667
|
const reason = String(controller.signal.reason ?? "");
|
|
3578
3668
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3579
|
-
|
|
3669
|
+
logger15.info("Subgraph operation interrupted by shutdown", {
|
|
3580
3670
|
operationId: operation.id,
|
|
3581
3671
|
subgraph: operation.subgraph_name
|
|
3582
3672
|
});
|
|
@@ -3587,7 +3677,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3587
3677
|
return;
|
|
3588
3678
|
}
|
|
3589
3679
|
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
|
|
3590
|
-
|
|
3680
|
+
logger15.error("Subgraph operation failed", {
|
|
3591
3681
|
operationId: operation.id,
|
|
3592
3682
|
subgraph: operation.subgraph_name,
|
|
3593
3683
|
error: getErrorMessage6(err)
|
|
@@ -3622,7 +3712,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3622
3712
|
await drain();
|
|
3623
3713
|
const stopListening = await listen2(CHANNEL_SUBGRAPH_OPERATIONS, () => {
|
|
3624
3714
|
drain();
|
|
3625
|
-
}, { connectionString:
|
|
3715
|
+
}, { connectionString: targetListenerUrl4() });
|
|
3626
3716
|
const pollInterval = setInterval(() => {
|
|
3627
3717
|
drain();
|
|
3628
3718
|
}, POLL_INTERVAL_MS);
|
|
@@ -3634,26 +3724,27 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3634
3724
|
controller.abort("shutdown");
|
|
3635
3725
|
}
|
|
3636
3726
|
await Promise.allSettled(activeRuns.values());
|
|
3637
|
-
|
|
3727
|
+
logger15.info("Subgraph operation runner stopped");
|
|
3638
3728
|
};
|
|
3639
3729
|
}
|
|
3640
3730
|
async function startSubgraphProcessor(opts) {
|
|
3641
3731
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3642
3732
|
let running = true;
|
|
3643
|
-
|
|
3733
|
+
logger15.info("Starting subgraph processor", { concurrency });
|
|
3644
3734
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3645
3735
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3646
3736
|
});
|
|
3647
|
-
const
|
|
3648
|
-
|
|
3649
|
-
await catchUpAll(activeSubgraphs, targetDb, concurrency);
|
|
3650
|
-
const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
|
|
3651
|
-
if (!running)
|
|
3737
|
+
const runCatchUp = async () => {
|
|
3738
|
+
if (!running || !isCatchUpLeader())
|
|
3652
3739
|
return;
|
|
3653
3740
|
const db = getTargetDb8();
|
|
3654
3741
|
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3655
3742
|
cleanupCaches(subgraphs);
|
|
3656
3743
|
await catchUpAll(subgraphs, db, concurrency);
|
|
3744
|
+
};
|
|
3745
|
+
const stopCatchUpLeader = startCatchUpLeader({ onAcquire: runCatchUp });
|
|
3746
|
+
const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
|
|
3747
|
+
await runCatchUp();
|
|
3657
3748
|
}, { connectionString: sourceListenerUrl() });
|
|
3658
3749
|
const stopReorgListening = await listen2("subgraph_reorg", async (payload) => {
|
|
3659
3750
|
if (!running)
|
|
@@ -3665,33 +3756,27 @@ async function startSubgraphProcessor(opts) {
|
|
|
3665
3756
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3666
3757
|
}
|
|
3667
3758
|
} catch (err) {
|
|
3668
|
-
|
|
3759
|
+
logger15.error("Subgraph reorg handling failed", {
|
|
3669
3760
|
error: getErrorMessage6(err)
|
|
3670
3761
|
});
|
|
3671
3762
|
}
|
|
3672
3763
|
}, { connectionString: sourceListenerUrl() });
|
|
3673
|
-
const pollInterval = setInterval(
|
|
3674
|
-
|
|
3675
|
-
return;
|
|
3676
|
-
const db = getTargetDb8();
|
|
3677
|
-
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3678
|
-
cleanupCaches(subgraphs);
|
|
3679
|
-
await catchUpAll(subgraphs, db, concurrency);
|
|
3764
|
+
const pollInterval = setInterval(() => {
|
|
3765
|
+
runCatchUp();
|
|
3680
3766
|
}, POLL_INTERVAL_MS);
|
|
3681
|
-
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll(
|
|
3682
|
-
const
|
|
3683
|
-
|
|
3684
|
-
logger14.info("Subgraph processor ready");
|
|
3767
|
+
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
3768
|
+
const stopSubscriptionPlane = await startSubscriptionPlane();
|
|
3769
|
+
logger15.info("Subgraph processor ready");
|
|
3685
3770
|
return async () => {
|
|
3686
3771
|
running = false;
|
|
3687
3772
|
clearInterval(pollInterval);
|
|
3773
|
+
await stopCatchUpLeader();
|
|
3688
3774
|
await stopListening();
|
|
3689
3775
|
await stopReorgListening();
|
|
3690
3776
|
stopStreamsReorgPoll?.();
|
|
3691
|
-
|
|
3777
|
+
await stopSubscriptionPlane();
|
|
3692
3778
|
await stopOperations();
|
|
3693
|
-
|
|
3694
|
-
logger14.info("Subgraph processor stopped");
|
|
3779
|
+
logger15.info("Subgraph processor stopped");
|
|
3695
3780
|
};
|
|
3696
3781
|
}
|
|
3697
3782
|
export {
|
|
@@ -3699,5 +3784,5 @@ export {
|
|
|
3699
3784
|
startSubgraphOperationRunner
|
|
3700
3785
|
};
|
|
3701
3786
|
|
|
3702
|
-
//# debugId=
|
|
3787
|
+
//# debugId=2915D9371C21F2BC64756E2164756E21
|
|
3703
3788
|
//# sourceMappingURL=processor.js.map
|