@secondlayer/subgraphs 3.7.2 → 3.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +7 -4
- package/dist/src/index.js.map +3 -3
- package/dist/src/runtime/block-processor.js +7 -4
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.js +7 -4
- package/dist/src/runtime/catchup.js.map +3 -3
- package/dist/src/runtime/processor.js +203 -117
- package/dist/src/runtime/processor.js.map +14 -11
- package/dist/src/runtime/reindex.js +7 -4
- package/dist/src/runtime/reindex.js.map +3 -3
- package/dist/src/runtime/reorg.js +7 -4
- package/dist/src/runtime/reorg.js.map +3 -3
- package/dist/src/runtime/replay.js +686 -4
- package/dist/src/runtime/replay.js.map +10 -5
- package/dist/src/service.js +206 -120
- package/dist/src/service.js.map +14 -11
- package/package.json +2 -2
package/dist/src/service.js
CHANGED
|
@@ -1242,6 +1242,11 @@ function resolveBlockSource(subgraph) {
|
|
|
1242
1242
|
import { createHash } from "node:crypto";
|
|
1243
1243
|
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
1244
1244
|
var loggedKillSwitch = false;
|
|
1245
|
+
var OP_VERB = {
|
|
1246
|
+
insert: "created",
|
|
1247
|
+
update: "updated",
|
|
1248
|
+
delete: "deleted"
|
|
1249
|
+
};
|
|
1245
1250
|
function isEmitOutboxEnabled() {
|
|
1246
1251
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
1247
1252
|
}
|
|
@@ -1269,12 +1274,10 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
1269
1274
|
return 0;
|
|
1270
1275
|
const rows = [];
|
|
1271
1276
|
for (const write of manifest.writes) {
|
|
1272
|
-
if (write.op !== "insert")
|
|
1273
|
-
continue;
|
|
1274
1277
|
const subs = matcher.match(subgraphName, write.table, write.row);
|
|
1275
1278
|
if (subs.length === 0)
|
|
1276
1279
|
continue;
|
|
1277
|
-
const eventType = `${subgraphName}.${write.table}.
|
|
1280
|
+
const eventType = `${subgraphName}.${write.table}.${OP_VERB[write.op]}`;
|
|
1278
1281
|
for (const s of subs) {
|
|
1279
1282
|
rows.push({
|
|
1280
1283
|
subscription_id: s.id,
|
|
@@ -2515,16 +2518,93 @@ import {
|
|
|
2515
2518
|
pgSchemaName as pgSchemaName2,
|
|
2516
2519
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
2517
2520
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2518
|
-
import { logger as
|
|
2521
|
+
import { logger as logger15 } from "@secondlayer/shared/logger";
|
|
2519
2522
|
import {
|
|
2520
2523
|
listen as listen2,
|
|
2521
2524
|
sourceListenerUrl,
|
|
2522
|
-
targetListenerUrl as
|
|
2525
|
+
targetListenerUrl as targetListenerUrl4
|
|
2523
2526
|
} from "@secondlayer/shared/queue/listener";
|
|
2524
2527
|
|
|
2528
|
+
// src/runtime/catchup-leader.ts
|
|
2529
|
+
import {
|
|
2530
|
+
SUBGRAPH_CATCHUP_LOCK_KEY,
|
|
2531
|
+
createPostgresLeaderBackend,
|
|
2532
|
+
withLeaderLock
|
|
2533
|
+
} from "@secondlayer/shared/leader";
|
|
2534
|
+
import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
2535
|
+
var catchUpLeader = false;
|
|
2536
|
+
function isCatchUpLeader() {
|
|
2537
|
+
return catchUpLeader;
|
|
2538
|
+
}
|
|
2539
|
+
function startCatchUpLeader(opts = {}) {
|
|
2540
|
+
return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
|
|
2541
|
+
catchUpLeader = true;
|
|
2542
|
+
await opts.onAcquire?.();
|
|
2543
|
+
return () => {
|
|
2544
|
+
catchUpLeader = false;
|
|
2545
|
+
};
|
|
2546
|
+
}, {
|
|
2547
|
+
pollMs: opts.pollMs,
|
|
2548
|
+
heartbeatMs: opts.heartbeatMs,
|
|
2549
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
|
|
2550
|
+
});
|
|
2551
|
+
}
|
|
2552
|
+
|
|
2553
|
+
// src/runtime/streams-reorg-poll.ts
|
|
2554
|
+
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
2555
|
+
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
2556
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2557
|
+
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
2558
|
+
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
2559
|
+
async function pollReorgsOnce(http, cursor, onReorg) {
|
|
2560
|
+
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
2561
|
+
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
2562
|
+
for (const r of sorted) {
|
|
2563
|
+
logger9.info("Streams reorg — rewinding", {
|
|
2564
|
+
forkPointHeight: r.fork_point_height
|
|
2565
|
+
});
|
|
2566
|
+
await onReorg(r.fork_point_height);
|
|
2567
|
+
}
|
|
2568
|
+
return next_since ?? cursor;
|
|
2569
|
+
}
|
|
2570
|
+
function startStreamsReorgPoll(onReorg) {
|
|
2571
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
2572
|
+
const http = new IndexHttpClient2({
|
|
2573
|
+
indexBaseUrl: baseUrl,
|
|
2574
|
+
streamsBaseUrl: baseUrl,
|
|
2575
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
2576
|
+
});
|
|
2577
|
+
let since = new Date(Date.now() - STARTUP_MARGIN_MS).toISOString();
|
|
2578
|
+
let running = true;
|
|
2579
|
+
let timer;
|
|
2580
|
+
const tick = async () => {
|
|
2581
|
+
if (!running)
|
|
2582
|
+
return;
|
|
2583
|
+
try {
|
|
2584
|
+
since = await pollReorgsOnce(http, since, onReorg);
|
|
2585
|
+
} catch (err) {
|
|
2586
|
+
logger9.error("Streams reorg poll failed", {
|
|
2587
|
+
error: getErrorMessage4(err)
|
|
2588
|
+
});
|
|
2589
|
+
}
|
|
2590
|
+
if (running)
|
|
2591
|
+
timer = setTimeout(tick, POLL_MS);
|
|
2592
|
+
};
|
|
2593
|
+
timer = setTimeout(tick, POLL_MS);
|
|
2594
|
+
logger9.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
2595
|
+
return () => {
|
|
2596
|
+
running = false;
|
|
2597
|
+
if (timer)
|
|
2598
|
+
clearTimeout(timer);
|
|
2599
|
+
};
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
// src/runtime/subscription-plane.ts
|
|
2603
|
+
import { logger as logger14 } from "@secondlayer/shared/logger";
|
|
2604
|
+
|
|
2525
2605
|
// src/runtime/chain-reorg.ts
|
|
2526
2606
|
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
2527
|
-
import { logger as
|
|
2607
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2528
2608
|
var MAX_ORPHANED_PER_SUB = 500;
|
|
2529
2609
|
async function handleChainReorg(forkHeight, db = getTargetDb5()) {
|
|
2530
2610
|
await db.deleteFrom("subscription_outbox").where("kind", "=", "chain").where("block_height", ">=", forkHeight).where("status", "=", "pending").where("event_type", "like", "chain.%.apply").execute();
|
|
@@ -2560,7 +2640,7 @@ async function handleChainReorg(forkHeight, db = getTargetDb5()) {
|
|
|
2560
2640
|
});
|
|
2561
2641
|
}
|
|
2562
2642
|
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
|
|
2563
|
-
|
|
2643
|
+
logger10.info("Chain reorg — emitted rollbacks", {
|
|
2564
2644
|
forkPointHeight: forkHeight,
|
|
2565
2645
|
subscriptions: bySub.size
|
|
2566
2646
|
});
|
|
@@ -2578,12 +2658,13 @@ import {
|
|
|
2578
2658
|
getTargetDb as getTargetDb6
|
|
2579
2659
|
} from "@secondlayer/shared/db";
|
|
2580
2660
|
import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2581
|
-
import { logger as
|
|
2582
|
-
import { listen, targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
2661
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
2662
|
+
import { listen, targetListenerUrl as targetListenerUrl2 } from "@secondlayer/shared/queue/listener";
|
|
2583
2663
|
import { sql as sql4 } from "kysely";
|
|
2584
2664
|
|
|
2585
2665
|
// src/runtime/formats/index.ts
|
|
2586
|
-
import {
|
|
2666
|
+
import { signSecondlayerWebhook } from "@secondlayer/shared/crypto/secondlayer-webhook";
|
|
2667
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
2587
2668
|
|
|
2588
2669
|
// src/runtime/formats/cloudevents.ts
|
|
2589
2670
|
function buildCloudEvents(outboxRow, _sub) {
|
|
@@ -2715,7 +2796,7 @@ function buildTrigger(outboxRow, sub) {
|
|
|
2715
2796
|
}
|
|
2716
2797
|
|
|
2717
2798
|
// src/runtime/formats/index.ts
|
|
2718
|
-
function
|
|
2799
|
+
function buildBody(outboxRow, sub, signingSecret) {
|
|
2719
2800
|
switch (sub.format) {
|
|
2720
2801
|
case "inngest":
|
|
2721
2802
|
return buildInngest(outboxRow);
|
|
@@ -2730,13 +2811,21 @@ function buildForFormat(outboxRow, sub, signingSecret) {
|
|
|
2730
2811
|
case "standard-webhooks":
|
|
2731
2812
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2732
2813
|
default:
|
|
2733
|
-
|
|
2814
|
+
logger11.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
2734
2815
|
format: sub.format,
|
|
2735
2816
|
subscriptionId: sub.id
|
|
2736
2817
|
});
|
|
2737
2818
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2738
2819
|
}
|
|
2739
2820
|
}
|
|
2821
|
+
function buildForFormat(outboxRow, sub, signingSecret) {
|
|
2822
|
+
const result = buildBody(outboxRow, sub, signingSecret);
|
|
2823
|
+
const sigHeaders = signSecondlayerWebhook(outboxRow.id, result.body);
|
|
2824
|
+
if (sigHeaders) {
|
|
2825
|
+
result.headers = { ...result.headers, ...sigHeaders };
|
|
2826
|
+
}
|
|
2827
|
+
return result;
|
|
2828
|
+
}
|
|
2740
2829
|
|
|
2741
2830
|
// src/runtime/emitter.ts
|
|
2742
2831
|
var BATCH_SIZE = 50;
|
|
@@ -2813,7 +2902,7 @@ async function dispatchOne(db, outboxRow, sub) {
|
|
|
2813
2902
|
let responseHeaders = {};
|
|
2814
2903
|
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
2815
2904
|
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
2816
|
-
|
|
2905
|
+
logger12.warn("[emitter] refused private egress", {
|
|
2817
2906
|
subscription: sub.name,
|
|
2818
2907
|
url: sub.url
|
|
2819
2908
|
});
|
|
@@ -2909,7 +2998,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
2909
2998
|
circuit_opened_at: new Date,
|
|
2910
2999
|
updated_at: new Date
|
|
2911
3000
|
}).where("id", "=", sub.id).execute();
|
|
2912
|
-
|
|
3001
|
+
logger12.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
2913
3002
|
subscription: sub.name,
|
|
2914
3003
|
failures: newFailures
|
|
2915
3004
|
});
|
|
@@ -2997,7 +3086,7 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
2997
3086
|
await settleFailed(db, row, sub, err);
|
|
2998
3087
|
}
|
|
2999
3088
|
} catch (err) {
|
|
3000
|
-
|
|
3089
|
+
logger12.error("Emitter dispatch crashed", {
|
|
3001
3090
|
outboxId: row.id,
|
|
3002
3091
|
error: err instanceof Error ? err.message : String(err)
|
|
3003
3092
|
});
|
|
@@ -3034,7 +3123,7 @@ async function startEmitter(opts) {
|
|
|
3034
3123
|
};
|
|
3035
3124
|
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
3036
3125
|
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
3037
|
-
|
|
3126
|
+
logger12.info("[emitter] started", { id: emitterId });
|
|
3038
3127
|
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
3039
3128
|
let lastErr = null;
|
|
3040
3129
|
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
@@ -3045,7 +3134,7 @@ async function startEmitter(opts) {
|
|
|
3045
3134
|
} catch (err) {
|
|
3046
3135
|
lastErr = err;
|
|
3047
3136
|
const delayMs = 500 * 2 ** i;
|
|
3048
|
-
|
|
3137
|
+
logger12.warn("[emitter] matcher refresh failed, retrying", {
|
|
3049
3138
|
attempt: i + 1,
|
|
3050
3139
|
delayMs,
|
|
3051
3140
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -3056,25 +3145,25 @@ async function startEmitter(opts) {
|
|
|
3056
3145
|
if (lastErr) {
|
|
3057
3146
|
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
3058
3147
|
}
|
|
3059
|
-
const listenUrl =
|
|
3148
|
+
const listenUrl = targetListenerUrl2();
|
|
3060
3149
|
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
3061
3150
|
if (!state.running)
|
|
3062
3151
|
return;
|
|
3063
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3152
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger12.error("[emitter] claim failed", {
|
|
3064
3153
|
error: err instanceof Error ? err.message : String(err)
|
|
3065
3154
|
}));
|
|
3066
3155
|
}, { connectionString: listenUrl });
|
|
3067
3156
|
const stopChanged = await listen("subscriptions:changed", () => {
|
|
3068
3157
|
if (!state.running)
|
|
3069
3158
|
return;
|
|
3070
|
-
refreshMatcher(db).catch((err) =>
|
|
3159
|
+
refreshMatcher(db).catch((err) => logger12.error("[emitter] matcher refresh failed", {
|
|
3071
3160
|
error: err instanceof Error ? err.message : String(err)
|
|
3072
3161
|
}));
|
|
3073
3162
|
}, { connectionString: listenUrl });
|
|
3074
3163
|
const poll = setInterval(() => {
|
|
3075
3164
|
if (!state.running)
|
|
3076
3165
|
return;
|
|
3077
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3166
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger12.error("[emitter] poll claim failed", {
|
|
3078
3167
|
error: err instanceof Error ? err.message : String(err)
|
|
3079
3168
|
}));
|
|
3080
3169
|
}, pollIntervalMs);
|
|
@@ -3082,7 +3171,7 @@ async function startEmitter(opts) {
|
|
|
3082
3171
|
const retention = setInterval(() => {
|
|
3083
3172
|
if (!state.running)
|
|
3084
3173
|
return;
|
|
3085
|
-
runRetention(db).catch((err) =>
|
|
3174
|
+
runRetention(db).catch((err) => logger12.error("[emitter] retention failed", {
|
|
3086
3175
|
error: err instanceof Error ? err.message : String(err)
|
|
3087
3176
|
}));
|
|
3088
3177
|
}, retentionIntervalMs);
|
|
@@ -3092,60 +3181,17 @@ async function startEmitter(opts) {
|
|
|
3092
3181
|
clearInterval(retention);
|
|
3093
3182
|
await stopNew();
|
|
3094
3183
|
await stopChanged();
|
|
3095
|
-
|
|
3184
|
+
logger12.info("[emitter] stopped", { id: emitterId });
|
|
3096
3185
|
};
|
|
3097
3186
|
}
|
|
3098
3187
|
|
|
3099
|
-
// src/runtime/
|
|
3100
|
-
import {
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
3107
|
-
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3108
|
-
for (const r of sorted) {
|
|
3109
|
-
logger12.info("Streams reorg — rewinding subgraphs", {
|
|
3110
|
-
forkPointHeight: r.fork_point_height
|
|
3111
|
-
});
|
|
3112
|
-
await handleReorg(r.fork_point_height, loadDef);
|
|
3113
|
-
if (handleChainReorg2)
|
|
3114
|
-
await handleChainReorg2(r.fork_point_height);
|
|
3115
|
-
}
|
|
3116
|
-
return next_since ?? cursor;
|
|
3117
|
-
}
|
|
3118
|
-
function startStreamsReorgPoll(handleReorg, loadDef, handleChainReorg2) {
|
|
3119
|
-
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
3120
|
-
const http = new IndexHttpClient2({
|
|
3121
|
-
indexBaseUrl: baseUrl,
|
|
3122
|
-
streamsBaseUrl: baseUrl,
|
|
3123
|
-
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
3124
|
-
});
|
|
3125
|
-
let since = new Date(Date.now() - STARTUP_MARGIN_MS).toISOString();
|
|
3126
|
-
let running = true;
|
|
3127
|
-
let timer;
|
|
3128
|
-
const tick = async () => {
|
|
3129
|
-
if (!running)
|
|
3130
|
-
return;
|
|
3131
|
-
try {
|
|
3132
|
-
since = await pollReorgsOnce(http, since, handleReorg, loadDef, handleChainReorg2);
|
|
3133
|
-
} catch (err) {
|
|
3134
|
-
logger12.error("Streams reorg poll failed", {
|
|
3135
|
-
error: getErrorMessage4(err)
|
|
3136
|
-
});
|
|
3137
|
-
}
|
|
3138
|
-
if (running)
|
|
3139
|
-
timer = setTimeout(tick, POLL_MS);
|
|
3140
|
-
};
|
|
3141
|
-
timer = setTimeout(tick, POLL_MS);
|
|
3142
|
-
logger12.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3143
|
-
return () => {
|
|
3144
|
-
running = false;
|
|
3145
|
-
if (timer)
|
|
3146
|
-
clearTimeout(timer);
|
|
3147
|
-
};
|
|
3148
|
-
}
|
|
3188
|
+
// src/runtime/subscription-leader.ts
|
|
3189
|
+
import {
|
|
3190
|
+
SUBSCRIPTION_EVALUATOR_LOCK_KEY,
|
|
3191
|
+
createPostgresLeaderBackend as createPostgresLeaderBackend2,
|
|
3192
|
+
withLeaderLock as withLeaderLock2
|
|
3193
|
+
} from "@secondlayer/shared/leader";
|
|
3194
|
+
import { targetListenerUrl as targetListenerUrl3 } from "@secondlayer/shared/queue/listener";
|
|
3149
3195
|
|
|
3150
3196
|
// src/runtime/trigger-evaluator-loop.ts
|
|
3151
3197
|
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
@@ -3222,10 +3268,11 @@ async function buildTraitContracts(db, chainSubs, asOfBlock) {
|
|
|
3222
3268
|
function evaluateBlock(block, sources, traitContracts) {
|
|
3223
3269
|
return matchSources(sources, block.txs, block.events, traitContracts);
|
|
3224
3270
|
}
|
|
3225
|
-
function chainDedupKey(subscriptionId, txId, eventIndex, blockHash) {
|
|
3226
|
-
|
|
3271
|
+
function chainDedupKey(subscriptionId, txId, eventIndex, blockHash, replayId) {
|
|
3272
|
+
const base = `chain:${subscriptionId}:${txId}:${eventIndex}:${blockHash}`;
|
|
3273
|
+
return replayId ? `replay:${replayId}:${base}` : base;
|
|
3227
3274
|
}
|
|
3228
|
-
function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event) {
|
|
3275
|
+
function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event, replayId) {
|
|
3229
3276
|
const payload = {
|
|
3230
3277
|
action: "apply",
|
|
3231
3278
|
block_hash: blockHash,
|
|
@@ -3245,10 +3292,12 @@ function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event) {
|
|
|
3245
3292
|
row_pk: { tx_id: txId, event_index: eventIndex },
|
|
3246
3293
|
event_type: `chain.${meta.triggerType}.apply`,
|
|
3247
3294
|
payload,
|
|
3248
|
-
dedup_key: chainDedupKey(meta.subscriptionId, txId, eventIndex, blockHash)
|
|
3295
|
+
dedup_key: chainDedupKey(meta.subscriptionId, txId, eventIndex, blockHash, replayId),
|
|
3296
|
+
...replayId ? { is_replay: true } : {}
|
|
3249
3297
|
};
|
|
3250
3298
|
}
|
|
3251
|
-
async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
3299
|
+
async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash, opts) {
|
|
3300
|
+
const replayId = opts?.replayId;
|
|
3252
3301
|
const rows = [];
|
|
3253
3302
|
for (const match of matches) {
|
|
3254
3303
|
const meta = keyMeta.get(match.sourceName);
|
|
@@ -3265,7 +3314,7 @@ async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
|
3265
3314
|
function_name: match.tx.function_name ?? null,
|
|
3266
3315
|
function_args: match.tx.function_args ?? null,
|
|
3267
3316
|
result_hex: match.tx.raw_result ?? null
|
|
3268
|
-
}));
|
|
3317
|
+
}, replayId));
|
|
3269
3318
|
} else {
|
|
3270
3319
|
for (const event of match.events) {
|
|
3271
3320
|
rows.push(applyRow(meta, blockHeight, blockHash, txId, event.event_index, {
|
|
@@ -3273,14 +3322,14 @@ async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
|
3273
3322
|
type: event.type,
|
|
3274
3323
|
event_index: event.event_index,
|
|
3275
3324
|
data: event.data
|
|
3276
|
-
}));
|
|
3325
|
+
}, replayId));
|
|
3277
3326
|
}
|
|
3278
3327
|
}
|
|
3279
3328
|
}
|
|
3280
3329
|
if (rows.length === 0)
|
|
3281
3330
|
return 0;
|
|
3282
|
-
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).
|
|
3283
|
-
return
|
|
3331
|
+
const result = await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).executeTakeFirst();
|
|
3332
|
+
return Number(result.numInsertedOrUpdatedRows ?? 0);
|
|
3284
3333
|
}
|
|
3285
3334
|
|
|
3286
3335
|
// src/runtime/trigger-evaluator-loop.ts
|
|
@@ -3362,6 +3411,48 @@ function startTriggerEvaluator() {
|
|
|
3362
3411
|
};
|
|
3363
3412
|
}
|
|
3364
3413
|
|
|
3414
|
+
// src/runtime/subscription-leader.ts
|
|
3415
|
+
var evaluatorLeader = false;
|
|
3416
|
+
function isEvaluatorLeader() {
|
|
3417
|
+
return evaluatorLeader;
|
|
3418
|
+
}
|
|
3419
|
+
function gateChainReorgOnLeader(handler, isLeader = isEvaluatorLeader) {
|
|
3420
|
+
return async (forkHeight) => {
|
|
3421
|
+
if (!isLeader())
|
|
3422
|
+
return;
|
|
3423
|
+
await handler(forkHeight);
|
|
3424
|
+
};
|
|
3425
|
+
}
|
|
3426
|
+
function startTriggerEvaluatorLeader(opts = {}) {
|
|
3427
|
+
const startWork = opts.startWork ?? startTriggerEvaluator;
|
|
3428
|
+
return withLeaderLock2(SUBSCRIPTION_EVALUATOR_LOCK_KEY, async () => {
|
|
3429
|
+
evaluatorLeader = true;
|
|
3430
|
+
const stop = await startWork();
|
|
3431
|
+
return () => {
|
|
3432
|
+
evaluatorLeader = false;
|
|
3433
|
+
stop();
|
|
3434
|
+
};
|
|
3435
|
+
}, {
|
|
3436
|
+
pollMs: opts.pollMs,
|
|
3437
|
+
heartbeatMs: opts.heartbeatMs,
|
|
3438
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend2(targetListenerUrl3()))
|
|
3439
|
+
});
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
// src/runtime/subscription-plane.ts
|
|
3443
|
+
async function startSubscriptionPlane() {
|
|
3444
|
+
const streamsIndex = process.env.SUBGRAPH_SOURCE === "streams-index";
|
|
3445
|
+
const stopChainReorgPoll = streamsIndex ? startStreamsReorgPoll(gateChainReorgOnLeader((forkHeight) => handleChainReorg(forkHeight))) : undefined;
|
|
3446
|
+
const stopTriggerEvaluator = streamsIndex ? startTriggerEvaluatorLeader() : undefined;
|
|
3447
|
+
const stopEmitter = await startEmitter();
|
|
3448
|
+
logger14.info("Subscription plane ready", { streamsIndex });
|
|
3449
|
+
return async () => {
|
|
3450
|
+
stopChainReorgPoll?.();
|
|
3451
|
+
await stopTriggerEvaluator?.();
|
|
3452
|
+
await stopEmitter();
|
|
3453
|
+
};
|
|
3454
|
+
}
|
|
3455
|
+
|
|
3365
3456
|
// src/runtime/processor.ts
|
|
3366
3457
|
var CHANNEL_NEW_BLOCK = "indexer:new_block";
|
|
3367
3458
|
var CHANNEL_SUBGRAPH_OPERATIONS = "subgraph_operations:new";
|
|
@@ -3385,7 +3476,7 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3385
3476
|
if (isHandlerNotFoundError(err)) {
|
|
3386
3477
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3387
3478
|
}
|
|
3388
|
-
|
|
3479
|
+
logger15.error("Subgraph catch-up failed", {
|
|
3389
3480
|
subgraph: sg.name,
|
|
3390
3481
|
error: msg
|
|
3391
3482
|
});
|
|
@@ -3425,7 +3516,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3425
3516
|
definitionCache.set(sg.name, def);
|
|
3426
3517
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3427
3518
|
invalidateSubgraphRoute(sg.name);
|
|
3428
|
-
|
|
3519
|
+
logger15.info("Subgraph handler reloaded", {
|
|
3429
3520
|
subgraph: sg.name,
|
|
3430
3521
|
from: prevVersion,
|
|
3431
3522
|
to: sg.version
|
|
@@ -3459,7 +3550,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3459
3550
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3460
3551
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3461
3552
|
});
|
|
3462
|
-
|
|
3553
|
+
logger15.info("Queued legacy reindex resume operation", {
|
|
3463
3554
|
subgraph: sg.name
|
|
3464
3555
|
});
|
|
3465
3556
|
} catch (err) {
|
|
@@ -3515,7 +3606,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3515
3606
|
const activeRuns = new Map;
|
|
3516
3607
|
let running = true;
|
|
3517
3608
|
let draining = false;
|
|
3518
|
-
|
|
3609
|
+
logger15.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3519
3610
|
const startOperation = (operation) => {
|
|
3520
3611
|
const controller = new AbortController;
|
|
3521
3612
|
active.set(operation.id, controller);
|
|
@@ -3523,7 +3614,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3523
3614
|
if (!running)
|
|
3524
3615
|
return;
|
|
3525
3616
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3526
|
-
|
|
3617
|
+
logger15.warn("Subgraph operation heartbeat failed", {
|
|
3527
3618
|
operationId: operation.id,
|
|
3528
3619
|
error: getErrorMessage6(err)
|
|
3529
3620
|
});
|
|
@@ -3535,7 +3626,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3535
3626
|
controller.abort("user-cancelled");
|
|
3536
3627
|
}
|
|
3537
3628
|
}).catch((err) => {
|
|
3538
|
-
|
|
3629
|
+
logger15.warn("Subgraph operation cancel poll failed", {
|
|
3539
3630
|
operationId: operation.id,
|
|
3540
3631
|
error: getErrorMessage6(err)
|
|
3541
3632
|
});
|
|
@@ -3552,14 +3643,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3552
3643
|
const reason = String(controller.signal.reason ?? "");
|
|
3553
3644
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3554
3645
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3555
|
-
|
|
3646
|
+
logger15.info("Subgraph operation cancelled", {
|
|
3556
3647
|
operationId: operation.id,
|
|
3557
3648
|
subgraph: operation.subgraph_name
|
|
3558
3649
|
});
|
|
3559
3650
|
return;
|
|
3560
3651
|
}
|
|
3561
3652
|
if (controller.signal.aborted) {
|
|
3562
|
-
|
|
3653
|
+
logger15.info("Subgraph operation interrupted", {
|
|
3563
3654
|
operationId: operation.id,
|
|
3564
3655
|
subgraph: operation.subgraph_name,
|
|
3565
3656
|
reason
|
|
@@ -3567,7 +3658,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3567
3658
|
return;
|
|
3568
3659
|
}
|
|
3569
3660
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3570
|
-
|
|
3661
|
+
logger15.info("Subgraph operation completed", {
|
|
3571
3662
|
operationId: operation.id,
|
|
3572
3663
|
subgraph: operation.subgraph_name,
|
|
3573
3664
|
processed
|
|
@@ -3575,7 +3666,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3575
3666
|
} catch (err) {
|
|
3576
3667
|
const reason = String(controller.signal.reason ?? "");
|
|
3577
3668
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3578
|
-
|
|
3669
|
+
logger15.info("Subgraph operation interrupted by shutdown", {
|
|
3579
3670
|
operationId: operation.id,
|
|
3580
3671
|
subgraph: operation.subgraph_name
|
|
3581
3672
|
});
|
|
@@ -3586,7 +3677,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3586
3677
|
return;
|
|
3587
3678
|
}
|
|
3588
3679
|
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
|
|
3589
|
-
|
|
3680
|
+
logger15.error("Subgraph operation failed", {
|
|
3590
3681
|
operationId: operation.id,
|
|
3591
3682
|
subgraph: operation.subgraph_name,
|
|
3592
3683
|
error: getErrorMessage6(err)
|
|
@@ -3621,7 +3712,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3621
3712
|
await drain();
|
|
3622
3713
|
const stopListening = await listen2(CHANNEL_SUBGRAPH_OPERATIONS, () => {
|
|
3623
3714
|
drain();
|
|
3624
|
-
}, { connectionString:
|
|
3715
|
+
}, { connectionString: targetListenerUrl4() });
|
|
3625
3716
|
const pollInterval = setInterval(() => {
|
|
3626
3717
|
drain();
|
|
3627
3718
|
}, POLL_INTERVAL_MS);
|
|
@@ -3633,26 +3724,27 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3633
3724
|
controller.abort("shutdown");
|
|
3634
3725
|
}
|
|
3635
3726
|
await Promise.allSettled(activeRuns.values());
|
|
3636
|
-
|
|
3727
|
+
logger15.info("Subgraph operation runner stopped");
|
|
3637
3728
|
};
|
|
3638
3729
|
}
|
|
3639
3730
|
async function startSubgraphProcessor(opts) {
|
|
3640
3731
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3641
3732
|
let running = true;
|
|
3642
|
-
|
|
3733
|
+
logger15.info("Starting subgraph processor", { concurrency });
|
|
3643
3734
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3644
3735
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3645
3736
|
});
|
|
3646
|
-
const
|
|
3647
|
-
|
|
3648
|
-
await catchUpAll(activeSubgraphs, targetDb, concurrency);
|
|
3649
|
-
const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
|
|
3650
|
-
if (!running)
|
|
3737
|
+
const runCatchUp = async () => {
|
|
3738
|
+
if (!running || !isCatchUpLeader())
|
|
3651
3739
|
return;
|
|
3652
3740
|
const db = getTargetDb8();
|
|
3653
3741
|
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3654
3742
|
cleanupCaches(subgraphs);
|
|
3655
3743
|
await catchUpAll(subgraphs, db, concurrency);
|
|
3744
|
+
};
|
|
3745
|
+
const stopCatchUpLeader = startCatchUpLeader({ onAcquire: runCatchUp });
|
|
3746
|
+
const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
|
|
3747
|
+
await runCatchUp();
|
|
3656
3748
|
}, { connectionString: sourceListenerUrl() });
|
|
3657
3749
|
const stopReorgListening = await listen2("subgraph_reorg", async (payload) => {
|
|
3658
3750
|
if (!running)
|
|
@@ -3664,39 +3756,33 @@ async function startSubgraphProcessor(opts) {
|
|
|
3664
3756
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3665
3757
|
}
|
|
3666
3758
|
} catch (err) {
|
|
3667
|
-
|
|
3759
|
+
logger15.error("Subgraph reorg handling failed", {
|
|
3668
3760
|
error: getErrorMessage6(err)
|
|
3669
3761
|
});
|
|
3670
3762
|
}
|
|
3671
3763
|
}, { connectionString: sourceListenerUrl() });
|
|
3672
|
-
const pollInterval = setInterval(
|
|
3673
|
-
|
|
3674
|
-
return;
|
|
3675
|
-
const db = getTargetDb8();
|
|
3676
|
-
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3677
|
-
cleanupCaches(subgraphs);
|
|
3678
|
-
await catchUpAll(subgraphs, db, concurrency);
|
|
3764
|
+
const pollInterval = setInterval(() => {
|
|
3765
|
+
runCatchUp();
|
|
3679
3766
|
}, POLL_INTERVAL_MS);
|
|
3680
|
-
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll(
|
|
3681
|
-
const
|
|
3682
|
-
|
|
3683
|
-
logger14.info("Subgraph processor ready");
|
|
3767
|
+
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
3768
|
+
const stopSubscriptionPlane = await startSubscriptionPlane();
|
|
3769
|
+
logger15.info("Subgraph processor ready");
|
|
3684
3770
|
return async () => {
|
|
3685
3771
|
running = false;
|
|
3686
3772
|
clearInterval(pollInterval);
|
|
3773
|
+
await stopCatchUpLeader();
|
|
3687
3774
|
await stopListening();
|
|
3688
3775
|
await stopReorgListening();
|
|
3689
3776
|
stopStreamsReorgPoll?.();
|
|
3690
|
-
|
|
3777
|
+
await stopSubscriptionPlane();
|
|
3691
3778
|
await stopOperations();
|
|
3692
|
-
|
|
3693
|
-
logger14.info("Subgraph processor stopped");
|
|
3779
|
+
logger15.info("Subgraph processor stopped");
|
|
3694
3780
|
};
|
|
3695
3781
|
}
|
|
3696
3782
|
|
|
3697
3783
|
// src/service.ts
|
|
3698
3784
|
import { assertDbSplit, getDb } from "@secondlayer/shared/db";
|
|
3699
|
-
import { logger as
|
|
3785
|
+
import { logger as logger16 } from "@secondlayer/shared/logger";
|
|
3700
3786
|
import { sql as sql5 } from "kysely";
|
|
3701
3787
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
3702
3788
|
var SERVICE_NAME = "subgraph-processor";
|
|
@@ -3704,7 +3790,7 @@ async function writeHeartbeat() {
|
|
|
3704
3790
|
try {
|
|
3705
3791
|
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql5`now()` })).execute();
|
|
3706
3792
|
} catch (err) {
|
|
3707
|
-
|
|
3793
|
+
logger16.warn("subgraph-processor heartbeat write failed", {
|
|
3708
3794
|
error: err instanceof Error ? err.message : String(err)
|
|
3709
3795
|
});
|
|
3710
3796
|
}
|
|
@@ -3716,7 +3802,7 @@ var processor = await startSubgraphProcessor({
|
|
|
3716
3802
|
await writeHeartbeat();
|
|
3717
3803
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
3718
3804
|
var shutdown = async () => {
|
|
3719
|
-
|
|
3805
|
+
logger16.info("Shutting down subgraph processor...");
|
|
3720
3806
|
clearInterval(heartbeatInterval);
|
|
3721
3807
|
await processor();
|
|
3722
3808
|
process.exit(0);
|
|
@@ -3724,5 +3810,5 @@ var shutdown = async () => {
|
|
|
3724
3810
|
process.on("SIGINT", shutdown);
|
|
3725
3811
|
process.on("SIGTERM", shutdown);
|
|
3726
3812
|
|
|
3727
|
-
//# debugId=
|
|
3813
|
+
//# debugId=6699532419F8689F64756E2164756E21
|
|
3728
3814
|
//# sourceMappingURL=service.js.map
|