@secondlayer/subgraphs 3.24.1 → 3.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +361 -39
- package/dist/src/index.js.map +7 -4
- package/dist/src/runtime/block-processor.d.ts +8 -1
- package/dist/src/runtime/block-processor.js +331 -9
- package/dist/src/runtime/block-processor.js.map +7 -4
- package/dist/src/runtime/catchup.js +340 -18
- package/dist/src/runtime/catchup.js.map +7 -4
- package/dist/src/runtime/processor.js +401 -79
- package/dist/src/runtime/processor.js.map +7 -4
- package/dist/src/runtime/reindex.js +361 -39
- package/dist/src/runtime/reindex.js.map +7 -4
- package/dist/src/runtime/reorg.js +347 -25
- package/dist/src/runtime/reorg.js.map +7 -4
- package/dist/src/service.js +404 -82
- package/dist/src/service.js.map +7 -4
- package/package.json +1 -1
package/dist/src/service.js
CHANGED
|
@@ -1639,7 +1639,7 @@ import {
|
|
|
1639
1639
|
rewindLiveProgress,
|
|
1640
1640
|
updateSubgraphStatus
|
|
1641
1641
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1642
|
-
import { logger as
|
|
1642
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1643
1643
|
import { sql as sql3 } from "kysely";
|
|
1644
1644
|
|
|
1645
1645
|
// src/runtime/block-source.ts
|
|
@@ -2080,6 +2080,315 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2080
2080
|
return rows.length;
|
|
2081
2081
|
}
|
|
2082
2082
|
|
|
2083
|
+
// src/runtime/sandbox/flag.ts
|
|
2084
|
+
function sandboxWorkersGloballyEnabled() {
|
|
2085
|
+
return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
|
|
2086
|
+
}
|
|
2087
|
+
function sandboxEnabled(subgraph) {
|
|
2088
|
+
return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
// src/runtime/sandbox/host.ts
|
|
2092
|
+
import { tmpdir } from "node:os";
|
|
2093
|
+
import { fileURLToPath } from "node:url";
|
|
2094
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2095
|
+
|
|
2096
|
+
// src/runtime/sandbox/bundle.ts
|
|
2097
|
+
import esbuild from "esbuild";
|
|
2098
|
+
var ALLOWED_BARE_SPECIFIERS = new Set;
|
|
2099
|
+
var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
|
|
2100
|
+
var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
|
|
2101
|
+
`;
|
|
2102
|
+
function resolverLockdownPlugin() {
|
|
2103
|
+
return {
|
|
2104
|
+
name: "f071-sandbox-resolver-lockdown",
|
|
2105
|
+
setup(build) {
|
|
2106
|
+
build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
|
|
2107
|
+
path: args.path,
|
|
2108
|
+
namespace: "f071-sandbox-subgraphs-stub"
|
|
2109
|
+
}));
|
|
2110
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
|
|
2111
|
+
build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
|
|
2112
|
+
path: args.path,
|
|
2113
|
+
namespace: "f071-sandbox-blocked"
|
|
2114
|
+
}));
|
|
2115
|
+
build.onResolve({ filter: /^[^./]/ }, (args) => {
|
|
2116
|
+
if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
|
|
2117
|
+
return;
|
|
2118
|
+
if (ALLOWED_BARE_SPECIFIERS.has(args.path))
|
|
2119
|
+
return;
|
|
2120
|
+
return { path: args.path, namespace: "f071-sandbox-blocked" };
|
|
2121
|
+
});
|
|
2122
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
|
|
2123
|
+
contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
|
|
2124
|
+
loader: "js"
|
|
2125
|
+
}));
|
|
2126
|
+
}
|
|
2127
|
+
};
|
|
2128
|
+
}
|
|
2129
|
+
async function bundleHandlerCode(handlerCode) {
|
|
2130
|
+
const result = await esbuild.build({
|
|
2131
|
+
stdin: {
|
|
2132
|
+
contents: handlerCode,
|
|
2133
|
+
loader: "ts",
|
|
2134
|
+
resolveDir: process.cwd()
|
|
2135
|
+
},
|
|
2136
|
+
bundle: true,
|
|
2137
|
+
platform: "neutral",
|
|
2138
|
+
format: "esm",
|
|
2139
|
+
write: false,
|
|
2140
|
+
logLevel: "silent",
|
|
2141
|
+
plugins: [resolverLockdownPlugin()]
|
|
2142
|
+
});
|
|
2143
|
+
const out = result.outputFiles?.[0];
|
|
2144
|
+
if (!out)
|
|
2145
|
+
throw new Error("esbuild produced no output for handler_code bundle");
|
|
2146
|
+
return new TextDecoder().decode(out.contents);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// src/runtime/sandbox/host.ts
|
|
2150
|
+
var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
|
|
2151
|
+
var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
|
|
2152
|
+
function spawnSandboxProcess() {
|
|
2153
|
+
let onMessage = null;
|
|
2154
|
+
const proc = Bun.spawn({
|
|
2155
|
+
cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
|
|
2156
|
+
env: { PATH: process.env.PATH ?? "" },
|
|
2157
|
+
cwd: tmpdir(),
|
|
2158
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
2159
|
+
serialization: "advanced",
|
|
2160
|
+
ipc(message) {
|
|
2161
|
+
onMessage?.(message);
|
|
2162
|
+
}
|
|
2163
|
+
});
|
|
2164
|
+
(async () => {
|
|
2165
|
+
const reader = proc.stderr.getReader();
|
|
2166
|
+
const decoder = new TextDecoder;
|
|
2167
|
+
try {
|
|
2168
|
+
while (true) {
|
|
2169
|
+
const { done, value } = await reader.read();
|
|
2170
|
+
if (done)
|
|
2171
|
+
break;
|
|
2172
|
+
const text = decoder.decode(value, { stream: true });
|
|
2173
|
+
if (text.trim()) {
|
|
2174
|
+
logger6.warn("Sandbox subprocess stderr", { stderr: text.trim() });
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
} catch {}
|
|
2178
|
+
})();
|
|
2179
|
+
return {
|
|
2180
|
+
send: (msg) => proc.send(msg),
|
|
2181
|
+
kill: () => proc.kill(),
|
|
2182
|
+
exited: proc.exited,
|
|
2183
|
+
setOnMessage: (handler) => {
|
|
2184
|
+
onMessage = handler;
|
|
2185
|
+
}
|
|
2186
|
+
};
|
|
2187
|
+
}
|
|
2188
|
+
var pool = new Map;
|
|
2189
|
+
|
|
2190
|
+
class SandboxWorkerCrash extends Error {
|
|
2191
|
+
constructor(subgraphName, cause) {
|
|
2192
|
+
super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
|
|
2193
|
+
this.name = "SandboxWorkerCrash";
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
|
|
2197
|
+
const bundledCode = await bundleHandlerCode(handlerCode);
|
|
2198
|
+
let settled = false;
|
|
2199
|
+
const ready = new Promise((resolve, reject) => {
|
|
2200
|
+
proc.setOnMessage((msg) => {
|
|
2201
|
+
if (msg.type === "ready" && msg.version === version) {
|
|
2202
|
+
settled = true;
|
|
2203
|
+
proc.setOnMessage(null);
|
|
2204
|
+
resolve();
|
|
2205
|
+
} else if (msg.type === "error") {
|
|
2206
|
+
settled = true;
|
|
2207
|
+
proc.setOnMessage(null);
|
|
2208
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2209
|
+
}
|
|
2210
|
+
});
|
|
2211
|
+
const init = { type: "init", bundledCode, version };
|
|
2212
|
+
proc.send(init);
|
|
2213
|
+
});
|
|
2214
|
+
const crashed = proc.exited.then((code) => {
|
|
2215
|
+
if (settled)
|
|
2216
|
+
return;
|
|
2217
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
|
|
2218
|
+
});
|
|
2219
|
+
await Promise.race([ready, crashed]);
|
|
2220
|
+
}
|
|
2221
|
+
async function ensureSandboxProcess(subgraphName, version, handlerCode) {
|
|
2222
|
+
if (handlerCode == null) {
|
|
2223
|
+
throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
|
|
2224
|
+
}
|
|
2225
|
+
const existing = pool.get(subgraphName);
|
|
2226
|
+
if (existing && existing.version === version)
|
|
2227
|
+
return existing;
|
|
2228
|
+
if (existing) {
|
|
2229
|
+
try {
|
|
2230
|
+
await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
|
|
2231
|
+
existing.version = version;
|
|
2232
|
+
logger6.info("Sandbox subprocess handler reloaded", {
|
|
2233
|
+
subgraph: subgraphName,
|
|
2234
|
+
version
|
|
2235
|
+
});
|
|
2236
|
+
return existing;
|
|
2237
|
+
} catch (err) {
|
|
2238
|
+
existing.proc.kill();
|
|
2239
|
+
pool.delete(subgraphName);
|
|
2240
|
+
logger6.warn("Sandbox subprocess re-init failed; respawning", {
|
|
2241
|
+
subgraph: subgraphName,
|
|
2242
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2243
|
+
});
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
const proc = spawnSandboxProcess();
|
|
2247
|
+
try {
|
|
2248
|
+
await initSandboxProcess(proc, subgraphName, version, handlerCode);
|
|
2249
|
+
} catch (err) {
|
|
2250
|
+
proc.kill();
|
|
2251
|
+
throw err;
|
|
2252
|
+
}
|
|
2253
|
+
const entry = { proc, version, chain: Promise.resolve() };
|
|
2254
|
+
pool.set(subgraphName, entry);
|
|
2255
|
+
return entry;
|
|
2256
|
+
}
|
|
2257
|
+
function evictSandboxWorker(subgraphName) {
|
|
2258
|
+
const entry = pool.get(subgraphName);
|
|
2259
|
+
if (!entry)
|
|
2260
|
+
return;
|
|
2261
|
+
entry.proc.kill();
|
|
2262
|
+
pool.delete(subgraphName);
|
|
2263
|
+
}
|
|
2264
|
+
async function runHandlersSandboxed(params) {
|
|
2265
|
+
const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
|
|
2266
|
+
const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
|
|
2267
|
+
const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
|
|
2268
|
+
entry.chain = run.catch(() => {
|
|
2269
|
+
return;
|
|
2270
|
+
});
|
|
2271
|
+
return run;
|
|
2272
|
+
}
|
|
2273
|
+
async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
|
|
2274
|
+
const { proc } = entry;
|
|
2275
|
+
let timeoutHandle;
|
|
2276
|
+
let settled = false;
|
|
2277
|
+
try {
|
|
2278
|
+
const done = new Promise((resolve, reject) => {
|
|
2279
|
+
proc.setOnMessage((msg) => {
|
|
2280
|
+
if (msg.type === "readRequest") {
|
|
2281
|
+
answerRead(hostCtx, msg).then((reply) => {
|
|
2282
|
+
proc.send({ type: "readResponse", id: msg.id, reply });
|
|
2283
|
+
}).catch((err) => {
|
|
2284
|
+
settled = true;
|
|
2285
|
+
reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
2286
|
+
});
|
|
2287
|
+
return;
|
|
2288
|
+
}
|
|
2289
|
+
if (msg.type === "blockDone") {
|
|
2290
|
+
settled = true;
|
|
2291
|
+
resolve({
|
|
2292
|
+
ops: msg.ops,
|
|
2293
|
+
result: { processed: msg.processed, errors: msg.errors }
|
|
2294
|
+
});
|
|
2295
|
+
return;
|
|
2296
|
+
}
|
|
2297
|
+
if (msg.type === "error") {
|
|
2298
|
+
settled = true;
|
|
2299
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2300
|
+
}
|
|
2301
|
+
});
|
|
2302
|
+
proc.send({ type: "runBlock", block, matched });
|
|
2303
|
+
});
|
|
2304
|
+
const crashed = proc.exited.then((code) => {
|
|
2305
|
+
if (settled)
|
|
2306
|
+
return;
|
|
2307
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
|
|
2308
|
+
});
|
|
2309
|
+
const deadline = new Promise((_, reject) => {
|
|
2310
|
+
timeoutHandle = setTimeout(() => {
|
|
2311
|
+
settled = true;
|
|
2312
|
+
reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
|
|
2313
|
+
}, BLOCK_TIMEOUT_MS);
|
|
2314
|
+
});
|
|
2315
|
+
const { ops, result } = await Promise.race([done, crashed, deadline]);
|
|
2316
|
+
for (const op of ops) {
|
|
2317
|
+
hostCtx.setTx(op.tx);
|
|
2318
|
+
replayOp(hostCtx, op);
|
|
2319
|
+
}
|
|
2320
|
+
return result;
|
|
2321
|
+
} catch (err) {
|
|
2322
|
+
evictSandboxWorker(subgraphName);
|
|
2323
|
+
logger6.warn("Sandbox subprocess run failed; subprocess evicted", {
|
|
2324
|
+
subgraph: subgraphName,
|
|
2325
|
+
blockHeight: block.height,
|
|
2326
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2327
|
+
});
|
|
2328
|
+
throw err;
|
|
2329
|
+
} finally {
|
|
2330
|
+
if (timeoutHandle !== undefined)
|
|
2331
|
+
clearTimeout(timeoutHandle);
|
|
2332
|
+
proc.setOnMessage(null);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
function replayOp(hostCtx, op) {
|
|
2336
|
+
switch (op.method) {
|
|
2337
|
+
case "insert":
|
|
2338
|
+
hostCtx.insert(op.table, op.args[0]);
|
|
2339
|
+
return;
|
|
2340
|
+
case "upsert":
|
|
2341
|
+
hostCtx.upsert(op.table, op.args[0], op.args[1]);
|
|
2342
|
+
return;
|
|
2343
|
+
case "increment":
|
|
2344
|
+
hostCtx.increment(op.table, op.args[0], op.args[1]);
|
|
2345
|
+
return;
|
|
2346
|
+
case "update":
|
|
2347
|
+
hostCtx.update(op.table, op.args[0], op.args[1]);
|
|
2348
|
+
return;
|
|
2349
|
+
case "delete":
|
|
2350
|
+
hostCtx.delete(op.table, op.args[0]);
|
|
2351
|
+
return;
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
async function answerRead(hostCtx, msg) {
|
|
2355
|
+
const { method, table, where, column } = msg;
|
|
2356
|
+
switch (method) {
|
|
2357
|
+
case "findOne":
|
|
2358
|
+
return { kind: "row", row: await hostCtx.findOne(table, where) };
|
|
2359
|
+
case "findMany":
|
|
2360
|
+
return { kind: "rows", rows: await hostCtx.findMany(table, where) };
|
|
2361
|
+
case "count":
|
|
2362
|
+
return { kind: "count", count: await hostCtx.count(table, where) };
|
|
2363
|
+
case "countDistinct": {
|
|
2364
|
+
if (!column)
|
|
2365
|
+
throw new Error("countDistinct read missing column");
|
|
2366
|
+
return {
|
|
2367
|
+
kind: "count",
|
|
2368
|
+
count: await hostCtx.countDistinct(table, column, where)
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
case "sum": {
|
|
2372
|
+
if (!column)
|
|
2373
|
+
throw new Error("sum read missing column");
|
|
2374
|
+
const v = await hostCtx.sum(table, column, where);
|
|
2375
|
+
return { kind: "amount", amount: v.toString() };
|
|
2376
|
+
}
|
|
2377
|
+
case "min": {
|
|
2378
|
+
if (!column)
|
|
2379
|
+
throw new Error("min read missing column");
|
|
2380
|
+
const v = await hostCtx.min(table, column, where);
|
|
2381
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2382
|
+
}
|
|
2383
|
+
case "max": {
|
|
2384
|
+
if (!column)
|
|
2385
|
+
throw new Error("max read missing column");
|
|
2386
|
+
const v = await hostCtx.max(table, column, where);
|
|
2387
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2083
2392
|
// src/runtime/subscription-state.ts
|
|
2084
2393
|
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2085
2394
|
import { sql as sql2 } from "kysely";
|
|
@@ -2264,13 +2573,17 @@ async function resolveRoute(subgraphName, targetDb) {
|
|
|
2264
2573
|
const route = {
|
|
2265
2574
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2266
2575
|
dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
|
|
2267
|
-
byo
|
|
2576
|
+
byo,
|
|
2577
|
+
sandboxWorkers: row?.sandbox_workers === true,
|
|
2578
|
+
handlerCode: row?.handler_code ?? null,
|
|
2579
|
+
version: row?.version ?? ""
|
|
2268
2580
|
};
|
|
2269
2581
|
routeCache.set(subgraphName, route);
|
|
2270
2582
|
return route;
|
|
2271
2583
|
}
|
|
2272
2584
|
function invalidateSubgraphRoute(subgraphName) {
|
|
2273
2585
|
routeCache.delete(subgraphName);
|
|
2586
|
+
evictSandboxWorker(subgraphName);
|
|
2274
2587
|
}
|
|
2275
2588
|
async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
|
|
2276
2589
|
const factories = new Map;
|
|
@@ -2376,7 +2689,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
|
|
|
2376
2689
|
const delay = retryDelaysMs[attempt];
|
|
2377
2690
|
if (delay === undefined)
|
|
2378
2691
|
break;
|
|
2379
|
-
|
|
2692
|
+
logger7.warn("Block processing failed, retrying", {
|
|
2380
2693
|
subgraph: subgraphName,
|
|
2381
2694
|
blockHeight,
|
|
2382
2695
|
attempt: attempt + 1,
|
|
@@ -2408,7 +2721,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2408
2721
|
} else {
|
|
2409
2722
|
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
2410
2723
|
if (!data) {
|
|
2411
|
-
|
|
2724
|
+
logger7.debug("Block not found or non-canonical for subgraph processing", {
|
|
2412
2725
|
subgraph: subgraphName,
|
|
2413
2726
|
blockHeight
|
|
2414
2727
|
});
|
|
@@ -2544,7 +2857,16 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2544
2857
|
}
|
|
2545
2858
|
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2546
2859
|
const handlerStart = performance.now();
|
|
2547
|
-
const runResult =
|
|
2860
|
+
const runResult = sandboxEnabled({
|
|
2861
|
+
sandbox_workers: route.sandboxWorkers
|
|
2862
|
+
}) ? await runHandlersSandboxed({
|
|
2863
|
+
subgraphName,
|
|
2864
|
+
version: route.version,
|
|
2865
|
+
handlerCode: route.handlerCode,
|
|
2866
|
+
hostCtx: ctx,
|
|
2867
|
+
block: blockMeta,
|
|
2868
|
+
matched
|
|
2869
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2548
2870
|
handlerMs = performance.now() - handlerStart;
|
|
2549
2871
|
result.processed = runResult.processed;
|
|
2550
2872
|
result.errors = runResult.errors;
|
|
@@ -2570,7 +2892,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2570
2892
|
});
|
|
2571
2893
|
} catch (err) {
|
|
2572
2894
|
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2573
|
-
|
|
2895
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2574
2896
|
subgraph: subgraphName,
|
|
2575
2897
|
blockHeight,
|
|
2576
2898
|
error: err.message
|
|
@@ -2596,7 +2918,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2596
2918
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
2597
2919
|
const count = Number(rows[0]?.count ?? 0);
|
|
2598
2920
|
if (count >= 1e7) {
|
|
2599
|
-
|
|
2921
|
+
logger7.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
2600
2922
|
subgraph: subgraphName,
|
|
2601
2923
|
table,
|
|
2602
2924
|
count
|
|
@@ -2604,7 +2926,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2604
2926
|
}
|
|
2605
2927
|
}
|
|
2606
2928
|
} catch (err) {
|
|
2607
|
-
|
|
2929
|
+
logger7.debug("Row count sample failed", {
|
|
2608
2930
|
subgraph: subgraphName,
|
|
2609
2931
|
error: err instanceof Error ? err.message : String(err)
|
|
2610
2932
|
});
|
|
@@ -2694,11 +3016,11 @@ import {
|
|
|
2694
3016
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2695
3017
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2696
3018
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2697
|
-
import { logger as
|
|
3019
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2698
3020
|
|
|
2699
3021
|
// src/runtime/reindex-notify.ts
|
|
2700
3022
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2701
|
-
import { logger as
|
|
3023
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2702
3024
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2703
3025
|
try {
|
|
2704
3026
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -2709,7 +3031,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2709
3031
|
return;
|
|
2710
3032
|
const resendKey = process.env.RESEND_API_KEY;
|
|
2711
3033
|
if (!resendKey) {
|
|
2712
|
-
|
|
3034
|
+
logger8.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2713
3035
|
subgraph: subgraphName
|
|
2714
3036
|
});
|
|
2715
3037
|
return;
|
|
@@ -2731,14 +3053,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2731
3053
|
});
|
|
2732
3054
|
if (!res.ok) {
|
|
2733
3055
|
const text = await res.text().catch(() => "");
|
|
2734
|
-
|
|
3056
|
+
logger8.warn("reindex-complete email failed", {
|
|
2735
3057
|
subgraph: subgraphName,
|
|
2736
3058
|
status: res.status,
|
|
2737
3059
|
body: text.slice(0, 200)
|
|
2738
3060
|
});
|
|
2739
3061
|
}
|
|
2740
3062
|
} catch (err) {
|
|
2741
|
-
|
|
3063
|
+
logger8.warn("reindex-complete email threw", {
|
|
2742
3064
|
subgraph: subgraphName,
|
|
2743
3065
|
error: getErrorMessage2(err)
|
|
2744
3066
|
});
|
|
@@ -2850,7 +3172,7 @@ async function processBlockRange(def, opts) {
|
|
|
2850
3172
|
while (currentHeight <= toBlock) {
|
|
2851
3173
|
if (opts.signal?.aborted) {
|
|
2852
3174
|
aborted = true;
|
|
2853
|
-
|
|
3175
|
+
logger9.info("Block processing aborted", {
|
|
2854
3176
|
subgraph: subgraphName,
|
|
2855
3177
|
currentBlock: currentHeight,
|
|
2856
3178
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2899,7 +3221,7 @@ async function processBlockRange(def, opts) {
|
|
|
2899
3221
|
});
|
|
2900
3222
|
} catch (err) {
|
|
2901
3223
|
const errorMsg = getErrorMessage3(err);
|
|
2902
|
-
|
|
3224
|
+
logger9.error("Block processing failed persistently", {
|
|
2903
3225
|
subgraph: subgraphName,
|
|
2904
3226
|
blockHeight: height,
|
|
2905
3227
|
error: errorMsg
|
|
@@ -2945,7 +3267,7 @@ async function processBlockRange(def, opts) {
|
|
|
2945
3267
|
lastProgressFlushAt = now;
|
|
2946
3268
|
}
|
|
2947
3269
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2948
|
-
|
|
3270
|
+
logger9.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2949
3271
|
subgraph: subgraphName,
|
|
2950
3272
|
processed: blocksProcessed,
|
|
2951
3273
|
total: totalBlocks,
|
|
@@ -2965,7 +3287,7 @@ async function processBlockRange(def, opts) {
|
|
|
2965
3287
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2966
3288
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2967
3289
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2968
|
-
|
|
3290
|
+
logger9.warn("Failed to record subgraph gaps", {
|
|
2969
3291
|
subgraph: subgraphName,
|
|
2970
3292
|
error: err instanceof Error ? err.message : String(err)
|
|
2971
3293
|
});
|
|
@@ -2982,7 +3304,7 @@ async function processBlockRange(def, opts) {
|
|
|
2982
3304
|
} else {
|
|
2983
3305
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2984
3306
|
}
|
|
2985
|
-
|
|
3307
|
+
logger9.info("Sparse skip", {
|
|
2986
3308
|
subgraph: subgraphName,
|
|
2987
3309
|
from: batchEnd + 1,
|
|
2988
3310
|
to: jumpTo - 1,
|
|
@@ -3022,11 +3344,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
3022
3344
|
const subgraphName = def.name;
|
|
3023
3345
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3024
3346
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3025
|
-
|
|
3347
|
+
logger9.info("Reindex starting", { subgraph: subgraphName });
|
|
3026
3348
|
try {
|
|
3027
3349
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3028
3350
|
if (fromBlock > toBlock) {
|
|
3029
|
-
|
|
3351
|
+
logger9.info("No blocks to reindex", {
|
|
3030
3352
|
subgraph: subgraphName,
|
|
3031
3353
|
fromBlock,
|
|
3032
3354
|
toBlock
|
|
@@ -3040,7 +3362,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3040
3362
|
await client.unsafe(stmt);
|
|
3041
3363
|
}
|
|
3042
3364
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3043
|
-
|
|
3365
|
+
logger9.info("Schema recreated for reindex", {
|
|
3044
3366
|
subgraph: subgraphName,
|
|
3045
3367
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3046
3368
|
});
|
|
@@ -3054,7 +3376,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3054
3376
|
last_error_at: null,
|
|
3055
3377
|
updated_at: new Date
|
|
3056
3378
|
}).where("name", "=", subgraphName).execute();
|
|
3057
|
-
|
|
3379
|
+
logger9.info("Reindexing blocks", {
|
|
3058
3380
|
subgraph: subgraphName,
|
|
3059
3381
|
fromBlock,
|
|
3060
3382
|
toBlock,
|
|
@@ -3076,9 +3398,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3076
3398
|
if (reason === "user-cancelled") {
|
|
3077
3399
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3078
3400
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3079
|
-
|
|
3401
|
+
logger9.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3080
3402
|
} else {
|
|
3081
|
-
|
|
3403
|
+
logger9.info("Reindex interrupted by shutdown, will resume", {
|
|
3082
3404
|
subgraph: subgraphName
|
|
3083
3405
|
});
|
|
3084
3406
|
}
|
|
@@ -3086,7 +3408,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3086
3408
|
}
|
|
3087
3409
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3088
3410
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3089
|
-
|
|
3411
|
+
logger9.info("Reindex complete", {
|
|
3090
3412
|
subgraph: subgraphName,
|
|
3091
3413
|
blocks: result.blocksProcessed,
|
|
3092
3414
|
events: result.totalEventsProcessed,
|
|
@@ -3099,7 +3421,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3099
3421
|
});
|
|
3100
3422
|
return { processed: result.blocksProcessed };
|
|
3101
3423
|
} catch (err) {
|
|
3102
|
-
|
|
3424
|
+
logger9.error("Reindex failed", {
|
|
3103
3425
|
subgraph: subgraphName,
|
|
3104
3426
|
error: getErrorMessage3(err)
|
|
3105
3427
|
});
|
|
@@ -3121,7 +3443,7 @@ async function resumeReindex(def, opts) {
|
|
|
3121
3443
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3122
3444
|
const toBlock = Number(row.reindex_to_block);
|
|
3123
3445
|
if (fromBlock == null) {
|
|
3124
|
-
|
|
3446
|
+
logger9.info("No reindex metadata, starting fresh reindex", {
|
|
3125
3447
|
subgraph: subgraphName
|
|
3126
3448
|
});
|
|
3127
3449
|
return reindexSubgraph(def, {
|
|
@@ -3130,12 +3452,12 @@ async function resumeReindex(def, opts) {
|
|
|
3130
3452
|
});
|
|
3131
3453
|
}
|
|
3132
3454
|
if (fromBlock > toBlock) {
|
|
3133
|
-
|
|
3455
|
+
logger9.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3134
3456
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3135
3457
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3136
3458
|
return { processed: 0 };
|
|
3137
3459
|
}
|
|
3138
|
-
|
|
3460
|
+
logger9.info("Resuming reindex", {
|
|
3139
3461
|
subgraph: subgraphName,
|
|
3140
3462
|
fromBlock,
|
|
3141
3463
|
toBlock,
|
|
@@ -3157,9 +3479,9 @@ async function resumeReindex(def, opts) {
|
|
|
3157
3479
|
if (reason === "user-cancelled") {
|
|
3158
3480
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3159
3481
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3160
|
-
|
|
3482
|
+
logger9.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3161
3483
|
} else {
|
|
3162
|
-
|
|
3484
|
+
logger9.info("Resume interrupted by shutdown, will resume again", {
|
|
3163
3485
|
subgraph: subgraphName
|
|
3164
3486
|
});
|
|
3165
3487
|
}
|
|
@@ -3167,13 +3489,13 @@ async function resumeReindex(def, opts) {
|
|
|
3167
3489
|
}
|
|
3168
3490
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3169
3491
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3170
|
-
|
|
3492
|
+
logger9.info("Resumed reindex complete", {
|
|
3171
3493
|
subgraph: subgraphName,
|
|
3172
3494
|
blocks: result.blocksProcessed
|
|
3173
3495
|
});
|
|
3174
3496
|
return { processed: result.blocksProcessed };
|
|
3175
3497
|
} catch (err) {
|
|
3176
|
-
|
|
3498
|
+
logger9.error("Resumed reindex failed", {
|
|
3177
3499
|
subgraph: subgraphName,
|
|
3178
3500
|
error: getErrorMessage3(err)
|
|
3179
3501
|
});
|
|
@@ -3184,7 +3506,7 @@ async function resumeReindex(def, opts) {
|
|
|
3184
3506
|
async function backfillSubgraph(def, opts) {
|
|
3185
3507
|
const targetDb = getTargetDb2();
|
|
3186
3508
|
const subgraphName = def.name;
|
|
3187
|
-
|
|
3509
|
+
logger9.info("Backfill starting", {
|
|
3188
3510
|
subgraph: subgraphName,
|
|
3189
3511
|
from: opts.fromBlock,
|
|
3190
3512
|
to: opts.toBlock
|
|
@@ -3202,17 +3524,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3202
3524
|
signal: opts.signal
|
|
3203
3525
|
});
|
|
3204
3526
|
if (result.aborted) {
|
|
3205
|
-
|
|
3527
|
+
logger9.info("Backfill aborted", { subgraph: subgraphName });
|
|
3206
3528
|
return { processed: result.blocksProcessed };
|
|
3207
3529
|
}
|
|
3208
3530
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3209
3531
|
if (resolved > 0) {
|
|
3210
|
-
|
|
3532
|
+
logger9.info("Resolved subgraph gaps via backfill", {
|
|
3211
3533
|
subgraph: subgraphName,
|
|
3212
3534
|
resolved
|
|
3213
3535
|
});
|
|
3214
3536
|
}
|
|
3215
|
-
|
|
3537
|
+
logger9.info("Backfill complete", {
|
|
3216
3538
|
subgraph: subgraphName,
|
|
3217
3539
|
blocks: result.blocksProcessed,
|
|
3218
3540
|
events: result.totalEventsProcessed,
|
|
@@ -3220,7 +3542,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3220
3542
|
});
|
|
3221
3543
|
return { processed: result.blocksProcessed };
|
|
3222
3544
|
} catch (err) {
|
|
3223
|
-
|
|
3545
|
+
logger9.error("Backfill failed", {
|
|
3224
3546
|
subgraph: subgraphName,
|
|
3225
3547
|
error: getErrorMessage3(err)
|
|
3226
3548
|
});
|
|
@@ -3231,7 +3553,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3231
3553
|
// src/runtime/catchup.ts
|
|
3232
3554
|
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3233
3555
|
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
3234
|
-
import { logger as
|
|
3556
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
3235
3557
|
|
|
3236
3558
|
// src/runtime/catchup-leader.ts
|
|
3237
3559
|
import {
|
|
@@ -3365,7 +3687,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3365
3687
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
3366
3688
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
3367
3689
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
3368
|
-
|
|
3690
|
+
logger10.info("Subgraph catch-up starting", {
|
|
3369
3691
|
subgraph: subgraphName,
|
|
3370
3692
|
from: startBlock,
|
|
3371
3693
|
to: chainTip,
|
|
@@ -3381,7 +3703,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3381
3703
|
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
3382
3704
|
while (currentHeight <= chainTip) {
|
|
3383
3705
|
if (!isCatchUpLeader()) {
|
|
3384
|
-
|
|
3706
|
+
logger10.info("Lost catch-up leadership, stopping catch-up", {
|
|
3385
3707
|
subgraph: subgraphName,
|
|
3386
3708
|
processed
|
|
3387
3709
|
});
|
|
@@ -3389,7 +3711,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3389
3711
|
}
|
|
3390
3712
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
3391
3713
|
if (!currentRow || currentRow.status !== "active") {
|
|
3392
|
-
|
|
3714
|
+
logger10.info("Subgraph status changed, stopping catch-up", {
|
|
3393
3715
|
subgraph: subgraphName,
|
|
3394
3716
|
status: currentRow?.status ?? "deleted"
|
|
3395
3717
|
});
|
|
@@ -3418,7 +3740,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3418
3740
|
blockData = (await source.loadBlockRange(height, height)).get(height);
|
|
3419
3741
|
}
|
|
3420
3742
|
if (!blockData) {
|
|
3421
|
-
|
|
3743
|
+
logger10.warn("Block missing during catch-up, deferring to next tick", {
|
|
3422
3744
|
subgraph: subgraphName,
|
|
3423
3745
|
blockHeight: height
|
|
3424
3746
|
});
|
|
@@ -3438,12 +3760,12 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3438
3760
|
});
|
|
3439
3761
|
} catch (err) {
|
|
3440
3762
|
if (err instanceof ReorgEpochChangedError) {
|
|
3441
|
-
|
|
3763
|
+
logger10.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
|
|
3442
3764
|
stopCatchup = true;
|
|
3443
3765
|
break;
|
|
3444
3766
|
}
|
|
3445
3767
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
3446
|
-
|
|
3768
|
+
logger10.error("Block processing failed persistently during catch-up", {
|
|
3447
3769
|
subgraph: subgraphName,
|
|
3448
3770
|
blockHeight: height,
|
|
3449
3771
|
error: errorMsg
|
|
@@ -3462,7 +3784,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3462
3784
|
}
|
|
3463
3785
|
}
|
|
3464
3786
|
if (processed % LOG_INTERVAL2 === 0) {
|
|
3465
|
-
|
|
3787
|
+
logger10.info("Subgraph catch-up progress", {
|
|
3466
3788
|
subgraph: subgraphName,
|
|
3467
3789
|
processed,
|
|
3468
3790
|
total: totalBlocks,
|
|
@@ -3478,7 +3800,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3478
3800
|
currentHeight = batchEnd + 1;
|
|
3479
3801
|
}
|
|
3480
3802
|
await stats.flush(targetDb);
|
|
3481
|
-
|
|
3803
|
+
logger10.info("Subgraph catch-up complete", {
|
|
3482
3804
|
subgraph: subgraphName,
|
|
3483
3805
|
processed
|
|
3484
3806
|
});
|
|
@@ -3495,14 +3817,14 @@ import {
|
|
|
3495
3817
|
listSubgraphs,
|
|
3496
3818
|
resolveSubgraphRawClient
|
|
3497
3819
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3498
|
-
import { logger as
|
|
3820
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
3499
3821
|
import { sql as sql4 } from "kysely";
|
|
3500
3822
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
3501
3823
|
const targetDb = getTargetDb4();
|
|
3502
3824
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
3503
3825
|
if (activeSubgraphs.length === 0)
|
|
3504
3826
|
return;
|
|
3505
|
-
|
|
3827
|
+
logger11.info("Propagating reorg to subgraphs", {
|
|
3506
3828
|
blockHeight,
|
|
3507
3829
|
subgraphCount: activeSubgraphs.length
|
|
3508
3830
|
});
|
|
@@ -3519,7 +3841,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
3519
3841
|
});
|
|
3520
3842
|
});
|
|
3521
3843
|
} catch (err) {
|
|
3522
|
-
|
|
3844
|
+
logger11.error("Subgraph reorg handling failed", {
|
|
3523
3845
|
subgraph: sg.name,
|
|
3524
3846
|
blockHeight,
|
|
3525
3847
|
error: getErrorMessage4(err)
|
|
@@ -3557,7 +3879,7 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3557
3879
|
await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
|
|
3558
3880
|
});
|
|
3559
3881
|
} else {
|
|
3560
|
-
|
|
3882
|
+
logger11.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
|
|
3561
3883
|
for (const tableName of tableNames) {
|
|
3562
3884
|
await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
|
|
3563
3885
|
}
|
|
@@ -3600,7 +3922,7 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3600
3922
|
}).as("row_pk"),
|
|
3601
3923
|
eb2.val("pending").as("status")
|
|
3602
3924
|
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
|
|
3603
|
-
|
|
3925
|
+
logger11.error("Failed to emit revert event for subscriptions", {
|
|
3604
3926
|
event: "reorg_revert_emit_dropped",
|
|
3605
3927
|
subgraph: sg.name,
|
|
3606
3928
|
table: tableName,
|
|
@@ -3609,14 +3931,14 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3609
3931
|
});
|
|
3610
3932
|
});
|
|
3611
3933
|
}
|
|
3612
|
-
|
|
3934
|
+
logger11.info("Subgraph reorg cleanup done", {
|
|
3613
3935
|
subgraph: sg.name,
|
|
3614
3936
|
blockHeight,
|
|
3615
3937
|
tablesAffected: Object.keys(revertedByTable).length
|
|
3616
3938
|
});
|
|
3617
3939
|
const def = await loadSubgraphDef(sg);
|
|
3618
3940
|
await processBlock(def, sg.name, blockHeight, { reorgRewind: true });
|
|
3619
|
-
|
|
3941
|
+
logger11.info("Subgraph reorg reprocessed", {
|
|
3620
3942
|
subgraph: sg.name,
|
|
3621
3943
|
blockHeight
|
|
3622
3944
|
});
|
|
@@ -3646,7 +3968,7 @@ import {
|
|
|
3646
3968
|
pgSchemaName as pgSchemaName2,
|
|
3647
3969
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
3648
3970
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3649
|
-
import { logger as
|
|
3971
|
+
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3650
3972
|
import {
|
|
3651
3973
|
listen,
|
|
3652
3974
|
sourceListenerUrl,
|
|
@@ -3656,7 +3978,7 @@ import {
|
|
|
3656
3978
|
// src/runtime/streams-reorg-poll.ts
|
|
3657
3979
|
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
3658
3980
|
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3659
|
-
import { logger as
|
|
3981
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
3660
3982
|
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3661
3983
|
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3662
3984
|
async function pollReorgsOnce(http2, cursor, onReorg, handled) {
|
|
@@ -3664,13 +3986,13 @@ async function pollReorgsOnce(http2, cursor, onReorg, handled) {
|
|
|
3664
3986
|
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3665
3987
|
for (const r of sorted) {
|
|
3666
3988
|
if (handled && r.id && handled.has(r.id)) {
|
|
3667
|
-
|
|
3989
|
+
logger12.info("Streams reorg already handled — skipping", {
|
|
3668
3990
|
forkPointHeight: r.fork_point_height,
|
|
3669
3991
|
reorgId: r.id
|
|
3670
3992
|
});
|
|
3671
3993
|
continue;
|
|
3672
3994
|
}
|
|
3673
|
-
|
|
3995
|
+
logger12.info("Streams reorg — rewinding", {
|
|
3674
3996
|
forkPointHeight: r.fork_point_height
|
|
3675
3997
|
});
|
|
3676
3998
|
await onReorg(r.fork_point_height);
|
|
@@ -3698,7 +4020,7 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3698
4020
|
try {
|
|
3699
4021
|
since = await pollReorgsOnce(http2, since, onReorg, handled);
|
|
3700
4022
|
} catch (err) {
|
|
3701
|
-
|
|
4023
|
+
logger12.error("Streams reorg poll failed", {
|
|
3702
4024
|
error: getErrorMessage5(err)
|
|
3703
4025
|
});
|
|
3704
4026
|
}
|
|
@@ -3706,7 +4028,7 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3706
4028
|
timer = setTimeout(tick, POLL_MS);
|
|
3707
4029
|
};
|
|
3708
4030
|
timer = setTimeout(tick, POLL_MS);
|
|
3709
|
-
|
|
4031
|
+
logger12.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3710
4032
|
return () => {
|
|
3711
4033
|
running = false;
|
|
3712
4034
|
if (timer)
|
|
@@ -3737,7 +4059,7 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3737
4059
|
if (isHandlerNotFoundError(err)) {
|
|
3738
4060
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3739
4061
|
}
|
|
3740
|
-
|
|
4062
|
+
logger13.error("Subgraph catch-up failed", {
|
|
3741
4063
|
subgraph: sg.name,
|
|
3742
4064
|
error: msg
|
|
3743
4065
|
});
|
|
@@ -3777,7 +4099,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3777
4099
|
definitionCache.set(sg.name, def);
|
|
3778
4100
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3779
4101
|
invalidateSubgraphRoute(sg.name);
|
|
3780
|
-
|
|
4102
|
+
logger13.info("Subgraph handler reloaded", {
|
|
3781
4103
|
subgraph: sg.name,
|
|
3782
4104
|
from: prevVersion,
|
|
3783
4105
|
to: sg.version
|
|
@@ -3811,7 +4133,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3811
4133
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3812
4134
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3813
4135
|
});
|
|
3814
|
-
|
|
4136
|
+
logger13.info("Queued legacy reindex resume operation", {
|
|
3815
4137
|
subgraph: sg.name
|
|
3816
4138
|
});
|
|
3817
4139
|
} catch (err) {
|
|
@@ -3874,7 +4196,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3874
4196
|
const activeRuns = new Map;
|
|
3875
4197
|
let running = true;
|
|
3876
4198
|
let draining = false;
|
|
3877
|
-
|
|
4199
|
+
logger13.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3878
4200
|
try {
|
|
3879
4201
|
const stranded = await db.selectFrom("subgraphs").select(["id", "name", "account_id"]).where("status", "=", "reindexing").where("reindex_from_block", "is not", null).where("reindex_to_block", "is not", null).where(({ not, exists, selectFrom }) => not(exists(selectFrom("subgraph_operations").select("id").whereRef("subgraph_id", "=", "subgraphs.id").where("status", "in", ["queued", "running"])))).execute();
|
|
3880
4202
|
for (const row of stranded) {
|
|
@@ -3885,14 +4207,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3885
4207
|
accountId: row.account_id,
|
|
3886
4208
|
kind: "reindex"
|
|
3887
4209
|
});
|
|
3888
|
-
|
|
4210
|
+
logger13.info("Re-enqueued stranded reindex", { subgraph: row.name });
|
|
3889
4211
|
} catch (err) {
|
|
3890
4212
|
if (!isActiveSubgraphOperationConflict(err))
|
|
3891
4213
|
throw err;
|
|
3892
4214
|
}
|
|
3893
4215
|
}
|
|
3894
4216
|
} catch (err) {
|
|
3895
|
-
|
|
4217
|
+
logger13.warn("Stranded-reindex sweep failed", {
|
|
3896
4218
|
error: getErrorMessage6(err)
|
|
3897
4219
|
});
|
|
3898
4220
|
}
|
|
@@ -3903,7 +4225,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3903
4225
|
if (!running)
|
|
3904
4226
|
return;
|
|
3905
4227
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3906
|
-
|
|
4228
|
+
logger13.warn("Subgraph operation heartbeat failed", {
|
|
3907
4229
|
operationId: operation.id,
|
|
3908
4230
|
error: getErrorMessage6(err)
|
|
3909
4231
|
});
|
|
@@ -3915,7 +4237,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3915
4237
|
controller.abort("user-cancelled");
|
|
3916
4238
|
}
|
|
3917
4239
|
}).catch((err) => {
|
|
3918
|
-
|
|
4240
|
+
logger13.warn("Subgraph operation cancel poll failed", {
|
|
3919
4241
|
operationId: operation.id,
|
|
3920
4242
|
error: getErrorMessage6(err)
|
|
3921
4243
|
});
|
|
@@ -3932,14 +4254,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3932
4254
|
const reason = String(controller.signal.reason ?? "");
|
|
3933
4255
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3934
4256
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3935
|
-
|
|
4257
|
+
logger13.info("Subgraph operation cancelled", {
|
|
3936
4258
|
operationId: operation.id,
|
|
3937
4259
|
subgraph: operation.subgraph_name
|
|
3938
4260
|
});
|
|
3939
4261
|
return;
|
|
3940
4262
|
}
|
|
3941
4263
|
if (controller.signal.aborted) {
|
|
3942
|
-
|
|
4264
|
+
logger13.info("Subgraph operation interrupted", {
|
|
3943
4265
|
operationId: operation.id,
|
|
3944
4266
|
subgraph: operation.subgraph_name,
|
|
3945
4267
|
reason
|
|
@@ -3947,7 +4269,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3947
4269
|
return;
|
|
3948
4270
|
}
|
|
3949
4271
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3950
|
-
|
|
4272
|
+
logger13.info("Subgraph operation completed", {
|
|
3951
4273
|
operationId: operation.id,
|
|
3952
4274
|
subgraph: operation.subgraph_name,
|
|
3953
4275
|
processed
|
|
@@ -3955,7 +4277,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3955
4277
|
} catch (err) {
|
|
3956
4278
|
const reason = String(controller.signal.reason ?? "");
|
|
3957
4279
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3958
|
-
|
|
4280
|
+
logger13.info("Subgraph operation interrupted by shutdown", {
|
|
3959
4281
|
operationId: operation.id,
|
|
3960
4282
|
subgraph: operation.subgraph_name
|
|
3961
4283
|
});
|
|
@@ -3966,7 +4288,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3966
4288
|
return;
|
|
3967
4289
|
}
|
|
3968
4290
|
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
|
|
3969
|
-
|
|
4291
|
+
logger13.error("Subgraph operation failed", {
|
|
3970
4292
|
operationId: operation.id,
|
|
3971
4293
|
subgraph: operation.subgraph_name,
|
|
3972
4294
|
error: getErrorMessage6(err)
|
|
@@ -4013,13 +4335,13 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
4013
4335
|
controller.abort("shutdown");
|
|
4014
4336
|
}
|
|
4015
4337
|
await Promise.allSettled(activeRuns.values());
|
|
4016
|
-
|
|
4338
|
+
logger13.info("Subgraph operation runner stopped");
|
|
4017
4339
|
};
|
|
4018
4340
|
}
|
|
4019
4341
|
async function startSubgraphProcessor(opts) {
|
|
4020
4342
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
4021
4343
|
let running = true;
|
|
4022
|
-
|
|
4344
|
+
logger13.info("Starting subgraph processor", { concurrency });
|
|
4023
4345
|
const stopOperations = await startSubgraphOperationRunner({
|
|
4024
4346
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
4025
4347
|
});
|
|
@@ -4045,7 +4367,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4045
4367
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
4046
4368
|
}
|
|
4047
4369
|
} catch (err) {
|
|
4048
|
-
|
|
4370
|
+
logger13.error("Subgraph reorg handling failed", {
|
|
4049
4371
|
error: getErrorMessage6(err)
|
|
4050
4372
|
});
|
|
4051
4373
|
}
|
|
@@ -4054,7 +4376,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4054
4376
|
runCatchUp();
|
|
4055
4377
|
}, POLL_INTERVAL_MS);
|
|
4056
4378
|
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
4057
|
-
|
|
4379
|
+
logger13.info("Subgraph processor ready");
|
|
4058
4380
|
return async () => {
|
|
4059
4381
|
running = false;
|
|
4060
4382
|
clearInterval(pollInterval);
|
|
@@ -4063,13 +4385,13 @@ async function startSubgraphProcessor(opts) {
|
|
|
4063
4385
|
await stopReorgListening();
|
|
4064
4386
|
stopStreamsReorgPoll?.();
|
|
4065
4387
|
await stopOperations();
|
|
4066
|
-
|
|
4388
|
+
logger13.info("Subgraph processor stopped");
|
|
4067
4389
|
};
|
|
4068
4390
|
}
|
|
4069
4391
|
|
|
4070
4392
|
// src/service.ts
|
|
4071
4393
|
import { assertDbSplit, getDb } from "@secondlayer/shared/db";
|
|
4072
|
-
import { logger as
|
|
4394
|
+
import { logger as logger14 } from "@secondlayer/shared/logger";
|
|
4073
4395
|
import { sql as sql5 } from "kysely";
|
|
4074
4396
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
4075
4397
|
var SERVICE_NAME = "subgraph-processor";
|
|
@@ -4077,7 +4399,7 @@ async function writeHeartbeat() {
|
|
|
4077
4399
|
try {
|
|
4078
4400
|
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql5`now()` })).execute();
|
|
4079
4401
|
} catch (err) {
|
|
4080
|
-
|
|
4402
|
+
logger14.warn("subgraph-processor heartbeat write failed", {
|
|
4081
4403
|
error: err instanceof Error ? err.message : String(err)
|
|
4082
4404
|
});
|
|
4083
4405
|
}
|
|
@@ -4089,7 +4411,7 @@ var processor = await startSubgraphProcessor({
|
|
|
4089
4411
|
await writeHeartbeat();
|
|
4090
4412
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
4091
4413
|
var shutdown = async () => {
|
|
4092
|
-
|
|
4414
|
+
logger14.info("Shutting down subgraph processor...");
|
|
4093
4415
|
clearInterval(heartbeatInterval);
|
|
4094
4416
|
await processor();
|
|
4095
4417
|
process.exit(0);
|
|
@@ -4097,5 +4419,5 @@ var shutdown = async () => {
|
|
|
4097
4419
|
process.on("SIGINT", shutdown);
|
|
4098
4420
|
process.on("SIGTERM", shutdown);
|
|
4099
4421
|
|
|
4100
|
-
//# debugId=
|
|
4422
|
+
//# debugId=9192F92CFA9186B564756E2164756E21
|
|
4101
4423
|
//# sourceMappingURL=service.js.map
|