@secondlayer/subgraphs 3.24.1 → 3.24.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +372 -39
- package/dist/src/index.js.map +7 -4
- package/dist/src/runtime/block-processor.d.ts +20 -2
- package/dist/src/runtime/block-processor.js +343 -9
- package/dist/src/runtime/block-processor.js.map +7 -4
- package/dist/src/runtime/catchup.js +351 -18
- package/dist/src/runtime/catchup.js.map +7 -4
- package/dist/src/runtime/processor.js +412 -79
- package/dist/src/runtime/processor.js.map +7 -4
- package/dist/src/runtime/reindex.js +372 -39
- package/dist/src/runtime/reindex.js.map +7 -4
- package/dist/src/runtime/reorg.js +358 -25
- package/dist/src/runtime/reorg.js.map +7 -4
- package/dist/src/service.js +415 -82
- package/dist/src/service.js.map +7 -4
- package/package.json +2 -2
package/dist/src/service.js
CHANGED
|
@@ -1627,6 +1627,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map, f
|
|
|
1627
1627
|
}
|
|
1628
1628
|
|
|
1629
1629
|
// src/runtime/block-processor.ts
|
|
1630
|
+
import { secretsKeyAvailable } from "@secondlayer/shared/crypto/secrets";
|
|
1630
1631
|
import { getTargetDb } from "@secondlayer/shared/db";
|
|
1631
1632
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
1632
1633
|
import { advanceOperationCursor } from "@secondlayer/shared/db/queries/subgraph-operations";
|
|
@@ -1639,7 +1640,7 @@ import {
|
|
|
1639
1640
|
rewindLiveProgress,
|
|
1640
1641
|
updateSubgraphStatus
|
|
1641
1642
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1642
|
-
import { logger as
|
|
1643
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1643
1644
|
import { sql as sql3 } from "kysely";
|
|
1644
1645
|
|
|
1645
1646
|
// src/runtime/block-source.ts
|
|
@@ -2080,6 +2081,315 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2080
2081
|
return rows.length;
|
|
2081
2082
|
}
|
|
2082
2083
|
|
|
2084
|
+
// src/runtime/sandbox/flag.ts
|
|
2085
|
+
function sandboxWorkersGloballyEnabled() {
|
|
2086
|
+
return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
|
|
2087
|
+
}
|
|
2088
|
+
function sandboxEnabled(subgraph) {
|
|
2089
|
+
return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
// src/runtime/sandbox/host.ts
|
|
2093
|
+
import { tmpdir } from "node:os";
|
|
2094
|
+
import { fileURLToPath } from "node:url";
|
|
2095
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2096
|
+
|
|
2097
|
+
// src/runtime/sandbox/bundle.ts
|
|
2098
|
+
import esbuild from "esbuild";
|
|
2099
|
+
var ALLOWED_BARE_SPECIFIERS = new Set;
|
|
2100
|
+
var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
|
|
2101
|
+
var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
|
|
2102
|
+
`;
|
|
2103
|
+
function resolverLockdownPlugin() {
|
|
2104
|
+
return {
|
|
2105
|
+
name: "f071-sandbox-resolver-lockdown",
|
|
2106
|
+
setup(build) {
|
|
2107
|
+
build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
|
|
2108
|
+
path: args.path,
|
|
2109
|
+
namespace: "f071-sandbox-subgraphs-stub"
|
|
2110
|
+
}));
|
|
2111
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
|
|
2112
|
+
build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
|
|
2113
|
+
path: args.path,
|
|
2114
|
+
namespace: "f071-sandbox-blocked"
|
|
2115
|
+
}));
|
|
2116
|
+
build.onResolve({ filter: /^[^./]/ }, (args) => {
|
|
2117
|
+
if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
|
|
2118
|
+
return;
|
|
2119
|
+
if (ALLOWED_BARE_SPECIFIERS.has(args.path))
|
|
2120
|
+
return;
|
|
2121
|
+
return { path: args.path, namespace: "f071-sandbox-blocked" };
|
|
2122
|
+
});
|
|
2123
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
|
|
2124
|
+
contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
|
|
2125
|
+
loader: "js"
|
|
2126
|
+
}));
|
|
2127
|
+
}
|
|
2128
|
+
};
|
|
2129
|
+
}
|
|
2130
|
+
async function bundleHandlerCode(handlerCode) {
|
|
2131
|
+
const result = await esbuild.build({
|
|
2132
|
+
stdin: {
|
|
2133
|
+
contents: handlerCode,
|
|
2134
|
+
loader: "ts",
|
|
2135
|
+
resolveDir: process.cwd()
|
|
2136
|
+
},
|
|
2137
|
+
bundle: true,
|
|
2138
|
+
platform: "neutral",
|
|
2139
|
+
format: "esm",
|
|
2140
|
+
write: false,
|
|
2141
|
+
logLevel: "silent",
|
|
2142
|
+
plugins: [resolverLockdownPlugin()]
|
|
2143
|
+
});
|
|
2144
|
+
const out = result.outputFiles?.[0];
|
|
2145
|
+
if (!out)
|
|
2146
|
+
throw new Error("esbuild produced no output for handler_code bundle");
|
|
2147
|
+
return new TextDecoder().decode(out.contents);
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
// src/runtime/sandbox/host.ts
|
|
2151
|
+
var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
|
|
2152
|
+
var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
|
|
2153
|
+
function spawnSandboxProcess() {
|
|
2154
|
+
let onMessage = null;
|
|
2155
|
+
const proc = Bun.spawn({
|
|
2156
|
+
cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
|
|
2157
|
+
env: { PATH: process.env.PATH ?? "" },
|
|
2158
|
+
cwd: tmpdir(),
|
|
2159
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
2160
|
+
serialization: "advanced",
|
|
2161
|
+
ipc(message) {
|
|
2162
|
+
onMessage?.(message);
|
|
2163
|
+
}
|
|
2164
|
+
});
|
|
2165
|
+
(async () => {
|
|
2166
|
+
const reader = proc.stderr.getReader();
|
|
2167
|
+
const decoder = new TextDecoder;
|
|
2168
|
+
try {
|
|
2169
|
+
while (true) {
|
|
2170
|
+
const { done, value } = await reader.read();
|
|
2171
|
+
if (done)
|
|
2172
|
+
break;
|
|
2173
|
+
const text = decoder.decode(value, { stream: true });
|
|
2174
|
+
if (text.trim()) {
|
|
2175
|
+
logger6.warn("Sandbox subprocess stderr", { stderr: text.trim() });
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
} catch {}
|
|
2179
|
+
})();
|
|
2180
|
+
return {
|
|
2181
|
+
send: (msg) => proc.send(msg),
|
|
2182
|
+
kill: () => proc.kill(),
|
|
2183
|
+
exited: proc.exited,
|
|
2184
|
+
setOnMessage: (handler) => {
|
|
2185
|
+
onMessage = handler;
|
|
2186
|
+
}
|
|
2187
|
+
};
|
|
2188
|
+
}
|
|
2189
|
+
var pool = new Map;
|
|
2190
|
+
|
|
2191
|
+
class SandboxWorkerCrash extends Error {
|
|
2192
|
+
constructor(subgraphName, cause) {
|
|
2193
|
+
super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
|
|
2194
|
+
this.name = "SandboxWorkerCrash";
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
|
|
2198
|
+
const bundledCode = await bundleHandlerCode(handlerCode);
|
|
2199
|
+
let settled = false;
|
|
2200
|
+
const ready = new Promise((resolve, reject) => {
|
|
2201
|
+
proc.setOnMessage((msg) => {
|
|
2202
|
+
if (msg.type === "ready" && msg.version === version) {
|
|
2203
|
+
settled = true;
|
|
2204
|
+
proc.setOnMessage(null);
|
|
2205
|
+
resolve();
|
|
2206
|
+
} else if (msg.type === "error") {
|
|
2207
|
+
settled = true;
|
|
2208
|
+
proc.setOnMessage(null);
|
|
2209
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2210
|
+
}
|
|
2211
|
+
});
|
|
2212
|
+
const init = { type: "init", bundledCode, version };
|
|
2213
|
+
proc.send(init);
|
|
2214
|
+
});
|
|
2215
|
+
const crashed = proc.exited.then((code) => {
|
|
2216
|
+
if (settled)
|
|
2217
|
+
return;
|
|
2218
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
|
|
2219
|
+
});
|
|
2220
|
+
await Promise.race([ready, crashed]);
|
|
2221
|
+
}
|
|
2222
|
+
async function ensureSandboxProcess(subgraphName, version, handlerCode) {
|
|
2223
|
+
if (handlerCode == null) {
|
|
2224
|
+
throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
|
|
2225
|
+
}
|
|
2226
|
+
const existing = pool.get(subgraphName);
|
|
2227
|
+
if (existing && existing.version === version)
|
|
2228
|
+
return existing;
|
|
2229
|
+
if (existing) {
|
|
2230
|
+
try {
|
|
2231
|
+
await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
|
|
2232
|
+
existing.version = version;
|
|
2233
|
+
logger6.info("Sandbox subprocess handler reloaded", {
|
|
2234
|
+
subgraph: subgraphName,
|
|
2235
|
+
version
|
|
2236
|
+
});
|
|
2237
|
+
return existing;
|
|
2238
|
+
} catch (err) {
|
|
2239
|
+
existing.proc.kill();
|
|
2240
|
+
pool.delete(subgraphName);
|
|
2241
|
+
logger6.warn("Sandbox subprocess re-init failed; respawning", {
|
|
2242
|
+
subgraph: subgraphName,
|
|
2243
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2244
|
+
});
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
const proc = spawnSandboxProcess();
|
|
2248
|
+
try {
|
|
2249
|
+
await initSandboxProcess(proc, subgraphName, version, handlerCode);
|
|
2250
|
+
} catch (err) {
|
|
2251
|
+
proc.kill();
|
|
2252
|
+
throw err;
|
|
2253
|
+
}
|
|
2254
|
+
const entry = { proc, version, chain: Promise.resolve() };
|
|
2255
|
+
pool.set(subgraphName, entry);
|
|
2256
|
+
return entry;
|
|
2257
|
+
}
|
|
2258
|
+
function evictSandboxWorker(subgraphName) {
|
|
2259
|
+
const entry = pool.get(subgraphName);
|
|
2260
|
+
if (!entry)
|
|
2261
|
+
return;
|
|
2262
|
+
entry.proc.kill();
|
|
2263
|
+
pool.delete(subgraphName);
|
|
2264
|
+
}
|
|
2265
|
+
async function runHandlersSandboxed(params) {
|
|
2266
|
+
const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
|
|
2267
|
+
const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
|
|
2268
|
+
const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
|
|
2269
|
+
entry.chain = run.catch(() => {
|
|
2270
|
+
return;
|
|
2271
|
+
});
|
|
2272
|
+
return run;
|
|
2273
|
+
}
|
|
2274
|
+
async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
|
|
2275
|
+
const { proc } = entry;
|
|
2276
|
+
let timeoutHandle;
|
|
2277
|
+
let settled = false;
|
|
2278
|
+
try {
|
|
2279
|
+
const done = new Promise((resolve, reject) => {
|
|
2280
|
+
proc.setOnMessage((msg) => {
|
|
2281
|
+
if (msg.type === "readRequest") {
|
|
2282
|
+
answerRead(hostCtx, msg).then((reply) => {
|
|
2283
|
+
proc.send({ type: "readResponse", id: msg.id, reply });
|
|
2284
|
+
}).catch((err) => {
|
|
2285
|
+
settled = true;
|
|
2286
|
+
reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
2287
|
+
});
|
|
2288
|
+
return;
|
|
2289
|
+
}
|
|
2290
|
+
if (msg.type === "blockDone") {
|
|
2291
|
+
settled = true;
|
|
2292
|
+
resolve({
|
|
2293
|
+
ops: msg.ops,
|
|
2294
|
+
result: { processed: msg.processed, errors: msg.errors }
|
|
2295
|
+
});
|
|
2296
|
+
return;
|
|
2297
|
+
}
|
|
2298
|
+
if (msg.type === "error") {
|
|
2299
|
+
settled = true;
|
|
2300
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2301
|
+
}
|
|
2302
|
+
});
|
|
2303
|
+
proc.send({ type: "runBlock", block, matched });
|
|
2304
|
+
});
|
|
2305
|
+
const crashed = proc.exited.then((code) => {
|
|
2306
|
+
if (settled)
|
|
2307
|
+
return;
|
|
2308
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
|
|
2309
|
+
});
|
|
2310
|
+
const deadline = new Promise((_, reject) => {
|
|
2311
|
+
timeoutHandle = setTimeout(() => {
|
|
2312
|
+
settled = true;
|
|
2313
|
+
reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
|
|
2314
|
+
}, BLOCK_TIMEOUT_MS);
|
|
2315
|
+
});
|
|
2316
|
+
const { ops, result } = await Promise.race([done, crashed, deadline]);
|
|
2317
|
+
for (const op of ops) {
|
|
2318
|
+
hostCtx.setTx(op.tx);
|
|
2319
|
+
replayOp(hostCtx, op);
|
|
2320
|
+
}
|
|
2321
|
+
return result;
|
|
2322
|
+
} catch (err) {
|
|
2323
|
+
evictSandboxWorker(subgraphName);
|
|
2324
|
+
logger6.warn("Sandbox subprocess run failed; subprocess evicted", {
|
|
2325
|
+
subgraph: subgraphName,
|
|
2326
|
+
blockHeight: block.height,
|
|
2327
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2328
|
+
});
|
|
2329
|
+
throw err;
|
|
2330
|
+
} finally {
|
|
2331
|
+
if (timeoutHandle !== undefined)
|
|
2332
|
+
clearTimeout(timeoutHandle);
|
|
2333
|
+
proc.setOnMessage(null);
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
function replayOp(hostCtx, op) {
|
|
2337
|
+
switch (op.method) {
|
|
2338
|
+
case "insert":
|
|
2339
|
+
hostCtx.insert(op.table, op.args[0]);
|
|
2340
|
+
return;
|
|
2341
|
+
case "upsert":
|
|
2342
|
+
hostCtx.upsert(op.table, op.args[0], op.args[1]);
|
|
2343
|
+
return;
|
|
2344
|
+
case "increment":
|
|
2345
|
+
hostCtx.increment(op.table, op.args[0], op.args[1]);
|
|
2346
|
+
return;
|
|
2347
|
+
case "update":
|
|
2348
|
+
hostCtx.update(op.table, op.args[0], op.args[1]);
|
|
2349
|
+
return;
|
|
2350
|
+
case "delete":
|
|
2351
|
+
hostCtx.delete(op.table, op.args[0]);
|
|
2352
|
+
return;
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
async function answerRead(hostCtx, msg) {
|
|
2356
|
+
const { method, table, where, column } = msg;
|
|
2357
|
+
switch (method) {
|
|
2358
|
+
case "findOne":
|
|
2359
|
+
return { kind: "row", row: await hostCtx.findOne(table, where) };
|
|
2360
|
+
case "findMany":
|
|
2361
|
+
return { kind: "rows", rows: await hostCtx.findMany(table, where) };
|
|
2362
|
+
case "count":
|
|
2363
|
+
return { kind: "count", count: await hostCtx.count(table, where) };
|
|
2364
|
+
case "countDistinct": {
|
|
2365
|
+
if (!column)
|
|
2366
|
+
throw new Error("countDistinct read missing column");
|
|
2367
|
+
return {
|
|
2368
|
+
kind: "count",
|
|
2369
|
+
count: await hostCtx.countDistinct(table, column, where)
|
|
2370
|
+
};
|
|
2371
|
+
}
|
|
2372
|
+
case "sum": {
|
|
2373
|
+
if (!column)
|
|
2374
|
+
throw new Error("sum read missing column");
|
|
2375
|
+
const v = await hostCtx.sum(table, column, where);
|
|
2376
|
+
return { kind: "amount", amount: v.toString() };
|
|
2377
|
+
}
|
|
2378
|
+
case "min": {
|
|
2379
|
+
if (!column)
|
|
2380
|
+
throw new Error("min read missing column");
|
|
2381
|
+
const v = await hostCtx.min(table, column, where);
|
|
2382
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2383
|
+
}
|
|
2384
|
+
case "max": {
|
|
2385
|
+
if (!column)
|
|
2386
|
+
throw new Error("max read missing column");
|
|
2387
|
+
const v = await hostCtx.max(table, column, where);
|
|
2388
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2083
2393
|
// src/runtime/subscription-state.ts
|
|
2084
2394
|
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2085
2395
|
import { sql as sql2 } from "kysely";
|
|
@@ -2255,22 +2565,36 @@ async function refreshMatcher(db) {
|
|
|
2255
2565
|
|
|
2256
2566
|
// src/runtime/block-processor.ts
|
|
2257
2567
|
var routeCache = new Map;
|
|
2568
|
+
|
|
2569
|
+
class ByoKeyUnavailableError extends Error {
|
|
2570
|
+
constructor(subgraphName) {
|
|
2571
|
+
super(`subgraph ${subgraphName} is BYO (database_url_enc set) but this process has no SECONDLAYER_SECRETS_KEY — the subgraph processor deliberately does not hold it (f072). Route BYO subgraphs to a process that does, or unset BYO.`);
|
|
2572
|
+
this.name = "ByoKeyUnavailableError";
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2258
2575
|
async function resolveRoute(subgraphName, targetDb) {
|
|
2259
2576
|
const cached = routeCache.get(subgraphName);
|
|
2260
2577
|
if (cached)
|
|
2261
2578
|
return cached;
|
|
2262
2579
|
const row = await targetDb.selectFrom("subgraphs").selectAll().where("name", "=", subgraphName).executeTakeFirst();
|
|
2263
2580
|
const byo = row ? isByoSubgraph(row) : false;
|
|
2581
|
+
if (row && byo && !secretsKeyAvailable()) {
|
|
2582
|
+
throw new ByoKeyUnavailableError(subgraphName);
|
|
2583
|
+
}
|
|
2264
2584
|
const route = {
|
|
2265
2585
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2266
2586
|
dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
|
|
2267
|
-
byo
|
|
2587
|
+
byo,
|
|
2588
|
+
sandboxWorkers: row?.sandbox_workers === true,
|
|
2589
|
+
handlerCode: row?.handler_code ?? null,
|
|
2590
|
+
version: row?.version ?? ""
|
|
2268
2591
|
};
|
|
2269
2592
|
routeCache.set(subgraphName, route);
|
|
2270
2593
|
return route;
|
|
2271
2594
|
}
|
|
2272
2595
|
function invalidateSubgraphRoute(subgraphName) {
|
|
2273
2596
|
routeCache.delete(subgraphName);
|
|
2597
|
+
evictSandboxWorker(subgraphName);
|
|
2274
2598
|
}
|
|
2275
2599
|
async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
|
|
2276
2600
|
const factories = new Map;
|
|
@@ -2376,7 +2700,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
|
|
|
2376
2700
|
const delay = retryDelaysMs[attempt];
|
|
2377
2701
|
if (delay === undefined)
|
|
2378
2702
|
break;
|
|
2379
|
-
|
|
2703
|
+
logger7.warn("Block processing failed, retrying", {
|
|
2380
2704
|
subgraph: subgraphName,
|
|
2381
2705
|
blockHeight,
|
|
2382
2706
|
attempt: attempt + 1,
|
|
@@ -2408,7 +2732,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2408
2732
|
} else {
|
|
2409
2733
|
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
2410
2734
|
if (!data) {
|
|
2411
|
-
|
|
2735
|
+
logger7.debug("Block not found or non-canonical for subgraph processing", {
|
|
2412
2736
|
subgraph: subgraphName,
|
|
2413
2737
|
blockHeight
|
|
2414
2738
|
});
|
|
@@ -2544,7 +2868,16 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2544
2868
|
}
|
|
2545
2869
|
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2546
2870
|
const handlerStart = performance.now();
|
|
2547
|
-
const runResult =
|
|
2871
|
+
const runResult = sandboxEnabled({
|
|
2872
|
+
sandbox_workers: route.sandboxWorkers
|
|
2873
|
+
}) ? await runHandlersSandboxed({
|
|
2874
|
+
subgraphName,
|
|
2875
|
+
version: route.version,
|
|
2876
|
+
handlerCode: route.handlerCode,
|
|
2877
|
+
hostCtx: ctx,
|
|
2878
|
+
block: blockMeta,
|
|
2879
|
+
matched
|
|
2880
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2548
2881
|
handlerMs = performance.now() - handlerStart;
|
|
2549
2882
|
result.processed = runResult.processed;
|
|
2550
2883
|
result.errors = runResult.errors;
|
|
@@ -2570,7 +2903,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2570
2903
|
});
|
|
2571
2904
|
} catch (err) {
|
|
2572
2905
|
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2573
|
-
|
|
2906
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2574
2907
|
subgraph: subgraphName,
|
|
2575
2908
|
blockHeight,
|
|
2576
2909
|
error: err.message
|
|
@@ -2596,7 +2929,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2596
2929
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
2597
2930
|
const count = Number(rows[0]?.count ?? 0);
|
|
2598
2931
|
if (count >= 1e7) {
|
|
2599
|
-
|
|
2932
|
+
logger7.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
2600
2933
|
subgraph: subgraphName,
|
|
2601
2934
|
table,
|
|
2602
2935
|
count
|
|
@@ -2604,7 +2937,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2604
2937
|
}
|
|
2605
2938
|
}
|
|
2606
2939
|
} catch (err) {
|
|
2607
|
-
|
|
2940
|
+
logger7.debug("Row count sample failed", {
|
|
2608
2941
|
subgraph: subgraphName,
|
|
2609
2942
|
error: err instanceof Error ? err.message : String(err)
|
|
2610
2943
|
});
|
|
@@ -2694,11 +3027,11 @@ import {
|
|
|
2694
3027
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2695
3028
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2696
3029
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2697
|
-
import { logger as
|
|
3030
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2698
3031
|
|
|
2699
3032
|
// src/runtime/reindex-notify.ts
|
|
2700
3033
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2701
|
-
import { logger as
|
|
3034
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2702
3035
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2703
3036
|
try {
|
|
2704
3037
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -2709,7 +3042,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2709
3042
|
return;
|
|
2710
3043
|
const resendKey = process.env.RESEND_API_KEY;
|
|
2711
3044
|
if (!resendKey) {
|
|
2712
|
-
|
|
3045
|
+
logger8.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2713
3046
|
subgraph: subgraphName
|
|
2714
3047
|
});
|
|
2715
3048
|
return;
|
|
@@ -2731,14 +3064,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2731
3064
|
});
|
|
2732
3065
|
if (!res.ok) {
|
|
2733
3066
|
const text = await res.text().catch(() => "");
|
|
2734
|
-
|
|
3067
|
+
logger8.warn("reindex-complete email failed", {
|
|
2735
3068
|
subgraph: subgraphName,
|
|
2736
3069
|
status: res.status,
|
|
2737
3070
|
body: text.slice(0, 200)
|
|
2738
3071
|
});
|
|
2739
3072
|
}
|
|
2740
3073
|
} catch (err) {
|
|
2741
|
-
|
|
3074
|
+
logger8.warn("reindex-complete email threw", {
|
|
2742
3075
|
subgraph: subgraphName,
|
|
2743
3076
|
error: getErrorMessage2(err)
|
|
2744
3077
|
});
|
|
@@ -2850,7 +3183,7 @@ async function processBlockRange(def, opts) {
|
|
|
2850
3183
|
while (currentHeight <= toBlock) {
|
|
2851
3184
|
if (opts.signal?.aborted) {
|
|
2852
3185
|
aborted = true;
|
|
2853
|
-
|
|
3186
|
+
logger9.info("Block processing aborted", {
|
|
2854
3187
|
subgraph: subgraphName,
|
|
2855
3188
|
currentBlock: currentHeight,
|
|
2856
3189
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2899,7 +3232,7 @@ async function processBlockRange(def, opts) {
|
|
|
2899
3232
|
});
|
|
2900
3233
|
} catch (err) {
|
|
2901
3234
|
const errorMsg = getErrorMessage3(err);
|
|
2902
|
-
|
|
3235
|
+
logger9.error("Block processing failed persistently", {
|
|
2903
3236
|
subgraph: subgraphName,
|
|
2904
3237
|
blockHeight: height,
|
|
2905
3238
|
error: errorMsg
|
|
@@ -2945,7 +3278,7 @@ async function processBlockRange(def, opts) {
|
|
|
2945
3278
|
lastProgressFlushAt = now;
|
|
2946
3279
|
}
|
|
2947
3280
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2948
|
-
|
|
3281
|
+
logger9.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2949
3282
|
subgraph: subgraphName,
|
|
2950
3283
|
processed: blocksProcessed,
|
|
2951
3284
|
total: totalBlocks,
|
|
@@ -2965,7 +3298,7 @@ async function processBlockRange(def, opts) {
|
|
|
2965
3298
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2966
3299
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2967
3300
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2968
|
-
|
|
3301
|
+
logger9.warn("Failed to record subgraph gaps", {
|
|
2969
3302
|
subgraph: subgraphName,
|
|
2970
3303
|
error: err instanceof Error ? err.message : String(err)
|
|
2971
3304
|
});
|
|
@@ -2982,7 +3315,7 @@ async function processBlockRange(def, opts) {
|
|
|
2982
3315
|
} else {
|
|
2983
3316
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2984
3317
|
}
|
|
2985
|
-
|
|
3318
|
+
logger9.info("Sparse skip", {
|
|
2986
3319
|
subgraph: subgraphName,
|
|
2987
3320
|
from: batchEnd + 1,
|
|
2988
3321
|
to: jumpTo - 1,
|
|
@@ -3022,11 +3355,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
3022
3355
|
const subgraphName = def.name;
|
|
3023
3356
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3024
3357
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3025
|
-
|
|
3358
|
+
logger9.info("Reindex starting", { subgraph: subgraphName });
|
|
3026
3359
|
try {
|
|
3027
3360
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3028
3361
|
if (fromBlock > toBlock) {
|
|
3029
|
-
|
|
3362
|
+
logger9.info("No blocks to reindex", {
|
|
3030
3363
|
subgraph: subgraphName,
|
|
3031
3364
|
fromBlock,
|
|
3032
3365
|
toBlock
|
|
@@ -3040,7 +3373,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3040
3373
|
await client.unsafe(stmt);
|
|
3041
3374
|
}
|
|
3042
3375
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3043
|
-
|
|
3376
|
+
logger9.info("Schema recreated for reindex", {
|
|
3044
3377
|
subgraph: subgraphName,
|
|
3045
3378
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3046
3379
|
});
|
|
@@ -3054,7 +3387,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3054
3387
|
last_error_at: null,
|
|
3055
3388
|
updated_at: new Date
|
|
3056
3389
|
}).where("name", "=", subgraphName).execute();
|
|
3057
|
-
|
|
3390
|
+
logger9.info("Reindexing blocks", {
|
|
3058
3391
|
subgraph: subgraphName,
|
|
3059
3392
|
fromBlock,
|
|
3060
3393
|
toBlock,
|
|
@@ -3076,9 +3409,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3076
3409
|
if (reason === "user-cancelled") {
|
|
3077
3410
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3078
3411
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3079
|
-
|
|
3412
|
+
logger9.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3080
3413
|
} else {
|
|
3081
|
-
|
|
3414
|
+
logger9.info("Reindex interrupted by shutdown, will resume", {
|
|
3082
3415
|
subgraph: subgraphName
|
|
3083
3416
|
});
|
|
3084
3417
|
}
|
|
@@ -3086,7 +3419,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3086
3419
|
}
|
|
3087
3420
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3088
3421
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3089
|
-
|
|
3422
|
+
logger9.info("Reindex complete", {
|
|
3090
3423
|
subgraph: subgraphName,
|
|
3091
3424
|
blocks: result.blocksProcessed,
|
|
3092
3425
|
events: result.totalEventsProcessed,
|
|
@@ -3099,7 +3432,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3099
3432
|
});
|
|
3100
3433
|
return { processed: result.blocksProcessed };
|
|
3101
3434
|
} catch (err) {
|
|
3102
|
-
|
|
3435
|
+
logger9.error("Reindex failed", {
|
|
3103
3436
|
subgraph: subgraphName,
|
|
3104
3437
|
error: getErrorMessage3(err)
|
|
3105
3438
|
});
|
|
@@ -3121,7 +3454,7 @@ async function resumeReindex(def, opts) {
|
|
|
3121
3454
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3122
3455
|
const toBlock = Number(row.reindex_to_block);
|
|
3123
3456
|
if (fromBlock == null) {
|
|
3124
|
-
|
|
3457
|
+
logger9.info("No reindex metadata, starting fresh reindex", {
|
|
3125
3458
|
subgraph: subgraphName
|
|
3126
3459
|
});
|
|
3127
3460
|
return reindexSubgraph(def, {
|
|
@@ -3130,12 +3463,12 @@ async function resumeReindex(def, opts) {
|
|
|
3130
3463
|
});
|
|
3131
3464
|
}
|
|
3132
3465
|
if (fromBlock > toBlock) {
|
|
3133
|
-
|
|
3466
|
+
logger9.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3134
3467
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3135
3468
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3136
3469
|
return { processed: 0 };
|
|
3137
3470
|
}
|
|
3138
|
-
|
|
3471
|
+
logger9.info("Resuming reindex", {
|
|
3139
3472
|
subgraph: subgraphName,
|
|
3140
3473
|
fromBlock,
|
|
3141
3474
|
toBlock,
|
|
@@ -3157,9 +3490,9 @@ async function resumeReindex(def, opts) {
|
|
|
3157
3490
|
if (reason === "user-cancelled") {
|
|
3158
3491
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3159
3492
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3160
|
-
|
|
3493
|
+
logger9.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3161
3494
|
} else {
|
|
3162
|
-
|
|
3495
|
+
logger9.info("Resume interrupted by shutdown, will resume again", {
|
|
3163
3496
|
subgraph: subgraphName
|
|
3164
3497
|
});
|
|
3165
3498
|
}
|
|
@@ -3167,13 +3500,13 @@ async function resumeReindex(def, opts) {
|
|
|
3167
3500
|
}
|
|
3168
3501
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3169
3502
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3170
|
-
|
|
3503
|
+
logger9.info("Resumed reindex complete", {
|
|
3171
3504
|
subgraph: subgraphName,
|
|
3172
3505
|
blocks: result.blocksProcessed
|
|
3173
3506
|
});
|
|
3174
3507
|
return { processed: result.blocksProcessed };
|
|
3175
3508
|
} catch (err) {
|
|
3176
|
-
|
|
3509
|
+
logger9.error("Resumed reindex failed", {
|
|
3177
3510
|
subgraph: subgraphName,
|
|
3178
3511
|
error: getErrorMessage3(err)
|
|
3179
3512
|
});
|
|
@@ -3184,7 +3517,7 @@ async function resumeReindex(def, opts) {
|
|
|
3184
3517
|
async function backfillSubgraph(def, opts) {
|
|
3185
3518
|
const targetDb = getTargetDb2();
|
|
3186
3519
|
const subgraphName = def.name;
|
|
3187
|
-
|
|
3520
|
+
logger9.info("Backfill starting", {
|
|
3188
3521
|
subgraph: subgraphName,
|
|
3189
3522
|
from: opts.fromBlock,
|
|
3190
3523
|
to: opts.toBlock
|
|
@@ -3202,17 +3535,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3202
3535
|
signal: opts.signal
|
|
3203
3536
|
});
|
|
3204
3537
|
if (result.aborted) {
|
|
3205
|
-
|
|
3538
|
+
logger9.info("Backfill aborted", { subgraph: subgraphName });
|
|
3206
3539
|
return { processed: result.blocksProcessed };
|
|
3207
3540
|
}
|
|
3208
3541
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3209
3542
|
if (resolved > 0) {
|
|
3210
|
-
|
|
3543
|
+
logger9.info("Resolved subgraph gaps via backfill", {
|
|
3211
3544
|
subgraph: subgraphName,
|
|
3212
3545
|
resolved
|
|
3213
3546
|
});
|
|
3214
3547
|
}
|
|
3215
|
-
|
|
3548
|
+
logger9.info("Backfill complete", {
|
|
3216
3549
|
subgraph: subgraphName,
|
|
3217
3550
|
blocks: result.blocksProcessed,
|
|
3218
3551
|
events: result.totalEventsProcessed,
|
|
@@ -3220,7 +3553,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3220
3553
|
});
|
|
3221
3554
|
return { processed: result.blocksProcessed };
|
|
3222
3555
|
} catch (err) {
|
|
3223
|
-
|
|
3556
|
+
logger9.error("Backfill failed", {
|
|
3224
3557
|
subgraph: subgraphName,
|
|
3225
3558
|
error: getErrorMessage3(err)
|
|
3226
3559
|
});
|
|
@@ -3231,7 +3564,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3231
3564
|
// src/runtime/catchup.ts
|
|
3232
3565
|
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3233
3566
|
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
3234
|
-
import { logger as
|
|
3567
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
3235
3568
|
|
|
3236
3569
|
// src/runtime/catchup-leader.ts
|
|
3237
3570
|
import {
|
|
@@ -3365,7 +3698,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3365
3698
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
3366
3699
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
3367
3700
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
3368
|
-
|
|
3701
|
+
logger10.info("Subgraph catch-up starting", {
|
|
3369
3702
|
subgraph: subgraphName,
|
|
3370
3703
|
from: startBlock,
|
|
3371
3704
|
to: chainTip,
|
|
@@ -3381,7 +3714,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3381
3714
|
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
3382
3715
|
while (currentHeight <= chainTip) {
|
|
3383
3716
|
if (!isCatchUpLeader()) {
|
|
3384
|
-
|
|
3717
|
+
logger10.info("Lost catch-up leadership, stopping catch-up", {
|
|
3385
3718
|
subgraph: subgraphName,
|
|
3386
3719
|
processed
|
|
3387
3720
|
});
|
|
@@ -3389,7 +3722,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3389
3722
|
}
|
|
3390
3723
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
3391
3724
|
if (!currentRow || currentRow.status !== "active") {
|
|
3392
|
-
|
|
3725
|
+
logger10.info("Subgraph status changed, stopping catch-up", {
|
|
3393
3726
|
subgraph: subgraphName,
|
|
3394
3727
|
status: currentRow?.status ?? "deleted"
|
|
3395
3728
|
});
|
|
@@ -3418,7 +3751,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3418
3751
|
blockData = (await source.loadBlockRange(height, height)).get(height);
|
|
3419
3752
|
}
|
|
3420
3753
|
if (!blockData) {
|
|
3421
|
-
|
|
3754
|
+
logger10.warn("Block missing during catch-up, deferring to next tick", {
|
|
3422
3755
|
subgraph: subgraphName,
|
|
3423
3756
|
blockHeight: height
|
|
3424
3757
|
});
|
|
@@ -3438,12 +3771,12 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3438
3771
|
});
|
|
3439
3772
|
} catch (err) {
|
|
3440
3773
|
if (err instanceof ReorgEpochChangedError) {
|
|
3441
|
-
|
|
3774
|
+
logger10.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
|
|
3442
3775
|
stopCatchup = true;
|
|
3443
3776
|
break;
|
|
3444
3777
|
}
|
|
3445
3778
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
3446
|
-
|
|
3779
|
+
logger10.error("Block processing failed persistently during catch-up", {
|
|
3447
3780
|
subgraph: subgraphName,
|
|
3448
3781
|
blockHeight: height,
|
|
3449
3782
|
error: errorMsg
|
|
@@ -3462,7 +3795,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3462
3795
|
}
|
|
3463
3796
|
}
|
|
3464
3797
|
if (processed % LOG_INTERVAL2 === 0) {
|
|
3465
|
-
|
|
3798
|
+
logger10.info("Subgraph catch-up progress", {
|
|
3466
3799
|
subgraph: subgraphName,
|
|
3467
3800
|
processed,
|
|
3468
3801
|
total: totalBlocks,
|
|
@@ -3478,7 +3811,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3478
3811
|
currentHeight = batchEnd + 1;
|
|
3479
3812
|
}
|
|
3480
3813
|
await stats.flush(targetDb);
|
|
3481
|
-
|
|
3814
|
+
logger10.info("Subgraph catch-up complete", {
|
|
3482
3815
|
subgraph: subgraphName,
|
|
3483
3816
|
processed
|
|
3484
3817
|
});
|
|
@@ -3495,14 +3828,14 @@ import {
|
|
|
3495
3828
|
listSubgraphs,
|
|
3496
3829
|
resolveSubgraphRawClient
|
|
3497
3830
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3498
|
-
import { logger as
|
|
3831
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
3499
3832
|
import { sql as sql4 } from "kysely";
|
|
3500
3833
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
3501
3834
|
const targetDb = getTargetDb4();
|
|
3502
3835
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
3503
3836
|
if (activeSubgraphs.length === 0)
|
|
3504
3837
|
return;
|
|
3505
|
-
|
|
3838
|
+
logger11.info("Propagating reorg to subgraphs", {
|
|
3506
3839
|
blockHeight,
|
|
3507
3840
|
subgraphCount: activeSubgraphs.length
|
|
3508
3841
|
});
|
|
@@ -3519,7 +3852,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
3519
3852
|
});
|
|
3520
3853
|
});
|
|
3521
3854
|
} catch (err) {
|
|
3522
|
-
|
|
3855
|
+
logger11.error("Subgraph reorg handling failed", {
|
|
3523
3856
|
subgraph: sg.name,
|
|
3524
3857
|
blockHeight,
|
|
3525
3858
|
error: getErrorMessage4(err)
|
|
@@ -3557,7 +3890,7 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3557
3890
|
await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
|
|
3558
3891
|
});
|
|
3559
3892
|
} else {
|
|
3560
|
-
|
|
3893
|
+
logger11.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
|
|
3561
3894
|
for (const tableName of tableNames) {
|
|
3562
3895
|
await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
|
|
3563
3896
|
}
|
|
@@ -3600,7 +3933,7 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3600
3933
|
}).as("row_pk"),
|
|
3601
3934
|
eb2.val("pending").as("status")
|
|
3602
3935
|
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
|
|
3603
|
-
|
|
3936
|
+
logger11.error("Failed to emit revert event for subscriptions", {
|
|
3604
3937
|
event: "reorg_revert_emit_dropped",
|
|
3605
3938
|
subgraph: sg.name,
|
|
3606
3939
|
table: tableName,
|
|
@@ -3609,14 +3942,14 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3609
3942
|
});
|
|
3610
3943
|
});
|
|
3611
3944
|
}
|
|
3612
|
-
|
|
3945
|
+
logger11.info("Subgraph reorg cleanup done", {
|
|
3613
3946
|
subgraph: sg.name,
|
|
3614
3947
|
blockHeight,
|
|
3615
3948
|
tablesAffected: Object.keys(revertedByTable).length
|
|
3616
3949
|
});
|
|
3617
3950
|
const def = await loadSubgraphDef(sg);
|
|
3618
3951
|
await processBlock(def, sg.name, blockHeight, { reorgRewind: true });
|
|
3619
|
-
|
|
3952
|
+
logger11.info("Subgraph reorg reprocessed", {
|
|
3620
3953
|
subgraph: sg.name,
|
|
3621
3954
|
blockHeight
|
|
3622
3955
|
});
|
|
@@ -3646,7 +3979,7 @@ import {
|
|
|
3646
3979
|
pgSchemaName as pgSchemaName2,
|
|
3647
3980
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
3648
3981
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3649
|
-
import { logger as
|
|
3982
|
+
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3650
3983
|
import {
|
|
3651
3984
|
listen,
|
|
3652
3985
|
sourceListenerUrl,
|
|
@@ -3656,7 +3989,7 @@ import {
|
|
|
3656
3989
|
// src/runtime/streams-reorg-poll.ts
|
|
3657
3990
|
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
3658
3991
|
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3659
|
-
import { logger as
|
|
3992
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
3660
3993
|
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3661
3994
|
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3662
3995
|
async function pollReorgsOnce(http2, cursor, onReorg, handled) {
|
|
@@ -3664,13 +3997,13 @@ async function pollReorgsOnce(http2, cursor, onReorg, handled) {
|
|
|
3664
3997
|
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3665
3998
|
for (const r of sorted) {
|
|
3666
3999
|
if (handled && r.id && handled.has(r.id)) {
|
|
3667
|
-
|
|
4000
|
+
logger12.info("Streams reorg already handled — skipping", {
|
|
3668
4001
|
forkPointHeight: r.fork_point_height,
|
|
3669
4002
|
reorgId: r.id
|
|
3670
4003
|
});
|
|
3671
4004
|
continue;
|
|
3672
4005
|
}
|
|
3673
|
-
|
|
4006
|
+
logger12.info("Streams reorg — rewinding", {
|
|
3674
4007
|
forkPointHeight: r.fork_point_height
|
|
3675
4008
|
});
|
|
3676
4009
|
await onReorg(r.fork_point_height);
|
|
@@ -3698,7 +4031,7 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3698
4031
|
try {
|
|
3699
4032
|
since = await pollReorgsOnce(http2, since, onReorg, handled);
|
|
3700
4033
|
} catch (err) {
|
|
3701
|
-
|
|
4034
|
+
logger12.error("Streams reorg poll failed", {
|
|
3702
4035
|
error: getErrorMessage5(err)
|
|
3703
4036
|
});
|
|
3704
4037
|
}
|
|
@@ -3706,7 +4039,7 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3706
4039
|
timer = setTimeout(tick, POLL_MS);
|
|
3707
4040
|
};
|
|
3708
4041
|
timer = setTimeout(tick, POLL_MS);
|
|
3709
|
-
|
|
4042
|
+
logger12.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3710
4043
|
return () => {
|
|
3711
4044
|
running = false;
|
|
3712
4045
|
if (timer)
|
|
@@ -3737,7 +4070,7 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3737
4070
|
if (isHandlerNotFoundError(err)) {
|
|
3738
4071
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3739
4072
|
}
|
|
3740
|
-
|
|
4073
|
+
logger13.error("Subgraph catch-up failed", {
|
|
3741
4074
|
subgraph: sg.name,
|
|
3742
4075
|
error: msg
|
|
3743
4076
|
});
|
|
@@ -3777,7 +4110,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3777
4110
|
definitionCache.set(sg.name, def);
|
|
3778
4111
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3779
4112
|
invalidateSubgraphRoute(sg.name);
|
|
3780
|
-
|
|
4113
|
+
logger13.info("Subgraph handler reloaded", {
|
|
3781
4114
|
subgraph: sg.name,
|
|
3782
4115
|
from: prevVersion,
|
|
3783
4116
|
to: sg.version
|
|
@@ -3811,7 +4144,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3811
4144
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3812
4145
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3813
4146
|
});
|
|
3814
|
-
|
|
4147
|
+
logger13.info("Queued legacy reindex resume operation", {
|
|
3815
4148
|
subgraph: sg.name
|
|
3816
4149
|
});
|
|
3817
4150
|
} catch (err) {
|
|
@@ -3874,7 +4207,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3874
4207
|
const activeRuns = new Map;
|
|
3875
4208
|
let running = true;
|
|
3876
4209
|
let draining = false;
|
|
3877
|
-
|
|
4210
|
+
logger13.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3878
4211
|
try {
|
|
3879
4212
|
const stranded = await db.selectFrom("subgraphs").select(["id", "name", "account_id"]).where("status", "=", "reindexing").where("reindex_from_block", "is not", null).where("reindex_to_block", "is not", null).where(({ not, exists, selectFrom }) => not(exists(selectFrom("subgraph_operations").select("id").whereRef("subgraph_id", "=", "subgraphs.id").where("status", "in", ["queued", "running"])))).execute();
|
|
3880
4213
|
for (const row of stranded) {
|
|
@@ -3885,14 +4218,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3885
4218
|
accountId: row.account_id,
|
|
3886
4219
|
kind: "reindex"
|
|
3887
4220
|
});
|
|
3888
|
-
|
|
4221
|
+
logger13.info("Re-enqueued stranded reindex", { subgraph: row.name });
|
|
3889
4222
|
} catch (err) {
|
|
3890
4223
|
if (!isActiveSubgraphOperationConflict(err))
|
|
3891
4224
|
throw err;
|
|
3892
4225
|
}
|
|
3893
4226
|
}
|
|
3894
4227
|
} catch (err) {
|
|
3895
|
-
|
|
4228
|
+
logger13.warn("Stranded-reindex sweep failed", {
|
|
3896
4229
|
error: getErrorMessage6(err)
|
|
3897
4230
|
});
|
|
3898
4231
|
}
|
|
@@ -3903,7 +4236,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3903
4236
|
if (!running)
|
|
3904
4237
|
return;
|
|
3905
4238
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3906
|
-
|
|
4239
|
+
logger13.warn("Subgraph operation heartbeat failed", {
|
|
3907
4240
|
operationId: operation.id,
|
|
3908
4241
|
error: getErrorMessage6(err)
|
|
3909
4242
|
});
|
|
@@ -3915,7 +4248,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3915
4248
|
controller.abort("user-cancelled");
|
|
3916
4249
|
}
|
|
3917
4250
|
}).catch((err) => {
|
|
3918
|
-
|
|
4251
|
+
logger13.warn("Subgraph operation cancel poll failed", {
|
|
3919
4252
|
operationId: operation.id,
|
|
3920
4253
|
error: getErrorMessage6(err)
|
|
3921
4254
|
});
|
|
@@ -3932,14 +4265,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3932
4265
|
const reason = String(controller.signal.reason ?? "");
|
|
3933
4266
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3934
4267
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3935
|
-
|
|
4268
|
+
logger13.info("Subgraph operation cancelled", {
|
|
3936
4269
|
operationId: operation.id,
|
|
3937
4270
|
subgraph: operation.subgraph_name
|
|
3938
4271
|
});
|
|
3939
4272
|
return;
|
|
3940
4273
|
}
|
|
3941
4274
|
if (controller.signal.aborted) {
|
|
3942
|
-
|
|
4275
|
+
logger13.info("Subgraph operation interrupted", {
|
|
3943
4276
|
operationId: operation.id,
|
|
3944
4277
|
subgraph: operation.subgraph_name,
|
|
3945
4278
|
reason
|
|
@@ -3947,7 +4280,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3947
4280
|
return;
|
|
3948
4281
|
}
|
|
3949
4282
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3950
|
-
|
|
4283
|
+
logger13.info("Subgraph operation completed", {
|
|
3951
4284
|
operationId: operation.id,
|
|
3952
4285
|
subgraph: operation.subgraph_name,
|
|
3953
4286
|
processed
|
|
@@ -3955,7 +4288,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3955
4288
|
} catch (err) {
|
|
3956
4289
|
const reason = String(controller.signal.reason ?? "");
|
|
3957
4290
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3958
|
-
|
|
4291
|
+
logger13.info("Subgraph operation interrupted by shutdown", {
|
|
3959
4292
|
operationId: operation.id,
|
|
3960
4293
|
subgraph: operation.subgraph_name
|
|
3961
4294
|
});
|
|
@@ -3966,7 +4299,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3966
4299
|
return;
|
|
3967
4300
|
}
|
|
3968
4301
|
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
|
|
3969
|
-
|
|
4302
|
+
logger13.error("Subgraph operation failed", {
|
|
3970
4303
|
operationId: operation.id,
|
|
3971
4304
|
subgraph: operation.subgraph_name,
|
|
3972
4305
|
error: getErrorMessage6(err)
|
|
@@ -4013,13 +4346,13 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
4013
4346
|
controller.abort("shutdown");
|
|
4014
4347
|
}
|
|
4015
4348
|
await Promise.allSettled(activeRuns.values());
|
|
4016
|
-
|
|
4349
|
+
logger13.info("Subgraph operation runner stopped");
|
|
4017
4350
|
};
|
|
4018
4351
|
}
|
|
4019
4352
|
async function startSubgraphProcessor(opts) {
|
|
4020
4353
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
4021
4354
|
let running = true;
|
|
4022
|
-
|
|
4355
|
+
logger13.info("Starting subgraph processor", { concurrency });
|
|
4023
4356
|
const stopOperations = await startSubgraphOperationRunner({
|
|
4024
4357
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
4025
4358
|
});
|
|
@@ -4045,7 +4378,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4045
4378
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
4046
4379
|
}
|
|
4047
4380
|
} catch (err) {
|
|
4048
|
-
|
|
4381
|
+
logger13.error("Subgraph reorg handling failed", {
|
|
4049
4382
|
error: getErrorMessage6(err)
|
|
4050
4383
|
});
|
|
4051
4384
|
}
|
|
@@ -4054,7 +4387,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4054
4387
|
runCatchUp();
|
|
4055
4388
|
}, POLL_INTERVAL_MS);
|
|
4056
4389
|
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
4057
|
-
|
|
4390
|
+
logger13.info("Subgraph processor ready");
|
|
4058
4391
|
return async () => {
|
|
4059
4392
|
running = false;
|
|
4060
4393
|
clearInterval(pollInterval);
|
|
@@ -4063,13 +4396,13 @@ async function startSubgraphProcessor(opts) {
|
|
|
4063
4396
|
await stopReorgListening();
|
|
4064
4397
|
stopStreamsReorgPoll?.();
|
|
4065
4398
|
await stopOperations();
|
|
4066
|
-
|
|
4399
|
+
logger13.info("Subgraph processor stopped");
|
|
4067
4400
|
};
|
|
4068
4401
|
}
|
|
4069
4402
|
|
|
4070
4403
|
// src/service.ts
|
|
4071
4404
|
import { assertDbSplit, getDb } from "@secondlayer/shared/db";
|
|
4072
|
-
import { logger as
|
|
4405
|
+
import { logger as logger14 } from "@secondlayer/shared/logger";
|
|
4073
4406
|
import { sql as sql5 } from "kysely";
|
|
4074
4407
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
4075
4408
|
var SERVICE_NAME = "subgraph-processor";
|
|
@@ -4077,7 +4410,7 @@ async function writeHeartbeat() {
|
|
|
4077
4410
|
try {
|
|
4078
4411
|
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql5`now()` })).execute();
|
|
4079
4412
|
} catch (err) {
|
|
4080
|
-
|
|
4413
|
+
logger14.warn("subgraph-processor heartbeat write failed", {
|
|
4081
4414
|
error: err instanceof Error ? err.message : String(err)
|
|
4082
4415
|
});
|
|
4083
4416
|
}
|
|
@@ -4089,7 +4422,7 @@ var processor = await startSubgraphProcessor({
|
|
|
4089
4422
|
await writeHeartbeat();
|
|
4090
4423
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
4091
4424
|
var shutdown = async () => {
|
|
4092
|
-
|
|
4425
|
+
logger14.info("Shutting down subgraph processor...");
|
|
4093
4426
|
clearInterval(heartbeatInterval);
|
|
4094
4427
|
await processor();
|
|
4095
4428
|
process.exit(0);
|
|
@@ -4097,5 +4430,5 @@ var shutdown = async () => {
|
|
|
4097
4430
|
process.on("SIGINT", shutdown);
|
|
4098
4431
|
process.on("SIGTERM", shutdown);
|
|
4099
4432
|
|
|
4100
|
-
//# debugId=
|
|
4433
|
+
//# debugId=93DDC9090FFF48C664756E2164756E21
|
|
4101
4434
|
//# sourceMappingURL=service.js.map
|