@secondlayer/subgraphs 3.24.1 → 3.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +361 -39
- package/dist/src/index.js.map +7 -4
- package/dist/src/runtime/block-processor.d.ts +8 -1
- package/dist/src/runtime/block-processor.js +331 -9
- package/dist/src/runtime/block-processor.js.map +7 -4
- package/dist/src/runtime/catchup.js +340 -18
- package/dist/src/runtime/catchup.js.map +7 -4
- package/dist/src/runtime/processor.js +401 -79
- package/dist/src/runtime/processor.js.map +7 -4
- package/dist/src/runtime/reindex.js +361 -39
- package/dist/src/runtime/reindex.js.map +7 -4
- package/dist/src/runtime/reorg.js +347 -25
- package/dist/src/runtime/reorg.js.map +7 -4
- package/dist/src/service.js +404 -82
- package/dist/src/service.js.map +7 -4
- package/package.json +1 -1
|
@@ -1639,7 +1639,7 @@ import {
|
|
|
1639
1639
|
rewindLiveProgress,
|
|
1640
1640
|
updateSubgraphStatus
|
|
1641
1641
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1642
|
-
import { logger as
|
|
1642
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1643
1643
|
import { sql as sql3 } from "kysely";
|
|
1644
1644
|
|
|
1645
1645
|
// src/runtime/block-source.ts
|
|
@@ -2080,6 +2080,315 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2080
2080
|
return rows.length;
|
|
2081
2081
|
}
|
|
2082
2082
|
|
|
2083
|
+
// src/runtime/sandbox/flag.ts
|
|
2084
|
+
function sandboxWorkersGloballyEnabled() {
|
|
2085
|
+
return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
|
|
2086
|
+
}
|
|
2087
|
+
function sandboxEnabled(subgraph) {
|
|
2088
|
+
return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
// src/runtime/sandbox/host.ts
|
|
2092
|
+
import { tmpdir } from "node:os";
|
|
2093
|
+
import { fileURLToPath } from "node:url";
|
|
2094
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2095
|
+
|
|
2096
|
+
// src/runtime/sandbox/bundle.ts
|
|
2097
|
+
import esbuild from "esbuild";
|
|
2098
|
+
var ALLOWED_BARE_SPECIFIERS = new Set;
|
|
2099
|
+
var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
|
|
2100
|
+
var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
|
|
2101
|
+
`;
|
|
2102
|
+
function resolverLockdownPlugin() {
|
|
2103
|
+
return {
|
|
2104
|
+
name: "f071-sandbox-resolver-lockdown",
|
|
2105
|
+
setup(build) {
|
|
2106
|
+
build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
|
|
2107
|
+
path: args.path,
|
|
2108
|
+
namespace: "f071-sandbox-subgraphs-stub"
|
|
2109
|
+
}));
|
|
2110
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
|
|
2111
|
+
build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
|
|
2112
|
+
path: args.path,
|
|
2113
|
+
namespace: "f071-sandbox-blocked"
|
|
2114
|
+
}));
|
|
2115
|
+
build.onResolve({ filter: /^[^./]/ }, (args) => {
|
|
2116
|
+
if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
|
|
2117
|
+
return;
|
|
2118
|
+
if (ALLOWED_BARE_SPECIFIERS.has(args.path))
|
|
2119
|
+
return;
|
|
2120
|
+
return { path: args.path, namespace: "f071-sandbox-blocked" };
|
|
2121
|
+
});
|
|
2122
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
|
|
2123
|
+
contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
|
|
2124
|
+
loader: "js"
|
|
2125
|
+
}));
|
|
2126
|
+
}
|
|
2127
|
+
};
|
|
2128
|
+
}
|
|
2129
|
+
async function bundleHandlerCode(handlerCode) {
|
|
2130
|
+
const result = await esbuild.build({
|
|
2131
|
+
stdin: {
|
|
2132
|
+
contents: handlerCode,
|
|
2133
|
+
loader: "ts",
|
|
2134
|
+
resolveDir: process.cwd()
|
|
2135
|
+
},
|
|
2136
|
+
bundle: true,
|
|
2137
|
+
platform: "neutral",
|
|
2138
|
+
format: "esm",
|
|
2139
|
+
write: false,
|
|
2140
|
+
logLevel: "silent",
|
|
2141
|
+
plugins: [resolverLockdownPlugin()]
|
|
2142
|
+
});
|
|
2143
|
+
const out = result.outputFiles?.[0];
|
|
2144
|
+
if (!out)
|
|
2145
|
+
throw new Error("esbuild produced no output for handler_code bundle");
|
|
2146
|
+
return new TextDecoder().decode(out.contents);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// src/runtime/sandbox/host.ts
|
|
2150
|
+
var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
|
|
2151
|
+
var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
|
|
2152
|
+
function spawnSandboxProcess() {
|
|
2153
|
+
let onMessage = null;
|
|
2154
|
+
const proc = Bun.spawn({
|
|
2155
|
+
cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
|
|
2156
|
+
env: { PATH: process.env.PATH ?? "" },
|
|
2157
|
+
cwd: tmpdir(),
|
|
2158
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
2159
|
+
serialization: "advanced",
|
|
2160
|
+
ipc(message) {
|
|
2161
|
+
onMessage?.(message);
|
|
2162
|
+
}
|
|
2163
|
+
});
|
|
2164
|
+
(async () => {
|
|
2165
|
+
const reader = proc.stderr.getReader();
|
|
2166
|
+
const decoder = new TextDecoder;
|
|
2167
|
+
try {
|
|
2168
|
+
while (true) {
|
|
2169
|
+
const { done, value } = await reader.read();
|
|
2170
|
+
if (done)
|
|
2171
|
+
break;
|
|
2172
|
+
const text = decoder.decode(value, { stream: true });
|
|
2173
|
+
if (text.trim()) {
|
|
2174
|
+
logger6.warn("Sandbox subprocess stderr", { stderr: text.trim() });
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
} catch {}
|
|
2178
|
+
})();
|
|
2179
|
+
return {
|
|
2180
|
+
send: (msg) => proc.send(msg),
|
|
2181
|
+
kill: () => proc.kill(),
|
|
2182
|
+
exited: proc.exited,
|
|
2183
|
+
setOnMessage: (handler) => {
|
|
2184
|
+
onMessage = handler;
|
|
2185
|
+
}
|
|
2186
|
+
};
|
|
2187
|
+
}
|
|
2188
|
+
var pool = new Map;
|
|
2189
|
+
|
|
2190
|
+
class SandboxWorkerCrash extends Error {
|
|
2191
|
+
constructor(subgraphName, cause) {
|
|
2192
|
+
super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
|
|
2193
|
+
this.name = "SandboxWorkerCrash";
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
|
|
2197
|
+
const bundledCode = await bundleHandlerCode(handlerCode);
|
|
2198
|
+
let settled = false;
|
|
2199
|
+
const ready = new Promise((resolve, reject) => {
|
|
2200
|
+
proc.setOnMessage((msg) => {
|
|
2201
|
+
if (msg.type === "ready" && msg.version === version) {
|
|
2202
|
+
settled = true;
|
|
2203
|
+
proc.setOnMessage(null);
|
|
2204
|
+
resolve();
|
|
2205
|
+
} else if (msg.type === "error") {
|
|
2206
|
+
settled = true;
|
|
2207
|
+
proc.setOnMessage(null);
|
|
2208
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2209
|
+
}
|
|
2210
|
+
});
|
|
2211
|
+
const init = { type: "init", bundledCode, version };
|
|
2212
|
+
proc.send(init);
|
|
2213
|
+
});
|
|
2214
|
+
const crashed = proc.exited.then((code) => {
|
|
2215
|
+
if (settled)
|
|
2216
|
+
return;
|
|
2217
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
|
|
2218
|
+
});
|
|
2219
|
+
await Promise.race([ready, crashed]);
|
|
2220
|
+
}
|
|
2221
|
+
async function ensureSandboxProcess(subgraphName, version, handlerCode) {
|
|
2222
|
+
if (handlerCode == null) {
|
|
2223
|
+
throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
|
|
2224
|
+
}
|
|
2225
|
+
const existing = pool.get(subgraphName);
|
|
2226
|
+
if (existing && existing.version === version)
|
|
2227
|
+
return existing;
|
|
2228
|
+
if (existing) {
|
|
2229
|
+
try {
|
|
2230
|
+
await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
|
|
2231
|
+
existing.version = version;
|
|
2232
|
+
logger6.info("Sandbox subprocess handler reloaded", {
|
|
2233
|
+
subgraph: subgraphName,
|
|
2234
|
+
version
|
|
2235
|
+
});
|
|
2236
|
+
return existing;
|
|
2237
|
+
} catch (err) {
|
|
2238
|
+
existing.proc.kill();
|
|
2239
|
+
pool.delete(subgraphName);
|
|
2240
|
+
logger6.warn("Sandbox subprocess re-init failed; respawning", {
|
|
2241
|
+
subgraph: subgraphName,
|
|
2242
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2243
|
+
});
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
const proc = spawnSandboxProcess();
|
|
2247
|
+
try {
|
|
2248
|
+
await initSandboxProcess(proc, subgraphName, version, handlerCode);
|
|
2249
|
+
} catch (err) {
|
|
2250
|
+
proc.kill();
|
|
2251
|
+
throw err;
|
|
2252
|
+
}
|
|
2253
|
+
const entry = { proc, version, chain: Promise.resolve() };
|
|
2254
|
+
pool.set(subgraphName, entry);
|
|
2255
|
+
return entry;
|
|
2256
|
+
}
|
|
2257
|
+
function evictSandboxWorker(subgraphName) {
|
|
2258
|
+
const entry = pool.get(subgraphName);
|
|
2259
|
+
if (!entry)
|
|
2260
|
+
return;
|
|
2261
|
+
entry.proc.kill();
|
|
2262
|
+
pool.delete(subgraphName);
|
|
2263
|
+
}
|
|
2264
|
+
async function runHandlersSandboxed(params) {
|
|
2265
|
+
const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
|
|
2266
|
+
const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
|
|
2267
|
+
const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
|
|
2268
|
+
entry.chain = run.catch(() => {
|
|
2269
|
+
return;
|
|
2270
|
+
});
|
|
2271
|
+
return run;
|
|
2272
|
+
}
|
|
2273
|
+
async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
|
|
2274
|
+
const { proc } = entry;
|
|
2275
|
+
let timeoutHandle;
|
|
2276
|
+
let settled = false;
|
|
2277
|
+
try {
|
|
2278
|
+
const done = new Promise((resolve, reject) => {
|
|
2279
|
+
proc.setOnMessage((msg) => {
|
|
2280
|
+
if (msg.type === "readRequest") {
|
|
2281
|
+
answerRead(hostCtx, msg).then((reply) => {
|
|
2282
|
+
proc.send({ type: "readResponse", id: msg.id, reply });
|
|
2283
|
+
}).catch((err) => {
|
|
2284
|
+
settled = true;
|
|
2285
|
+
reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
2286
|
+
});
|
|
2287
|
+
return;
|
|
2288
|
+
}
|
|
2289
|
+
if (msg.type === "blockDone") {
|
|
2290
|
+
settled = true;
|
|
2291
|
+
resolve({
|
|
2292
|
+
ops: msg.ops,
|
|
2293
|
+
result: { processed: msg.processed, errors: msg.errors }
|
|
2294
|
+
});
|
|
2295
|
+
return;
|
|
2296
|
+
}
|
|
2297
|
+
if (msg.type === "error") {
|
|
2298
|
+
settled = true;
|
|
2299
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2300
|
+
}
|
|
2301
|
+
});
|
|
2302
|
+
proc.send({ type: "runBlock", block, matched });
|
|
2303
|
+
});
|
|
2304
|
+
const crashed = proc.exited.then((code) => {
|
|
2305
|
+
if (settled)
|
|
2306
|
+
return;
|
|
2307
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
|
|
2308
|
+
});
|
|
2309
|
+
const deadline = new Promise((_, reject) => {
|
|
2310
|
+
timeoutHandle = setTimeout(() => {
|
|
2311
|
+
settled = true;
|
|
2312
|
+
reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
|
|
2313
|
+
}, BLOCK_TIMEOUT_MS);
|
|
2314
|
+
});
|
|
2315
|
+
const { ops, result } = await Promise.race([done, crashed, deadline]);
|
|
2316
|
+
for (const op of ops) {
|
|
2317
|
+
hostCtx.setTx(op.tx);
|
|
2318
|
+
replayOp(hostCtx, op);
|
|
2319
|
+
}
|
|
2320
|
+
return result;
|
|
2321
|
+
} catch (err) {
|
|
2322
|
+
evictSandboxWorker(subgraphName);
|
|
2323
|
+
logger6.warn("Sandbox subprocess run failed; subprocess evicted", {
|
|
2324
|
+
subgraph: subgraphName,
|
|
2325
|
+
blockHeight: block.height,
|
|
2326
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2327
|
+
});
|
|
2328
|
+
throw err;
|
|
2329
|
+
} finally {
|
|
2330
|
+
if (timeoutHandle !== undefined)
|
|
2331
|
+
clearTimeout(timeoutHandle);
|
|
2332
|
+
proc.setOnMessage(null);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
function replayOp(hostCtx, op) {
|
|
2336
|
+
switch (op.method) {
|
|
2337
|
+
case "insert":
|
|
2338
|
+
hostCtx.insert(op.table, op.args[0]);
|
|
2339
|
+
return;
|
|
2340
|
+
case "upsert":
|
|
2341
|
+
hostCtx.upsert(op.table, op.args[0], op.args[1]);
|
|
2342
|
+
return;
|
|
2343
|
+
case "increment":
|
|
2344
|
+
hostCtx.increment(op.table, op.args[0], op.args[1]);
|
|
2345
|
+
return;
|
|
2346
|
+
case "update":
|
|
2347
|
+
hostCtx.update(op.table, op.args[0], op.args[1]);
|
|
2348
|
+
return;
|
|
2349
|
+
case "delete":
|
|
2350
|
+
hostCtx.delete(op.table, op.args[0]);
|
|
2351
|
+
return;
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
async function answerRead(hostCtx, msg) {
|
|
2355
|
+
const { method, table, where, column } = msg;
|
|
2356
|
+
switch (method) {
|
|
2357
|
+
case "findOne":
|
|
2358
|
+
return { kind: "row", row: await hostCtx.findOne(table, where) };
|
|
2359
|
+
case "findMany":
|
|
2360
|
+
return { kind: "rows", rows: await hostCtx.findMany(table, where) };
|
|
2361
|
+
case "count":
|
|
2362
|
+
return { kind: "count", count: await hostCtx.count(table, where) };
|
|
2363
|
+
case "countDistinct": {
|
|
2364
|
+
if (!column)
|
|
2365
|
+
throw new Error("countDistinct read missing column");
|
|
2366
|
+
return {
|
|
2367
|
+
kind: "count",
|
|
2368
|
+
count: await hostCtx.countDistinct(table, column, where)
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
case "sum": {
|
|
2372
|
+
if (!column)
|
|
2373
|
+
throw new Error("sum read missing column");
|
|
2374
|
+
const v = await hostCtx.sum(table, column, where);
|
|
2375
|
+
return { kind: "amount", amount: v.toString() };
|
|
2376
|
+
}
|
|
2377
|
+
case "min": {
|
|
2378
|
+
if (!column)
|
|
2379
|
+
throw new Error("min read missing column");
|
|
2380
|
+
const v = await hostCtx.min(table, column, where);
|
|
2381
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2382
|
+
}
|
|
2383
|
+
case "max": {
|
|
2384
|
+
if (!column)
|
|
2385
|
+
throw new Error("max read missing column");
|
|
2386
|
+
const v = await hostCtx.max(table, column, where);
|
|
2387
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2083
2392
|
// src/runtime/subscription-state.ts
|
|
2084
2393
|
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2085
2394
|
import { sql as sql2 } from "kysely";
|
|
@@ -2264,13 +2573,17 @@ async function resolveRoute(subgraphName, targetDb) {
|
|
|
2264
2573
|
const route = {
|
|
2265
2574
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2266
2575
|
dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
|
|
2267
|
-
byo
|
|
2576
|
+
byo,
|
|
2577
|
+
sandboxWorkers: row?.sandbox_workers === true,
|
|
2578
|
+
handlerCode: row?.handler_code ?? null,
|
|
2579
|
+
version: row?.version ?? ""
|
|
2268
2580
|
};
|
|
2269
2581
|
routeCache.set(subgraphName, route);
|
|
2270
2582
|
return route;
|
|
2271
2583
|
}
|
|
2272
2584
|
function invalidateSubgraphRoute(subgraphName) {
|
|
2273
2585
|
routeCache.delete(subgraphName);
|
|
2586
|
+
evictSandboxWorker(subgraphName);
|
|
2274
2587
|
}
|
|
2275
2588
|
async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
|
|
2276
2589
|
const factories = new Map;
|
|
@@ -2376,7 +2689,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
|
|
|
2376
2689
|
const delay = retryDelaysMs[attempt];
|
|
2377
2690
|
if (delay === undefined)
|
|
2378
2691
|
break;
|
|
2379
|
-
|
|
2692
|
+
logger7.warn("Block processing failed, retrying", {
|
|
2380
2693
|
subgraph: subgraphName,
|
|
2381
2694
|
blockHeight,
|
|
2382
2695
|
attempt: attempt + 1,
|
|
@@ -2408,7 +2721,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2408
2721
|
} else {
|
|
2409
2722
|
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
2410
2723
|
if (!data) {
|
|
2411
|
-
|
|
2724
|
+
logger7.debug("Block not found or non-canonical for subgraph processing", {
|
|
2412
2725
|
subgraph: subgraphName,
|
|
2413
2726
|
blockHeight
|
|
2414
2727
|
});
|
|
@@ -2544,7 +2857,16 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2544
2857
|
}
|
|
2545
2858
|
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2546
2859
|
const handlerStart = performance.now();
|
|
2547
|
-
const runResult =
|
|
2860
|
+
const runResult = sandboxEnabled({
|
|
2861
|
+
sandbox_workers: route.sandboxWorkers
|
|
2862
|
+
}) ? await runHandlersSandboxed({
|
|
2863
|
+
subgraphName,
|
|
2864
|
+
version: route.version,
|
|
2865
|
+
handlerCode: route.handlerCode,
|
|
2866
|
+
hostCtx: ctx,
|
|
2867
|
+
block: blockMeta,
|
|
2868
|
+
matched
|
|
2869
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2548
2870
|
handlerMs = performance.now() - handlerStart;
|
|
2549
2871
|
result.processed = runResult.processed;
|
|
2550
2872
|
result.errors = runResult.errors;
|
|
@@ -2570,7 +2892,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2570
2892
|
});
|
|
2571
2893
|
} catch (err) {
|
|
2572
2894
|
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2573
|
-
|
|
2895
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2574
2896
|
subgraph: subgraphName,
|
|
2575
2897
|
blockHeight,
|
|
2576
2898
|
error: err.message
|
|
@@ -2596,7 +2918,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2596
2918
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
2597
2919
|
const count = Number(rows[0]?.count ?? 0);
|
|
2598
2920
|
if (count >= 1e7) {
|
|
2599
|
-
|
|
2921
|
+
logger7.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
2600
2922
|
subgraph: subgraphName,
|
|
2601
2923
|
table,
|
|
2602
2924
|
count
|
|
@@ -2604,7 +2926,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2604
2926
|
}
|
|
2605
2927
|
}
|
|
2606
2928
|
} catch (err) {
|
|
2607
|
-
|
|
2929
|
+
logger7.debug("Row count sample failed", {
|
|
2608
2930
|
subgraph: subgraphName,
|
|
2609
2931
|
error: err instanceof Error ? err.message : String(err)
|
|
2610
2932
|
});
|
|
@@ -2694,11 +3016,11 @@ import {
|
|
|
2694
3016
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2695
3017
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2696
3018
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2697
|
-
import { logger as
|
|
3019
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2698
3020
|
|
|
2699
3021
|
// src/runtime/reindex-notify.ts
|
|
2700
3022
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2701
|
-
import { logger as
|
|
3023
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2702
3024
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2703
3025
|
try {
|
|
2704
3026
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -2709,7 +3031,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2709
3031
|
return;
|
|
2710
3032
|
const resendKey = process.env.RESEND_API_KEY;
|
|
2711
3033
|
if (!resendKey) {
|
|
2712
|
-
|
|
3034
|
+
logger8.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2713
3035
|
subgraph: subgraphName
|
|
2714
3036
|
});
|
|
2715
3037
|
return;
|
|
@@ -2731,14 +3053,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2731
3053
|
});
|
|
2732
3054
|
if (!res.ok) {
|
|
2733
3055
|
const text = await res.text().catch(() => "");
|
|
2734
|
-
|
|
3056
|
+
logger8.warn("reindex-complete email failed", {
|
|
2735
3057
|
subgraph: subgraphName,
|
|
2736
3058
|
status: res.status,
|
|
2737
3059
|
body: text.slice(0, 200)
|
|
2738
3060
|
});
|
|
2739
3061
|
}
|
|
2740
3062
|
} catch (err) {
|
|
2741
|
-
|
|
3063
|
+
logger8.warn("reindex-complete email threw", {
|
|
2742
3064
|
subgraph: subgraphName,
|
|
2743
3065
|
error: getErrorMessage2(err)
|
|
2744
3066
|
});
|
|
@@ -2850,7 +3172,7 @@ async function processBlockRange(def, opts) {
|
|
|
2850
3172
|
while (currentHeight <= toBlock) {
|
|
2851
3173
|
if (opts.signal?.aborted) {
|
|
2852
3174
|
aborted = true;
|
|
2853
|
-
|
|
3175
|
+
logger9.info("Block processing aborted", {
|
|
2854
3176
|
subgraph: subgraphName,
|
|
2855
3177
|
currentBlock: currentHeight,
|
|
2856
3178
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2899,7 +3221,7 @@ async function processBlockRange(def, opts) {
|
|
|
2899
3221
|
});
|
|
2900
3222
|
} catch (err) {
|
|
2901
3223
|
const errorMsg = getErrorMessage3(err);
|
|
2902
|
-
|
|
3224
|
+
logger9.error("Block processing failed persistently", {
|
|
2903
3225
|
subgraph: subgraphName,
|
|
2904
3226
|
blockHeight: height,
|
|
2905
3227
|
error: errorMsg
|
|
@@ -2945,7 +3267,7 @@ async function processBlockRange(def, opts) {
|
|
|
2945
3267
|
lastProgressFlushAt = now;
|
|
2946
3268
|
}
|
|
2947
3269
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2948
|
-
|
|
3270
|
+
logger9.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2949
3271
|
subgraph: subgraphName,
|
|
2950
3272
|
processed: blocksProcessed,
|
|
2951
3273
|
total: totalBlocks,
|
|
@@ -2965,7 +3287,7 @@ async function processBlockRange(def, opts) {
|
|
|
2965
3287
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2966
3288
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2967
3289
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2968
|
-
|
|
3290
|
+
logger9.warn("Failed to record subgraph gaps", {
|
|
2969
3291
|
subgraph: subgraphName,
|
|
2970
3292
|
error: err instanceof Error ? err.message : String(err)
|
|
2971
3293
|
});
|
|
@@ -2982,7 +3304,7 @@ async function processBlockRange(def, opts) {
|
|
|
2982
3304
|
} else {
|
|
2983
3305
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2984
3306
|
}
|
|
2985
|
-
|
|
3307
|
+
logger9.info("Sparse skip", {
|
|
2986
3308
|
subgraph: subgraphName,
|
|
2987
3309
|
from: batchEnd + 1,
|
|
2988
3310
|
to: jumpTo - 1,
|
|
@@ -3022,11 +3344,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
3022
3344
|
const subgraphName = def.name;
|
|
3023
3345
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3024
3346
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3025
|
-
|
|
3347
|
+
logger9.info("Reindex starting", { subgraph: subgraphName });
|
|
3026
3348
|
try {
|
|
3027
3349
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3028
3350
|
if (fromBlock > toBlock) {
|
|
3029
|
-
|
|
3351
|
+
logger9.info("No blocks to reindex", {
|
|
3030
3352
|
subgraph: subgraphName,
|
|
3031
3353
|
fromBlock,
|
|
3032
3354
|
toBlock
|
|
@@ -3040,7 +3362,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3040
3362
|
await client.unsafe(stmt);
|
|
3041
3363
|
}
|
|
3042
3364
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3043
|
-
|
|
3365
|
+
logger9.info("Schema recreated for reindex", {
|
|
3044
3366
|
subgraph: subgraphName,
|
|
3045
3367
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3046
3368
|
});
|
|
@@ -3054,7 +3376,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3054
3376
|
last_error_at: null,
|
|
3055
3377
|
updated_at: new Date
|
|
3056
3378
|
}).where("name", "=", subgraphName).execute();
|
|
3057
|
-
|
|
3379
|
+
logger9.info("Reindexing blocks", {
|
|
3058
3380
|
subgraph: subgraphName,
|
|
3059
3381
|
fromBlock,
|
|
3060
3382
|
toBlock,
|
|
@@ -3076,9 +3398,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3076
3398
|
if (reason === "user-cancelled") {
|
|
3077
3399
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3078
3400
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3079
|
-
|
|
3401
|
+
logger9.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3080
3402
|
} else {
|
|
3081
|
-
|
|
3403
|
+
logger9.info("Reindex interrupted by shutdown, will resume", {
|
|
3082
3404
|
subgraph: subgraphName
|
|
3083
3405
|
});
|
|
3084
3406
|
}
|
|
@@ -3086,7 +3408,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3086
3408
|
}
|
|
3087
3409
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3088
3410
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3089
|
-
|
|
3411
|
+
logger9.info("Reindex complete", {
|
|
3090
3412
|
subgraph: subgraphName,
|
|
3091
3413
|
blocks: result.blocksProcessed,
|
|
3092
3414
|
events: result.totalEventsProcessed,
|
|
@@ -3099,7 +3421,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3099
3421
|
});
|
|
3100
3422
|
return { processed: result.blocksProcessed };
|
|
3101
3423
|
} catch (err) {
|
|
3102
|
-
|
|
3424
|
+
logger9.error("Reindex failed", {
|
|
3103
3425
|
subgraph: subgraphName,
|
|
3104
3426
|
error: getErrorMessage3(err)
|
|
3105
3427
|
});
|
|
@@ -3121,7 +3443,7 @@ async function resumeReindex(def, opts) {
|
|
|
3121
3443
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3122
3444
|
const toBlock = Number(row.reindex_to_block);
|
|
3123
3445
|
if (fromBlock == null) {
|
|
3124
|
-
|
|
3446
|
+
logger9.info("No reindex metadata, starting fresh reindex", {
|
|
3125
3447
|
subgraph: subgraphName
|
|
3126
3448
|
});
|
|
3127
3449
|
return reindexSubgraph(def, {
|
|
@@ -3130,12 +3452,12 @@ async function resumeReindex(def, opts) {
|
|
|
3130
3452
|
});
|
|
3131
3453
|
}
|
|
3132
3454
|
if (fromBlock > toBlock) {
|
|
3133
|
-
|
|
3455
|
+
logger9.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3134
3456
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3135
3457
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3136
3458
|
return { processed: 0 };
|
|
3137
3459
|
}
|
|
3138
|
-
|
|
3460
|
+
logger9.info("Resuming reindex", {
|
|
3139
3461
|
subgraph: subgraphName,
|
|
3140
3462
|
fromBlock,
|
|
3141
3463
|
toBlock,
|
|
@@ -3157,9 +3479,9 @@ async function resumeReindex(def, opts) {
|
|
|
3157
3479
|
if (reason === "user-cancelled") {
|
|
3158
3480
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3159
3481
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3160
|
-
|
|
3482
|
+
logger9.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3161
3483
|
} else {
|
|
3162
|
-
|
|
3484
|
+
logger9.info("Resume interrupted by shutdown, will resume again", {
|
|
3163
3485
|
subgraph: subgraphName
|
|
3164
3486
|
});
|
|
3165
3487
|
}
|
|
@@ -3167,13 +3489,13 @@ async function resumeReindex(def, opts) {
|
|
|
3167
3489
|
}
|
|
3168
3490
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3169
3491
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3170
|
-
|
|
3492
|
+
logger9.info("Resumed reindex complete", {
|
|
3171
3493
|
subgraph: subgraphName,
|
|
3172
3494
|
blocks: result.blocksProcessed
|
|
3173
3495
|
});
|
|
3174
3496
|
return { processed: result.blocksProcessed };
|
|
3175
3497
|
} catch (err) {
|
|
3176
|
-
|
|
3498
|
+
logger9.error("Resumed reindex failed", {
|
|
3177
3499
|
subgraph: subgraphName,
|
|
3178
3500
|
error: getErrorMessage3(err)
|
|
3179
3501
|
});
|
|
@@ -3184,7 +3506,7 @@ async function resumeReindex(def, opts) {
|
|
|
3184
3506
|
async function backfillSubgraph(def, opts) {
|
|
3185
3507
|
const targetDb = getTargetDb2();
|
|
3186
3508
|
const subgraphName = def.name;
|
|
3187
|
-
|
|
3509
|
+
logger9.info("Backfill starting", {
|
|
3188
3510
|
subgraph: subgraphName,
|
|
3189
3511
|
from: opts.fromBlock,
|
|
3190
3512
|
to: opts.toBlock
|
|
@@ -3202,17 +3524,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3202
3524
|
signal: opts.signal
|
|
3203
3525
|
});
|
|
3204
3526
|
if (result.aborted) {
|
|
3205
|
-
|
|
3527
|
+
logger9.info("Backfill aborted", { subgraph: subgraphName });
|
|
3206
3528
|
return { processed: result.blocksProcessed };
|
|
3207
3529
|
}
|
|
3208
3530
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3209
3531
|
if (resolved > 0) {
|
|
3210
|
-
|
|
3532
|
+
logger9.info("Resolved subgraph gaps via backfill", {
|
|
3211
3533
|
subgraph: subgraphName,
|
|
3212
3534
|
resolved
|
|
3213
3535
|
});
|
|
3214
3536
|
}
|
|
3215
|
-
|
|
3537
|
+
logger9.info("Backfill complete", {
|
|
3216
3538
|
subgraph: subgraphName,
|
|
3217
3539
|
blocks: result.blocksProcessed,
|
|
3218
3540
|
events: result.totalEventsProcessed,
|
|
@@ -3220,7 +3542,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3220
3542
|
});
|
|
3221
3543
|
return { processed: result.blocksProcessed };
|
|
3222
3544
|
} catch (err) {
|
|
3223
|
-
|
|
3545
|
+
logger9.error("Backfill failed", {
|
|
3224
3546
|
subgraph: subgraphName,
|
|
3225
3547
|
error: getErrorMessage3(err)
|
|
3226
3548
|
});
|
|
@@ -3236,5 +3558,5 @@ export {
|
|
|
3236
3558
|
backfillSubgraph
|
|
3237
3559
|
};
|
|
3238
3560
|
|
|
3239
|
-
//# debugId=
|
|
3561
|
+
//# debugId=06DD68AD526188D464756E2164756E21
|
|
3240
3562
|
//# sourceMappingURL=reindex.js.map
|