@secondlayer/subgraphs 3.24.1 → 3.24.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +372 -39
- package/dist/src/index.js.map +7 -4
- package/dist/src/runtime/block-processor.d.ts +20 -2
- package/dist/src/runtime/block-processor.js +343 -9
- package/dist/src/runtime/block-processor.js.map +7 -4
- package/dist/src/runtime/catchup.js +351 -18
- package/dist/src/runtime/catchup.js.map +7 -4
- package/dist/src/runtime/processor.js +412 -79
- package/dist/src/runtime/processor.js.map +7 -4
- package/dist/src/runtime/reindex.js +372 -39
- package/dist/src/runtime/reindex.js.map +7 -4
- package/dist/src/runtime/reorg.js +358 -25
- package/dist/src/runtime/reorg.js.map +7 -4
- package/dist/src/service.js +415 -82
- package/dist/src/service.js.map +7 -4
- package/package.json +2 -2
|
@@ -1627,6 +1627,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map, f
|
|
|
1627
1627
|
}
|
|
1628
1628
|
|
|
1629
1629
|
// src/runtime/block-processor.ts
|
|
1630
|
+
import { secretsKeyAvailable } from "@secondlayer/shared/crypto/secrets";
|
|
1630
1631
|
import { getTargetDb } from "@secondlayer/shared/db";
|
|
1631
1632
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
1632
1633
|
import { advanceOperationCursor } from "@secondlayer/shared/db/queries/subgraph-operations";
|
|
@@ -1639,7 +1640,7 @@ import {
|
|
|
1639
1640
|
rewindLiveProgress,
|
|
1640
1641
|
updateSubgraphStatus
|
|
1641
1642
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1642
|
-
import { logger as
|
|
1643
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1643
1644
|
import { sql as sql3 } from "kysely";
|
|
1644
1645
|
|
|
1645
1646
|
// src/runtime/block-source.ts
|
|
@@ -2080,6 +2081,315 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2080
2081
|
return rows.length;
|
|
2081
2082
|
}
|
|
2082
2083
|
|
|
2084
|
+
// src/runtime/sandbox/flag.ts
|
|
2085
|
+
function sandboxWorkersGloballyEnabled() {
|
|
2086
|
+
return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
|
|
2087
|
+
}
|
|
2088
|
+
function sandboxEnabled(subgraph) {
|
|
2089
|
+
return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
// src/runtime/sandbox/host.ts
|
|
2093
|
+
import { tmpdir } from "node:os";
|
|
2094
|
+
import { fileURLToPath } from "node:url";
|
|
2095
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2096
|
+
|
|
2097
|
+
// src/runtime/sandbox/bundle.ts
|
|
2098
|
+
import esbuild from "esbuild";
|
|
2099
|
+
var ALLOWED_BARE_SPECIFIERS = new Set;
|
|
2100
|
+
var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
|
|
2101
|
+
var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
|
|
2102
|
+
`;
|
|
2103
|
+
function resolverLockdownPlugin() {
|
|
2104
|
+
return {
|
|
2105
|
+
name: "f071-sandbox-resolver-lockdown",
|
|
2106
|
+
setup(build) {
|
|
2107
|
+
build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
|
|
2108
|
+
path: args.path,
|
|
2109
|
+
namespace: "f071-sandbox-subgraphs-stub"
|
|
2110
|
+
}));
|
|
2111
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
|
|
2112
|
+
build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
|
|
2113
|
+
path: args.path,
|
|
2114
|
+
namespace: "f071-sandbox-blocked"
|
|
2115
|
+
}));
|
|
2116
|
+
build.onResolve({ filter: /^[^./]/ }, (args) => {
|
|
2117
|
+
if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
|
|
2118
|
+
return;
|
|
2119
|
+
if (ALLOWED_BARE_SPECIFIERS.has(args.path))
|
|
2120
|
+
return;
|
|
2121
|
+
return { path: args.path, namespace: "f071-sandbox-blocked" };
|
|
2122
|
+
});
|
|
2123
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
|
|
2124
|
+
contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
|
|
2125
|
+
loader: "js"
|
|
2126
|
+
}));
|
|
2127
|
+
}
|
|
2128
|
+
};
|
|
2129
|
+
}
|
|
2130
|
+
async function bundleHandlerCode(handlerCode) {
|
|
2131
|
+
const result = await esbuild.build({
|
|
2132
|
+
stdin: {
|
|
2133
|
+
contents: handlerCode,
|
|
2134
|
+
loader: "ts",
|
|
2135
|
+
resolveDir: process.cwd()
|
|
2136
|
+
},
|
|
2137
|
+
bundle: true,
|
|
2138
|
+
platform: "neutral",
|
|
2139
|
+
format: "esm",
|
|
2140
|
+
write: false,
|
|
2141
|
+
logLevel: "silent",
|
|
2142
|
+
plugins: [resolverLockdownPlugin()]
|
|
2143
|
+
});
|
|
2144
|
+
const out = result.outputFiles?.[0];
|
|
2145
|
+
if (!out)
|
|
2146
|
+
throw new Error("esbuild produced no output for handler_code bundle");
|
|
2147
|
+
return new TextDecoder().decode(out.contents);
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
// src/runtime/sandbox/host.ts
|
|
2151
|
+
var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
|
|
2152
|
+
var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
|
|
2153
|
+
function spawnSandboxProcess() {
|
|
2154
|
+
let onMessage = null;
|
|
2155
|
+
const proc = Bun.spawn({
|
|
2156
|
+
cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
|
|
2157
|
+
env: { PATH: process.env.PATH ?? "" },
|
|
2158
|
+
cwd: tmpdir(),
|
|
2159
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
2160
|
+
serialization: "advanced",
|
|
2161
|
+
ipc(message) {
|
|
2162
|
+
onMessage?.(message);
|
|
2163
|
+
}
|
|
2164
|
+
});
|
|
2165
|
+
(async () => {
|
|
2166
|
+
const reader = proc.stderr.getReader();
|
|
2167
|
+
const decoder = new TextDecoder;
|
|
2168
|
+
try {
|
|
2169
|
+
while (true) {
|
|
2170
|
+
const { done, value } = await reader.read();
|
|
2171
|
+
if (done)
|
|
2172
|
+
break;
|
|
2173
|
+
const text = decoder.decode(value, { stream: true });
|
|
2174
|
+
if (text.trim()) {
|
|
2175
|
+
logger6.warn("Sandbox subprocess stderr", { stderr: text.trim() });
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
} catch {}
|
|
2179
|
+
})();
|
|
2180
|
+
return {
|
|
2181
|
+
send: (msg) => proc.send(msg),
|
|
2182
|
+
kill: () => proc.kill(),
|
|
2183
|
+
exited: proc.exited,
|
|
2184
|
+
setOnMessage: (handler) => {
|
|
2185
|
+
onMessage = handler;
|
|
2186
|
+
}
|
|
2187
|
+
};
|
|
2188
|
+
}
|
|
2189
|
+
var pool = new Map;
|
|
2190
|
+
|
|
2191
|
+
class SandboxWorkerCrash extends Error {
|
|
2192
|
+
constructor(subgraphName, cause) {
|
|
2193
|
+
super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
|
|
2194
|
+
this.name = "SandboxWorkerCrash";
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
|
|
2198
|
+
const bundledCode = await bundleHandlerCode(handlerCode);
|
|
2199
|
+
let settled = false;
|
|
2200
|
+
const ready = new Promise((resolve, reject) => {
|
|
2201
|
+
proc.setOnMessage((msg) => {
|
|
2202
|
+
if (msg.type === "ready" && msg.version === version) {
|
|
2203
|
+
settled = true;
|
|
2204
|
+
proc.setOnMessage(null);
|
|
2205
|
+
resolve();
|
|
2206
|
+
} else if (msg.type === "error") {
|
|
2207
|
+
settled = true;
|
|
2208
|
+
proc.setOnMessage(null);
|
|
2209
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2210
|
+
}
|
|
2211
|
+
});
|
|
2212
|
+
const init = { type: "init", bundledCode, version };
|
|
2213
|
+
proc.send(init);
|
|
2214
|
+
});
|
|
2215
|
+
const crashed = proc.exited.then((code) => {
|
|
2216
|
+
if (settled)
|
|
2217
|
+
return;
|
|
2218
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
|
|
2219
|
+
});
|
|
2220
|
+
await Promise.race([ready, crashed]);
|
|
2221
|
+
}
|
|
2222
|
+
async function ensureSandboxProcess(subgraphName, version, handlerCode) {
|
|
2223
|
+
if (handlerCode == null) {
|
|
2224
|
+
throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
|
|
2225
|
+
}
|
|
2226
|
+
const existing = pool.get(subgraphName);
|
|
2227
|
+
if (existing && existing.version === version)
|
|
2228
|
+
return existing;
|
|
2229
|
+
if (existing) {
|
|
2230
|
+
try {
|
|
2231
|
+
await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
|
|
2232
|
+
existing.version = version;
|
|
2233
|
+
logger6.info("Sandbox subprocess handler reloaded", {
|
|
2234
|
+
subgraph: subgraphName,
|
|
2235
|
+
version
|
|
2236
|
+
});
|
|
2237
|
+
return existing;
|
|
2238
|
+
} catch (err) {
|
|
2239
|
+
existing.proc.kill();
|
|
2240
|
+
pool.delete(subgraphName);
|
|
2241
|
+
logger6.warn("Sandbox subprocess re-init failed; respawning", {
|
|
2242
|
+
subgraph: subgraphName,
|
|
2243
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2244
|
+
});
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
const proc = spawnSandboxProcess();
|
|
2248
|
+
try {
|
|
2249
|
+
await initSandboxProcess(proc, subgraphName, version, handlerCode);
|
|
2250
|
+
} catch (err) {
|
|
2251
|
+
proc.kill();
|
|
2252
|
+
throw err;
|
|
2253
|
+
}
|
|
2254
|
+
const entry = { proc, version, chain: Promise.resolve() };
|
|
2255
|
+
pool.set(subgraphName, entry);
|
|
2256
|
+
return entry;
|
|
2257
|
+
}
|
|
2258
|
+
function evictSandboxWorker(subgraphName) {
|
|
2259
|
+
const entry = pool.get(subgraphName);
|
|
2260
|
+
if (!entry)
|
|
2261
|
+
return;
|
|
2262
|
+
entry.proc.kill();
|
|
2263
|
+
pool.delete(subgraphName);
|
|
2264
|
+
}
|
|
2265
|
+
async function runHandlersSandboxed(params) {
|
|
2266
|
+
const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
|
|
2267
|
+
const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
|
|
2268
|
+
const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
|
|
2269
|
+
entry.chain = run.catch(() => {
|
|
2270
|
+
return;
|
|
2271
|
+
});
|
|
2272
|
+
return run;
|
|
2273
|
+
}
|
|
2274
|
+
async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
|
|
2275
|
+
const { proc } = entry;
|
|
2276
|
+
let timeoutHandle;
|
|
2277
|
+
let settled = false;
|
|
2278
|
+
try {
|
|
2279
|
+
const done = new Promise((resolve, reject) => {
|
|
2280
|
+
proc.setOnMessage((msg) => {
|
|
2281
|
+
if (msg.type === "readRequest") {
|
|
2282
|
+
answerRead(hostCtx, msg).then((reply) => {
|
|
2283
|
+
proc.send({ type: "readResponse", id: msg.id, reply });
|
|
2284
|
+
}).catch((err) => {
|
|
2285
|
+
settled = true;
|
|
2286
|
+
reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
2287
|
+
});
|
|
2288
|
+
return;
|
|
2289
|
+
}
|
|
2290
|
+
if (msg.type === "blockDone") {
|
|
2291
|
+
settled = true;
|
|
2292
|
+
resolve({
|
|
2293
|
+
ops: msg.ops,
|
|
2294
|
+
result: { processed: msg.processed, errors: msg.errors }
|
|
2295
|
+
});
|
|
2296
|
+
return;
|
|
2297
|
+
}
|
|
2298
|
+
if (msg.type === "error") {
|
|
2299
|
+
settled = true;
|
|
2300
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2301
|
+
}
|
|
2302
|
+
});
|
|
2303
|
+
proc.send({ type: "runBlock", block, matched });
|
|
2304
|
+
});
|
|
2305
|
+
const crashed = proc.exited.then((code) => {
|
|
2306
|
+
if (settled)
|
|
2307
|
+
return;
|
|
2308
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
|
|
2309
|
+
});
|
|
2310
|
+
const deadline = new Promise((_, reject) => {
|
|
2311
|
+
timeoutHandle = setTimeout(() => {
|
|
2312
|
+
settled = true;
|
|
2313
|
+
reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
|
|
2314
|
+
}, BLOCK_TIMEOUT_MS);
|
|
2315
|
+
});
|
|
2316
|
+
const { ops, result } = await Promise.race([done, crashed, deadline]);
|
|
2317
|
+
for (const op of ops) {
|
|
2318
|
+
hostCtx.setTx(op.tx);
|
|
2319
|
+
replayOp(hostCtx, op);
|
|
2320
|
+
}
|
|
2321
|
+
return result;
|
|
2322
|
+
} catch (err) {
|
|
2323
|
+
evictSandboxWorker(subgraphName);
|
|
2324
|
+
logger6.warn("Sandbox subprocess run failed; subprocess evicted", {
|
|
2325
|
+
subgraph: subgraphName,
|
|
2326
|
+
blockHeight: block.height,
|
|
2327
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2328
|
+
});
|
|
2329
|
+
throw err;
|
|
2330
|
+
} finally {
|
|
2331
|
+
if (timeoutHandle !== undefined)
|
|
2332
|
+
clearTimeout(timeoutHandle);
|
|
2333
|
+
proc.setOnMessage(null);
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
function replayOp(hostCtx, op) {
|
|
2337
|
+
switch (op.method) {
|
|
2338
|
+
case "insert":
|
|
2339
|
+
hostCtx.insert(op.table, op.args[0]);
|
|
2340
|
+
return;
|
|
2341
|
+
case "upsert":
|
|
2342
|
+
hostCtx.upsert(op.table, op.args[0], op.args[1]);
|
|
2343
|
+
return;
|
|
2344
|
+
case "increment":
|
|
2345
|
+
hostCtx.increment(op.table, op.args[0], op.args[1]);
|
|
2346
|
+
return;
|
|
2347
|
+
case "update":
|
|
2348
|
+
hostCtx.update(op.table, op.args[0], op.args[1]);
|
|
2349
|
+
return;
|
|
2350
|
+
case "delete":
|
|
2351
|
+
hostCtx.delete(op.table, op.args[0]);
|
|
2352
|
+
return;
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
async function answerRead(hostCtx, msg) {
|
|
2356
|
+
const { method, table, where, column } = msg;
|
|
2357
|
+
switch (method) {
|
|
2358
|
+
case "findOne":
|
|
2359
|
+
return { kind: "row", row: await hostCtx.findOne(table, where) };
|
|
2360
|
+
case "findMany":
|
|
2361
|
+
return { kind: "rows", rows: await hostCtx.findMany(table, where) };
|
|
2362
|
+
case "count":
|
|
2363
|
+
return { kind: "count", count: await hostCtx.count(table, where) };
|
|
2364
|
+
case "countDistinct": {
|
|
2365
|
+
if (!column)
|
|
2366
|
+
throw new Error("countDistinct read missing column");
|
|
2367
|
+
return {
|
|
2368
|
+
kind: "count",
|
|
2369
|
+
count: await hostCtx.countDistinct(table, column, where)
|
|
2370
|
+
};
|
|
2371
|
+
}
|
|
2372
|
+
case "sum": {
|
|
2373
|
+
if (!column)
|
|
2374
|
+
throw new Error("sum read missing column");
|
|
2375
|
+
const v = await hostCtx.sum(table, column, where);
|
|
2376
|
+
return { kind: "amount", amount: v.toString() };
|
|
2377
|
+
}
|
|
2378
|
+
case "min": {
|
|
2379
|
+
if (!column)
|
|
2380
|
+
throw new Error("min read missing column");
|
|
2381
|
+
const v = await hostCtx.min(table, column, where);
|
|
2382
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2383
|
+
}
|
|
2384
|
+
case "max": {
|
|
2385
|
+
if (!column)
|
|
2386
|
+
throw new Error("max read missing column");
|
|
2387
|
+
const v = await hostCtx.max(table, column, where);
|
|
2388
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2083
2393
|
// src/runtime/subscription-state.ts
|
|
2084
2394
|
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2085
2395
|
import { sql as sql2 } from "kysely";
|
|
@@ -2255,22 +2565,36 @@ async function refreshMatcher(db) {
|
|
|
2255
2565
|
|
|
2256
2566
|
// src/runtime/block-processor.ts
|
|
2257
2567
|
var routeCache = new Map;
|
|
2568
|
+
|
|
2569
|
+
class ByoKeyUnavailableError extends Error {
|
|
2570
|
+
constructor(subgraphName) {
|
|
2571
|
+
super(`subgraph ${subgraphName} is BYO (database_url_enc set) but this process has no SECONDLAYER_SECRETS_KEY — the subgraph processor deliberately does not hold it (f072). Route BYO subgraphs to a process that does, or unset BYO.`);
|
|
2572
|
+
this.name = "ByoKeyUnavailableError";
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2258
2575
|
async function resolveRoute(subgraphName, targetDb) {
|
|
2259
2576
|
const cached = routeCache.get(subgraphName);
|
|
2260
2577
|
if (cached)
|
|
2261
2578
|
return cached;
|
|
2262
2579
|
const row = await targetDb.selectFrom("subgraphs").selectAll().where("name", "=", subgraphName).executeTakeFirst();
|
|
2263
2580
|
const byo = row ? isByoSubgraph(row) : false;
|
|
2581
|
+
if (row && byo && !secretsKeyAvailable()) {
|
|
2582
|
+
throw new ByoKeyUnavailableError(subgraphName);
|
|
2583
|
+
}
|
|
2264
2584
|
const route = {
|
|
2265
2585
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2266
2586
|
dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
|
|
2267
|
-
byo
|
|
2587
|
+
byo,
|
|
2588
|
+
sandboxWorkers: row?.sandbox_workers === true,
|
|
2589
|
+
handlerCode: row?.handler_code ?? null,
|
|
2590
|
+
version: row?.version ?? ""
|
|
2268
2591
|
};
|
|
2269
2592
|
routeCache.set(subgraphName, route);
|
|
2270
2593
|
return route;
|
|
2271
2594
|
}
|
|
2272
2595
|
function invalidateSubgraphRoute(subgraphName) {
|
|
2273
2596
|
routeCache.delete(subgraphName);
|
|
2597
|
+
evictSandboxWorker(subgraphName);
|
|
2274
2598
|
}
|
|
2275
2599
|
async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
|
|
2276
2600
|
const factories = new Map;
|
|
@@ -2376,7 +2700,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
|
|
|
2376
2700
|
const delay = retryDelaysMs[attempt];
|
|
2377
2701
|
if (delay === undefined)
|
|
2378
2702
|
break;
|
|
2379
|
-
|
|
2703
|
+
logger7.warn("Block processing failed, retrying", {
|
|
2380
2704
|
subgraph: subgraphName,
|
|
2381
2705
|
blockHeight,
|
|
2382
2706
|
attempt: attempt + 1,
|
|
@@ -2408,7 +2732,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2408
2732
|
} else {
|
|
2409
2733
|
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
2410
2734
|
if (!data) {
|
|
2411
|
-
|
|
2735
|
+
logger7.debug("Block not found or non-canonical for subgraph processing", {
|
|
2412
2736
|
subgraph: subgraphName,
|
|
2413
2737
|
blockHeight
|
|
2414
2738
|
});
|
|
@@ -2544,7 +2868,16 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2544
2868
|
}
|
|
2545
2869
|
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2546
2870
|
const handlerStart = performance.now();
|
|
2547
|
-
const runResult =
|
|
2871
|
+
const runResult = sandboxEnabled({
|
|
2872
|
+
sandbox_workers: route.sandboxWorkers
|
|
2873
|
+
}) ? await runHandlersSandboxed({
|
|
2874
|
+
subgraphName,
|
|
2875
|
+
version: route.version,
|
|
2876
|
+
handlerCode: route.handlerCode,
|
|
2877
|
+
hostCtx: ctx,
|
|
2878
|
+
block: blockMeta,
|
|
2879
|
+
matched
|
|
2880
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2548
2881
|
handlerMs = performance.now() - handlerStart;
|
|
2549
2882
|
result.processed = runResult.processed;
|
|
2550
2883
|
result.errors = runResult.errors;
|
|
@@ -2570,7 +2903,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2570
2903
|
});
|
|
2571
2904
|
} catch (err) {
|
|
2572
2905
|
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2573
|
-
|
|
2906
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2574
2907
|
subgraph: subgraphName,
|
|
2575
2908
|
blockHeight,
|
|
2576
2909
|
error: err.message
|
|
@@ -2596,7 +2929,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2596
2929
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
2597
2930
|
const count = Number(rows[0]?.count ?? 0);
|
|
2598
2931
|
if (count >= 1e7) {
|
|
2599
|
-
|
|
2932
|
+
logger7.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
2600
2933
|
subgraph: subgraphName,
|
|
2601
2934
|
table,
|
|
2602
2935
|
count
|
|
@@ -2604,7 +2937,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2604
2937
|
}
|
|
2605
2938
|
}
|
|
2606
2939
|
} catch (err) {
|
|
2607
|
-
|
|
2940
|
+
logger7.debug("Row count sample failed", {
|
|
2608
2941
|
subgraph: subgraphName,
|
|
2609
2942
|
error: err instanceof Error ? err.message : String(err)
|
|
2610
2943
|
});
|
|
@@ -2694,11 +3027,11 @@ import {
|
|
|
2694
3027
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2695
3028
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2696
3029
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2697
|
-
import { logger as
|
|
3030
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2698
3031
|
|
|
2699
3032
|
// src/runtime/reindex-notify.ts
|
|
2700
3033
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2701
|
-
import { logger as
|
|
3034
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2702
3035
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2703
3036
|
try {
|
|
2704
3037
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -2709,7 +3042,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2709
3042
|
return;
|
|
2710
3043
|
const resendKey = process.env.RESEND_API_KEY;
|
|
2711
3044
|
if (!resendKey) {
|
|
2712
|
-
|
|
3045
|
+
logger8.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2713
3046
|
subgraph: subgraphName
|
|
2714
3047
|
});
|
|
2715
3048
|
return;
|
|
@@ -2731,14 +3064,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2731
3064
|
});
|
|
2732
3065
|
if (!res.ok) {
|
|
2733
3066
|
const text = await res.text().catch(() => "");
|
|
2734
|
-
|
|
3067
|
+
logger8.warn("reindex-complete email failed", {
|
|
2735
3068
|
subgraph: subgraphName,
|
|
2736
3069
|
status: res.status,
|
|
2737
3070
|
body: text.slice(0, 200)
|
|
2738
3071
|
});
|
|
2739
3072
|
}
|
|
2740
3073
|
} catch (err) {
|
|
2741
|
-
|
|
3074
|
+
logger8.warn("reindex-complete email threw", {
|
|
2742
3075
|
subgraph: subgraphName,
|
|
2743
3076
|
error: getErrorMessage2(err)
|
|
2744
3077
|
});
|
|
@@ -2850,7 +3183,7 @@ async function processBlockRange(def, opts) {
|
|
|
2850
3183
|
while (currentHeight <= toBlock) {
|
|
2851
3184
|
if (opts.signal?.aborted) {
|
|
2852
3185
|
aborted = true;
|
|
2853
|
-
|
|
3186
|
+
logger9.info("Block processing aborted", {
|
|
2854
3187
|
subgraph: subgraphName,
|
|
2855
3188
|
currentBlock: currentHeight,
|
|
2856
3189
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2899,7 +3232,7 @@ async function processBlockRange(def, opts) {
|
|
|
2899
3232
|
});
|
|
2900
3233
|
} catch (err) {
|
|
2901
3234
|
const errorMsg = getErrorMessage3(err);
|
|
2902
|
-
|
|
3235
|
+
logger9.error("Block processing failed persistently", {
|
|
2903
3236
|
subgraph: subgraphName,
|
|
2904
3237
|
blockHeight: height,
|
|
2905
3238
|
error: errorMsg
|
|
@@ -2945,7 +3278,7 @@ async function processBlockRange(def, opts) {
|
|
|
2945
3278
|
lastProgressFlushAt = now;
|
|
2946
3279
|
}
|
|
2947
3280
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2948
|
-
|
|
3281
|
+
logger9.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2949
3282
|
subgraph: subgraphName,
|
|
2950
3283
|
processed: blocksProcessed,
|
|
2951
3284
|
total: totalBlocks,
|
|
@@ -2965,7 +3298,7 @@ async function processBlockRange(def, opts) {
|
|
|
2965
3298
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2966
3299
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2967
3300
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2968
|
-
|
|
3301
|
+
logger9.warn("Failed to record subgraph gaps", {
|
|
2969
3302
|
subgraph: subgraphName,
|
|
2970
3303
|
error: err instanceof Error ? err.message : String(err)
|
|
2971
3304
|
});
|
|
@@ -2982,7 +3315,7 @@ async function processBlockRange(def, opts) {
|
|
|
2982
3315
|
} else {
|
|
2983
3316
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2984
3317
|
}
|
|
2985
|
-
|
|
3318
|
+
logger9.info("Sparse skip", {
|
|
2986
3319
|
subgraph: subgraphName,
|
|
2987
3320
|
from: batchEnd + 1,
|
|
2988
3321
|
to: jumpTo - 1,
|
|
@@ -3022,11 +3355,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
3022
3355
|
const subgraphName = def.name;
|
|
3023
3356
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3024
3357
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3025
|
-
|
|
3358
|
+
logger9.info("Reindex starting", { subgraph: subgraphName });
|
|
3026
3359
|
try {
|
|
3027
3360
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3028
3361
|
if (fromBlock > toBlock) {
|
|
3029
|
-
|
|
3362
|
+
logger9.info("No blocks to reindex", {
|
|
3030
3363
|
subgraph: subgraphName,
|
|
3031
3364
|
fromBlock,
|
|
3032
3365
|
toBlock
|
|
@@ -3040,7 +3373,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3040
3373
|
await client.unsafe(stmt);
|
|
3041
3374
|
}
|
|
3042
3375
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3043
|
-
|
|
3376
|
+
logger9.info("Schema recreated for reindex", {
|
|
3044
3377
|
subgraph: subgraphName,
|
|
3045
3378
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3046
3379
|
});
|
|
@@ -3054,7 +3387,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3054
3387
|
last_error_at: null,
|
|
3055
3388
|
updated_at: new Date
|
|
3056
3389
|
}).where("name", "=", subgraphName).execute();
|
|
3057
|
-
|
|
3390
|
+
logger9.info("Reindexing blocks", {
|
|
3058
3391
|
subgraph: subgraphName,
|
|
3059
3392
|
fromBlock,
|
|
3060
3393
|
toBlock,
|
|
@@ -3076,9 +3409,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3076
3409
|
if (reason === "user-cancelled") {
|
|
3077
3410
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3078
3411
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3079
|
-
|
|
3412
|
+
logger9.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3080
3413
|
} else {
|
|
3081
|
-
|
|
3414
|
+
logger9.info("Reindex interrupted by shutdown, will resume", {
|
|
3082
3415
|
subgraph: subgraphName
|
|
3083
3416
|
});
|
|
3084
3417
|
}
|
|
@@ -3086,7 +3419,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3086
3419
|
}
|
|
3087
3420
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3088
3421
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3089
|
-
|
|
3422
|
+
logger9.info("Reindex complete", {
|
|
3090
3423
|
subgraph: subgraphName,
|
|
3091
3424
|
blocks: result.blocksProcessed,
|
|
3092
3425
|
events: result.totalEventsProcessed,
|
|
@@ -3099,7 +3432,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3099
3432
|
});
|
|
3100
3433
|
return { processed: result.blocksProcessed };
|
|
3101
3434
|
} catch (err) {
|
|
3102
|
-
|
|
3435
|
+
logger9.error("Reindex failed", {
|
|
3103
3436
|
subgraph: subgraphName,
|
|
3104
3437
|
error: getErrorMessage3(err)
|
|
3105
3438
|
});
|
|
@@ -3121,7 +3454,7 @@ async function resumeReindex(def, opts) {
|
|
|
3121
3454
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3122
3455
|
const toBlock = Number(row.reindex_to_block);
|
|
3123
3456
|
if (fromBlock == null) {
|
|
3124
|
-
|
|
3457
|
+
logger9.info("No reindex metadata, starting fresh reindex", {
|
|
3125
3458
|
subgraph: subgraphName
|
|
3126
3459
|
});
|
|
3127
3460
|
return reindexSubgraph(def, {
|
|
@@ -3130,12 +3463,12 @@ async function resumeReindex(def, opts) {
|
|
|
3130
3463
|
});
|
|
3131
3464
|
}
|
|
3132
3465
|
if (fromBlock > toBlock) {
|
|
3133
|
-
|
|
3466
|
+
logger9.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3134
3467
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3135
3468
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3136
3469
|
return { processed: 0 };
|
|
3137
3470
|
}
|
|
3138
|
-
|
|
3471
|
+
logger9.info("Resuming reindex", {
|
|
3139
3472
|
subgraph: subgraphName,
|
|
3140
3473
|
fromBlock,
|
|
3141
3474
|
toBlock,
|
|
@@ -3157,9 +3490,9 @@ async function resumeReindex(def, opts) {
|
|
|
3157
3490
|
if (reason === "user-cancelled") {
|
|
3158
3491
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3159
3492
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3160
|
-
|
|
3493
|
+
logger9.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3161
3494
|
} else {
|
|
3162
|
-
|
|
3495
|
+
logger9.info("Resume interrupted by shutdown, will resume again", {
|
|
3163
3496
|
subgraph: subgraphName
|
|
3164
3497
|
});
|
|
3165
3498
|
}
|
|
@@ -3167,13 +3500,13 @@ async function resumeReindex(def, opts) {
|
|
|
3167
3500
|
}
|
|
3168
3501
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3169
3502
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3170
|
-
|
|
3503
|
+
logger9.info("Resumed reindex complete", {
|
|
3171
3504
|
subgraph: subgraphName,
|
|
3172
3505
|
blocks: result.blocksProcessed
|
|
3173
3506
|
});
|
|
3174
3507
|
return { processed: result.blocksProcessed };
|
|
3175
3508
|
} catch (err) {
|
|
3176
|
-
|
|
3509
|
+
logger9.error("Resumed reindex failed", {
|
|
3177
3510
|
subgraph: subgraphName,
|
|
3178
3511
|
error: getErrorMessage3(err)
|
|
3179
3512
|
});
|
|
@@ -3184,7 +3517,7 @@ async function resumeReindex(def, opts) {
|
|
|
3184
3517
|
async function backfillSubgraph(def, opts) {
|
|
3185
3518
|
const targetDb = getTargetDb2();
|
|
3186
3519
|
const subgraphName = def.name;
|
|
3187
|
-
|
|
3520
|
+
logger9.info("Backfill starting", {
|
|
3188
3521
|
subgraph: subgraphName,
|
|
3189
3522
|
from: opts.fromBlock,
|
|
3190
3523
|
to: opts.toBlock
|
|
@@ -3202,17 +3535,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3202
3535
|
signal: opts.signal
|
|
3203
3536
|
});
|
|
3204
3537
|
if (result.aborted) {
|
|
3205
|
-
|
|
3538
|
+
logger9.info("Backfill aborted", { subgraph: subgraphName });
|
|
3206
3539
|
return { processed: result.blocksProcessed };
|
|
3207
3540
|
}
|
|
3208
3541
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3209
3542
|
if (resolved > 0) {
|
|
3210
|
-
|
|
3543
|
+
logger9.info("Resolved subgraph gaps via backfill", {
|
|
3211
3544
|
subgraph: subgraphName,
|
|
3212
3545
|
resolved
|
|
3213
3546
|
});
|
|
3214
3547
|
}
|
|
3215
|
-
|
|
3548
|
+
logger9.info("Backfill complete", {
|
|
3216
3549
|
subgraph: subgraphName,
|
|
3217
3550
|
blocks: result.blocksProcessed,
|
|
3218
3551
|
events: result.totalEventsProcessed,
|
|
@@ -3220,7 +3553,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3220
3553
|
});
|
|
3221
3554
|
return { processed: result.blocksProcessed };
|
|
3222
3555
|
} catch (err) {
|
|
3223
|
-
|
|
3556
|
+
logger9.error("Backfill failed", {
|
|
3224
3557
|
subgraph: subgraphName,
|
|
3225
3558
|
error: getErrorMessage3(err)
|
|
3226
3559
|
});
|
|
@@ -3236,5 +3569,5 @@ export {
|
|
|
3236
3569
|
backfillSubgraph
|
|
3237
3570
|
};
|
|
3238
3571
|
|
|
3239
|
-
//# debugId=
|
|
3572
|
+
//# debugId=492BCFB971F513C864756E2164756E21
|
|
3240
3573
|
//# sourceMappingURL=reindex.js.map
|