@secondlayer/subgraphs 3.24.0 → 3.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +379 -52
- package/dist/src/index.js.map +9 -6
- package/dist/src/runtime/block-processor.d.ts +8 -1
- package/dist/src/runtime/block-processor.js +349 -22
- package/dist/src/runtime/block-processor.js.map +9 -6
- package/dist/src/runtime/catchup.js +358 -31
- package/dist/src/runtime/catchup.js.map +9 -6
- package/dist/src/runtime/context.js +19 -14
- package/dist/src/runtime/context.js.map +4 -4
- package/dist/src/runtime/processor.js +419 -92
- package/dist/src/runtime/processor.js.map +9 -6
- package/dist/src/runtime/reindex.js +379 -52
- package/dist/src/runtime/reindex.js.map +9 -6
- package/dist/src/runtime/reorg.js +365 -38
- package/dist/src/runtime/reorg.js.map +9 -6
- package/dist/src/runtime/replay.js +6 -1
- package/dist/src/runtime/replay.js.map +3 -3
- package/dist/src/schema/index.js +19 -14
- package/dist/src/schema/index.js.map +4 -4
- package/dist/src/service.js +422 -95
- package/dist/src/service.js.map +9 -6
- package/dist/src/testing/index.js +19 -14
- package/dist/src/testing/index.js.map +4 -4
- package/package.json +1 -1
|
@@ -11,6 +11,11 @@ import { createHash } from "node:crypto";
|
|
|
11
11
|
|
|
12
12
|
// src/schema/utils.ts
|
|
13
13
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
14
|
+
function quotePgIdent(name) {
|
|
15
|
+
if (/^[a-z_][a-z0-9_]*$/i.test(name))
|
|
16
|
+
return name;
|
|
17
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
18
|
+
}
|
|
14
19
|
|
|
15
20
|
// src/schema/generator.ts
|
|
16
21
|
var TYPE_MAP = {
|
|
@@ -35,7 +40,7 @@ function tableNeedsTrgm(tableDef) {
|
|
|
35
40
|
return Object.values(tableDef.columns).some((col) => col.search);
|
|
36
41
|
}
|
|
37
42
|
function emitTableDDL(schemaName, tableName, tableDef) {
|
|
38
|
-
const qualifiedName = `${schemaName}.${tableName}`;
|
|
43
|
+
const qualifiedName = `${quotePgIdent(schemaName)}.${quotePgIdent(tableName)}`;
|
|
39
44
|
const statements = [];
|
|
40
45
|
const columnDefs = [
|
|
41
46
|
"_id BIGSERIAL PRIMARY KEY",
|
|
@@ -46,12 +51,12 @@ function emitTableDDL(schemaName, tableName, tableDef) {
|
|
|
46
51
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
47
52
|
const sqlType = TYPE_MAP[col.type];
|
|
48
53
|
const nullable = col.nullable ? "" : " NOT NULL";
|
|
49
|
-
let colDef = `${colName} ${sqlType}${nullable}`;
|
|
54
|
+
let colDef = `${quotePgIdent(colName)} ${sqlType}${nullable}`;
|
|
50
55
|
if (col.default !== undefined) {
|
|
51
56
|
colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
|
|
52
57
|
}
|
|
53
58
|
if (col.type === "uint") {
|
|
54
|
-
colDef += ` CHECK (${colName} >= 0)`;
|
|
59
|
+
colDef += ` CHECK (${quotePgIdent(colName)} >= 0)`;
|
|
55
60
|
}
|
|
56
61
|
columnDefs.push(colDef);
|
|
57
62
|
}
|
|
@@ -59,30 +64,30 @@ function emitTableDDL(schemaName, tableName, tableDef) {
|
|
|
59
64
|
${columnDefs.join(`,
|
|
60
65
|
`)}
|
|
61
66
|
)`);
|
|
62
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
|
|
63
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
|
|
67
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_block_height`)} ON ${qualifiedName} (_block_height)`);
|
|
68
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_tx_id`)} ON ${qualifiedName} (_tx_id)`);
|
|
64
69
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
65
70
|
if (col.indexed) {
|
|
66
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
|
|
71
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_${colName}`)} ON ${qualifiedName} (${quotePgIdent(colName)})`);
|
|
67
72
|
}
|
|
68
73
|
}
|
|
69
74
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
70
75
|
if (col.search) {
|
|
71
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
|
|
76
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_${colName}_trgm`)} ON ${qualifiedName} USING gin (${quotePgIdent(colName)} gin_trgm_ops)`);
|
|
72
77
|
}
|
|
73
78
|
}
|
|
74
79
|
if (tableDef.indexes) {
|
|
75
80
|
for (let i = 0;i < tableDef.indexes.length; i++) {
|
|
76
81
|
const cols = tableDef.indexes[i];
|
|
77
|
-
const idxName = `idx_${schemaName}_${tableName}_composite_${i}
|
|
78
|
-
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
|
|
82
|
+
const idxName = quotePgIdent(`idx_${schemaName}_${tableName}_composite_${i}`);
|
|
83
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.map(quotePgIdent).join(", ")})`);
|
|
79
84
|
}
|
|
80
85
|
}
|
|
81
86
|
if (tableDef.uniqueKeys) {
|
|
82
87
|
for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
|
|
83
88
|
const cols = tableDef.uniqueKeys[i];
|
|
84
|
-
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}
|
|
85
|
-
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
|
|
89
|
+
const constraintName = quotePgIdent(`uq_${schemaName}_${tableName}_${cols.join("_")}`);
|
|
90
|
+
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.map(quotePgIdent).join(", ")})`);
|
|
86
91
|
}
|
|
87
92
|
}
|
|
88
93
|
return statements;
|
|
@@ -114,8 +119,8 @@ function emitFactoryDDL(schemaName) {
|
|
|
114
119
|
}
|
|
115
120
|
function emitForeignKeyDDL(schemaName, tableName, tableDef) {
|
|
116
121
|
return (tableDef.relations ?? []).map((rel) => {
|
|
117
|
-
const constraintName = `fk_${schemaName}_${tableName}_${rel.name}
|
|
118
|
-
return `ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`;
|
|
122
|
+
const constraintName = quotePgIdent(`fk_${schemaName}_${tableName}_${rel.name}`);
|
|
123
|
+
return `ALTER TABLE ${quotePgIdent(schemaName)}.${quotePgIdent(tableName)} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.map(quotePgIdent).join(", ")}) ` + `REFERENCES ${quotePgIdent(schemaName)}.${quotePgIdent(rel.references)} (${rel.referencedColumns.map(quotePgIdent).join(", ")})`;
|
|
119
124
|
});
|
|
120
125
|
}
|
|
121
126
|
function generateSubgraphSQL(def, schemaNameOverride) {
|
|
@@ -1634,7 +1639,7 @@ import {
|
|
|
1634
1639
|
rewindLiveProgress,
|
|
1635
1640
|
updateSubgraphStatus
|
|
1636
1641
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1637
|
-
import { logger as
|
|
1642
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1638
1643
|
import { sql as sql3 } from "kysely";
|
|
1639
1644
|
|
|
1640
1645
|
// src/runtime/block-source.ts
|
|
@@ -2075,6 +2080,315 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2075
2080
|
return rows.length;
|
|
2076
2081
|
}
|
|
2077
2082
|
|
|
2083
|
+
// src/runtime/sandbox/flag.ts
|
|
2084
|
+
function sandboxWorkersGloballyEnabled() {
|
|
2085
|
+
return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
|
|
2086
|
+
}
|
|
2087
|
+
function sandboxEnabled(subgraph) {
|
|
2088
|
+
return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
// src/runtime/sandbox/host.ts
|
|
2092
|
+
import { tmpdir } from "node:os";
|
|
2093
|
+
import { fileURLToPath } from "node:url";
|
|
2094
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2095
|
+
|
|
2096
|
+
// src/runtime/sandbox/bundle.ts
|
|
2097
|
+
import esbuild from "esbuild";
|
|
2098
|
+
var ALLOWED_BARE_SPECIFIERS = new Set;
|
|
2099
|
+
var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
|
|
2100
|
+
var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
|
|
2101
|
+
`;
|
|
2102
|
+
function resolverLockdownPlugin() {
|
|
2103
|
+
return {
|
|
2104
|
+
name: "f071-sandbox-resolver-lockdown",
|
|
2105
|
+
setup(build) {
|
|
2106
|
+
build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
|
|
2107
|
+
path: args.path,
|
|
2108
|
+
namespace: "f071-sandbox-subgraphs-stub"
|
|
2109
|
+
}));
|
|
2110
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
|
|
2111
|
+
build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
|
|
2112
|
+
path: args.path,
|
|
2113
|
+
namespace: "f071-sandbox-blocked"
|
|
2114
|
+
}));
|
|
2115
|
+
build.onResolve({ filter: /^[^./]/ }, (args) => {
|
|
2116
|
+
if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
|
|
2117
|
+
return;
|
|
2118
|
+
if (ALLOWED_BARE_SPECIFIERS.has(args.path))
|
|
2119
|
+
return;
|
|
2120
|
+
return { path: args.path, namespace: "f071-sandbox-blocked" };
|
|
2121
|
+
});
|
|
2122
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
|
|
2123
|
+
contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
|
|
2124
|
+
loader: "js"
|
|
2125
|
+
}));
|
|
2126
|
+
}
|
|
2127
|
+
};
|
|
2128
|
+
}
|
|
2129
|
+
async function bundleHandlerCode(handlerCode) {
|
|
2130
|
+
const result = await esbuild.build({
|
|
2131
|
+
stdin: {
|
|
2132
|
+
contents: handlerCode,
|
|
2133
|
+
loader: "ts",
|
|
2134
|
+
resolveDir: process.cwd()
|
|
2135
|
+
},
|
|
2136
|
+
bundle: true,
|
|
2137
|
+
platform: "neutral",
|
|
2138
|
+
format: "esm",
|
|
2139
|
+
write: false,
|
|
2140
|
+
logLevel: "silent",
|
|
2141
|
+
plugins: [resolverLockdownPlugin()]
|
|
2142
|
+
});
|
|
2143
|
+
const out = result.outputFiles?.[0];
|
|
2144
|
+
if (!out)
|
|
2145
|
+
throw new Error("esbuild produced no output for handler_code bundle");
|
|
2146
|
+
return new TextDecoder().decode(out.contents);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// src/runtime/sandbox/host.ts
|
|
2150
|
+
var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
|
|
2151
|
+
var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
|
|
2152
|
+
function spawnSandboxProcess() {
|
|
2153
|
+
let onMessage = null;
|
|
2154
|
+
const proc = Bun.spawn({
|
|
2155
|
+
cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
|
|
2156
|
+
env: { PATH: process.env.PATH ?? "" },
|
|
2157
|
+
cwd: tmpdir(),
|
|
2158
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
2159
|
+
serialization: "advanced",
|
|
2160
|
+
ipc(message) {
|
|
2161
|
+
onMessage?.(message);
|
|
2162
|
+
}
|
|
2163
|
+
});
|
|
2164
|
+
(async () => {
|
|
2165
|
+
const reader = proc.stderr.getReader();
|
|
2166
|
+
const decoder = new TextDecoder;
|
|
2167
|
+
try {
|
|
2168
|
+
while (true) {
|
|
2169
|
+
const { done, value } = await reader.read();
|
|
2170
|
+
if (done)
|
|
2171
|
+
break;
|
|
2172
|
+
const text = decoder.decode(value, { stream: true });
|
|
2173
|
+
if (text.trim()) {
|
|
2174
|
+
logger6.warn("Sandbox subprocess stderr", { stderr: text.trim() });
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
} catch {}
|
|
2178
|
+
})();
|
|
2179
|
+
return {
|
|
2180
|
+
send: (msg) => proc.send(msg),
|
|
2181
|
+
kill: () => proc.kill(),
|
|
2182
|
+
exited: proc.exited,
|
|
2183
|
+
setOnMessage: (handler) => {
|
|
2184
|
+
onMessage = handler;
|
|
2185
|
+
}
|
|
2186
|
+
};
|
|
2187
|
+
}
|
|
2188
|
+
var pool = new Map;
|
|
2189
|
+
|
|
2190
|
+
class SandboxWorkerCrash extends Error {
|
|
2191
|
+
constructor(subgraphName, cause) {
|
|
2192
|
+
super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
|
|
2193
|
+
this.name = "SandboxWorkerCrash";
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
|
|
2197
|
+
const bundledCode = await bundleHandlerCode(handlerCode);
|
|
2198
|
+
let settled = false;
|
|
2199
|
+
const ready = new Promise((resolve, reject) => {
|
|
2200
|
+
proc.setOnMessage((msg) => {
|
|
2201
|
+
if (msg.type === "ready" && msg.version === version) {
|
|
2202
|
+
settled = true;
|
|
2203
|
+
proc.setOnMessage(null);
|
|
2204
|
+
resolve();
|
|
2205
|
+
} else if (msg.type === "error") {
|
|
2206
|
+
settled = true;
|
|
2207
|
+
proc.setOnMessage(null);
|
|
2208
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2209
|
+
}
|
|
2210
|
+
});
|
|
2211
|
+
const init = { type: "init", bundledCode, version };
|
|
2212
|
+
proc.send(init);
|
|
2213
|
+
});
|
|
2214
|
+
const crashed = proc.exited.then((code) => {
|
|
2215
|
+
if (settled)
|
|
2216
|
+
return;
|
|
2217
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
|
|
2218
|
+
});
|
|
2219
|
+
await Promise.race([ready, crashed]);
|
|
2220
|
+
}
|
|
2221
|
+
async function ensureSandboxProcess(subgraphName, version, handlerCode) {
|
|
2222
|
+
if (handlerCode == null) {
|
|
2223
|
+
throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
|
|
2224
|
+
}
|
|
2225
|
+
const existing = pool.get(subgraphName);
|
|
2226
|
+
if (existing && existing.version === version)
|
|
2227
|
+
return existing;
|
|
2228
|
+
if (existing) {
|
|
2229
|
+
try {
|
|
2230
|
+
await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
|
|
2231
|
+
existing.version = version;
|
|
2232
|
+
logger6.info("Sandbox subprocess handler reloaded", {
|
|
2233
|
+
subgraph: subgraphName,
|
|
2234
|
+
version
|
|
2235
|
+
});
|
|
2236
|
+
return existing;
|
|
2237
|
+
} catch (err) {
|
|
2238
|
+
existing.proc.kill();
|
|
2239
|
+
pool.delete(subgraphName);
|
|
2240
|
+
logger6.warn("Sandbox subprocess re-init failed; respawning", {
|
|
2241
|
+
subgraph: subgraphName,
|
|
2242
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2243
|
+
});
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
const proc = spawnSandboxProcess();
|
|
2247
|
+
try {
|
|
2248
|
+
await initSandboxProcess(proc, subgraphName, version, handlerCode);
|
|
2249
|
+
} catch (err) {
|
|
2250
|
+
proc.kill();
|
|
2251
|
+
throw err;
|
|
2252
|
+
}
|
|
2253
|
+
const entry = { proc, version, chain: Promise.resolve() };
|
|
2254
|
+
pool.set(subgraphName, entry);
|
|
2255
|
+
return entry;
|
|
2256
|
+
}
|
|
2257
|
+
function evictSandboxWorker(subgraphName) {
|
|
2258
|
+
const entry = pool.get(subgraphName);
|
|
2259
|
+
if (!entry)
|
|
2260
|
+
return;
|
|
2261
|
+
entry.proc.kill();
|
|
2262
|
+
pool.delete(subgraphName);
|
|
2263
|
+
}
|
|
2264
|
+
async function runHandlersSandboxed(params) {
|
|
2265
|
+
const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
|
|
2266
|
+
const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
|
|
2267
|
+
const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
|
|
2268
|
+
entry.chain = run.catch(() => {
|
|
2269
|
+
return;
|
|
2270
|
+
});
|
|
2271
|
+
return run;
|
|
2272
|
+
}
|
|
2273
|
+
async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
|
|
2274
|
+
const { proc } = entry;
|
|
2275
|
+
let timeoutHandle;
|
|
2276
|
+
let settled = false;
|
|
2277
|
+
try {
|
|
2278
|
+
const done = new Promise((resolve, reject) => {
|
|
2279
|
+
proc.setOnMessage((msg) => {
|
|
2280
|
+
if (msg.type === "readRequest") {
|
|
2281
|
+
answerRead(hostCtx, msg).then((reply) => {
|
|
2282
|
+
proc.send({ type: "readResponse", id: msg.id, reply });
|
|
2283
|
+
}).catch((err) => {
|
|
2284
|
+
settled = true;
|
|
2285
|
+
reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
2286
|
+
});
|
|
2287
|
+
return;
|
|
2288
|
+
}
|
|
2289
|
+
if (msg.type === "blockDone") {
|
|
2290
|
+
settled = true;
|
|
2291
|
+
resolve({
|
|
2292
|
+
ops: msg.ops,
|
|
2293
|
+
result: { processed: msg.processed, errors: msg.errors }
|
|
2294
|
+
});
|
|
2295
|
+
return;
|
|
2296
|
+
}
|
|
2297
|
+
if (msg.type === "error") {
|
|
2298
|
+
settled = true;
|
|
2299
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2300
|
+
}
|
|
2301
|
+
});
|
|
2302
|
+
proc.send({ type: "runBlock", block, matched });
|
|
2303
|
+
});
|
|
2304
|
+
const crashed = proc.exited.then((code) => {
|
|
2305
|
+
if (settled)
|
|
2306
|
+
return;
|
|
2307
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
|
|
2308
|
+
});
|
|
2309
|
+
const deadline = new Promise((_, reject) => {
|
|
2310
|
+
timeoutHandle = setTimeout(() => {
|
|
2311
|
+
settled = true;
|
|
2312
|
+
reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
|
|
2313
|
+
}, BLOCK_TIMEOUT_MS);
|
|
2314
|
+
});
|
|
2315
|
+
const { ops, result } = await Promise.race([done, crashed, deadline]);
|
|
2316
|
+
for (const op of ops) {
|
|
2317
|
+
hostCtx.setTx(op.tx);
|
|
2318
|
+
replayOp(hostCtx, op);
|
|
2319
|
+
}
|
|
2320
|
+
return result;
|
|
2321
|
+
} catch (err) {
|
|
2322
|
+
evictSandboxWorker(subgraphName);
|
|
2323
|
+
logger6.warn("Sandbox subprocess run failed; subprocess evicted", {
|
|
2324
|
+
subgraph: subgraphName,
|
|
2325
|
+
blockHeight: block.height,
|
|
2326
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2327
|
+
});
|
|
2328
|
+
throw err;
|
|
2329
|
+
} finally {
|
|
2330
|
+
if (timeoutHandle !== undefined)
|
|
2331
|
+
clearTimeout(timeoutHandle);
|
|
2332
|
+
proc.setOnMessage(null);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
function replayOp(hostCtx, op) {
|
|
2336
|
+
switch (op.method) {
|
|
2337
|
+
case "insert":
|
|
2338
|
+
hostCtx.insert(op.table, op.args[0]);
|
|
2339
|
+
return;
|
|
2340
|
+
case "upsert":
|
|
2341
|
+
hostCtx.upsert(op.table, op.args[0], op.args[1]);
|
|
2342
|
+
return;
|
|
2343
|
+
case "increment":
|
|
2344
|
+
hostCtx.increment(op.table, op.args[0], op.args[1]);
|
|
2345
|
+
return;
|
|
2346
|
+
case "update":
|
|
2347
|
+
hostCtx.update(op.table, op.args[0], op.args[1]);
|
|
2348
|
+
return;
|
|
2349
|
+
case "delete":
|
|
2350
|
+
hostCtx.delete(op.table, op.args[0]);
|
|
2351
|
+
return;
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
async function answerRead(hostCtx, msg) {
|
|
2355
|
+
const { method, table, where, column } = msg;
|
|
2356
|
+
switch (method) {
|
|
2357
|
+
case "findOne":
|
|
2358
|
+
return { kind: "row", row: await hostCtx.findOne(table, where) };
|
|
2359
|
+
case "findMany":
|
|
2360
|
+
return { kind: "rows", rows: await hostCtx.findMany(table, where) };
|
|
2361
|
+
case "count":
|
|
2362
|
+
return { kind: "count", count: await hostCtx.count(table, where) };
|
|
2363
|
+
case "countDistinct": {
|
|
2364
|
+
if (!column)
|
|
2365
|
+
throw new Error("countDistinct read missing column");
|
|
2366
|
+
return {
|
|
2367
|
+
kind: "count",
|
|
2368
|
+
count: await hostCtx.countDistinct(table, column, where)
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
case "sum": {
|
|
2372
|
+
if (!column)
|
|
2373
|
+
throw new Error("sum read missing column");
|
|
2374
|
+
const v = await hostCtx.sum(table, column, where);
|
|
2375
|
+
return { kind: "amount", amount: v.toString() };
|
|
2376
|
+
}
|
|
2377
|
+
case "min": {
|
|
2378
|
+
if (!column)
|
|
2379
|
+
throw new Error("min read missing column");
|
|
2380
|
+
const v = await hostCtx.min(table, column, where);
|
|
2381
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2382
|
+
}
|
|
2383
|
+
case "max": {
|
|
2384
|
+
if (!column)
|
|
2385
|
+
throw new Error("max read missing column");
|
|
2386
|
+
const v = await hostCtx.max(table, column, where);
|
|
2387
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2078
2392
|
// src/runtime/subscription-state.ts
|
|
2079
2393
|
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2080
2394
|
import { sql as sql2 } from "kysely";
|
|
@@ -2259,13 +2573,17 @@ async function resolveRoute(subgraphName, targetDb) {
|
|
|
2259
2573
|
const route = {
|
|
2260
2574
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2261
2575
|
dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
|
|
2262
|
-
byo
|
|
2576
|
+
byo,
|
|
2577
|
+
sandboxWorkers: row?.sandbox_workers === true,
|
|
2578
|
+
handlerCode: row?.handler_code ?? null,
|
|
2579
|
+
version: row?.version ?? ""
|
|
2263
2580
|
};
|
|
2264
2581
|
routeCache.set(subgraphName, route);
|
|
2265
2582
|
return route;
|
|
2266
2583
|
}
|
|
2267
2584
|
function invalidateSubgraphRoute(subgraphName) {
|
|
2268
2585
|
routeCache.delete(subgraphName);
|
|
2586
|
+
evictSandboxWorker(subgraphName);
|
|
2269
2587
|
}
|
|
2270
2588
|
async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
|
|
2271
2589
|
const factories = new Map;
|
|
@@ -2371,7 +2689,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
|
|
|
2371
2689
|
const delay = retryDelaysMs[attempt];
|
|
2372
2690
|
if (delay === undefined)
|
|
2373
2691
|
break;
|
|
2374
|
-
|
|
2692
|
+
logger7.warn("Block processing failed, retrying", {
|
|
2375
2693
|
subgraph: subgraphName,
|
|
2376
2694
|
blockHeight,
|
|
2377
2695
|
attempt: attempt + 1,
|
|
@@ -2403,7 +2721,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2403
2721
|
} else {
|
|
2404
2722
|
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
2405
2723
|
if (!data) {
|
|
2406
|
-
|
|
2724
|
+
logger7.debug("Block not found or non-canonical for subgraph processing", {
|
|
2407
2725
|
subgraph: subgraphName,
|
|
2408
2726
|
blockHeight
|
|
2409
2727
|
});
|
|
@@ -2539,7 +2857,16 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2539
2857
|
}
|
|
2540
2858
|
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2541
2859
|
const handlerStart = performance.now();
|
|
2542
|
-
const runResult =
|
|
2860
|
+
const runResult = sandboxEnabled({
|
|
2861
|
+
sandbox_workers: route.sandboxWorkers
|
|
2862
|
+
}) ? await runHandlersSandboxed({
|
|
2863
|
+
subgraphName,
|
|
2864
|
+
version: route.version,
|
|
2865
|
+
handlerCode: route.handlerCode,
|
|
2866
|
+
hostCtx: ctx,
|
|
2867
|
+
block: blockMeta,
|
|
2868
|
+
matched
|
|
2869
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2543
2870
|
handlerMs = performance.now() - handlerStart;
|
|
2544
2871
|
result.processed = runResult.processed;
|
|
2545
2872
|
result.errors = runResult.errors;
|
|
@@ -2565,7 +2892,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2565
2892
|
});
|
|
2566
2893
|
} catch (err) {
|
|
2567
2894
|
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2568
|
-
|
|
2895
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2569
2896
|
subgraph: subgraphName,
|
|
2570
2897
|
blockHeight,
|
|
2571
2898
|
error: err.message
|
|
@@ -2591,7 +2918,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2591
2918
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
2592
2919
|
const count = Number(rows[0]?.count ?? 0);
|
|
2593
2920
|
if (count >= 1e7) {
|
|
2594
|
-
|
|
2921
|
+
logger7.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
2595
2922
|
subgraph: subgraphName,
|
|
2596
2923
|
table,
|
|
2597
2924
|
count
|
|
@@ -2599,7 +2926,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2599
2926
|
}
|
|
2600
2927
|
}
|
|
2601
2928
|
} catch (err) {
|
|
2602
|
-
|
|
2929
|
+
logger7.debug("Row count sample failed", {
|
|
2603
2930
|
subgraph: subgraphName,
|
|
2604
2931
|
error: err instanceof Error ? err.message : String(err)
|
|
2605
2932
|
});
|
|
@@ -2689,11 +3016,11 @@ import {
|
|
|
2689
3016
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2690
3017
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2691
3018
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2692
|
-
import { logger as
|
|
3019
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2693
3020
|
|
|
2694
3021
|
// src/runtime/reindex-notify.ts
|
|
2695
3022
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2696
|
-
import { logger as
|
|
3023
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2697
3024
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2698
3025
|
try {
|
|
2699
3026
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -2704,7 +3031,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2704
3031
|
return;
|
|
2705
3032
|
const resendKey = process.env.RESEND_API_KEY;
|
|
2706
3033
|
if (!resendKey) {
|
|
2707
|
-
|
|
3034
|
+
logger8.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2708
3035
|
subgraph: subgraphName
|
|
2709
3036
|
});
|
|
2710
3037
|
return;
|
|
@@ -2726,14 +3053,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2726
3053
|
});
|
|
2727
3054
|
if (!res.ok) {
|
|
2728
3055
|
const text = await res.text().catch(() => "");
|
|
2729
|
-
|
|
3056
|
+
logger8.warn("reindex-complete email failed", {
|
|
2730
3057
|
subgraph: subgraphName,
|
|
2731
3058
|
status: res.status,
|
|
2732
3059
|
body: text.slice(0, 200)
|
|
2733
3060
|
});
|
|
2734
3061
|
}
|
|
2735
3062
|
} catch (err) {
|
|
2736
|
-
|
|
3063
|
+
logger8.warn("reindex-complete email threw", {
|
|
2737
3064
|
subgraph: subgraphName,
|
|
2738
3065
|
error: getErrorMessage2(err)
|
|
2739
3066
|
});
|
|
@@ -2845,7 +3172,7 @@ async function processBlockRange(def, opts) {
|
|
|
2845
3172
|
while (currentHeight <= toBlock) {
|
|
2846
3173
|
if (opts.signal?.aborted) {
|
|
2847
3174
|
aborted = true;
|
|
2848
|
-
|
|
3175
|
+
logger9.info("Block processing aborted", {
|
|
2849
3176
|
subgraph: subgraphName,
|
|
2850
3177
|
currentBlock: currentHeight,
|
|
2851
3178
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -2894,7 +3221,7 @@ async function processBlockRange(def, opts) {
|
|
|
2894
3221
|
});
|
|
2895
3222
|
} catch (err) {
|
|
2896
3223
|
const errorMsg = getErrorMessage3(err);
|
|
2897
|
-
|
|
3224
|
+
logger9.error("Block processing failed persistently", {
|
|
2898
3225
|
subgraph: subgraphName,
|
|
2899
3226
|
blockHeight: height,
|
|
2900
3227
|
error: errorMsg
|
|
@@ -2940,7 +3267,7 @@ async function processBlockRange(def, opts) {
|
|
|
2940
3267
|
lastProgressFlushAt = now;
|
|
2941
3268
|
}
|
|
2942
3269
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
2943
|
-
|
|
3270
|
+
logger9.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
2944
3271
|
subgraph: subgraphName,
|
|
2945
3272
|
processed: blocksProcessed,
|
|
2946
3273
|
total: totalBlocks,
|
|
@@ -2960,7 +3287,7 @@ async function processBlockRange(def, opts) {
|
|
|
2960
3287
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
2961
3288
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
2962
3289
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
2963
|
-
|
|
3290
|
+
logger9.warn("Failed to record subgraph gaps", {
|
|
2964
3291
|
subgraph: subgraphName,
|
|
2965
3292
|
error: err instanceof Error ? err.message : String(err)
|
|
2966
3293
|
});
|
|
@@ -2977,7 +3304,7 @@ async function processBlockRange(def, opts) {
|
|
|
2977
3304
|
} else {
|
|
2978
3305
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
2979
3306
|
}
|
|
2980
|
-
|
|
3307
|
+
logger9.info("Sparse skip", {
|
|
2981
3308
|
subgraph: subgraphName,
|
|
2982
3309
|
from: batchEnd + 1,
|
|
2983
3310
|
to: jumpTo - 1,
|
|
@@ -3017,11 +3344,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
3017
3344
|
const subgraphName = def.name;
|
|
3018
3345
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3019
3346
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3020
|
-
|
|
3347
|
+
logger9.info("Reindex starting", { subgraph: subgraphName });
|
|
3021
3348
|
try {
|
|
3022
3349
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3023
3350
|
if (fromBlock > toBlock) {
|
|
3024
|
-
|
|
3351
|
+
logger9.info("No blocks to reindex", {
|
|
3025
3352
|
subgraph: subgraphName,
|
|
3026
3353
|
fromBlock,
|
|
3027
3354
|
toBlock
|
|
@@ -3035,7 +3362,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3035
3362
|
await client.unsafe(stmt);
|
|
3036
3363
|
}
|
|
3037
3364
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3038
|
-
|
|
3365
|
+
logger9.info("Schema recreated for reindex", {
|
|
3039
3366
|
subgraph: subgraphName,
|
|
3040
3367
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3041
3368
|
});
|
|
@@ -3049,7 +3376,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3049
3376
|
last_error_at: null,
|
|
3050
3377
|
updated_at: new Date
|
|
3051
3378
|
}).where("name", "=", subgraphName).execute();
|
|
3052
|
-
|
|
3379
|
+
logger9.info("Reindexing blocks", {
|
|
3053
3380
|
subgraph: subgraphName,
|
|
3054
3381
|
fromBlock,
|
|
3055
3382
|
toBlock,
|
|
@@ -3071,9 +3398,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3071
3398
|
if (reason === "user-cancelled") {
|
|
3072
3399
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3073
3400
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3074
|
-
|
|
3401
|
+
logger9.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3075
3402
|
} else {
|
|
3076
|
-
|
|
3403
|
+
logger9.info("Reindex interrupted by shutdown, will resume", {
|
|
3077
3404
|
subgraph: subgraphName
|
|
3078
3405
|
});
|
|
3079
3406
|
}
|
|
@@ -3081,7 +3408,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3081
3408
|
}
|
|
3082
3409
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3083
3410
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3084
|
-
|
|
3411
|
+
logger9.info("Reindex complete", {
|
|
3085
3412
|
subgraph: subgraphName,
|
|
3086
3413
|
blocks: result.blocksProcessed,
|
|
3087
3414
|
events: result.totalEventsProcessed,
|
|
@@ -3094,7 +3421,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3094
3421
|
});
|
|
3095
3422
|
return { processed: result.blocksProcessed };
|
|
3096
3423
|
} catch (err) {
|
|
3097
|
-
|
|
3424
|
+
logger9.error("Reindex failed", {
|
|
3098
3425
|
subgraph: subgraphName,
|
|
3099
3426
|
error: getErrorMessage3(err)
|
|
3100
3427
|
});
|
|
@@ -3116,7 +3443,7 @@ async function resumeReindex(def, opts) {
|
|
|
3116
3443
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3117
3444
|
const toBlock = Number(row.reindex_to_block);
|
|
3118
3445
|
if (fromBlock == null) {
|
|
3119
|
-
|
|
3446
|
+
logger9.info("No reindex metadata, starting fresh reindex", {
|
|
3120
3447
|
subgraph: subgraphName
|
|
3121
3448
|
});
|
|
3122
3449
|
return reindexSubgraph(def, {
|
|
@@ -3125,12 +3452,12 @@ async function resumeReindex(def, opts) {
|
|
|
3125
3452
|
});
|
|
3126
3453
|
}
|
|
3127
3454
|
if (fromBlock > toBlock) {
|
|
3128
|
-
|
|
3455
|
+
logger9.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3129
3456
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3130
3457
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3131
3458
|
return { processed: 0 };
|
|
3132
3459
|
}
|
|
3133
|
-
|
|
3460
|
+
logger9.info("Resuming reindex", {
|
|
3134
3461
|
subgraph: subgraphName,
|
|
3135
3462
|
fromBlock,
|
|
3136
3463
|
toBlock,
|
|
@@ -3152,9 +3479,9 @@ async function resumeReindex(def, opts) {
|
|
|
3152
3479
|
if (reason === "user-cancelled") {
|
|
3153
3480
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3154
3481
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3155
|
-
|
|
3482
|
+
logger9.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3156
3483
|
} else {
|
|
3157
|
-
|
|
3484
|
+
logger9.info("Resume interrupted by shutdown, will resume again", {
|
|
3158
3485
|
subgraph: subgraphName
|
|
3159
3486
|
});
|
|
3160
3487
|
}
|
|
@@ -3162,13 +3489,13 @@ async function resumeReindex(def, opts) {
|
|
|
3162
3489
|
}
|
|
3163
3490
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3164
3491
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3165
|
-
|
|
3492
|
+
logger9.info("Resumed reindex complete", {
|
|
3166
3493
|
subgraph: subgraphName,
|
|
3167
3494
|
blocks: result.blocksProcessed
|
|
3168
3495
|
});
|
|
3169
3496
|
return { processed: result.blocksProcessed };
|
|
3170
3497
|
} catch (err) {
|
|
3171
|
-
|
|
3498
|
+
logger9.error("Resumed reindex failed", {
|
|
3172
3499
|
subgraph: subgraphName,
|
|
3173
3500
|
error: getErrorMessage3(err)
|
|
3174
3501
|
});
|
|
@@ -3179,7 +3506,7 @@ async function resumeReindex(def, opts) {
|
|
|
3179
3506
|
async function backfillSubgraph(def, opts) {
|
|
3180
3507
|
const targetDb = getTargetDb2();
|
|
3181
3508
|
const subgraphName = def.name;
|
|
3182
|
-
|
|
3509
|
+
logger9.info("Backfill starting", {
|
|
3183
3510
|
subgraph: subgraphName,
|
|
3184
3511
|
from: opts.fromBlock,
|
|
3185
3512
|
to: opts.toBlock
|
|
@@ -3197,17 +3524,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3197
3524
|
signal: opts.signal
|
|
3198
3525
|
});
|
|
3199
3526
|
if (result.aborted) {
|
|
3200
|
-
|
|
3527
|
+
logger9.info("Backfill aborted", { subgraph: subgraphName });
|
|
3201
3528
|
return { processed: result.blocksProcessed };
|
|
3202
3529
|
}
|
|
3203
3530
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3204
3531
|
if (resolved > 0) {
|
|
3205
|
-
|
|
3532
|
+
logger9.info("Resolved subgraph gaps via backfill", {
|
|
3206
3533
|
subgraph: subgraphName,
|
|
3207
3534
|
resolved
|
|
3208
3535
|
});
|
|
3209
3536
|
}
|
|
3210
|
-
|
|
3537
|
+
logger9.info("Backfill complete", {
|
|
3211
3538
|
subgraph: subgraphName,
|
|
3212
3539
|
blocks: result.blocksProcessed,
|
|
3213
3540
|
events: result.totalEventsProcessed,
|
|
@@ -3215,7 +3542,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3215
3542
|
});
|
|
3216
3543
|
return { processed: result.blocksProcessed };
|
|
3217
3544
|
} catch (err) {
|
|
3218
|
-
|
|
3545
|
+
logger9.error("Backfill failed", {
|
|
3219
3546
|
subgraph: subgraphName,
|
|
3220
3547
|
error: getErrorMessage3(err)
|
|
3221
3548
|
});
|
|
@@ -3226,7 +3553,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3226
3553
|
// src/runtime/catchup.ts
|
|
3227
3554
|
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3228
3555
|
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
3229
|
-
import { logger as
|
|
3556
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
3230
3557
|
|
|
3231
3558
|
// src/runtime/catchup-leader.ts
|
|
3232
3559
|
import {
|
|
@@ -3360,7 +3687,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3360
3687
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
3361
3688
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
3362
3689
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
3363
|
-
|
|
3690
|
+
logger10.info("Subgraph catch-up starting", {
|
|
3364
3691
|
subgraph: subgraphName,
|
|
3365
3692
|
from: startBlock,
|
|
3366
3693
|
to: chainTip,
|
|
@@ -3376,7 +3703,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3376
3703
|
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
3377
3704
|
while (currentHeight <= chainTip) {
|
|
3378
3705
|
if (!isCatchUpLeader()) {
|
|
3379
|
-
|
|
3706
|
+
logger10.info("Lost catch-up leadership, stopping catch-up", {
|
|
3380
3707
|
subgraph: subgraphName,
|
|
3381
3708
|
processed
|
|
3382
3709
|
});
|
|
@@ -3384,7 +3711,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3384
3711
|
}
|
|
3385
3712
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
3386
3713
|
if (!currentRow || currentRow.status !== "active") {
|
|
3387
|
-
|
|
3714
|
+
logger10.info("Subgraph status changed, stopping catch-up", {
|
|
3388
3715
|
subgraph: subgraphName,
|
|
3389
3716
|
status: currentRow?.status ?? "deleted"
|
|
3390
3717
|
});
|
|
@@ -3413,7 +3740,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3413
3740
|
blockData = (await source.loadBlockRange(height, height)).get(height);
|
|
3414
3741
|
}
|
|
3415
3742
|
if (!blockData) {
|
|
3416
|
-
|
|
3743
|
+
logger10.warn("Block missing during catch-up, deferring to next tick", {
|
|
3417
3744
|
subgraph: subgraphName,
|
|
3418
3745
|
blockHeight: height
|
|
3419
3746
|
});
|
|
@@ -3433,12 +3760,12 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3433
3760
|
});
|
|
3434
3761
|
} catch (err) {
|
|
3435
3762
|
if (err instanceof ReorgEpochChangedError) {
|
|
3436
|
-
|
|
3763
|
+
logger10.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
|
|
3437
3764
|
stopCatchup = true;
|
|
3438
3765
|
break;
|
|
3439
3766
|
}
|
|
3440
3767
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
3441
|
-
|
|
3768
|
+
logger10.error("Block processing failed persistently during catch-up", {
|
|
3442
3769
|
subgraph: subgraphName,
|
|
3443
3770
|
blockHeight: height,
|
|
3444
3771
|
error: errorMsg
|
|
@@ -3457,7 +3784,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3457
3784
|
}
|
|
3458
3785
|
}
|
|
3459
3786
|
if (processed % LOG_INTERVAL2 === 0) {
|
|
3460
|
-
|
|
3787
|
+
logger10.info("Subgraph catch-up progress", {
|
|
3461
3788
|
subgraph: subgraphName,
|
|
3462
3789
|
processed,
|
|
3463
3790
|
total: totalBlocks,
|
|
@@ -3473,7 +3800,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
3473
3800
|
currentHeight = batchEnd + 1;
|
|
3474
3801
|
}
|
|
3475
3802
|
await stats.flush(targetDb);
|
|
3476
|
-
|
|
3803
|
+
logger10.info("Subgraph catch-up complete", {
|
|
3477
3804
|
subgraph: subgraphName,
|
|
3478
3805
|
processed
|
|
3479
3806
|
});
|
|
@@ -3490,14 +3817,14 @@ import {
|
|
|
3490
3817
|
listSubgraphs,
|
|
3491
3818
|
resolveSubgraphRawClient
|
|
3492
3819
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3493
|
-
import { logger as
|
|
3820
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
3494
3821
|
import { sql as sql4 } from "kysely";
|
|
3495
3822
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
3496
3823
|
const targetDb = getTargetDb4();
|
|
3497
3824
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
3498
3825
|
if (activeSubgraphs.length === 0)
|
|
3499
3826
|
return;
|
|
3500
|
-
|
|
3827
|
+
logger11.info("Propagating reorg to subgraphs", {
|
|
3501
3828
|
blockHeight,
|
|
3502
3829
|
subgraphCount: activeSubgraphs.length
|
|
3503
3830
|
});
|
|
@@ -3514,7 +3841,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
3514
3841
|
});
|
|
3515
3842
|
});
|
|
3516
3843
|
} catch (err) {
|
|
3517
|
-
|
|
3844
|
+
logger11.error("Subgraph reorg handling failed", {
|
|
3518
3845
|
subgraph: sg.name,
|
|
3519
3846
|
blockHeight,
|
|
3520
3847
|
error: getErrorMessage4(err)
|
|
@@ -3552,7 +3879,7 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3552
3879
|
await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
|
|
3553
3880
|
});
|
|
3554
3881
|
} else {
|
|
3555
|
-
|
|
3882
|
+
logger11.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
|
|
3556
3883
|
for (const tableName of tableNames) {
|
|
3557
3884
|
await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
|
|
3558
3885
|
}
|
|
@@ -3595,7 +3922,7 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3595
3922
|
}).as("row_pk"),
|
|
3596
3923
|
eb2.val("pending").as("status")
|
|
3597
3924
|
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
|
|
3598
|
-
|
|
3925
|
+
logger11.error("Failed to emit revert event for subscriptions", {
|
|
3599
3926
|
event: "reorg_revert_emit_dropped",
|
|
3600
3927
|
subgraph: sg.name,
|
|
3601
3928
|
table: tableName,
|
|
@@ -3604,14 +3931,14 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
3604
3931
|
});
|
|
3605
3932
|
});
|
|
3606
3933
|
}
|
|
3607
|
-
|
|
3934
|
+
logger11.info("Subgraph reorg cleanup done", {
|
|
3608
3935
|
subgraph: sg.name,
|
|
3609
3936
|
blockHeight,
|
|
3610
3937
|
tablesAffected: Object.keys(revertedByTable).length
|
|
3611
3938
|
});
|
|
3612
3939
|
const def = await loadSubgraphDef(sg);
|
|
3613
3940
|
await processBlock(def, sg.name, blockHeight, { reorgRewind: true });
|
|
3614
|
-
|
|
3941
|
+
logger11.info("Subgraph reorg reprocessed", {
|
|
3615
3942
|
subgraph: sg.name,
|
|
3616
3943
|
blockHeight
|
|
3617
3944
|
});
|
|
@@ -3641,7 +3968,7 @@ import {
|
|
|
3641
3968
|
pgSchemaName as pgSchemaName2,
|
|
3642
3969
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
3643
3970
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3644
|
-
import { logger as
|
|
3971
|
+
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3645
3972
|
import {
|
|
3646
3973
|
listen,
|
|
3647
3974
|
sourceListenerUrl,
|
|
@@ -3651,7 +3978,7 @@ import {
|
|
|
3651
3978
|
// src/runtime/streams-reorg-poll.ts
|
|
3652
3979
|
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
3653
3980
|
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3654
|
-
import { logger as
|
|
3981
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
3655
3982
|
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3656
3983
|
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3657
3984
|
async function pollReorgsOnce(http2, cursor, onReorg, handled) {
|
|
@@ -3659,13 +3986,13 @@ async function pollReorgsOnce(http2, cursor, onReorg, handled) {
|
|
|
3659
3986
|
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3660
3987
|
for (const r of sorted) {
|
|
3661
3988
|
if (handled && r.id && handled.has(r.id)) {
|
|
3662
|
-
|
|
3989
|
+
logger12.info("Streams reorg already handled — skipping", {
|
|
3663
3990
|
forkPointHeight: r.fork_point_height,
|
|
3664
3991
|
reorgId: r.id
|
|
3665
3992
|
});
|
|
3666
3993
|
continue;
|
|
3667
3994
|
}
|
|
3668
|
-
|
|
3995
|
+
logger12.info("Streams reorg — rewinding", {
|
|
3669
3996
|
forkPointHeight: r.fork_point_height
|
|
3670
3997
|
});
|
|
3671
3998
|
await onReorg(r.fork_point_height);
|
|
@@ -3693,7 +4020,7 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3693
4020
|
try {
|
|
3694
4021
|
since = await pollReorgsOnce(http2, since, onReorg, handled);
|
|
3695
4022
|
} catch (err) {
|
|
3696
|
-
|
|
4023
|
+
logger12.error("Streams reorg poll failed", {
|
|
3697
4024
|
error: getErrorMessage5(err)
|
|
3698
4025
|
});
|
|
3699
4026
|
}
|
|
@@ -3701,7 +4028,7 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
3701
4028
|
timer = setTimeout(tick, POLL_MS);
|
|
3702
4029
|
};
|
|
3703
4030
|
timer = setTimeout(tick, POLL_MS);
|
|
3704
|
-
|
|
4031
|
+
logger12.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3705
4032
|
return () => {
|
|
3706
4033
|
running = false;
|
|
3707
4034
|
if (timer)
|
|
@@ -3732,7 +4059,7 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3732
4059
|
if (isHandlerNotFoundError(err)) {
|
|
3733
4060
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3734
4061
|
}
|
|
3735
|
-
|
|
4062
|
+
logger13.error("Subgraph catch-up failed", {
|
|
3736
4063
|
subgraph: sg.name,
|
|
3737
4064
|
error: msg
|
|
3738
4065
|
});
|
|
@@ -3772,7 +4099,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3772
4099
|
definitionCache.set(sg.name, def);
|
|
3773
4100
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3774
4101
|
invalidateSubgraphRoute(sg.name);
|
|
3775
|
-
|
|
4102
|
+
logger13.info("Subgraph handler reloaded", {
|
|
3776
4103
|
subgraph: sg.name,
|
|
3777
4104
|
from: prevVersion,
|
|
3778
4105
|
to: sg.version
|
|
@@ -3806,7 +4133,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3806
4133
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3807
4134
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3808
4135
|
});
|
|
3809
|
-
|
|
4136
|
+
logger13.info("Queued legacy reindex resume operation", {
|
|
3810
4137
|
subgraph: sg.name
|
|
3811
4138
|
});
|
|
3812
4139
|
} catch (err) {
|
|
@@ -3869,7 +4196,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3869
4196
|
const activeRuns = new Map;
|
|
3870
4197
|
let running = true;
|
|
3871
4198
|
let draining = false;
|
|
3872
|
-
|
|
4199
|
+
logger13.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3873
4200
|
try {
|
|
3874
4201
|
const stranded = await db.selectFrom("subgraphs").select(["id", "name", "account_id"]).where("status", "=", "reindexing").where("reindex_from_block", "is not", null).where("reindex_to_block", "is not", null).where(({ not, exists, selectFrom }) => not(exists(selectFrom("subgraph_operations").select("id").whereRef("subgraph_id", "=", "subgraphs.id").where("status", "in", ["queued", "running"])))).execute();
|
|
3875
4202
|
for (const row of stranded) {
|
|
@@ -3880,14 +4207,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3880
4207
|
accountId: row.account_id,
|
|
3881
4208
|
kind: "reindex"
|
|
3882
4209
|
});
|
|
3883
|
-
|
|
4210
|
+
logger13.info("Re-enqueued stranded reindex", { subgraph: row.name });
|
|
3884
4211
|
} catch (err) {
|
|
3885
4212
|
if (!isActiveSubgraphOperationConflict(err))
|
|
3886
4213
|
throw err;
|
|
3887
4214
|
}
|
|
3888
4215
|
}
|
|
3889
4216
|
} catch (err) {
|
|
3890
|
-
|
|
4217
|
+
logger13.warn("Stranded-reindex sweep failed", {
|
|
3891
4218
|
error: getErrorMessage6(err)
|
|
3892
4219
|
});
|
|
3893
4220
|
}
|
|
@@ -3898,7 +4225,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3898
4225
|
if (!running)
|
|
3899
4226
|
return;
|
|
3900
4227
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3901
|
-
|
|
4228
|
+
logger13.warn("Subgraph operation heartbeat failed", {
|
|
3902
4229
|
operationId: operation.id,
|
|
3903
4230
|
error: getErrorMessage6(err)
|
|
3904
4231
|
});
|
|
@@ -3910,7 +4237,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3910
4237
|
controller.abort("user-cancelled");
|
|
3911
4238
|
}
|
|
3912
4239
|
}).catch((err) => {
|
|
3913
|
-
|
|
4240
|
+
logger13.warn("Subgraph operation cancel poll failed", {
|
|
3914
4241
|
operationId: operation.id,
|
|
3915
4242
|
error: getErrorMessage6(err)
|
|
3916
4243
|
});
|
|
@@ -3927,14 +4254,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3927
4254
|
const reason = String(controller.signal.reason ?? "");
|
|
3928
4255
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3929
4256
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3930
|
-
|
|
4257
|
+
logger13.info("Subgraph operation cancelled", {
|
|
3931
4258
|
operationId: operation.id,
|
|
3932
4259
|
subgraph: operation.subgraph_name
|
|
3933
4260
|
});
|
|
3934
4261
|
return;
|
|
3935
4262
|
}
|
|
3936
4263
|
if (controller.signal.aborted) {
|
|
3937
|
-
|
|
4264
|
+
logger13.info("Subgraph operation interrupted", {
|
|
3938
4265
|
operationId: operation.id,
|
|
3939
4266
|
subgraph: operation.subgraph_name,
|
|
3940
4267
|
reason
|
|
@@ -3942,7 +4269,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3942
4269
|
return;
|
|
3943
4270
|
}
|
|
3944
4271
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3945
|
-
|
|
4272
|
+
logger13.info("Subgraph operation completed", {
|
|
3946
4273
|
operationId: operation.id,
|
|
3947
4274
|
subgraph: operation.subgraph_name,
|
|
3948
4275
|
processed
|
|
@@ -3950,7 +4277,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3950
4277
|
} catch (err) {
|
|
3951
4278
|
const reason = String(controller.signal.reason ?? "");
|
|
3952
4279
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3953
|
-
|
|
4280
|
+
logger13.info("Subgraph operation interrupted by shutdown", {
|
|
3954
4281
|
operationId: operation.id,
|
|
3955
4282
|
subgraph: operation.subgraph_name
|
|
3956
4283
|
});
|
|
@@ -3961,7 +4288,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3961
4288
|
return;
|
|
3962
4289
|
}
|
|
3963
4290
|
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
|
|
3964
|
-
|
|
4291
|
+
logger13.error("Subgraph operation failed", {
|
|
3965
4292
|
operationId: operation.id,
|
|
3966
4293
|
subgraph: operation.subgraph_name,
|
|
3967
4294
|
error: getErrorMessage6(err)
|
|
@@ -4008,13 +4335,13 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
4008
4335
|
controller.abort("shutdown");
|
|
4009
4336
|
}
|
|
4010
4337
|
await Promise.allSettled(activeRuns.values());
|
|
4011
|
-
|
|
4338
|
+
logger13.info("Subgraph operation runner stopped");
|
|
4012
4339
|
};
|
|
4013
4340
|
}
|
|
4014
4341
|
async function startSubgraphProcessor(opts) {
|
|
4015
4342
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
4016
4343
|
let running = true;
|
|
4017
|
-
|
|
4344
|
+
logger13.info("Starting subgraph processor", { concurrency });
|
|
4018
4345
|
const stopOperations = await startSubgraphOperationRunner({
|
|
4019
4346
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
4020
4347
|
});
|
|
@@ -4040,7 +4367,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4040
4367
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
4041
4368
|
}
|
|
4042
4369
|
} catch (err) {
|
|
4043
|
-
|
|
4370
|
+
logger13.error("Subgraph reorg handling failed", {
|
|
4044
4371
|
error: getErrorMessage6(err)
|
|
4045
4372
|
});
|
|
4046
4373
|
}
|
|
@@ -4049,7 +4376,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4049
4376
|
runCatchUp();
|
|
4050
4377
|
}, POLL_INTERVAL_MS);
|
|
4051
4378
|
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
4052
|
-
|
|
4379
|
+
logger13.info("Subgraph processor ready");
|
|
4053
4380
|
return async () => {
|
|
4054
4381
|
running = false;
|
|
4055
4382
|
clearInterval(pollInterval);
|
|
@@ -4058,7 +4385,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
4058
4385
|
await stopReorgListening();
|
|
4059
4386
|
stopStreamsReorgPoll?.();
|
|
4060
4387
|
await stopOperations();
|
|
4061
|
-
|
|
4388
|
+
logger13.info("Subgraph processor stopped");
|
|
4062
4389
|
};
|
|
4063
4390
|
}
|
|
4064
4391
|
export {
|
|
@@ -4066,5 +4393,5 @@ export {
|
|
|
4066
4393
|
startSubgraphOperationRunner
|
|
4067
4394
|
};
|
|
4068
4395
|
|
|
4069
|
-
//# debugId=
|
|
4396
|
+
//# debugId=030615F15870ECBC64756E2164756E21
|
|
4070
4397
|
//# sourceMappingURL=processor.js.map
|