@secondlayer/subgraphs 3.24.0 → 3.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +379 -52
- package/dist/src/index.js.map +9 -6
- package/dist/src/runtime/block-processor.d.ts +8 -1
- package/dist/src/runtime/block-processor.js +349 -22
- package/dist/src/runtime/block-processor.js.map +9 -6
- package/dist/src/runtime/catchup.js +358 -31
- package/dist/src/runtime/catchup.js.map +9 -6
- package/dist/src/runtime/context.js +19 -14
- package/dist/src/runtime/context.js.map +4 -4
- package/dist/src/runtime/processor.js +419 -92
- package/dist/src/runtime/processor.js.map +9 -6
- package/dist/src/runtime/reindex.js +379 -52
- package/dist/src/runtime/reindex.js.map +9 -6
- package/dist/src/runtime/reorg.js +365 -38
- package/dist/src/runtime/reorg.js.map +9 -6
- package/dist/src/runtime/replay.js +6 -1
- package/dist/src/runtime/replay.js.map +3 -3
- package/dist/src/schema/index.js +19 -14
- package/dist/src/schema/index.js.map +4 -4
- package/dist/src/service.js +422 -95
- package/dist/src/service.js.map +9 -6
- package/dist/src/testing/index.js +19 -14
- package/dist/src/testing/index.js.map +4 -4
- package/package.json +1 -1
package/dist/src/index.js
CHANGED
|
@@ -226,6 +226,11 @@ import { createHash } from "node:crypto";
|
|
|
226
226
|
|
|
227
227
|
// src/schema/utils.ts
|
|
228
228
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
229
|
+
function quotePgIdent(name) {
|
|
230
|
+
if (/^[a-z_][a-z0-9_]*$/i.test(name))
|
|
231
|
+
return name;
|
|
232
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
233
|
+
}
|
|
229
234
|
|
|
230
235
|
// src/schema/generator.ts
|
|
231
236
|
var TYPE_MAP = {
|
|
@@ -250,7 +255,7 @@ function tableNeedsTrgm(tableDef) {
|
|
|
250
255
|
return Object.values(tableDef.columns).some((col) => col.search);
|
|
251
256
|
}
|
|
252
257
|
function emitTableDDL(schemaName, tableName, tableDef) {
|
|
253
|
-
const qualifiedName = `${schemaName}.${tableName}`;
|
|
258
|
+
const qualifiedName = `${quotePgIdent(schemaName)}.${quotePgIdent(tableName)}`;
|
|
254
259
|
const statements = [];
|
|
255
260
|
const columnDefs = [
|
|
256
261
|
"_id BIGSERIAL PRIMARY KEY",
|
|
@@ -261,12 +266,12 @@ function emitTableDDL(schemaName, tableName, tableDef) {
|
|
|
261
266
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
262
267
|
const sqlType = TYPE_MAP[col.type];
|
|
263
268
|
const nullable = col.nullable ? "" : " NOT NULL";
|
|
264
|
-
let colDef = `${colName} ${sqlType}${nullable}`;
|
|
269
|
+
let colDef = `${quotePgIdent(colName)} ${sqlType}${nullable}`;
|
|
265
270
|
if (col.default !== undefined) {
|
|
266
271
|
colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
|
|
267
272
|
}
|
|
268
273
|
if (col.type === "uint") {
|
|
269
|
-
colDef += ` CHECK (${colName} >= 0)`;
|
|
274
|
+
colDef += ` CHECK (${quotePgIdent(colName)} >= 0)`;
|
|
270
275
|
}
|
|
271
276
|
columnDefs.push(colDef);
|
|
272
277
|
}
|
|
@@ -274,30 +279,30 @@ function emitTableDDL(schemaName, tableName, tableDef) {
|
|
|
274
279
|
${columnDefs.join(`,
|
|
275
280
|
`)}
|
|
276
281
|
)`);
|
|
277
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
|
|
278
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
|
|
282
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_block_height`)} ON ${qualifiedName} (_block_height)`);
|
|
283
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_tx_id`)} ON ${qualifiedName} (_tx_id)`);
|
|
279
284
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
280
285
|
if (col.indexed) {
|
|
281
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
|
|
286
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_${colName}`)} ON ${qualifiedName} (${quotePgIdent(colName)})`);
|
|
282
287
|
}
|
|
283
288
|
}
|
|
284
289
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
285
290
|
if (col.search) {
|
|
286
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
|
|
291
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${quotePgIdent(`idx_${schemaName}_${tableName}_${colName}_trgm`)} ON ${qualifiedName} USING gin (${quotePgIdent(colName)} gin_trgm_ops)`);
|
|
287
292
|
}
|
|
288
293
|
}
|
|
289
294
|
if (tableDef.indexes) {
|
|
290
295
|
for (let i = 0;i < tableDef.indexes.length; i++) {
|
|
291
296
|
const cols = tableDef.indexes[i];
|
|
292
|
-
const idxName = `idx_${schemaName}_${tableName}_composite_${i}
|
|
293
|
-
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
|
|
297
|
+
const idxName = quotePgIdent(`idx_${schemaName}_${tableName}_composite_${i}`);
|
|
298
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.map(quotePgIdent).join(", ")})`);
|
|
294
299
|
}
|
|
295
300
|
}
|
|
296
301
|
if (tableDef.uniqueKeys) {
|
|
297
302
|
for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
|
|
298
303
|
const cols = tableDef.uniqueKeys[i];
|
|
299
|
-
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}
|
|
300
|
-
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
|
|
304
|
+
const constraintName = quotePgIdent(`uq_${schemaName}_${tableName}_${cols.join("_")}`);
|
|
305
|
+
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.map(quotePgIdent).join(", ")})`);
|
|
301
306
|
}
|
|
302
307
|
}
|
|
303
308
|
return statements;
|
|
@@ -329,8 +334,8 @@ function emitFactoryDDL(schemaName) {
|
|
|
329
334
|
}
|
|
330
335
|
function emitForeignKeyDDL(schemaName, tableName, tableDef) {
|
|
331
336
|
return (tableDef.relations ?? []).map((rel) => {
|
|
332
|
-
const constraintName = `fk_${schemaName}_${tableName}_${rel.name}
|
|
333
|
-
return `ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`;
|
|
337
|
+
const constraintName = quotePgIdent(`fk_${schemaName}_${tableName}_${rel.name}`);
|
|
338
|
+
return `ALTER TABLE ${quotePgIdent(schemaName)}.${quotePgIdent(tableName)} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.map(quotePgIdent).join(", ")}) ` + `REFERENCES ${quotePgIdent(schemaName)}.${quotePgIdent(rel.references)} (${rel.referencedColumns.map(quotePgIdent).join(", ")})`;
|
|
334
339
|
});
|
|
335
340
|
}
|
|
336
341
|
function generateSubgraphSQL(def, schemaNameOverride) {
|
|
@@ -1849,7 +1854,7 @@ import {
|
|
|
1849
1854
|
rewindLiveProgress,
|
|
1850
1855
|
updateSubgraphStatus
|
|
1851
1856
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1852
|
-
import { logger as
|
|
1857
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1853
1858
|
import { sql as sql3 } from "kysely";
|
|
1854
1859
|
|
|
1855
1860
|
// src/runtime/block-source.ts
|
|
@@ -2290,6 +2295,315 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2290
2295
|
return rows.length;
|
|
2291
2296
|
}
|
|
2292
2297
|
|
|
2298
|
+
// src/runtime/sandbox/flag.ts
|
|
2299
|
+
function sandboxWorkersGloballyEnabled() {
|
|
2300
|
+
return process.env.SUBGRAPH_SANDBOX_WORKERS === "1";
|
|
2301
|
+
}
|
|
2302
|
+
function sandboxEnabled(subgraph) {
|
|
2303
|
+
return sandboxWorkersGloballyEnabled() && subgraph.sandbox_workers === true;
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
// src/runtime/sandbox/host.ts
|
|
2307
|
+
import { tmpdir } from "node:os";
|
|
2308
|
+
import { fileURLToPath } from "node:url";
|
|
2309
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
2310
|
+
|
|
2311
|
+
// src/runtime/sandbox/bundle.ts
|
|
2312
|
+
import esbuild from "esbuild";
|
|
2313
|
+
var ALLOWED_BARE_SPECIFIERS = new Set;
|
|
2314
|
+
var SUBGRAPHS_PACKAGE_SPECIFIER = "@secondlayer/subgraphs";
|
|
2315
|
+
var SUBGRAPHS_STUB_SOURCE = `export function defineSubgraph(def) { return def; }
|
|
2316
|
+
`;
|
|
2317
|
+
function resolverLockdownPlugin() {
|
|
2318
|
+
return {
|
|
2319
|
+
name: "f071-sandbox-resolver-lockdown",
|
|
2320
|
+
setup(build) {
|
|
2321
|
+
build.onResolve({ filter: /^@secondlayer\/subgraphs$/ }, (args) => ({
|
|
2322
|
+
path: args.path,
|
|
2323
|
+
namespace: "f071-sandbox-subgraphs-stub"
|
|
2324
|
+
}));
|
|
2325
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-subgraphs-stub" }, () => ({ contents: SUBGRAPHS_STUB_SOURCE, loader: "js" }));
|
|
2326
|
+
build.onResolve({ filter: /^(node:|bun:)/ }, (args) => ({
|
|
2327
|
+
path: args.path,
|
|
2328
|
+
namespace: "f071-sandbox-blocked"
|
|
2329
|
+
}));
|
|
2330
|
+
build.onResolve({ filter: /^[^./]/ }, (args) => {
|
|
2331
|
+
if (args.path === SUBGRAPHS_PACKAGE_SPECIFIER)
|
|
2332
|
+
return;
|
|
2333
|
+
if (ALLOWED_BARE_SPECIFIERS.has(args.path))
|
|
2334
|
+
return;
|
|
2335
|
+
return { path: args.path, namespace: "f071-sandbox-blocked" };
|
|
2336
|
+
});
|
|
2337
|
+
build.onLoad({ filter: /.*/, namespace: "f071-sandbox-blocked" }, (args) => ({
|
|
2338
|
+
contents: `throw new Error(${JSON.stringify(`subgraph sandbox: import of "${args.path}" is blocked in sandboxed handler code — handlers must use the injected \`ctx\` for all I/O`)});`,
|
|
2339
|
+
loader: "js"
|
|
2340
|
+
}));
|
|
2341
|
+
}
|
|
2342
|
+
};
|
|
2343
|
+
}
|
|
2344
|
+
async function bundleHandlerCode(handlerCode) {
|
|
2345
|
+
const result = await esbuild.build({
|
|
2346
|
+
stdin: {
|
|
2347
|
+
contents: handlerCode,
|
|
2348
|
+
loader: "ts",
|
|
2349
|
+
resolveDir: process.cwd()
|
|
2350
|
+
},
|
|
2351
|
+
bundle: true,
|
|
2352
|
+
platform: "neutral",
|
|
2353
|
+
format: "esm",
|
|
2354
|
+
write: false,
|
|
2355
|
+
logLevel: "silent",
|
|
2356
|
+
plugins: [resolverLockdownPlugin()]
|
|
2357
|
+
});
|
|
2358
|
+
const out = result.outputFiles?.[0];
|
|
2359
|
+
if (!out)
|
|
2360
|
+
throw new Error("esbuild produced no output for handler_code bundle");
|
|
2361
|
+
return new TextDecoder().decode(out.contents);
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
// src/runtime/sandbox/host.ts
|
|
2365
|
+
var BLOCK_TIMEOUT_MS = Number.parseInt(process.env.SUBGRAPH_SANDBOX_BLOCK_TIMEOUT_MS ?? "60000");
|
|
2366
|
+
var SUBPROCESS_ENTRY_PATH = fileURLToPath(new URL("./subprocess-entry.ts", import.meta.url));
|
|
2367
|
+
function spawnSandboxProcess() {
|
|
2368
|
+
let onMessage = null;
|
|
2369
|
+
const proc = Bun.spawn({
|
|
2370
|
+
cmd: [process.execPath, "--no-env-file", SUBPROCESS_ENTRY_PATH],
|
|
2371
|
+
env: { PATH: process.env.PATH ?? "" },
|
|
2372
|
+
cwd: tmpdir(),
|
|
2373
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
2374
|
+
serialization: "advanced",
|
|
2375
|
+
ipc(message) {
|
|
2376
|
+
onMessage?.(message);
|
|
2377
|
+
}
|
|
2378
|
+
});
|
|
2379
|
+
(async () => {
|
|
2380
|
+
const reader = proc.stderr.getReader();
|
|
2381
|
+
const decoder = new TextDecoder;
|
|
2382
|
+
try {
|
|
2383
|
+
while (true) {
|
|
2384
|
+
const { done, value } = await reader.read();
|
|
2385
|
+
if (done)
|
|
2386
|
+
break;
|
|
2387
|
+
const text = decoder.decode(value, { stream: true });
|
|
2388
|
+
if (text.trim()) {
|
|
2389
|
+
logger6.warn("Sandbox subprocess stderr", { stderr: text.trim() });
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
} catch {}
|
|
2393
|
+
})();
|
|
2394
|
+
return {
|
|
2395
|
+
send: (msg) => proc.send(msg),
|
|
2396
|
+
kill: () => proc.kill(),
|
|
2397
|
+
exited: proc.exited,
|
|
2398
|
+
setOnMessage: (handler) => {
|
|
2399
|
+
onMessage = handler;
|
|
2400
|
+
}
|
|
2401
|
+
};
|
|
2402
|
+
}
|
|
2403
|
+
var pool = new Map;
|
|
2404
|
+
|
|
2405
|
+
class SandboxWorkerCrash extends Error {
|
|
2406
|
+
constructor(subgraphName, cause) {
|
|
2407
|
+
super(`sandbox subprocess for "${subgraphName}" failed: ${cause}`);
|
|
2408
|
+
this.name = "SandboxWorkerCrash";
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
async function initSandboxProcess(proc, subgraphName, version, handlerCode) {
|
|
2412
|
+
const bundledCode = await bundleHandlerCode(handlerCode);
|
|
2413
|
+
let settled = false;
|
|
2414
|
+
const ready = new Promise((resolve, reject) => {
|
|
2415
|
+
proc.setOnMessage((msg) => {
|
|
2416
|
+
if (msg.type === "ready" && msg.version === version) {
|
|
2417
|
+
settled = true;
|
|
2418
|
+
proc.setOnMessage(null);
|
|
2419
|
+
resolve();
|
|
2420
|
+
} else if (msg.type === "error") {
|
|
2421
|
+
settled = true;
|
|
2422
|
+
proc.setOnMessage(null);
|
|
2423
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2424
|
+
}
|
|
2425
|
+
});
|
|
2426
|
+
const init = { type: "init", bundledCode, version };
|
|
2427
|
+
proc.send(init);
|
|
2428
|
+
});
|
|
2429
|
+
const crashed = proc.exited.then((code) => {
|
|
2430
|
+
if (settled)
|
|
2431
|
+
return;
|
|
2432
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited (code ${code}) during init`);
|
|
2433
|
+
});
|
|
2434
|
+
await Promise.race([ready, crashed]);
|
|
2435
|
+
}
|
|
2436
|
+
async function ensureSandboxProcess(subgraphName, version, handlerCode) {
|
|
2437
|
+
if (handlerCode == null) {
|
|
2438
|
+
throw new SandboxWorkerCrash(subgraphName, "subgraph has no handler_code; cannot bundle for the sandbox path");
|
|
2439
|
+
}
|
|
2440
|
+
const existing = pool.get(subgraphName);
|
|
2441
|
+
if (existing && existing.version === version)
|
|
2442
|
+
return existing;
|
|
2443
|
+
if (existing) {
|
|
2444
|
+
try {
|
|
2445
|
+
await initSandboxProcess(existing.proc, subgraphName, version, handlerCode);
|
|
2446
|
+
existing.version = version;
|
|
2447
|
+
logger6.info("Sandbox subprocess handler reloaded", {
|
|
2448
|
+
subgraph: subgraphName,
|
|
2449
|
+
version
|
|
2450
|
+
});
|
|
2451
|
+
return existing;
|
|
2452
|
+
} catch (err) {
|
|
2453
|
+
existing.proc.kill();
|
|
2454
|
+
pool.delete(subgraphName);
|
|
2455
|
+
logger6.warn("Sandbox subprocess re-init failed; respawning", {
|
|
2456
|
+
subgraph: subgraphName,
|
|
2457
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2458
|
+
});
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
const proc = spawnSandboxProcess();
|
|
2462
|
+
try {
|
|
2463
|
+
await initSandboxProcess(proc, subgraphName, version, handlerCode);
|
|
2464
|
+
} catch (err) {
|
|
2465
|
+
proc.kill();
|
|
2466
|
+
throw err;
|
|
2467
|
+
}
|
|
2468
|
+
const entry = { proc, version, chain: Promise.resolve() };
|
|
2469
|
+
pool.set(subgraphName, entry);
|
|
2470
|
+
return entry;
|
|
2471
|
+
}
|
|
2472
|
+
function evictSandboxWorker(subgraphName) {
|
|
2473
|
+
const entry = pool.get(subgraphName);
|
|
2474
|
+
if (!entry)
|
|
2475
|
+
return;
|
|
2476
|
+
entry.proc.kill();
|
|
2477
|
+
pool.delete(subgraphName);
|
|
2478
|
+
}
|
|
2479
|
+
async function runHandlersSandboxed(params) {
|
|
2480
|
+
const { subgraphName, version, handlerCode, hostCtx, block, matched } = params;
|
|
2481
|
+
const entry = await ensureSandboxProcess(subgraphName, version, handlerCode);
|
|
2482
|
+
const run = entry.chain.then(() => runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched));
|
|
2483
|
+
entry.chain = run.catch(() => {
|
|
2484
|
+
return;
|
|
2485
|
+
});
|
|
2486
|
+
return run;
|
|
2487
|
+
}
|
|
2488
|
+
async function runBlockThroughProcess(entry, subgraphName, hostCtx, block, matched) {
|
|
2489
|
+
const { proc } = entry;
|
|
2490
|
+
let timeoutHandle;
|
|
2491
|
+
let settled = false;
|
|
2492
|
+
try {
|
|
2493
|
+
const done = new Promise((resolve, reject) => {
|
|
2494
|
+
proc.setOnMessage((msg) => {
|
|
2495
|
+
if (msg.type === "readRequest") {
|
|
2496
|
+
answerRead(hostCtx, msg).then((reply) => {
|
|
2497
|
+
proc.send({ type: "readResponse", id: msg.id, reply });
|
|
2498
|
+
}).catch((err) => {
|
|
2499
|
+
settled = true;
|
|
2500
|
+
reject(new SandboxWorkerCrash(subgraphName, `host read failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
2501
|
+
});
|
|
2502
|
+
return;
|
|
2503
|
+
}
|
|
2504
|
+
if (msg.type === "blockDone") {
|
|
2505
|
+
settled = true;
|
|
2506
|
+
resolve({
|
|
2507
|
+
ops: msg.ops,
|
|
2508
|
+
result: { processed: msg.processed, errors: msg.errors }
|
|
2509
|
+
});
|
|
2510
|
+
return;
|
|
2511
|
+
}
|
|
2512
|
+
if (msg.type === "error") {
|
|
2513
|
+
settled = true;
|
|
2514
|
+
reject(new SandboxWorkerCrash(subgraphName, msg.message));
|
|
2515
|
+
}
|
|
2516
|
+
});
|
|
2517
|
+
proc.send({ type: "runBlock", block, matched });
|
|
2518
|
+
});
|
|
2519
|
+
const crashed = proc.exited.then((code) => {
|
|
2520
|
+
if (settled)
|
|
2521
|
+
return;
|
|
2522
|
+
throw new SandboxWorkerCrash(subgraphName, `subprocess exited unexpectedly (code ${code}) mid-block`);
|
|
2523
|
+
});
|
|
2524
|
+
const deadline = new Promise((_, reject) => {
|
|
2525
|
+
timeoutHandle = setTimeout(() => {
|
|
2526
|
+
settled = true;
|
|
2527
|
+
reject(new SandboxWorkerCrash(subgraphName, `block ${block.height} exceeded ${BLOCK_TIMEOUT_MS}ms deadline`));
|
|
2528
|
+
}, BLOCK_TIMEOUT_MS);
|
|
2529
|
+
});
|
|
2530
|
+
const { ops, result } = await Promise.race([done, crashed, deadline]);
|
|
2531
|
+
for (const op of ops) {
|
|
2532
|
+
hostCtx.setTx(op.tx);
|
|
2533
|
+
replayOp(hostCtx, op);
|
|
2534
|
+
}
|
|
2535
|
+
return result;
|
|
2536
|
+
} catch (err) {
|
|
2537
|
+
evictSandboxWorker(subgraphName);
|
|
2538
|
+
logger6.warn("Sandbox subprocess run failed; subprocess evicted", {
|
|
2539
|
+
subgraph: subgraphName,
|
|
2540
|
+
blockHeight: block.height,
|
|
2541
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2542
|
+
});
|
|
2543
|
+
throw err;
|
|
2544
|
+
} finally {
|
|
2545
|
+
if (timeoutHandle !== undefined)
|
|
2546
|
+
clearTimeout(timeoutHandle);
|
|
2547
|
+
proc.setOnMessage(null);
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2550
|
+
function replayOp(hostCtx, op) {
|
|
2551
|
+
switch (op.method) {
|
|
2552
|
+
case "insert":
|
|
2553
|
+
hostCtx.insert(op.table, op.args[0]);
|
|
2554
|
+
return;
|
|
2555
|
+
case "upsert":
|
|
2556
|
+
hostCtx.upsert(op.table, op.args[0], op.args[1]);
|
|
2557
|
+
return;
|
|
2558
|
+
case "increment":
|
|
2559
|
+
hostCtx.increment(op.table, op.args[0], op.args[1]);
|
|
2560
|
+
return;
|
|
2561
|
+
case "update":
|
|
2562
|
+
hostCtx.update(op.table, op.args[0], op.args[1]);
|
|
2563
|
+
return;
|
|
2564
|
+
case "delete":
|
|
2565
|
+
hostCtx.delete(op.table, op.args[0]);
|
|
2566
|
+
return;
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
async function answerRead(hostCtx, msg) {
|
|
2570
|
+
const { method, table, where, column } = msg;
|
|
2571
|
+
switch (method) {
|
|
2572
|
+
case "findOne":
|
|
2573
|
+
return { kind: "row", row: await hostCtx.findOne(table, where) };
|
|
2574
|
+
case "findMany":
|
|
2575
|
+
return { kind: "rows", rows: await hostCtx.findMany(table, where) };
|
|
2576
|
+
case "count":
|
|
2577
|
+
return { kind: "count", count: await hostCtx.count(table, where) };
|
|
2578
|
+
case "countDistinct": {
|
|
2579
|
+
if (!column)
|
|
2580
|
+
throw new Error("countDistinct read missing column");
|
|
2581
|
+
return {
|
|
2582
|
+
kind: "count",
|
|
2583
|
+
count: await hostCtx.countDistinct(table, column, where)
|
|
2584
|
+
};
|
|
2585
|
+
}
|
|
2586
|
+
case "sum": {
|
|
2587
|
+
if (!column)
|
|
2588
|
+
throw new Error("sum read missing column");
|
|
2589
|
+
const v = await hostCtx.sum(table, column, where);
|
|
2590
|
+
return { kind: "amount", amount: v.toString() };
|
|
2591
|
+
}
|
|
2592
|
+
case "min": {
|
|
2593
|
+
if (!column)
|
|
2594
|
+
throw new Error("min read missing column");
|
|
2595
|
+
const v = await hostCtx.min(table, column, where);
|
|
2596
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2597
|
+
}
|
|
2598
|
+
case "max": {
|
|
2599
|
+
if (!column)
|
|
2600
|
+
throw new Error("max read missing column");
|
|
2601
|
+
const v = await hostCtx.max(table, column, where);
|
|
2602
|
+
return { kind: "amount", amount: v == null ? null : v.toString() };
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2293
2607
|
// src/runtime/subscription-state.ts
|
|
2294
2608
|
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2295
2609
|
import { sql as sql2 } from "kysely";
|
|
@@ -2474,13 +2788,17 @@ async function resolveRoute(subgraphName, targetDb) {
|
|
|
2474
2788
|
const route = {
|
|
2475
2789
|
schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
|
|
2476
2790
|
dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
|
|
2477
|
-
byo
|
|
2791
|
+
byo,
|
|
2792
|
+
sandboxWorkers: row?.sandbox_workers === true,
|
|
2793
|
+
handlerCode: row?.handler_code ?? null,
|
|
2794
|
+
version: row?.version ?? ""
|
|
2478
2795
|
};
|
|
2479
2796
|
routeCache.set(subgraphName, route);
|
|
2480
2797
|
return route;
|
|
2481
2798
|
}
|
|
2482
2799
|
function invalidateSubgraphRoute(subgraphName) {
|
|
2483
2800
|
routeCache.delete(subgraphName);
|
|
2801
|
+
evictSandboxWorker(subgraphName);
|
|
2484
2802
|
}
|
|
2485
2803
|
async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
|
|
2486
2804
|
const factories = new Map;
|
|
@@ -2586,7 +2904,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
|
|
|
2586
2904
|
const delay = retryDelaysMs[attempt];
|
|
2587
2905
|
if (delay === undefined)
|
|
2588
2906
|
break;
|
|
2589
|
-
|
|
2907
|
+
logger7.warn("Block processing failed, retrying", {
|
|
2590
2908
|
subgraph: subgraphName,
|
|
2591
2909
|
blockHeight,
|
|
2592
2910
|
attempt: attempt + 1,
|
|
@@ -2618,7 +2936,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2618
2936
|
} else {
|
|
2619
2937
|
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
2620
2938
|
if (!data) {
|
|
2621
|
-
|
|
2939
|
+
logger7.debug("Block not found or non-canonical for subgraph processing", {
|
|
2622
2940
|
subgraph: subgraphName,
|
|
2623
2941
|
blockHeight
|
|
2624
2942
|
});
|
|
@@ -2754,7 +3072,16 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2754
3072
|
}
|
|
2755
3073
|
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
|
|
2756
3074
|
const handlerStart = performance.now();
|
|
2757
|
-
const runResult =
|
|
3075
|
+
const runResult = sandboxEnabled({
|
|
3076
|
+
sandbox_workers: route.sandboxWorkers
|
|
3077
|
+
}) ? await runHandlersSandboxed({
|
|
3078
|
+
subgraphName,
|
|
3079
|
+
version: route.version,
|
|
3080
|
+
handlerCode: route.handlerCode,
|
|
3081
|
+
hostCtx: ctx,
|
|
3082
|
+
block: blockMeta,
|
|
3083
|
+
matched
|
|
3084
|
+
}) : await runHandlers(subgraph, matched, ctx);
|
|
2758
3085
|
handlerMs = performance.now() - handlerStart;
|
|
2759
3086
|
result.processed = runResult.processed;
|
|
2760
3087
|
result.errors = runResult.errors;
|
|
@@ -2780,7 +3107,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2780
3107
|
});
|
|
2781
3108
|
} catch (err) {
|
|
2782
3109
|
if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
|
|
2783
|
-
|
|
3110
|
+
logger7.warn("cursor race lost — block already covered", {
|
|
2784
3111
|
subgraph: subgraphName,
|
|
2785
3112
|
blockHeight,
|
|
2786
3113
|
error: err.message
|
|
@@ -2806,7 +3133,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2806
3133
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
2807
3134
|
const count = Number(rows[0]?.count ?? 0);
|
|
2808
3135
|
if (count >= 1e7) {
|
|
2809
|
-
|
|
3136
|
+
logger7.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
2810
3137
|
subgraph: subgraphName,
|
|
2811
3138
|
table,
|
|
2812
3139
|
count
|
|
@@ -2814,7 +3141,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
2814
3141
|
}
|
|
2815
3142
|
}
|
|
2816
3143
|
} catch (err) {
|
|
2817
|
-
|
|
3144
|
+
logger7.debug("Row count sample failed", {
|
|
2818
3145
|
subgraph: subgraphName,
|
|
2819
3146
|
error: err instanceof Error ? err.message : String(err)
|
|
2820
3147
|
});
|
|
@@ -2904,11 +3231,11 @@ import {
|
|
|
2904
3231
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
2905
3232
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
2906
3233
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2907
|
-
import { logger as
|
|
3234
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2908
3235
|
|
|
2909
3236
|
// src/runtime/reindex-notify.ts
|
|
2910
3237
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
2911
|
-
import { logger as
|
|
3238
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2912
3239
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
2913
3240
|
try {
|
|
2914
3241
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -2919,7 +3246,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2919
3246
|
return;
|
|
2920
3247
|
const resendKey = process.env.RESEND_API_KEY;
|
|
2921
3248
|
if (!resendKey) {
|
|
2922
|
-
|
|
3249
|
+
logger8.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
2923
3250
|
subgraph: subgraphName
|
|
2924
3251
|
});
|
|
2925
3252
|
return;
|
|
@@ -2941,14 +3268,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
2941
3268
|
});
|
|
2942
3269
|
if (!res.ok) {
|
|
2943
3270
|
const text = await res.text().catch(() => "");
|
|
2944
|
-
|
|
3271
|
+
logger8.warn("reindex-complete email failed", {
|
|
2945
3272
|
subgraph: subgraphName,
|
|
2946
3273
|
status: res.status,
|
|
2947
3274
|
body: text.slice(0, 200)
|
|
2948
3275
|
});
|
|
2949
3276
|
}
|
|
2950
3277
|
} catch (err) {
|
|
2951
|
-
|
|
3278
|
+
logger8.warn("reindex-complete email threw", {
|
|
2952
3279
|
subgraph: subgraphName,
|
|
2953
3280
|
error: getErrorMessage2(err)
|
|
2954
3281
|
});
|
|
@@ -3060,7 +3387,7 @@ async function processBlockRange(def, opts) {
|
|
|
3060
3387
|
while (currentHeight <= toBlock) {
|
|
3061
3388
|
if (opts.signal?.aborted) {
|
|
3062
3389
|
aborted = true;
|
|
3063
|
-
|
|
3390
|
+
logger9.info("Block processing aborted", {
|
|
3064
3391
|
subgraph: subgraphName,
|
|
3065
3392
|
currentBlock: currentHeight,
|
|
3066
3393
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -3109,7 +3436,7 @@ async function processBlockRange(def, opts) {
|
|
|
3109
3436
|
});
|
|
3110
3437
|
} catch (err) {
|
|
3111
3438
|
const errorMsg = getErrorMessage3(err);
|
|
3112
|
-
|
|
3439
|
+
logger9.error("Block processing failed persistently", {
|
|
3113
3440
|
subgraph: subgraphName,
|
|
3114
3441
|
blockHeight: height,
|
|
3115
3442
|
error: errorMsg
|
|
@@ -3155,7 +3482,7 @@ async function processBlockRange(def, opts) {
|
|
|
3155
3482
|
lastProgressFlushAt = now;
|
|
3156
3483
|
}
|
|
3157
3484
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
3158
|
-
|
|
3485
|
+
logger9.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
3159
3486
|
subgraph: subgraphName,
|
|
3160
3487
|
processed: blocksProcessed,
|
|
3161
3488
|
total: totalBlocks,
|
|
@@ -3175,7 +3502,7 @@ async function processBlockRange(def, opts) {
|
|
|
3175
3502
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
3176
3503
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
3177
3504
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
3178
|
-
|
|
3505
|
+
logger9.warn("Failed to record subgraph gaps", {
|
|
3179
3506
|
subgraph: subgraphName,
|
|
3180
3507
|
error: err instanceof Error ? err.message : String(err)
|
|
3181
3508
|
});
|
|
@@ -3192,7 +3519,7 @@ async function processBlockRange(def, opts) {
|
|
|
3192
3519
|
} else {
|
|
3193
3520
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
3194
3521
|
}
|
|
3195
|
-
|
|
3522
|
+
logger9.info("Sparse skip", {
|
|
3196
3523
|
subgraph: subgraphName,
|
|
3197
3524
|
from: batchEnd + 1,
|
|
3198
3525
|
to: jumpTo - 1,
|
|
@@ -3232,11 +3559,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
3232
3559
|
const subgraphName = def.name;
|
|
3233
3560
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3234
3561
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3235
|
-
|
|
3562
|
+
logger9.info("Reindex starting", { subgraph: subgraphName });
|
|
3236
3563
|
try {
|
|
3237
3564
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3238
3565
|
if (fromBlock > toBlock) {
|
|
3239
|
-
|
|
3566
|
+
logger9.info("No blocks to reindex", {
|
|
3240
3567
|
subgraph: subgraphName,
|
|
3241
3568
|
fromBlock,
|
|
3242
3569
|
toBlock
|
|
@@ -3250,7 +3577,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3250
3577
|
await client.unsafe(stmt);
|
|
3251
3578
|
}
|
|
3252
3579
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3253
|
-
|
|
3580
|
+
logger9.info("Schema recreated for reindex", {
|
|
3254
3581
|
subgraph: subgraphName,
|
|
3255
3582
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3256
3583
|
});
|
|
@@ -3264,7 +3591,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3264
3591
|
last_error_at: null,
|
|
3265
3592
|
updated_at: new Date
|
|
3266
3593
|
}).where("name", "=", subgraphName).execute();
|
|
3267
|
-
|
|
3594
|
+
logger9.info("Reindexing blocks", {
|
|
3268
3595
|
subgraph: subgraphName,
|
|
3269
3596
|
fromBlock,
|
|
3270
3597
|
toBlock,
|
|
@@ -3286,9 +3613,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3286
3613
|
if (reason === "user-cancelled") {
|
|
3287
3614
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3288
3615
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3289
|
-
|
|
3616
|
+
logger9.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3290
3617
|
} else {
|
|
3291
|
-
|
|
3618
|
+
logger9.info("Reindex interrupted by shutdown, will resume", {
|
|
3292
3619
|
subgraph: subgraphName
|
|
3293
3620
|
});
|
|
3294
3621
|
}
|
|
@@ -3296,7 +3623,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3296
3623
|
}
|
|
3297
3624
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3298
3625
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3299
|
-
|
|
3626
|
+
logger9.info("Reindex complete", {
|
|
3300
3627
|
subgraph: subgraphName,
|
|
3301
3628
|
blocks: result.blocksProcessed,
|
|
3302
3629
|
events: result.totalEventsProcessed,
|
|
@@ -3309,7 +3636,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3309
3636
|
});
|
|
3310
3637
|
return { processed: result.blocksProcessed };
|
|
3311
3638
|
} catch (err) {
|
|
3312
|
-
|
|
3639
|
+
logger9.error("Reindex failed", {
|
|
3313
3640
|
subgraph: subgraphName,
|
|
3314
3641
|
error: getErrorMessage3(err)
|
|
3315
3642
|
});
|
|
@@ -3331,7 +3658,7 @@ async function resumeReindex(def, opts) {
|
|
|
3331
3658
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3332
3659
|
const toBlock = Number(row.reindex_to_block);
|
|
3333
3660
|
if (fromBlock == null) {
|
|
3334
|
-
|
|
3661
|
+
logger9.info("No reindex metadata, starting fresh reindex", {
|
|
3335
3662
|
subgraph: subgraphName
|
|
3336
3663
|
});
|
|
3337
3664
|
return reindexSubgraph(def, {
|
|
@@ -3340,12 +3667,12 @@ async function resumeReindex(def, opts) {
|
|
|
3340
3667
|
});
|
|
3341
3668
|
}
|
|
3342
3669
|
if (fromBlock > toBlock) {
|
|
3343
|
-
|
|
3670
|
+
logger9.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3344
3671
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3345
3672
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3346
3673
|
return { processed: 0 };
|
|
3347
3674
|
}
|
|
3348
|
-
|
|
3675
|
+
logger9.info("Resuming reindex", {
|
|
3349
3676
|
subgraph: subgraphName,
|
|
3350
3677
|
fromBlock,
|
|
3351
3678
|
toBlock,
|
|
@@ -3367,9 +3694,9 @@ async function resumeReindex(def, opts) {
|
|
|
3367
3694
|
if (reason === "user-cancelled") {
|
|
3368
3695
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3369
3696
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3370
|
-
|
|
3697
|
+
logger9.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3371
3698
|
} else {
|
|
3372
|
-
|
|
3699
|
+
logger9.info("Resume interrupted by shutdown, will resume again", {
|
|
3373
3700
|
subgraph: subgraphName
|
|
3374
3701
|
});
|
|
3375
3702
|
}
|
|
@@ -3377,13 +3704,13 @@ async function resumeReindex(def, opts) {
|
|
|
3377
3704
|
}
|
|
3378
3705
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3379
3706
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3380
|
-
|
|
3707
|
+
logger9.info("Resumed reindex complete", {
|
|
3381
3708
|
subgraph: subgraphName,
|
|
3382
3709
|
blocks: result.blocksProcessed
|
|
3383
3710
|
});
|
|
3384
3711
|
return { processed: result.blocksProcessed };
|
|
3385
3712
|
} catch (err) {
|
|
3386
|
-
|
|
3713
|
+
logger9.error("Resumed reindex failed", {
|
|
3387
3714
|
subgraph: subgraphName,
|
|
3388
3715
|
error: getErrorMessage3(err)
|
|
3389
3716
|
});
|
|
@@ -3394,7 +3721,7 @@ async function resumeReindex(def, opts) {
|
|
|
3394
3721
|
async function backfillSubgraph(def, opts) {
|
|
3395
3722
|
const targetDb = getTargetDb2();
|
|
3396
3723
|
const subgraphName = def.name;
|
|
3397
|
-
|
|
3724
|
+
logger9.info("Backfill starting", {
|
|
3398
3725
|
subgraph: subgraphName,
|
|
3399
3726
|
from: opts.fromBlock,
|
|
3400
3727
|
to: opts.toBlock
|
|
@@ -3412,17 +3739,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3412
3739
|
signal: opts.signal
|
|
3413
3740
|
});
|
|
3414
3741
|
if (result.aborted) {
|
|
3415
|
-
|
|
3742
|
+
logger9.info("Backfill aborted", { subgraph: subgraphName });
|
|
3416
3743
|
return { processed: result.blocksProcessed };
|
|
3417
3744
|
}
|
|
3418
3745
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3419
3746
|
if (resolved > 0) {
|
|
3420
|
-
|
|
3747
|
+
logger9.info("Resolved subgraph gaps via backfill", {
|
|
3421
3748
|
subgraph: subgraphName,
|
|
3422
3749
|
resolved
|
|
3423
3750
|
});
|
|
3424
3751
|
}
|
|
3425
|
-
|
|
3752
|
+
logger9.info("Backfill complete", {
|
|
3426
3753
|
subgraph: subgraphName,
|
|
3427
3754
|
blocks: result.blocksProcessed,
|
|
3428
3755
|
events: result.totalEventsProcessed,
|
|
@@ -3430,7 +3757,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3430
3757
|
});
|
|
3431
3758
|
return { processed: result.blocksProcessed };
|
|
3432
3759
|
} catch (err) {
|
|
3433
|
-
|
|
3760
|
+
logger9.error("Backfill failed", {
|
|
3434
3761
|
subgraph: subgraphName,
|
|
3435
3762
|
error: getErrorMessage3(err)
|
|
3436
3763
|
});
|
|
@@ -4736,5 +5063,5 @@ export {
|
|
|
4736
5063
|
ByoBreakingChangeError
|
|
4737
5064
|
};
|
|
4738
5065
|
|
|
4739
|
-
//# debugId=
|
|
5066
|
+
//# debugId=F67C948CCFE0BB8264756E2164756E21
|
|
4740
5067
|
//# sourceMappingURL=index.js.map
|