@secondlayer/subgraphs 3.23.1 → 3.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@ import { createRequire } from "node:module";
2
2
  var __require = /* @__PURE__ */ createRequire(import.meta.url);
3
3
 
4
4
  // src/runtime/context.ts
5
- import { logger } from "@secondlayer/shared/logger";
5
+ import { logger as logger2 } from "@secondlayer/shared/logger";
6
6
  import { formatUnits } from "@secondlayer/stacks/utils";
7
7
  import { sql } from "kysely";
8
8
 
@@ -147,6 +147,7 @@ function generateSubgraphSQL(def, schemaNameOverride) {
147
147
 
148
148
  // src/runtime/chain-read.ts
149
149
  import { getSourceDb } from "@secondlayer/shared/db";
150
+ import { logger } from "@secondlayer/shared/logger";
150
151
  import { http, createPublicClient } from "@secondlayer/stacks";
151
152
  import {
152
153
  ContractResponseError,
@@ -232,6 +233,14 @@ function createChainReadClient(ctx) {
232
233
  const pinnedTo = constant ? null : ctx.indexBlockHash;
233
234
  const cached = await db.selectFrom("chain_read_cache").select("result_hex").where("contract_id", "=", contractId).where("function_name", "=", fn.name).where("args_hash", "=", argsHash).where((eb) => pinnedTo === null ? eb("index_block_hash", "is", null) : eb("index_block_hash", "=", pinnedTo)).executeTakeFirst();
234
235
  const resultHex = cached?.result_hex ?? await gate.run(async () => {
236
+ if (!ctx.indexBlockHash) {
237
+ logger.warn("Chain read is not pinned to a block (no index_block_hash)", {
238
+ contractId,
239
+ functionName: fn.name,
240
+ blockHeight: ctx.blockHeight,
241
+ note: 'cache: "contract-constant" asserts this value cannot change; the node answered at its own tip'
242
+ });
243
+ }
235
244
  const value2 = await readContract(nodeClient(rpcUrl), {
236
245
  contract: contractId,
237
246
  functionName: fn.name,
@@ -346,7 +355,7 @@ class SubgraphContext {
346
355
  data: { ...key, ...row, ...meta, _upsert_keys: keyColumns }
347
356
  });
348
357
  } else {
349
- logger.warn("upsert called without matching uniqueKeys constraint, using fallback", {
358
+ logger2.warn("upsert called without matching uniqueKeys constraint, using fallback", {
350
359
  table,
351
360
  keys: keyColumns
352
361
  });
@@ -906,7 +915,7 @@ function decodeFunctionArgs(args) {
906
915
 
907
916
  // src/runtime/runner.ts
908
917
  import { getErrorMessage } from "@secondlayer/shared";
909
- import { logger as logger2 } from "@secondlayer/shared/logger";
918
+ import { logger as logger3 } from "@secondlayer/shared/logger";
910
919
  import {
911
920
  clarityValueToJS,
912
921
  deserializeCV as deserializeCV3,
@@ -1257,7 +1266,7 @@ async function runHandlers(subgraph, matched, ctx, opts) {
1257
1266
  units.sort((a, b) => (a.tx.tx_index ?? 0) - (b.tx.tx_index ?? 0) || (a.event?.event_index ?? -1) - (b.event?.event_index ?? -1));
1258
1267
  for (const { tx, event, sourceName } of units) {
1259
1268
  if (errors >= threshold) {
1260
- logger2.error("Subgraph error threshold reached, skipping remaining events", {
1269
+ logger3.error("Subgraph error threshold reached, skipping remaining events", {
1261
1270
  subgraph: subgraph.name,
1262
1271
  errors,
1263
1272
  threshold
@@ -1266,7 +1275,7 @@ async function runHandlers(subgraph, matched, ctx, opts) {
1266
1275
  }
1267
1276
  const handler = subgraph.handlers[sourceName] ?? subgraph.handlers["*"] ?? null;
1268
1277
  if (!handler) {
1269
- logger2.warn("No handler found for source", {
1278
+ logger3.warn("No handler found for source", {
1270
1279
  subgraph: subgraph.name,
1271
1280
  sourceName,
1272
1281
  txId: tx.tx_id
@@ -1323,7 +1332,7 @@ async function runHandlers(subgraph, matched, ctx, opts) {
1323
1332
  const verdict = validatePrintPayload(filter.prints, printPayload.topic ?? "", printPayload.data ?? {});
1324
1333
  if (!verdict.ok) {
1325
1334
  skipped++;
1326
- logger2.warn("Print payload does not match declared prints schema", {
1335
+ logger3.warn("Print payload does not match declared prints schema", {
1327
1336
  subgraph: subgraph.name,
1328
1337
  sourceName,
1329
1338
  topic: printPayload.topic,
@@ -1338,7 +1347,7 @@ async function runHandlers(subgraph, matched, ctx, opts) {
1338
1347
  } catch (err) {
1339
1348
  ctx.rollbackTo(checkpoint);
1340
1349
  errors++;
1341
- logger2.error("Subgraph handler error", {
1350
+ logger3.error("Subgraph handler error", {
1342
1351
  subgraph: subgraph.name,
1343
1352
  sourceName,
1344
1353
  txId: tx.tx_id,
@@ -1618,12 +1627,14 @@ import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contract
1618
1627
  import { advanceOperationCursor } from "@secondlayer/shared/db/queries/subgraph-operations";
1619
1628
  import {
1620
1629
  isByoSubgraph,
1630
+ recordLiveError,
1621
1631
  recordLiveProgress,
1622
1632
  recordSubgraphProcessed,
1623
1633
  resolveSubgraphDb,
1634
+ rewindLiveProgress,
1624
1635
  updateSubgraphStatus
1625
1636
  } from "@secondlayer/shared/db/queries/subgraphs";
1626
- import { logger as logger5 } from "@secondlayer/shared/logger";
1637
+ import { logger as logger6 } from "@secondlayer/shared/logger";
1627
1638
  import { sql as sql3 } from "kysely";
1628
1639
 
1629
1640
  // src/runtime/block-source.ts
@@ -1631,7 +1642,7 @@ import { getSourceDb as getSourceDb2 } from "@secondlayer/shared/db";
1631
1642
  import {
1632
1643
  IndexHttpClient
1633
1644
  } from "@secondlayer/shared/index-http";
1634
- import { logger as logger3 } from "@secondlayer/shared/logger";
1645
+ import { logger as logger4 } from "@secondlayer/shared/logger";
1635
1646
 
1636
1647
  // src/runtime/batch-loader.ts
1637
1648
  async function loadBlockRange(db, fromHeight, toHeight) {
@@ -1955,7 +1966,7 @@ class FallbackBlockSource {
1955
1966
  try {
1956
1967
  return await this.primary.getTip();
1957
1968
  } catch (err) {
1958
- logger3.warn("block source primary getTip failed — using DB tap", {
1969
+ logger4.warn("block source primary getTip failed — using DB tap", {
1959
1970
  error: err instanceof Error ? err.message : String(err)
1960
1971
  });
1961
1972
  return this.fallback.getTip();
@@ -1965,7 +1976,7 @@ class FallbackBlockSource {
1965
1976
  try {
1966
1977
  return await this.primary.loadBlockRange(fromHeight, toHeight);
1967
1978
  } catch (err) {
1968
- logger3.warn("block source primary loadBlockRange failed — using DB tap", {
1979
+ logger4.warn("block source primary loadBlockRange failed — using DB tap", {
1969
1980
  from: fromHeight,
1970
1981
  to: toHeight,
1971
1982
  error: err instanceof Error ? err.message : String(err)
@@ -1997,7 +2008,7 @@ function resolveBlockSource(subgraph) {
1997
2008
  return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph), canSparseScan(subgraph) ? sparseProbeTargets(subgraph) : undefined, needsTransactionData(subgraph)), postgresBlockSource);
1998
2009
  }
1999
2010
  if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
2000
- logger3.debug("Subgraph not streams-index eligible, using DB tap", {
2011
+ logger4.debug("Subgraph not streams-index eligible, using DB tap", {
2001
2012
  subgraph: subgraph.name
2002
2013
  });
2003
2014
  }
@@ -2006,7 +2017,7 @@ function resolveBlockSource(subgraph) {
2006
2017
 
2007
2018
  // src/runtime/outbox-emit.ts
2008
2019
  import { createHash as createHash2 } from "node:crypto";
2009
- import { logger as logger4 } from "@secondlayer/shared/logger";
2020
+ import { logger as logger5 } from "@secondlayer/shared/logger";
2010
2021
  var loggedKillSwitch = false;
2011
2022
  var OP_VERB = {
2012
2023
  insert: "created",
@@ -2030,7 +2041,7 @@ function stableStringify(obj) {
2030
2041
  async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
2031
2042
  if (!isEmitOutboxEnabled()) {
2032
2043
  if (!loggedKillSwitch) {
2033
- logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
2044
+ logger5.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
2034
2045
  loggedKillSwitch = true;
2035
2046
  }
2036
2047
  return 0;
@@ -2328,10 +2339,20 @@ class CursorRaceLostError extends Error {
2328
2339
  this.name = "CursorRaceLostError";
2329
2340
  }
2330
2341
  }
2342
+
2343
+ class LiveCursorRaceLostError extends Error {
2344
+ constructor(subgraphName, height) {
2345
+ super(`live cursor race lost for ${subgraphName} at block ${height}`);
2346
+ this.name = "LiveCursorRaceLostError";
2347
+ }
2348
+ }
2331
2349
  function opCursorMode(opts) {
2332
2350
  const ap = opts?.atomicProgress;
2333
2351
  return ap && "operationId" in ap ? ap : undefined;
2334
2352
  }
2353
+ function isReorgRewind(opts) {
2354
+ return opts?.reorgRewind === true;
2355
+ }
2335
2356
  function statusMode(opts) {
2336
2357
  const ap = opts?.atomicProgress;
2337
2358
  return ap && "status" in ap ? ap : undefined;
@@ -2350,7 +2371,7 @@ async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts,
2350
2371
  const delay = retryDelaysMs[attempt];
2351
2372
  if (delay === undefined)
2352
2373
  break;
2353
- logger5.warn("Block processing failed, retrying", {
2374
+ logger6.warn("Block processing failed, retrying", {
2354
2375
  subgraph: subgraphName,
2355
2376
  blockHeight,
2356
2377
  attempt: attempt + 1,
@@ -2382,7 +2403,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2382
2403
  } else {
2383
2404
  const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
2384
2405
  if (!data) {
2385
- logger5.debug("Block not found or non-canonical for subgraph processing", {
2406
+ logger6.debug("Block not found or non-canonical for subgraph processing", {
2386
2407
  subgraph: subgraphName,
2387
2408
  blockHeight
2388
2409
  });
@@ -2401,7 +2422,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2401
2422
  result.matched = matched.length;
2402
2423
  if (matched.length === 0) {
2403
2424
  if (!opts?.skipProgressUpdate) {
2404
- await recordLiveProgress(targetDb, subgraphName, blockHeight);
2425
+ if (isReorgRewind(opts)) {
2426
+ await rewindLiveProgress(targetDb, subgraphName, blockHeight);
2427
+ } else {
2428
+ await recordLiveProgress(targetDb, subgraphName, blockHeight);
2429
+ }
2405
2430
  }
2406
2431
  return result;
2407
2432
  }
@@ -2420,13 +2445,23 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2420
2445
  };
2421
2446
  let handlerMs = 0;
2422
2447
  let flushMs = 0;
2423
- const applyProgress = async (tx, rr) => {
2448
+ const applyProgress = async (tx, rr, flushedWrites) => {
2424
2449
  if (opts?.skipProgressUpdate)
2425
2450
  return;
2451
+ const rewind = isReorgRewind(opts);
2426
2452
  if (rr.errors > 0 && rr.processed === 0) {
2427
- await updateSubgraphStatus(tx, subgraphName, "error", blockHeight);
2453
+ if (rewind) {
2454
+ await updateSubgraphStatus(tx, subgraphName, "error", blockHeight);
2455
+ } else {
2456
+ await recordLiveError(tx, subgraphName, blockHeight);
2457
+ }
2458
+ } else if (rewind) {
2459
+ await rewindLiveProgress(tx, subgraphName, blockHeight);
2428
2460
  } else {
2429
- await recordLiveProgress(tx, subgraphName, blockHeight);
2461
+ const advanced = await recordLiveProgress(tx, subgraphName, blockHeight);
2462
+ if (!advanced && flushedWrites && !route.byo) {
2463
+ throw new LiveCursorRaceLostError(subgraphName, blockHeight);
2464
+ }
2430
2465
  }
2431
2466
  if (rr.processed > 0 || rr.errors > 0) {
2432
2467
  const lastError = rr.errors > 0 ? `${rr.errors} error(s) at block ${blockHeight}` : undefined;
@@ -2475,13 +2510,15 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2475
2510
  } else if (byoOm && manifest && manifest.count > 0) {
2476
2511
  await advanceOperationCursor(tx, byoOm.operationId, blockHeight);
2477
2512
  }
2478
- await applyProgress(tx, runResult);
2513
+ await applyProgress(tx, runResult, manifest ? manifest.count > 0 : false);
2479
2514
  });
2480
2515
  } else {
2481
2516
  try {
2482
2517
  await targetDb.transaction().execute(async (tx) => {
2483
2518
  const opMode = opCursorMode(opts);
2484
- if (statusMode(opts)) {
2519
+ const sm = statusMode(opts);
2520
+ const rewind = isReorgRewind(opts);
2521
+ if (sm) {
2485
2522
  const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
2486
2523
  if (row && Number(row.last_processed_block) >= blockHeight) {
2487
2524
  result.skipped = true;
@@ -2493,6 +2530,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2493
2530
  result.skipped = true;
2494
2531
  return;
2495
2532
  }
2533
+ } else if (!rewind) {
2534
+ const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
2535
+ if (row && Number(row.last_processed_block) >= blockHeight) {
2536
+ result.skipped = true;
2537
+ return;
2538
+ }
2496
2539
  }
2497
2540
  const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
2498
2541
  const handlerStart = performance.now();
@@ -2510,7 +2553,6 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2510
2553
  }
2511
2554
  flushMs = performance.now() - flushStart;
2512
2555
  }
2513
- const sm = statusMode(opts);
2514
2556
  if (sm && flushedWrites) {
2515
2557
  await updateSubgraphStatus(tx, subgraphName, sm.status, blockHeight);
2516
2558
  } else if (opMode && flushedWrites) {
@@ -2519,11 +2561,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2519
2561
  throw new CursorRaceLostError(opMode.operationId, blockHeight);
2520
2562
  }
2521
2563
  }
2522
- await applyProgress(tx, runResult);
2564
+ await applyProgress(tx, runResult, flushedWrites);
2523
2565
  });
2524
2566
  } catch (err) {
2525
- if (err instanceof CursorRaceLostError) {
2526
- logger5.warn("cursor race lost — block already covered", {
2567
+ if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
2568
+ logger6.warn("cursor race lost — block already covered", {
2527
2569
  subgraph: subgraphName,
2528
2570
  blockHeight,
2529
2571
  error: err.message
@@ -2535,11 +2577,13 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2535
2577
  }
2536
2578
  }
2537
2579
  const totalMs = performance.now() - blockStart;
2538
- result.timing = {
2539
- totalMs: Math.round(totalMs),
2540
- handlerMs: Math.round(handlerMs),
2541
- flushMs: Math.round(flushMs)
2542
- };
2580
+ if (!result.skipped) {
2581
+ result.timing = {
2582
+ totalMs: Math.round(totalMs),
2583
+ handlerMs: Math.round(handlerMs),
2584
+ flushMs: Math.round(flushMs)
2585
+ };
2586
+ }
2543
2587
  if (blockHeight % 1000 === 0) {
2544
2588
  try {
2545
2589
  const tables = Object.keys(subgraph.schema);
@@ -2547,7 +2591,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2547
2591
  const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
2548
2592
  const count = Number(rows[0]?.count ?? 0);
2549
2593
  if (count >= 1e7) {
2550
- logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
2594
+ logger6.warn("Subgraph table exceeds 10M rows (estimate)", {
2551
2595
  subgraph: subgraphName,
2552
2596
  table,
2553
2597
  count
@@ -2555,7 +2599,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
2555
2599
  }
2556
2600
  }
2557
2601
  } catch (err) {
2558
- logger5.debug("Row count sample failed", {
2602
+ logger6.debug("Row count sample failed", {
2559
2603
  subgraph: subgraphName,
2560
2604
  error: err instanceof Error ? err.message : String(err)
2561
2605
  });
@@ -2645,11 +2689,11 @@ import {
2645
2689
  recordSubgraphProcessed as recordSubgraphProcessed2,
2646
2690
  updateSubgraphStatus as updateSubgraphStatus2
2647
2691
  } from "@secondlayer/shared/db/queries/subgraphs";
2648
- import { logger as logger7 } from "@secondlayer/shared/logger";
2692
+ import { logger as logger8 } from "@secondlayer/shared/logger";
2649
2693
 
2650
2694
  // src/runtime/reindex-notify.ts
2651
2695
  import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
2652
- import { logger as logger6 } from "@secondlayer/shared/logger";
2696
+ import { logger as logger7 } from "@secondlayer/shared/logger";
2653
2697
  async function notifyReindexComplete(db, subgraphName, stats) {
2654
2698
  try {
2655
2699
  const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
@@ -2660,7 +2704,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
2660
2704
  return;
2661
2705
  const resendKey = process.env.RESEND_API_KEY;
2662
2706
  if (!resendKey) {
2663
- logger6.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
2707
+ logger7.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
2664
2708
  subgraph: subgraphName
2665
2709
  });
2666
2710
  return;
@@ -2682,14 +2726,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
2682
2726
  });
2683
2727
  if (!res.ok) {
2684
2728
  const text = await res.text().catch(() => "");
2685
- logger6.warn("reindex-complete email failed", {
2729
+ logger7.warn("reindex-complete email failed", {
2686
2730
  subgraph: subgraphName,
2687
2731
  status: res.status,
2688
2732
  body: text.slice(0, 200)
2689
2733
  });
2690
2734
  }
2691
2735
  } catch (err) {
2692
- logger6.warn("reindex-complete email threw", {
2736
+ logger7.warn("reindex-complete email threw", {
2693
2737
  subgraph: subgraphName,
2694
2738
  error: getErrorMessage2(err)
2695
2739
  });
@@ -2801,7 +2845,7 @@ async function processBlockRange(def, opts) {
2801
2845
  while (currentHeight <= toBlock) {
2802
2846
  if (opts.signal?.aborted) {
2803
2847
  aborted = true;
2804
- logger7.info("Block processing aborted", {
2848
+ logger8.info("Block processing aborted", {
2805
2849
  subgraph: subgraphName,
2806
2850
  currentBlock: currentHeight,
2807
2851
  reason: String(opts.signal.reason ?? "unknown")
@@ -2850,7 +2894,7 @@ async function processBlockRange(def, opts) {
2850
2894
  });
2851
2895
  } catch (err) {
2852
2896
  const errorMsg = getErrorMessage3(err);
2853
- logger7.error("Block processing failed persistently", {
2897
+ logger8.error("Block processing failed persistently", {
2854
2898
  subgraph: subgraphName,
2855
2899
  blockHeight: height,
2856
2900
  error: errorMsg
@@ -2896,7 +2940,7 @@ async function processBlockRange(def, opts) {
2896
2940
  lastProgressFlushAt = now;
2897
2941
  }
2898
2942
  if (blocksProcessed % LOG_INTERVAL === 0) {
2899
- logger7.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
2943
+ logger8.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
2900
2944
  subgraph: subgraphName,
2901
2945
  processed: blocksProcessed,
2902
2946
  total: totalBlocks,
@@ -2916,7 +2960,7 @@ async function processBlockRange(def, opts) {
2916
2960
  if (batchFailedBlocks.length > 0 && opts.subgraphId) {
2917
2961
  const gaps = coalesceFailedBlocks(batchFailedBlocks);
2918
2962
  await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
2919
- logger7.warn("Failed to record subgraph gaps", {
2963
+ logger8.warn("Failed to record subgraph gaps", {
2920
2964
  subgraph: subgraphName,
2921
2965
  error: err instanceof Error ? err.message : String(err)
2922
2966
  });
@@ -2933,7 +2977,7 @@ async function processBlockRange(def, opts) {
2933
2977
  } else {
2934
2978
  await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
2935
2979
  }
2936
- logger7.info("Sparse skip", {
2980
+ logger8.info("Sparse skip", {
2937
2981
  subgraph: subgraphName,
2938
2982
  from: batchEnd + 1,
2939
2983
  to: jumpTo - 1,
@@ -2973,11 +3017,11 @@ async function reindexSubgraph(def, opts) {
2973
3017
  const subgraphName = def.name;
2974
3018
  const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
2975
3019
  await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
2976
- logger7.info("Reindex starting", { subgraph: subgraphName });
3020
+ logger8.info("Reindex starting", { subgraph: subgraphName });
2977
3021
  try {
2978
3022
  const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
2979
3023
  if (fromBlock > toBlock) {
2980
- logger7.info("No blocks to reindex", {
3024
+ logger8.info("No blocks to reindex", {
2981
3025
  subgraph: subgraphName,
2982
3026
  fromBlock,
2983
3027
  toBlock
@@ -2991,7 +3035,7 @@ async function reindexSubgraph(def, opts) {
2991
3035
  await client.unsafe(stmt);
2992
3036
  }
2993
3037
  await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
2994
- logger7.info("Schema recreated for reindex", {
3038
+ logger8.info("Schema recreated for reindex", {
2995
3039
  subgraph: subgraphName,
2996
3040
  cursorResetTo: Math.max(0, fromBlock - 1)
2997
3041
  });
@@ -3005,7 +3049,7 @@ async function reindexSubgraph(def, opts) {
3005
3049
  last_error_at: null,
3006
3050
  updated_at: new Date
3007
3051
  }).where("name", "=", subgraphName).execute();
3008
- logger7.info("Reindexing blocks", {
3052
+ logger8.info("Reindexing blocks", {
3009
3053
  subgraph: subgraphName,
3010
3054
  fromBlock,
3011
3055
  toBlock,
@@ -3027,9 +3071,9 @@ async function reindexSubgraph(def, opts) {
3027
3071
  if (reason === "user-cancelled") {
3028
3072
  await updateSubgraphStatus2(targetDb, subgraphName, "active");
3029
3073
  await clearReindexMetadata(targetDb, subgraphName);
3030
- logger7.info("Reindex cancelled by user", { subgraph: subgraphName });
3074
+ logger8.info("Reindex cancelled by user", { subgraph: subgraphName });
3031
3075
  } else {
3032
- logger7.info("Reindex interrupted by shutdown, will resume", {
3076
+ logger8.info("Reindex interrupted by shutdown, will resume", {
3033
3077
  subgraph: subgraphName
3034
3078
  });
3035
3079
  }
@@ -3037,7 +3081,7 @@ async function reindexSubgraph(def, opts) {
3037
3081
  }
3038
3082
  await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
3039
3083
  await clearReindexMetadata(targetDb, subgraphName);
3040
- logger7.info("Reindex complete", {
3084
+ logger8.info("Reindex complete", {
3041
3085
  subgraph: subgraphName,
3042
3086
  blocks: result.blocksProcessed,
3043
3087
  events: result.totalEventsProcessed,
@@ -3050,7 +3094,7 @@ async function reindexSubgraph(def, opts) {
3050
3094
  });
3051
3095
  return { processed: result.blocksProcessed };
3052
3096
  } catch (err) {
3053
- logger7.error("Reindex failed", {
3097
+ logger8.error("Reindex failed", {
3054
3098
  subgraph: subgraphName,
3055
3099
  error: getErrorMessage3(err)
3056
3100
  });
@@ -3072,7 +3116,7 @@ async function resumeReindex(def, opts) {
3072
3116
  const fromBlock = resolveReindexResumeBlock(row);
3073
3117
  const toBlock = Number(row.reindex_to_block);
3074
3118
  if (fromBlock == null) {
3075
- logger7.info("No reindex metadata, starting fresh reindex", {
3119
+ logger8.info("No reindex metadata, starting fresh reindex", {
3076
3120
  subgraph: subgraphName
3077
3121
  });
3078
3122
  return reindexSubgraph(def, {
@@ -3081,12 +3125,12 @@ async function resumeReindex(def, opts) {
3081
3125
  });
3082
3126
  }
3083
3127
  if (fromBlock > toBlock) {
3084
- logger7.info("Resume: no remaining blocks", { subgraph: subgraphName });
3128
+ logger8.info("Resume: no remaining blocks", { subgraph: subgraphName });
3085
3129
  await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
3086
3130
  await clearReindexMetadata(targetDb, subgraphName);
3087
3131
  return { processed: 0 };
3088
3132
  }
3089
- logger7.info("Resuming reindex", {
3133
+ logger8.info("Resuming reindex", {
3090
3134
  subgraph: subgraphName,
3091
3135
  fromBlock,
3092
3136
  toBlock,
@@ -3108,9 +3152,9 @@ async function resumeReindex(def, opts) {
3108
3152
  if (reason === "user-cancelled") {
3109
3153
  await updateSubgraphStatus2(targetDb, subgraphName, "active");
3110
3154
  await clearReindexMetadata(targetDb, subgraphName);
3111
- logger7.info("Resume cancelled by user", { subgraph: subgraphName });
3155
+ logger8.info("Resume cancelled by user", { subgraph: subgraphName });
3112
3156
  } else {
3113
- logger7.info("Resume interrupted by shutdown, will resume again", {
3157
+ logger8.info("Resume interrupted by shutdown, will resume again", {
3114
3158
  subgraph: subgraphName
3115
3159
  });
3116
3160
  }
@@ -3118,13 +3162,13 @@ async function resumeReindex(def, opts) {
3118
3162
  }
3119
3163
  await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
3120
3164
  await clearReindexMetadata(targetDb, subgraphName);
3121
- logger7.info("Resumed reindex complete", {
3165
+ logger8.info("Resumed reindex complete", {
3122
3166
  subgraph: subgraphName,
3123
3167
  blocks: result.blocksProcessed
3124
3168
  });
3125
3169
  return { processed: result.blocksProcessed };
3126
3170
  } catch (err) {
3127
- logger7.error("Resumed reindex failed", {
3171
+ logger8.error("Resumed reindex failed", {
3128
3172
  subgraph: subgraphName,
3129
3173
  error: getErrorMessage3(err)
3130
3174
  });
@@ -3135,7 +3179,7 @@ async function resumeReindex(def, opts) {
3135
3179
  async function backfillSubgraph(def, opts) {
3136
3180
  const targetDb = getTargetDb2();
3137
3181
  const subgraphName = def.name;
3138
- logger7.info("Backfill starting", {
3182
+ logger8.info("Backfill starting", {
3139
3183
  subgraph: subgraphName,
3140
3184
  from: opts.fromBlock,
3141
3185
  to: opts.toBlock
@@ -3153,17 +3197,17 @@ async function backfillSubgraph(def, opts) {
3153
3197
  signal: opts.signal
3154
3198
  });
3155
3199
  if (result.aborted) {
3156
- logger7.info("Backfill aborted", { subgraph: subgraphName });
3200
+ logger8.info("Backfill aborted", { subgraph: subgraphName });
3157
3201
  return { processed: result.blocksProcessed };
3158
3202
  }
3159
3203
  const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
3160
3204
  if (resolved > 0) {
3161
- logger7.info("Resolved subgraph gaps via backfill", {
3205
+ logger8.info("Resolved subgraph gaps via backfill", {
3162
3206
  subgraph: subgraphName,
3163
3207
  resolved
3164
3208
  });
3165
3209
  }
3166
- logger7.info("Backfill complete", {
3210
+ logger8.info("Backfill complete", {
3167
3211
  subgraph: subgraphName,
3168
3212
  blocks: result.blocksProcessed,
3169
3213
  events: result.totalEventsProcessed,
@@ -3171,7 +3215,7 @@ async function backfillSubgraph(def, opts) {
3171
3215
  });
3172
3216
  return { processed: result.blocksProcessed };
3173
3217
  } catch (err) {
3174
- logger7.error("Backfill failed", {
3218
+ logger8.error("Backfill failed", {
3175
3219
  subgraph: subgraphName,
3176
3220
  error: getErrorMessage3(err)
3177
3221
  });
@@ -3182,7 +3226,34 @@ async function backfillSubgraph(def, opts) {
3182
3226
  // src/runtime/catchup.ts
3183
3227
  import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
3184
3228
  import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
3185
- import { logger as logger8 } from "@secondlayer/shared/logger";
3229
+ import { logger as logger9 } from "@secondlayer/shared/logger";
3230
+
3231
+ // src/runtime/catchup-leader.ts
3232
+ import {
3233
+ SUBGRAPH_CATCHUP_LOCK_KEY,
3234
+ createPostgresLeaderBackend,
3235
+ withLeaderLock
3236
+ } from "@secondlayer/shared/leader";
3237
+ import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
3238
+ var catchUpLeader = false;
3239
+ function isCatchUpLeader() {
3240
+ return catchUpLeader;
3241
+ }
3242
+ function startCatchUpLeader(opts = {}) {
3243
+ return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
3244
+ catchUpLeader = true;
3245
+ await opts.onAcquire?.();
3246
+ return () => {
3247
+ catchUpLeader = false;
3248
+ };
3249
+ }, {
3250
+ pollMs: opts.pollMs,
3251
+ heartbeatMs: opts.heartbeatMs,
3252
+ createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
3253
+ });
3254
+ }
3255
+
3256
+ // src/runtime/catchup.ts
3186
3257
  var LOG_INTERVAL2 = 1000;
3187
3258
  var STANDARD_CATCHUP_BATCH_CONFIG = {
3188
3259
  defaultBatchSize: 500,
@@ -3289,7 +3360,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
3289
3360
  const subgraphStart = Number(subgraphRow.start_block) || 1;
3290
3361
  const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
3291
3362
  const totalBlocks = chainTip - lastProcessedBlock;
3292
- logger8.info("Subgraph catch-up starting", {
3363
+ logger9.info("Subgraph catch-up starting", {
3293
3364
  subgraph: subgraphName,
3294
3365
  from: startBlock,
3295
3366
  to: chainTip,
@@ -3304,9 +3375,16 @@ async function catchUpSubgraph(subgraph, subgraphName) {
3304
3375
  let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
3305
3376
  let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
3306
3377
  while (currentHeight <= chainTip) {
3378
+ if (!isCatchUpLeader()) {
3379
+ logger9.info("Lost catch-up leadership, stopping catch-up", {
3380
+ subgraph: subgraphName,
3381
+ processed
3382
+ });
3383
+ break;
3384
+ }
3307
3385
  const currentRow = await getSubgraph(targetDb, subgraphName);
3308
3386
  if (!currentRow || currentRow.status !== "active") {
3309
- logger8.info("Subgraph status changed, stopping catch-up", {
3387
+ logger9.info("Subgraph status changed, stopping catch-up", {
3310
3388
  subgraph: subgraphName,
3311
3389
  status: currentRow?.status ?? "deleted"
3312
3390
  });
@@ -3335,7 +3413,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
3335
3413
  blockData = (await source.loadBlockRange(height, height)).get(height);
3336
3414
  }
3337
3415
  if (!blockData) {
3338
- logger8.warn("Block missing during catch-up, deferring to next tick", {
3416
+ logger9.warn("Block missing during catch-up, deferring to next tick", {
3339
3417
  subgraph: subgraphName,
3340
3418
  blockHeight: height
3341
3419
  });
@@ -3355,12 +3433,12 @@ async function catchUpSubgraph(subgraph, subgraphName) {
3355
3433
  });
3356
3434
  } catch (err) {
3357
3435
  if (err instanceof ReorgEpochChangedError) {
3358
- logger8.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
3436
+ logger9.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
3359
3437
  stopCatchup = true;
3360
3438
  break;
3361
3439
  }
3362
3440
  const errorMsg = err instanceof Error ? err.message : String(err);
3363
- logger8.error("Block processing failed persistently during catch-up", {
3441
+ logger9.error("Block processing failed persistently during catch-up", {
3364
3442
  subgraph: subgraphName,
3365
3443
  blockHeight: height,
3366
3444
  error: errorMsg
@@ -3379,7 +3457,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
3379
3457
  }
3380
3458
  }
3381
3459
  if (processed % LOG_INTERVAL2 === 0) {
3382
- logger8.info("Subgraph catch-up progress", {
3460
+ logger9.info("Subgraph catch-up progress", {
3383
3461
  subgraph: subgraphName,
3384
3462
  processed,
3385
3463
  total: totalBlocks,
@@ -3395,7 +3473,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
3395
3473
  currentHeight = batchEnd + 1;
3396
3474
  }
3397
3475
  await stats.flush(targetDb);
3398
- logger8.info("Subgraph catch-up complete", {
3476
+ logger9.info("Subgraph catch-up complete", {
3399
3477
  subgraph: subgraphName,
3400
3478
  processed
3401
3479
  });
@@ -3412,13 +3490,14 @@ import {
3412
3490
  listSubgraphs,
3413
3491
  resolveSubgraphRawClient
3414
3492
  } from "@secondlayer/shared/db/queries/subgraphs";
3415
- import { logger as logger9 } from "@secondlayer/shared/logger";
3493
+ import { logger as logger10 } from "@secondlayer/shared/logger";
3494
+ import { sql as sql4 } from "kysely";
3416
3495
  async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
3417
3496
  const targetDb = getTargetDb4();
3418
3497
  const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
3419
3498
  if (activeSubgraphs.length === 0)
3420
3499
  return;
3421
- logger9.info("Propagating reorg to subgraphs", {
3500
+ logger10.info("Propagating reorg to subgraphs", {
3422
3501
  blockHeight,
3423
3502
  subgraphCount: activeSubgraphs.length
3424
3503
  });
@@ -3429,100 +3508,13 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
3429
3508
  continue;
3430
3509
  bumpReorgEpoch(sg.name);
3431
3510
  await withSubgraphBlockLock(sg.name, async () => {
3432
- const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
3433
- const client = resolveSubgraphRawClient(sg);
3434
- const tableNames = Object.keys(schema);
3435
- const revertedByTable = {};
3436
- for (const tableName of tableNames) {
3437
- const rows = await client.unsafe(`SELECT * FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1 LIMIT 1000`, [blockHeight]);
3438
- if (rows.length > 0)
3439
- revertedByTable[tableName] = rows;
3440
- }
3441
- const hasJournal = (await client.unsafe(`SELECT to_regclass('"${schemaName}"."_journal"') AS r`))[0]?.r;
3442
- if (hasJournal) {
3443
- await client.begin(async (tx) => {
3444
- for (const tableName of tableNames) {
3445
- const earliest = `
3446
- SELECT DISTINCT ON (row_key) row_key, prev_row
3447
- FROM "${schemaName}"."_journal"
3448
- WHERE block_height >= $1 AND table_name = $2
3449
- ORDER BY row_key, _jid ASC`;
3450
- await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" t USING (${earliest}) e WHERE to_jsonb(t.*) @> e.row_key`, [blockHeight, tableName]);
3451
- await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
3452
- await tx.unsafe(`INSERT INTO "${schemaName}"."${tableName}"
3453
- SELECT r.* FROM (${earliest}) e
3454
- CROSS JOIN LATERAL jsonb_populate_record(NULL::"${schemaName}"."${tableName}", e.prev_row) r
3455
- WHERE e.prev_row IS NOT NULL`, [blockHeight, tableName]);
3456
- }
3457
- await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
3458
- });
3459
- } else {
3460
- logger9.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
3461
- for (const tableName of tableNames) {
3462
- await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
3463
- }
3464
- }
3465
- await client.unsafe(`DO $$ BEGIN
3466
- IF to_regclass('"${schemaName}"."_factory_addresses"') IS NOT NULL THEN
3467
- DELETE FROM "${schemaName}"."_factory_addresses" WHERE block_height >= ${Number(blockHeight)};
3468
- END IF;
3469
- END $$`);
3470
- for (const [tableName, rows] of Object.entries(revertedByTable)) {
3471
- if (rows.length === 0)
3472
- continue;
3473
- await targetDb.insertInto("subscription_outbox").columns([
3474
- "subscription_id",
3475
- "subgraph_name",
3476
- "table_name",
3477
- "block_height",
3478
- "event_type",
3479
- "payload",
3480
- "dedup_key",
3481
- "row_pk",
3482
- "status"
3483
- ]).expression((eb) => eb.selectFrom("subscriptions").select((eb2) => [
3484
- "id as subscription_id",
3485
- eb2.val(sg.name).as("subgraph_name"),
3486
- eb2.val(tableName).as("table_name"),
3487
- eb2.val(blockHeight).as("block_height"),
3488
- eb2.val(`${sg.name}.${tableName}.reverted`).as("event_type"),
3489
- eb2.val(JSON.stringify({
3490
- subgraph: sg.name,
3491
- table: tableName,
3492
- reorgFromHeight: blockHeight,
3493
- rows
3494
- })).as("payload"),
3495
- eb2.val(`reorg:${sg.name}:${tableName}:${blockHeight}`).as("dedup_key"),
3496
- eb2.val({
3497
- subgraph: sg.name,
3498
- table: tableName,
3499
- reorgRoot: blockHeight
3500
- }).as("row_pk"),
3501
- eb2.val("pending").as("status")
3502
- ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
3503
- logger9.error("Failed to emit revert event for subscriptions", {
3504
- event: "reorg_revert_emit_dropped",
3505
- subgraph: sg.name,
3506
- table: tableName,
3507
- blockHeight,
3508
- error: getErrorMessage4(err)
3509
- });
3510
- });
3511
- }
3512
- logger9.info("Subgraph reorg cleanup done", {
3513
- subgraph: sg.name,
3514
- blockHeight,
3515
- tablesAffected: Object.keys(revertedByTable).length
3516
- });
3517
- const def = await loadSubgraphDef(sg);
3518
- await processBlock(def, sg.name, blockHeight);
3519
- logger9.info("Subgraph reorg reprocessed", {
3520
- subgraph: sg.name,
3521
- blockHeight
3511
+ await getTargetDb4().transaction().execute(async (lockTx) => {
3512
+ await sql4`SELECT pg_advisory_xact_lock(hashtext(${`subgraph-reorg:${sg.name}`}))`.execute(lockTx);
3513
+ await reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef);
3522
3514
  });
3523
3515
  });
3524
3516
  } catch (err) {
3525
- logger9.error("Subgraph reorg handling failed", {
3517
+ logger10.error("Subgraph reorg handling failed", {
3526
3518
  subgraph: sg.name,
3527
3519
  blockHeight,
3528
3520
  error: getErrorMessage4(err)
@@ -3530,6 +3522,100 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
3530
3522
  }
3531
3523
  }
3532
3524
  }
3525
+ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
3526
+ const targetDb = getTargetDb4();
3527
+ const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
3528
+ const client = resolveSubgraphRawClient(sg);
3529
+ const tableNames = Object.keys(schema);
3530
+ const revertedByTable = {};
3531
+ for (const tableName of tableNames) {
3532
+ const rows = await client.unsafe(`SELECT * FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1 LIMIT 1000`, [blockHeight]);
3533
+ if (rows.length > 0)
3534
+ revertedByTable[tableName] = rows;
3535
+ }
3536
+ const hasJournal = (await client.unsafe(`SELECT to_regclass('"${schemaName}"."_journal"') AS r`))[0]?.r;
3537
+ if (hasJournal) {
3538
+ await client.begin(async (tx) => {
3539
+ for (const tableName of tableNames) {
3540
+ const earliest = `
3541
+ SELECT DISTINCT ON (row_key) row_key, prev_row
3542
+ FROM "${schemaName}"."_journal"
3543
+ WHERE block_height >= $1 AND table_name = $2
3544
+ ORDER BY row_key, _jid ASC`;
3545
+ await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" t USING (${earliest}) e WHERE to_jsonb(t.*) @> e.row_key`, [blockHeight, tableName]);
3546
+ await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
3547
+ await tx.unsafe(`INSERT INTO "${schemaName}"."${tableName}"
3548
+ SELECT r.* FROM (${earliest}) e
3549
+ CROSS JOIN LATERAL jsonb_populate_record(NULL::"${schemaName}"."${tableName}", e.prev_row) r
3550
+ WHERE e.prev_row IS NOT NULL`, [blockHeight, tableName]);
3551
+ }
3552
+ await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
3553
+ });
3554
+ } else {
3555
+ logger10.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
3556
+ for (const tableName of tableNames) {
3557
+ await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
3558
+ }
3559
+ }
3560
+ await client.unsafe(`DO $$ BEGIN
3561
+ IF to_regclass('"${schemaName}"."_factory_addresses"') IS NOT NULL THEN
3562
+ DELETE FROM "${schemaName}"."_factory_addresses" WHERE block_height >= ${Number(blockHeight)};
3563
+ END IF;
3564
+ END $$`);
3565
+ for (const [tableName, rows] of Object.entries(revertedByTable)) {
3566
+ if (rows.length === 0)
3567
+ continue;
3568
+ await targetDb.insertInto("subscription_outbox").columns([
3569
+ "subscription_id",
3570
+ "subgraph_name",
3571
+ "table_name",
3572
+ "block_height",
3573
+ "event_type",
3574
+ "payload",
3575
+ "dedup_key",
3576
+ "row_pk",
3577
+ "status"
3578
+ ]).expression((eb) => eb.selectFrom("subscriptions").select((eb2) => [
3579
+ "id as subscription_id",
3580
+ eb2.val(sg.name).as("subgraph_name"),
3581
+ eb2.val(tableName).as("table_name"),
3582
+ eb2.val(blockHeight).as("block_height"),
3583
+ eb2.val(`${sg.name}.${tableName}.reverted`).as("event_type"),
3584
+ eb2.val(JSON.stringify({
3585
+ subgraph: sg.name,
3586
+ table: tableName,
3587
+ reorgFromHeight: blockHeight,
3588
+ rows
3589
+ })).as("payload"),
3590
+ eb2.val(`reorg:${sg.name}:${tableName}:${blockHeight}`).as("dedup_key"),
3591
+ eb2.val({
3592
+ subgraph: sg.name,
3593
+ table: tableName,
3594
+ reorgRoot: blockHeight
3595
+ }).as("row_pk"),
3596
+ eb2.val("pending").as("status")
3597
+ ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
3598
+ logger10.error("Failed to emit revert event for subscriptions", {
3599
+ event: "reorg_revert_emit_dropped",
3600
+ subgraph: sg.name,
3601
+ table: tableName,
3602
+ blockHeight,
3603
+ error: getErrorMessage4(err)
3604
+ });
3605
+ });
3606
+ }
3607
+ logger10.info("Subgraph reorg cleanup done", {
3608
+ subgraph: sg.name,
3609
+ blockHeight,
3610
+ tablesAffected: Object.keys(revertedByTable).length
3611
+ });
3612
+ const def = await loadSubgraphDef(sg);
3613
+ await processBlock(def, sg.name, blockHeight, { reorgRewind: true });
3614
+ logger10.info("Subgraph reorg reprocessed", {
3615
+ subgraph: sg.name,
3616
+ blockHeight
3617
+ });
3618
+ }
3533
3619
 
3534
3620
  // src/runtime/processor.ts
3535
3621
  import { randomUUID } from "node:crypto";
@@ -3555,42 +3641,17 @@ import {
3555
3641
  pgSchemaName as pgSchemaName2,
3556
3642
  updateSubgraphStatus as updateSubgraphStatus3
3557
3643
  } from "@secondlayer/shared/db/queries/subgraphs";
3558
- import { logger as logger11 } from "@secondlayer/shared/logger";
3644
+ import { logger as logger12 } from "@secondlayer/shared/logger";
3559
3645
  import {
3560
3646
  listen,
3561
3647
  sourceListenerUrl,
3562
3648
  targetListenerUrl as targetListenerUrl2
3563
3649
  } from "@secondlayer/shared/queue/listener";
3564
3650
 
3565
- // src/runtime/catchup-leader.ts
3566
- import {
3567
- SUBGRAPH_CATCHUP_LOCK_KEY,
3568
- createPostgresLeaderBackend,
3569
- withLeaderLock
3570
- } from "@secondlayer/shared/leader";
3571
- import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
3572
- var catchUpLeader = false;
3573
- function isCatchUpLeader() {
3574
- return catchUpLeader;
3575
- }
3576
- function startCatchUpLeader(opts = {}) {
3577
- return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
3578
- catchUpLeader = true;
3579
- await opts.onAcquire?.();
3580
- return () => {
3581
- catchUpLeader = false;
3582
- };
3583
- }, {
3584
- pollMs: opts.pollMs,
3585
- heartbeatMs: opts.heartbeatMs,
3586
- createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
3587
- });
3588
- }
3589
-
3590
3651
  // src/runtime/streams-reorg-poll.ts
3591
3652
  import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
3592
3653
  import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
3593
- import { logger as logger10 } from "@secondlayer/shared/logger";
3654
+ import { logger as logger11 } from "@secondlayer/shared/logger";
3594
3655
  var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
3595
3656
  var STARTUP_MARGIN_MS = 60 * 60 * 1000;
3596
3657
  async function pollReorgsOnce(http2, cursor, onReorg, handled) {
@@ -3598,13 +3659,13 @@ async function pollReorgsOnce(http2, cursor, onReorg, handled) {
3598
3659
  const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
3599
3660
  for (const r of sorted) {
3600
3661
  if (handled && r.id && handled.has(r.id)) {
3601
- logger10.info("Streams reorg already handled — skipping", {
3662
+ logger11.info("Streams reorg already handled — skipping", {
3602
3663
  forkPointHeight: r.fork_point_height,
3603
3664
  reorgId: r.id
3604
3665
  });
3605
3666
  continue;
3606
3667
  }
3607
- logger10.info("Streams reorg — rewinding", {
3668
+ logger11.info("Streams reorg — rewinding", {
3608
3669
  forkPointHeight: r.fork_point_height
3609
3670
  });
3610
3671
  await onReorg(r.fork_point_height);
@@ -3632,7 +3693,7 @@ function startStreamsReorgPoll(onReorg) {
3632
3693
  try {
3633
3694
  since = await pollReorgsOnce(http2, since, onReorg, handled);
3634
3695
  } catch (err) {
3635
- logger10.error("Streams reorg poll failed", {
3696
+ logger11.error("Streams reorg poll failed", {
3636
3697
  error: getErrorMessage5(err)
3637
3698
  });
3638
3699
  }
@@ -3640,7 +3701,7 @@ function startStreamsReorgPoll(onReorg) {
3640
3701
  timer = setTimeout(tick, POLL_MS);
3641
3702
  };
3642
3703
  timer = setTimeout(tick, POLL_MS);
3643
- logger10.info("Streams reorg poll started", { pollMs: POLL_MS });
3704
+ logger11.info("Streams reorg poll started", { pollMs: POLL_MS });
3644
3705
  return () => {
3645
3706
  running = false;
3646
3707
  if (timer)
@@ -3671,7 +3732,7 @@ async function catchUpAll(subgraphs, db, concurrency) {
3671
3732
  if (isHandlerNotFoundError(err)) {
3672
3733
  await updateSubgraphStatus3(db, sg.name, "error");
3673
3734
  }
3674
- logger11.error("Subgraph catch-up failed", {
3735
+ logger12.error("Subgraph catch-up failed", {
3675
3736
  subgraph: sg.name,
3676
3737
  error: msg
3677
3738
  });
@@ -3711,7 +3772,7 @@ async function loadSubgraphDefinition(sg) {
3711
3772
  definitionCache.set(sg.name, def);
3712
3773
  if (prevVersion && prevVersion !== sg.version) {
3713
3774
  invalidateSubgraphRoute(sg.name);
3714
- logger11.info("Subgraph handler reloaded", {
3775
+ logger12.info("Subgraph handler reloaded", {
3715
3776
  subgraph: sg.name,
3716
3777
  from: prevVersion,
3717
3778
  to: sg.version
@@ -3745,7 +3806,7 @@ async function synthesizeLegacyReindexOperations() {
3745
3806
  fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
3746
3807
  toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
3747
3808
  });
3748
- logger11.info("Queued legacy reindex resume operation", {
3809
+ logger12.info("Queued legacy reindex resume operation", {
3749
3810
  subgraph: sg.name
3750
3811
  });
3751
3812
  } catch (err) {
@@ -3808,7 +3869,7 @@ async function startSubgraphOperationRunner(opts) {
3808
3869
  const activeRuns = new Map;
3809
3870
  let running = true;
3810
3871
  let draining = false;
3811
- logger11.info("Starting subgraph operation runner", { concurrency, lockedBy });
3872
+ logger12.info("Starting subgraph operation runner", { concurrency, lockedBy });
3812
3873
  try {
3813
3874
  const stranded = await db.selectFrom("subgraphs").select(["id", "name", "account_id"]).where("status", "=", "reindexing").where("reindex_from_block", "is not", null).where("reindex_to_block", "is not", null).where(({ not, exists, selectFrom }) => not(exists(selectFrom("subgraph_operations").select("id").whereRef("subgraph_id", "=", "subgraphs.id").where("status", "in", ["queued", "running"])))).execute();
3814
3875
  for (const row of stranded) {
@@ -3819,14 +3880,14 @@ async function startSubgraphOperationRunner(opts) {
3819
3880
  accountId: row.account_id,
3820
3881
  kind: "reindex"
3821
3882
  });
3822
- logger11.info("Re-enqueued stranded reindex", { subgraph: row.name });
3883
+ logger12.info("Re-enqueued stranded reindex", { subgraph: row.name });
3823
3884
  } catch (err) {
3824
3885
  if (!isActiveSubgraphOperationConflict(err))
3825
3886
  throw err;
3826
3887
  }
3827
3888
  }
3828
3889
  } catch (err) {
3829
- logger11.warn("Stranded-reindex sweep failed", {
3890
+ logger12.warn("Stranded-reindex sweep failed", {
3830
3891
  error: getErrorMessage6(err)
3831
3892
  });
3832
3893
  }
@@ -3837,7 +3898,7 @@ async function startSubgraphOperationRunner(opts) {
3837
3898
  if (!running)
3838
3899
  return;
3839
3900
  heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
3840
- logger11.warn("Subgraph operation heartbeat failed", {
3901
+ logger12.warn("Subgraph operation heartbeat failed", {
3841
3902
  operationId: operation.id,
3842
3903
  error: getErrorMessage6(err)
3843
3904
  });
@@ -3849,7 +3910,7 @@ async function startSubgraphOperationRunner(opts) {
3849
3910
  controller.abort("user-cancelled");
3850
3911
  }
3851
3912
  }).catch((err) => {
3852
- logger11.warn("Subgraph operation cancel poll failed", {
3913
+ logger12.warn("Subgraph operation cancel poll failed", {
3853
3914
  operationId: operation.id,
3854
3915
  error: getErrorMessage6(err)
3855
3916
  });
@@ -3866,14 +3927,14 @@ async function startSubgraphOperationRunner(opts) {
3866
3927
  const reason = String(controller.signal.reason ?? "");
3867
3928
  if (controller.signal.aborted && reason === "user-cancelled") {
3868
3929
  await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
3869
- logger11.info("Subgraph operation cancelled", {
3930
+ logger12.info("Subgraph operation cancelled", {
3870
3931
  operationId: operation.id,
3871
3932
  subgraph: operation.subgraph_name
3872
3933
  });
3873
3934
  return;
3874
3935
  }
3875
3936
  if (controller.signal.aborted) {
3876
- logger11.info("Subgraph operation interrupted", {
3937
+ logger12.info("Subgraph operation interrupted", {
3877
3938
  operationId: operation.id,
3878
3939
  subgraph: operation.subgraph_name,
3879
3940
  reason
@@ -3881,7 +3942,7 @@ async function startSubgraphOperationRunner(opts) {
3881
3942
  return;
3882
3943
  }
3883
3944
  await completeSubgraphOperation(db, operation.id, lockedBy, processed);
3884
- logger11.info("Subgraph operation completed", {
3945
+ logger12.info("Subgraph operation completed", {
3885
3946
  operationId: operation.id,
3886
3947
  subgraph: operation.subgraph_name,
3887
3948
  processed
@@ -3889,7 +3950,7 @@ async function startSubgraphOperationRunner(opts) {
3889
3950
  } catch (err) {
3890
3951
  const reason = String(controller.signal.reason ?? "");
3891
3952
  if (controller.signal.aborted && reason === "shutdown") {
3892
- logger11.info("Subgraph operation interrupted by shutdown", {
3953
+ logger12.info("Subgraph operation interrupted by shutdown", {
3893
3954
  operationId: operation.id,
3894
3955
  subgraph: operation.subgraph_name
3895
3956
  });
@@ -3900,7 +3961,7 @@ async function startSubgraphOperationRunner(opts) {
3900
3961
  return;
3901
3962
  }
3902
3963
  await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage6(err), processed);
3903
- logger11.error("Subgraph operation failed", {
3964
+ logger12.error("Subgraph operation failed", {
3904
3965
  operationId: operation.id,
3905
3966
  subgraph: operation.subgraph_name,
3906
3967
  error: getErrorMessage6(err)
@@ -3947,13 +4008,13 @@ async function startSubgraphOperationRunner(opts) {
3947
4008
  controller.abort("shutdown");
3948
4009
  }
3949
4010
  await Promise.allSettled(activeRuns.values());
3950
- logger11.info("Subgraph operation runner stopped");
4011
+ logger12.info("Subgraph operation runner stopped");
3951
4012
  };
3952
4013
  }
3953
4014
  async function startSubgraphProcessor(opts) {
3954
4015
  const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
3955
4016
  let running = true;
3956
- logger11.info("Starting subgraph processor", { concurrency });
4017
+ logger12.info("Starting subgraph processor", { concurrency });
3957
4018
  const stopOperations = await startSubgraphOperationRunner({
3958
4019
  concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
3959
4020
  });
@@ -3979,7 +4040,7 @@ async function startSubgraphProcessor(opts) {
3979
4040
  await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
3980
4041
  }
3981
4042
  } catch (err) {
3982
- logger11.error("Subgraph reorg handling failed", {
4043
+ logger12.error("Subgraph reorg handling failed", {
3983
4044
  error: getErrorMessage6(err)
3984
4045
  });
3985
4046
  }
@@ -3988,7 +4049,7 @@ async function startSubgraphProcessor(opts) {
3988
4049
  runCatchUp();
3989
4050
  }, POLL_INTERVAL_MS);
3990
4051
  const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
3991
- logger11.info("Subgraph processor ready");
4052
+ logger12.info("Subgraph processor ready");
3992
4053
  return async () => {
3993
4054
  running = false;
3994
4055
  clearInterval(pollInterval);
@@ -3997,21 +4058,21 @@ async function startSubgraphProcessor(opts) {
3997
4058
  await stopReorgListening();
3998
4059
  stopStreamsReorgPoll?.();
3999
4060
  await stopOperations();
4000
- logger11.info("Subgraph processor stopped");
4061
+ logger12.info("Subgraph processor stopped");
4001
4062
  };
4002
4063
  }
4003
4064
 
4004
4065
  // src/service.ts
4005
4066
  import { assertDbSplit, getDb } from "@secondlayer/shared/db";
4006
- import { logger as logger12 } from "@secondlayer/shared/logger";
4007
- import { sql as sql4 } from "kysely";
4067
+ import { logger as logger13 } from "@secondlayer/shared/logger";
4068
+ import { sql as sql5 } from "kysely";
4008
4069
  var HEARTBEAT_INTERVAL_MS2 = 30000;
4009
4070
  var SERVICE_NAME = "subgraph-processor";
4010
4071
  async function writeHeartbeat() {
4011
4072
  try {
4012
- await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql4`now()` })).execute();
4073
+ await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql5`now()` })).execute();
4013
4074
  } catch (err) {
4014
- logger12.warn("subgraph-processor heartbeat write failed", {
4075
+ logger13.warn("subgraph-processor heartbeat write failed", {
4015
4076
  error: err instanceof Error ? err.message : String(err)
4016
4077
  });
4017
4078
  }
@@ -4023,7 +4084,7 @@ var processor = await startSubgraphProcessor({
4023
4084
  await writeHeartbeat();
4024
4085
  var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
4025
4086
  var shutdown = async () => {
4026
- logger12.info("Shutting down subgraph processor...");
4087
+ logger13.info("Shutting down subgraph processor...");
4027
4088
  clearInterval(heartbeatInterval);
4028
4089
  await processor();
4029
4090
  process.exit(0);
@@ -4031,5 +4092,5 @@ var shutdown = async () => {
4031
4092
  process.on("SIGINT", shutdown);
4032
4093
  process.on("SIGTERM", shutdown);
4033
4094
 
4034
- //# debugId=12163624A2820BC464756E2164756E21
4095
+ //# debugId=77535A6AB51F4C4C64756E2164756E21
4035
4096
  //# sourceMappingURL=service.js.map