@secondlayer/subgraphs 3.23.2 → 3.24.0

package/dist/src/runtime/reorg.js

@@ -1627,9 +1627,11 @@ import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contract
 import { advanceOperationCursor } from "@secondlayer/shared/db/queries/subgraph-operations";
 import {
   isByoSubgraph,
+  recordLiveError,
   recordLiveProgress,
   recordSubgraphProcessed,
   resolveSubgraphDb,
+  rewindLiveProgress,
   updateSubgraphStatus
 } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger6 } from "@secondlayer/shared/logger";
@@ -2337,10 +2339,20 @@ class CursorRaceLostError extends Error {
     this.name = "CursorRaceLostError";
   }
 }
+
+class LiveCursorRaceLostError extends Error {
+  constructor(subgraphName, height) {
+    super(`live cursor race lost for ${subgraphName} at block ${height}`);
+    this.name = "LiveCursorRaceLostError";
+  }
+}
 function opCursorMode(opts) {
   const ap = opts?.atomicProgress;
   return ap && "operationId" in ap ? ap : undefined;
 }
+function isReorgRewind(opts) {
+  return opts?.reorgRewind === true;
+}
 function statusMode(opts) {
   const ap = opts?.atomicProgress;
   return ap && "status" in ap ? ap : undefined;
@@ -2410,7 +2422,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   result.matched = matched.length;
   if (matched.length === 0) {
     if (!opts?.skipProgressUpdate) {
-      await recordLiveProgress(targetDb, subgraphName, blockHeight);
+      if (isReorgRewind(opts)) {
+        await rewindLiveProgress(targetDb, subgraphName, blockHeight);
+      } else {
+        await recordLiveProgress(targetDb, subgraphName, blockHeight);
+      }
     }
     return result;
   }
@@ -2429,13 +2445,23 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   };
   let handlerMs = 0;
   let flushMs = 0;
-  const applyProgress = async (tx, rr) => {
+  const applyProgress = async (tx, rr, flushedWrites) => {
     if (opts?.skipProgressUpdate)
       return;
+    const rewind = isReorgRewind(opts);
     if (rr.errors > 0 && rr.processed === 0) {
-      await updateSubgraphStatus(tx, subgraphName, "error", blockHeight);
+      if (rewind) {
+        await updateSubgraphStatus(tx, subgraphName, "error", blockHeight);
+      } else {
+        await recordLiveError(tx, subgraphName, blockHeight);
+      }
+    } else if (rewind) {
+      await rewindLiveProgress(tx, subgraphName, blockHeight);
     } else {
-      await recordLiveProgress(tx, subgraphName, blockHeight);
+      const advanced = await recordLiveProgress(tx, subgraphName, blockHeight);
+      if (!advanced && flushedWrites && !route.byo) {
+        throw new LiveCursorRaceLostError(subgraphName, blockHeight);
+      }
     }
     if (rr.processed > 0 || rr.errors > 0) {
       const lastError = rr.errors > 0 ? `${rr.errors} error(s) at block ${blockHeight}` : undefined;
@@ -2484,13 +2510,15 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
       } else if (byoOm && manifest && manifest.count > 0) {
         await advanceOperationCursor(tx, byoOm.operationId, blockHeight);
       }
-      await applyProgress(tx, runResult);
+      await applyProgress(tx, runResult, manifest ? manifest.count > 0 : false);
     });
   } else {
     try {
       await targetDb.transaction().execute(async (tx) => {
         const opMode = opCursorMode(opts);
-        if (statusMode(opts)) {
+        const sm = statusMode(opts);
+        const rewind = isReorgRewind(opts);
+        if (sm) {
           const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
           if (row && Number(row.last_processed_block) >= blockHeight) {
             result.skipped = true;
@@ -2502,6 +2530,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
             result.skipped = true;
             return;
           }
+        } else if (!rewind) {
+          const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
+          if (row && Number(row.last_processed_block) >= blockHeight) {
+            result.skipped = true;
+            return;
+          }
         }
         const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
         const handlerStart = performance.now();
@@ -2519,7 +2553,6 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
           }
           flushMs = performance.now() - flushStart;
         }
-        const sm = statusMode(opts);
         if (sm && flushedWrites) {
           await updateSubgraphStatus(tx, subgraphName, sm.status, blockHeight);
         } else if (opMode && flushedWrites) {
@@ -2528,10 +2561,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
             throw new CursorRaceLostError(opMode.operationId, blockHeight);
           }
         }
-        await applyProgress(tx, runResult);
+        await applyProgress(tx, runResult, flushedWrites);
       });
     } catch (err) {
-      if (err instanceof CursorRaceLostError) {
+      if (err instanceof CursorRaceLostError || err instanceof LiveCursorRaceLostError) {
         logger6.warn("cursor race lost — block already covered", {
           subgraph: subgraphName,
           blockHeight,
@@ -2544,11 +2577,13 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     }
   }
   const totalMs = performance.now() - blockStart;
-  result.timing = {
-    totalMs: Math.round(totalMs),
-    handlerMs: Math.round(handlerMs),
-    flushMs: Math.round(flushMs)
-  };
+  if (!result.skipped) {
+    result.timing = {
+      totalMs: Math.round(totalMs),
+      handlerMs: Math.round(handlerMs),
+      flushMs: Math.round(flushMs)
+    };
+  }
   if (blockHeight % 1000 === 0) {
     try {
       const tables = Object.keys(subgraph.schema);
@@ -2643,6 +2678,33 @@ class StatsAccumulator {
 import { getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
 import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger7 } from "@secondlayer/shared/logger";
+
+// src/runtime/catchup-leader.ts
+import {
+  SUBGRAPH_CATCHUP_LOCK_KEY,
+  createPostgresLeaderBackend,
+  withLeaderLock
+} from "@secondlayer/shared/leader";
+import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
+var catchUpLeader = false;
+function isCatchUpLeader() {
+  return catchUpLeader;
+}
+function startCatchUpLeader(opts = {}) {
+  return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
+    catchUpLeader = true;
+    await opts.onAcquire?.();
+    return () => {
+      catchUpLeader = false;
+    };
+  }, {
+    pollMs: opts.pollMs,
+    heartbeatMs: opts.heartbeatMs,
+    createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
+  });
+}
+
+// src/runtime/catchup.ts
 var LOG_INTERVAL = 1000;
 var STANDARD_CATCHUP_BATCH_CONFIG = {
   defaultBatchSize: 500,
@@ -2764,6 +2826,13 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
     let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
     while (currentHeight <= chainTip) {
+      if (!isCatchUpLeader()) {
+        logger7.info("Lost catch-up leadership, stopping catch-up", {
+          subgraph: subgraphName,
+          processed
+        });
+        break;
+      }
       const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
         logger7.info("Subgraph status changed, stopping catch-up", {
@@ -2873,6 +2942,7 @@ import {
   resolveSubgraphRawClient
 } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger8 } from "@secondlayer/shared/logger";
+import { sql as sql4 } from "kysely";
 async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
   const targetDb = getTargetDb3();
   const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
@@ -2889,96 +2959,9 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
         continue;
       bumpReorgEpoch(sg.name);
       await withSubgraphBlockLock(sg.name, async () => {
-        const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
-        const client = resolveSubgraphRawClient(sg);
-        const tableNames = Object.keys(schema);
-        const revertedByTable = {};
-        for (const tableName of tableNames) {
-          const rows = await client.unsafe(`SELECT * FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1 LIMIT 1000`, [blockHeight]);
-          if (rows.length > 0)
-            revertedByTable[tableName] = rows;
-        }
-        const hasJournal = (await client.unsafe(`SELECT to_regclass('"${schemaName}"."_journal"') AS r`))[0]?.r;
-        if (hasJournal) {
-          await client.begin(async (tx) => {
-            for (const tableName of tableNames) {
-              const earliest = `
-								SELECT DISTINCT ON (row_key) row_key, prev_row
-								FROM "${schemaName}"."_journal"
-								WHERE block_height >= $1 AND table_name = $2
-								ORDER BY row_key, _jid ASC`;
-              await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" t USING (${earliest}) e WHERE to_jsonb(t.*) @> e.row_key`, [blockHeight, tableName]);
-              await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
-              await tx.unsafe(`INSERT INTO "${schemaName}"."${tableName}"
-								 SELECT r.* FROM (${earliest}) e
-								 CROSS JOIN LATERAL jsonb_populate_record(NULL::"${schemaName}"."${tableName}", e.prev_row) r
-								 WHERE e.prev_row IS NOT NULL`, [blockHeight, tableName]);
-            }
-            await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
-          });
-        } else {
-          logger8.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
-          for (const tableName of tableNames) {
-            await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
-          }
-        }
-        await client.unsafe(`DO $$ BEGIN
-						IF to_regclass('"${schemaName}"."_factory_addresses"') IS NOT NULL THEN
-							DELETE FROM "${schemaName}"."_factory_addresses" WHERE block_height >= ${Number(blockHeight)};
-						END IF;
-					END $$`);
-        for (const [tableName, rows] of Object.entries(revertedByTable)) {
-          if (rows.length === 0)
-            continue;
-          await targetDb.insertInto("subscription_outbox").columns([
-            "subscription_id",
-            "subgraph_name",
-            "table_name",
-            "block_height",
-            "event_type",
-            "payload",
-            "dedup_key",
-            "row_pk",
-            "status"
-          ]).expression((eb) => eb.selectFrom("subscriptions").select((eb2) => [
-            "id as subscription_id",
-            eb2.val(sg.name).as("subgraph_name"),
-            eb2.val(tableName).as("table_name"),
-            eb2.val(blockHeight).as("block_height"),
-            eb2.val(`${sg.name}.${tableName}.reverted`).as("event_type"),
-            eb2.val(JSON.stringify({
-              subgraph: sg.name,
-              table: tableName,
-              reorgFromHeight: blockHeight,
-              rows
-            })).as("payload"),
-            eb2.val(`reorg:${sg.name}:${tableName}:${blockHeight}`).as("dedup_key"),
-            eb2.val({
-              subgraph: sg.name,
-              table: tableName,
-              reorgRoot: blockHeight
-            }).as("row_pk"),
-            eb2.val("pending").as("status")
-          ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
-            logger8.error("Failed to emit revert event for subscriptions", {
-              event: "reorg_revert_emit_dropped",
-              subgraph: sg.name,
-              table: tableName,
-              blockHeight,
-              error: getErrorMessage2(err)
-            });
-          });
-        }
-        logger8.info("Subgraph reorg cleanup done", {
-          subgraph: sg.name,
-          blockHeight,
-          tablesAffected: Object.keys(revertedByTable).length
-        });
-        const def = await loadSubgraphDef(sg);
-        await processBlock(def, sg.name, blockHeight);
-        logger8.info("Subgraph reorg reprocessed", {
-          subgraph: sg.name,
-          blockHeight
+        await getTargetDb3().transaction().execute(async (lockTx) => {
+          await sql4`SELECT pg_advisory_xact_lock(hashtext(${`subgraph-reorg:${sg.name}`}))`.execute(lockTx);
+          await reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef);
         });
       });
     } catch (err) {
@@ -2990,9 +2973,103 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
     }
   }
 }
+async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
+  const targetDb = getTargetDb3();
+  const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
+  const client = resolveSubgraphRawClient(sg);
+  const tableNames = Object.keys(schema);
+  const revertedByTable = {};
+  for (const tableName of tableNames) {
+    const rows = await client.unsafe(`SELECT * FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1 LIMIT 1000`, [blockHeight]);
+    if (rows.length > 0)
+      revertedByTable[tableName] = rows;
+  }
+  const hasJournal = (await client.unsafe(`SELECT to_regclass('"${schemaName}"."_journal"') AS r`))[0]?.r;
+  if (hasJournal) {
+    await client.begin(async (tx) => {
+      for (const tableName of tableNames) {
+        const earliest = `
+					SELECT DISTINCT ON (row_key) row_key, prev_row
+					FROM "${schemaName}"."_journal"
+					WHERE block_height >= $1 AND table_name = $2
+					ORDER BY row_key, _jid ASC`;
+        await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" t USING (${earliest}) e WHERE to_jsonb(t.*) @> e.row_key`, [blockHeight, tableName]);
+        await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
+        await tx.unsafe(`INSERT INTO "${schemaName}"."${tableName}"
+					 SELECT r.* FROM (${earliest}) e
+					 CROSS JOIN LATERAL jsonb_populate_record(NULL::"${schemaName}"."${tableName}", e.prev_row) r
+					 WHERE e.prev_row IS NOT NULL`, [blockHeight, tableName]);
+      }
+      await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
+    });
+  } else {
+    logger8.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
+    for (const tableName of tableNames) {
+      await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
+    }
+  }
+  await client.unsafe(`DO $$ BEGIN
+			IF to_regclass('"${schemaName}"."_factory_addresses"') IS NOT NULL THEN
+				DELETE FROM "${schemaName}"."_factory_addresses" WHERE block_height >= ${Number(blockHeight)};
+			END IF;
+		END $$`);
+  for (const [tableName, rows] of Object.entries(revertedByTable)) {
+    if (rows.length === 0)
+      continue;
+    await targetDb.insertInto("subscription_outbox").columns([
+      "subscription_id",
+      "subgraph_name",
+      "table_name",
+      "block_height",
+      "event_type",
+      "payload",
+      "dedup_key",
+      "row_pk",
+      "status"
+    ]).expression((eb) => eb.selectFrom("subscriptions").select((eb2) => [
+      "id as subscription_id",
+      eb2.val(sg.name).as("subgraph_name"),
+      eb2.val(tableName).as("table_name"),
+      eb2.val(blockHeight).as("block_height"),
+      eb2.val(`${sg.name}.${tableName}.reverted`).as("event_type"),
+      eb2.val(JSON.stringify({
+        subgraph: sg.name,
+        table: tableName,
+        reorgFromHeight: blockHeight,
+        rows
+      })).as("payload"),
+      eb2.val(`reorg:${sg.name}:${tableName}:${blockHeight}`).as("dedup_key"),
+      eb2.val({
+        subgraph: sg.name,
+        table: tableName,
+        reorgRoot: blockHeight
+      }).as("row_pk"),
+      eb2.val("pending").as("status")
+    ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
+      logger8.error("Failed to emit revert event for subscriptions", {
+        event: "reorg_revert_emit_dropped",
+        subgraph: sg.name,
+        table: tableName,
+        blockHeight,
+        error: getErrorMessage2(err)
+      });
+    });
+  }
+  logger8.info("Subgraph reorg cleanup done", {
+    subgraph: sg.name,
+    blockHeight,
+    tablesAffected: Object.keys(revertedByTable).length
+  });
+  const def = await loadSubgraphDef(sg);
+  await processBlock(def, sg.name, blockHeight, { reorgRewind: true });
+  logger8.info("Subgraph reorg reprocessed", {
+    subgraph: sg.name,
+    blockHeight
+  });
+}
 export {
   handleSubgraphReorg
 };
 
-//# debugId=33E820804CBE066864756E2164756E21
+//# debugId=314C75DB8B9D518F64756E2164756E21
 //# sourceMappingURL=reorg.js.map