@secondlayer/subgraphs 3.19.2 → 3.19.4

package/dist/src/runtime/reorg.js

@@ -2238,20 +2238,309 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   return result;
 }
 
+// src/runtime/stats.ts
+var FLUSH_INTERVAL_BLOCKS = 100;
+var FLUSH_INTERVAL_MS = 60000;
+
+class StatsAccumulator {
+  subgraphName;
+  isCatchup;
+  current;
+  lastFlush = Date.now();
+  constructor(subgraphName, isCatchup) {
+    this.subgraphName = subgraphName;
+    this.isCatchup = isCatchup;
+    this.current = this.newEntry();
+  }
+  record(timing, opsCount) {
+    this.current.blocksProcessed++;
+    this.current.totalTimeMs += timing.totalMs;
+    this.current.handlerTimeMs += timing.handlerMs;
+    this.current.flushTimeMs += timing.flushMs;
+    this.current.maxBlockTimeMs = Math.max(this.current.maxBlockTimeMs, timing.totalMs);
+    this.current.maxHandlerTimeMs = Math.max(this.current.maxHandlerTimeMs, timing.handlerMs);
+    this.current.totalOps += opsCount;
+  }
+  shouldFlush() {
+    return this.current.blocksProcessed >= FLUSH_INTERVAL_BLOCKS || Date.now() - this.lastFlush >= FLUSH_INTERVAL_MS;
+  }
+  async flush(db) {
+    if (this.current.blocksProcessed === 0)
+      return;
+    const entry = this.current;
+    this.current = this.newEntry();
+    this.lastFlush = Date.now();
+    const avgOpsPerBlock = entry.blocksProcessed > 0 ? entry.totalOps / entry.blocksProcessed : 0;
+    await db.insertInto("subgraph_processing_stats").values({
+      subgraph_name: entry.subgraphName,
+      bucket_start: entry.bucketStart,
+      bucket_end: new Date,
+      blocks_processed: entry.blocksProcessed,
+      total_time_ms: Math.round(entry.totalTimeMs),
+      handler_time_ms: Math.round(entry.handlerTimeMs),
+      flush_time_ms: Math.round(entry.flushTimeMs),
+      max_block_time_ms: Math.round(entry.maxBlockTimeMs),
+      max_handler_time_ms: Math.round(entry.maxHandlerTimeMs),
+      avg_ops_per_block: Number.parseFloat(avgOpsPerBlock.toFixed(2)),
+      is_catchup: entry.isCatchup
+    }).execute();
+  }
+  newEntry() {
+    return {
+      subgraphName: this.subgraphName,
+      bucketStart: new Date,
+      blocksProcessed: 0,
+      totalTimeMs: 0,
+      handlerTimeMs: 0,
+      flushTimeMs: 0,
+      maxBlockTimeMs: 0,
+      maxHandlerTimeMs: 0,
+      totalOps: 0,
+      isCatchup: this.isCatchup
+    };
+  }
+}
+
+// src/runtime/catchup.ts
+import { getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
+import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
+import { logger as logger6 } from "@secondlayer/shared/logger";
+var LOG_INTERVAL = 1000;
+var STANDARD_CATCHUP_BATCH_CONFIG = {
+  defaultBatchSize: 500,
+  minBatchSize: 100,
+  maxBatchSize: 1000,
+  prefetch: true
+};
+var catchingUp = new Set;
+var subgraphBlockLockHeld = new Set;
+var subgraphBlockLockWaiters = new Map;
+var reorgEpoch = new Map;
+async function acquireSubgraphBlockLock(name) {
+  if (!subgraphBlockLockHeld.has(name)) {
+    subgraphBlockLockHeld.add(name);
+    return () => releaseSubgraphBlockLock(name);
+  }
+  return new Promise((resolve) => {
+    const waiters = subgraphBlockLockWaiters.get(name) ?? [];
+    waiters.push(() => resolve(() => releaseSubgraphBlockLock(name)));
+    subgraphBlockLockWaiters.set(name, waiters);
+  });
+}
+function releaseSubgraphBlockLock(name) {
+  const waiters = subgraphBlockLockWaiters.get(name);
+  if (waiters && waiters.length > 0) {
+    const next = waiters.shift();
+    if (waiters.length === 0)
+      subgraphBlockLockWaiters.delete(name);
+    next();
+    return;
+  }
+  subgraphBlockLockHeld.delete(name);
+}
+async function withSubgraphBlockLock(name, fn) {
+  const release = await acquireSubgraphBlockLock(name);
+  try {
+    return await fn();
+  } finally {
+    release();
+  }
+}
+function bumpReorgEpoch(name) {
+  reorgEpoch.set(name, (reorgEpoch.get(name) ?? 0) + 1);
+}
+function getReorgEpoch(name) {
+  return reorgEpoch.get(name) ?? 0;
+}
+
+class ReorgEpochChangedError extends Error {
+  constructor(name) {
+    super(`reorg epoch changed for ${name} mid catch-up`);
+    this.name = "ReorgEpochChangedError";
+  }
+}
+function parsePositiveInt(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+function parseBoolean(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "true")
+    return true;
+  if (normalized === "false")
+    return false;
+  return;
+}
+function resolveCatchupBatchConfig(env = process.env) {
+  const base = STANDARD_CATCHUP_BATCH_CONFIG;
+  const minBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MIN_BATCH_SIZE) ?? base.minBatchSize;
+  const maxBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MAX_BATCH_SIZE) ?? base.maxBatchSize;
+  const defaultBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_BATCH_SIZE) ?? base.defaultBatchSize;
+  return {
+    minBatchSize,
+    maxBatchSize,
+    defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize),
+    prefetch: parseBoolean(env.SUBGRAPH_CATCHUP_PREFETCH) ?? base.prefetch
+  };
+}
+function adjustBatchSize(current, avgEvents, config) {
+  if (avgEvents > 50)
+    return Math.max(Math.round(current * 0.5), config.minBatchSize);
+  if (avgEvents < 10)
+    return Math.min(Math.round(current * 1.5), config.maxBatchSize);
+  return current;
+}
+async function catchUpSubgraph(subgraph, subgraphName) {
+  if (catchingUp.has(subgraphName))
+    return 0;
+  catchingUp.add(subgraphName);
+  try {
+    const source = resolveBlockSource(subgraph);
+    const targetDb = getTargetDb2();
+    const subgraphRow = await getSubgraph(targetDb, subgraphName);
+    if (!subgraphRow)
+      return 0;
+    const lastProcessedBlock = Number(subgraphRow.last_processed_block);
+    const chainTip = await source.getTip();
+    if (chainTip <= 0 || lastProcessedBlock >= chainTip)
+      return 0;
+    const subgraphStart = Number(subgraphRow.start_block) || 1;
+    const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
+    const totalBlocks = chainTip - lastProcessedBlock;
+    logger6.info("Subgraph catch-up starting", {
+      subgraph: subgraphName,
+      from: startBlock,
+      to: chainTip,
+      blocks: totalBlocks
+    });
+    const stats = new StatsAccumulator(subgraphName, true);
+    let processed = 0;
+    const batchConfig = resolveCatchupBatchConfig();
+    let batchSize = batchConfig.defaultBatchSize;
+    let currentHeight = startBlock;
+    const walkEpoch = getReorgEpoch(subgraphName);
+    let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
+    let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
+    while (currentHeight <= chainTip) {
+      const currentRow = await getSubgraph(targetDb, subgraphName);
+      if (!currentRow || currentRow.status !== "active") {
+        logger6.info("Subgraph status changed, stopping catch-up", {
+          subgraph: subgraphName,
+          status: currentRow?.status ?? "deleted"
+        });
+        break;
+      }
+      let batchEnd;
+      let batch;
+      if (nextBatchPromise) {
+        batch = await nextBatchPromise;
+        batchEnd = prefetchedBatchEnd;
+        const nextStart = batchEnd + 1;
+        if (nextStart <= chainTip) {
+          prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
+          nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
+        } else {
+          nextBatchPromise = undefined;
+        }
+      } else {
+        batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
+        batch = await source.loadBlockRange(currentHeight, batchEnd);
+      }
+      let stopCatchup = false;
+      for (let height = currentHeight;height <= batchEnd; height++) {
+        let blockData = batch.get(height);
+        if (!blockData) {
+          blockData = (await source.loadBlockRange(height, height)).get(height);
+        }
+        if (!blockData) {
+          logger6.warn("Block missing during catch-up, deferring to next tick", {
+            subgraph: subgraphName,
+            blockHeight: height
+          });
+          stopCatchup = true;
+          break;
+        }
+        const preloaded = blockData;
+        let result;
+        try {
+          result = await withSubgraphBlockLock(subgraphName, async () => {
+            if (getReorgEpoch(subgraphName) !== walkEpoch) {
+              throw new ReorgEpochChangedError(subgraphName);
+            }
+            return processBlockWithRetry(subgraph, subgraphName, height, {
+              preloaded
+            });
+          });
+        } catch (err) {
+          if (err instanceof ReorgEpochChangedError) {
+            logger6.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
+            stopCatchup = true;
+            break;
+          }
+          const errorMsg = err instanceof Error ? err.message : String(err);
+          logger6.error("Block processing failed persistently during catch-up", {
+            subgraph: subgraphName,
+            blockHeight: height,
+            error: errorMsg
+          });
+          const { updateSubgraphStatus: updateSubgraphStatus2, recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
+          await recordSubgraphProcessed2(targetDb, subgraphName, 0, 1, `catch-up halted at block ${height}: ${errorMsg}`).catch(() => {});
+          await updateSubgraphStatus2(targetDb, subgraphName, "error").catch(() => {});
+          stopCatchup = true;
+          break;
+        }
+        processed++;
+        if (result.timing) {
+          stats.record(result.timing, result.processed);
+          if (stats.shouldFlush()) {
+            await stats.flush(targetDb);
+          }
+        }
+        if (processed % LOG_INTERVAL === 0) {
+          logger6.info("Subgraph catch-up progress", {
+            subgraph: subgraphName,
+            processed,
+            total: totalBlocks,
+            currentBlock: height,
+            pct: Math.round(processed / totalBlocks * 100)
+          });
+        }
+      }
+      if (stopCatchup)
+        break;
+      const avg = avgEventsPerBlock(batch);
+      batchSize = adjustBatchSize(batchSize, avg, batchConfig);
+      currentHeight = batchEnd + 1;
+    }
+    await stats.flush(targetDb);
+    logger6.info("Subgraph catch-up complete", {
+      subgraph: subgraphName,
+      processed
+    });
+    return processed;
+  } finally {
+    catchingUp.delete(subgraphName);
+  }
+}
+
 // src/runtime/reorg.ts
 import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
-import { getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
+import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
 import {
   listSubgraphs,
   resolveSubgraphRawClient
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger6 } from "@secondlayer/shared/logger";
+import { logger as logger7 } from "@secondlayer/shared/logger";
 async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
-  const targetDb = getTargetDb2();
+  const targetDb = getTargetDb3();
   const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
   if (activeSubgraphs.length === 0)
     return;
-  logger6.info("Propagating reorg to subgraphs", {
+  logger7.info("Propagating reorg to subgraphs", {
     blockHeight,
     subgraphCount: activeSubgraphs.length
   });
@@ -2260,88 +2549,97 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
       const schema = sg.definition.schema;
       if (!schema)
         continue;
-      const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
-      const client = resolveSubgraphRawClient(sg);
-      const tableNames = Object.keys(schema);
-      const revertedByTable = {};
-      for (const tableName of tableNames) {
-        const rows = await client.unsafe(`SELECT * FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1 LIMIT 1000`, [blockHeight]);
-        if (rows.length > 0)
-          revertedByTable[tableName] = rows;
-      }
-      const hasJournal = (await client.unsafe(`SELECT to_regclass('"${schemaName}"."_journal"') AS r`))[0]?.r;
-      if (hasJournal) {
-        await client.begin(async (tx) => {
+      bumpReorgEpoch(sg.name);
+      await withSubgraphBlockLock(sg.name, async () => {
+        const schemaName = sg.schema_name ?? pgSchemaName(sg.name);
+        const client = resolveSubgraphRawClient(sg);
+        const tableNames = Object.keys(schema);
+        const revertedByTable = {};
+        for (const tableName of tableNames) {
+          const rows = await client.unsafe(`SELECT * FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1 LIMIT 1000`, [blockHeight]);
+          if (rows.length > 0)
+            revertedByTable[tableName] = rows;
+        }
+        const hasJournal = (await client.unsafe(`SELECT to_regclass('"${schemaName}"."_journal"') AS r`))[0]?.r;
+        if (hasJournal) {
+          await client.begin(async (tx) => {
+            for (const tableName of tableNames) {
+              const earliest = `
+								SELECT DISTINCT ON (row_key) row_key, prev_row
+								FROM "${schemaName}"."_journal"
+								WHERE block_height >= $1 AND table_name = $2
+								ORDER BY row_key, _jid ASC`;
+              await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" t USING (${earliest}) e WHERE to_jsonb(t.*) @> e.row_key`, [blockHeight, tableName]);
+              await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
+              await tx.unsafe(`INSERT INTO "${schemaName}"."${tableName}"
+								 SELECT r.* FROM (${earliest}) e
+								 CROSS JOIN LATERAL jsonb_populate_record(NULL::"${schemaName}"."${tableName}", e.prev_row) r
+								 WHERE e.prev_row IS NOT NULL`, [blockHeight, tableName]);
+            }
+            await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
+          });
+        } else {
+          logger7.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
           for (const tableName of tableNames) {
-            const earliest = `
-							SELECT DISTINCT ON (row_key) row_key, prev_row
-							FROM "${schemaName}"."_journal"
-							WHERE block_height >= $1 AND table_name = $2
-							ORDER BY row_key, _jid ASC`;
-            await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" t USING (${earliest}) e WHERE to_jsonb(t.*) @> e.row_key`, [blockHeight, tableName]);
-            await tx.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
-            await tx.unsafe(`INSERT INTO "${schemaName}"."${tableName}"
-							 SELECT r.* FROM (${earliest}) e
-							 CROSS JOIN LATERAL jsonb_populate_record(NULL::"${schemaName}"."${tableName}", e.prev_row) r
-							 WHERE e.prev_row IS NOT NULL`, [blockHeight, tableName]);
+            await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
           }
-          await tx.unsafe(`DELETE FROM "${schemaName}"."_journal" WHERE block_height >= $1`, [blockHeight]);
-        });
-      } else {
-        logger6.warn("Subgraph has no revert journal — falling back to height delete (accumulator rows may lose history)", { subgraph: sg.name, blockHeight });
-        for (const tableName of tableNames) {
-          await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
         }
-      }
-      for (const [tableName, rows] of Object.entries(revertedByTable)) {
-        if (rows.length === 0)
-          continue;
-        await targetDb.insertInto("subscription_outbox").columns([
-          "subscription_id",
-          "event_type",
-          "payload",
-          "dedup_key",
-          "row_pk",
-          "status"
-        ]).expression((eb) => eb.selectFrom("subscriptions").select((eb2) => [
-          "id as subscription_id",
-          eb2.val(`${sg.name}.${tableName}.reverted`).as("event_type"),
-          eb2.val(JSON.stringify({
-            subgraph: sg.name,
-            table: tableName,
-            reorgFromHeight: blockHeight,
-            rows
-          })).as("payload"),
-          eb2.val(`reorg:${sg.name}:${tableName}:${blockHeight}`).as("dedup_key"),
-          eb2.val({
-            subgraph: sg.name,
-            table: tableName,
-            reorgRoot: blockHeight
-          }).as("row_pk"),
-          eb2.val("pending").as("status")
-        ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.column("dedup_key").doNothing()).execute().catch((err) => {
-          logger6.error("Failed to emit revert event for subscriptions", {
-            event: "reorg_revert_emit_dropped",
-            subgraph: sg.name,
-            table: tableName,
-            blockHeight,
-            error: getErrorMessage2(err)
+        for (const [tableName, rows] of Object.entries(revertedByTable)) {
+          if (rows.length === 0)
+            continue;
+          await targetDb.insertInto("subscription_outbox").columns([
+            "subscription_id",
+            "subgraph_name",
+            "table_name",
+            "block_height",
+            "event_type",
+            "payload",
+            "dedup_key",
+            "row_pk",
+            "status"
+          ]).expression((eb) => eb.selectFrom("subscriptions").select((eb2) => [
+            "id as subscription_id",
+            eb2.val(sg.name).as("subgraph_name"),
+            eb2.val(tableName).as("table_name"),
+            eb2.val(blockHeight).as("block_height"),
+            eb2.val(`${sg.name}.${tableName}.reverted`).as("event_type"),
+            eb2.val(JSON.stringify({
+              subgraph: sg.name,
+              table: tableName,
+              reorgFromHeight: blockHeight,
+              rows
+            })).as("payload"),
+            eb2.val(`reorg:${sg.name}:${tableName}:${blockHeight}`).as("dedup_key"),
+            eb2.val({
+              subgraph: sg.name,
+              table: tableName,
+              reorgRoot: blockHeight
+            }).as("row_pk"),
+            eb2.val("pending").as("status")
+          ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute().catch((err) => {
+            logger7.error("Failed to emit revert event for subscriptions", {
+              event: "reorg_revert_emit_dropped",
+              subgraph: sg.name,
+              table: tableName,
+              blockHeight,
+              error: getErrorMessage2(err)
+            });
           });
+        }
+        logger7.info("Subgraph reorg cleanup done", {
+          subgraph: sg.name,
+          blockHeight,
+          tablesAffected: Object.keys(revertedByTable).length
+        });
+        const def = await loadSubgraphDef(sg);
+        await processBlock(def, sg.name, blockHeight);
+        logger7.info("Subgraph reorg reprocessed", {
+          subgraph: sg.name,
+          blockHeight
         });
-      }
-      logger6.info("Subgraph reorg cleanup done", {
-        subgraph: sg.name,
-        blockHeight,
-        tablesAffected: Object.keys(revertedByTable).length
-      });
-      const def = await loadSubgraphDef(sg);
-      await processBlock(def, sg.name, blockHeight);
-      logger6.info("Subgraph reorg reprocessed", {
-        subgraph: sg.name,
-        blockHeight
       });
     } catch (err) {
-      logger6.error("Subgraph reorg handling failed", {
+      logger7.error("Subgraph reorg handling failed", {
         subgraph: sg.name,
         blockHeight,
         error: getErrorMessage2(err)
@@ -2353,5 +2651,5 @@ export {
   handleSubgraphReorg
 };
 
-//# debugId=9AC73085D64D8A9364756E2164756E21
+//# debugId=283CF7962DBBE8DC64756E2164756E21
 //# sourceMappingURL=reorg.js.map