@secondlayer/subgraphs 1.2.2 → 1.3.1

package/dist/src/runtime/processor.js

@@ -1299,14 +1299,103 @@ class StatsAccumulator {
   }
 }
 
-// src/runtime/catchup.ts
-import { getSourceDb as getSourceDb2, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
+// src/runtime/reindex.ts
+import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
+import { getRawClient, getSourceDb as getSourceDb2, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
 import {
-  recordGapBatch
+  recordGapBatch,
+  resolveGaps
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
-import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
+import {
+  recordSubgraphProcessed as recordSubgraphProcessed2,
+  updateSubgraphStatus as updateSubgraphStatus2
+} from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger5 } from "@secondlayer/shared/logger";
 
+// src/schema/generator.ts
+var TYPE_MAP = {
+  text: "TEXT",
+  uint: "NUMERIC",
+  int: "NUMERIC",
+  principal: "TEXT",
+  boolean: "BOOLEAN",
+  timestamp: "TIMESTAMPTZ",
+  jsonb: "JSONB"
+};
+function escapeLiteralDefault(value) {
+  if (value === null || value === undefined)
+    return "NULL";
+  if (typeof value === "number" || typeof value === "bigint")
+    return String(value);
+  if (typeof value === "boolean")
+    return value ? "TRUE" : "FALSE";
+  return `'${String(value).replace(/'/g, "''")}'`;
+}
+function generateSubgraphSQL(def, schemaNameOverride) {
+  const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
+  const statements = [];
+  const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
+  if (needsTrgm) {
+    statements.push(`CREATE EXTENSION IF NOT EXISTS pg_trgm`);
+  }
+  statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
+  for (const [tableName, tableDef] of Object.entries(def.schema)) {
+    const qualifiedName = `${schemaName}.${tableName}`;
+    const columnDefs = [
+      `_id BIGSERIAL PRIMARY KEY`,
+      `_block_height BIGINT NOT NULL`,
+      `_tx_id TEXT NOT NULL`,
+      `_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()`
+    ];
+    for (const [colName, col] of Object.entries(tableDef.columns)) {
+      const sqlType = TYPE_MAP[col.type];
+      const nullable = col.nullable ? "" : " NOT NULL";
+      let colDef = `${colName} ${sqlType}${nullable}`;
+      if (col.default !== undefined) {
+        colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
+      }
+      columnDefs.push(colDef);
+    }
+    statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
+  ${columnDefs.join(`,
+  `)}
+)`);
+    statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
+    statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
+    for (const [colName, col] of Object.entries(tableDef.columns)) {
+      if (col.indexed) {
+        statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
+      }
+    }
+    for (const [colName, col] of Object.entries(tableDef.columns)) {
+      if (col.search) {
+        statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
+      }
+    }
+    if (tableDef.indexes) {
+      for (let i = 0;i < tableDef.indexes.length; i++) {
+        const cols = tableDef.indexes[i];
+        const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
+        statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
+      }
+    }
+    if (tableDef.uniqueKeys) {
+      for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
+        const cols = tableDef.uniqueKeys[i];
+        const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
+        statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
+      }
+    }
+  }
+  const hashInput = JSON.stringify({
+    name: def.name,
+    schema: def.schema,
+    sources: def.sources
+  }, (_key, value) => typeof value === "bigint" ? value.toString() : value);
+  const hash = String(Bun.hash(hashInput));
+  return { statements, hash };
+}
+
 // src/runtime/batch-loader.ts
 async function loadBlockRange(db, fromHeight, toHeight) {
   const [blocks, txs, events] = await Promise.all([
@@ -1349,8 +1438,417 @@ function avgEventsPerBlock(batch) {
   return totalEvents / batch.size;
 }
 
-// src/runtime/catchup.ts
+// src/runtime/reindex.ts
 var LOG_INTERVAL = 1000;
+var HEALTH_FLUSH_INTERVAL = 1000;
+var HOBBY_REINDEX_BATCH_CONFIG = {
+  defaultBatchSize: 50,
+  minBatchSize: 25,
+  maxBatchSize: 100
+};
+var STANDARD_REINDEX_BATCH_CONFIG = {
+  defaultBatchSize: 500,
+  minBatchSize: 100,
+  maxBatchSize: 1000
+};
+function initialReindexProgressBlock(fromBlock) {
+  return Math.max(0, fromBlock - 1);
+}
+function resolveReindexResumeBlock(row) {
+  if (row.reindex_from_block == null || row.reindex_to_block == null) {
+    return null;
+  }
+  const lastProcessedBlock = Number(row.last_processed_block);
+  const reindexFromBlock = Number(row.reindex_from_block);
+  return Math.max(lastProcessedBlock + 1, reindexFromBlock);
+}
+function parsePositiveInt(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+function resolveReindexBatchConfig(env = process.env) {
+  const base = env.TENANT_PLAN?.trim().toLowerCase() === "hobby" ? HOBBY_REINDEX_BATCH_CONFIG : STANDARD_REINDEX_BATCH_CONFIG;
+  const minBatchSize = parsePositiveInt(env.SUBGRAPH_REINDEX_MIN_BATCH_SIZE) ?? base.minBatchSize;
+  const maxBatchSize = parsePositiveInt(env.SUBGRAPH_REINDEX_MAX_BATCH_SIZE) ?? base.maxBatchSize;
+  const defaultBatchSize = parsePositiveInt(env.SUBGRAPH_REINDEX_BATCH_SIZE) ?? base.defaultBatchSize;
+  return {
+    minBatchSize,
+    maxBatchSize,
+    defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize)
+  };
+}
+function coalesceFailedBlocks(blocks) {
+  if (blocks.length === 0)
+    return [];
+  blocks.sort((a, b) => a.height - b.height);
+  const ranges = [];
+  let start = blocks[0].height;
+  let end = blocks[0].height;
+  let reason = blocks[0].reason;
+  for (let i = 1;i < blocks.length; i++) {
+    const b = blocks[i];
+    if (b.height === end + 1 && b.reason === reason) {
+      end = b.height;
+    } else {
+      ranges.push({ start, end, reason });
+      start = b.height;
+      end = b.height;
+      reason = b.reason;
+    }
+  }
+  ranges.push({ start, end, reason });
+  return ranges;
+}
+async function processBlockRange(def, opts) {
+  const sourceDb = getSourceDb2();
+  const targetDb = getTargetDb2();
+  const subgraphName = def.name;
+  const { fromBlock, toBlock, status } = opts;
+  const totalBlocks = toBlock - fromBlock + 1;
+  const stats = new StatsAccumulator(subgraphName, opts.isCatchup);
+  let blocksProcessed = 0;
+  let totalEventsProcessed = 0;
+  let totalErrors = 0;
+  let pendingEventsProcessed = 0;
+  let pendingErrors = 0;
+  let pendingLastError;
+  let lastHealthFlushBlock = 0;
+  const batchConfig = resolveReindexBatchConfig();
+  let batchSize = batchConfig.defaultBatchSize;
+  let currentHeight = fromBlock;
+  let aborted = false;
+  const flushHealth = async () => {
+    if (pendingEventsProcessed === 0 && pendingErrors === 0)
+      return;
+    await recordSubgraphProcessed2(targetDb, subgraphName, pendingEventsProcessed, pendingErrors, pendingLastError);
+    pendingEventsProcessed = 0;
+    pendingErrors = 0;
+    pendingLastError = undefined;
+    lastHealthFlushBlock = blocksProcessed;
+  };
+  let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
+  let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
+  while (currentHeight <= toBlock) {
+    if (opts.signal?.aborted) {
+      aborted = true;
+      logger5.info("Block processing aborted", {
+        subgraph: subgraphName,
+        currentBlock: currentHeight,
+        reason: String(opts.signal.reason ?? "unknown")
+      });
+      break;
+    }
+    const batch = await nextBatchPromise;
+    const batchEnd = nextBatchEnd;
+    const nextStart = batchEnd + 1;
+    if (nextStart <= toBlock) {
+      nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
+      nextBatchPromise = loadBlockRange(sourceDb, nextStart, nextBatchEnd);
+    }
+    const batchFailedBlocks = [];
+    for (let height = currentHeight;height <= batchEnd; height++) {
+      const blockData = batch.get(height);
+      if (!blockData) {
+        batchFailedBlocks.push({ height, reason: "block_missing" });
+        blocksProcessed++;
+        continue;
+      }
+      let result;
+      try {
+        result = await processBlock(def, subgraphName, height, {
+          skipProgressUpdate: true,
+          preloaded: blockData
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        logger5.error("Block processing error", {
+          subgraph: subgraphName,
+          blockHeight: height,
+          error: errorMsg
+        });
+        batchFailedBlocks.push({ height, reason: "processing_error" });
+        await updateSubgraphStatus2(targetDb, subgraphName, status, height).catch(() => {});
+        blocksProcessed++;
+        totalErrors++;
+        pendingErrors++;
+        pendingLastError = errorMsg;
+        continue;
+      }
+      blocksProcessed++;
+      totalEventsProcessed += result.processed;
+      totalErrors += result.errors;
+      pendingEventsProcessed += result.processed;
+      pendingErrors += result.errors;
+      if (result.errors > 0) {
+        pendingLastError = `${result.errors} error(s) while processing block ${height}`;
+      }
+      if (result.timing) {
+        stats.record(result.timing, result.processed);
+        if (stats.shouldFlush()) {
+          await stats.flush(targetDb);
+        }
+      }
+      if (blocksProcessed % 100 === 0) {
+        await updateSubgraphStatus2(targetDb, subgraphName, status, height);
+      }
+      if (blocksProcessed % LOG_INTERVAL === 0) {
+        logger5.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
+          subgraph: subgraphName,
+          processed: blocksProcessed,
+          total: totalBlocks,
+          currentBlock: height,
+          pct: Math.round(blocksProcessed / totalBlocks * 100)
+        });
+      }
+    }
+    if (blocksProcessed - lastHealthFlushBlock >= HEALTH_FLUSH_INTERVAL) {
+      await flushHealth();
+    }
+    if (batchFailedBlocks.length > 0 && opts.subgraphId) {
+      const gaps = coalesceFailedBlocks(batchFailedBlocks);
+      await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
+        logger5.warn("Failed to record subgraph gaps", {
+          subgraph: subgraphName,
+          error: err instanceof Error ? err.message : String(err)
+        });
+      });
+    }
+    const avg = avgEventsPerBlock(batch);
+    if (avg > 50)
+      batchSize = Math.max(Math.round(batchSize * 0.5), batchConfig.minBatchSize);
+    else if (avg < 10)
+      batchSize = Math.min(Math.round(batchSize * 1.5), batchConfig.maxBatchSize);
+    currentHeight = batchEnd + 1;
+  }
+  await stats.flush(targetDb);
+  await flushHealth();
+  return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
+}
+async function resolveBlockRange(sourceDb, def, opts) {
+  const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
+  let toBlock;
+  if (opts?.toBlock != null) {
+    toBlock = opts.toBlock;
+  } else {
+    const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    toBlock = progress?.highest_seen_block ?? 0;
+  }
+  return { fromBlock, toBlock };
+}
+async function clearReindexMetadata(db, subgraphName) {
+  await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
+}
+async function reindexSubgraph(def, opts) {
+  const sourceDb = getSourceDb2();
+  const targetDb = getTargetDb2();
+  const client = getRawClient("target");
+  const subgraphName = def.name;
+  const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
+  await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
+  logger5.info("Reindex starting", { subgraph: subgraphName });
+  try {
+    const { fromBlock, toBlock } = await resolveBlockRange(sourceDb, def, opts);
+    if (fromBlock > toBlock) {
+      logger5.info("No blocks to reindex", {
+        subgraph: subgraphName,
+        fromBlock,
+        toBlock
+      });
+      await updateSubgraphStatus2(targetDb, subgraphName, "active", 0);
+      return { processed: 0 };
+    }
+    await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
+    const { statements } = generateSubgraphSQL(def, schemaName);
+    for (const stmt of statements) {
+      await client.unsafe(stmt);
+    }
+    logger5.info("Schema recreated for reindex", { subgraph: subgraphName });
+    await targetDb.updateTable("subgraphs").set({
+      last_processed_block: initialReindexProgressBlock(fromBlock),
+      reindex_from_block: fromBlock,
+      reindex_to_block: toBlock,
+      total_processed: 0,
+      total_errors: 0,
+      last_error: null,
+      last_error_at: null,
+      updated_at: new Date
+    }).where("name", "=", subgraphName).execute();
+    logger5.info("Reindexing blocks", {
+      subgraph: subgraphName,
+      fromBlock,
+      toBlock,
+      totalBlocks: toBlock - fromBlock + 1
+    });
+    const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
+    const result = await processBlockRange(def, {
+      fromBlock,
+      toBlock,
+      status: "reindexing",
+      isCatchup: false,
+      apiKeyId: null,
+      subgraphId: subgraphRow?.id,
+      signal: opts?.signal
+    });
+    if (result.aborted) {
+      const reason = String(opts?.signal?.reason ?? "unknown");
+      if (reason === "user-cancelled") {
+        await updateSubgraphStatus2(targetDb, subgraphName, "active");
+        await clearReindexMetadata(targetDb, subgraphName);
+        logger5.info("Reindex cancelled by user", { subgraph: subgraphName });
+      } else {
+        logger5.info("Reindex interrupted by shutdown, will resume", {
+          subgraph: subgraphName
+        });
+      }
+      return { processed: result.blocksProcessed };
+    }
+    await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
+    await clearReindexMetadata(targetDb, subgraphName);
+    logger5.info("Reindex complete", {
+      subgraph: subgraphName,
+      blocks: result.blocksProcessed,
+      events: result.totalEventsProcessed,
+      errors: result.totalErrors
+    });
+    return { processed: result.blocksProcessed };
+  } catch (err) {
+    logger5.error("Reindex failed", {
+      subgraph: subgraphName,
+      error: getErrorMessage2(err)
+    });
+    await updateSubgraphStatus2(targetDb, subgraphName, "error");
+    throw err;
+  }
+}
+async function resumeReindex(def, opts) {
+  const targetDb = getTargetDb2();
+  const subgraphName = def.name;
+  const row = await targetDb.selectFrom("subgraphs").select([
+    "id",
+    "last_processed_block",
+    "reindex_from_block",
+    "reindex_to_block"
+  ]).where("name", "=", subgraphName).executeTakeFirst();
+  if (!row)
+    throw new Error(`Subgraph "${subgraphName}" not found`);
+  const fromBlock = resolveReindexResumeBlock(row);
+  const toBlock = Number(row.reindex_to_block);
+  if (fromBlock == null) {
+    logger5.info("No reindex metadata, starting fresh reindex", {
+      subgraph: subgraphName
+    });
+    return reindexSubgraph(def, {
+      schemaName: opts.schemaName,
+      signal: opts.signal
+    });
+  }
+  if (fromBlock > toBlock) {
+    logger5.info("Resume: no remaining blocks", { subgraph: subgraphName });
+    await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
+    await clearReindexMetadata(targetDb, subgraphName);
+    return { processed: 0 };
+  }
+  logger5.info("Resuming reindex", {
+    subgraph: subgraphName,
+    fromBlock,
+    toBlock,
+    remaining: toBlock - fromBlock + 1
+  });
+  try {
+    const result = await processBlockRange(def, {
+      fromBlock,
+      toBlock,
+      status: "reindexing",
+      isCatchup: false,
+      apiKeyId: null,
+      subgraphId: row.id,
+      signal: opts.signal
+    });
+    if (result.aborted) {
+      const reason = String(opts.signal?.reason ?? "unknown");
+      if (reason === "user-cancelled") {
+        await updateSubgraphStatus2(targetDb, subgraphName, "active");
+        await clearReindexMetadata(targetDb, subgraphName);
+        logger5.info("Resume cancelled by user", { subgraph: subgraphName });
+      } else {
+        logger5.info("Resume interrupted by shutdown, will resume again", {
+          subgraph: subgraphName
+        });
+      }
+      return { processed: result.blocksProcessed };
+    }
+    await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
+    await clearReindexMetadata(targetDb, subgraphName);
+    logger5.info("Resumed reindex complete", {
+      subgraph: subgraphName,
+      blocks: result.blocksProcessed
+    });
+    return { processed: result.blocksProcessed };
+  } catch (err) {
+    logger5.error("Resumed reindex failed", {
+      subgraph: subgraphName,
+      error: getErrorMessage2(err)
+    });
+    await updateSubgraphStatus2(targetDb, subgraphName, "error");
+    throw err;
+  }
+}
+async function backfillSubgraph(def, opts) {
+  const targetDb = getTargetDb2();
+  const subgraphName = def.name;
+  logger5.info("Backfill starting", {
+    subgraph: subgraphName,
+    from: opts.fromBlock,
+    to: opts.toBlock
+  });
+  try {
+    const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
+    const result = await processBlockRange(def, {
+      fromBlock: opts.fromBlock,
+      toBlock: opts.toBlock,
+      status: "active",
+      isCatchup: false,
+      apiKeyId: null,
+      subgraphId: subgraphRow?.id,
+      signal: opts.signal
+    });
+    if (result.aborted) {
+      logger5.info("Backfill aborted", { subgraph: subgraphName });
+      return { processed: result.blocksProcessed };
+    }
+    const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
+    if (resolved > 0) {
+      logger5.info("Resolved subgraph gaps via backfill", {
+        subgraph: subgraphName,
+        resolved
+      });
+    }
+    logger5.info("Backfill complete", {
+      subgraph: subgraphName,
+      blocks: result.blocksProcessed,
+      events: result.totalEventsProcessed,
+      errors: result.totalErrors
+    });
+    return { processed: result.blocksProcessed };
+  } catch (err) {
+    logger5.error("Backfill failed", {
+      subgraph: subgraphName,
+      error: getErrorMessage2(err)
+    });
+    throw err;
+  }
+}
+
+// src/runtime/catchup.ts
+import { getSourceDb as getSourceDb3, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
+import {
+  recordGapBatch as recordGapBatch2
+} from "@secondlayer/shared/db/queries/subgraph-gaps";
+import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
+import { logger as logger6 } from "@secondlayer/shared/logger";
+var LOG_INTERVAL2 = 1000;
 var DEFAULT_BATCH_SIZE = 500;
 var MIN_BATCH_SIZE = 100;
 var MAX_BATCH_SIZE = 1000;
@@ -1389,8 +1887,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     return 0;
   catchingUp.add(subgraphName);
   try {
-    const sourceDb = getSourceDb2();
-    const targetDb = getTargetDb2();
+    const sourceDb = getSourceDb3();
+    const targetDb = getTargetDb3();
     const subgraphRow = await getSubgraph(targetDb, subgraphName);
     if (!subgraphRow)
       return 0;
@@ -1404,7 +1902,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     const subgraphStart = Number(subgraphRow.start_block) || 1;
     const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
     const totalBlocks = chainTip - lastProcessedBlock;
-    logger5.info("Subgraph catch-up starting", {
+    logger6.info("Subgraph catch-up starting", {
       subgraph: subgraphName,
       from: startBlock,
       to: chainTip,
@@ -1419,7 +1917,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     while (currentHeight <= chainTip) {
       const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
-        logger5.info("Subgraph status changed, stopping catch-up", {
+        logger6.info("Subgraph status changed, stopping catch-up", {
           subgraph: subgraphName,
           status: currentRow?.status ?? "deleted"
         });
@@ -1446,14 +1944,14 @@ async function catchUpSubgraph(subgraph, subgraphName) {
             preloaded: blockData
           });
         } catch (err) {
-          logger5.error("Block processing error during catch-up", {
+          logger6.error("Block processing error during catch-up", {
             subgraph: subgraphName,
             blockHeight: height,
             error: err instanceof Error ? err.message : String(err)
           });
           batchFailedBlocks.push({ height, reason: "processing_error" });
-          const { updateSubgraphStatus: updateSubgraphStatus2 } = await import("@secondlayer/shared/db/queries/subgraphs");
-          await updateSubgraphStatus2(targetDb, subgraphName, "active", height).catch(() => {});
+          const { updateSubgraphStatus: updateSubgraphStatus3 } = await import("@secondlayer/shared/db/queries/subgraphs");
+          await updateSubgraphStatus3(targetDb, subgraphName, "active", height).catch(() => {});
           processed++;
           continue;
         }
@@ -1464,8 +1962,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
             await stats.flush(targetDb);
           }
         }
-        if (processed % LOG_INTERVAL === 0) {
-          logger5.info("Subgraph catch-up progress", {
+        if (processed % LOG_INTERVAL2 === 0) {
+          logger6.info("Subgraph catch-up progress", {
             subgraph: subgraphName,
             processed,
             total: totalBlocks,
@@ -1476,8 +1974,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       }
       if (batchFailedBlocks.length > 0) {
         const gaps = coalesceGaps(batchFailedBlocks);
-        await recordGapBatch(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
-          logger5.warn("Failed to record subgraph gaps", {
+        await recordGapBatch2(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
+          logger6.warn("Failed to record subgraph gaps", {
             subgraph: subgraphName,
             error: err instanceof Error ? err.message : String(err)
           });
@@ -1488,7 +1986,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       currentHeight = batchEnd + 1;
     }
     await stats.flush(targetDb);
-    logger5.info("Subgraph catch-up complete", {
+    logger6.info("Subgraph catch-up complete", {
       subgraph: subgraphName,
       processed
     });
@@ -1499,17 +1997,17 @@ async function catchUpSubgraph(subgraph, subgraphName) {
 }
 
 // src/runtime/reorg.ts
-import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
-import { getRawClient, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
+import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
+import { getRawClient as getRawClient2, getTargetDb as getTargetDb4 } from "@secondlayer/shared/db";
 import { listSubgraphs } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger6 } from "@secondlayer/shared/logger";
+import { logger as logger7 } from "@secondlayer/shared/logger";
 async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
-  const targetDb = getTargetDb3();
-  const client = getRawClient("target");
+  const targetDb = getTargetDb4();
+  const client = getRawClient2("target");
   const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
   if (activeSubgraphs.length === 0)
     return;
-  logger6.info("Propagating reorg to subgraphs", {
+  logger7.info("Propagating reorg to subgraphs", {
     blockHeight,
     subgraphCount: activeSubgraphs.length
   });
@@ -1522,51 +2020,65 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
       for (const tableName of Object.keys(schema)) {
         await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" = $1`, [blockHeight]);
       }
-      logger6.info("Subgraph reorg cleanup done", {
+      logger7.info("Subgraph reorg cleanup done", {
         subgraph: sg.name,
         blockHeight
       });
       const def = await loadSubgraphDef(sg);
       await processBlock(def, sg.name, blockHeight);
-      logger6.info("Subgraph reorg reprocessed", {
+      logger7.info("Subgraph reorg reprocessed", {
         subgraph: sg.name,
         blockHeight
       });
     } catch (err) {
-      logger6.error("Subgraph reorg handling failed", {
+      logger7.error("Subgraph reorg handling failed", {
         subgraph: sg.name,
         blockHeight,
-        error: getErrorMessage2(err)
+        error: getErrorMessage3(err)
       });
     }
   }
 }
 
 // src/runtime/processor.ts
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
 import { resolve } from "node:path";
 import { pathToFileURL } from "node:url";
-import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
-import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
+import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
+import { getTargetDb as getTargetDb6 } from "@secondlayer/shared/db";
+import {
+  cancelSubgraphOperation,
+  claimSubgraphOperation,
+  completeSubgraphOperation,
+  createSubgraphOperation,
+  failSubgraphOperation,
+  findActiveSubgraphOperation,
+  getSubgraphOperation,
+  heartbeatSubgraphOperation,
+  isActiveSubgraphOperationConflict
+} from "@secondlayer/shared/db/queries/subgraph-operations";
 import {
   listSubgraphs as listSubgraphs2,
-  updateSubgraphStatus as updateSubgraphStatus2
+  pgSchemaName as pgSchemaName2,
+  updateSubgraphStatus as updateSubgraphStatus3
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger9 } from "@secondlayer/shared/logger";
+import { logger as logger10 } from "@secondlayer/shared/logger";
 import { listen as listen2 } from "@secondlayer/shared/queue/listener";
 
 // src/runtime/emitter.ts
 import {
-  getTargetDb as getTargetDb4
+  getTargetDb as getTargetDb5
 } from "@secondlayer/shared/db";
 import {
   getSubscriptionSigningSecret
 } from "@secondlayer/shared/db/queries/subscriptions";
-import { logger as logger8 } from "@secondlayer/shared/logger";
+import { logger as logger9 } from "@secondlayer/shared/logger";
 import { listen } from "@secondlayer/shared/queue/listener";
 import { sql as sql4 } from "kysely";
 
 // src/runtime/formats/index.ts
-import { logger as logger7 } from "@secondlayer/shared/logger";
+import { logger as logger8 } from "@secondlayer/shared/logger";
 
 // src/runtime/formats/cloudevents.ts
 function buildCloudEvents(outboxRow, _sub) {
@@ -1713,7 +2225,7 @@ function buildForFormat(outboxRow, sub, signingSecret) {
     case "standard-webhooks":
       return buildStandardWebhooks(outboxRow, signingSecret);
     default:
-      logger7.warn("Unknown subscription format, falling back to standard-webhooks", {
+      logger8.warn("Unknown subscription format, falling back to standard-webhooks", {
         format: sub.format,
         subscriptionId: sub.id
       });
@@ -1796,7 +2308,7 @@ async function dispatchOne(db, outboxRow, sub) {
   let responseHeaders = {};
   if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
     error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
-    logger8.warn("[emitter] refused private egress", {
+    logger9.warn("[emitter] refused private egress", {
       subscription: sub.name,
       url: sub.url
     });
@@ -1892,7 +2404,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
         circuit_opened_at: new Date,
         updated_at: new Date
       }).where("id", "=", sub.id).execute();
-      logger8.warn("Subscription circuit tripped — paused after consecutive failures", {
+      logger9.warn("Subscription circuit tripped — paused after consecutive failures", {
         subscription: sub.name,
         failures: newFailures
       });
@@ -1980,7 +2492,7 @@ async function drainForSub(db, state, sub, rows) {
             await settleFailed(db, row, sub, err);
           }
         } catch (err) {
-          logger8.error("Emitter dispatch crashed", {
+          logger9.error("Emitter dispatch crashed", {
             outboxId: row.id,
             error: err instanceof Error ? err.message : String(err)
           });
@@ -2009,7 +2521,7 @@ async function runRetention(db) {
 }
 async function startEmitter(opts) {
   const emitterId = `emitter-${Math.random().toString(36).slice(2, 10)}`;
-  const db = getTargetDb4();
+  const db = getTargetDb5();
   const state = {
     running: true,
     inFlightBySub: new Map,
@@ -2017,7 +2529,7 @@ async function startEmitter(opts) {
   };
   const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
   const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
-  logger8.info("[emitter] started", { id: emitterId });
+  logger9.info("[emitter] started", { id: emitterId });
   const MATCHER_BOOT_ATTEMPTS = 5;
   let lastErr = null;
   for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
@@ -2028,7 +2540,7 @@ async function startEmitter(opts) {
     } catch (err) {
       lastErr = err;
       const delayMs = 500 * 2 ** i;
-      logger8.warn("[emitter] matcher refresh failed, retrying", {
+      logger9.warn("[emitter] matcher refresh failed, retrying", {
         attempt: i + 1,
         delayMs,
         error: err instanceof Error ? err.message : String(err)
@@ -2042,21 +2554,21 @@ async function startEmitter(opts) {
   const stopNew = await listen("subscriptions:new_outbox", () => {
     if (!state.running)
       return;
-    claimAndDrain(db, state, emitterId).catch((err) => logger8.error("[emitter] claim failed", {
+    claimAndDrain(db, state, emitterId).catch((err) => logger9.error("[emitter] claim failed", {
       error: err instanceof Error ? err.message : String(err)
     }));
   });
   const stopChanged = await listen("subscriptions:changed", () => {
     if (!state.running)
       return;
-    refreshMatcher(db).catch((err) => logger8.error("[emitter] matcher refresh failed", {
+    refreshMatcher(db).catch((err) => logger9.error("[emitter] matcher refresh failed", {
       error: err instanceof Error ? err.message : String(err)
     }));
   });
   const poll = setInterval(() => {
     if (!state.running)
       return;
-    claimAndDrain(db, state, emitterId).catch((err) => logger8.error("[emitter] poll claim failed", {
+    claimAndDrain(db, state, emitterId).catch((err) => logger9.error("[emitter] poll claim failed", {
       error: err instanceof Error ? err.message : String(err)
     }));
   }, pollIntervalMs);
@@ -2064,7 +2576,7 @@ async function startEmitter(opts) {
   const retention = setInterval(() => {
     if (!state.running)
       return;
-    runRetention(db).catch((err) => logger8.error("[emitter] retention failed", {
+    runRetention(db).catch((err) => logger9.error("[emitter] retention failed", {
       error: err instanceof Error ? err.message : String(err)
     }));
   }, retentionIntervalMs);
@@ -2074,20 +2586,27 @@ async function startEmitter(opts) {
     clearInterval(retention);
     await stopNew();
     await stopChanged();
-    logger8.info("[emitter] stopped", { id: emitterId });
+    logger9.info("[emitter] stopped", { id: emitterId });
   };
 }
 
 // src/runtime/processor.ts
 var CHANNEL_NEW_BLOCK = "indexer:new_block";
+var CHANNEL_SUBGRAPH_OPERATIONS = "subgraph_operations:new";
 var DEFAULT_CONCURRENCY = 5;
+var DEFAULT_OPERATION_CONCURRENCY = 1;
 var POLL_INTERVAL_MS = 5000;
+var HEARTBEAT_INTERVAL_MS = 15000;
+var CANCEL_POLL_INTERVAL_MS = 1000;
 function handlerImportUrl(handlerPath, cacheBust = Date.now()) {
   return `${pathToFileURL(resolve(handlerPath)).href}?t=${cacheBust}`;
 }
 function sourceListenerUrl() {
   return process.env.SOURCE_DATABASE_URL ?? process.env.DATABASE_URL;
 }
+function targetListenerUrl() {
+  return process.env.TARGET_DATABASE_URL ?? process.env.DATABASE_URL;
+}
 function isHandlerNotFoundError(err) {
   if (!(err instanceof Error))
     return false;
@@ -2115,7 +2634,7 @@ async function loadSubgraphDefinition(sg) {
   knownVersions.set(sg.name, sg.version);
   definitionCache.set(sg.name, def);
   if (prevVersion && prevVersion !== sg.version) {
-    logger9.info("Subgraph handler reloaded", {
+    logger10.info("Subgraph handler reloaded", {
       subgraph: sg.name,
       from: prevVersion,
       to: sg.version
@@ -2132,22 +2651,215 @@ function cleanupCaches(active) {
     }
   }
 }
+async function synthesizeLegacyReindexOperations() {
+  const db = getTargetDb6();
+  const stale = (await listSubgraphs2(db)).filter((sg) => sg.status === "reindexing");
+  for (const sg of stale) {
+    const active = await findActiveSubgraphOperation(db, sg.id);
+    if (active)
+      continue;
+    try {
+      await createSubgraphOperation(db, {
+        subgraphId: sg.id,
+        subgraphName: sg.name,
+        accountId: sg.account_id,
+        kind: "reindex",
+        fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
+        toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
+      });
+      logger10.info("Queued legacy reindex resume operation", {
+        subgraph: sg.name
+      });
+    } catch (err) {
+      if (isActiveSubgraphOperationConflict(err))
+        continue;
+      throw err;
+    }
+  }
+}
+async function runSubgraphOperation(operation, signal) {
+  if (operation.cancel_requested) {
+    return 0;
+  }
+  const db = getTargetDb6();
+  const subgraph = await db.selectFrom("subgraphs").selectAll().where("id", "=", operation.subgraph_id).executeTakeFirst();
+  if (!subgraph)
+    throw new Error(`Subgraph not found: ${operation.subgraph_id}`);
+  const def = await loadSubgraphDefinition(subgraph);
+  const schemaName = subgraph.schema_name ?? pgSchemaName2(subgraph.name);
+  if (operation.kind === "backfill") {
+    if (operation.from_block == null || operation.to_block == null) {
+      throw new Error("Backfill operation is missing from_block or to_block");
+    }
+    const result2 = await backfillSubgraph(def, {
+      fromBlock: Number(operation.from_block),
+      toBlock: Number(operation.to_block),
+      schemaName,
+      signal
+    });
+    return result2.processed;
+  }
+  const hasResumeMetadata = subgraph.status === "reindexing" && subgraph.reindex_from_block != null && subgraph.reindex_to_block != null;
+  if (hasResumeMetadata) {
+    const result2 = await resumeReindex(def, { schemaName, signal });
+    return result2.processed;
+  }
+  const result = await reindexSubgraph(def, {
+    fromBlock: operation.from_block == null ? undefined : Number(operation.from_block),
+    toBlock: operation.to_block == null ? undefined : Number(operation.to_block),
+    schemaName,
+    signal
+  });
+  return result.processed;
+}
+async function startSubgraphOperationRunner(opts) {
+  const concurrency = opts?.concurrency ?? DEFAULT_OPERATION_CONCURRENCY;
+  const db = getTargetDb6();
+  const lockedBy = `${hostname()}:${process.pid}:${randomUUID()}`;
+  const active = new Map;
+  const activeRuns = new Map;
+  let running = true;
+  let draining = false;
+  logger10.info("Starting subgraph operation runner", { concurrency, lockedBy });
+  const startOperation = (operation) => {
+    const controller = new AbortController;
+    active.set(operation.id, controller);
+    const heartbeat = setInterval(() => {
+      if (!running)
+        return;
+      heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
+        logger10.warn("Subgraph operation heartbeat failed", {
+          operationId: operation.id,
+          error: getErrorMessage4(err)
+        });
+      });
+    }, HEARTBEAT_INTERVAL_MS);
+    const cancelPoll = setInterval(() => {
+      getSubgraphOperation(db, operation.id).then((row) => {
+        if (row?.cancel_requested && !controller.signal.aborted) {
+          controller.abort("user-cancelled");
+        }
+      }).catch((err) => {
+        logger10.warn("Subgraph operation cancel poll failed", {
+          operationId: operation.id,
+          error: getErrorMessage4(err)
+        });
+      });
+    }, CANCEL_POLL_INTERVAL_MS);
+    const run = (async () => {
+      let processed = 0;
+      try {
+        if (operation.cancel_requested) {
+          controller.abort("user-cancelled");
+        } else {
+          processed = await runSubgraphOperation(operation, controller.signal);
+        }
+        const reason = String(controller.signal.reason ?? "");
+        if (controller.signal.aborted && reason === "user-cancelled") {
+          await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
+          logger10.info("Subgraph operation cancelled", {
+            operationId: operation.id,
+            subgraph: operation.subgraph_name
+          });
+          return;
+        }
+        if (controller.signal.aborted) {
+          logger10.info("Subgraph operation interrupted", {
+            operationId: operation.id,
+            subgraph: operation.subgraph_name,
+            reason
+          });
+          return;
+        }
+        await completeSubgraphOperation(db, operation.id, lockedBy, processed);
+        logger10.info("Subgraph operation completed", {
+          operationId: operation.id,
+          subgraph: operation.subgraph_name,
+          processed
+        });
+      } catch (err) {
+        const reason = String(controller.signal.reason ?? "");
+        if (controller.signal.aborted && reason === "shutdown") {
+          logger10.info("Subgraph operation interrupted by shutdown", {
+            operationId: operation.id,
+            subgraph: operation.subgraph_name
+          });
+          return;
+        }
+        if (controller.signal.aborted && reason === "user-cancelled") {
+          await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
+          return;
+        }
+        await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage4(err), processed);
+        logger10.error("Subgraph operation failed", {
+          operationId: operation.id,
+          subgraph: operation.subgraph_name,
+          error: getErrorMessage4(err)
+        });
+      } finally {
+        clearInterval(heartbeat);
+        clearInterval(cancelPoll);
+        active.delete(operation.id);
+        activeRuns.delete(operation.id);
+        if (running)
+          drain();
+      }
+    })();
+    activeRuns.set(operation.id, run);
+  };
+  const drain = async () => {
+    if (!running || draining)
+      return;
+    draining = true;
+    try {
+      while (running && active.size < concurrency) {
+        const operation = await claimSubgraphOperation(db, lockedBy);
+        if (!operation)
+          break;
+        startOperation(operation);
+      }
+    } finally {
+      draining = false;
+    }
+  };
+  await synthesizeLegacyReindexOperations();
+  await drain();
+  const stopListening = await listen2(CHANNEL_SUBGRAPH_OPERATIONS, () => {
+    drain();
+  }, { connectionString: targetListenerUrl() });
+  const pollInterval = setInterval(() => {
+    drain();
+  }, POLL_INTERVAL_MS);
+  return async () => {
+    running = false;
+    clearInterval(pollInterval);
+    await stopListening();
+    for (const controller of active.values()) {
+      controller.abort("shutdown");
+    }
+    await Promise.allSettled(activeRuns.values());
+    logger10.info("Subgraph operation runner stopped");
+  };
+}
 async function startSubgraphProcessor(opts) {
   const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
   let running = true;
-  logger9.info("Starting subgraph processor", { concurrency });
-  const targetDb = getTargetDb5();
+  logger10.info("Starting subgraph processor", { concurrency });
+  const stopOperations = await startSubgraphOperationRunner({
+    concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
+  });
+  const targetDb = getTargetDb6();
   const activeSubgraphs = (await listSubgraphs2(targetDb)).filter((v) => v.status === "active");
   for (const sg of activeSubgraphs) {
     try {
       const def = await loadSubgraphDefinition(sg);
       await catchUpSubgraph(def, sg.name);
     } catch (err) {
-      const msg = getErrorMessage3(err);
+      const msg = getErrorMessage4(err);
       if (isHandlerNotFoundError(err)) {
-        await updateSubgraphStatus2(targetDb, sg.name, "error");
+        await updateSubgraphStatus3(targetDb, sg.name, "error");
       }
-      logger9.error("Subgraph catch-up failed on startup", {
+      logger10.error("Subgraph catch-up failed on startup", {
         subgraph: sg.name,
         error: msg
       });
@@ -2156,7 +2868,7 @@ async function startSubgraphProcessor(opts) {
   const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
     if (!running)
       return;
-    const db = getTargetDb5();
+    const db = getTargetDb6();
     const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
     cleanupCaches(subgraphs);
     for (const sg of subgraphs) {
@@ -2164,11 +2876,11 @@ async function startSubgraphProcessor(opts) {
         const def = await loadSubgraphDefinition(sg);
         await catchUpSubgraph(def, sg.name);
       } catch (err) {
-        const msg = getErrorMessage3(err);
+        const msg = getErrorMessage4(err);
         if (isHandlerNotFoundError(err)) {
-          await updateSubgraphStatus2(db, sg.name, "error");
+          await updateSubgraphStatus3(db, sg.name, "error");
         }
-        logger9.error("Subgraph processing failed", {
+        logger10.error("Subgraph processing failed", {
           subgraph: sg.name,
           error: msg
         });
@@ -2185,15 +2897,15 @@ async function startSubgraphProcessor(opts) {
         await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
       }
     } catch (err) {
-      logger9.error("Subgraph reorg handling failed", {
-        error: getErrorMessage3(err)
+      logger10.error("Subgraph reorg handling failed", {
+        error: getErrorMessage4(err)
       });
     }
   }, { connectionString: sourceListenerUrl() });
   const pollInterval = setInterval(async () => {
     if (!running)
       return;
-    const db = getTargetDb5();
+    const db = getTargetDb6();
     const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
     cleanupCaches(subgraphs);
     for (const sg of subgraphs) {
@@ -2201,27 +2913,29 @@ async function startSubgraphProcessor(opts) {
         const def = await loadSubgraphDefinition(sg);
         await catchUpSubgraph(def, sg.name);
       } catch (err) {
-        logger9.error("Subgraph poll processing failed", {
+        logger10.error("Subgraph poll processing failed", {
           subgraph: sg.name,
-          error: getErrorMessage3(err)
+          error: getErrorMessage4(err)
         });
       }
     }
   }, POLL_INTERVAL_MS);
   const stopEmitter = await startEmitter();
-  logger9.info("Subgraph processor ready");
+  logger10.info("Subgraph processor ready");
   return async () => {
     running = false;
     clearInterval(pollInterval);
     await stopListening();
     await stopReorgListening();
+    await stopOperations();
     await stopEmitter();
-    logger9.info("Subgraph processor stopped");
+    logger10.info("Subgraph processor stopped");
   };
 }
 export {
-  startSubgraphProcessor
+  startSubgraphProcessor,
+  startSubgraphOperationRunner
 };
 
-//# debugId=5B3674F8B5BD730264756E2164756E21
+//# debugId=1929DF522C4FE6AB64756E2164756E21
 //# sourceMappingURL=processor.js.map