@secondlayer/subgraphs 0.5.7 → 0.6.0

package/dist/src/runtime/reindex.d.ts

@@ -81,4 +81,16 @@ interface ReindexOptions {
 declare function reindexSubgraph(def: SubgraphDefinition, opts?: ReindexOptions): Promise<{
 	processed: number
 }>;
-export { reindexSubgraph, ReindexOptions };
+/**
+* Backfill a subgraph by re-processing a block range WITHOUT dropping the schema.
+* Uses upserts so existing data is updated, not duplicated. Safe to run while
+* the subgraph is actively syncing.
+*/
+declare function backfillSubgraph(def: SubgraphDefinition, opts: {
+	fromBlock: number
+	toBlock: number
+	schemaName?: string
+}): Promise<{
+	processed: number
+}>;
+export { reindexSubgraph, backfillSubgraph, ReindexOptions };

package/dist/src/runtime/reindex.js

@@ -122,53 +122,83 @@ class SubgraphContext {
     }
     return opsToFlush.length;
   }
+  prepareInsert(op) {
+    const upsertKeys = op.data._upsert_keys;
+    const data = { ...op.data };
+    delete data._upsert_keys;
+    delete data._upsert_fallback_keys;
+    delete data._upsert_fallback_set;
+    data._block_height = this.block.height;
+    data._tx_id = this._tx.txId;
+    data._created_at = "NOW()";
+    const cols = Object.keys(data);
+    cols.forEach(validateColumnName);
+    const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
+    const batchKey = `${op.table}:${[...cols].sort().join(",")}:${upsertKeys ? [...upsertKeys].sort().join(",") : ""}`;
+    return { data, cols, vals, upsertKeys, batchKey };
+  }
   buildStatements(ops) {
     const statements = [];
+    let currentBatch = null;
+    let currentBatchKey = "";
+    const flushInsertBatch = () => {
+      if (!currentBatch)
+        return;
+      const qualifiedTable = `"${this.pgSchemaName}"."${currentBatch.table}"`;
+      const colList = currentBatch.cols.map((c) => `"${c}"`).join(", ");
+      let rows = currentBatch.rows;
+      if (currentBatch.upsertKeys && currentBatch.upsertKeys.length > 0) {
+        const keyIndices = currentBatch.upsertKeys.map((k) => currentBatch.cols.indexOf(k));
+        const seen = new Map;
+        for (let i = 0;i < rows.length; i++) {
+          const key = keyIndices.map((ki) => rows[i][ki]).join("\x00");
+          seen.set(key, i);
+        }
+        if (seen.size < rows.length) {
+          rows = Array.from(seen.values()).map((i) => rows[i]);
+        }
+      }
+      const valuesList = rows.map((r) => `(${r.join(", ")})`).join(", ");
+      let stmt = `INSERT INTO ${qualifiedTable} (${colList}) VALUES ${valuesList}`;
+      if (currentBatch.upsertKeys && currentBatch.upsertKeys.length > 0) {
+        const updateCols = currentBatch.cols.filter((c) => !currentBatch.upsertKeys.includes(c) && !c.startsWith("_"));
+        if (updateCols.length > 0) {
+          const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
+          stmt += ` ON CONFLICT (${currentBatch.upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
+        } else {
+          stmt += ` ON CONFLICT (${currentBatch.upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
+        }
+      }
+      statements.push(stmt);
+      currentBatch = null;
+      currentBatchKey = "";
+    };
     for (const op of ops) {
       const qualifiedTable = `"${this.pgSchemaName}"."${op.table}"`;
-      switch (op.kind) {
-        case "insert": {
-          const upsertKeys = op.data._upsert_keys;
-          const fallbackKeys = op.data._upsert_fallback_keys;
-          const fallbackSet = op.data._upsert_fallback_set;
-          const data = { ...op.data };
-          delete data._upsert_keys;
-          delete data._upsert_fallback_keys;
-          delete data._upsert_fallback_set;
-          data._block_height = this.block.height;
-          data._tx_id = this._tx.txId;
-          data._created_at = "NOW()";
-          const cols = Object.keys(data);
-          cols.forEach(validateColumnName);
-          const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
-          let stmt = `INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) VALUES (${vals.join(", ")})`;
-          if (upsertKeys && upsertKeys.length > 0) {
-            const updateCols = cols.filter((c) => !upsertKeys.includes(c) && !c.startsWith("_"));
-            if (updateCols.length > 0) {
-              const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
-              stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
-            } else {
-              stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
-            }
-          } else if (fallbackKeys && fallbackSet) {}
-          statements.push(stmt);
-          break;
+      if (op.kind === "insert") {
+        const { cols, vals, upsertKeys, batchKey } = this.prepareInsert(op);
+        if (batchKey === currentBatchKey && currentBatch) {
+          currentBatch.rows.push(vals);
+        } else {
+          flushInsertBatch();
+          currentBatch = { table: op.table, cols, rows: [vals], upsertKeys };
+          currentBatchKey = batchKey;
         }
-        case "update": {
+      } else {
+        flushInsertBatch();
+        if (op.kind === "update") {
           const setEntries = Object.entries(op.set);
           setEntries.forEach(([k]) => validateColumnName(k));
           const setClauses = setEntries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
           const { clause } = buildWhereClause(op.data);
           statements.push(`UPDATE ${qualifiedTable} SET ${setClauses.join(", ")} WHERE ${clause}`);
-          break;
-        }
-        case "delete": {
+        } else if (op.kind === "delete") {
           const { clause } = buildWhereClause(op.data);
           statements.push(`DELETE FROM ${qualifiedTable} WHERE ${clause}`);
-          break;
         }
       }
     }
+    flushInsertBatch();
     return statements;
   }
   validateTable(table) {
@@ -418,6 +448,7 @@ import { updateSubgraphStatus, recordSubgraphProcessed } from "@secondlayer/shar
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 
 // src/runtime/block-processor.ts
+var schemaNameCache = new Map;
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   const db = getDb();
   const blockStart = performance.now();
@@ -428,19 +459,26 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     errors: 0,
     skipped: false
   };
-  const block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
-  if (!block) {
-    logger3.warn("Block not found for subgraph processing", { subgraph: subgraphName, blockHeight });
-    result.skipped = true;
-    return result;
-  }
-  if (!block.canonical) {
-    logger3.debug("Skipping non-canonical block", { subgraph: subgraphName, blockHeight });
-    result.skipped = true;
-    return result;
+  let block, txs, evts;
+  if (opts?.preloaded) {
+    block = opts.preloaded.block;
+    txs = opts.preloaded.txs;
+    evts = opts.preloaded.events;
+  } else {
+    block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
+    if (!block) {
+      logger3.warn("Block not found for subgraph processing", { subgraph: subgraphName, blockHeight });
+      result.skipped = true;
+      return result;
+    }
+    if (!block.canonical) {
+      logger3.debug("Skipping non-canonical block", { subgraph: subgraphName, blockHeight });
+      result.skipped = true;
+      return result;
+    }
+    txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
+    evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   }
-  const txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
-  const evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   const matched = matchSources(subgraph.sources, txs, evts);
   result.matched = matched.length;
   if (matched.length === 0) {
@@ -449,8 +487,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     }
     return result;
   }
-  const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
-  const schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
+  let schemaName = schemaNameCache.get(subgraphName);
+  if (!schemaName) {
+    const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
+    schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
+    schemaNameCache.set(subgraphName, schemaName);
+  }
   const blockMeta = {
     height: block.height,
     hash: block.hash,
@@ -665,45 +707,96 @@ function generateSubgraphSQL(def, schemaNameOverride) {
   return { statements, hash };
 }
 
+// src/runtime/batch-loader.ts
+async function loadBlockRange(db, fromHeight, toHeight) {
+  const [blocks, txs, events] = await Promise.all([
+    db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
+    db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
+    db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
+  ]);
+  const txsByHeight = new Map;
+  for (const tx of txs) {
+    const h = Number(tx.block_height);
+    const list = txsByHeight.get(h) ?? [];
+    list.push(tx);
+    txsByHeight.set(h, list);
+  }
+  const eventsByHeight = new Map;
+  for (const evt of events) {
+    const h = Number(evt.block_height);
+    const list = eventsByHeight.get(h) ?? [];
+    list.push(evt);
+    eventsByHeight.set(h, list);
+  }
+  const result = new Map;
+  for (const block of blocks) {
+    const h = Number(block.height);
+    result.set(h, {
+      block,
+      txs: txsByHeight.get(h) ?? [],
+      events: eventsByHeight.get(h) ?? []
+    });
+  }
+  return result;
+}
+function avgEventsPerBlock(batch) {
+  if (batch.size === 0)
+    return 0;
+  let totalEvents = 0;
+  for (const data of batch.values()) {
+    totalEvents += data.events.length;
+  }
+  return totalEvents / batch.size;
+}
+
 // src/runtime/reindex.ts
 var LOG_INTERVAL = 1000;
-async function reindexSubgraph(def, opts) {
+var DEFAULT_BATCH_SIZE = 500;
+var MIN_BATCH_SIZE = 100;
+var MAX_BATCH_SIZE = 1000;
+async function processBlockRange(def, opts) {
   const db = getDb2();
-  const client = getRawClient();
   const subgraphName = def.name;
-  const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
-  await updateSubgraphStatus2(db, subgraphName, "reindexing");
-  logger4.info("Reindex starting", { subgraph: subgraphName });
-  try {
-    await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
-    const { statements } = generateSubgraphSQL(def, schemaName);
-    for (const stmt of statements) {
-      await client.unsafe(stmt);
+  const { fromBlock, toBlock, status } = opts;
+  const totalBlocks = toBlock - fromBlock + 1;
+  const stats = new StatsAccumulator(subgraphName, opts.apiKeyId, opts.isCatchup);
+  let blocksProcessed = 0;
+  let totalEventsProcessed = 0;
+  let totalErrors = 0;
+  let batchSize = DEFAULT_BATCH_SIZE;
+  let currentHeight = fromBlock;
+  let nextBatchPromise = loadBlockRange(db, currentHeight, Math.min(currentHeight + batchSize - 1, toBlock));
+  while (currentHeight <= toBlock) {
+    const batch = await nextBatchPromise;
+    const batchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
+    const nextStart = batchEnd + 1;
+    if (nextStart <= toBlock) {
+      const nextEnd = Math.min(nextStart + batchSize - 1, toBlock);
+      nextBatchPromise = loadBlockRange(db, nextStart, nextEnd);
     }
-    logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
-    const fromBlock = opts?.fromBlock ?? 1;
-    let toBlock;
-    if (opts?.toBlock != null) {
-      toBlock = opts.toBlock;
-    } else {
-      const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
-      toBlock = progress?.last_indexed_block ?? progress?.last_contiguous_block ?? 0;
-    }
-    if (fromBlock > toBlock) {
-      logger4.info("No blocks to reindex", { subgraph: subgraphName, fromBlock, toBlock });
-      await updateSubgraphStatus2(db, subgraphName, "active", 0);
-      return { processed: 0 };
-    }
-    const totalBlocks = toBlock - fromBlock + 1;
-    logger4.info("Reindexing blocks", { subgraph: subgraphName, fromBlock, toBlock, totalBlocks });
-    const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
-    const stats = new StatsAccumulator(subgraphName, subgraphRow?.api_key_id ?? null, false);
-    let blocksProcessed = 0;
-    let totalEventsProcessed = 0;
-    let totalErrors = 0;
-    const PROGRESS_INTERVAL = 100;
-    for (let height = fromBlock;height <= toBlock; height++) {
-      const result = await processBlock(def, subgraphName, height, { skipProgressUpdate: true });
+    for (let height = currentHeight;height <= batchEnd; height++) {
+      const blockData = batch.get(height);
+      if (!blockData) {
+        blocksProcessed++;
+        continue;
+      }
+      let result;
+      try {
+        result = await processBlock(def, subgraphName, height, {
+          skipProgressUpdate: true,
+          preloaded: blockData
+        });
+      } catch (err) {
+        logger4.error("Block processing error", {
+          subgraph: subgraphName,
+          blockHeight: height,
+          error: err instanceof Error ? err.message : String(err)
+        });
+        await updateSubgraphStatus2(db, subgraphName, status, height).catch(() => {});
+        blocksProcessed++;
+        totalErrors++;
+        continue;
+      }
       blocksProcessed++;
       totalEventsProcessed += result.processed;
       totalErrors += result.errors;
@@ -713,11 +806,11 @@ async function reindexSubgraph(def, opts) {
           await stats.flush(db);
         }
       }
-      if (blocksProcessed % PROGRESS_INTERVAL === 0) {
-        await updateSubgraphStatus2(db, subgraphName, "reindexing", height);
+      if (blocksProcessed % 100 === 0) {
+        await updateSubgraphStatus2(db, subgraphName, status, height);
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
-        logger4.info("Reindex progress", {
+        logger4.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
           subgraph: subgraphName,
           processed: blocksProcessed,
           total: totalBlocks,
@@ -726,26 +819,97 @@ async function reindexSubgraph(def, opts) {
         });
       }
     }
-    await stats.flush(db);
+    const avg = avgEventsPerBlock(batch);
+    if (avg > 50)
+      batchSize = Math.max(Math.round(batchSize * 0.5), MIN_BATCH_SIZE);
+    else if (avg < 10)
+      batchSize = Math.min(Math.round(batchSize * 1.5), MAX_BATCH_SIZE);
+    currentHeight = batchEnd + 1;
+  }
+  await stats.flush(db);
+  return { blocksProcessed, totalEventsProcessed, totalErrors };
+}
+async function resolveBlockRange(db, opts) {
+  const fromBlock = opts?.fromBlock ?? 1;
+  let toBlock;
+  if (opts?.toBlock != null) {
+    toBlock = opts.toBlock;
+  } else {
+    const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    toBlock = progress?.last_indexed_block ?? progress?.last_contiguous_block ?? 0;
+  }
+  return { fromBlock, toBlock };
+}
+async function reindexSubgraph(def, opts) {
+  const db = getDb2();
+  const client = getRawClient();
+  const subgraphName = def.name;
+  const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
+  await updateSubgraphStatus2(db, subgraphName, "reindexing");
+  logger4.info("Reindex starting", { subgraph: subgraphName });
+  try {
+    await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
+    const { statements } = generateSubgraphSQL(def, schemaName);
+    for (const stmt of statements) {
+      await client.unsafe(stmt);
+    }
+    logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
+    const { fromBlock, toBlock } = await resolveBlockRange(db, opts);
+    if (fromBlock > toBlock) {
+      logger4.info("No blocks to reindex", { subgraph: subgraphName, fromBlock, toBlock });
+      await updateSubgraphStatus2(db, subgraphName, "active", 0);
+      return { processed: 0 };
+    }
+    logger4.info("Reindexing blocks", { subgraph: subgraphName, fromBlock, toBlock, totalBlocks: toBlock - fromBlock + 1 });
+    const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
+    const result = await processBlockRange(def, {
+      fromBlock,
+      toBlock,
+      status: "reindexing",
+      isCatchup: false,
+      apiKeyId: subgraphRow?.api_key_id ?? null
+    });
     const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
-    if (totalEventsProcessed > 0 || totalErrors > 0) {
-      await recordSubgraphProcessed2(db, subgraphName, totalEventsProcessed, totalErrors, totalErrors > 0 ? `${totalErrors} error(s) during reindex` : undefined);
+    if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
+      await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
     }
     await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
-    logger4.info("Reindex complete", { subgraph: subgraphName, blocks: blocksProcessed, events: totalEventsProcessed, errors: totalErrors });
-    return { processed: blocksProcessed };
+    logger4.info("Reindex complete", { subgraph: subgraphName, blocks: result.blocksProcessed, events: result.totalEventsProcessed, errors: result.totalErrors });
+    return { processed: result.blocksProcessed };
   } catch (err) {
-    logger4.error("Reindex failed", {
-      subgraph: subgraphName,
-      error: getErrorMessage2(err)
-    });
+    logger4.error("Reindex failed", { subgraph: subgraphName, error: getErrorMessage2(err) });
     await updateSubgraphStatus2(db, subgraphName, "error");
     throw err;
   }
 }
+async function backfillSubgraph(def, opts) {
+  const db = getDb2();
+  const subgraphName = def.name;
+  logger4.info("Backfill starting", { subgraph: subgraphName, from: opts.fromBlock, to: opts.toBlock });
+  try {
+    const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
+    const result = await processBlockRange(def, {
+      fromBlock: opts.fromBlock,
+      toBlock: opts.toBlock,
+      status: "active",
+      isCatchup: false,
+      apiKeyId: subgraphRow?.api_key_id ?? null
+    });
+    const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
+    if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
+      await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
+    }
+    logger4.info("Backfill complete", { subgraph: subgraphName, blocks: result.blocksProcessed, events: result.totalEventsProcessed, errors: result.totalErrors });
+    return { processed: result.blocksProcessed };
+  } catch (err) {
+    logger4.error("Backfill failed", { subgraph: subgraphName, error: getErrorMessage2(err) });
+    throw err;
+  }
+}
 export {
-  reindexSubgraph
+  reindexSubgraph,
+  backfillSubgraph
 };
 
-//# debugId=3876D18534CDDF5864756E2164756E21
+//# debugId=15F1B59510B2A45164756E2164756E21
 //# sourceMappingURL=reindex.js.map