@secondlayer/subgraphs 0.6.0 → 0.7.1

package/dist/src/index.d.ts

@@ -70,6 +70,8 @@ interface SubgraphDefinition {
 	version?: string;
 	/** Human description */
 	description?: string;
+	/** Block height to start indexing from (default: 1) */
+	startBlock?: number;
 	/** What blockchain data to process — one or more source filters */
 	sources: SubgraphSource[];
 	/** Tables in this subgraph */
@@ -138,8 +140,8 @@ interface GeneratedSQL {
 */
 declare function generateSubgraphSQL(def: SubgraphDefinition, schemaNameOverride?: string): GeneratedSQL;
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
-import { Kysely } from "kysely";
 import { Database } from "@secondlayer/shared/db";
+import { Kysely } from "kysely";
 type AnyDb = Kysely<Database>;
 interface TableDiff {
 	/** Tables added to the schema */

package/dist/src/index.js

@@ -60,8 +60,8 @@ function validateSubgraphDefinition(def) {
 }
 
 // src/runtime/context.ts
-import { sql } from "kysely";
 import { logger } from "@secondlayer/shared/logger";
+import { sql } from "kysely";
 function validateColumnName(name) {
   if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
     throw new Error(`Invalid column name: ${name}`);
@@ -102,13 +102,26 @@ class SubgraphContext {
     const keyColumns = Object.keys(key);
     const hasUniqueConstraint = tableDef.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
     if (hasUniqueConstraint) {
-      this.ops.push({ kind: "insert", table, data: { ...key, ...row, _upsert_keys: keyColumns } });
+      this.ops.push({
+        kind: "insert",
+        table,
+        data: { ...key, ...row, _upsert_keys: keyColumns }
+      });
     } else {
       logger.warn("upsert called without matching uniqueKeys constraint, using fallback", {
         table,
         keys: keyColumns
       });
-      this.ops.push({ kind: "insert", table, data: { ...key, ...row, _upsert_fallback_keys: keyColumns, _upsert_fallback_set: row } });
+      this.ops.push({
+        kind: "insert",
+        table,
+        data: {
+          ...key,
+          ...row,
+          _upsert_fallback_keys: keyColumns,
+          _upsert_fallback_set: row
+        }
+      });
     }
   }
   delete(table, where) {
@@ -271,95 +284,6 @@ function buildWhereClause(where) {
   return { clause: parts.join(" AND "), values: [] };
 }
 
-// src/runtime/source-matcher.ts
-function matchPattern(value, pattern) {
-  if (!pattern.includes("*"))
-    return value === pattern;
-  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-  return new RegExp(`^${regex}$`).test(value);
-}
-function matchSource(source, transactions, eventsByTx) {
-  const results = [];
-  const key = sourceKey(source);
-  for (const tx of transactions) {
-    if (source.type) {
-      if (!matchPattern(tx.type, source.type))
-        continue;
-      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
-      let matchedEvents = txEvents;
-      if (source.minAmount !== undefined) {
-        const amountEvents = matchedEvents.filter((e) => {
-          const data = e.data;
-          const rawAmount = data?.amount;
-          if (rawAmount === undefined)
-            return false;
-          const amount = BigInt(rawAmount);
-          return amount >= source.minAmount;
-        });
-        if (amountEvents.length === 0)
-          continue;
-        matchedEvents = amountEvents;
-      }
-      results.push({ tx, events: matchedEvents, sourceKey: key });
-      continue;
-    }
-    if (source.contract) {
-      const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
-      if (source.function && tx.function_name) {
-        if (!matchPattern(tx.function_name, source.function))
-          continue;
-      } else if (source.function && !tx.function_name) {
-        continue;
-      }
-      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
-      let matchedEvents = txEvents;
-      if (!txContractMatch) {
-        matchedEvents = txEvents.filter((e) => {
-          const data = e.data;
-          const contractIdentifier = data?.contract_identifier;
-          return contractIdentifier && matchPattern(contractIdentifier, source.contract);
-        });
-        if (matchedEvents.length === 0)
-          continue;
-      }
-      if (source.event) {
-        matchedEvents = matchedEvents.filter((e) => {
-          if (matchPattern(e.type, source.event))
-            return true;
-          const data = e.data;
-          const topic = data?.topic;
-          return topic ? matchPattern(topic, source.event) : false;
-        });
-      }
-      if (txContractMatch || matchedEvents.length > 0) {
-        results.push({ tx, events: matchedEvents, sourceKey: key });
-      }
-    }
-  }
-  return results;
-}
-function matchSources(sources, transactions, events) {
-  const eventsByTx = new Map;
-  for (const event of events) {
-    const list = eventsByTx.get(event.tx_id) ?? [];
-    list.push(event);
-    eventsByTx.set(event.tx_id, list);
-  }
-  const seen = new Set;
-  const results = [];
-  for (const source of sources) {
-    const matches = matchSource(source, transactions, eventsByTx);
-    for (const match of matches) {
-      const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
-      if (!seen.has(dedupeKey)) {
-        seen.add(dedupeKey);
-        results.push(match);
-      }
-    }
-  }
-  return results;
-}
-
 // src/runtime/clarity.ts
 import { cvToJSON, deserializeCV } from "@secondlayer/stacks/clarity";
 function decodeClarityValue(hex) {
@@ -392,8 +316,8 @@ function decodeFunctionArgs(args) {
 }
 
 // src/runtime/runner.ts
-import { logger as logger2 } from "@secondlayer/shared/logger";
 import { getErrorMessage } from "@secondlayer/shared";
+import { logger as logger2 } from "@secondlayer/shared/logger";
 var DEFAULT_ERROR_THRESHOLD = 50;
 function resolveHandler(handlers, key) {
   return handlers[key] ?? handlers["*"] ?? null;
@@ -405,7 +329,11 @@ async function runHandlers(subgraph, matched, ctx, opts) {
   for (const { tx, events, sourceKey: sourceKey2 } of matched) {
     const handler = resolveHandler(subgraph.handlers, sourceKey2);
     if (!handler) {
-      logger2.warn("No handler found for source key", { subgraph: subgraph.name, sourceKey: sourceKey2, txId: tx.tx_id });
+      logger2.warn("No handler found for source key", {
+        subgraph: subgraph.name,
+        sourceKey: sourceKey2,
+        txId: tx.tx_id
+      });
       continue;
     }
     ctx.setTx({
@@ -482,11 +410,103 @@ async function runHandlers(subgraph, matched, ctx, opts) {
   return { processed, errors };
 }
 
+// src/runtime/source-matcher.ts
+function matchPattern(value, pattern) {
+  if (!pattern.includes("*"))
+    return value === pattern;
+  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  return new RegExp(`^${regex}$`).test(value);
+}
+function matchSource(source, transactions, eventsByTx) {
+  const results = [];
+  const key = sourceKey(source);
+  for (const tx of transactions) {
+    if (source.type) {
+      if (!matchPattern(tx.type, source.type))
+        continue;
+      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
+      let matchedEvents = txEvents;
+      if (source.minAmount !== undefined) {
+        const amountEvents = matchedEvents.filter((e) => {
+          const data = e.data;
+          const rawAmount = data?.amount;
+          if (rawAmount === undefined)
+            return false;
+          const amount = BigInt(rawAmount);
+          return amount >= source.minAmount;
+        });
+        if (amountEvents.length === 0)
+          continue;
+        matchedEvents = amountEvents;
+      }
+      results.push({ tx, events: matchedEvents, sourceKey: key });
+      continue;
+    }
+    if (source.contract) {
+      const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
+      if (source.function && tx.function_name) {
+        if (!matchPattern(tx.function_name, source.function))
+          continue;
+      } else if (source.function && !tx.function_name) {
+        continue;
+      }
+      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
+      let matchedEvents = txEvents;
+      if (!txContractMatch) {
+        matchedEvents = txEvents.filter((e) => {
+          const data = e.data;
+          const contractIdentifier = data?.contract_identifier;
+          return contractIdentifier && matchPattern(contractIdentifier, source.contract);
+        });
+        if (matchedEvents.length === 0)
+          continue;
+      }
+      if (source.event) {
+        matchedEvents = matchedEvents.filter((e) => {
+          if (matchPattern(e.type, source.event))
+            return true;
+          const data = e.data;
+          const topic = data?.topic;
+          return topic ? matchPattern(topic, source.event) : false;
+        });
+      }
+      if (txContractMatch || matchedEvents.length > 0) {
+        results.push({ tx, events: matchedEvents, sourceKey: key });
+      }
+    }
+  }
+  return results;
+}
+function matchSources(sources, transactions, events) {
+  const eventsByTx = new Map;
+  for (const event of events) {
+    const list = eventsByTx.get(event.tx_id) ?? [];
+    list.push(event);
+    eventsByTx.set(event.tx_id, list);
+  }
+  const seen = new Set;
+  const results = [];
+  for (const source of sources) {
+    const matches = matchSource(source, transactions, eventsByTx);
+    for (const match of matches) {
+      const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
+      if (!seen.has(dedupeKey)) {
+        seen.add(dedupeKey);
+        results.push(match);
+      }
+    }
+  }
+  return results;
+}
+
 // src/runtime/block-processor.ts
-import { sql as sql2 } from "kysely";
 import { getDb } from "@secondlayer/shared/db";
+import {
+  recordSubgraphProcessed,
+  updateSubgraphStatus
+} from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger3 } from "@secondlayer/shared/logger";
-import { updateSubgraphStatus, recordSubgraphProcessed } from "@secondlayer/shared/db/queries/subgraphs";
+import { sql as sql2 } from "kysely";
 
 // src/schema/utils.ts
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
@@ -511,12 +531,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   } else {
     block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
     if (!block) {
-      logger3.warn("Block not found for subgraph processing", { subgraph: subgraphName, blockHeight });
+      logger3.warn("Block not found for subgraph processing", {
+        subgraph: subgraphName,
+        blockHeight
+      });
       result.skipped = true;
       return result;
     }
     if (!block.canonical) {
-      logger3.debug("Skipping non-canonical block", { subgraph: subgraphName, blockHeight });
+      logger3.debug("Skipping non-canonical block", {
+        subgraph: subgraphName,
+        blockHeight
+      });
       result.skipped = true;
       return result;
     }
@@ -585,7 +611,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         const { rows } = await sql2.raw(`SELECT COUNT(*) as count FROM "${schemaName}"."${table}"`).execute(db);
         const count = Number(rows[0].count);
         if (count >= 1e7) {
-          logger3.warn("Subgraph table exceeds 10M rows", { subgraph: subgraphName, table, count });
+          logger3.warn("Subgraph table exceeds 10M rows", {
+            subgraph: subgraphName,
+            table,
+            count
+          });
         }
       }
     } catch {}
@@ -639,7 +669,7 @@ class StatsAccumulator {
       flush_time_ms: Math.round(entry.flushTimeMs),
       max_block_time_ms: Math.round(entry.maxBlockTimeMs),
       max_handler_time_ms: Math.round(entry.maxHandlerTimeMs),
-      avg_ops_per_block: parseFloat(avgOpsPerBlock.toFixed(2)),
+      avg_ops_per_block: Number.parseFloat(avgOpsPerBlock.toFixed(2)),
       is_catchup: entry.isCatchup
     }).execute();
   }
@@ -661,10 +691,14 @@ class StatsAccumulator {
 }
 
 // src/runtime/reindex.ts
+import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
 import { getDb as getDb2, getRawClient } from "@secondlayer/shared/db";
+import {
+  recordGapBatch,
+  resolveGaps
+} from "@secondlayer/shared/db/queries/subgraph-gaps";
 import { updateSubgraphStatus as updateSubgraphStatus2 } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger4 } from "@secondlayer/shared/logger";
-import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
 
 // src/schema/generator.ts
 var TYPE_MAP = {
@@ -798,6 +832,28 @@ var LOG_INTERVAL = 1000;
 var DEFAULT_BATCH_SIZE = 500;
 var MIN_BATCH_SIZE = 100;
 var MAX_BATCH_SIZE = 1000;
+function coalesceFailedBlocks(blocks) {
+  if (blocks.length === 0)
+    return [];
+  blocks.sort((a, b) => a.height - b.height);
+  const ranges = [];
+  let start = blocks[0].height;
+  let end = blocks[0].height;
+  let reason = blocks[0].reason;
+  for (let i = 1;i < blocks.length; i++) {
+    const b = blocks[i];
+    if (b.height === end + 1 && b.reason === reason) {
+      end = b.height;
+    } else {
+      ranges.push({ start, end, reason });
+      start = b.height;
+      end = b.height;
+      reason = b.reason;
+    }
+  }
+  ranges.push({ start, end, reason });
+  return ranges;
+}
 async function processBlockRange(def, opts) {
   const db = getDb2();
   const subgraphName = def.name;
@@ -818,9 +874,11 @@ async function processBlockRange(def, opts) {
       const nextEnd = Math.min(nextStart + batchSize - 1, toBlock);
       nextBatchPromise = loadBlockRange(db, nextStart, nextEnd);
     }
+    const batchFailedBlocks = [];
     for (let height = currentHeight;height <= batchEnd; height++) {
       const blockData = batch.get(height);
       if (!blockData) {
+        batchFailedBlocks.push({ height, reason: "block_missing" });
         blocksProcessed++;
         continue;
       }
@@ -836,6 +894,7 @@ async function processBlockRange(def, opts) {
           blockHeight: height,
           error: err instanceof Error ? err.message : String(err)
         });
+        batchFailedBlocks.push({ height, reason: "processing_error" });
         await updateSubgraphStatus2(db, subgraphName, status, height).catch(() => {});
         blocksProcessed++;
         totalErrors++;
@@ -863,6 +922,15 @@ async function processBlockRange(def, opts) {
         });
       }
     }
+    if (batchFailedBlocks.length > 0 && opts.subgraphId) {
+      const gaps = coalesceFailedBlocks(batchFailedBlocks);
+      await recordGapBatch(db, opts.subgraphId, subgraphName, gaps).catch((err) => {
+        logger4.warn("Failed to record subgraph gaps", {
+          subgraph: subgraphName,
+          error: err instanceof Error ? err.message : String(err)
+        });
+      });
+    }
     const avg = avgEventsPerBlock(batch);
     if (avg > 50)
       batchSize = Math.max(Math.round(batchSize * 0.5), MIN_BATCH_SIZE);
@@ -873,14 +941,14 @@ async function processBlockRange(def, opts) {
   await stats.flush(db);
   return { blocksProcessed, totalEventsProcessed, totalErrors };
 }
-async function resolveBlockRange(db, opts) {
-  const fromBlock = opts?.fromBlock ?? 1;
+async function resolveBlockRange(db, def, opts) {
+  const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
   let toBlock;
   if (opts?.toBlock != null) {
     toBlock = opts.toBlock;
   } else {
     const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
-    toBlock = progress?.last_indexed_block ?? progress?.last_contiguous_block ?? 0;
+    toBlock = progress?.highest_seen_block ?? 0;
   }
   return { fromBlock, toBlock };
 }
@@ -898,30 +966,48 @@ async function reindexSubgraph(def, opts) {
       await client.unsafe(stmt);
     }
     logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
-    const { fromBlock, toBlock } = await resolveBlockRange(db, opts);
+    const { fromBlock, toBlock } = await resolveBlockRange(db, def, opts);
     if (fromBlock > toBlock) {
-      logger4.info("No blocks to reindex", { subgraph: subgraphName, fromBlock, toBlock });
+      logger4.info("No blocks to reindex", {
+        subgraph: subgraphName,
+        fromBlock,
+        toBlock
+      });
       await updateSubgraphStatus2(db, subgraphName, "active", 0);
       return { processed: 0 };
     }
-    logger4.info("Reindexing blocks", { subgraph: subgraphName, fromBlock, toBlock, totalBlocks: toBlock - fromBlock + 1 });
-    const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
+    logger4.info("Reindexing blocks", {
+      subgraph: subgraphName,
+      fromBlock,
+      toBlock,
+      totalBlocks: toBlock - fromBlock + 1
+    });
+    const subgraphRow = await db.selectFrom("subgraphs").select(["id", "api_key_id"]).where("name", "=", subgraphName).executeTakeFirst();
     const result = await processBlockRange(def, {
       fromBlock,
       toBlock,
       status: "reindexing",
       isCatchup: false,
-      apiKeyId: subgraphRow?.api_key_id ?? null
+      apiKeyId: subgraphRow?.api_key_id ?? null,
+      subgraphId: subgraphRow?.id
     });
     const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
       await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
     }
     await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
-    logger4.info("Reindex complete", { subgraph: subgraphName, blocks: result.blocksProcessed, events: result.totalEventsProcessed, errors: result.totalErrors });
+    logger4.info("Reindex complete", {
+      subgraph: subgraphName,
+      blocks: result.blocksProcessed,
+      events: result.totalEventsProcessed,
+      errors: result.totalErrors
+    });
     return { processed: result.blocksProcessed };
   } catch (err) {
-    logger4.error("Reindex failed", { subgraph: subgraphName, error: getErrorMessage2(err) });
+    logger4.error("Reindex failed", {
+      subgraph: subgraphName,
+      error: getErrorMessage2(err)
+    });
     await updateSubgraphStatus2(db, subgraphName, "error");
     throw err;
   }
@@ -929,24 +1015,44 @@ async function reindexSubgraph(def, opts) {
 async function backfillSubgraph(def, opts) {
   const db = getDb2();
   const subgraphName = def.name;
-  logger4.info("Backfill starting", { subgraph: subgraphName, from: opts.fromBlock, to: opts.toBlock });
+  logger4.info("Backfill starting", {
+    subgraph: subgraphName,
+    from: opts.fromBlock,
+    to: opts.toBlock
+  });
   try {
-    const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
+    const subgraphRow = await db.selectFrom("subgraphs").select(["id", "api_key_id"]).where("name", "=", subgraphName).executeTakeFirst();
     const result = await processBlockRange(def, {
       fromBlock: opts.fromBlock,
       toBlock: opts.toBlock,
       status: "active",
       isCatchup: false,
-      apiKeyId: subgraphRow?.api_key_id ?? null
+      apiKeyId: subgraphRow?.api_key_id ?? null,
+      subgraphId: subgraphRow?.id
     });
+    const resolved = await resolveGaps(db, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
+    if (resolved > 0) {
+      logger4.info("Resolved subgraph gaps via backfill", {
+        subgraph: subgraphName,
+        resolved
+      });
+    }
     const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
       await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
     }
-    logger4.info("Backfill complete", { subgraph: subgraphName, blocks: result.blocksProcessed, events: result.totalEventsProcessed, errors: result.totalErrors });
+    logger4.info("Backfill complete", {
+      subgraph: subgraphName,
+      blocks: result.blocksProcessed,
+      events: result.totalEventsProcessed,
+      errors: result.totalErrors
+    });
     return { processed: result.blocksProcessed };
   } catch (err) {
-    logger4.error("Backfill failed", { subgraph: subgraphName, error: getErrorMessage2(err) });
+    logger4.error("Backfill failed", {
+      subgraph: subgraphName,
+      error: getErrorMessage2(err)
+    });
     throw err;
   }
 }
@@ -1008,11 +1114,19 @@ async function deploySchema(db, def, handlerPath, opts) {
   const regData = {
     name: def.name,
     version: def.version || "1.0.0",
-    definition: toJsonSafe({ name: def.name, version: def.version, description: def.description, sources: def.sources, schema: def.schema }),
+    definition: toJsonSafe({
+      name: def.name,
+      version: def.version,
+      description: def.description,
+      startBlock: def.startBlock,
+      sources: def.sources,
+      schema: def.schema
+    }),
     schemaHash: hash,
     handlerPath,
     apiKeyId: opts?.apiKeyId,
-    schemaName
+    schemaName,
+    startBlock: def.startBlock
   };
   if (existing) {
     if (existing.schema_hash === hash && !opts?.forceReindex) {
@@ -1128,5 +1242,5 @@ export {
   backfillSubgraph
 };
 
-//# debugId=FAC592B26F4F381E64756E2164756E21
+//# debugId=DED4A221B27D85F264756E2164756E21
 //# sourceMappingURL=index.js.map