@secondlayer/subgraphs 0.7.2 → 0.8.0

package/dist/src/index.d.ts

@@ -87,15 +87,28 @@ interface ReindexOptions {
 	fromBlock?: number;
 	toBlock?: number;
 	schemaName?: string;
+	signal?: AbortSignal;
 }
 /**
 * Reindex a subgraph by dropping its tables, recreating them, and reprocessing
 * all historical blocks through the handler pipeline.
+* Supports cancellation via AbortSignal — on shutdown abort, status stays
+* "reindexing" for auto-resume; on user cancel, status resets to "active".
 */
 declare function reindexSubgraph(def: SubgraphDefinition, opts?: ReindexOptions): Promise<{
 	processed: number
 }>;
 /**
+* Resume a previously interrupted reindex. Skips schema drop (already done)
+* and continues from last_processed_block + 1.
+*/
+declare function resumeReindex(def: SubgraphDefinition, opts: {
+	schemaName: string
+	signal?: AbortSignal
+}): Promise<{
+	processed: number
+}>;
+/**
 * Backfill a subgraph by re-processing a block range WITHOUT dropping the schema.
 * Uses upserts so existing data is updated, not duplicated. Safe to run while
 * the subgraph is actively syncing.
@@ -104,6 +117,7 @@ declare function backfillSubgraph(def: SubgraphDefinition, opts: {
 	fromBlock: number
 	toBlock: number
 	schemaName?: string
+	signal?: AbortSignal
 }): Promise<{
 	processed: number
 }>;
@@ -230,4 +244,4 @@ interface SubgraphTableClient<TRow> {
 type InferSubgraphClient<T> = T extends {
 	schema: infer S
 } ? { [K in keyof S] : S[K] extends SubgraphTable ? SubgraphTableClient<InferTableRow<S[K]>> : never } : never;
-export { validateSubgraphDefinition, sourceKey, reindexSubgraph, pgSchemaName, generateSubgraphSQL, diffSchema, deploySchema, defineSubgraph, backfillSubgraph, WhereInput, TableDiff, SystemRow, SubgraphTableClient, SubgraphTable, SubgraphSource, SubgraphSchema, SubgraphHandler, SubgraphDefinition, SubgraphContext, SubgraphColumn, ReindexOptions, InferTableRow, InferSubgraphClient, InferColumnType, GeneratedSQL, FindManyOptions, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff };
+export { validateSubgraphDefinition, sourceKey, resumeReindex, reindexSubgraph, pgSchemaName, generateSubgraphSQL, diffSchema, deploySchema, defineSubgraph, backfillSubgraph, WhereInput, TableDiff, SystemRow, SubgraphTableClient, SubgraphTable, SubgraphSource, SubgraphSchema, SubgraphHandler, SubgraphDefinition, SubgraphContext, SubgraphColumn, ReindexOptions, InferTableRow, InferSubgraphClient, InferColumnType, GeneratedSQL, FindManyOptions, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff };

package/dist/src/index.js

@@ -411,11 +411,17 @@ async function runHandlers(subgraph, matched, ctx, opts) {
 }
 
 // src/runtime/source-matcher.ts
+var patternCache = new Map;
 function matchPattern(value, pattern) {
   if (!pattern.includes("*"))
     return value === pattern;
-  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-  return new RegExp(`^${regex}$`).test(value);
+  let re = patternCache.get(pattern);
+  if (!re) {
+    const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+    re = new RegExp(`^${regex}$`);
+    patternCache.set(pattern, re);
+  }
+  return re.test(value);
 }
 function matchSource(source, transactions, eventsByTx) {
   const results = [];
@@ -608,10 +614,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     try {
       const tables = Object.keys(subgraph.schema);
       for (const table of tables) {
-        const { rows } = await sql2.raw(`SELECT COUNT(*) as count FROM "${schemaName}"."${table}"`).execute(db);
-        const count = Number(rows[0].count);
+        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(db);
+        const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
-          logger3.warn("Subgraph table exceeds 10M rows", {
+          logger3.warn("Subgraph table exceeds 10M rows (estimate)", {
             subgraph: subgraphName,
             table,
             count
@@ -865,8 +871,18 @@ async function processBlockRange(def, opts) {
   let totalErrors = 0;
   let batchSize = DEFAULT_BATCH_SIZE;
   let currentHeight = fromBlock;
+  let aborted = false;
   let nextBatchPromise = loadBlockRange(db, currentHeight, Math.min(currentHeight + batchSize - 1, toBlock));
   while (currentHeight <= toBlock) {
+    if (opts.signal?.aborted) {
+      aborted = true;
+      logger4.info("Block processing aborted", {
+        subgraph: subgraphName,
+        currentBlock: currentHeight,
+        reason: String(opts.signal.reason ?? "unknown")
+      });
+      break;
+    }
     const batch = await nextBatchPromise;
     const batchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
     const nextStart = batchEnd + 1;
@@ -939,7 +955,7 @@ async function processBlockRange(def, opts) {
     currentHeight = batchEnd + 1;
   }
   await stats.flush(db);
-  return { blocksProcessed, totalEventsProcessed, totalErrors };
+  return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
 }
 async function resolveBlockRange(db, def, opts) {
   const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
@@ -952,6 +968,9 @@ async function resolveBlockRange(db, def, opts) {
   }
   return { fromBlock, toBlock };
 }
+async function clearReindexMetadata(db, subgraphName) {
+  await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
+}
 async function reindexSubgraph(def, opts) {
   const db = getDb2();
   const client = getRawClient();
@@ -960,12 +979,6 @@ async function reindexSubgraph(def, opts) {
   await updateSubgraphStatus2(db, subgraphName, "reindexing");
   logger4.info("Reindex starting", { subgraph: subgraphName });
   try {
-    await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
-    const { statements } = generateSubgraphSQL(def, schemaName);
-    for (const stmt of statements) {
-      await client.unsafe(stmt);
-    }
-    logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
     const { fromBlock, toBlock } = await resolveBlockRange(db, def, opts);
     if (fromBlock > toBlock) {
       logger4.info("No blocks to reindex", {
@@ -976,6 +989,13 @@ async function reindexSubgraph(def, opts) {
       await updateSubgraphStatus2(db, subgraphName, "active", 0);
       return { processed: 0 };
     }
+    await db.updateTable("subgraphs").set({ reindex_from_block: fromBlock, reindex_to_block: toBlock }).where("name", "=", subgraphName).execute();
+    await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
+    const { statements } = generateSubgraphSQL(def, schemaName);
+    for (const stmt of statements) {
+      await client.unsafe(stmt);
+    }
+    logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
     logger4.info("Reindexing blocks", {
       subgraph: subgraphName,
       fromBlock,
@@ -989,13 +1009,28 @@ async function reindexSubgraph(def, opts) {
       status: "reindexing",
       isCatchup: false,
       apiKeyId: subgraphRow?.api_key_id ?? null,
-      subgraphId: subgraphRow?.id
+      subgraphId: subgraphRow?.id,
+      signal: opts?.signal
     });
+    if (result.aborted) {
+      const reason = String(opts?.signal?.reason ?? "unknown");
+      if (reason === "user-cancelled") {
+        await updateSubgraphStatus2(db, subgraphName, "active");
+        await clearReindexMetadata(db, subgraphName);
+        logger4.info("Reindex cancelled by user", { subgraph: subgraphName });
+      } else {
+        logger4.info("Reindex interrupted by shutdown, will resume", {
+          subgraph: subgraphName
+        });
+      }
+      return { processed: result.blocksProcessed };
+    }
     const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
       await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
     }
     await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
+    await clearReindexMetadata(db, subgraphName);
     logger4.info("Reindex complete", {
       subgraph: subgraphName,
       blocks: result.blocksProcessed,
@@ -1012,6 +1047,84 @@ async function reindexSubgraph(def, opts) {
     throw err;
   }
 }
+async function resumeReindex(def, opts) {
+  const db = getDb2();
+  const subgraphName = def.name;
+  const row = await db.selectFrom("subgraphs").select([
+    "id",
+    "api_key_id",
+    "last_processed_block",
+    "reindex_from_block",
+    "reindex_to_block"
+  ]).where("name", "=", subgraphName).executeTakeFirst();
+  if (!row)
+    throw new Error(`Subgraph "${subgraphName}" not found`);
+  if (row.reindex_from_block == null || row.reindex_to_block == null) {
+    logger4.info("No reindex metadata, starting fresh reindex", {
+      subgraph: subgraphName
+    });
+    return reindexSubgraph(def, {
+      schemaName: opts.schemaName,
+      signal: opts.signal
+    });
+  }
+  const fromBlock = Math.max(row.last_processed_block + 1, row.reindex_from_block);
+  const toBlock = row.reindex_to_block;
+  if (fromBlock > toBlock) {
+    logger4.info("Resume: no remaining blocks", { subgraph: subgraphName });
+    await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
+    await clearReindexMetadata(db, subgraphName);
+    return { processed: 0 };
+  }
+  logger4.info("Resuming reindex", {
+    subgraph: subgraphName,
+    fromBlock,
+    toBlock,
+    remaining: toBlock - fromBlock + 1
+  });
+  try {
+    const result = await processBlockRange(def, {
+      fromBlock,
+      toBlock,
+      status: "reindexing",
+      isCatchup: false,
+      apiKeyId: row.api_key_id ?? null,
+      subgraphId: row.id,
+      signal: opts.signal
+    });
+    if (result.aborted) {
+      const reason = String(opts.signal?.reason ?? "unknown");
+      if (reason === "user-cancelled") {
+        await updateSubgraphStatus2(db, subgraphName, "active");
+        await clearReindexMetadata(db, subgraphName);
+        logger4.info("Resume cancelled by user", { subgraph: subgraphName });
+      } else {
+        logger4.info("Resume interrupted by shutdown, will resume again", {
+          subgraph: subgraphName
+        });
+      }
+      return { processed: result.blocksProcessed };
+    }
+    const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
+    if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
+      await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during resumed reindex` : undefined);
+    }
+    await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
+    await clearReindexMetadata(db, subgraphName);
+    logger4.info("Resumed reindex complete", {
+      subgraph: subgraphName,
+      blocks: result.blocksProcessed
+    });
+    return { processed: result.blocksProcessed };
+  } catch (err) {
+    logger4.error("Resumed reindex failed", {
+      subgraph: subgraphName,
+      error: getErrorMessage2(err)
+    });
+    await updateSubgraphStatus2(db, subgraphName, "error");
+    throw err;
+  }
+}
 async function backfillSubgraph(def, opts) {
   const db = getDb2();
   const subgraphName = def.name;
@@ -1028,8 +1141,13 @@ async function backfillSubgraph(def, opts) {
       status: "active",
       isCatchup: false,
       apiKeyId: subgraphRow?.api_key_id ?? null,
-      subgraphId: subgraphRow?.id
+      subgraphId: subgraphRow?.id,
+      signal: opts.signal
     });
+    if (result.aborted) {
+      logger4.info("Backfill aborted", { subgraph: subgraphName });
+      return { processed: result.blocksProcessed };
+    }
     const resolved = await resolveGaps(db, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
     if (resolved > 0) {
       logger4.info("Resolved subgraph gaps via backfill", {
@@ -1233,6 +1351,7 @@ function getDefault(type) {
 export {
   validateSubgraphDefinition,
   sourceKey,
+  resumeReindex,
   reindexSubgraph,
   pgSchemaName,
   generateSubgraphSQL,
@@ -1242,5 +1361,5 @@ export {
   backfillSubgraph
 };
 
-//# debugId=DED4A221B27D85F264756E2164756E21
+//# debugId=7A65C6186191C40964756E2164756E21
 //# sourceMappingURL=index.js.map