@secondlayer/subgraphs 3.10.0 → 3.12.0

package/dist/src/runtime/processor.js

@@ -1119,6 +1119,28 @@ function reconstructEvent(e) {
 }
 
 // src/runtime/block-source.ts
+function canSparseScan(subgraph) {
+  if (Array.isArray(subgraph.sources))
+    return false;
+  const filters = sourceFilters(subgraph);
+  if (filters.length === 0)
+    return false;
+  return filters.every((f) => Boolean(EVENT_FILTER_TO_INDEX_TYPE[f.type]));
+}
+function sparseProbeTargets(subgraph) {
+  const targets = new Map;
+  for (const f of sourceFilters(subgraph)) {
+    const eventType = EVENT_FILTER_TO_INDEX_TYPE[f.type];
+    if (!eventType)
+      continue;
+    const scoped = f;
+    const contractId = scoped.contractId ?? scoped.assetIdentifier?.split("::")[0];
+    const key = `${eventType}|${contractId ?? ""}`;
+    targets.set(key, { eventType, ...contractId ? { contractId } : {} });
+  }
+  return [...targets.values()];
+}
+
 class PostgresBlockSource {
   async getTip() {
     const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
@@ -1181,9 +1203,18 @@ function isStreamsIndexEligible(subgraph) {
 class PublicApiBlockSource {
   http;
   eventTypes;
-  constructor(http, eventTypes) {
+  probeTargets;
+  constructor(http, eventTypes, probeTargets) {
     this.http = http;
     this.eventTypes = eventTypes;
+    this.probeTargets = probeTargets;
+  }
+  async nextDataHeight(afterHeight, untilHeight) {
+    if (!this.probeTargets?.length)
+      return afterHeight + 1;
+    const hits = await Promise.all(this.probeTargets.map((t) => this.http.firstEventHeight(t.eventType, afterHeight + 1, untilHeight, t.contractId)));
+    const found = hits.filter((h) => h !== null);
+    return found.length ? Math.min(...found) : null;
   }
   getTip() {
     return this.http.getIndexTip();
@@ -1247,6 +1278,15 @@ class FallbackBlockSource {
       return this.fallback.loadBlockRange(fromHeight, toHeight);
     }
   }
+  async nextDataHeight(afterHeight, untilHeight) {
+    if (!this.primary.nextDataHeight)
+      return afterHeight + 1;
+    try {
+      return await this.primary.nextDataHeight(afterHeight, untilHeight);
+    } catch {
+      return afterHeight + 1;
+    }
+  }
 }
 var postgresBlockSource = new PostgresBlockSource;
 function buildHttpClient() {
@@ -1259,7 +1299,7 @@ function buildHttpClient() {
 }
 function resolveBlockSource(subgraph) {
   if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
-    return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph)), postgresBlockSource);
+    return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph), canSparseScan(subgraph) ? sparseProbeTargets(subgraph) : undefined), postgresBlockSource);
   }
   if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
     logger3.debug("Subgraph not streams-index eligible, using DB tap", {
@@ -1742,6 +1782,7 @@ import {
   recordGapBatch,
   resolveGaps
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
+import { updateOperationProcessedEvents } from "@secondlayer/shared/db/queries/subgraph-operations";
 import {
   recordSubgraphProcessed as recordSubgraphProcessed2,
   updateSubgraphStatus as updateSubgraphStatus2
@@ -1929,6 +1970,7 @@ async function processBlockRange(def, opts) {
   let batchSize = batchConfig.defaultBatchSize;
   let currentHeight = fromBlock;
   let aborted = false;
+  const sparse = Boolean(source.nextDataHeight && canSparseScan(def));
   const flushHealth = async () => {
     if (pendingEventsProcessed === 0 && pendingErrors === 0)
       return;
@@ -1959,6 +2001,7 @@ async function processBlockRange(def, opts) {
       nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
     }
     const batchFailedBlocks = [];
+    let batchMatched = 0;
     for (let height = currentHeight;height <= batchEnd; height++) {
       const blockData = batch.get(height);
       if (!blockData) {
@@ -1988,6 +2031,7 @@ async function processBlockRange(def, opts) {
         continue;
       }
       blocksProcessed++;
+      batchMatched += result.matched;
       totalEventsProcessed += result.processed;
       totalErrors += result.errors;
       pendingEventsProcessed += result.processed;
@@ -2005,6 +2049,9 @@ async function processBlockRange(def, opts) {
       const shouldFlushProgress = blocksProcessed % 100 === 0 || now - lastProgressFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
       if (shouldFlushProgress) {
         await updateSubgraphStatus2(targetDb, subgraphName, status, height);
+        if (opts.operationId) {
+          await updateOperationProcessedEvents(targetDb, opts.operationId, totalEventsProcessed).catch(() => {});
+        }
         lastProgressFlushAt = now;
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
@@ -2033,6 +2080,27 @@ async function processBlockRange(def, opts) {
         });
       });
     }
+    if (sparse && batchMatched === 0 && batchEnd < toBlock && source.nextDataHeight) {
+      const next = await source.nextDataHeight(batchEnd, toBlock);
+      const jumpTo = next === null ? toBlock + 1 : Math.max(next, batchEnd + 1);
+      if (jumpTo > batchEnd + 1) {
+        const skipped = Math.min(jumpTo, toBlock + 1) - (batchEnd + 1);
+        blocksProcessed += skipped;
+        await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
+        logger6.info("Sparse skip", {
+          subgraph: subgraphName,
+          from: batchEnd + 1,
+          to: jumpTo - 1,
+          skipped
+        });
+        currentHeight = jumpTo;
+        if (currentHeight <= toBlock) {
+          nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
+          nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
+        }
+        continue;
+      }
+    }
     const avg = avgEventsPerBlock(batch);
     if (avg > 50)
       batchSize = Math.max(Math.round(batchSize * 0.5), batchConfig.minBatchSize);
@@ -2101,6 +2169,7 @@ async function reindexSubgraph(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: subgraphRow?.id,
+      operationId: opts?.operationId,
       signal: opts?.signal
     });
     if (result.aborted) {
@@ -2176,6 +2245,7 @@ async function resumeReindex(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: row.id,
+      operationId: opts.operationId,
       signal: opts.signal
     });
     if (result.aborted) {
@@ -2224,6 +2294,7 @@ async function backfillSubgraph(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: subgraphRow?.id,
+      operationId: opts.operationId,
       signal: opts.signal
     });
     if (result.aborted) {
@@ -2766,6 +2837,7 @@ async function runSubgraphOperation(operation, signal) {
       fromBlock: Number(operation.from_block),
       toBlock: Number(operation.to_block),
       schemaName,
+      operationId: operation.id,
       signal
     });
     return result2.processed;
@@ -2775,13 +2847,18 @@ async function runSubgraphOperation(operation, signal) {
   }
   const hasResumeMetadata = subgraph.status === "reindexing" && subgraph.reindex_from_block != null && subgraph.reindex_to_block != null;
   if (hasResumeMetadata) {
-    const result2 = await resumeReindex(def, { schemaName, signal });
+    const result2 = await resumeReindex(def, {
+      schemaName,
+      operationId: operation.id,
+      signal
+    });
     return result2.processed;
   }
   const result = await reindexSubgraph(def, {
     fromBlock: operation.from_block == null ? undefined : Number(operation.from_block),
     toBlock: operation.to_block == null ? undefined : Number(operation.to_block),
     schemaName,
+    operationId: operation.id,
     signal
   });
   return result.processed;
@@ -2795,6 +2872,27 @@ async function startSubgraphOperationRunner(opts) {
   let running = true;
   let draining = false;
   logger10.info("Starting subgraph operation runner", { concurrency, lockedBy });
+  try {
+    const stranded = await db.selectFrom("subgraphs").select(["id", "name", "account_id"]).where("status", "=", "reindexing").where("reindex_from_block", "is not", null).where("reindex_to_block", "is not", null).where(({ not, exists, selectFrom }) => not(exists(selectFrom("subgraph_operations").select("id").whereRef("subgraph_id", "=", "subgraphs.id").where("status", "in", ["queued", "running"])))).execute();
+    for (const row of stranded) {
+      try {
+        await createSubgraphOperation(db, {
+          subgraphId: row.id,
+          subgraphName: row.name,
+          accountId: row.account_id,
+          kind: "reindex"
+        });
+        logger10.info("Re-enqueued stranded reindex", { subgraph: row.name });
+      } catch (err) {
+        if (!isActiveSubgraphOperationConflict(err))
+          throw err;
+      }
+    }
+  } catch (err) {
+    logger10.warn("Stranded-reindex sweep failed", {
+      error: getErrorMessage5(err)
+    });
+  }
   const startOperation = (operation) => {
     const controller = new AbortController;
     active.set(operation.id, controller);
@@ -2970,5 +3068,5 @@ export {
   startSubgraphOperationRunner
 };
 
-//# debugId=663C20C581D3E38564756E2164756E21
+//# debugId=CA67CC38C44D43DF64756E2164756E21
 //# sourceMappingURL=processor.js.map