@secondlayer/subgraphs 1.3.1 → 1.3.3

package/dist/src/service.js

@@ -187,9 +187,9 @@ class SubgraphContext {
       const blockHeight = op.data._block_height ?? this.block.height;
       const txId = op.data._tx_id ?? this._tx.txId;
       const baseRow = op.kind === "update" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
-      delete baseRow._upsert_keys;
-      delete baseRow._upsert_fallback_keys;
-      delete baseRow._upsert_fallback_set;
+      baseRow._upsert_keys = undefined;
+      baseRow._upsert_fallback_keys = undefined;
+      baseRow._upsert_fallback_set = undefined;
       return {
         op: op.kind,
         table: op.table,
@@ -1336,16 +1336,16 @@ function generateSubgraphSQL(def, schemaNameOverride) {
   const statements = [];
   const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
   if (needsTrgm) {
-    statements.push(`CREATE EXTENSION IF NOT EXISTS pg_trgm`);
+    statements.push("CREATE EXTENSION IF NOT EXISTS pg_trgm");
   }
   statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
   for (const [tableName, tableDef] of Object.entries(def.schema)) {
     const qualifiedName = `${schemaName}.${tableName}`;
     const columnDefs = [
-      `_id BIGSERIAL PRIMARY KEY`,
-      `_block_height BIGINT NOT NULL`,
-      `_tx_id TEXT NOT NULL`,
-      `_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()`
+      "_id BIGSERIAL PRIMARY KEY",
+      "_block_height BIGINT NOT NULL",
+      "_tx_id TEXT NOT NULL",
+      "_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
     ];
     for (const [colName, col] of Object.entries(tableDef.columns)) {
       const sqlType = TYPE_MAP[col.type];
@@ -1441,6 +1441,7 @@ function avgEventsPerBlock(batch) {
 // src/runtime/reindex.ts
 var LOG_INTERVAL = 1000;
 var HEALTH_FLUSH_INTERVAL = 1000;
+var PROGRESS_FLUSH_INTERVAL_MS = 5000;
 var HOBBY_REINDEX_BATCH_CONFIG = {
   defaultBatchSize: 50,
   minBatchSize: 25,
@@ -1515,6 +1516,8 @@ async function processBlockRange(def, opts) {
   let pendingErrors = 0;
   let pendingLastError;
   let lastHealthFlushBlock = 0;
+  let lastHealthFlushAt = Date.now();
+  let lastProgressFlushAt = Date.now();
   const batchConfig = resolveReindexBatchConfig();
   let batchSize = batchConfig.defaultBatchSize;
   let currentHeight = fromBlock;
@@ -1527,6 +1530,7 @@ async function processBlockRange(def, opts) {
     pendingErrors = 0;
     pendingLastError = undefined;
     lastHealthFlushBlock = blocksProcessed;
+    lastHealthFlushAt = Date.now();
   };
   let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
   let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
@@ -1590,8 +1594,11 @@ async function processBlockRange(def, opts) {
           await stats.flush(targetDb);
         }
       }
-      if (blocksProcessed % 100 === 0) {
+      const now = Date.now();
+      const shouldFlushProgress = blocksProcessed % 100 === 0 || now - lastProgressFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
+      if (shouldFlushProgress) {
         await updateSubgraphStatus2(targetDb, subgraphName, status, height);
+        lastProgressFlushAt = now;
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
         logger5.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
@@ -1603,7 +1610,11 @@ async function processBlockRange(def, opts) {
         });
       }
     }
-    if (blocksProcessed - lastHealthFlushBlock >= HEALTH_FLUSH_INTERVAL) {
+    if (status === "reindexing" && blocksProcessed > 0) {
+      await updateSubgraphStatus2(targetDb, subgraphName, status, batchEnd);
+    }
+    const shouldFlushHealth = blocksProcessed - lastHealthFlushBlock >= HEALTH_FLUSH_INTERVAL || Date.now() - lastHealthFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
+    if (shouldFlushHealth) {
       await flushHealth();
     }
     if (batchFailedBlocks.length > 0 && opts.subgraphId) {
@@ -1849,10 +1860,47 @@ import {
 import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger6 } from "@secondlayer/shared/logger";
 var LOG_INTERVAL2 = 1000;
-var DEFAULT_BATCH_SIZE = 500;
-var MIN_BATCH_SIZE = 100;
-var MAX_BATCH_SIZE = 1000;
+var HOBBY_CATCHUP_BATCH_CONFIG = {
+  defaultBatchSize: 50,
+  minBatchSize: 25,
+  maxBatchSize: 100,
+  prefetch: false
+};
+var STANDARD_CATCHUP_BATCH_CONFIG = {
+  defaultBatchSize: 500,
+  minBatchSize: 100,
+  maxBatchSize: 1000,
+  prefetch: true
+};
 var catchingUp = new Set;
+function parsePositiveInt2(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+function parseBoolean(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "true")
+    return true;
+  if (normalized === "false")
+    return false;
+  return;
+}
+function resolveCatchupBatchConfig(env = process.env) {
+  const base = env.TENANT_PLAN?.trim().toLowerCase() === "hobby" ? HOBBY_CATCHUP_BATCH_CONFIG : STANDARD_CATCHUP_BATCH_CONFIG;
+  const minBatchSize = parsePositiveInt2(env.SUBGRAPH_CATCHUP_MIN_BATCH_SIZE) ?? base.minBatchSize;
+  const maxBatchSize = parsePositiveInt2(env.SUBGRAPH_CATCHUP_MAX_BATCH_SIZE) ?? base.maxBatchSize;
+  const defaultBatchSize = parsePositiveInt2(env.SUBGRAPH_CATCHUP_BATCH_SIZE) ?? base.defaultBatchSize;
+  return {
+    minBatchSize,
+    maxBatchSize,
+    defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize),
+    prefetch: parseBoolean(env.SUBGRAPH_CATCHUP_PREFETCH) ?? base.prefetch
+  };
+}
 function coalesceGaps(blocks) {
   if (blocks.length === 0)
     return [];
@@ -1875,11 +1923,11 @@ function coalesceGaps(blocks) {
   ranges.push({ start, end, reason });
   return ranges;
 }
-function adjustBatchSize(current, avgEvents) {
+function adjustBatchSize(current, avgEvents, config) {
   if (avgEvents > 50)
-    return Math.max(Math.round(current * 0.5), MIN_BATCH_SIZE);
+    return Math.max(Math.round(current * 0.5), config.minBatchSize);
   if (avgEvents < 10)
-    return Math.min(Math.round(current * 1.5), MAX_BATCH_SIZE);
+    return Math.min(Math.round(current * 1.5), config.maxBatchSize);
   return current;
 }
 async function catchUpSubgraph(subgraph, subgraphName) {
@@ -1910,10 +1958,11 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     });
     const stats = new StatsAccumulator(subgraphName, true);
     let processed = 0;
-    let batchSize = DEFAULT_BATCH_SIZE;
+    const batchConfig = resolveCatchupBatchConfig();
+    let batchSize = batchConfig.defaultBatchSize;
     let currentHeight = startBlock;
-    let nextBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
-    let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
+    let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
+    let nextBatchPromise = batchConfig.prefetch ? loadBlockRange(sourceDb, currentHeight, prefetchedBatchEnd) : undefined;
     while (currentHeight <= chainTip) {
       const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
@@ -1923,12 +1972,21 @@ async function catchUpSubgraph(subgraph, subgraphName) {
         });
         break;
       }
-      const batch = await nextBatchPromise;
-      const batchEnd = nextBatchEnd;
-      const nextStart = batchEnd + 1;
-      if (nextStart <= chainTip) {
-        nextBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
-        nextBatchPromise = loadBlockRange(sourceDb, nextStart, nextBatchEnd);
+      let batchEnd;
+      let batch;
+      if (nextBatchPromise) {
+        batch = await nextBatchPromise;
+        batchEnd = prefetchedBatchEnd;
+        const nextStart = batchEnd + 1;
+        if (nextStart <= chainTip) {
+          prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
+          nextBatchPromise = loadBlockRange(sourceDb, nextStart, prefetchedBatchEnd);
+        } else {
+          nextBatchPromise = undefined;
+        }
+      } else {
+        batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
+        batch = await loadBlockRange(sourceDb, currentHeight, batchEnd);
       }
       const batchFailedBlocks = [];
       for (let height = currentHeight;height <= batchEnd; height++) {
@@ -1982,7 +2040,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
         });
       }
       const avg = avgEventsPerBlock(batch);
-      batchSize = adjustBatchSize(batchSize, avg);
+      batchSize = adjustBatchSize(batchSize, avg, batchConfig);
       currentHeight = batchEnd + 1;
     }
     await stats.flush(targetDb);
@@ -2070,9 +2128,7 @@ import { listen as listen2 } from "@secondlayer/shared/queue/listener";
 import {
   getTargetDb as getTargetDb5
 } from "@secondlayer/shared/db";
-import {
-  getSubscriptionSigningSecret
-} from "@secondlayer/shared/db/queries/subscriptions";
+import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
 import { logger as logger9 } from "@secondlayer/shared/logger";
 import { listen } from "@secondlayer/shared/queue/listener";
 import { sql as sql4 } from "kysely";
@@ -2736,7 +2792,7 @@ async function startSubgraphOperationRunner(opts) {
     }, HEARTBEAT_INTERVAL_MS);
     const cancelPoll = setInterval(() => {
       getSubgraphOperation(db, operation.id).then((row) => {
-        if (row?.cancel_requested && !controller.signal.aborted) {
+        if ((!row || row.cancel_requested) && !controller.signal.aborted) {
           controller.abort("user-cancelled");
         }
       }).catch((err) => {
@@ -2946,5 +3002,5 @@ var shutdown = async () => {
 process.on("SIGINT", shutdown);
 process.on("SIGTERM", shutdown);
 
-//# debugId=C4D9D2D38E25E18D64756E2164756E21
+//# debugId=A1AAE72D48D7A5B964756E2164756E21
 //# sourceMappingURL=service.js.map