@secondlayer/subgraphs 1.3.2 → 1.3.3

package/dist/src/service.js

@@ -187,9 +187,9 @@ class SubgraphContext {
       const blockHeight = op.data._block_height ?? this.block.height;
       const txId = op.data._tx_id ?? this._tx.txId;
       const baseRow = op.kind === "update" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
-      delete baseRow._upsert_keys;
-      delete baseRow._upsert_fallback_keys;
-      delete baseRow._upsert_fallback_set;
+      baseRow._upsert_keys = undefined;
+      baseRow._upsert_fallback_keys = undefined;
+      baseRow._upsert_fallback_set = undefined;
       return {
         op: op.kind,
         table: op.table,
@@ -1336,16 +1336,16 @@ function generateSubgraphSQL(def, schemaNameOverride) {
   const statements = [];
   const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
   if (needsTrgm) {
-    statements.push(`CREATE EXTENSION IF NOT EXISTS pg_trgm`);
+    statements.push("CREATE EXTENSION IF NOT EXISTS pg_trgm");
   }
   statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
   for (const [tableName, tableDef] of Object.entries(def.schema)) {
     const qualifiedName = `${schemaName}.${tableName}`;
     const columnDefs = [
-      `_id BIGSERIAL PRIMARY KEY`,
-      `_block_height BIGINT NOT NULL`,
-      `_tx_id TEXT NOT NULL`,
-      `_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()`
+      "_id BIGSERIAL PRIMARY KEY",
+      "_block_height BIGINT NOT NULL",
+      "_tx_id TEXT NOT NULL",
+      "_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
     ];
     for (const [colName, col] of Object.entries(tableDef.columns)) {
       const sqlType = TYPE_MAP[col.type];
@@ -1860,10 +1860,47 @@ import {
 import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger6 } from "@secondlayer/shared/logger";
 var LOG_INTERVAL2 = 1000;
-var DEFAULT_BATCH_SIZE = 500;
-var MIN_BATCH_SIZE = 100;
-var MAX_BATCH_SIZE = 1000;
+var HOBBY_CATCHUP_BATCH_CONFIG = {
+  defaultBatchSize: 50,
+  minBatchSize: 25,
+  maxBatchSize: 100,
+  prefetch: false
+};
+var STANDARD_CATCHUP_BATCH_CONFIG = {
+  defaultBatchSize: 500,
+  minBatchSize: 100,
+  maxBatchSize: 1000,
+  prefetch: true
+};
 var catchingUp = new Set;
+function parsePositiveInt2(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+function parseBoolean(value) {
+  if (value == null || value.trim() === "")
+    return;
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "true")
+    return true;
+  if (normalized === "false")
+    return false;
+  return;
+}
+function resolveCatchupBatchConfig(env = process.env) {
+  const base = env.TENANT_PLAN?.trim().toLowerCase() === "hobby" ? HOBBY_CATCHUP_BATCH_CONFIG : STANDARD_CATCHUP_BATCH_CONFIG;
+  const minBatchSize = parsePositiveInt2(env.SUBGRAPH_CATCHUP_MIN_BATCH_SIZE) ?? base.minBatchSize;
+  const maxBatchSize = parsePositiveInt2(env.SUBGRAPH_CATCHUP_MAX_BATCH_SIZE) ?? base.maxBatchSize;
+  const defaultBatchSize = parsePositiveInt2(env.SUBGRAPH_CATCHUP_BATCH_SIZE) ?? base.defaultBatchSize;
+  return {
+    minBatchSize,
+    maxBatchSize,
+    defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize),
+    prefetch: parseBoolean(env.SUBGRAPH_CATCHUP_PREFETCH) ?? base.prefetch
+  };
+}
 function coalesceGaps(blocks) {
   if (blocks.length === 0)
     return [];
@@ -1886,11 +1923,11 @@ function coalesceGaps(blocks) {
   ranges.push({ start, end, reason });
   return ranges;
 }
-function adjustBatchSize(current, avgEvents) {
+function adjustBatchSize(current, avgEvents, config) {
   if (avgEvents > 50)
-    return Math.max(Math.round(current * 0.5), MIN_BATCH_SIZE);
+    return Math.max(Math.round(current * 0.5), config.minBatchSize);
   if (avgEvents < 10)
-    return Math.min(Math.round(current * 1.5), MAX_BATCH_SIZE);
+    return Math.min(Math.round(current * 1.5), config.maxBatchSize);
   return current;
 }
 async function catchUpSubgraph(subgraph, subgraphName) {
@@ -1921,10 +1958,11 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     });
     const stats = new StatsAccumulator(subgraphName, true);
     let processed = 0;
-    let batchSize = DEFAULT_BATCH_SIZE;
+    const batchConfig = resolveCatchupBatchConfig();
+    let batchSize = batchConfig.defaultBatchSize;
     let currentHeight = startBlock;
-    let nextBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
-    let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
+    let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
+    let nextBatchPromise = batchConfig.prefetch ? loadBlockRange(sourceDb, currentHeight, prefetchedBatchEnd) : undefined;
     while (currentHeight <= chainTip) {
       const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
@@ -1934,12 +1972,21 @@ async function catchUpSubgraph(subgraph, subgraphName) {
         });
         break;
       }
-      const batch = await nextBatchPromise;
-      const batchEnd = nextBatchEnd;
-      const nextStart = batchEnd + 1;
-      if (nextStart <= chainTip) {
-        nextBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
-        nextBatchPromise = loadBlockRange(sourceDb, nextStart, nextBatchEnd);
+      let batchEnd;
+      let batch;
+      if (nextBatchPromise) {
+        batch = await nextBatchPromise;
+        batchEnd = prefetchedBatchEnd;
+        const nextStart = batchEnd + 1;
+        if (nextStart <= chainTip) {
+          prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
+          nextBatchPromise = loadBlockRange(sourceDb, nextStart, prefetchedBatchEnd);
+        } else {
+          nextBatchPromise = undefined;
+        }
+      } else {
+        batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
+        batch = await loadBlockRange(sourceDb, currentHeight, batchEnd);
       }
       const batchFailedBlocks = [];
       for (let height = currentHeight;height <= batchEnd; height++) {
@@ -1993,7 +2040,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
         });
       }
       const avg = avgEventsPerBlock(batch);
-      batchSize = adjustBatchSize(batchSize, avg);
+      batchSize = adjustBatchSize(batchSize, avg, batchConfig);
       currentHeight = batchEnd + 1;
     }
     await stats.flush(targetDb);
@@ -2081,9 +2128,7 @@ import { listen as listen2 } from "@secondlayer/shared/queue/listener";
 import {
   getTargetDb as getTargetDb5
 } from "@secondlayer/shared/db";
-import {
-  getSubscriptionSigningSecret
-} from "@secondlayer/shared/db/queries/subscriptions";
+import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
 import { logger as logger9 } from "@secondlayer/shared/logger";
 import { listen } from "@secondlayer/shared/queue/listener";
 import { sql as sql4 } from "kysely";
@@ -2957,5 +3002,5 @@ var shutdown = async () => {
 process.on("SIGINT", shutdown);
 process.on("SIGTERM", shutdown);
 
-//# debugId=C53337E6DAF60BED64756E2164756E21
+//# debugId=A1AAE72D48D7A5B964756E2164756E21
 //# sourceMappingURL=service.js.map