@secondlayer/subgraphs 0.11.7 → 1.0.0-alpha.0

package/dist/src/runtime/reindex.js

@@ -849,7 +849,10 @@ function matchSources(sources, transactions, events) {
 }
 
 // src/runtime/block-processor.ts
-import { getDb } from "@secondlayer/shared/db";
+import {
+  getSourceDb,
+  getTargetDb
+} from "@secondlayer/shared/db";
 import {
   recordSubgraphProcessed,
   updateSubgraphStatus
@@ -863,7 +866,8 @@ import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 // src/runtime/block-processor.ts
 var schemaNameCache = new Map;
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
-  const db = getDb();
+  const sourceDb = getSourceDb();
+  const targetDb = getTargetDb();
   const blockStart = performance.now();
   const result = {
     blockHeight,
@@ -872,13 +876,15 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     errors: 0,
     skipped: false
   };
-  let block, txs, evts;
+  let block;
+  let txs;
+  let evts;
   if (opts?.preloaded) {
     block = opts.preloaded.block;
     txs = opts.preloaded.txs;
     evts = opts.preloaded.events;
   } else {
-    block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
+    block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
     if (!block) {
       logger3.warn("Block not found for subgraph processing", {
         subgraph: subgraphName,
@@ -895,20 +901,20 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
       result.skipped = true;
       return result;
     }
-    txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
-    evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
+    txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
+    evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   }
   const matched = matchSources(subgraph.sources, txs, evts);
   result.matched = matched.length;
   if (matched.length === 0) {
     if (!opts?.skipProgressUpdate) {
-      await updateSubgraphStatus(db, subgraphName, "active", blockHeight);
+      await updateSubgraphStatus(targetDb, subgraphName, "active", blockHeight);
     }
     return result;
   }
   let schemaName = schemaNameCache.get(subgraphName);
   if (!schemaName) {
-    const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
+    const subgraphRecord = await targetDb.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
     schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
     schemaNameCache.set(subgraphName, schemaName);
   }
@@ -926,7 +932,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   };
   let handlerMs = 0;
   let flushMs = 0;
-  await db.transaction().execute(async (tx) => {
+  await targetDb.transaction().execute(async (tx) => {
     const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx);
     const handlerStart = performance.now();
     const runResult = await runHandlers(subgraph, matched, ctx);
@@ -957,7 +963,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     try {
       const tables = Object.keys(subgraph.schema);
       for (const table of tables) {
-        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(db);
+        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
           logger3.warn("Subgraph table exceeds 10M rows (estimate)", {
@@ -978,13 +984,11 @@ var FLUSH_INTERVAL_MS = 60000;
 
 class StatsAccumulator {
   subgraphName;
-  apiKeyId;
   isCatchup;
   current;
   lastFlush = Date.now();
-  constructor(subgraphName, apiKeyId, isCatchup) {
+  constructor(subgraphName, isCatchup) {
     this.subgraphName = subgraphName;
-    this.apiKeyId = apiKeyId;
     this.isCatchup = isCatchup;
     this.current = this.newEntry();
   }
@@ -1009,7 +1013,6 @@ class StatsAccumulator {
     const avgOpsPerBlock = entry.blocksProcessed > 0 ? entry.totalOps / entry.blocksProcessed : 0;
     await db.insertInto("subgraph_processing_stats").values({
       subgraph_name: entry.subgraphName,
-      api_key_id: entry.apiKeyId,
       bucket_start: entry.bucketStart,
       bucket_end: new Date,
       blocks_processed: entry.blocksProcessed,
@@ -1025,7 +1028,6 @@ class StatsAccumulator {
   newEntry() {
     return {
       subgraphName: this.subgraphName,
-      apiKeyId: this.apiKeyId,
       bucketStart: new Date,
       blocksProcessed: 0,
       totalTimeMs: 0,
@@ -1041,7 +1043,7 @@ class StatsAccumulator {
 
 // src/runtime/reindex.ts
 import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
-import { getDb as getDb2, getRawClient } from "@secondlayer/shared/db";
+import { getRawClient, getSourceDb as getSourceDb2, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
 import {
   recordGapBatch,
   resolveGaps
@@ -1206,11 +1208,12 @@ function coalesceFailedBlocks(blocks) {
   return ranges;
 }
 async function processBlockRange(def, opts) {
-  const db = getDb2();
+  const sourceDb = getSourceDb2();
+  const targetDb = getTargetDb2();
   const subgraphName = def.name;
   const { fromBlock, toBlock, status } = opts;
   const totalBlocks = toBlock - fromBlock + 1;
-  const stats = new StatsAccumulator(subgraphName, opts.apiKeyId, opts.isCatchup);
+  const stats = new StatsAccumulator(subgraphName, opts.isCatchup);
   let blocksProcessed = 0;
   let totalEventsProcessed = 0;
   let totalErrors = 0;
@@ -1218,7 +1221,7 @@ async function processBlockRange(def, opts) {
   let currentHeight = fromBlock;
   let aborted = false;
   let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
-  let nextBatchPromise = loadBlockRange(db, currentHeight, nextBatchEnd);
+  let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
   while (currentHeight <= toBlock) {
     if (opts.signal?.aborted) {
       aborted = true;
@@ -1234,7 +1237,7 @@ async function processBlockRange(def, opts) {
     const nextStart = batchEnd + 1;
     if (nextStart <= toBlock) {
       nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
-      nextBatchPromise = loadBlockRange(db, nextStart, nextBatchEnd);
+      nextBatchPromise = loadBlockRange(sourceDb, nextStart, nextBatchEnd);
     }
     const batchFailedBlocks = [];
     for (let height = currentHeight;height <= batchEnd; height++) {
@@ -1258,8 +1261,8 @@ async function processBlockRange(def, opts) {
           error: errorMsg
         });
         batchFailedBlocks.push({ height, reason: "processing_error" });
-        await updateSubgraphStatus2(db, subgraphName, status, height).catch(() => {});
-        await recordSubgraphProcessed2(db, subgraphName, 0, 1, errorMsg).catch(() => {});
+        await updateSubgraphStatus2(targetDb, subgraphName, status, height).catch(() => {});
+        await recordSubgraphProcessed2(targetDb, subgraphName, 0, 1, errorMsg).catch(() => {});
         blocksProcessed++;
         totalErrors++;
         continue;
@@ -1270,11 +1273,11 @@ async function processBlockRange(def, opts) {
       if (result.timing) {
         stats.record(result.timing, result.processed);
         if (stats.shouldFlush()) {
-          await stats.flush(db);
+          await stats.flush(targetDb);
         }
       }
       if (blocksProcessed % 100 === 0) {
-        await updateSubgraphStatus2(db, subgraphName, status, height);
+        await updateSubgraphStatus2(targetDb, subgraphName, status, height);
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
         logger4.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
@@ -1288,7 +1291,7 @@ async function processBlockRange(def, opts) {
     }
     if (batchFailedBlocks.length > 0 && opts.subgraphId) {
       const gaps = coalesceFailedBlocks(batchFailedBlocks);
-      await recordGapBatch(db, opts.subgraphId, subgraphName, gaps).catch((err) => {
+      await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
         logger4.warn("Failed to record subgraph gaps", {
           subgraph: subgraphName,
           error: err instanceof Error ? err.message : String(err)
@@ -1302,16 +1305,16 @@ async function processBlockRange(def, opts) {
       batchSize = Math.min(Math.round(batchSize * 1.5), MAX_BATCH_SIZE);
     currentHeight = batchEnd + 1;
   }
-  await stats.flush(db);
+  await stats.flush(targetDb);
   return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
 }
-async function resolveBlockRange(db, def, opts) {
+async function resolveBlockRange(sourceDb, def, opts) {
   const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
   let toBlock;
   if (opts?.toBlock != null) {
     toBlock = opts.toBlock;
   } else {
-    const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
     toBlock = progress?.highest_seen_block ?? 0;
   }
   return { fromBlock, toBlock };
@@ -1320,24 +1323,25 @@ async function clearReindexMetadata(db, subgraphName) {
   await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
 }
 async function reindexSubgraph(def, opts) {
-  const db = getDb2();
-  const client = getRawClient();
+  const sourceDb = getSourceDb2();
+  const targetDb = getTargetDb2();
+  const client = getRawClient("target");
   const subgraphName = def.name;
   const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
-  await updateSubgraphStatus2(db, subgraphName, "reindexing");
+  await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
   logger4.info("Reindex starting", { subgraph: subgraphName });
   try {
-    const { fromBlock, toBlock } = await resolveBlockRange(db, def, opts);
+    const { fromBlock, toBlock } = await resolveBlockRange(sourceDb, def, opts);
     if (fromBlock > toBlock) {
       logger4.info("No blocks to reindex", {
         subgraph: subgraphName,
         fromBlock,
         toBlock
       });
-      await updateSubgraphStatus2(db, subgraphName, "active", 0);
+      await updateSubgraphStatus2(targetDb, subgraphName, "active", 0);
       return { processed: 0 };
     }
-    await db.updateTable("subgraphs").set({ reindex_from_block: fromBlock, reindex_to_block: toBlock }).where("name", "=", subgraphName).execute();
+    await targetDb.updateTable("subgraphs").set({ reindex_from_block: fromBlock, reindex_to_block: toBlock }).where("name", "=", subgraphName).execute();
     await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
     const { statements } = generateSubgraphSQL(def, schemaName);
     for (const stmt of statements) {
@@ -1350,21 +1354,21 @@ async function reindexSubgraph(def, opts) {
       toBlock,
       totalBlocks: toBlock - fromBlock + 1
     });
-    const subgraphRow = await db.selectFrom("subgraphs").select(["id", "api_key_id"]).where("name", "=", subgraphName).executeTakeFirst();
+    const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
     const result = await processBlockRange(def, {
       fromBlock,
       toBlock,
       status: "reindexing",
       isCatchup: false,
-      apiKeyId: subgraphRow?.api_key_id ?? null,
+      apiKeyId: null,
       subgraphId: subgraphRow?.id,
       signal: opts?.signal
     });
     if (result.aborted) {
       const reason = String(opts?.signal?.reason ?? "unknown");
       if (reason === "user-cancelled") {
-        await updateSubgraphStatus2(db, subgraphName, "active");
-        await clearReindexMetadata(db, subgraphName);
+        await updateSubgraphStatus2(targetDb, subgraphName, "active");
+        await clearReindexMetadata(targetDb, subgraphName);
         logger4.info("Reindex cancelled by user", { subgraph: subgraphName });
       } else {
         logger4.info("Reindex interrupted by shutdown, will resume", {
@@ -1375,10 +1379,10 @@ async function reindexSubgraph(def, opts) {
     }
     const { recordSubgraphProcessed: recordSubgraphProcessed3 } = await import("@secondlayer/shared/db/queries/subgraphs");
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
-      await recordSubgraphProcessed3(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
+      await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
     }
-    await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
-    await clearReindexMetadata(db, subgraphName);
+    await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
+    await clearReindexMetadata(targetDb, subgraphName);
     logger4.info("Reindex complete", {
       subgraph: subgraphName,
       blocks: result.blocksProcessed,
@@ -1391,16 +1395,15 @@ async function reindexSubgraph(def, opts) {
       subgraph: subgraphName,
       error: getErrorMessage2(err)
     });
-    await updateSubgraphStatus2(db, subgraphName, "error");
+    await updateSubgraphStatus2(targetDb, subgraphName, "error");
     throw err;
   }
 }
 async function resumeReindex(def, opts) {
-  const db = getDb2();
+  const targetDb = getTargetDb2();
   const subgraphName = def.name;
-  const row = await db.selectFrom("subgraphs").select([
+  const row = await targetDb.selectFrom("subgraphs").select([
     "id",
-    "api_key_id",
     "last_processed_block",
     "reindex_from_block",
     "reindex_to_block"
@@ -1420,8 +1423,8 @@ async function resumeReindex(def, opts) {
   const toBlock = row.reindex_to_block;
   if (fromBlock > toBlock) {
     logger4.info("Resume: no remaining blocks", { subgraph: subgraphName });
-    await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
-    await clearReindexMetadata(db, subgraphName);
+    await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
+    await clearReindexMetadata(targetDb, subgraphName);
     return { processed: 0 };
   }
   logger4.info("Resuming reindex", {
@@ -1436,15 +1439,15 @@ async function resumeReindex(def, opts) {
       toBlock,
       status: "reindexing",
       isCatchup: false,
-      apiKeyId: row.api_key_id ?? null,
+      apiKeyId: null,
       subgraphId: row.id,
       signal: opts.signal
     });
     if (result.aborted) {
       const reason = String(opts.signal?.reason ?? "unknown");
       if (reason === "user-cancelled") {
-        await updateSubgraphStatus2(db, subgraphName, "active");
-        await clearReindexMetadata(db, subgraphName);
+        await updateSubgraphStatus2(targetDb, subgraphName, "active");
+        await clearReindexMetadata(targetDb, subgraphName);
         logger4.info("Resume cancelled by user", { subgraph: subgraphName });
       } else {
         logger4.info("Resume interrupted by shutdown, will resume again", {
@@ -1455,10 +1458,10 @@ async function resumeReindex(def, opts) {
     }
     const { recordSubgraphProcessed: recordSubgraphProcessed3 } = await import("@secondlayer/shared/db/queries/subgraphs");
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
-      await recordSubgraphProcessed3(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during resumed reindex` : undefined);
+      await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during resumed reindex` : undefined);
     }
-    await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
-    await clearReindexMetadata(db, subgraphName);
+    await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
+    await clearReindexMetadata(targetDb, subgraphName);
     logger4.info("Resumed reindex complete", {
       subgraph: subgraphName,
       blocks: result.blocksProcessed
@@ -1469,12 +1472,12 @@ async function resumeReindex(def, opts) {
       subgraph: subgraphName,
       error: getErrorMessage2(err)
     });
-    await updateSubgraphStatus2(db, subgraphName, "error");
+    await updateSubgraphStatus2(targetDb, subgraphName, "error");
     throw err;
   }
 }
 async function backfillSubgraph(def, opts) {
-  const db = getDb2();
+  const targetDb = getTargetDb2();
   const subgraphName = def.name;
   logger4.info("Backfill starting", {
     subgraph: subgraphName,
@@ -1482,13 +1485,13 @@ async function backfillSubgraph(def, opts) {
     to: opts.toBlock
   });
   try {
-    const subgraphRow = await db.selectFrom("subgraphs").select(["id", "api_key_id"]).where("name", "=", subgraphName).executeTakeFirst();
+    const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
     const result = await processBlockRange(def, {
       fromBlock: opts.fromBlock,
       toBlock: opts.toBlock,
       status: "active",
       isCatchup: false,
-      apiKeyId: subgraphRow?.api_key_id ?? null,
+      apiKeyId: null,
       subgraphId: subgraphRow?.id,
       signal: opts.signal
     });
@@ -1496,7 +1499,7 @@ async function backfillSubgraph(def, opts) {
       logger4.info("Backfill aborted", { subgraph: subgraphName });
       return { processed: result.blocksProcessed };
     }
-    const resolved = await resolveGaps(db, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
+    const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
     if (resolved > 0) {
       logger4.info("Resolved subgraph gaps via backfill", {
         subgraph: subgraphName,
@@ -1505,7 +1508,7 @@ async function backfillSubgraph(def, opts) {
     }
     const { recordSubgraphProcessed: recordSubgraphProcessed3 } = await import("@secondlayer/shared/db/queries/subgraphs");
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
-      await recordSubgraphProcessed3(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
+      await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
     }
     logger4.info("Backfill complete", {
       subgraph: subgraphName,
@@ -1528,5 +1531,5 @@ export {
   backfillSubgraph
 };
 
-//# debugId=2538A408ADD5D94F64756E2164756E21
+//# debugId=CA98B860698EE36C64756E2164756E21
 //# sourceMappingURL=reindex.js.map