@secondlayer/subgraphs 0.11.7 → 1.0.0-alpha.0

package/dist/src/runtime/processor.js

@@ -849,7 +849,10 @@ function matchSources(sources, transactions, events) {
 }
 
 // src/runtime/block-processor.ts
-import { getDb } from "@secondlayer/shared/db";
+import {
+  getSourceDb,
+  getTargetDb
+} from "@secondlayer/shared/db";
 import {
   recordSubgraphProcessed,
   updateSubgraphStatus
@@ -863,7 +866,8 @@ import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 // src/runtime/block-processor.ts
 var schemaNameCache = new Map;
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
-  const db = getDb();
+  const sourceDb = getSourceDb();
+  const targetDb = getTargetDb();
   const blockStart = performance.now();
   const result = {
     blockHeight,
@@ -872,13 +876,15 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     errors: 0,
     skipped: false
   };
-  let block, txs, evts;
+  let block;
+  let txs;
+  let evts;
   if (opts?.preloaded) {
     block = opts.preloaded.block;
     txs = opts.preloaded.txs;
     evts = opts.preloaded.events;
   } else {
-    block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
+    block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
     if (!block) {
       logger3.warn("Block not found for subgraph processing", {
         subgraph: subgraphName,
@@ -895,20 +901,20 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
       result.skipped = true;
       return result;
     }
-    txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
-    evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
+    txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
+    evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   }
   const matched = matchSources(subgraph.sources, txs, evts);
   result.matched = matched.length;
   if (matched.length === 0) {
     if (!opts?.skipProgressUpdate) {
-      await updateSubgraphStatus(db, subgraphName, "active", blockHeight);
+      await updateSubgraphStatus(targetDb, subgraphName, "active", blockHeight);
     }
     return result;
   }
   let schemaName = schemaNameCache.get(subgraphName);
   if (!schemaName) {
-    const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
+    const subgraphRecord = await targetDb.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
     schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
     schemaNameCache.set(subgraphName, schemaName);
   }
@@ -926,7 +932,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   };
   let handlerMs = 0;
   let flushMs = 0;
-  await db.transaction().execute(async (tx) => {
+  await targetDb.transaction().execute(async (tx) => {
     const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx);
     const handlerStart = performance.now();
     const runResult = await runHandlers(subgraph, matched, ctx);
@@ -957,7 +963,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     try {
       const tables = Object.keys(subgraph.schema);
       for (const table of tables) {
-        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(db);
+        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
           logger3.warn("Subgraph table exceeds 10M rows (estimate)", {
@@ -978,13 +984,11 @@ var FLUSH_INTERVAL_MS = 60000;
 
 class StatsAccumulator {
   subgraphName;
-  apiKeyId;
   isCatchup;
   current;
   lastFlush = Date.now();
-  constructor(subgraphName, apiKeyId, isCatchup) {
+  constructor(subgraphName, isCatchup) {
     this.subgraphName = subgraphName;
-    this.apiKeyId = apiKeyId;
     this.isCatchup = isCatchup;
     this.current = this.newEntry();
   }
@@ -1009,7 +1013,6 @@ class StatsAccumulator {
     const avgOpsPerBlock = entry.blocksProcessed > 0 ? entry.totalOps / entry.blocksProcessed : 0;
     await db.insertInto("subgraph_processing_stats").values({
       subgraph_name: entry.subgraphName,
-      api_key_id: entry.apiKeyId,
       bucket_start: entry.bucketStart,
       bucket_end: new Date,
       blocks_processed: entry.blocksProcessed,
@@ -1025,7 +1028,6 @@ class StatsAccumulator {
   newEntry() {
     return {
       subgraphName: this.subgraphName,
-      apiKeyId: this.apiKeyId,
       bucketStart: new Date,
       blocksProcessed: 0,
       totalTimeMs: 0,
@@ -1040,7 +1042,7 @@ class StatsAccumulator {
 }
 
 // src/runtime/catchup.ts
-import { getDb as getDb2 } from "@secondlayer/shared/db";
+import { getSourceDb as getSourceDb2, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
 import {
   recordGapBatch
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
@@ -1129,12 +1131,13 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     return 0;
   catchingUp.add(subgraphName);
   try {
-    const db = getDb2();
-    const subgraphRow = await getSubgraph(db, subgraphName);
+    const sourceDb = getSourceDb2();
+    const targetDb = getTargetDb2();
+    const subgraphRow = await getSubgraph(targetDb, subgraphName);
     if (!subgraphRow)
       return 0;
     const lastProcessedBlock = Number(subgraphRow.last_processed_block);
-    const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
     if (!progress)
       return 0;
     const chainTip = Number(progress.highest_seen_block);
@@ -1149,14 +1152,14 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       to: chainTip,
       blocks: totalBlocks
     });
-    const stats = new StatsAccumulator(subgraphName, subgraphRow.api_key_id, true);
+    const stats = new StatsAccumulator(subgraphName, true);
     let processed = 0;
     let batchSize = DEFAULT_BATCH_SIZE;
     let currentHeight = startBlock;
     let nextBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
-    let nextBatchPromise = loadBlockRange(db, currentHeight, nextBatchEnd);
+    let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
     while (currentHeight <= chainTip) {
-      const currentRow = await getSubgraph(db, subgraphName);
+      const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
         logger4.info("Subgraph status changed, stopping catch-up", {
           subgraph: subgraphName,
@@ -1169,7 +1172,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       const nextStart = batchEnd + 1;
       if (nextStart <= chainTip) {
         nextBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
-        nextBatchPromise = loadBlockRange(db, nextStart, nextBatchEnd);
+        nextBatchPromise = loadBlockRange(sourceDb, nextStart, nextBatchEnd);
       }
       const batchFailedBlocks = [];
       for (let height = currentHeight;height <= batchEnd; height++) {
@@ -1192,7 +1195,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
           });
           batchFailedBlocks.push({ height, reason: "processing_error" });
           const { updateSubgraphStatus: updateSubgraphStatus2 } = await import("@secondlayer/shared/db/queries/subgraphs");
-          await updateSubgraphStatus2(db, subgraphName, "active", height).catch(() => {});
+          await updateSubgraphStatus2(targetDb, subgraphName, "active", height).catch(() => {});
           processed++;
           continue;
         }
@@ -1200,7 +1203,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
         if (result.timing) {
           stats.record(result.timing, result.processed);
           if (stats.shouldFlush()) {
-            await stats.flush(db);
+            await stats.flush(targetDb);
           }
         }
         if (processed % LOG_INTERVAL === 0) {
@@ -1215,7 +1218,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       }
       if (batchFailedBlocks.length > 0) {
         const gaps = coalesceGaps(batchFailedBlocks);
-        await recordGapBatch(db, subgraphRow.id, subgraphName, gaps).catch((err) => {
+        await recordGapBatch(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
           logger4.warn("Failed to record subgraph gaps", {
             subgraph: subgraphName,
             error: err instanceof Error ? err.message : String(err)
@@ -1226,7 +1229,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       batchSize = adjustBatchSize(batchSize, avg);
       currentHeight = batchEnd + 1;
     }
-    await stats.flush(db);
+    await stats.flush(targetDb);
     logger4.info("Subgraph catch-up complete", {
       subgraph: subgraphName,
       processed
@@ -1239,13 +1242,13 @@ async function catchUpSubgraph(subgraph, subgraphName) {
 
 // src/runtime/reorg.ts
 import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
-import { getDb as getDb3, getRawClient } from "@secondlayer/shared/db";
+import { getRawClient, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
 import { listSubgraphs } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger5 } from "@secondlayer/shared/logger";
 async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
-  const db = getDb3();
-  const client = getRawClient();
-  const activeSubgraphs = (await listSubgraphs(db)).filter((v) => v.status === "active");
+  const targetDb = getTargetDb3();
+  const client = getRawClient("target");
+  const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
   if (activeSubgraphs.length === 0)
     return;
   logger5.info("Propagating reorg to subgraphs", {
@@ -1283,7 +1286,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
 
 // src/runtime/processor.ts
 import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
-import { getDb as getDb4 } from "@secondlayer/shared/db";
+import { getTargetDb as getTargetDb4 } from "@secondlayer/shared/db";
 import {
   listSubgraphs as listSubgraphs2,
   updateSubgraphStatus as updateSubgraphStatus2
@@ -1293,6 +1296,9 @@ import { listen } from "@secondlayer/shared/queue/listener";
 var CHANNEL_NEW_BLOCK = "indexer:new_block";
 var DEFAULT_CONCURRENCY = 5;
 var POLL_INTERVAL_MS = 5000;
+function sourceListenerUrl() {
+  return process.env.SOURCE_DATABASE_URL ?? process.env.DATABASE_URL;
+}
 function isHandlerNotFoundError(err) {
   if (!(err instanceof Error))
     return false;
@@ -1341,8 +1347,8 @@ async function startSubgraphProcessor(opts) {
   const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
   let running = true;
   logger6.info("Starting subgraph processor", { concurrency });
-  const db = getDb4();
-  const activeSubgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
+  const targetDb = getTargetDb4();
+  const activeSubgraphs = (await listSubgraphs2(targetDb)).filter((v) => v.status === "active");
   for (const sg of activeSubgraphs) {
     try {
       const def = await loadSubgraphDefinition(sg);
@@ -1350,7 +1356,7 @@ async function startSubgraphProcessor(opts) {
     } catch (err) {
       const msg = getErrorMessage3(err);
       if (isHandlerNotFoundError(err)) {
-        await updateSubgraphStatus2(db, sg.name, "error");
+        await updateSubgraphStatus2(targetDb, sg.name, "error");
       }
       logger6.error("Subgraph catch-up failed on startup", {
         subgraph: sg.name,
@@ -1361,8 +1367,8 @@ async function startSubgraphProcessor(opts) {
   const stopListening = await listen(CHANNEL_NEW_BLOCK, async () => {
     if (!running)
       return;
-    const db2 = getDb4();
-    const subgraphs = (await listSubgraphs2(db2)).filter((v) => v.status === "active");
+    const db = getTargetDb4();
+    const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
     cleanupCaches(subgraphs);
     for (const sg of subgraphs) {
       try {
@@ -1371,7 +1377,7 @@ async function startSubgraphProcessor(opts) {
       } catch (err) {
         const msg = getErrorMessage3(err);
         if (isHandlerNotFoundError(err)) {
-          await updateSubgraphStatus2(db2, sg.name, "error");
+          await updateSubgraphStatus2(db, sg.name, "error");
         }
         logger6.error("Subgraph processing failed", {
           subgraph: sg.name,
@@ -1379,7 +1385,7 @@ async function startSubgraphProcessor(opts) {
         });
       }
     }
-  });
+  }, { connectionString: sourceListenerUrl() });
   const stopReorgListening = await listen("subgraph_reorg", async (payload) => {
     if (!running)
       return;
@@ -1394,12 +1400,12 @@ async function startSubgraphProcessor(opts) {
         error: getErrorMessage3(err)
       });
     }
-  });
+  }, { connectionString: sourceListenerUrl() });
   const pollInterval = setInterval(async () => {
     if (!running)
       return;
-    const db2 = getDb4();
-    const subgraphs = (await listSubgraphs2(db2)).filter((v) => v.status === "active");
+    const db = getTargetDb4();
+    const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
     cleanupCaches(subgraphs);
     for (const sg of subgraphs) {
       try {
@@ -1426,5 +1432,5 @@ export {
   startSubgraphProcessor
 };
 
-//# debugId=F4C1ACE0964756F564756E2164756E21
+//# debugId=F8E2C460E99ED57564756E2164756E21
 //# sourceMappingURL=processor.js.map