@secondlayer/subgraphs 3.5.0 → 3.6.0

package/dist/src/runtime/catchup.js

@@ -518,21 +518,21 @@ function buildEventPayload(filter, tx, event) {
       return {
         sender: decoded.sender,
         recipient: decoded.recipient,
-        tokenId: decoded.value,
+        tokenId: decoded.raw_value ?? decoded.value,
         assetIdentifier: decoded.asset_identifier,
         tx: txMeta
       };
     case "nft_mint":
       return {
         recipient: decoded.recipient,
-        tokenId: decoded.value,
+        tokenId: decoded.raw_value ?? decoded.value,
         assetIdentifier: decoded.asset_identifier,
         tx: txMeta
       };
     case "nft_burn":
       return {
         sender: decoded.sender,
-        tokenId: decoded.value,
+        tokenId: decoded.raw_value ?? decoded.value,
         assetIdentifier: decoded.asset_identifier,
         tx: txMeta
       };
@@ -564,8 +564,8 @@ function buildEventPayload(filter, tx, event) {
         tx: txMeta
       };
     case "print_event": {
-      const decodedRawValue = decoded.raw_value;
-      const rawValue = decodedRawValue && typeof decodedRawValue === "object" && !Array.isArray(decodedRawValue) ? decodedRawValue : decoded.value;
+      const rawHex = event.data?.raw_value;
+      const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
       const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
       const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
       const { topic: _, ...rest } = clarityObj ?? {};
@@ -949,10 +949,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
 }
 
 // src/runtime/block-processor.ts
-import {
-  getSourceDb,
-  getTargetDb
-} from "@secondlayer/shared/db";
+import { getTargetDb } from "@secondlayer/shared/db";
 import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
 import {
   isByoSubgraph,
@@ -960,15 +957,287 @@ import {
   resolveSubgraphDb,
   updateSubgraphStatus
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger4 } from "@secondlayer/shared/logger";
+import { logger as logger5 } from "@secondlayer/shared/logger";
 import { sql as sql3 } from "kysely";
 
 // src/schema/utils.ts
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 
+// src/runtime/block-source.ts
+import { getSourceDb } from "@secondlayer/shared/db";
+import { IndexHttpClient } from "@secondlayer/shared/index-http";
+import { logger as logger3 } from "@secondlayer/shared/logger";
+
+// src/runtime/batch-loader.ts
+async function loadBlockRange(db, fromHeight, toHeight) {
+  const [blocks, txs, events] = await Promise.all([
+    db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
+    db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
+    db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
+  ]);
+  const txsByHeight = new Map;
+  for (const tx of txs) {
+    const h = Number(tx.block_height);
+    const list = txsByHeight.get(h) ?? [];
+    list.push(tx);
+    txsByHeight.set(h, list);
+  }
+  const eventsByHeight = new Map;
+  for (const evt of events) {
+    const h = Number(evt.block_height);
+    const list = eventsByHeight.get(h) ?? [];
+    list.push(evt);
+    eventsByHeight.set(h, list);
+  }
+  const result = new Map;
+  for (const block of blocks) {
+    const h = Number(block.height);
+    result.set(h, {
+      block,
+      txs: txsByHeight.get(h) ?? [],
+      events: eventsByHeight.get(h) ?? []
+    });
+  }
+  return result;
+}
+function avgEventsPerBlock(batch) {
+  if (batch.size === 0)
+    return 0;
+  let totalEvents = 0;
+  for (const data of batch.values()) {
+    totalEvents += data.events.length;
+  }
+  return totalEvents / batch.size;
+}
+
+// src/runtime/reconstruct.ts
+function isoToUnixSeconds(iso) {
+  if (!iso)
+    return 0;
+  return Math.floor(new Date(iso).getTime() / 1000);
+}
+function reconstructBlock(b) {
+  return {
+    height: b.block_height,
+    hash: b.block_hash,
+    parent_hash: b.parent_hash,
+    burn_block_height: b.burn_block_height,
+    burn_block_hash: b.burn_block_hash,
+    timestamp: isoToUnixSeconds(b.block_time),
+    canonical: true,
+    created_at: new Date(0)
+  };
+}
+function reconstructTransaction(t) {
+  return {
+    tx_id: t.tx_id,
+    block_height: t.block_height,
+    tx_index: t.tx_index,
+    type: t.tx_type,
+    sender: t.sender,
+    status: t.status,
+    contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
+    function_name: t.contract_call?.function_name ?? null,
+    function_args: t.contract_call?.function_args_hex ?? [],
+    raw_result: t.contract_call?.result_hex ?? null,
+    raw_tx: "",
+    created_at: new Date(0)
+  };
+}
+function reconstructEvent(e) {
+  const base = {
+    id: `${e.tx_id}#${e.event_index}`,
+    tx_id: e.tx_id,
+    block_height: e.block_height,
+    event_index: e.event_index,
+    created_at: new Date(0)
+  };
+  switch (e.event_type) {
+    case "ft_transfer":
+    case "ft_mint":
+    case "ft_burn":
+      return {
+        ...base,
+        type: `${e.event_type}_event`,
+        data: {
+          asset_identifier: e.asset_identifier,
+          sender: e.sender,
+          recipient: e.recipient,
+          amount: e.amount
+        }
+      };
+    case "nft_transfer":
+    case "nft_mint":
+    case "nft_burn":
+      return {
+        ...base,
+        type: `${e.event_type}_event`,
+        data: {
+          asset_identifier: e.asset_identifier,
+          sender: e.sender,
+          recipient: e.recipient,
+          raw_value: e.value
+        }
+      };
+    case "stx_transfer":
+    case "stx_mint":
+    case "stx_burn":
+      return {
+        ...base,
+        type: `${e.event_type}_event`,
+        data: {
+          sender: e.sender,
+          recipient: e.recipient,
+          amount: e.amount,
+          ..."memo" in e ? { memo: e.memo ?? undefined } : {}
+        }
+      };
+    case "stx_lock":
+      return {
+        ...base,
+        type: "stx_lock_event",
+        data: {
+          locked_address: e.sender,
+          locked_amount: e.amount,
+          unlock_height: e.payload.unlock_height
+        }
+      };
+    case "print":
+      return {
+        ...base,
+        type: "contract_event",
+        data: {
+          topic: e.payload.topic,
+          contract_id: e.contract_id,
+          contract_identifier: e.contract_id,
+          value: e.payload.value,
+          raw_value: e.payload.raw_value
+        }
+      };
+  }
+}
+
+// src/runtime/block-source.ts
+class PostgresBlockSource {
+  async getTip() {
+    const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    return progress ? Number(progress.highest_seen_block) : 0;
+  }
+  loadBlockRange(fromHeight, toHeight) {
+    return loadBlockRange(getSourceDb(), fromHeight, toHeight);
+  }
+}
+var EVENT_FILTER_TO_INDEX_TYPE = {
+  stx_transfer: "stx_transfer",
+  stx_mint: "stx_mint",
+  stx_burn: "stx_burn",
+  stx_lock: "stx_lock",
+  ft_transfer: "ft_transfer",
+  ft_mint: "ft_mint",
+  ft_burn: "ft_burn",
+  nft_transfer: "nft_transfer",
+  nft_mint: "nft_mint",
+  nft_burn: "nft_burn",
+  print_event: "print"
+};
+var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
+var ALL_INDEX_EVENT_TYPES = [
+  ...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
+];
+function sourceFilters(subgraph) {
+  const sources = subgraph.sources;
+  return Array.isArray(sources) ? sources : Object.values(sources);
+}
+function referencedIndexEventTypes(subgraph) {
+  const filters = sourceFilters(subgraph);
+  if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
+    return ALL_INDEX_EVENT_TYPES;
+  }
+  const types = new Set;
+  for (const f of filters) {
+    const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
+    if (t)
+      types.add(t);
+  }
+  return [...types];
+}
+function isStreamsIndexEligible(subgraph) {
+  if (Array.isArray(subgraph.sources))
+    return false;
+  const filters = sourceFilters(subgraph);
+  if (filters.length === 0)
+    return false;
+  for (const f of filters) {
+    const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
+    if (!known)
+      return false;
+  }
+  return true;
+}
+
+class PublicApiBlockSource {
+  http;
+  eventTypes;
+  constructor(http, eventTypes) {
+    this.http = http;
+    this.eventTypes = eventTypes;
+  }
+  getTip() {
+    return this.http.getIndexTip();
+  }
+  async loadBlockRange(fromHeight, toHeight) {
+    const [blocks, txs, eventLists] = await Promise.all([
+      this.http.walkBlocks(fromHeight, toHeight),
+      this.http.walkTransactions(fromHeight, toHeight),
+      Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
+    ]);
+    const map = new Map;
+    for (const b of blocks) {
+      map.set(b.block_height, {
+        block: reconstructBlock(b),
+        txs: [],
+        events: []
+      });
+    }
+    for (const t of txs) {
+      map.get(t.block_height)?.txs.push(reconstructTransaction(t));
+    }
+    for (const list of eventLists) {
+      for (const e of list) {
+        map.get(e.block_height)?.events.push(reconstructEvent(e));
+      }
+    }
+    for (const bd of map.values()) {
+      bd.txs.sort((a, b) => a.tx_index - b.tx_index);
+      bd.events.sort((a, b) => a.event_index - b.event_index);
+    }
+    return map;
+  }
+}
+var postgresBlockSource = new PostgresBlockSource;
+function buildHttpClient() {
+  const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
+  return new IndexHttpClient({
+    indexBaseUrl: baseUrl,
+    streamsBaseUrl: baseUrl,
+    streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
+  });
+}
+function resolveBlockSource(subgraph) {
+  if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
+    return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
+  }
+  if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
+    logger3.debug("Subgraph not streams-index eligible, using DB tap", {
+      subgraph: subgraph.name
+    });
+  }
+  return postgresBlockSource;
+}
+
 // src/runtime/outbox-emit.ts
 import { createHash } from "node:crypto";
-import { logger as logger3 } from "@secondlayer/shared/logger";
+import { logger as logger4 } from "@secondlayer/shared/logger";
 var loggedKillSwitch = false;
 function isEmitOutboxEnabled() {
   return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
@@ -987,7 +1256,7 @@ function stableStringify(obj) {
 async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
   if (!isEmitOutboxEnabled()) {
     if (!loggedKillSwitch) {
-      logger3.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
+      logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
       loggedKillSwitch = true;
     }
     return 0;
@@ -1228,7 +1497,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
   return resolved;
 }
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
-  const sourceDb = getSourceDb();
   const targetDb = getTargetDb();
   const blockStart = performance.now();
   const result = {
@@ -1246,25 +1514,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     txs = opts.preloaded.txs;
     evts = opts.preloaded.events;
   } else {
-    block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
-    if (!block) {
-      logger4.warn("Block not found for subgraph processing", {
+    const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
+    if (!data) {
+      logger5.debug("Block not found or non-canonical for subgraph processing", {
         subgraph: subgraphName,
         blockHeight
       });
       result.skipped = true;
       return result;
     }
-    if (!block.canonical) {
-      logger4.debug("Skipping non-canonical block", {
-        subgraph: subgraphName,
-        blockHeight
-      });
-      result.skipped = true;
-      return result;
-    }
-    txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
-    evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
+    block = data.block;
+    txs = data.txs;
+    evts = data.events;
   }
   const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
   const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
@@ -1355,7 +1616,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
-          logger4.warn("Subgraph table exceeds 10M rows (estimate)", {
+          logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
             subgraph: subgraphName,
             table,
             count
@@ -1363,7 +1624,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         }
       }
     } catch (err) {
-      logger4.debug("Row count sample failed", {
+      logger5.debug("Row count sample failed", {
         subgraph: subgraphName,
         error: err instanceof Error ? err.message : String(err)
       });
@@ -1436,56 +1697,12 @@ class StatsAccumulator {
 }
 
 // src/runtime/catchup.ts
-import { getSourceDb as getSourceDb2, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
+import { getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
 import {
   recordGapBatch
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
 import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger5 } from "@secondlayer/shared/logger";
-
-// src/runtime/batch-loader.ts
-async function loadBlockRange(db, fromHeight, toHeight) {
-  const [blocks, txs, events] = await Promise.all([
-    db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
-    db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
-    db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
-  ]);
-  const txsByHeight = new Map;
-  for (const tx of txs) {
-    const h = Number(tx.block_height);
-    const list = txsByHeight.get(h) ?? [];
-    list.push(tx);
-    txsByHeight.set(h, list);
-  }
-  const eventsByHeight = new Map;
-  for (const evt of events) {
-    const h = Number(evt.block_height);
-    const list = eventsByHeight.get(h) ?? [];
-    list.push(evt);
-    eventsByHeight.set(h, list);
-  }
-  const result = new Map;
-  for (const block of blocks) {
-    const h = Number(block.height);
-    result.set(h, {
-      block,
-      txs: txsByHeight.get(h) ?? [],
-      events: eventsByHeight.get(h) ?? []
-    });
-  }
-  return result;
-}
-function avgEventsPerBlock(batch) {
-  if (batch.size === 0)
-    return 0;
-  let totalEvents = 0;
-  for (const data of batch.values()) {
-    totalEvents += data.events.length;
-  }
-  return totalEvents / batch.size;
-}
-
-// src/runtime/catchup.ts
+import { logger as logger6 } from "@secondlayer/shared/logger";
 var LOG_INTERVAL = 1000;
 var STANDARD_CATCHUP_BATCH_CONFIG = {
   defaultBatchSize: 500,
@@ -1556,22 +1773,19 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     return 0;
   catchingUp.add(subgraphName);
   try {
-    const sourceDb = getSourceDb2();
+    const source = resolveBlockSource(subgraph);
     const targetDb = getTargetDb2();
     const subgraphRow = await getSubgraph(targetDb, subgraphName);
     if (!subgraphRow)
       return 0;
     const lastProcessedBlock = Number(subgraphRow.last_processed_block);
-    const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
-    if (!progress)
-      return 0;
-    const chainTip = Number(progress.highest_seen_block);
-    if (lastProcessedBlock >= chainTip)
+    const chainTip = await source.getTip();
+    if (chainTip <= 0 || lastProcessedBlock >= chainTip)
       return 0;
     const subgraphStart = Number(subgraphRow.start_block) || 1;
     const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
     const totalBlocks = chainTip - lastProcessedBlock;
-    logger5.info("Subgraph catch-up starting", {
+    logger6.info("Subgraph catch-up starting", {
       subgraph: subgraphName,
       from: startBlock,
       to: chainTip,
@@ -1583,11 +1797,11 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     let batchSize = batchConfig.defaultBatchSize;
     let currentHeight = startBlock;
     let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
-    let nextBatchPromise = batchConfig.prefetch ? loadBlockRange(sourceDb, currentHeight, prefetchedBatchEnd) : undefined;
+    let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
     while (currentHeight <= chainTip) {
       const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
-        logger5.info("Subgraph status changed, stopping catch-up", {
+        logger6.info("Subgraph status changed, stopping catch-up", {
           subgraph: subgraphName,
           status: currentRow?.status ?? "deleted"
         });
@@ -1601,13 +1815,13 @@ async function catchUpSubgraph(subgraph, subgraphName) {
         const nextStart = batchEnd + 1;
         if (nextStart <= chainTip) {
           prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
-          nextBatchPromise = loadBlockRange(sourceDb, nextStart, prefetchedBatchEnd);
+          nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
         } else {
           nextBatchPromise = undefined;
         }
       } else {
         batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
-        batch = await loadBlockRange(sourceDb, currentHeight, batchEnd);
+        batch = await source.loadBlockRange(currentHeight, batchEnd);
       }
       const batchFailedBlocks = [];
       for (let height = currentHeight;height <= batchEnd; height++) {
@@ -1623,7 +1837,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
             preloaded: blockData
           });
         } catch (err) {
-          logger5.error("Block processing error during catch-up", {
+          logger6.error("Block processing error during catch-up", {
             subgraph: subgraphName,
             blockHeight: height,
             error: err instanceof Error ? err.message : String(err)
@@ -1642,7 +1856,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
           }
         }
         if (processed % LOG_INTERVAL === 0) {
-          logger5.info("Subgraph catch-up progress", {
+          logger6.info("Subgraph catch-up progress", {
             subgraph: subgraphName,
             processed,
             total: totalBlocks,
@@ -1654,7 +1868,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       if (batchFailedBlocks.length > 0) {
         const gaps = coalesceGaps(batchFailedBlocks);
         await recordGapBatch(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
-          logger5.warn("Failed to record subgraph gaps", {
+          logger6.warn("Failed to record subgraph gaps", {
             subgraph: subgraphName,
             error: err instanceof Error ? err.message : String(err)
           });
@@ -1665,7 +1879,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       currentHeight = batchEnd + 1;
     }
     await stats.flush(targetDb);
-    logger5.info("Subgraph catch-up complete", {
+    logger6.info("Subgraph catch-up complete", {
       subgraph: subgraphName,
       processed
     });
@@ -1679,5 +1893,5 @@ export {
   catchUpSubgraph
 };
 
-//# debugId=3947491E2917E90B64756E2164756E21
+//# debugId=9D92106B9D08283764756E2164756E21
 //# sourceMappingURL=catchup.js.map