@secondlayer/subgraphs 3.4.0 → 3.6.0

package/dist/src/runtime/reorg.js

@@ -518,21 +518,21 @@ function buildEventPayload(filter, tx, event) {
       return {
         sender: decoded.sender,
         recipient: decoded.recipient,
-        tokenId: decoded.value,
+        tokenId: decoded.raw_value ?? decoded.value,
         assetIdentifier: decoded.asset_identifier,
         tx: txMeta
       };
     case "nft_mint":
       return {
         recipient: decoded.recipient,
-        tokenId: decoded.value,
+        tokenId: decoded.raw_value ?? decoded.value,
         assetIdentifier: decoded.asset_identifier,
         tx: txMeta
       };
     case "nft_burn":
       return {
         sender: decoded.sender,
-        tokenId: decoded.value,
+        tokenId: decoded.raw_value ?? decoded.value,
         assetIdentifier: decoded.asset_identifier,
         tx: txMeta
       };
@@ -564,8 +564,8 @@ function buildEventPayload(filter, tx, event) {
         tx: txMeta
       };
     case "print_event": {
-      const decodedRawValue = decoded.raw_value;
-      const rawValue = decodedRawValue && typeof decodedRawValue === "object" && !Array.isArray(decodedRawValue) ? decodedRawValue : decoded.value;
+      const rawHex = event.data?.raw_value;
+      const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
       const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
       const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
       const { topic: _, ...rest } = clarityObj ?? {};
@@ -949,10 +949,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
 }
 
 // src/runtime/block-processor.ts
-import {
-  getSourceDb,
-  getTargetDb
-} from "@secondlayer/shared/db";
+import { getTargetDb } from "@secondlayer/shared/db";
 import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
 import {
   isByoSubgraph,
@@ -960,15 +957,287 @@ import {
   resolveSubgraphDb,
   updateSubgraphStatus
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger4 } from "@secondlayer/shared/logger";
+import { logger as logger5 } from "@secondlayer/shared/logger";
 import { sql as sql3 } from "kysely";
 
 // src/schema/utils.ts
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 
+// src/runtime/block-source.ts
+import { getSourceDb } from "@secondlayer/shared/db";
+import { IndexHttpClient } from "@secondlayer/shared/index-http";
+import { logger as logger3 } from "@secondlayer/shared/logger";
+
+// src/runtime/batch-loader.ts
+async function loadBlockRange(db, fromHeight, toHeight) {
+  const [blocks, txs, events] = await Promise.all([
+    db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
+    db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
+    db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
+  ]);
+  const txsByHeight = new Map;
+  for (const tx of txs) {
+    const h = Number(tx.block_height);
+    const list = txsByHeight.get(h) ?? [];
+    list.push(tx);
+    txsByHeight.set(h, list);
+  }
+  const eventsByHeight = new Map;
+  for (const evt of events) {
+    const h = Number(evt.block_height);
+    const list = eventsByHeight.get(h) ?? [];
+    list.push(evt);
+    eventsByHeight.set(h, list);
+  }
+  const result = new Map;
+  for (const block of blocks) {
+    const h = Number(block.height);
+    result.set(h, {
+      block,
+      txs: txsByHeight.get(h) ?? [],
+      events: eventsByHeight.get(h) ?? []
+    });
+  }
+  return result;
+}
+function avgEventsPerBlock(batch) {
+  if (batch.size === 0)
+    return 0;
+  let totalEvents = 0;
+  for (const data of batch.values()) {
+    totalEvents += data.events.length;
+  }
+  return totalEvents / batch.size;
+}
+
+// src/runtime/reconstruct.ts
+function isoToUnixSeconds(iso) {
+  if (!iso)
+    return 0;
+  return Math.floor(new Date(iso).getTime() / 1000);
+}
+function reconstructBlock(b) {
+  return {
+    height: b.block_height,
+    hash: b.block_hash,
+    parent_hash: b.parent_hash,
+    burn_block_height: b.burn_block_height,
+    burn_block_hash: b.burn_block_hash,
+    timestamp: isoToUnixSeconds(b.block_time),
+    canonical: true,
+    created_at: new Date(0)
+  };
+}
+function reconstructTransaction(t) {
+  return {
+    tx_id: t.tx_id,
+    block_height: t.block_height,
+    tx_index: t.tx_index,
+    type: t.tx_type,
+    sender: t.sender,
+    status: t.status,
+    contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
+    function_name: t.contract_call?.function_name ?? null,
+    function_args: t.contract_call?.function_args_hex ?? [],
+    raw_result: t.contract_call?.result_hex ?? null,
+    raw_tx: "",
+    created_at: new Date(0)
+  };
+}
+function reconstructEvent(e) {
+  const base = {
+    id: `${e.tx_id}#${e.event_index}`,
+    tx_id: e.tx_id,
+    block_height: e.block_height,
+    event_index: e.event_index,
+    created_at: new Date(0)
+  };
+  switch (e.event_type) {
+    case "ft_transfer":
+    case "ft_mint":
+    case "ft_burn":
+      return {
+        ...base,
+        type: `${e.event_type}_event`,
+        data: {
+          asset_identifier: e.asset_identifier,
+          sender: e.sender,
+          recipient: e.recipient,
+          amount: e.amount
+        }
+      };
+    case "nft_transfer":
+    case "nft_mint":
+    case "nft_burn":
+      return {
+        ...base,
+        type: `${e.event_type}_event`,
+        data: {
+          asset_identifier: e.asset_identifier,
+          sender: e.sender,
+          recipient: e.recipient,
+          raw_value: e.value
+        }
+      };
+    case "stx_transfer":
+    case "stx_mint":
+    case "stx_burn":
+      return {
+        ...base,
+        type: `${e.event_type}_event`,
+        data: {
+          sender: e.sender,
+          recipient: e.recipient,
+          amount: e.amount,
+          ..."memo" in e ? { memo: e.memo ?? undefined } : {}
+        }
+      };
+    case "stx_lock":
+      return {
+        ...base,
+        type: "stx_lock_event",
+        data: {
+          locked_address: e.sender,
+          locked_amount: e.amount,
+          unlock_height: e.payload.unlock_height
+        }
+      };
+    case "print":
+      return {
+        ...base,
+        type: "contract_event",
+        data: {
+          topic: e.payload.topic,
+          contract_id: e.contract_id,
+          contract_identifier: e.contract_id,
+          value: e.payload.value,
+          raw_value: e.payload.raw_value
+        }
+      };
+  }
+}
+
+// src/runtime/block-source.ts
+class PostgresBlockSource {
+  async getTip() {
+    const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    return progress ? Number(progress.highest_seen_block) : 0;
+  }
+  loadBlockRange(fromHeight, toHeight) {
+    return loadBlockRange(getSourceDb(), fromHeight, toHeight);
+  }
+}
+var EVENT_FILTER_TO_INDEX_TYPE = {
+  stx_transfer: "stx_transfer",
+  stx_mint: "stx_mint",
+  stx_burn: "stx_burn",
+  stx_lock: "stx_lock",
+  ft_transfer: "ft_transfer",
+  ft_mint: "ft_mint",
+  ft_burn: "ft_burn",
+  nft_transfer: "nft_transfer",
+  nft_mint: "nft_mint",
+  nft_burn: "nft_burn",
+  print_event: "print"
+};
+var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
+var ALL_INDEX_EVENT_TYPES = [
+  ...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
+];
+function sourceFilters(subgraph) {
+  const sources = subgraph.sources;
+  return Array.isArray(sources) ? sources : Object.values(sources);
+}
+function referencedIndexEventTypes(subgraph) {
+  const filters = sourceFilters(subgraph);
+  if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
+    return ALL_INDEX_EVENT_TYPES;
+  }
+  const types = new Set;
+  for (const f of filters) {
+    const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
+    if (t)
+      types.add(t);
+  }
+  return [...types];
+}
+function isStreamsIndexEligible(subgraph) {
+  if (Array.isArray(subgraph.sources))
+    return false;
+  const filters = sourceFilters(subgraph);
+  if (filters.length === 0)
+    return false;
+  for (const f of filters) {
+    const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
+    if (!known)
+      return false;
+  }
+  return true;
+}
+
+class PublicApiBlockSource {
+  http;
+  eventTypes;
+  constructor(http, eventTypes) {
+    this.http = http;
+    this.eventTypes = eventTypes;
+  }
+  getTip() {
+    return this.http.getIndexTip();
+  }
+  async loadBlockRange(fromHeight, toHeight) {
+    const [blocks, txs, eventLists] = await Promise.all([
+      this.http.walkBlocks(fromHeight, toHeight),
+      this.http.walkTransactions(fromHeight, toHeight),
+      Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
+    ]);
+    const map = new Map;
+    for (const b of blocks) {
+      map.set(b.block_height, {
+        block: reconstructBlock(b),
+        txs: [],
+        events: []
+      });
+    }
+    for (const t of txs) {
+      map.get(t.block_height)?.txs.push(reconstructTransaction(t));
+    }
+    for (const list of eventLists) {
+      for (const e of list) {
+        map.get(e.block_height)?.events.push(reconstructEvent(e));
+      }
+    }
+    for (const bd of map.values()) {
+      bd.txs.sort((a, b) => a.tx_index - b.tx_index);
+      bd.events.sort((a, b) => a.event_index - b.event_index);
+    }
+    return map;
+  }
+}
+var postgresBlockSource = new PostgresBlockSource;
+function buildHttpClient() {
+  const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
+  return new IndexHttpClient({
+    indexBaseUrl: baseUrl,
+    streamsBaseUrl: baseUrl,
+    streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
+  });
+}
+function resolveBlockSource(subgraph) {
+  if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
+    return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
+  }
+  if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
+    logger3.debug("Subgraph not streams-index eligible, using DB tap", {
+      subgraph: subgraph.name
+    });
+  }
+  return postgresBlockSource;
+}
+
 // src/runtime/outbox-emit.ts
 import { createHash } from "node:crypto";
-import { logger as logger3 } from "@secondlayer/shared/logger";
+import { logger as logger4 } from "@secondlayer/shared/logger";
 var loggedKillSwitch = false;
 function isEmitOutboxEnabled() {
   return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
@@ -987,7 +1256,7 @@ function stableStringify(obj) {
 async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
   if (!isEmitOutboxEnabled()) {
     if (!loggedKillSwitch) {
-      logger3.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
+      logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
       loggedKillSwitch = true;
     }
     return 0;
@@ -1228,7 +1497,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
   return resolved;
 }
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
-  const sourceDb = getSourceDb();
   const targetDb = getTargetDb();
   const blockStart = performance.now();
   const result = {
@@ -1246,25 +1514,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     txs = opts.preloaded.txs;
     evts = opts.preloaded.events;
   } else {
-    block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
-    if (!block) {
-      logger4.warn("Block not found for subgraph processing", {
+    const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
+    if (!data) {
+      logger5.debug("Block not found or non-canonical for subgraph processing", {
         subgraph: subgraphName,
         blockHeight
       });
       result.skipped = true;
       return result;
     }
-    if (!block.canonical) {
-      logger4.debug("Skipping non-canonical block", {
-        subgraph: subgraphName,
-        blockHeight
-      });
-      result.skipped = true;
-      return result;
-    }
-    txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
-    evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
+    block = data.block;
+    txs = data.txs;
+    evts = data.events;
   }
   const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
   const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
@@ -1355,7 +1616,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
-          logger4.warn("Subgraph table exceeds 10M rows (estimate)", {
+          logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
             subgraph: subgraphName,
             table,
             count
@@ -1363,7 +1624,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         }
       }
     } catch (err) {
-      logger4.debug("Row count sample failed", {
+      logger5.debug("Row count sample failed", {
         subgraph: subgraphName,
         error: err instanceof Error ? err.message : String(err)
       });
@@ -1379,13 +1640,13 @@ import {
   listSubgraphs,
   resolveSubgraphRawClient
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger5 } from "@secondlayer/shared/logger";
+import { logger as logger6 } from "@secondlayer/shared/logger";
 async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
   const targetDb = getTargetDb2();
   const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
   if (activeSubgraphs.length === 0)
     return;
-  logger5.info("Propagating reorg to subgraphs", {
+  logger6.info("Propagating reorg to subgraphs", {
     blockHeight,
     subgraphCount: activeSubgraphs.length
   });
@@ -1433,7 +1694,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
           }).as("row_pk"),
           eb2.val("pending").as("status")
         ]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.column("dedup_key").doNothing()).execute().catch((err) => {
-          logger5.warn("Failed to emit revert event for subscriptions", {
+          logger6.warn("Failed to emit revert event for subscriptions", {
             subgraph: sg.name,
             table: tableName,
             blockHeight,
@@ -1441,19 +1702,19 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
           });
         });
       }
-      logger5.info("Subgraph reorg cleanup done", {
+      logger6.info("Subgraph reorg cleanup done", {
         subgraph: sg.name,
         blockHeight,
         tablesAffected: Object.keys(revertedByTable).length
       });
       const def = await loadSubgraphDef(sg);
       await processBlock(def, sg.name, blockHeight);
-      logger5.info("Subgraph reorg reprocessed", {
+      logger6.info("Subgraph reorg reprocessed", {
         subgraph: sg.name,
         blockHeight
       });
     } catch (err) {
-      logger5.error("Subgraph reorg handling failed", {
+      logger6.error("Subgraph reorg handling failed", {
         subgraph: sg.name,
         blockHeight,
         error: getErrorMessage2(err)
@@ -1465,5 +1726,5 @@ export {
   handleSubgraphReorg
 };
 
-//# debugId=39C8DEB104D91A4564756E2164756E21
+//# debugId=BCA72BAE451BCFF564756E2164756E21
 //# sourceMappingURL=reorg.js.map