@secondlayer/subgraphs 3.21.0 → 3.23.0

package/dist/src/runtime/processor.js

@@ -100,6 +100,18 @@ function emitJournalDDL(schemaName) {
     `CREATE INDEX IF NOT EXISTS idx_${schemaName}_journal_height ON ${schemaName}._journal (block_height)`
   ];
 }
+function emitFactoryDDL(schemaName) {
+  return [
+    `CREATE TABLE IF NOT EXISTS ${schemaName}._factory_addresses (
+  source_name TEXT NOT NULL,
+  address TEXT NOT NULL,
+  block_height BIGINT NOT NULL,
+  _created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  PRIMARY KEY (source_name, address)
+)`,
+    `CREATE INDEX IF NOT EXISTS idx_${schemaName}_factory_height ON ${schemaName}._factory_addresses (block_height)`
+  ];
+}
 function emitForeignKeyDDL(schemaName, tableName, tableDef) {
   return (tableDef.relations ?? []).map((rel) => {
     const constraintName = `fk_${schemaName}_${tableName}_${rel.name}`;
@@ -118,6 +130,9 @@ function generateSubgraphSQL(def, schemaNameOverride) {
     statements.push(...emitTableDDL(schemaName, tableName, tableDef));
   }
   statements.push(...emitJournalDDL(schemaName));
+  if (Object.values(def.sources ?? {}).some((src) => src.factory !== undefined)) {
+    statements.push(...emitFactoryDDL(schemaName));
+  }
   for (const [tableName, tableDef] of Object.entries(def.schema)) {
     statements.push(...emitForeignKeyDDL(schemaName, tableName, tableDef));
   }
@@ -130,6 +145,139 @@ function generateSubgraphSQL(def, schemaNameOverride) {
   return { statements, hash };
 }
 
+// src/runtime/chain-read.ts
+import { getSourceDb } from "@secondlayer/shared/db";
+import { http, createPublicClient } from "@secondlayer/stacks";
+import {
+  ContractResponseError,
+  buildFunctionArgs,
+  isResponseOutput,
+  readContract
+} from "@secondlayer/stacks/actions";
+import {
+  clarityValueToJSUntyped,
+  deserializeCV,
+  normalizeAbi,
+  serializeCV,
+  toCamelCase
+} from "@secondlayer/stacks/clarity";
+function readContractAt(ctx, contractId, abi, options) {
+  return ctx.client.contract(contractId, abi, options);
+}
+
+class ChainReadError extends Error {
+  name = "ChainReadError";
+}
+function nodeClient(rpcUrl) {
+  return createPublicClient({
+    transport: http(rpcUrl, { retryCount: 0, timeout: 1e4 })
+  });
+}
+
+class ConcurrencyGate {
+  limit;
+  active = 0;
+  queue = [];
+  constructor(limit) {
+    this.limit = limit;
+  }
+  async run(fn) {
+    if (this.active >= this.limit) {
+      await new Promise((resolve) => this.queue.push(resolve));
+    }
+    this.active++;
+    try {
+      return await fn();
+    } finally {
+      this.active--;
+      this.queue.shift()?.();
+    }
+  }
+}
+var DEFAULT_MAX_CONCURRENT_READS = 4;
+function maxConcurrentReads() {
+  const raw = process.env.SUBGRAPH_CHAIN_READ_CONCURRENCY;
+  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_MAX_CONCURRENT_READS;
+}
+var gate = new ConcurrencyGate(maxConcurrentReads());
+function hashArgs(args) {
+  const serialized = args.map((arg) => serializeCV(arg)).join(",");
+  return Bun.hash(serialized).toString(16);
+}
+function findFunction(abi, name) {
+  const fn = abi.functions.find((f) => f.name === name);
+  if (!fn) {
+    throw new ChainReadError(`No function "${name}" in the supplied ABI.`);
+  }
+  if (fn.access !== "read-only") {
+    throw new ChainReadError(`"${name}" is ${fn.access}, not read-only — a handler cannot call it.`);
+  }
+  return fn;
+}
+function createChainReadClient(ctx) {
+  const readValue = async (contractId, abi, functionName, args, options) => {
+    const rpcUrl = ctx.rpcUrl ?? process.env.STACKS_NODE_RPC_URL;
+    if (!rpcUrl) {
+      throw new ChainReadError("ctx.client needs a Stacks node: set STACKS_NODE_RPC_URL on the subgraph processor.");
+    }
+    const constant = options.cache === "contract-constant";
+    if (!constant && !ctx.indexBlockHash) {
+      throw new ChainReadError(`Block ${ctx.blockHeight} has no index_block_hash, so a read cannot be pinned to it. Reading at the node's tip would make this handler non-deterministic (the same reindex would produce different rows). Re-ingest the block, or pass cache: "contract-constant" if the value genuinely cannot change.`);
+    }
+    const fn = findFunction(abi, functionName);
+    const clarityArgs = buildFunctionArgs(fn, args);
+    const db = ctx.db ?? getSourceDb();
+    const argsHash = hashArgs(clarityArgs);
+    const pinnedTo = constant ? null : ctx.indexBlockHash;
+    const cached = await db.selectFrom("chain_read_cache").select("result_hex").where("contract_id", "=", contractId).where("function_name", "=", fn.name).where("args_hash", "=", argsHash).where((eb) => pinnedTo === null ? eb("index_block_hash", "is", null) : eb("index_block_hash", "=", pinnedTo)).executeTakeFirst();
+    const resultHex = cached?.result_hex ?? await gate.run(async () => {
+      const value2 = await readContract(nodeClient(rpcUrl), {
+        contract: contractId,
+        functionName: fn.name,
+        args: clarityArgs,
+        tip: ctx.indexBlockHash ?? undefined
+      });
+      return serializeCV(value2);
+    });
+    if (!cached) {
+      await db.insertInto("chain_read_cache").values({
+        contract_id: contractId,
+        function_name: fn.name,
+        args_hash: argsHash,
+        index_block_hash: pinnedTo,
+        block_height: constant ? null : ctx.blockHeight,
+        result_hex: resultHex
+      }).onConflict((oc) => oc.doNothing()).execute();
+    }
+    const value = deserializeCV(resultHex);
+    const js = clarityValueToJSUntyped(fn.outputs, value);
+    if (isResponseOutput(fn.outputs)) {
+      const response = js;
+      if ("ok" in response)
+        return response.ok;
+      throw new ContractResponseError(`${contractId}::${fn.name} returned an err response.`, response.err);
+    }
+    return js;
+  };
+  return {
+    contract(contractId, rawAbi, options = {}) {
+      const abi = normalizeAbi(rawAbi);
+      const camelToKebab = new Map;
+      for (const fn of abi.functions) {
+        camelToKebab.set(toCamelCase(fn.name), fn.name);
+      }
+      const read = new Proxy({}, {
+        get(_target, prop) {
+          const fnName = camelToKebab.get(prop) ?? prop;
+          return (args = {}) => readValue(contractId, abi, fnName, args, options);
+        }
+      });
+      return { read };
+    }
+  };
+}
+
 // src/runtime/context.ts
 var JOURNAL_RETENTION_BLOCKS = 300;
 var journalEnsured = new Set;
@@ -148,6 +296,7 @@ class SubgraphContext {
   ops = [];
   byo;
   journal;
+  _client;
   constructor(db, pgSchemaName2, subgraphSchema, block, tx, byo = false, journal = false) {
     this.db = db;
     this.pgSchemaName = pgSchemaName2;
@@ -157,6 +306,13 @@ class SubgraphContext {
     this.byo = byo;
     this.journal = journal;
   }
+  get client() {
+    this._client ??= createChainReadClient({
+      blockHeight: this.block.height,
+      indexBlockHash: this.block.indexBlockHash
+    });
+    return this._client;
+  }
   get tx() {
     return this._tx;
   }
@@ -262,23 +418,21 @@ class SubgraphContext {
   formatUnits(value, decimals) {
     return formatUnits(value, decimals);
   }
-  async findOne(table, where) {
-    this.validateTable(table);
+  async readRows(table, where, limit) {
     const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
     const { clause } = buildWhereClause(where);
-    const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause} LIMIT 1`;
+    const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause}${limit !== undefined ? ` LIMIT ${limit}` : ""}`;
     const { rows } = await sql.raw(query).execute(this.db);
-    const row = rows[0] ?? null;
-    return this.overlayOne(table, where, row ? this.coerceRow(table, row) : null);
+    return rows.map((r) => this.coerceRow(table, r));
+  }
+  async findOne(table, where) {
+    this.validateTable(table);
+    const row = (await this.readRows(table, where, 1))[0] ?? null;
+    return this.overlayOne(table, where, row);
   }
   async findMany(table, where) {
     this.validateTable(table);
-    const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
-    const { clause } = buildWhereClause(where);
-    const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause}`;
-    const { rows } = await sql.raw(query).execute(this.db);
-    const dbRows = rows.map((r) => this.coerceRow(table, r));
-    return this.overlayMany(table, where, dbRows);
+    return this.overlayMany(table, where, await this.readRows(table, where));
   }
   overlayOne(table, where, dbRow) {
     if (this.ops.length === 0)
@@ -720,11 +874,11 @@ function buildWhereClause(where) {
 }
 
 // src/runtime/clarity.ts
-import { cvToValue, deserializeCV } from "@secondlayer/stacks/clarity";
+import { cvToValue, deserializeCV as deserializeCV2 } from "@secondlayer/stacks/clarity";
 function decodeClarityValue(hex) {
   try {
     const cleanHex = hex.startsWith("0x") ? hex.slice(2) : hex;
-    const cv = deserializeCV(cleanHex);
+    const cv = deserializeCV2(cleanHex);
     return cvToValue(cv);
   } catch {
     return hex;
@@ -755,8 +909,8 @@ import { getErrorMessage } from "@secondlayer/shared";
 import { logger as logger2 } from "@secondlayer/shared/logger";
 import {
   clarityValueToJS,
-  deserializeCV as deserializeCV2,
-  toCamelCase
+  deserializeCV as deserializeCV3,
+  toCamelCase as toCamelCase2
 } from "@secondlayer/stacks/clarity";
 
 // src/runtime/print-validate.ts
@@ -919,7 +1073,7 @@ function buildContractCallInput(filter, tx) {
       return;
     try {
       const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
-      input[toCamelCase(arg.name)] = decodeArg(arg.type, deserializeCV2(clean));
+      input[toCamelCase2(arg.name)] = decodeArg(arg.type, deserializeCV3(clean));
     } catch {}
   });
   return input;
@@ -1198,6 +1352,9 @@ async function runHandlers(subgraph, matched, ctx, opts) {
 
 // src/runtime/source-matcher.ts
 var patternCache = new Map;
+function matchAny(value, pattern) {
+  return Array.isArray(pattern) ? pattern.some((p) => matchPattern(value, p)) : matchPattern(value, pattern);
+}
 function matchPattern(value, pattern) {
   if (!pattern.includes("*"))
     return value === pattern;
@@ -1210,6 +1367,22 @@ function matchPattern(value, pattern) {
   return re.test(value);
 }
 var EMPTY_SET = new Set;
+function factoryAllows(filter, contractId, factoryContracts) {
+  if (!filter.factory)
+    return true;
+  if (!contractId)
+    return false;
+  return (factoryContracts.get(filter.factory.from) ?? EMPTY_SET).has(contractId);
+}
+function readPath(payload, path) {
+  let cur = payload;
+  for (const key of path.split(".")) {
+    if (cur === null || typeof cur !== "object")
+      return;
+    cur = cur[key];
+  }
+  return cur;
+}
 function traitAllows(filter, contractId, traitContracts) {
   const trait = filter.trait;
   if (!trait)
@@ -1221,7 +1394,7 @@ function traitAllows(filter, contractId, traitContracts) {
 function assetContract(assetId) {
   return assetId?.split("::")[0];
 }
-function matchFilter(filter, transactions, eventsByTx, traitContracts) {
+function matchFilter(filter, transactions, eventsByTx, traitContracts, factoryContracts = new Map) {
   const results = [];
   switch (filter.type) {
     case "stx_transfer":
@@ -1348,9 +1521,11 @@ function matchFilter(filter, transactions, eventsByTx, traitContracts) {
         if (tx.type !== "contract_call")
           continue;
         if (filter.contractId) {
-          if (!tx.contract_id || !matchPattern(tx.contract_id, filter.contractId))
+          if (!tx.contract_id || !matchAny(tx.contract_id, filter.contractId))
             continue;
         }
+        if (!factoryAllows(filter, tx.contract_id, factoryContracts))
+          continue;
         if (filter.functionName) {
           if (!tx.function_name || !matchPattern(tx.function_name, filter.functionName))
             continue;
@@ -1397,11 +1572,13 @@ function matchFilter(filter, transactions, eventsByTx, traitContracts) {
             return false;
           const printContractId = data.contract_identifier ?? data.contract_id;
           if (filter.contractId) {
-            if (!printContractId || !matchPattern(printContractId, filter.contractId))
+            if (!printContractId || !matchAny(printContractId, filter.contractId))
               return false;
           }
           if (!traitAllows(filter, printContractId, traitContracts))
             return false;
+          if (!factoryAllows(filter, printContractId, factoryContracts))
+            return false;
           return true;
         });
         if (matched.length > 0) {
@@ -1413,7 +1590,7 @@ function matchFilter(filter, transactions, eventsByTx, traitContracts) {
   }
   return results;
 }
-function matchSources(sources, transactions, events, traitContracts = new Map) {
+function matchSources(sources, transactions, events, traitContracts = new Map, factoryContracts = new Map) {
   const eventsByTx = new Map;
   for (const event of events) {
     const list = eventsByTx.get(event.tx_id) ?? [];
@@ -1423,7 +1600,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
   const seen = new Set;
   const results = [];
   for (const [sourceName, filter] of Object.entries(sources)) {
-    const matches = matchFilter(filter, transactions, eventsByTx, traitContracts);
+    const matches = matchFilter(filter, transactions, eventsByTx, traitContracts, factoryContracts);
     for (const match of matches) {
       const dedupeKey = `${match.tx.tx_id}:${sourceName}`;
       if (!seen.has(dedupeKey)) {
@@ -1450,7 +1627,7 @@ import { logger as logger5 } from "@secondlayer/shared/logger";
 import { sql as sql3 } from "kysely";
 
 // src/runtime/block-source.ts
-import { getSourceDb } from "@secondlayer/shared/db";
+import { getSourceDb as getSourceDb2 } from "@secondlayer/shared/db";
 import {
   IndexHttpClient
 } from "@secondlayer/shared/index-http";
@@ -1511,6 +1688,7 @@ function reconstructBlock(b) {
     parent_hash: b.parent_hash,
     burn_block_height: b.burn_block_height,
     burn_block_hash: b.burn_block_hash,
+    index_block_hash: b.index_block_hash ?? null,
     timestamp: isoToUnixSeconds(b.block_time),
     canonical: true,
     created_at: new Date(0)
@@ -1645,11 +1823,11 @@ function sparseProbeTargets(subgraph) {
 
 class PostgresBlockSource {
   async getTip() {
-    const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
+    const progress = await getSourceDb2().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
     return progress ? Number(progress.highest_seen_block) : 0;
   }
   loadBlockRange(fromHeight, toHeight) {
-    return loadBlockRange(getSourceDb(), fromHeight, toHeight);
+    return loadBlockRange(getSourceDb2(), fromHeight, toHeight);
   }
 }
 var EVENT_FILTER_TO_INDEX_TYPE = {
@@ -1718,8 +1896,8 @@ class PublicApiBlockSource {
   eventTypes;
   probeTargets;
   needsTransactions;
-  constructor(http, eventTypes, probeTargets, needsTransactions = true) {
-    this.http = http;
+  constructor(http2, eventTypes, probeTargets, needsTransactions = true) {
+    this.http = http2;
     this.eventTypes = eventTypes;
     this.probeTargets = probeTargets;
     this.needsTransactions = needsTransactions;
@@ -2078,6 +2256,57 @@ async function resolveRoute(subgraphName, targetDb) {
 function invalidateSubgraphRoute(subgraphName) {
   routeCache.delete(subgraphName);
 }
+async function resolveFactoryContracts(subgraph, blockHeight, schemaName, db, txs, evts) {
+  const factories = new Map;
+  for (const source of Object.values(subgraph.sources)) {
+    const factory = source.factory;
+    if (factory)
+      factories.set(factory.from, factory);
+  }
+  const resolved = new Map;
+  const discovered = [];
+  if (factories.size === 0)
+    return { resolved, discovered };
+  for (const [discoveringSource, factory] of factories) {
+    const known = new Set;
+    try {
+      const rows = await sql3`
+				SELECT address FROM ${sql3.raw(`"${schemaName}"."_factory_addresses"`)}
+				WHERE source_name = ${discoveringSource} AND block_height <= ${blockHeight}
+			`.execute(db);
+      for (const row of rows.rows)
+        known.add(row.address);
+    } catch {}
+    const discovering = subgraph.sources[factory.from];
+    if (discovering) {
+      const matches = matchSources({ [factory.from]: discovering }, txs, evts, new Map, new Map);
+      for (const match of matches) {
+        for (const event of match.events ?? []) {
+          const payload = buildEventPayload(discovering, match.tx, event);
+          const value = readPath(payload, factory.field);
+          if (typeof value === "string" && value.length > 0 && !known.has(value)) {
+            known.add(value);
+            discovered.push({ sourceName: discoveringSource, address: value });
+          }
+        }
+      }
+    }
+    resolved.set(discoveringSource, known);
+  }
+  return { resolved, discovered };
+}
+async function persistFactoryDiscoveries(schemaName, db, blockHeight, discovered) {
+  if (discovered.length === 0)
+    return;
+  for (const { sourceName, address } of discovered) {
+    await sql3`
+			INSERT INTO ${sql3.raw(`"${schemaName}"."_factory_addresses"`)}
+				(source_name, address, block_height)
+			VALUES (${sourceName}, ${address}, ${blockHeight})
+			ON CONFLICT (source_name, address) DO NOTHING
+		`.execute(db);
+  }
+}
 async function resolveTraitContracts(subgraph, blockHeight, db) {
   const traits = new Set;
   for (const source of Object.values(subgraph.sources)) {
@@ -2164,8 +2393,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     txs = data.txs;
     evts = data.events;
   }
+  const route = await resolveRoute(subgraphName, targetDb);
+  const schemaName = route.schemaName;
   const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
-  const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
+  const { resolved: factoryContracts, discovered } = await resolveFactoryContracts(subgraph, blockHeight, schemaName, targetDb, txs, evts);
+  const matched = matchSources(subgraph.sources, txs, evts, traitContracts, factoryContracts);
   result.matched = matched.length;
   if (matched.length === 0) {
     if (!opts?.skipProgressUpdate) {
@@ -2173,13 +2405,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     }
     return result;
   }
-  const route = await resolveRoute(subgraphName, targetDb);
-  const schemaName = route.schemaName;
   const blockMeta = {
     height: block.height,
     hash: block.hash,
     timestamp: block.timestamp,
-    burnBlockHeight: block.burn_block_height
+    burnBlockHeight: block.burn_block_height,
+    indexBlockHash: block.index_block_hash
   };
   const initialTx = {
     txId: "",
@@ -2232,6 +2463,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     });
     result.processed = runResult.processed;
     result.errors = runResult.errors;
+    await persistFactoryDiscoveries(schemaName, route.dataDb, blockHeight, discovered);
     await targetDb.transaction().execute(async (tx) => {
       if (manifest && manifest.count > 0) {
         await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
@@ -3230,6 +3462,11 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
             await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" >= $1`, [blockHeight]);
           }
         }
+        await client.unsafe(`DO $$ BEGIN
+						IF to_regclass('"${schemaName}"."_factory_addresses"') IS NOT NULL THEN
+							DELETE FROM "${schemaName}"."_factory_addresses" WHERE block_height >= ${Number(blockHeight)};
+						END IF;
+					END $$`);
         for (const [tableName, rows] of Object.entries(revertedByTable)) {
           if (rows.length === 0)
             continue;
@@ -3356,8 +3593,8 @@ import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-h
 import { logger as logger10 } from "@secondlayer/shared/logger";
 var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
 var STARTUP_MARGIN_MS = 60 * 60 * 1000;
-async function pollReorgsOnce(http, cursor, onReorg, handled) {
-  const { reorgs, next_since } = await http.listReorgs(cursor);
+async function pollReorgsOnce(http2, cursor, onReorg, handled) {
+  const { reorgs, next_since } = await http2.listReorgs(cursor);
   const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
   for (const r of sorted) {
     if (handled && r.id && handled.has(r.id)) {
@@ -3378,7 +3615,7 @@ async function pollReorgsOnce(http, cursor, onReorg, handled) {
 }
 function startStreamsReorgPoll(onReorg) {
   const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
-  const http = new IndexHttpClient2({
+  const http2 = new IndexHttpClient2({
     indexBaseUrl: baseUrl,
     streamsBaseUrl: baseUrl,
     streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_decode_internal"
@@ -3393,7 +3630,7 @@ function startStreamsReorgPoll(onReorg) {
     if (handled.size > 1e4)
       handled.clear();
     try {
-      since = await pollReorgsOnce(http, since, onReorg, handled);
+      since = await pollReorgsOnce(http2, since, onReorg, handled);
     } catch (err) {
       logger10.error("Streams reorg poll failed", {
         error: getErrorMessage5(err)
@@ -3768,5 +4005,5 @@ export {
   startSubgraphOperationRunner
 };
 
-//# debugId=0DCB2AA2FDAD1E6864756E2164756E21
+//# debugId=72D00DA8ABE4D8A264756E2164756E21
 //# sourceMappingURL=processor.js.map