@secondlayer/subgraphs 3.2.1 → 3.3.0

package/dist/src/runtime/reindex.d.ts

@@ -9,6 +9,22 @@ interface SubgraphColumn {
 	search?: boolean;
 	default?: string | number | boolean;
 }
+/**
+* A foreign-key relation to another table in the same subgraph. Drives DDL FK
+* constraints and ORM codegen (`@relation` in Prisma, `relations()` in Drizzle)
+* so generated clients get typed joins. The referenced columns must form a
+* `uniqueKeys` entry on the target table.
+*/
+interface SubgraphRelation {
+	/** Relation field name on this table's generated model (e.g. "pool"). */
+	name: string;
+	/** Target table name in this subgraph. */
+	references: string;
+	/** Local column(s) holding the foreign key. */
+	fields: string[];
+	/** Target column(s) the fields point at (a uniqueKeys entry on the target). */
+	referencedColumns: string[];
+}
 /** Table definition within a subgraph schema */
 interface SubgraphTable {
 	columns: Record<string, SubgraphColumn>;
@@ -16,6 +32,8 @@ interface SubgraphTable {
 	indexes?: string[][];
 	/** Unique key constraints (each entry is an array of column names). Required for upsert. */
 	uniqueKeys?: string[][];
+	/** Foreign-key relations to other tables (for typed ORM joins). */
+	relations?: SubgraphRelation[];
 }
 /** Subgraph schema — maps table names to table definitions */
 type SubgraphSchema = Record<string, SubgraphTable>;
@@ -42,45 +60,53 @@ interface StxLockFilter {
 	lockedAddress?: string;
 	minAmount?: bigint;
 }
+/**
+* Restrict a source to contracts conforming to a trait/standard (e.g. "sip-010")
+* instead of a fixed contract — resolved from the contract registry at match time,
+* as-of each processed block. Lets a source index "all SIP-010 tokens" etc.
+*/
+type TraitScope = {
+	trait?: string
+};
 /** FT event filters */
-interface FtTransferFilter {
+interface FtTransferFilter extends TraitScope {
 	type: "ft_transfer";
 	assetIdentifier?: string;
 	sender?: string;
 	recipient?: string;
 	minAmount?: bigint;
 }
-interface FtMintFilter {
+interface FtMintFilter extends TraitScope {
 	type: "ft_mint";
 	assetIdentifier?: string;
 	recipient?: string;
 	minAmount?: bigint;
 }
-interface FtBurnFilter {
+interface FtBurnFilter extends TraitScope {
 	type: "ft_burn";
 	assetIdentifier?: string;
 	sender?: string;
 	minAmount?: bigint;
 }
 /** NFT event filters */
-interface NftTransferFilter {
+interface NftTransferFilter extends TraitScope {
 	type: "nft_transfer";
 	assetIdentifier?: string;
 	sender?: string;
 	recipient?: string;
 }
-interface NftMintFilter {
+interface NftMintFilter extends TraitScope {
 	type: "nft_mint";
 	assetIdentifier?: string;
 	recipient?: string;
 }
-interface NftBurnFilter {
+interface NftBurnFilter extends TraitScope {
 	type: "nft_burn";
 	assetIdentifier?: string;
 	sender?: string;
 }
 /** Contract event filters */
-interface ContractCallFilter {
+interface ContractCallFilter extends TraitScope {
 	type: "contract_call";
 	contractId?: string;
 	functionName?: string;
@@ -98,7 +124,7 @@ interface ContractDeployFilter {
 	deployer?: string;
 	contractName?: string;
 }
-interface PrintEventFilter {
+interface PrintEventFilter extends TraitScope {
 	type: "print_event";
 	contractId?: string;
 	topic?: string;

package/dist/src/runtime/reindex.js

@@ -18,12 +18,14 @@ class SubgraphContext {
   pgSchemaName;
   subgraphSchema;
   ops = [];
-  constructor(db, pgSchemaName, subgraphSchema, block, tx) {
+  byo;
+  constructor(db, pgSchemaName, subgraphSchema, block, tx, byo = false) {
     this.db = db;
     this.pgSchemaName = pgSchemaName;
     this.subgraphSchema = subgraphSchema;
     this.block = block;
     this._tx = tx;
+    this.byo = byo;
   }
   get tx() {
     return this._tx;
@@ -218,6 +220,15 @@ class SubgraphContext {
   }
   buildStatements(ops) {
     const statements = [];
+    if (this.byo) {
+      const insertTables = new Set;
+      for (const op of ops)
+        if (op.kind === "insert")
+          insertTables.add(op.table);
+      for (const table of insertTables) {
+        statements.push(`DELETE FROM "${this.pgSchemaName}"."${table}" WHERE "_block_height" = ${this.block.height}`);
+      }
+    }
     let currentBatch = null;
     let currentBatchKey = "";
     const flushInsertBatch = () => {
@@ -711,7 +722,19 @@ function matchPattern(value, pattern) {
   }
   return re.test(value);
 }
-function matchFilter(filter, transactions, eventsByTx) {
+var EMPTY_SET = new Set;
+function traitAllows(filter, contractId, traitContracts) {
+  const trait = filter.trait;
+  if (!trait)
+    return true;
+  if (!contractId)
+    return false;
+  return (traitContracts.get(trait) ?? EMPTY_SET).has(contractId);
+}
+function assetContract(assetId) {
+  return assetId?.split("::")[0];
+}
+function matchFilter(filter, transactions, eventsByTx, traitContracts) {
   const results = [];
   switch (filter.type) {
     case "stx_transfer":
@@ -775,6 +798,8 @@ function matchFilter(filter, transactions, eventsByTx) {
             if (!assetId || !matchPattern(assetId, filter.assetIdentifier))
               return false;
           }
+          if (!traitAllows(filter, assetContract(data.asset_identifier), traitContracts))
+            return false;
           if ("sender" in filter && filter.sender) {
             if (!matchPattern(data.sender, filter.sender))
               return false;
@@ -813,6 +838,8 @@ function matchFilter(filter, transactions, eventsByTx) {
             if (!assetId || !matchPattern(assetId, filter.assetIdentifier))
               return false;
           }
+          if (!traitAllows(filter, assetContract(data.asset_identifier), traitContracts))
+            return false;
           if ("sender" in filter && filter.sender) {
             if (!matchPattern(data.sender, filter.sender))
               return false;
@@ -845,6 +872,8 @@ function matchFilter(filter, transactions, eventsByTx) {
           if (!matchPattern(tx.sender, filter.caller))
             continue;
         }
+        if (!traitAllows(filter, tx.contract_id, traitContracts))
+          continue;
         const txEvents = eventsByTx.get(tx.tx_id) ?? [];
         results.push({ tx, events: txEvents });
       }
@@ -879,11 +908,13 @@ function matchFilter(filter, transactions, eventsByTx) {
             return false;
           if (data.topic !== "print")
             return false;
+          const printContractId = data.contract_identifier ?? data.contract_id;
           if (filter.contractId) {
-            const contractId = data.contract_identifier ?? data.contract_id;
-            if (!contractId || !matchPattern(contractId, filter.contractId))
+            if (!printContractId || !matchPattern(printContractId, filter.contractId))
               return false;
           }
+          if (!traitAllows(filter, printContractId, traitContracts))
+            return false;
           return true;
         });
         if (matched.length > 0) {
@@ -895,7 +926,7 @@ function matchFilter(filter, transactions, eventsByTx) {
   }
   return results;
 }
-function matchSources(sources, transactions, events) {
+function matchSources(sources, transactions, events, traitContracts = new Map) {
   const eventsByTx = new Map;
   for (const event of events) {
     const list = eventsByTx.get(event.tx_id) ?? [];
@@ -905,7 +936,7 @@ function matchSources(sources, transactions, events) {
   const seen = new Set;
   const results = [];
   for (const [sourceName, filter] of Object.entries(sources)) {
-    const matches = matchFilter(filter, transactions, eventsByTx);
+    const matches = matchFilter(filter, transactions, eventsByTx, traitContracts);
     for (const match of matches) {
       const dedupeKey = `${match.tx.tx_id}:${sourceName}`;
       if (!seen.has(dedupeKey)) {
@@ -922,8 +953,11 @@ import {
   getSourceDb,
   getTargetDb
 } from "@secondlayer/shared/db";
+import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
 import {
+  isByoSubgraph,
   recordSubgraphProcessed,
+  resolveSubgraphDb,
   updateSubgraphStatus
 } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger4 } from "@secondlayer/shared/logger";
@@ -1161,7 +1195,38 @@ async function refreshMatcher(db) {
 }
 
 // src/runtime/block-processor.ts
-var schemaNameCache = new Map;
+var routeCache = new Map;
+async function resolveRoute(subgraphName, targetDb) {
+  const cached = routeCache.get(subgraphName);
+  if (cached)
+    return cached;
+  const row = await targetDb.selectFrom("subgraphs").selectAll().where("name", "=", subgraphName).executeTakeFirst();
+  const byo = row ? isByoSubgraph(row) : false;
+  const route = {
+    schemaName: row?.schema_name ?? pgSchemaName(subgraphName),
+    dataDb: row && byo ? resolveSubgraphDb(row) : targetDb,
+    byo
+  };
+  routeCache.set(subgraphName, route);
+  return route;
+}
+function invalidateSubgraphRoute(subgraphName) {
+  routeCache.delete(subgraphName);
+}
+async function resolveTraitContracts(subgraph, blockHeight, db) {
+  const traits = new Set;
+  for (const source of Object.values(subgraph.sources)) {
+    const trait = source.trait;
+    if (trait)
+      traits.add(trait);
+  }
+  const resolved = new Map;
+  for (const trait of traits) {
+    const ids = await resolveTraitContractIds(db, trait, blockHeight);
+    resolved.set(trait, new Set(ids));
+  }
+  return resolved;
+}
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   const sourceDb = getSourceDb();
   const targetDb = getTargetDb();
@@ -1201,7 +1266,8 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
     evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   }
-  const matched = matchSources(subgraph.sources, txs, evts);
+  const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
+  const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
   result.matched = matched.length;
   if (matched.length === 0) {
     if (!opts?.skipProgressUpdate) {
@@ -1209,12 +1275,8 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     }
     return result;
   }
-  let schemaName = schemaNameCache.get(subgraphName);
-  if (!schemaName) {
-    const subgraphRecord = await targetDb.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
-    schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
-    schemaNameCache.set(subgraphName, schemaName);
-  }
+  const route = await resolveRoute(subgraphName, targetDb);
+  const schemaName = route.schemaName;
   const blockMeta = {
     height: block.height,
     hash: block.hash,
@@ -1229,30 +1291,57 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   };
   let handlerMs = 0;
   let flushMs = 0;
-  await targetDb.transaction().execute(async (tx) => {
-    const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx);
-    const handlerStart = performance.now();
-    const runResult = await runHandlers(subgraph, matched, ctx);
-    handlerMs = performance.now() - handlerStart;
+  const applyProgress = async (tx, rr) => {
+    if (opts?.skipProgressUpdate)
+      return;
+    const status = rr.errors > 0 && rr.processed === 0 ? "error" : "active";
+    await updateSubgraphStatus(tx, subgraphName, status, blockHeight);
+    if (rr.processed > 0 || rr.errors > 0) {
+      const lastError = rr.errors > 0 ? `${rr.errors} error(s) at block ${blockHeight}` : undefined;
+      await recordSubgraphProcessed(tx, subgraphName, rr.processed, rr.errors, lastError);
+    }
+  };
+  if (route.byo) {
+    let runResult = { processed: 0, errors: 0 };
+    let manifest;
+    await route.dataDb.transaction().execute(async (tx) => {
+      const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, true);
+      const handlerStart = performance.now();
+      runResult = await runHandlers(subgraph, matched, ctx);
+      handlerMs = performance.now() - handlerStart;
+      if (ctx.pendingOps > 0) {
+        const flushStart = performance.now();
+        manifest = await ctx.flush();
+        flushMs = performance.now() - flushStart;
+      }
+    });
     result.processed = runResult.processed;
     result.errors = runResult.errors;
-    if (ctx.pendingOps > 0) {
-      const flushStart = performance.now();
-      const manifest = await ctx.flush();
-      if (manifest.count > 0) {
+    await targetDb.transaction().execute(async (tx) => {
+      if (manifest && manifest.count > 0) {
         await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
       }
-      flushMs = performance.now() - flushStart;
-    }
-    if (!opts?.skipProgressUpdate) {
-      const status = runResult.errors > 0 && runResult.processed === 0 ? "error" : "active";
-      await updateSubgraphStatus(tx, subgraphName, status, blockHeight);
-    }
-    if (!opts?.skipProgressUpdate && (runResult.processed > 0 || runResult.errors > 0)) {
-      const lastError = runResult.errors > 0 ? `${runResult.errors} error(s) at block ${blockHeight}` : undefined;
-      await recordSubgraphProcessed(tx, subgraphName, runResult.processed, runResult.errors, lastError);
-    }
-  });
+      await applyProgress(tx, runResult);
+    });
+  } else {
+    await targetDb.transaction().execute(async (tx) => {
+      const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx);
+      const handlerStart = performance.now();
+      const runResult = await runHandlers(subgraph, matched, ctx);
+      handlerMs = performance.now() - handlerStart;
+      result.processed = runResult.processed;
+      result.errors = runResult.errors;
+      if (ctx.pendingOps > 0) {
+        const flushStart = performance.now();
+        const manifest = await ctx.flush();
+        if (manifest.count > 0) {
+          await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
+        }
+        flushMs = performance.now() - flushStart;
+      }
+      await applyProgress(tx, runResult);
+    });
+  }
   const totalMs = performance.now() - blockStart;
   result.timing = {
     totalMs: Math.round(totalMs),
@@ -1263,7 +1352,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     try {
       const tables = Object.keys(subgraph.schema);
       for (const table of tables) {
-        const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
+        const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
           logger4.warn("Subgraph table exceeds 10M rows (estimate)", {
@@ -1435,6 +1524,12 @@ function generateSubgraphSQL(def, schemaNameOverride) {
       }
     }
   }
+  for (const [tableName, tableDef] of Object.entries(def.schema)) {
+    for (const rel of tableDef.relations ?? []) {
+      const constraintName = `fk_${schemaName}_${tableName}_${rel.name}`;
+      statements.push(`ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`);
+    }
+  }
   const hashInput = JSON.stringify({
     name: def.name,
     schema: def.schema,
@@ -1903,5 +1998,5 @@ export {
   backfillSubgraph
 };
 
-//# debugId=1CF94A64247CCC5564756E2164756E21
+//# debugId=C6A12A80FA03F24E64756E2164756E21
 //# sourceMappingURL=reindex.js.map