@secondlayer/subgraphs 0.5.7 → 0.7.0

package/dist/src/runtime/block-processor.js

@@ -16,8 +16,8 @@ function sourceKey(source) {
 }
 
 // src/runtime/context.ts
-import { sql } from "kysely";
 import { logger } from "@secondlayer/shared/logger";
+import { sql } from "kysely";
 function validateColumnName(name) {
   if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
     throw new Error(`Invalid column name: ${name}`);
@@ -58,13 +58,26 @@ class SubgraphContext {
     const keyColumns = Object.keys(key);
     const hasUniqueConstraint = tableDef.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
     if (hasUniqueConstraint) {
-      this.ops.push({ kind: "insert", table, data: { ...key, ...row, _upsert_keys: keyColumns } });
+      this.ops.push({
+        kind: "insert",
+        table,
+        data: { ...key, ...row, _upsert_keys: keyColumns }
+      });
     } else {
       logger.warn("upsert called without matching uniqueKeys constraint, using fallback", {
         table,
         keys: keyColumns
       });
-      this.ops.push({ kind: "insert", table, data: { ...key, ...row, _upsert_fallback_keys: keyColumns, _upsert_fallback_set: row } });
+      this.ops.push({
+        kind: "insert",
+        table,
+        data: {
+          ...key,
+          ...row,
+          _upsert_fallback_keys: keyColumns,
+          _upsert_fallback_set: row
+        }
+      });
     }
   }
   delete(table, where) {
@@ -122,53 +135,83 @@ class SubgraphContext {
     }
     return opsToFlush.length;
   }
+  prepareInsert(op) {
+    const upsertKeys = op.data._upsert_keys;
+    const data = { ...op.data };
+    delete data._upsert_keys;
+    delete data._upsert_fallback_keys;
+    delete data._upsert_fallback_set;
+    data._block_height = this.block.height;
+    data._tx_id = this._tx.txId;
+    data._created_at = "NOW()";
+    const cols = Object.keys(data);
+    cols.forEach(validateColumnName);
+    const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
+    const batchKey = `${op.table}:${[...cols].sort().join(",")}:${upsertKeys ? [...upsertKeys].sort().join(",") : ""}`;
+    return { data, cols, vals, upsertKeys, batchKey };
+  }
   buildStatements(ops) {
     const statements = [];
+    let currentBatch = null;
+    let currentBatchKey = "";
+    const flushInsertBatch = () => {
+      if (!currentBatch)
+        return;
+      const qualifiedTable = `"${this.pgSchemaName}"."${currentBatch.table}"`;
+      const colList = currentBatch.cols.map((c) => `"${c}"`).join(", ");
+      let rows = currentBatch.rows;
+      if (currentBatch.upsertKeys && currentBatch.upsertKeys.length > 0) {
+        const keyIndices = currentBatch.upsertKeys.map((k) => currentBatch.cols.indexOf(k));
+        const seen = new Map;
+        for (let i = 0;i < rows.length; i++) {
+          const key = keyIndices.map((ki) => rows[i][ki]).join("\x00");
+          seen.set(key, i);
+        }
+        if (seen.size < rows.length) {
+          rows = Array.from(seen.values()).map((i) => rows[i]);
+        }
+      }
+      const valuesList = rows.map((r) => `(${r.join(", ")})`).join(", ");
+      let stmt = `INSERT INTO ${qualifiedTable} (${colList}) VALUES ${valuesList}`;
+      if (currentBatch.upsertKeys && currentBatch.upsertKeys.length > 0) {
+        const updateCols = currentBatch.cols.filter((c) => !currentBatch.upsertKeys.includes(c) && !c.startsWith("_"));
+        if (updateCols.length > 0) {
+          const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
+          stmt += ` ON CONFLICT (${currentBatch.upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
+        } else {
+          stmt += ` ON CONFLICT (${currentBatch.upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
+        }
+      }
+      statements.push(stmt);
+      currentBatch = null;
+      currentBatchKey = "";
+    };
     for (const op of ops) {
       const qualifiedTable = `"${this.pgSchemaName}"."${op.table}"`;
-      switch (op.kind) {
-        case "insert": {
-          const upsertKeys = op.data._upsert_keys;
-          const fallbackKeys = op.data._upsert_fallback_keys;
-          const fallbackSet = op.data._upsert_fallback_set;
-          const data = { ...op.data };
-          delete data._upsert_keys;
-          delete data._upsert_fallback_keys;
-          delete data._upsert_fallback_set;
-          data._block_height = this.block.height;
-          data._tx_id = this._tx.txId;
-          data._created_at = "NOW()";
-          const cols = Object.keys(data);
-          cols.forEach(validateColumnName);
-          const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
-          let stmt = `INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) VALUES (${vals.join(", ")})`;
-          if (upsertKeys && upsertKeys.length > 0) {
-            const updateCols = cols.filter((c) => !upsertKeys.includes(c) && !c.startsWith("_"));
-            if (updateCols.length > 0) {
-              const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
-              stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
-            } else {
-              stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
-            }
-          } else if (fallbackKeys && fallbackSet) {}
-          statements.push(stmt);
-          break;
+      if (op.kind === "insert") {
+        const { cols, vals, upsertKeys, batchKey } = this.prepareInsert(op);
+        if (batchKey === currentBatchKey && currentBatch) {
+          currentBatch.rows.push(vals);
+        } else {
+          flushInsertBatch();
+          currentBatch = { table: op.table, cols, rows: [vals], upsertKeys };
+          currentBatchKey = batchKey;
         }
-        case "update": {
+      } else {
+        flushInsertBatch();
+        if (op.kind === "update") {
           const setEntries = Object.entries(op.set);
           setEntries.forEach(([k]) => validateColumnName(k));
           const setClauses = setEntries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
           const { clause } = buildWhereClause(op.data);
           statements.push(`UPDATE ${qualifiedTable} SET ${setClauses.join(", ")} WHERE ${clause}`);
-          break;
-        }
-        case "delete": {
+        } else if (op.kind === "delete") {
           const { clause } = buildWhereClause(op.data);
           statements.push(`DELETE FROM ${qualifiedTable} WHERE ${clause}`);
-          break;
         }
       }
     }
+    flushInsertBatch();
     return statements;
   }
   validateTable(table) {
@@ -197,95 +240,6 @@ function buildWhereClause(where) {
   return { clause: parts.join(" AND "), values: [] };
 }
 
-// src/runtime/source-matcher.ts
-function matchPattern(value, pattern) {
-  if (!pattern.includes("*"))
-    return value === pattern;
-  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-  return new RegExp(`^${regex}$`).test(value);
-}
-function matchSource(source, transactions, eventsByTx) {
-  const results = [];
-  const key = sourceKey(source);
-  for (const tx of transactions) {
-    if (source.type) {
-      if (!matchPattern(tx.type, source.type))
-        continue;
-      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
-      let matchedEvents = txEvents;
-      if (source.minAmount !== undefined) {
-        const amountEvents = matchedEvents.filter((e) => {
-          const data = e.data;
-          const rawAmount = data?.amount;
-          if (rawAmount === undefined)
-            return false;
-          const amount = BigInt(rawAmount);
-          return amount >= source.minAmount;
-        });
-        if (amountEvents.length === 0)
-          continue;
-        matchedEvents = amountEvents;
-      }
-      results.push({ tx, events: matchedEvents, sourceKey: key });
-      continue;
-    }
-    if (source.contract) {
-      const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
-      if (source.function && tx.function_name) {
-        if (!matchPattern(tx.function_name, source.function))
-          continue;
-      } else if (source.function && !tx.function_name) {
-        continue;
-      }
-      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
-      let matchedEvents = txEvents;
-      if (!txContractMatch) {
-        matchedEvents = txEvents.filter((e) => {
-          const data = e.data;
-          const contractIdentifier = data?.contract_identifier;
-          return contractIdentifier && matchPattern(contractIdentifier, source.contract);
-        });
-        if (matchedEvents.length === 0)
-          continue;
-      }
-      if (source.event) {
-        matchedEvents = matchedEvents.filter((e) => {
-          if (matchPattern(e.type, source.event))
-            return true;
-          const data = e.data;
-          const topic = data?.topic;
-          return topic ? matchPattern(topic, source.event) : false;
-        });
-      }
-      if (txContractMatch || matchedEvents.length > 0) {
-        results.push({ tx, events: matchedEvents, sourceKey: key });
-      }
-    }
-  }
-  return results;
-}
-function matchSources(sources, transactions, events) {
-  const eventsByTx = new Map;
-  for (const event of events) {
-    const list = eventsByTx.get(event.tx_id) ?? [];
-    list.push(event);
-    eventsByTx.set(event.tx_id, list);
-  }
-  const seen = new Set;
-  const results = [];
-  for (const source of sources) {
-    const matches = matchSource(source, transactions, eventsByTx);
-    for (const match of matches) {
-      const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
-      if (!seen.has(dedupeKey)) {
-        seen.add(dedupeKey);
-        results.push(match);
-      }
-    }
-  }
-  return results;
-}
-
 // src/runtime/clarity.ts
 import { cvToJSON, deserializeCV } from "@secondlayer/stacks/clarity";
 function decodeClarityValue(hex) {
@@ -318,8 +272,8 @@ function decodeFunctionArgs(args) {
 }
 
 // src/runtime/runner.ts
-import { logger as logger2 } from "@secondlayer/shared/logger";
 import { getErrorMessage } from "@secondlayer/shared";
+import { logger as logger2 } from "@secondlayer/shared/logger";
 var DEFAULT_ERROR_THRESHOLD = 50;
 function resolveHandler(handlers, key) {
   return handlers[key] ?? handlers["*"] ?? null;
@@ -331,7 +285,11 @@ async function runHandlers(subgraph, matched, ctx, opts) {
   for (const { tx, events, sourceKey: sourceKey2 } of matched) {
     const handler = resolveHandler(subgraph.handlers, sourceKey2);
     if (!handler) {
-      logger2.warn("No handler found for source key", { subgraph: subgraph.name, sourceKey: sourceKey2, txId: tx.tx_id });
+      logger2.warn("No handler found for source key", {
+        subgraph: subgraph.name,
+        sourceKey: sourceKey2,
+        txId: tx.tx_id
+      });
       continue;
     }
     ctx.setTx({
@@ -408,16 +366,109 @@ async function runHandlers(subgraph, matched, ctx, opts) {
   return { processed, errors };
 }
 
+// src/runtime/source-matcher.ts
+function matchPattern(value, pattern) {
+  if (!pattern.includes("*"))
+    return value === pattern;
+  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  return new RegExp(`^${regex}$`).test(value);
+}
+function matchSource(source, transactions, eventsByTx) {
+  const results = [];
+  const key = sourceKey(source);
+  for (const tx of transactions) {
+    if (source.type) {
+      if (!matchPattern(tx.type, source.type))
+        continue;
+      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
+      let matchedEvents = txEvents;
+      if (source.minAmount !== undefined) {
+        const amountEvents = matchedEvents.filter((e) => {
+          const data = e.data;
+          const rawAmount = data?.amount;
+          if (rawAmount === undefined)
+            return false;
+          const amount = BigInt(rawAmount);
+          return amount >= source.minAmount;
+        });
+        if (amountEvents.length === 0)
+          continue;
+        matchedEvents = amountEvents;
+      }
+      results.push({ tx, events: matchedEvents, sourceKey: key });
+      continue;
+    }
+    if (source.contract) {
+      const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
+      if (source.function && tx.function_name) {
+        if (!matchPattern(tx.function_name, source.function))
+          continue;
+      } else if (source.function && !tx.function_name) {
+        continue;
+      }
+      const txEvents = eventsByTx.get(tx.tx_id) ?? [];
+      let matchedEvents = txEvents;
+      if (!txContractMatch) {
+        matchedEvents = txEvents.filter((e) => {
+          const data = e.data;
+          const contractIdentifier = data?.contract_identifier;
+          return contractIdentifier && matchPattern(contractIdentifier, source.contract);
+        });
+        if (matchedEvents.length === 0)
+          continue;
+      }
+      if (source.event) {
+        matchedEvents = matchedEvents.filter((e) => {
+          if (matchPattern(e.type, source.event))
+            return true;
+          const data = e.data;
+          const topic = data?.topic;
+          return topic ? matchPattern(topic, source.event) : false;
+        });
+      }
+      if (txContractMatch || matchedEvents.length > 0) {
+        results.push({ tx, events: matchedEvents, sourceKey: key });
+      }
+    }
+  }
+  return results;
+}
+function matchSources(sources, transactions, events) {
+  const eventsByTx = new Map;
+  for (const event of events) {
+    const list = eventsByTx.get(event.tx_id) ?? [];
+    list.push(event);
+    eventsByTx.set(event.tx_id, list);
+  }
+  const seen = new Set;
+  const results = [];
+  for (const source of sources) {
+    const matches = matchSource(source, transactions, eventsByTx);
+    for (const match of matches) {
+      const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
+      if (!seen.has(dedupeKey)) {
+        seen.add(dedupeKey);
+        results.push(match);
+      }
+    }
+  }
+  return results;
+}
+
 // src/runtime/block-processor.ts
-import { sql as sql2 } from "kysely";
 import { getDb } from "@secondlayer/shared/db";
+import {
+  recordSubgraphProcessed,
+  updateSubgraphStatus
+} from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger3 } from "@secondlayer/shared/logger";
-import { updateSubgraphStatus, recordSubgraphProcessed } from "@secondlayer/shared/db/queries/subgraphs";
+import { sql as sql2 } from "kysely";
 
 // src/schema/utils.ts
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 
 // src/runtime/block-processor.ts
+var schemaNameCache = new Map;
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   const db = getDb();
   const blockStart = performance.now();
@@ -428,19 +479,32 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     errors: 0,
     skipped: false
   };
-  const block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
-  if (!block) {
-    logger3.warn("Block not found for subgraph processing", { subgraph: subgraphName, blockHeight });
-    result.skipped = true;
-    return result;
-  }
-  if (!block.canonical) {
-    logger3.debug("Skipping non-canonical block", { subgraph: subgraphName, blockHeight });
-    result.skipped = true;
-    return result;
+  let block, txs, evts;
+  if (opts?.preloaded) {
+    block = opts.preloaded.block;
+    txs = opts.preloaded.txs;
+    evts = opts.preloaded.events;
+  } else {
+    block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
+    if (!block) {
+      logger3.warn("Block not found for subgraph processing", {
+        subgraph: subgraphName,
+        blockHeight
+      });
+      result.skipped = true;
+      return result;
+    }
+    if (!block.canonical) {
+      logger3.debug("Skipping non-canonical block", {
+        subgraph: subgraphName,
+        blockHeight
+      });
+      result.skipped = true;
+      return result;
+    }
+    txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
+    evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   }
-  const txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
-  const evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
   const matched = matchSources(subgraph.sources, txs, evts);
   result.matched = matched.length;
   if (matched.length === 0) {
@@ -449,8 +513,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     }
     return result;
   }
-  const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
-  const schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
+  let schemaName = schemaNameCache.get(subgraphName);
+  if (!schemaName) {
+    const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
+    schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
+    schemaNameCache.set(subgraphName, schemaName);
+  }
   const blockMeta = {
     height: block.height,
     hash: block.hash,
@@ -499,7 +567,11 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         const { rows } = await sql2.raw(`SELECT COUNT(*) as count FROM "${schemaName}"."${table}"`).execute(db);
         const count = Number(rows[0].count);
         if (count >= 1e7) {
-          logger3.warn("Subgraph table exceeds 10M rows", { subgraph: subgraphName, table, count });
+          logger3.warn("Subgraph table exceeds 10M rows", {
+            subgraph: subgraphName,
+            table,
+            count
+          });
         }
       }
     } catch {}
@@ -510,5 +582,5 @@ export {
   processBlock
 };
 
-//# debugId=11059A8A65310A0964756E2164756E21
+//# debugId=256005336347A12F64756E2164756E21
 //# sourceMappingURL=block-processor.js.map