@secondlayer/subgraphs 6.4.0 → 6.6.0

package/dist/testing/index.js

@@ -1,6 +1,3 @@
-// src/testing/harness.ts
-import { normalizeAbi as normalizeAbi2, toCamelCase as toCamelCase3 } from "@secondlayer/stacks/clarity";
-
 // src/runtime/context.ts
 import { logger as logger2 } from "@secondlayer/shared/logger";
 import { sql } from "kysely";
@@ -862,13 +859,127 @@ function buildWhereClause(where) {
   return { clause: parts.join(" AND "), values: [] };
 }
 
+// src/runtime/memory-store.ts
+import { normalizeAbi as normalizeAbi2, toCamelCase as toCamelCase2 } from "@secondlayer/stacks/clarity";
+class MemoryStore {
+  tables = new Map;
+}
+
+class MemorySubgraphContext extends SubgraphContext {
+  store;
+  tables;
+  constructor(store, schema, block, tx) {
+    super(undefined, "memory", schema, block, tx, false);
+    this.store = store;
+    this.tables = schema;
+  }
+  async readRows(table, where, limit) {
+    const rows = (this.store.tables.get(table) ?? []).filter((row) => Object.entries(where).every(([k, v]) => sameValue(row[k], v)));
+    return limit === undefined ? rows : rows.slice(0, limit);
+  }
+  readDependent = null;
+  dependsOnPrior(what) {
+    this.readDependent ??= what;
+  }
+  findOne(table, where) {
+    this.dependsOnPrior(`findOne("${table}")`);
+    return super.findOne(table, where);
+  }
+  findMany(table, where) {
+    this.dependsOnPrior(`findMany("${table}")`);
+    return super.findMany(table, where);
+  }
+  update(table, where, set) {
+    this.dependsOnPrior(`update("${table}")`);
+    super.update(table, where, set);
+  }
+  delete(table, where) {
+    this.dependsOnPrior(`delete("${table}")`);
+    super.delete(table, where);
+  }
+  increment(table, key, deltas) {
+    this.dependsOnPrior(`increment("${table}")`);
+    super.increment(table, key, deltas);
+  }
+  upsert(table, key, row) {
+    const def = this.tables[table];
+    if (def) {
+      const keyCols = Object.keys(key);
+      const keyed = def.uniqueKeys?.some((uk) => uk.length === keyCols.length && uk.every((c) => keyCols.includes(c)));
+      const omitted = Object.keys(def.columns).find((c) => !(c in key) && !(c in row));
+      if (!keyed)
+        this.dependsOnPrior(`upsert("${table}") without a unique key`);
+      else if (omitted)
+        this.dependsOnPrior(`upsert("${table}") leaves "${omitted}" as it was`);
+    }
+    super.upsert(table, key, row);
+  }
+  async rowsOf(table) {
+    return this.overlayMany(table, {}, await this.readRows(table, {}));
+  }
+  setReads(reads) {
+    this._client = {
+      contract(contractId, abi) {
+        const camelToKebab = new Map;
+        for (const fn of normalizeAbi2(abi).functions) {
+          camelToKebab.set(toCamelCase2(fn.name), fn.name);
+        }
+        const read = new Proxy({}, {
+          get(_target, prop) {
+            const fnName = camelToKebab.get(prop) ?? prop;
+            const key = `${contractId}.${fnName}`;
+            return async () => {
+              if (!(key in reads)) {
+                throw new Error(`No stubbed chain read for "${key}" — pass it via createTestContext(schema, { reads: { "${key}": … } }).`);
+              }
+              return reads[key];
+            };
+          }
+        });
+        return { read };
+      }
+    };
+  }
+  async commitOps() {
+    const tables = new Set;
+    for (const op of this.ops)
+      tables.add(op.table);
+    for (const table of tables) {
+      this.store.tables.set(table, await this.rowsOf(table));
+    }
+    this.ops.length = 0;
+  }
+  insertsSince(checkpoint) {
+    const out = [];
+    for (const op of this.ops.slice(Math.max(0, checkpoint))) {
+      if (op.kind !== "insert")
+        continue;
+      const row = { ...op.data };
+      out.push({
+        table: op.table,
+        keys: Object.keys(row).filter((k) => !k.startsWith("_")),
+        row
+      });
+    }
+    return out;
+  }
+}
+function sameValue(a, b) {
+  if (a === b)
+    return true;
+  if ((typeof a === "bigint" || typeof a === "number") && (typeof b === "bigint" || typeof b === "number")) {
+    return BigInt(a) === BigInt(b);
+  }
+  return String(a) === String(b);
+}
+
 // src/runtime/runner.ts
 import { getErrorMessage } from "@secondlayer/shared";
 import { logger as logger4 } from "@secondlayer/shared/logger";
 import {
   clarityValueToJS,
   deserializeCV as deserializeCV4,
-  toCamelCase as toCamelCase2
+  toCamelCase as toCamelCase3
 } from "@secondlayer/stacks/clarity";
 
 // src/print-schema.ts
@@ -1253,6 +1364,74 @@ var COLUMN_TYPES = new Set([
   "timestamp",
   "jsonb"
 ]);
+function isColumnType(field) {
+  return typeof field === "string" && COLUMN_TYPES.has(field);
+}
+function matchesField(field, value) {
+  if (isColumnType(field)) {
+    switch (field) {
+      case "uint":
+      case "int":
+        return typeof value === "bigint" || typeof value === "number";
+      case "boolean":
+        return typeof value === "boolean";
+      case "text":
+      case "principal":
+      case "timestamp":
+        return typeof value === "string" || typeof value === "number" || typeof value === "bigint";
+      case "jsonb":
+        return value !== undefined;
+    }
+  }
+  if (typeof field !== "object" || field === null)
+    return false;
+  if ("optional" in field) {
+    return value === undefined || value === null || matchesField(field.type, value);
+  }
+  if ("tuple" in field) {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+      return false;
+    }
+    return matchesTopic(field.tuple, value).ok;
+  }
+  if ("list" in field) {
+    return Array.isArray(value) && value.every((item) => matchesField(field.list, item));
+  }
+  return false;
+}
+function matchesTopic(declared, data) {
+  for (const [name, field] of Object.entries(declared)) {
+    const present = name in data;
+    const optional = typeof field === "object" && field !== null && "optional" in field;
+    if (!present) {
+      if (optional)
+        continue;
+      return { ok: false, reason: `missing declared field "${name}"` };
+    }
+    if (!matchesField(field, data[name])) {
+      return {
+        ok: false,
+        reason: `field "${name}" does not match its declared type (got ${describe(data[name])})`
+      };
+    }
+  }
+  return { ok: true };
+}
+function describe(value) {
+  if (value === null)
+    return "null";
+  if (Array.isArray(value))
+    return "array";
+  return typeof value;
+}
+function validatePrintPayload(prints, topic, data) {
+  if (!prints)
+    return { ok: true };
+  const declared = prints[topic];
+  if (!declared)
+    return { ok: true };
+  return matchesTopic(declared, data);
+}
 
 // src/runtime/realm.ts
 import vm from "node:vm";
@@ -1262,7 +1441,7 @@ import { createHash as createHash2 } from "node:crypto";
 // package.json
 var package_default = {
   name: "@secondlayer/subgraphs",
-  version: "6.4.0",
+  version: "6.6.0",
   description: "Describe the Stacks data you want as one TypeScript file; it fills itself and survives reorgs.",
   license: "MIT",
   repository: {
@@ -1306,6 +1485,10 @@ var package_default = {
     "./testing": {
       types: "./dist/testing/index.d.ts",
       import: "./dist/testing/index.js"
+    },
+    "./verify": {
+      types: "./dist/verify/index.d.ts",
+      import: "./dist/verify/index.js"
     }
   },
   files: ["dist"],
@@ -1317,14 +1500,16 @@ var package_default = {
     prepublishOnly: "bun run build"
   },
   dependencies: {
-    "@secondlayer/shared": "^11.22.0",
-    "@secondlayer/stacks": "^6.1.0",
+    "@secondlayer/shared": "^11.24.0",
+    "@secondlayer/stacks": "^6.2.0",
+    "@secondlayer/verify": "^0.3.0",
     esbuild: "^0.28.2",
     kysely: "0.28.17",
     zod: "^4.3.6"
   },
   devDependencies: {
     "@types/bun": "latest",
+    bunup: "0.16.32",
     typescript: "^5"
   },
   publishConfig: {
@@ -1454,7 +1639,6 @@ function formatFinding(f) {
 var STATE_SOURCE_TYPES = new Set([
   "var_set",
   "map_set",
-  "map_insert",
   "map_delete"
 ]);
 var LEVEL_RANK = { state: 0, events: 1, none: 2 };
@@ -1471,6 +1655,9 @@ function deriveVerification(def, findings = []) {
     if (filter.trait) {
       raise("none");
       reasons.push(`source "${name}": trait scope needs a proven contract registry`);
+    } else if (filter.type === "map_insert") {
+      raise("events");
+      reasons.push(`map_insert source "${name}": map_insert vs map_set needs event proofs`);
     } else if (!STATE_SOURCE_TYPES.has(filter.type)) {
       raise("events");
       reasons.push(`${filter.type} source "${name}" needs event proofs`);
@@ -1488,14 +1675,12 @@ function deriveVerification(def, findings = []) {
   if (findings.length > MAX_LISTED_FINDINGS) {
     reasons.push(`…and ${findings.length - MAX_LISTED_FINDINGS} more handler findings`);
   }
-  const writesMapInsert = Object.values(def.sources ?? {}).some((s) => s.type === "map_insert");
   return {
     level,
     reasons,
     unproven: reasons.length === 0 ? [
       "tx attribution of writes: event.tx, _tx_id (needs event proofs)",
-      "intermediate writes within a block (needs event proofs)",
-      ...writesMapInsert ? ["map_insert vs map_set (needs event proofs)"] : []
+      "intermediate writes within a block (needs event proofs)"
     ] : []
   };
 }
@@ -1505,15 +1690,18 @@ function isDeterminismViolation(v) {
 function sha256(input) {
   return createHash2("sha256").update(input).digest("hex");
 }
-function computePin(input) {
-  const canonical = JSON.stringify({
+function pinPreimage(input) {
+  const fields = {
     handlerHash: sha256(input.handlerCode),
     network: input.network,
     runtime: input.runtime ?? SUBGRAPHS_RUNTIME,
     schemaHash: input.schemaHash,
     startBlock: input.startBlock ?? null
-  });
-  return sha256(canonical);
+  };
+  return JSON.stringify(fields);
+}
+function computePin(input) {
+  return sha256(pinPreimage(input));
 }
 
 // src/runtime/realm.ts
@@ -1524,6 +1712,9 @@ class NondeterminismError extends Error {
 class HandlerReadError extends Error {
   name = "HandlerReadError";
 }
+function abortsBlock(err) {
+  return err instanceof NondeterminismError || err instanceof HandlerReadError;
+}
 var HOST_GLOBALS = [
   "ArrayBuffer",
   "DataView",
@@ -1695,6 +1886,7 @@ function deterministicContext(ctx, schema) {
     }
   };
 }
+var realmDefinitions = new WeakSet;
 async function loadDeterministicDefinition(handlerCode) {
   const { transform } = await import("esbuild");
   const { code } = await transform(handlerCode, {
@@ -1723,7 +1915,9 @@ ${code}
       return handler(event, facade);
     };
   }
-  return { ...def, sources, schema, handlers };
+  const loaded = { ...def, sources, schema, handlers };
+  realmDefinitions.add(loaded);
+  return loaded;
 }
 
 // src/runtime/source-matcher.ts
@@ -2046,8 +2240,58 @@ function matchSources(sources, transactions, events, traitContracts = new Map, f
 import { getTargetDb } from "@secondlayer/shared/db";
 import { logger as logger3 } from "@secondlayer/shared/logger";
 import { sql as sql2 } from "kysely";
+var VIOLATIONS_CAP_PER_SUBGRAPH = 100;
+function jsonSafe2(value) {
+  if (typeof value === "bigint")
+    return value.toString();
+  if (value === null || value === undefined)
+    return value;
+  if (Array.isArray(value))
+    return value.map(jsonSafe2);
+  if (typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = jsonSafe2(v);
+    }
+    return out;
+  }
+  return value;
+}
+async function defaultRecordPrintViolation(input) {
+  try {
+    const db = getTargetDb();
+    const sample = jsonSafe2(input.samplePayload);
+    await db.insertInto("subgraph_violations").values({
+      subgraph_name: input.subgraphName,
+      source_name: input.sourceName,
+      block_height: input.blockHeight,
+      tx_id: input.txId,
+      reason: input.reason,
+      sample_payload: sample
+    }).execute();
+    await sql2`
+			DELETE FROM subgraph_violations
+			WHERE id IN (
+				SELECT id FROM subgraph_violations
+				WHERE subgraph_name = ${input.subgraphName}
+				ORDER BY seen_at DESC
+				OFFSET ${VIOLATIONS_CAP_PER_SUBGRAPH}
+			)
+		`.execute(db);
+  } catch (err) {
+    logger3.warn("Failed to record print violation", {
+      subgraph: input.subgraphName,
+      error: err instanceof Error ? err.message : String(err)
+    });
+  }
+}
+var recorder = defaultRecordPrintViolation;
+async function recordPrintViolation(input) {
+  await recorder(input);
+}
 
 // src/runtime/runner.ts
+var DEFAULT_ERROR_THRESHOLD = 50;
 function decodeFunctionArgs(args) {
   let parsed = args;
   if (typeof parsed === "string") {
@@ -2117,7 +2361,7 @@ function buildContractCallInput(filter, tx) {
       return;
     try {
       const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
-      input[toCamelCase2(arg.name)] = decodeArg(arg.type, deserializeCV4(clean));
+      input[toCamelCase3(arg.name)] = decodeArg(arg.type, deserializeCV4(clean));
     } catch {}
   });
   return input;
@@ -2330,9 +2574,144 @@ function buildEventPayload(filter, tx, event) {
       };
   }
 }
+async function runHandlers(subgraph, matched, ctx, opts) {
+  let processed = 0;
+  let errors = 0;
+  let skipped = 0;
+  let delivered = 0;
+  const threshold = opts?.errorThreshold ?? DEFAULT_ERROR_THRESHOLD;
+  const filterLookup = new Map;
+  if (!Array.isArray(subgraph.sources)) {
+    for (const [name, filter] of Object.entries(subgraph.sources)) {
+      filterLookup.set(name, filter);
+    }
+  }
+  const units = [];
+  for (const { tx, events, sourceName } of matched) {
+    if (events.length === 0) {
+      units.push({ tx, sourceName, event: null });
+    } else {
+      for (const event of events)
+        units.push({ tx, sourceName, event });
+    }
+  }
+  const clockRank = (e) => e?.clock === "vm" ? 1 : 0;
+  units.sort((a, b) => (a.tx.tx_index ?? 0) - (b.tx.tx_index ?? 0) || clockRank(a.event) - clockRank(b.event) || (a.event?.event_index ?? -1) - (b.event?.event_index ?? -1));
+  for (const { tx, event, sourceName } of units) {
+    if (errors >= threshold) {
+      logger4.error("Subgraph error threshold reached, skipping remaining events", {
+        subgraph: subgraph.name,
+        errors,
+        threshold
+      });
+      return {
+        processed,
+        errors,
+        delivered,
+        ...skipped > 0 ? { skipped } : {}
+      };
+    }
+    const filter = filterLookup.get(sourceName);
+    const handler = subgraph.handlers[sourceName] ?? null;
+    if (!handler) {
+      logger4.warn("No handler found for source", {
+        subgraph: subgraph.name,
+        sourceName,
+        txId: tx.tx_id
+      });
+      continue;
+    }
+    ctx.setTx({
+      txId: tx.tx_id,
+      sender: tx.sender,
+      type: tx.type,
+      status: tx.status,
+      contractId: tx.contract_id ?? null,
+      functionName: tx.function_name ?? null
+    });
+    const checkpoint = ctx.opsCheckpoint();
+    try {
+      let payload;
+      if (event === null) {
+        payload = filter ? buildEventPayload(filter, tx, null) : {
+          tx: {
+            txId: tx.tx_id,
+            sender: tx.sender,
+            type: tx.type,
+            status: tx.status,
+            contractId: tx.contract_id,
+            functionName: tx.function_name
+          }
+        };
+      } else if (filter) {
+        payload = buildEventPayload(filter, tx, event);
+      } else {
+        const decoded = decodeEventData(event.data);
+        payload = {
+          ...decoded,
+          _eventId: event.id,
+          _eventType: event.type,
+          _eventIndex: event.event_index,
+          tx: {
+            txId: tx.tx_id,
+            sender: tx.sender,
+            type: tx.type,
+            status: tx.status,
+            contractId: tx.contract_id,
+            functionName: tx.function_name
+          }
+        };
+      }
+      if (event !== null && filter?.type === "print_event" && filter.topic && payload.topic !== filter.topic) {
+        continue;
+      }
+      delivered++;
+      if (event !== null && filter?.type === "print_event" && filter.prints) {
+        const printPayload = payload;
+        const verdict = validatePrintPayload(filter.prints, printPayload.topic ?? "", printPayload.data ?? {});
+        if (!verdict.ok) {
+          skipped++;
+          logger4.warn("Print payload does not match declared prints schema", {
+            subgraph: subgraph.name,
+            sourceName,
+            topic: printPayload.topic,
+            txId: tx.tx_id,
+            reason: verdict.reason
+          });
+          recordPrintViolation({
+            subgraphName: subgraph.name,
+            sourceName,
+            blockHeight: ctx.block.height,
+            txId: tx.tx_id,
+            reason: verdict.reason,
+            samplePayload: {
+              topic: printPayload.topic ?? "",
+              data: printPayload.data ?? {}
+            }
+          });
+          continue;
+        }
+      }
+      await handler(payload, ctx);
+      processed++;
+    } catch (err) {
+      if (abortsBlock(err))
+        throw err;
+      ctx.rollbackTo(checkpoint);
+      errors++;
+      logger4.error("Subgraph handler error", {
+        subgraph: subgraph.name,
+        sourceName,
+        txId: tx.tx_id,
+        ...event !== null ? { eventId: event.id, eventType: event.type } : {},
+        error: getErrorMessage(err)
+      });
+    }
+  }
+  return { processed, errors, delivered, ...skipped > 0 ? { skipped } : {} };
+}
 
 // src/testing/harness.ts
-var TEST_SCHEMA_NAME = "subgraph_test";
 function defaultBlock(overrides = {}) {
   return {
     height: 1,
@@ -2353,76 +2732,8 @@ function defaultTx(overrides = {}) {
     ...overrides
   };
 }
-
-class InMemorySubgraphContext extends SubgraphContext {
-  store = new Map;
-  constructor(schema, block, tx) {
-    super(undefined, TEST_SCHEMA_NAME, schema, block, tx, false);
-  }
-  async readRows(table, where, limit) {
-    const rows = (this.store.get(table) ?? []).filter((row) => Object.entries(where).every(([k, v]) => sameValue(row[k], v)));
-    return limit === undefined ? rows : rows.slice(0, limit);
-  }
-  async rowsOf(table) {
-    return this.overlayMany(table, {}, await this.readRows(table, {}));
-  }
-  setReads(reads) {
-    this._client = {
-      contract(contractId, abi) {
-        const camelToKebab = new Map;
-        for (const fn of normalizeAbi2(abi).functions) {
-          camelToKebab.set(toCamelCase3(fn.name), fn.name);
-        }
-        const read = new Proxy({}, {
-          get(_target, prop) {
-            const fnName = camelToKebab.get(prop) ?? prop;
-            const key = `${contractId}.${fnName}`;
-            return async () => {
-              if (!(key in reads)) {
-                throw new Error(`No stubbed chain read for "${key}" — pass it via createTestContext(schema, { reads: { "${key}": … } }).`);
-              }
-              return reads[key];
-            };
-          }
-        });
-        return { read };
-      }
-    };
-  }
-  async commitOps() {
-    const tables = new Set;
-    for (const op of this.ops)
-      tables.add(op.table);
-    for (const table of tables) {
-      this.store.set(table, await this.rowsOf(table));
-    }
-    this.ops.length = 0;
-  }
-  insertsSince(checkpoint) {
-    const out = [];
-    for (const op of this.ops.slice(Math.max(0, checkpoint))) {
-      if (op.kind !== "insert")
-        continue;
-      const row = { ...op.data };
-      out.push({
-        table: op.table,
-        keys: Object.keys(row).filter((k) => !k.startsWith("_")),
-        row
-      });
-    }
-    return out;
-  }
-}
-function sameValue(a, b) {
-  if (a === b)
-    return true;
-  if ((typeof a === "bigint" || typeof a === "number") && (typeof b === "bigint" || typeof b === "number")) {
-    return BigInt(a) === BigInt(b);
-  }
-  return String(a) === String(b);
-}
 function createTestContext(schema, options = {}) {
-  const impl = new InMemorySubgraphContext(schema, defaultBlock(options.block), defaultTx(options.tx));
+  const impl = new MemorySubgraphContext(new MemoryStore, schema, defaultBlock(options.block), defaultTx(options.tx));
   impl.setReads(options.reads ?? {});
   const ctx = impl;
   ctx.rows = (table) => impl.rowsOf(table);
@@ -2537,17 +2848,17 @@ function toContractCallPayload(row) {
     }
   };
 }
-function jsonSafe2(value) {
+function jsonSafe3(value) {
   if (typeof value === "bigint")
     return value.toString();
   if (value === null || value === undefined)
     return value;
   if (Array.isArray(value))
-    return value.map(jsonSafe2);
+    return value.map(jsonSafe3);
   if (typeof value === "object") {
     const out = {};
     for (const [k, v] of Object.entries(value)) {
-      out[k] = jsonSafe2(v);
+      out[k] = jsonSafe3(v);
     }
     return out;
   }
@@ -2658,7 +2969,7 @@ async function runSubgraphTest(input) {
     tables.push({
       name: table,
       rows: tableRows.length,
-      ...sample ? { sampleRow: jsonSafe2(sample) } : {}
+      ...sample ? { sampleRow: jsonSafe3(sample) } : {}
     });
   }
   const fetched = Object.values(events).reduce((n, r) => n + r.length, 0);
@@ -2687,7 +2998,7 @@ async function runSubgraphTest(input) {
       matched,
       written: 0,
       tables,
-      firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe2(firstEvent.data) } : undefined,
+      firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe3(firstEvent.data) } : undefined,
       hint: `${matched} event${matched === 1 ? "" : "s"} matched, but NO rows were written — the shape of the field-mapping bug that ships a 0-row subgraph.${keyPart} Map only observed keys; do not invent fields.`,
       ...traceFields
     };
@@ -2701,7 +3012,7 @@ async function runSubgraphTest(input) {
       matched,
       written,
       tables,
-      firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe2(firstEvent.data) } : undefined,
+      firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe3(firstEvent.data) } : undefined,
       hint: `At least one matched event wrote 0 rows.${keyPart}`,
       ...traceFields
     };
@@ -2711,7 +3022,7 @@ async function runSubgraphTest(input) {
     matched,
     written,
     tables,
-    firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe2(firstEvent.data) } : undefined,
+    firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe3(firstEvent.data) } : undefined,
     ...traceFields
   };
 }
@@ -2757,5 +3068,5 @@ export {
   toHandlerPayload
 };
 
-//# debugId=124EC2B77481A62F64756E2164756E21
+//# debugId=C0C7992E00C29F7064756E2164756E21
 //# sourceMappingURL=index.js.map