@secondlayer/subgraphs 4.1.5 → 4.2.0

package/dist/src/testing/index.js

@@ -1212,7 +1212,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map, f
 
 // src/runtime/runner.ts
 import { getErrorMessage } from "@secondlayer/shared";
-import { logger as logger3 } from "@secondlayer/shared/logger";
+import { logger as logger4 } from "@secondlayer/shared/logger";
 import {
   clarityValueToJS,
   deserializeCV as deserializeCV4,
@@ -1226,6 +1226,19 @@ import {
 function camelizeDataKey(str) {
   return str.replace(/-([a-z0-9])/g, (_, c) => c.toUpperCase());
 }
+function camelizeKeys(obj) {
+  if (obj === null || obj === undefined)
+    return obj;
+  if (typeof obj !== "object")
+    return obj;
+  if (Array.isArray(obj))
+    return obj.map(camelizeKeys);
+  const result = {};
+  for (const [k, v] of Object.entries(obj)) {
+    result[camelizeDataKey(k)] = camelizeKeys(v);
+  }
+  return result;
+}
 function cvToTree(cv) {
   switch (cv.type) {
     case "uint":
@@ -1550,6 +1563,58 @@ function inferPrintTopics(samples) {
   return out.sort((a, b) => b.count - a.count);
 }
 
+// src/runtime/materialize.ts
+function resolveColumn(col, event, tx, block) {
+  if ("fromTx" in col) {
+    return { ok: true, value: tx[col.fromTx] };
+  }
+  if ("fromBlock" in col) {
+    return { ok: true, value: block[col.fromBlock] };
+  }
+  const data = event.data;
+  const dataObj = data !== null && typeof data === "object" && !Array.isArray(data) ? data : undefined;
+  if (dataObj && Object.prototype.hasOwnProperty.call(dataObj, col.from)) {
+    return { ok: true, value: dataObj[col.from] };
+  }
+  if (col.from === "topic" && "topic" in event) {
+    return { ok: true, value: event.topic };
+  }
+  if (Object.prototype.hasOwnProperty.call(event, col.from)) {
+    return { ok: true, value: event[col.from] };
+  }
+  return { ok: false, missing: col.from };
+}
+function buildMaterializeRow(spec, event, ctx, tableColumns) {
+  const row = {};
+  for (const [colName, mapping] of Object.entries(spec.columns)) {
+    const resolved = resolveColumn(mapping, event, ctx.tx, ctx.block);
+    if (!resolved.ok) {
+      const colDef = tableColumns[colName];
+      if (colDef?.nullable) {
+        row[colName] = null;
+        continue;
+      }
+      return {
+        ok: false,
+        reason: `required materialize field "${resolved.missing}" missing for column "${colName}"`
+      };
+    }
+    row[colName] = resolved.value ?? null;
+  }
+  return { ok: true, row };
+}
+function applyMaterializeInsert(spec, event, ctx, schema) {
+  const table = schema[spec.table];
+  if (!table) {
+    return { ok: false, reason: `materialize table "${spec.table}" missing` };
+  }
+  const built = buildMaterializeRow(spec, event, ctx, table.columns);
+  if (!built.ok)
+    return built;
+  ctx.insert(spec.table, built.row);
+  return { ok: true };
+}
+
 // src/runtime/print-validate.ts
 var COLUMN_TYPES = new Set([
   "text",
@@ -1629,21 +1694,62 @@ function validatePrintPayload(prints, topic, data) {
   return matchesTopic(declared, data);
 }
 
-// src/runtime/runner.ts
-var DEFAULT_ERROR_THRESHOLD = 50;
-function camelizeKeys(obj) {
-  if (obj === null || obj === undefined)
-    return obj;
-  if (typeof obj !== "object")
-    return obj;
-  if (Array.isArray(obj))
-    return obj.map(camelizeKeys);
-  const result = {};
-  for (const [k, v] of Object.entries(obj)) {
-    result[camelizeDataKey(k)] = camelizeKeys(v);
+// src/runtime/violations.ts
+import { getTargetDb } from "@secondlayer/shared/db";
+import { logger as logger3 } from "@secondlayer/shared/logger";
+import { sql as sql2 } from "kysely";
+var VIOLATIONS_CAP_PER_SUBGRAPH = 100;
+function jsonSafe2(value) {
+  if (typeof value === "bigint")
+    return value.toString();
+  if (value === null || value === undefined)
+    return value;
+  if (Array.isArray(value))
+    return value.map(jsonSafe2);
+  if (typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = jsonSafe2(v);
+    }
+    return out;
+  }
+  return value;
+}
+async function defaultRecordPrintViolation(input) {
+  try {
+    const db = getTargetDb();
+    const sample = jsonSafe2(input.samplePayload);
+    await db.insertInto("subgraph_violations").values({
+      subgraph_name: input.subgraphName,
+      source_name: input.sourceName,
+      block_height: input.blockHeight,
+      tx_id: input.txId,
+      reason: input.reason,
+      sample_payload: sample
+    }).execute();
+    await sql2`
+			DELETE FROM subgraph_violations
+			WHERE id IN (
+				SELECT id FROM subgraph_violations
+				WHERE subgraph_name = ${input.subgraphName}
+				ORDER BY seen_at DESC
+				OFFSET ${VIOLATIONS_CAP_PER_SUBGRAPH}
+			)
+		`.execute(db);
+  } catch (err) {
+    logger3.warn("Failed to record print violation", {
+      subgraph: input.subgraphName,
+      error: err instanceof Error ? err.message : String(err)
+    });
   }
-  return result;
 }
+var recorder = defaultRecordPrintViolation;
+async function recordPrintViolation(input) {
+  await recorder(input);
+}
+
+// src/runtime/runner.ts
+var DEFAULT_ERROR_THRESHOLD = 50;
 function decodeFunctionArgs2(args) {
   let parsed = args;
   if (typeof parsed === "string") {
@@ -1891,16 +1997,18 @@ async function runHandlers(subgraph, matched, ctx, opts) {
   units.sort((a, b) => (a.tx.tx_index ?? 0) - (b.tx.tx_index ?? 0) || (a.event?.event_index ?? -1) - (b.event?.event_index ?? -1));
   for (const { tx, event, sourceName } of units) {
     if (errors >= threshold) {
-      logger3.error("Subgraph error threshold reached, skipping remaining events", {
+      logger4.error("Subgraph error threshold reached, skipping remaining events", {
         subgraph: subgraph.name,
         errors,
         threshold
       });
       return { processed, errors, ...skipped > 0 ? { skipped } : {} };
     }
+    const filter = filterLookup.get(sourceName);
+    const materialize = filter && "materialize" in filter && filter.materialize !== undefined ? filter.materialize : undefined;
     const handler = subgraph.handlers[sourceName] ?? subgraph.handlers["*"] ?? null;
-    if (!handler) {
-      logger3.warn("No handler found for source", {
+    if (!handler && !materialize) {
+      logger4.warn("No handler found for source", {
         subgraph: subgraph.name,
         sourceName,
         txId: tx.tx_id
@@ -1915,7 +2023,6 @@ async function runHandlers(subgraph, matched, ctx, opts) {
       contractId: tx.contract_id ?? null,
       functionName: tx.function_name ?? null
     });
-    const filter = filterLookup.get(sourceName);
     const checkpoint = ctx.opsCheckpoint();
     try {
       let payload;
@@ -1957,22 +2064,48 @@ async function runHandlers(subgraph, matched, ctx, opts) {
         const verdict = validatePrintPayload(filter.prints, printPayload.topic ?? "", printPayload.data ?? {});
         if (!verdict.ok) {
           skipped++;
-          logger3.warn("Print payload does not match declared prints schema", {
+          logger4.warn("Print payload does not match declared prints schema", {
             subgraph: subgraph.name,
             sourceName,
             topic: printPayload.topic,
             txId: tx.tx_id,
             reason: verdict.reason
           });
+          recordPrintViolation({
+            subgraphName: subgraph.name,
+            sourceName,
+            blockHeight: ctx.block.height,
+            txId: tx.tx_id,
+            reason: verdict.reason,
+            samplePayload: {
+              topic: printPayload.topic ?? "",
+              data: printPayload.data ?? {}
+            }
+          });
           continue;
         }
       }
-      await handler(payload, ctx);
-      processed++;
+      if (materialize) {
+        const result = applyMaterializeInsert(materialize, payload, ctx, subgraph.schema);
+        if (!result.ok) {
+          skipped++;
+          logger4.warn("Materialize skipped: required field missing", {
+            subgraph: subgraph.name,
+            sourceName,
+            txId: tx.tx_id,
+            reason: result.reason
+          });
+          continue;
+        }
+        processed++;
+      } else if (handler) {
+        await handler(payload, ctx);
+        processed++;
+      }
     } catch (err) {
       ctx.rollbackTo(checkpoint);
       errors++;
-      logger3.error("Subgraph handler error", {
+      logger4.error("Subgraph handler error", {
         subgraph: subgraph.name,
         sourceName,
         txId: tx.tx_id,
@@ -1984,7 +2117,7 @@ async function runHandlers(subgraph, matched, ctx, opts) {
   return { processed, errors, ...skipped > 0 ? { skipped } : {} };
 }
 
-// src/testing/index.ts
+// src/testing/harness.ts
 import { normalizeAbi as normalizeAbi2, toCamelCase as toCamelCase3 } from "@secondlayer/stacks/clarity";
 var TEST_SCHEMA_NAME = "subgraph_test";
 function defaultBlock(overrides = {}) {
@@ -2052,6 +2185,20 @@ class InMemorySubgraphContext extends SubgraphContext {
     }
     this.ops.length = 0;
   }
+  insertsSince(checkpoint) {
+    const out = [];
+    for (const op of this.ops.slice(Math.max(0, checkpoint))) {
+      if (op.kind !== "insert")
+        continue;
+      const row = { ...op.data };
+      out.push({
+        table: op.table,
+        keys: Object.keys(row).filter((k) => !k.startsWith("_")),
+        row
+      });
+    }
+    return out;
+  }
 }
 function sameValue(a, b) {
   if (a === b)
@@ -2067,7 +2214,10 @@ function createTestContext(schema, options = {}) {
   const ctx = impl;
   ctx.rows = (table) => impl.rowsOf(table);
   ctx.commit = () => impl.commitOps();
-  ctx.setTx = (tx) => impl.setTx(defaultTx(tx));
+  ctx.setTx = (tx) => {
+    SubgraphContext.prototype.setTx.call(impl, defaultTx(tx));
+  };
+  ctx.pendingInsertsSince = (checkpoint) => impl.insertsSince(checkpoint);
   return ctx;
 }
 function buildEvent(source, payload, options = {}) {
@@ -2087,11 +2237,284 @@ function buildEvent(source, payload, options = {}) {
   }
   return { ...base, ...payload };
 }
+// src/testing/run.ts
+function toHandlerPayload(_filter, row) {
+  if (row.event_type === "print") {
+    const payload = row.payload;
+    return {
+      contractId: row.contract_id ?? "",
+      topic: payload?.topic ?? "",
+      data: camelizeKeys(payload?.value) ?? {}
+    };
+  }
+  const r = row;
+  return {
+    ...r.sender !== undefined ? { sender: r.sender } : {},
+    ...r.recipient !== undefined ? { recipient: r.recipient } : {},
+    ...r.amount !== undefined ? { amount: BigInt(String(r.amount)) } : {},
+    ...r.asset_identifier !== undefined ? { assetIdentifier: r.asset_identifier } : {},
+    ...r.value !== undefined ? { tokenId: r.value } : {}
+  };
+}
+function toContractCallPayload(row) {
+  return {
+    type: "contract_call",
+    sender: row.sender,
+    contractId: row.contract_id ?? "",
+    functionName: row.function_name ?? "",
+    args: Array.isArray(row.args) ? row.args : [],
+    result: row.result ?? null,
+    resultHex: row.result_hex ?? null,
+    tx: {
+      txId: row.tx_id,
+      sender: row.sender,
+      type: "contract_call",
+      status: row.status,
+      contractId: row.contract_id ?? null,
+      functionName: row.function_name ?? null
+    }
+  };
+}
+function jsonSafe3(value) {
+  if (typeof value === "bigint")
+    return value.toString();
+  if (value === null || value === undefined)
+    return value;
+  if (Array.isArray(value))
+    return value.map(jsonSafe3);
+  if (typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      out[k] = jsonSafe3(v);
+    }
+    return out;
+  }
+  return value;
+}
+function dataKeysOf(data) {
+  if (data && typeof data === "object" && !Array.isArray(data)) {
+    return Object.keys(data);
+  }
+  return [];
+}
+function rowMeta(row) {
+  const r = row;
+  return {
+    ...typeof r.block_height === "number" ? { blockHeight: r.block_height } : {},
+    ...typeof r.tx_id === "string" ? { txId: r.tx_id } : {}
+  };
+}
+function inKeysOf(filter, payload) {
+  if (filter?.type === "contract_call") {
+    return ["functionName", "args", "sender", "contractId"].filter((k) => payload[k] !== undefined);
+  }
+  return dataKeysOf(payload.data);
+}
+function unusedInKeysAcross(traces) {
+  if (traces.length === 0)
+    return;
+  const alwaysIn = new Set(traces[0]?.inKeys ?? []);
+  for (const t of traces.slice(1)) {
+    const keys = new Set(t.inKeys);
+    for (const k of [...alwaysIn]) {
+      if (!keys.has(k))
+        alwaysIn.delete(k);
+    }
+  }
+  const outKeys = new Set;
+  for (const t of traces) {
+    for (const o of t.outs) {
+      for (const k of o.keys)
+        outKeys.add(k);
+    }
+  }
+  const outNormalized = new Set([...outKeys].map((k) => k.replace(/_/g, "").toLowerCase()));
+  const unused = [...alwaysIn].filter((k) => !outNormalized.has(k.replace(/_/g, "").toLowerCase()));
+  return unused.length > 0 ? unused : undefined;
+}
+async function runSubgraphTest(input) {
+  const { schema, handlers, sources, events, trace } = input;
+  const sourceNames = Object.keys(sources);
+  if (sourceNames.length === 0) {
+    return {
+      ok: false,
+      code: "NO_SOURCES",
+      matched: 0,
+      written: 0,
+      tables: [],
+      hint: "No sources to test."
+    };
+  }
+  const ctx = createTestContext(schema);
+  let matched = 0;
+  let firstEvent;
+  const traces = [];
+  for (const [name, rows] of Object.entries(events)) {
+    const filter = sources[name];
+    const handler = handlers[name] ?? handlers["*"];
+    const materialize = filter && "materialize" in filter && filter.materialize !== undefined ? filter.materialize : undefined;
+    if (typeof handler !== "function" && !materialize)
+      continue;
+    for (const row of rows) {
+      const payload = filter?.type === "contract_call" ? toContractCallPayload(row) : toHandlerPayload(filter, row);
+      if (filter?.type === "print_event" && filter.topic && payload.topic !== filter.topic) {
+        continue;
+      }
+      if (!firstEvent) {
+        firstEvent = {
+          source: name,
+          data: filter?.type === "contract_call" ? {
+            functionName: payload.functionName,
+            args: payload.args
+          } : payload.data
+        };
+      }
+      matched++;
+      const checkpoint = ctx.opsCheckpoint();
+      try {
+        const event = buildEvent(filter, payload);
+        if (materialize) {
+          applyMaterializeInsert(materialize, event, {
+            tx: ctx.tx,
+            block: ctx.block,
+            insert: (table, insertRow) => {
+              ctx.insert(table, insertRow);
+            }
+          }, schema);
+        } else {
+          await handler(event, ctx);
+        }
+      } catch {}
+      if (trace) {
+        const inserts = ctx.pendingInsertsSince(checkpoint);
+        traces.push({
+          source: name,
+          ...rowMeta(row),
+          inKeys: inKeysOf(filter, payload),
+          outs: inserts.map((i) => ({ table: i.table, keys: i.keys }))
+        });
+      }
+    }
+  }
+  const tableNames = Object.keys(schema);
+  const tables = [];
+  let written = 0;
+  for (const table of tableNames) {
+    const tableRows = await ctx.rows(table);
+    written += tableRows.length;
+    const sample = tableRows[0];
+    tables.push({
+      name: table,
+      rows: tableRows.length,
+      ...sample ? { sampleRow: jsonSafe3(sample) } : {}
+    });
+  }
+  const fetched = Object.values(events).reduce((n, r) => n + r.length, 0);
+  const unusedInKeys = trace ? unusedInKeysAcross(traces) : undefined;
+  const traceFields = {
+    ...trace ? { traces } : {},
+    ...unusedInKeys ? { unusedInKeys } : {}
+  };
+  if (fetched === 0) {
+    return {
+      ok: false,
+      code: "NO_EVENTS",
+      matched: 0,
+      written,
+      tables,
+      hint: "No events matched these sources in the given range. Widen the range or check the source filters.",
+      ...traceFields
+    };
+  }
+  if (written === 0) {
+    const keys = dataKeysOf(firstEvent?.data);
+    const keyPart = keys.length > 0 ? ` Observed event.data keys on first event (${firstEvent?.source}): ${keys.join(", ")}.` : firstEvent ? ` First event source: ${firstEvent.source}.` : "";
+    return {
+      ok: false,
+      code: "EMPTY_MAPPING",
+      matched,
+      written: 0,
+      tables,
+      firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe3(firstEvent.data) } : undefined,
+      hint: `${matched} event${matched === 1 ? "" : "s"} matched, but NO rows were written — the shape of the field-mapping bug that ships a 0-row subgraph.${keyPart} Map only observed keys; do not invent fields.`,
+      ...traceFields
+    };
+  }
+  if (trace && traces.some((t) => t.outs.length === 0)) {
+    const empty = traces.find((t) => t.outs.length === 0);
+    const keyPart = empty && empty.inKeys.length > 0 ? ` Empty event (${empty.source}) IN keys: ${empty.inKeys.join(", ")}.` : "";
+    return {
+      ok: false,
+      code: "EMPTY_MAPPING",
+      matched,
+      written,
+      tables,
+      firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe3(firstEvent.data) } : undefined,
+      hint: `At least one matched event wrote 0 rows.${keyPart}`,
+      ...traceFields
+    };
+  }
+  return {
+    ok: true,
+    matched,
+    written,
+    tables,
+    firstEvent: firstEvent ? { ...firstEvent, data: jsonSafe3(firstEvent.data) } : undefined,
+    ...traceFields
+  };
+}
+async function probeHandlers(def, samples) {
+  const ctx = createTestContext(def.schema);
+  let matched = 0;
+  let firstEventKeys;
+  for (const sample of samples) {
+    const filter = def.sources[sample.source];
+    if (!filter)
+      continue;
+    const handler = def.handlers[sample.source] ?? def.handlers["*"];
+    const materialize = "materialize" in filter && filter.materialize !== undefined ? filter.materialize : undefined;
+    if (typeof handler !== "function" && !materialize)
+      continue;
+    if (firstEventKeys === undefined) {
+      firstEventKeys = dataKeysOf(sample.event.data);
+    }
+    matched++;
+    try {
+      const event = buildEvent(filter, sample.event);
+      if (materialize) {
+        applyMaterializeInsert(materialize, event, {
+          tx: ctx.tx,
+          block: ctx.block,
+          insert: (table, row) => {
+            ctx.insert(table, row);
+          }
+        }, def.schema);
+      } else {
+        await handler(event, ctx);
+      }
+    } catch {}
+  }
+  const tables = Object.keys(def.schema);
+  let written = 0;
+  for (const table of tables) {
+    written += (await ctx.rows(table)).length;
+  }
+  return {
+    matched,
+    written,
+    tables,
+    ...firstEventKeys !== undefined ? { firstEventKeys } : {}
+  };
+}
 export {
+  toHandlerPayload,
+  toContractCallPayload,
+  runSubgraphTest,
+  probeHandlers,
   createTestContext,
   buildEventPayload,
   buildEvent
 };
 
-//# debugId=A471864525222B1C64756E2164756E21
+//# debugId=ABB2AE2A4DA0D8AC64756E2164756E21
 //# sourceMappingURL=index.js.map