@secondlayer/subgraphs 1.0.0-alpha.0 → 1.0.0-beta.1

package/dist/src/runtime/catchup.js

@@ -168,7 +168,7 @@ class SubgraphContext {
   }
   async flush() {
     if (this.ops.length === 0)
-      return 0;
+      return { count: 0, writes: [] };
     const opsToFlush = [...this.ops];
     this.ops.length = 0;
     const statements = this.buildStatements(opsToFlush);
@@ -183,7 +183,21 @@ class SubgraphContext {
         }
       });
     }
-    return opsToFlush.length;
+    const writes = opsToFlush.map((op, rowIndex) => {
+      const blockHeight = op.data._block_height ?? this.block.height;
+      const txId = op.data._tx_id ?? this._tx.txId;
+      const baseRow = op.kind === "update" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
+      delete baseRow._upsert_keys;
+      delete baseRow._upsert_fallback_keys;
+      delete baseRow._upsert_fallback_set;
+      return {
+        op: op.kind,
+        table: op.table,
+        row: jsonSafe(baseRow),
+        pk: { blockHeight, txId, rowIndex }
+      };
+    });
+    return { count: opsToFlush.length, writes };
   }
   prepareInsert(op) {
     const upsertKeys = op.data._upsert_keys;
@@ -276,6 +290,13 @@ class SubgraphContext {
     }
   }
 }
+function jsonSafe(row) {
+  const out = {};
+  for (const [k, v] of Object.entries(row)) {
+    out[k] = typeof v === "bigint" ? v.toString() : v;
+  }
+  return out;
+}
 function escapeLiteral(value) {
   if (value === null || value === undefined)
     return "NULL";
@@ -857,12 +878,200 @@ import {
   recordSubgraphProcessed,
   updateSubgraphStatus
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger3 } from "@secondlayer/shared/logger";
-import { sql as sql2 } from "kysely";
+import { logger as logger4 } from "@secondlayer/shared/logger";
+import { sql as sql3 } from "kysely";
 
 // src/schema/utils.ts
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 
+// src/runtime/outbox-emit.ts
+import { createHash } from "node:crypto";
+import { logger as logger3 } from "@secondlayer/shared/logger";
+var loggedKillSwitch = false;
+function isEmitOutboxEnabled() {
+  return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
+}
+function dedupKey(subgraphName, tableName, blockHeight, txId, rowIndex, row) {
+  const canonical = `${subgraphName}:${tableName}:${blockHeight}:${txId}:${rowIndex}:${stableStringify(row)}`;
+  return createHash("sha256").update(canonical).digest("hex").slice(0, 32);
+}
+function stableStringify(obj) {
+  const keys = Object.keys(obj).sort();
+  return JSON.stringify(keys.reduce((acc, k) => {
+    acc[k] = obj[k];
+    return acc;
+  }, {}));
+}
+async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
+  if (!isEmitOutboxEnabled()) {
+    if (!loggedKillSwitch) {
+      logger3.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
+      loggedKillSwitch = true;
+    }
+    return 0;
+  }
+  loggedKillSwitch = false;
+  if (manifest.count === 0 || matcher.size() === 0)
+    return 0;
+  const rows = [];
+  for (const write of manifest.writes) {
+    if (write.op !== "insert")
+      continue;
+    const subs = matcher.match(subgraphName, write.table, write.row);
+    if (subs.length === 0)
+      continue;
+    const eventType = `${subgraphName}.${write.table}.created`;
+    for (const s of subs) {
+      rows.push({
+        subscription_id: s.id,
+        subgraph_name: subgraphName,
+        table_name: write.table,
+        block_height: blockHeight,
+        tx_id: write.pk.txId || null,
+        row_pk: write.pk,
+        event_type: eventType,
+        payload: write.row,
+        dedup_key: dedupKey(subgraphName, write.table, write.pk.blockHeight, write.pk.txId, write.pk.rowIndex, write.row)
+      });
+    }
+  }
+  if (rows.length === 0)
+    return 0;
+  await tx.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
+  return rows.length;
+}
+
+// src/runtime/subscription-state.ts
+import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
+import { sql as sql2 } from "kysely";
+
+// src/runtime/emitter-matcher.ts
+function isPrimitive(v) {
+  const t = typeof v;
+  return t === "string" || t === "number" || t === "boolean";
+}
+function coerceNumeric(v) {
+  if (typeof v === "number" && Number.isFinite(v))
+    return v;
+  if (typeof v === "bigint")
+    return Number(v);
+  if (typeof v === "string" && v !== "" && !Number.isNaN(Number(v))) {
+    return Number(v);
+  }
+  return null;
+}
+function matchClause(rowValue, clause) {
+  if (isPrimitive(clause)) {
+    if (typeof clause === "number" || typeof rowValue === "number") {
+      const a = coerceNumeric(rowValue);
+      const b = coerceNumeric(clause);
+      if (a !== null && b !== null)
+        return a === b;
+    }
+    return rowValue === clause || String(rowValue) === String(clause);
+  }
+  if (clause === null || typeof clause !== "object" || Array.isArray(clause)) {
+    return false;
+  }
+  const keys = Object.keys(clause);
+  if (keys.length !== 1)
+    return false;
+  const op = keys[0];
+  const c = clause;
+  switch (op) {
+    case "eq":
+      return matchClause(rowValue, c.eq);
+    case "neq":
+      return !matchClause(rowValue, c.neq);
+    case "gt":
+    case "gte":
+    case "lt":
+    case "lte": {
+      const a = coerceNumeric(rowValue);
+      const b = coerceNumeric(c[op]);
+      if (a === null || b === null)
+        return false;
+      if (op === "gt")
+        return a > b;
+      if (op === "gte")
+        return a >= b;
+      if (op === "lt")
+        return a < b;
+      return a <= b;
+    }
+    case "in": {
+      const list = c.in;
+      if (!Array.isArray(list))
+        return false;
+      return list.some((item) => isPrimitive(item) && matchClause(rowValue, item));
+    }
+    default:
+      return false;
+  }
+}
+function matchesFilter(filter, row) {
+  if (!filter || Object.keys(filter).length === 0)
+    return true;
+  for (const [col, clause] of Object.entries(filter)) {
+    if (!matchClause(row[col], clause))
+      return false;
+  }
+  return true;
+}
+function key(subgraphName, tableName) {
+  return `${subgraphName}\x00${tableName}`;
+}
+
+class SubscriptionMatcher {
+  byKey = new Map;
+  byId = new Map;
+  setAll(subs) {
+    this.byKey.clear();
+    this.byId.clear();
+    for (const sub of subs) {
+      if (sub.status !== "active")
+        continue;
+      this.byId.set(sub.id, sub);
+      const k = key(sub.subgraph_name, sub.table_name);
+      const arr = this.byKey.get(k);
+      if (arr)
+        arr.push(sub);
+      else
+        this.byKey.set(k, [sub]);
+    }
+  }
+  match(subgraphName, tableName, row) {
+    const bucket = this.byKey.get(key(subgraphName, tableName));
+    if (!bucket)
+      return [];
+    const hits = [];
+    for (const sub of bucket) {
+      if (matchesFilter(sub.filter, row))
+        hits.push(sub);
+    }
+    return hits;
+  }
+  has(subgraphName, tableName) {
+    return this.byKey.has(key(subgraphName, tableName));
+  }
+  size() {
+    return this.byId.size;
+  }
+  get(id) {
+    return this.byId.get(id);
+  }
+}
+
+// src/runtime/subscription-state.ts
+var matcher = new SubscriptionMatcher;
+async function refreshMatcher(db) {
+  const rows = await sql2`
+		SELECT * FROM subscriptions WHERE status = 'active'
+	`.execute(db);
+  matcher.setAll(rows.rows);
+  return matcher.size();
+}
+
 // src/runtime/block-processor.ts
 var schemaNameCache = new Map;
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
@@ -886,7 +1095,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   } else {
     block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
     if (!block) {
-      logger3.warn("Block not found for subgraph processing", {
+      logger4.warn("Block not found for subgraph processing", {
         subgraph: subgraphName,
         blockHeight
       });
@@ -894,7 +1103,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
       return result;
     }
     if (!block.canonical) {
-      logger3.debug("Skipping non-canonical block", {
+      logger4.debug("Skipping non-canonical block", {
         subgraph: subgraphName,
         blockHeight
       });
@@ -941,7 +1150,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     result.errors = runResult.errors;
     if (ctx.pendingOps > 0) {
       const flushStart = performance.now();
-      await ctx.flush();
+      const manifest = await ctx.flush();
+      if (manifest.count > 0) {
+        await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
+      }
       flushMs = performance.now() - flushStart;
     }
     if (!opts?.skipProgressUpdate) {
@@ -963,10 +1175,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     try {
       const tables = Object.keys(subgraph.schema);
       for (const table of tables) {
-        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
+        const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
-          logger3.warn("Subgraph table exceeds 10M rows (estimate)", {
+          logger4.warn("Subgraph table exceeds 10M rows (estimate)", {
             subgraph: subgraphName,
             table,
             count
@@ -1047,7 +1259,7 @@ import {
   recordGapBatch
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
 import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger4 } from "@secondlayer/shared/logger";
+import { logger as logger5 } from "@secondlayer/shared/logger";
 
 // src/runtime/batch-loader.ts
 async function loadBlockRange(db, fromHeight, toHeight) {
@@ -1146,7 +1358,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     const subgraphStart = Number(subgraphRow.start_block) || 1;
     const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
     const totalBlocks = chainTip - lastProcessedBlock;
-    logger4.info("Subgraph catch-up starting", {
+    logger5.info("Subgraph catch-up starting", {
       subgraph: subgraphName,
       from: startBlock,
       to: chainTip,
@@ -1161,7 +1373,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
     while (currentHeight <= chainTip) {
       const currentRow = await getSubgraph(targetDb, subgraphName);
       if (!currentRow || currentRow.status !== "active") {
-        logger4.info("Subgraph status changed, stopping catch-up", {
+        logger5.info("Subgraph status changed, stopping catch-up", {
           subgraph: subgraphName,
           status: currentRow?.status ?? "deleted"
         });
@@ -1188,7 +1400,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
             preloaded: blockData
           });
         } catch (err) {
-          logger4.error("Block processing error during catch-up", {
+          logger5.error("Block processing error during catch-up", {
             subgraph: subgraphName,
             blockHeight: height,
             error: err instanceof Error ? err.message : String(err)
@@ -1207,7 +1419,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
           }
         }
         if (processed % LOG_INTERVAL === 0) {
-          logger4.info("Subgraph catch-up progress", {
+          logger5.info("Subgraph catch-up progress", {
             subgraph: subgraphName,
             processed,
             total: totalBlocks,
@@ -1219,7 +1431,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       if (batchFailedBlocks.length > 0) {
         const gaps = coalesceGaps(batchFailedBlocks);
         await recordGapBatch(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
-          logger4.warn("Failed to record subgraph gaps", {
+          logger5.warn("Failed to record subgraph gaps", {
             subgraph: subgraphName,
             error: err instanceof Error ? err.message : String(err)
           });
@@ -1230,7 +1442,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
       currentHeight = batchEnd + 1;
     }
     await stats.flush(targetDb);
-    logger4.info("Subgraph catch-up complete", {
+    logger5.info("Subgraph catch-up complete", {
       subgraph: subgraphName,
       processed
     });
@@ -1243,5 +1455,5 @@ export {
   catchUpSubgraph
 };
 
-//# debugId=6C16DAE24C1ADF9B64756E2164756E21
+//# debugId=CC931C2EA467385764756E2164756E21
 //# sourceMappingURL=catchup.js.map