@secondlayer/subgraphs 1.0.0-alpha.0 → 1.0.0-beta.1

package/dist/src/runtime/reindex.js

@@ -168,7 +168,7 @@ class SubgraphContext {
   }
   async flush() {
     if (this.ops.length === 0)
-      return 0;
+      return { count: 0, writes: [] };
     const opsToFlush = [...this.ops];
     this.ops.length = 0;
     const statements = this.buildStatements(opsToFlush);
@@ -183,7 +183,21 @@ class SubgraphContext {
         }
       });
     }
-    return opsToFlush.length;
+    const writes = opsToFlush.map((op, rowIndex) => {
+      const blockHeight = op.data._block_height ?? this.block.height;
+      const txId = op.data._tx_id ?? this._tx.txId;
+      const baseRow = op.kind === "update" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
+      delete baseRow._upsert_keys;
+      delete baseRow._upsert_fallback_keys;
+      delete baseRow._upsert_fallback_set;
+      return {
+        op: op.kind,
+        table: op.table,
+        row: jsonSafe(baseRow),
+        pk: { blockHeight, txId, rowIndex }
+      };
+    });
+    return { count: opsToFlush.length, writes };
   }
   prepareInsert(op) {
     const upsertKeys = op.data._upsert_keys;
@@ -276,6 +290,13 @@ class SubgraphContext {
     }
   }
 }
+function jsonSafe(row) {
+  const out = {};
+  for (const [k, v] of Object.entries(row)) {
+    out[k] = typeof v === "bigint" ? v.toString() : v;
+  }
+  return out;
+}
 function escapeLiteral(value) {
   if (value === null || value === undefined)
     return "NULL";
@@ -857,12 +878,200 @@ import {
   recordSubgraphProcessed,
   updateSubgraphStatus
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger3 } from "@secondlayer/shared/logger";
-import { sql as sql2 } from "kysely";
+import { logger as logger4 } from "@secondlayer/shared/logger";
+import { sql as sql3 } from "kysely";
 
 // src/schema/utils.ts
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 
+// src/runtime/outbox-emit.ts
+import { createHash } from "node:crypto";
+import { logger as logger3 } from "@secondlayer/shared/logger";
+var loggedKillSwitch = false;
+function isEmitOutboxEnabled() {
+  return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
+}
+function dedupKey(subgraphName, tableName, blockHeight, txId, rowIndex, row) {
+  const canonical = `${subgraphName}:${tableName}:${blockHeight}:${txId}:${rowIndex}:${stableStringify(row)}`;
+  return createHash("sha256").update(canonical).digest("hex").slice(0, 32);
+}
+function stableStringify(obj) {
+  const keys = Object.keys(obj).sort();
+  return JSON.stringify(keys.reduce((acc, k) => {
+    acc[k] = obj[k];
+    return acc;
+  }, {}));
+}
+async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
+  if (!isEmitOutboxEnabled()) {
+    if (!loggedKillSwitch) {
+      logger3.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
+      loggedKillSwitch = true;
+    }
+    return 0;
+  }
+  loggedKillSwitch = false;
+  if (manifest.count === 0 || matcher.size() === 0)
+    return 0;
+  const rows = [];
+  for (const write of manifest.writes) {
+    if (write.op !== "insert")
+      continue;
+    const subs = matcher.match(subgraphName, write.table, write.row);
+    if (subs.length === 0)
+      continue;
+    const eventType = `${subgraphName}.${write.table}.created`;
+    for (const s of subs) {
+      rows.push({
+        subscription_id: s.id,
+        subgraph_name: subgraphName,
+        table_name: write.table,
+        block_height: blockHeight,
+        tx_id: write.pk.txId || null,
+        row_pk: write.pk,
+        event_type: eventType,
+        payload: write.row,
+        dedup_key: dedupKey(subgraphName, write.table, write.pk.blockHeight, write.pk.txId, write.pk.rowIndex, write.row)
+      });
+    }
+  }
+  if (rows.length === 0)
+    return 0;
+  await tx.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
+  return rows.length;
+}
+
+// src/runtime/subscription-state.ts
+import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
+import { sql as sql2 } from "kysely";
+
+// src/runtime/emitter-matcher.ts
+function isPrimitive(v) {
+  const t = typeof v;
+  return t === "string" || t === "number" || t === "boolean";
+}
+function coerceNumeric(v) {
+  if (typeof v === "number" && Number.isFinite(v))
+    return v;
+  if (typeof v === "bigint")
+    return Number(v);
+  if (typeof v === "string" && v !== "" && !Number.isNaN(Number(v))) {
+    return Number(v);
+  }
+  return null;
+}
+function matchClause(rowValue, clause) {
+  if (isPrimitive(clause)) {
+    if (typeof clause === "number" || typeof rowValue === "number") {
+      const a = coerceNumeric(rowValue);
+      const b = coerceNumeric(clause);
+      if (a !== null && b !== null)
+        return a === b;
+    }
+    return rowValue === clause || String(rowValue) === String(clause);
+  }
+  if (clause === null || typeof clause !== "object" || Array.isArray(clause)) {
+    return false;
+  }
+  const keys = Object.keys(clause);
+  if (keys.length !== 1)
+    return false;
+  const op = keys[0];
+  const c = clause;
+  switch (op) {
+    case "eq":
+      return matchClause(rowValue, c.eq);
+    case "neq":
+      return !matchClause(rowValue, c.neq);
+    case "gt":
+    case "gte":
+    case "lt":
+    case "lte": {
+      const a = coerceNumeric(rowValue);
+      const b = coerceNumeric(c[op]);
+      if (a === null || b === null)
+        return false;
+      if (op === "gt")
+        return a > b;
+      if (op === "gte")
+        return a >= b;
+      if (op === "lt")
+        return a < b;
+      return a <= b;
+    }
+    case "in": {
+      const list = c.in;
+      if (!Array.isArray(list))
+        return false;
+      return list.some((item) => isPrimitive(item) && matchClause(rowValue, item));
+    }
+    default:
+      return false;
+  }
+}
+function matchesFilter(filter, row) {
+  if (!filter || Object.keys(filter).length === 0)
+    return true;
+  for (const [col, clause] of Object.entries(filter)) {
+    if (!matchClause(row[col], clause))
+      return false;
+  }
+  return true;
+}
+function key(subgraphName, tableName) {
+  return `${subgraphName}\x00${tableName}`;
+}
+
+class SubscriptionMatcher {
+  byKey = new Map;
+  byId = new Map;
+  setAll(subs) {
+    this.byKey.clear();
+    this.byId.clear();
+    for (const sub of subs) {
+      if (sub.status !== "active")
+        continue;
+      this.byId.set(sub.id, sub);
+      const k = key(sub.subgraph_name, sub.table_name);
+      const arr = this.byKey.get(k);
+      if (arr)
+        arr.push(sub);
+      else
+        this.byKey.set(k, [sub]);
+    }
+  }
+  match(subgraphName, tableName, row) {
+    const bucket = this.byKey.get(key(subgraphName, tableName));
+    if (!bucket)
+      return [];
+    const hits = [];
+    for (const sub of bucket) {
+      if (matchesFilter(sub.filter, row))
+        hits.push(sub);
+    }
+    return hits;
+  }
+  has(subgraphName, tableName) {
+    return this.byKey.has(key(subgraphName, tableName));
+  }
+  size() {
+    return this.byId.size;
+  }
+  get(id) {
+    return this.byId.get(id);
+  }
+}
+
+// src/runtime/subscription-state.ts
+var matcher = new SubscriptionMatcher;
+async function refreshMatcher(db) {
+  const rows = await sql2`
+		SELECT * FROM subscriptions WHERE status = 'active'
+	`.execute(db);
+  matcher.setAll(rows.rows);
+  return matcher.size();
+}
+
 // src/runtime/block-processor.ts
 var schemaNameCache = new Map;
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
@@ -886,7 +1095,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   } else {
     block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
     if (!block) {
-      logger3.warn("Block not found for subgraph processing", {
+      logger4.warn("Block not found for subgraph processing", {
         subgraph: subgraphName,
         blockHeight
       });
@@ -894,7 +1103,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
       return result;
     }
     if (!block.canonical) {
-      logger3.debug("Skipping non-canonical block", {
+      logger4.debug("Skipping non-canonical block", {
         subgraph: subgraphName,
         blockHeight
       });
@@ -941,7 +1150,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     result.errors = runResult.errors;
     if (ctx.pendingOps > 0) {
       const flushStart = performance.now();
-      await ctx.flush();
+      const manifest = await ctx.flush();
+      if (manifest.count > 0) {
+        await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
+      }
       flushMs = performance.now() - flushStart;
     }
     if (!opts?.skipProgressUpdate) {
@@ -963,10 +1175,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     try {
       const tables = Object.keys(subgraph.schema);
       for (const table of tables) {
-        const { rows } = await sql2.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
+        const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
         const count = Number(rows[0]?.count ?? 0);
         if (count >= 1e7) {
-          logger3.warn("Subgraph table exceeds 10M rows (estimate)", {
+          logger4.warn("Subgraph table exceeds 10M rows (estimate)", {
             subgraph: subgraphName,
             table,
             count
@@ -1052,7 +1264,7 @@ import {
   recordSubgraphProcessed as recordSubgraphProcessed2,
   updateSubgraphStatus as updateSubgraphStatus2
 } from "@secondlayer/shared/db/queries/subgraphs";
-import { logger as logger4 } from "@secondlayer/shared/logger";
+import { logger as logger5 } from "@secondlayer/shared/logger";
 
 // src/schema/generator.ts
 var TYPE_MAP = {
@@ -1225,7 +1437,7 @@ async function processBlockRange(def, opts) {
   while (currentHeight <= toBlock) {
     if (opts.signal?.aborted) {
       aborted = true;
-      logger4.info("Block processing aborted", {
+      logger5.info("Block processing aborted", {
         subgraph: subgraphName,
         currentBlock: currentHeight,
         reason: String(opts.signal.reason ?? "unknown")
@@ -1255,7 +1467,7 @@ async function processBlockRange(def, opts) {
         });
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
-        logger4.error("Block processing error", {
+        logger5.error("Block processing error", {
           subgraph: subgraphName,
           blockHeight: height,
           error: errorMsg
@@ -1280,7 +1492,7 @@ async function processBlockRange(def, opts) {
         await updateSubgraphStatus2(targetDb, subgraphName, status, height);
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
-        logger4.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
+        logger5.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
           subgraph: subgraphName,
           processed: blocksProcessed,
           total: totalBlocks,
@@ -1292,7 +1504,7 @@ async function processBlockRange(def, opts) {
     if (batchFailedBlocks.length > 0 && opts.subgraphId) {
       const gaps = coalesceFailedBlocks(batchFailedBlocks);
       await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
-        logger4.warn("Failed to record subgraph gaps", {
+        logger5.warn("Failed to record subgraph gaps", {
           subgraph: subgraphName,
           error: err instanceof Error ? err.message : String(err)
         });
@@ -1329,11 +1541,11 @@ async function reindexSubgraph(def, opts) {
   const subgraphName = def.name;
   const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
   await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
-  logger4.info("Reindex starting", { subgraph: subgraphName });
+  logger5.info("Reindex starting", { subgraph: subgraphName });
   try {
     const { fromBlock, toBlock } = await resolveBlockRange(sourceDb, def, opts);
     if (fromBlock > toBlock) {
-      logger4.info("No blocks to reindex", {
+      logger5.info("No blocks to reindex", {
         subgraph: subgraphName,
         fromBlock,
         toBlock
@@ -1347,8 +1559,8 @@ async function reindexSubgraph(def, opts) {
     for (const stmt of statements) {
       await client.unsafe(stmt);
     }
-    logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
-    logger4.info("Reindexing blocks", {
+    logger5.info("Schema recreated for reindex", { subgraph: subgraphName });
+    logger5.info("Reindexing blocks", {
       subgraph: subgraphName,
       fromBlock,
       toBlock,
@@ -1369,9 +1581,9 @@ async function reindexSubgraph(def, opts) {
       if (reason === "user-cancelled") {
         await updateSubgraphStatus2(targetDb, subgraphName, "active");
         await clearReindexMetadata(targetDb, subgraphName);
-        logger4.info("Reindex cancelled by user", { subgraph: subgraphName });
+        logger5.info("Reindex cancelled by user", { subgraph: subgraphName });
       } else {
-        logger4.info("Reindex interrupted by shutdown, will resume", {
+        logger5.info("Reindex interrupted by shutdown, will resume", {
           subgraph: subgraphName
         });
       }
@@ -1383,7 +1595,7 @@ async function reindexSubgraph(def, opts) {
     }
     await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
     await clearReindexMetadata(targetDb, subgraphName);
-    logger4.info("Reindex complete", {
+    logger5.info("Reindex complete", {
       subgraph: subgraphName,
       blocks: result.blocksProcessed,
       events: result.totalEventsProcessed,
@@ -1391,7 +1603,7 @@ async function reindexSubgraph(def, opts) {
     });
     return { processed: result.blocksProcessed };
   } catch (err) {
-    logger4.error("Reindex failed", {
+    logger5.error("Reindex failed", {
       subgraph: subgraphName,
       error: getErrorMessage2(err)
     });
@@ -1411,7 +1623,7 @@ async function resumeReindex(def, opts) {
   if (!row)
     throw new Error(`Subgraph "${subgraphName}" not found`);
   if (row.reindex_from_block == null || row.reindex_to_block == null) {
-    logger4.info("No reindex metadata, starting fresh reindex", {
+    logger5.info("No reindex metadata, starting fresh reindex", {
       subgraph: subgraphName
     });
     return reindexSubgraph(def, {
@@ -1422,12 +1634,12 @@ async function resumeReindex(def, opts) {
   const fromBlock = Math.max(row.last_processed_block + 1, row.reindex_from_block);
   const toBlock = row.reindex_to_block;
   if (fromBlock > toBlock) {
-    logger4.info("Resume: no remaining blocks", { subgraph: subgraphName });
+    logger5.info("Resume: no remaining blocks", { subgraph: subgraphName });
     await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
     await clearReindexMetadata(targetDb, subgraphName);
     return { processed: 0 };
   }
-  logger4.info("Resuming reindex", {
+  logger5.info("Resuming reindex", {
     subgraph: subgraphName,
     fromBlock,
     toBlock,
@@ -1448,9 +1660,9 @@ async function resumeReindex(def, opts) {
       if (reason === "user-cancelled") {
         await updateSubgraphStatus2(targetDb, subgraphName, "active");
         await clearReindexMetadata(targetDb, subgraphName);
-        logger4.info("Resume cancelled by user", { subgraph: subgraphName });
+        logger5.info("Resume cancelled by user", { subgraph: subgraphName });
       } else {
-        logger4.info("Resume interrupted by shutdown, will resume again", {
+        logger5.info("Resume interrupted by shutdown, will resume again", {
           subgraph: subgraphName
         });
       }
@@ -1462,13 +1674,13 @@ async function resumeReindex(def, opts) {
     }
     await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
     await clearReindexMetadata(targetDb, subgraphName);
-    logger4.info("Resumed reindex complete", {
+    logger5.info("Resumed reindex complete", {
       subgraph: subgraphName,
       blocks: result.blocksProcessed
     });
     return { processed: result.blocksProcessed };
   } catch (err) {
-    logger4.error("Resumed reindex failed", {
+    logger5.error("Resumed reindex failed", {
       subgraph: subgraphName,
       error: getErrorMessage2(err)
     });
@@ -1479,7 +1691,7 @@ async function resumeReindex(def, opts) {
 async function backfillSubgraph(def, opts) {
   const targetDb = getTargetDb2();
   const subgraphName = def.name;
-  logger4.info("Backfill starting", {
+  logger5.info("Backfill starting", {
     subgraph: subgraphName,
     from: opts.fromBlock,
     to: opts.toBlock
@@ -1496,12 +1708,12 @@ async function backfillSubgraph(def, opts) {
       signal: opts.signal
     });
     if (result.aborted) {
-      logger4.info("Backfill aborted", { subgraph: subgraphName });
+      logger5.info("Backfill aborted", { subgraph: subgraphName });
       return { processed: result.blocksProcessed };
     }
     const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
     if (resolved > 0) {
-      logger4.info("Resolved subgraph gaps via backfill", {
+      logger5.info("Resolved subgraph gaps via backfill", {
         subgraph: subgraphName,
         resolved
       });
@@ -1510,7 +1722,7 @@ async function backfillSubgraph(def, opts) {
     if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
       await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
     }
-    logger4.info("Backfill complete", {
+    logger5.info("Backfill complete", {
       subgraph: subgraphName,
       blocks: result.blocksProcessed,
       events: result.totalEventsProcessed,
@@ -1518,7 +1730,7 @@ async function backfillSubgraph(def, opts) {
     });
     return { processed: result.blocksProcessed };
   } catch (err) {
-    logger4.error("Backfill failed", {
+    logger5.error("Backfill failed", {
       subgraph: subgraphName,
       error: getErrorMessage2(err)
     });
@@ -1531,5 +1743,5 @@ export {
   backfillSubgraph
 };
 
-//# debugId=CA98B860698EE36C64756E2164756E21
+//# debugId=A2112359A7EA0BBD64756E2164756E21
 //# sourceMappingURL=reindex.js.map