@secondlayer/subgraphs 1.0.0 → 1.1.0

package/README.md

@@ -0,0 +1,94 @@
+# @secondlayer/subgraphs
+
+Typed on-chain indexing for Stacks. Declare event filters + column schema with `defineSubgraph()`; the runtime decodes blocks, matches filters, runs your handlers inside a transactional context, and exposes the result as a Postgres schema you query over REST or SQL.
+
+Subgraph rows fan out to HTTP subscribers through a post-flush outbox emitter — signed Standard Webhooks POSTs with retries, circuit breaker, and replay.
+
+## Install
+
+```bash
+bun add @secondlayer/subgraphs
+```
+
+## Quick Start
+
+```typescript
+import { defineSubgraph } from "@secondlayer/subgraphs";
+
+export default defineSubgraph({
+  name: "token-transfers",
+  version: "1.0.0",
+  sources: {
+    // Named event sources — the key becomes the handler name.
+    transfer: {
+      type: "ft_transfer",
+      assetIdentifier: "SP2X0TZ59D5SZ8ACQ6YMCHHNR2ZN51Z32E2CJ173.stx-token::stx",
+    },
+  },
+  schema: {
+    transfers: {
+      columns: {
+        sender: { type: "principal" },
+        recipient: { type: "principal" },
+        amount: { type: "uint" },
+      },
+      // Auto-added: _block_height, _tx_id, _created_at
+    },
+  },
+  handlers: {
+    async transfer(event, ctx) {
+      ctx.insert("transfers", {
+        sender: event.sender,
+        recipient: event.recipient,
+        amount: event.amount,
+      });
+    },
+  },
+});
+```
+
+Deploy via CLI (`sl subgraphs deploy path/to/definition.ts`), SDK (`sl.subgraphs.deploy({...})`), or MCP (`subgraphs_deploy`). The dashboard is read-only — creation always happens through an API surface.
+
+## Exports
+
+| Subpath | Description |
+| --- | --- |
+| `.` | `defineSubgraph`, `validateSubgraphDefinition`, `deploySchema`, `diffSchema`, `reindexSubgraph`, `backfillSubgraph`, `generateSubgraphSQL`, `pgSchemaName` |
+| `./types` | All schema + filter + handler types (`SubgraphDefinition`, `SubgraphFilter`, `StxTransferFilter`, etc.) |
+| `./schema` | Generator + deployer internals |
+| `./validate` | Shape + filter validation for deploys |
+| `./runtime/source-matcher` | Pure fn: match txs+events against a `SubgraphFilter` — used by the processor hot path |
+| `./runtime/replay` | `replaySubscription({ accountId, subscriptionId, fromBlock, toBlock })` — re-enqueue historical rows as outbox entries |
+
+## Runtime components
+
+The runtime ships behind these entrypoints (import from the package root):
+
+- `startSubgraphProcessor(opts?)` — boots the block processor. LISTENs on `indexer:new_block`, matches sources, runs handlers, flushes writes inside a transaction, and emits outbox rows for matching subscriptions. Also boots the emitter worker.
+- `processBlock(subgraph, name, height, opts?)` — single-block entry point used by catch-up, reindex, and tests.
+- `catchUpSubgraph(def, name)` — drains pending blocks up to chain tip.
+- `reindexSubgraph(def, opts)` — drop + rebuild schema tables from a start block. Breaking schema changes trigger this automatically on deploy.
+
+## Subscription emitter
+
+Every row written through `ctx.insert()` / `ctx.upsert()` is atomically enqueued to `subscription_outbox` for every active subscription whose filter matches — inside the same transaction as the flush, so a processor crash rolls back both.
+
+The emitter drains the outbox via `LISTEN subscriptions:new_outbox` and `FOR UPDATE SKIP LOCKED` batch claims. Live deliveries win a 90/10 split over replays. Each row dispatches through the format builder matching the subscription's `format` column (`standard-webhooks`, `inngest`, `trigger`, `cloudflare`, `cloudevents`, `raw`). Retries follow `30s → 2m → 10m → 1h → 6h → 24h → 72h`. Twenty consecutive failures trips the per-sub circuit breaker and pauses the subscription.
+
+Delivery bodies and response previews land in `subscription_deliveries`. Rows whose retries exhaust mark `status = 'dead'` in the outbox and surface in the dashboard's dead-letter queue for one-click requeue.
+
+## Environment
+
+| Variable | Default | Description |
+| --- | --- | --- |
+| `SECONDLAYER_EMIT_OUTBOX` | `true` | Set `false` to bypass outbox emission on every block (kill-switch). |
+| `SECONDLAYER_ALLOW_PRIVATE_EGRESS` | `false` | Allow the emitter to deliver to private IP ranges (localhost, 10/8, 172.16/12, 192.168/16, link-local, v6 mapped). Leave off in production. |
+| `SECONDLAYER_SECRETS_KEY` | — | 32-byte hex key for the AES-GCM envelope around subscription signing secrets. OSS mode auto-generates + persists to `.env.local`. |
+
+## Postgres + pool mode
+
+The emitter holds a persistent `LISTEN` on `subscriptions:new_outbox` and `subscriptions:changed`, so it MUST connect through a session-mode pool. pgbouncer in transaction mode silently breaks it. Run the emitter against a session-mode port (`pool_mode = session`), or connect directly to Postgres as the default docker-compose setup does.
+
+## License
+
+MIT

package/dist/src/index.js

@@ -996,7 +996,7 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
         subgraph_name: subgraphName,
         table_name: write.table,
         block_height: blockHeight,
-        tx_id: write.pk.txId || null,
+        tx_id: write.pk.txId ?? null,
         row_pk: write.pk,
         event_type: eventType,
         payload: write.row,
@@ -1019,7 +1019,26 @@ function isPrimitive(v) {
   const t = typeof v;
   return t === "string" || t === "number" || t === "boolean";
 }
-function coerceNumeric(v) {
+function coerceBigInt(v) {
+  if (typeof v === "bigint")
+    return v;
+  if (typeof v === "number") {
+    if (!Number.isFinite(v))
+      return null;
+    if (!Number.isInteger(v))
+      return null;
+    return BigInt(v);
+  }
+  if (typeof v === "string" && /^-?\d+$/.test(v)) {
+    try {
+      return BigInt(v);
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+function coerceFloat(v) {
   if (typeof v === "number" && Number.isFinite(v))
     return v;
   if (typeof v === "bigint")
@@ -1029,13 +1048,31 @@ function coerceNumeric(v) {
   }
   return null;
 }
+function compareNumeric(a, b) {
+  const ba = coerceBigInt(a);
+  const bb = coerceBigInt(b);
+  if (ba !== null && bb !== null) {
+    if (ba === bb)
+      return 0;
+    return ba > bb ? 1 : -1;
+  }
+  const fa = coerceFloat(a);
+  const fb = coerceFloat(b);
+  if (fa === null || fb === null)
+    return null;
+  if (fa === fb)
+    return 0;
+  return fa > fb ? 1 : -1;
+}
 function matchClause(rowValue, clause) {
+  const rowIsPrimitive = isPrimitive(rowValue) || typeof rowValue === "bigint";
   if (isPrimitive(clause)) {
-    if (typeof clause === "number" || typeof rowValue === "number") {
-      const a = coerceNumeric(rowValue);
-      const b = coerceNumeric(clause);
-      if (a !== null && b !== null)
-        return a === b;
+    if (!rowIsPrimitive)
+      return false;
+    if (typeof clause === "number" || typeof rowValue === "number" || typeof rowValue === "bigint") {
+      const cmp = compareNumeric(rowValue, clause);
+      if (cmp !== null)
+        return cmp === 0;
     }
     return rowValue === clause || String(rowValue) === String(clause);
   }
@@ -1056,22 +1093,25 @@ function matchClause(rowValue, clause) {
     case "gte":
     case "lt":
     case "lte": {
-      const a = coerceNumeric(rowValue);
-      const b = coerceNumeric(c[op]);
-      if (a === null || b === null)
+      if (!rowIsPrimitive)
+        return false;
+      const cmp = compareNumeric(rowValue, c[op]);
+      if (cmp === null)
         return false;
       if (op === "gt")
-        return a > b;
+        return cmp > 0;
       if (op === "gte")
-        return a >= b;
+        return cmp >= 0;
       if (op === "lt")
-        return a < b;
-      return a <= b;
+        return cmp < 0;
+      return cmp <= 0;
     }
     case "in": {
       const list = c.in;
       if (!Array.isArray(list))
         return false;
+      if (!rowIsPrimitive)
+        return false;
       return list.some((item) => isPrimitive(item) && matchClause(rowValue, item));
     }
     default:
@@ -1254,7 +1294,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
           });
         }
       }
-    } catch {}
+    } catch (err) {
+      logger4.debug("Row count sample failed", {
+        subgraph: subgraphName,
+        error: err instanceof Error ? err.message : String(err)
+      });
+    }
   }
   return result;
 }
@@ -2044,5 +2089,5 @@ export {
   backfillSubgraph
 };
 
-//# debugId=C453EF737BBEFDDD64756E2164756E21
+//# debugId=E6D35F38651A38BA64756E2164756E21
 //# sourceMappingURL=index.js.map