@prisma/streams-server 0.1.0 → 0.1.1

package/src/db/db.ts

@@ -964,60 +964,6 @@ export class SqliteDurableStore {
     }
   }
 
-  findFirstWalOffsetForRoutingKeys(
-    stream: string,
-    startOffsetExclusive: bigint,
-    endOffsetInclusive: bigint,
-    routingKeys: Uint8Array[]
-  ): bigint | null {
-    if (routingKeys.length === 0) return null;
-    const start = this.bindInt(startOffsetExclusive);
-
-    // Avoid tripping SQLite variable limits (commonly 999) when callers watch a
-    // large number of keys. Chunking is cheap because the touch WAL ranges we
-    // scan are typically small and indexed by (stream, routing_key, offset).
-    const MAX_KEYS_PER_QUERY = 400;
-    let best: bigint | null = null;
-    let endLimit = endOffsetInclusive;
-    const touchNameLooksDefault = stream.endsWith(".__touch");
-    const touchLikeClause = touchNameLooksDefault ? " AND stream LIKE '%.__touch'" : "";
-
-    for (let i = 0; i < routingKeys.length; i += MAX_KEYS_PER_QUERY) {
-      const chunk = routingKeys.slice(i, i + MAX_KEYS_PER_QUERY);
-      if (chunk.length === 0) continue;
-      const end = this.bindInt(endLimit);
-      const placeholders = chunk.map(() => "?").join(", ");
-      const stmt = this.db.query(
-        `SELECT MIN(offset) as offset
-         FROM wal
-         WHERE stream = ?${touchLikeClause}
-           AND offset > ?
-           AND offset <= ?
-           AND routing_key IN (${placeholders});`
-      );
-      try {
-        const row = stmt.get(stream, start, end, ...chunk) as any;
-        if (!row || row.offset == null) continue;
-        const off = this.toBigInt(row.offset);
-        if (best == null || off < best) {
-          best = off;
-          // No need to search beyond the best match so far.
-          endLimit = off - 1n;
-          if (off <= startOffsetExclusive + 1n) break;
-        }
-      } finally {
-        try {
-          stmt.finalize?.();
-        } catch {
-          // ignore
-        }
-      }
-      if (endLimit < startOffsetExclusive + 1n) break;
-    }
-
-    return best;
-  }
-
   nextSegmentIndexForStream(stream: string): number {
     const row = this.stmts.nextSegmentIndex.get(stream) as any;
     return Number(row?.next_idx ?? 0);

package/src/db/schema.ts

@@ -9,7 +9,7 @@ import { dsError } from "../util/ds_error.ts";
  *  - local metadata store (streams/segments/manifests/schemas)
  */
 
-export const SCHEMA_VERSION = 10;
+export const SCHEMA_VERSION = 11;
 
 export const DEFAULT_PRAGMAS_SQL = `
 PRAGMA journal_mode = WAL;
@@ -75,8 +75,6 @@ CREATE TABLE IF NOT EXISTS wal (
 CREATE UNIQUE INDEX IF NOT EXISTS wal_stream_offset_uniq ON wal(stream, offset);
 CREATE INDEX IF NOT EXISTS wal_stream_offset_idx ON wal(stream, offset);
 CREATE INDEX IF NOT EXISTS wal_ts_idx ON wal(ts_ms);
--- Partial index for internal companion touch streams (WAL-only, routing-key heavy).
-CREATE INDEX IF NOT EXISTS wal_touch_stream_rk_offset_idx ON wal(stream, routing_key, offset) WHERE stream LIKE '%.__touch';
 
 CREATE TABLE IF NOT EXISTS segments (
   segment_id TEXT PRIMARY KEY,
@@ -134,7 +132,7 @@ CREATE TABLE IF NOT EXISTS stream_interpreters (
   updated_at_ms INTEGER NOT NULL
 );
 
--- Live Query V2 dynamic template registry (per base stream).
+-- Live dynamic template registry (per base stream).
 CREATE TABLE IF NOT EXISTS live_templates (
   stream TEXT NOT NULL,
   template_id TEXT NOT NULL,
@@ -267,7 +265,6 @@ const CREATE_INDEXES_V4_SQL = `
 CREATE UNIQUE INDEX IF NOT EXISTS wal_stream_offset_uniq ON wal(stream, offset);
 CREATE INDEX IF NOT EXISTS wal_stream_offset_idx ON wal(stream, offset);
 CREATE INDEX IF NOT EXISTS wal_ts_idx ON wal(ts_ms);
-CREATE INDEX IF NOT EXISTS wal_touch_stream_rk_offset_idx ON wal(stream, routing_key, offset) WHERE stream LIKE '%.__touch';
 
 CREATE INDEX IF NOT EXISTS streams_pending_bytes_idx ON streams(pending_bytes);
 CREATE INDEX IF NOT EXISTS streams_last_cut_idx ON streams(last_segment_cut_ms);
@@ -326,6 +323,8 @@ export function initSchema(db: SqliteDatabase, opts: { skipMigrations?: boolean
       migrateV8ToV9(db);
     } else if (version === 9) {
       migrateV9ToV10(db);
+    } else if (version === 10) {
+      migrateV10ToV11(db);
     } else {
       throw dsError(`unexpected schema version: ${version} (expected ${SCHEMA_VERSION})`);
     }
@@ -555,7 +554,6 @@ function migrateV6ToV7(db: SqliteDatabase): void {
 
 function migrateV7ToV8(db: SqliteDatabase): void {
   const tx = db.transaction(() => {
-    db.exec(`CREATE INDEX IF NOT EXISTS wal_touch_stream_rk_offset_idx ON wal(stream, routing_key, offset) WHERE stream LIKE '%.__touch';`);
     db.exec(`UPDATE schema_version SET version = 8;`);
   });
   tx();
@@ -613,6 +611,14 @@ function migrateV9ToV10(db: SqliteDatabase): void {
     `);
     db.exec(`DROP TABLE wal_stats;`);
 
+    db.exec(`UPDATE schema_version SET version = 10;`);
+  });
+  tx();
+}
+
+function migrateV10ToV11(db: SqliteDatabase): void {
+  const tx = db.transaction(() => {
+    db.exec(`DROP INDEX IF EXISTS wal_touch_stream_rk_offset_idx;`);
     db.exec(`UPDATE schema_version SET version = ${SCHEMA_VERSION};`);
   });
   tx();

package/src/touch/interpreter_worker.ts

@@ -5,9 +5,8 @@ import { SqliteDurableStore } from "../db/db.ts";
 import { initConsoleLogging } from "../util/log.ts";
 import type { ProcessRequest } from "./worker_protocol.ts";
 import { interpretRecordToChanges } from "./engine.ts";
-import { encodeTemplateArg, tableKeyFor, tableKeyIdFor, templateKeyFor, templateKeyIdFor, watchKeyFor, watchKeyIdFor, type TemplateEncoding } from "./live_keys.ts";
+import { encodeTemplateArg, tableKeyIdFor, templateKeyIdFor, watchKeyIdFor, type TemplateEncoding } from "./live_keys.ts";
 import { isTouchEnabled } from "./spec.ts";
-import { resolveTouchStreamName } from "./naming.ts";
 
 initConsoleLogging();
 
@@ -50,9 +49,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
     return;
   }
   const touch = interpreter.touch;
-  const derivedStream = resolveTouchStreamName(stream, touch);
-  const touchStorage = touch.storage ?? "memory";
-  const emitRoutingKey = touchStorage === "sqlite";
 
   const fineBudgetRaw = msg.fineTouchBudget ?? touch.fineTouchBudgetPerBatch;
   const fineBudget = fineBudgetRaw == null ? null : Math.max(0, Math.floor(fineBudgetRaw));
@@ -125,7 +121,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
 
   type PendingTouch = {
     keyId: number;
-    key?: string;
     windowStartMs: number;
     watermark: string;
     entity: string;
@@ -136,17 +131,16 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
 
   const pending = new Map<string, PendingTouch>();
   const templateOnlyEntityTouch = new Map<string, EntityTemplateOnlyTouch>();
-  const touches: Array<{ keyId: number; key?: string; watermark: string; entity: string; kind: "table" | "template"; templateId?: string }> = [];
+  const touches: Array<{ keyId: number; watermark: string; entity: string; kind: "table" | "template"; templateId?: string }> = [];
   let fineTouchesDroppedDueToBudget = 0;
   let fineTouchesSkippedColdTemplate = 0;
 
   const flush = (_mapKey: string, p: PendingTouch) => {
-    touches.push({ keyId: p.keyId >>> 0, key: p.key, watermark: p.watermark, entity: p.entity, kind: p.kind, templateId: p.templateId });
+    touches.push({ keyId: p.keyId >>> 0, watermark: p.watermark, entity: p.entity, kind: p.kind, templateId: p.templateId });
   };
 
   const queueTouch = (args: {
     keyId: number;
-    key?: string;
     tsMs: number;
     watermark: string;
     entity: string;
@@ -154,7 +148,7 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
     templateId?: string;
     windowMs: number;
   }) => {
-    const mapKey = args.key ? `k:${args.key}` : `i:${args.keyId >>> 0}`;
+    const mapKey = `i:${args.keyId >>> 0}`;
     const prev = pending.get(mapKey);
 
     // Guardrail: cap fine/template touches (key cardinality) per batch.
@@ -177,7 +171,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
     if (!prev) {
       pending.set(mapKey, {
         keyId: args.keyId >>> 0,
-        key: args.key,
         windowStartMs: args.tsMs,
         watermark: args.watermark,
         entity: args.entity,
@@ -194,7 +187,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
     flush(mapKey, prev);
     pending.set(mapKey, {
       keyId: args.keyId >>> 0,
-      key: args.key,
       windowStartMs: args.tsMs,
       watermark: args.watermark,
       entity: args.entity,
@@ -237,7 +229,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
       const coarseKeyId = tableKeyIdFor(entity);
       queueTouch({
         keyId: coarseKeyId,
-        key: emitRoutingKey ? tableKeyFor(entity) : undefined,
         tsMs,
         watermark,
         entity,
@@ -267,7 +258,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
         if (fineGranularity === "template") {
           queueTouch({
             keyId: templateKeyIdFor(tpl.templateId) >>> 0,
-            key: emitRoutingKey ? templateKeyFor(tpl.templateId) : undefined,
             tsMs,
             watermark,
             entity,
@@ -282,9 +272,9 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
         const afterObj = ch.after;
         const beforeObj = ch.before;
 
-        const watchKeys = new Map<number, string | undefined>();
+        const watchKeyIds = new Set<number>();
 
-        const compute = (obj: unknown): { keyId: number; key?: string } | null => {
+        const compute = (obj: unknown): number | null => {
           if (!obj || typeof obj !== "object" || Array.isArray(obj)) return null;
           const args: string[] = [];
           for (let i = 0; i < tpl.fields.length; i++) {
@@ -295,19 +285,15 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
             if (encoded == null) return null;
             args.push(encoded);
           }
-          if (emitRoutingKey) {
-            const key = watchKeyFor(tpl.templateId, args);
-            return { keyId: Number.parseInt(key.slice(8), 16) >>> 0, key };
-          }
-          return { keyId: watchKeyIdFor(tpl.templateId, args) >>> 0 };
+          return watchKeyIdFor(tpl.templateId, args) >>> 0;
         };
 
         if (ch.op === "insert") {
           const k = compute(afterObj);
-          if (k) watchKeys.set(k.keyId >>> 0, k.key);
+          if (k != null) watchKeyIds.add(k >>> 0);
         } else if (ch.op === "delete") {
           const k = compute(beforeObj);
-          if (k) watchKeys.set(k.keyId >>> 0, k.key);
+          if (k != null) watchKeyIds.add(k >>> 0);
         } else {
           // update: compute touches from both before and after (when possible).
           // Policy for missing/insufficient before image:
@@ -317,9 +303,9 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
           const kAfter = compute(afterObj);
           const kBefore = compute(beforeObj);
 
-          if (kBefore) {
-            watchKeys.set(kBefore.keyId >>> 0, kBefore.key);
-            if (kAfter) watchKeys.set(kAfter.keyId >>> 0, kAfter.key);
+          if (kBefore != null) {
+            watchKeyIds.add(kBefore >>> 0);
+            if (kAfter != null) watchKeyIds.add(kAfter >>> 0);
           } else {
             if (beforeObj === undefined) {
               if (onMissingBefore === "error") {
@@ -335,17 +321,16 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
             }
 
             if (onMissingBefore === "skipBefore") {
-              if (kAfter) watchKeys.set(kAfter.keyId >>> 0, kAfter.key);
+              if (kAfter != null) watchKeyIds.add(kAfter >>> 0);
             } else {
               // coarse: no fine touches
             }
           }
         }
 
-        for (const [watchKeyId, watchKey] of watchKeys.entries()) {
+        for (const watchKeyId of watchKeyIds) {
           queueTouch({
             keyId: watchKeyId >>> 0,
-            key: watchKey,
             tsMs,
             watermark,
             entity,
@@ -369,7 +354,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
         if (agg.offset < tpl.activeFromSourceOffset) continue;
         queueTouch({
           keyId: templateKeyIdFor(tpl.templateId) >>> 0,
-          key: emitRoutingKey ? templateKeyFor(tpl.templateId) : undefined,
           tsMs: agg.tsMs,
           watermark: agg.watermark,
           entity,
@@ -386,8 +370,8 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
   }
 
   touches.sort((a, b) => {
-    const ak = a.key ?? `~${(a.keyId >>> 0).toString(16).padStart(8, "0")}`;
-    const bk = b.key ?? `~${(b.keyId >>> 0).toString(16).padStart(8, "0")}`;
+    const ak = a.keyId >>> 0;
+    const bk = b.keyId >>> 0;
     if (ak < bk) return -1;
     if (ak > bk) return 1;
     const aw = BigInt(a.watermark);
@@ -408,7 +392,6 @@ async function handleProcess(msg: ProcessRequest): Promise<void> {
     type: "result",
     id: msg.id,
     stream,
-    derivedStream,
     processedThrough,
     touches,
     stats: {

package/src/touch/live_metrics.ts

@@ -4,8 +4,6 @@ import { STREAM_FLAG_TOUCH } from "../db/db";
 import { encodeOffset } from "../offset";
 import type { TouchConfig } from "./spec";
 import type { TemplateLifecycleEvent } from "./live_templates";
-import { resolveTouchStreamName } from "./naming";
-import type { RoutingKeyNotifier } from "./routing_key_notifier";
 import type { TouchJournalIntervalStats, TouchJournalMeta } from "./touch_journal";
 import { Result } from "better-result";
 
@@ -207,8 +205,7 @@ export class LiveMetricsV2 {
   private readonly snapshotIntervalMs: number;
   private readonly snapshotChunkSize: number;
   private readonly retentionMs: number;
-  private readonly routingKeyNotifier?: RoutingKeyNotifier;
-  private readonly getTouchJournal?: (derivedStream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
+  private readonly getTouchJournal?: (stream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
   private timer: any | null = null;
   private snapshotTimer: any | null = null;
   private retentionTimer: any | null = null;
@@ -234,8 +231,7 @@ export class LiveMetricsV2 {
       snapshotIntervalMs?: number;
       snapshotChunkSize?: number;
       retentionMs?: number;
-      routingKeyNotifier?: RoutingKeyNotifier;
-      getTouchJournal?: (derivedStream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
+      getTouchJournal?: (stream: string) => { meta: TouchJournalMeta; interval: TouchJournalIntervalStats } | null;
     }
   ) {
     this.db = db;
@@ -246,7 +242,6 @@ export class LiveMetricsV2 {
     this.snapshotIntervalMs = opts?.snapshotIntervalMs ?? 60_000;
     this.snapshotChunkSize = opts?.snapshotChunkSize ?? 200;
     this.retentionMs = opts?.retentionMs ?? 7 * 24 * 60 * 60 * 1000;
-    this.routingKeyNotifier = opts?.routingKeyNotifier;
     this.getTouchJournal = opts?.getTouchJournal;
   }
 
@@ -500,8 +495,6 @@ export class LiveMetricsV2 {
     this.lagSumMs = 0;
     this.lagSamples = 0;
 
-    const rkInterval = this.routingKeyNotifier?.snapshotAndResetIntervalStats() ?? null;
-
     for (const st of states) {
       const stream = st.stream;
       const regRow = this.db.getStream(stream);
@@ -522,26 +515,8 @@ export class LiveMetricsV2 {
       if (!touchCfg) continue;
 
       const c = this.get(stream, touchCfg);
-      const storage = (touchCfg.storage ?? "memory") as "memory" | "sqlite";
-      const derived = resolveTouchStreamName(stream, touchCfg);
-      const journal = storage === "memory" ? this.getTouchJournal?.(derived) ?? null : null;
-      const trow = (() => {
-        try {
-          return this.db.getStream(derived);
-        } catch {
-          return null;
-        }
-      })();
-      const touchTailSeq = trow ? (trow.next_offset > 0n ? trow.next_offset - 1n : -1n) : -1n;
-      let touchWalOldestOffset: string | null = null;
-      try {
-        const oldest = this.db.getWalOldestOffset(derived);
-        touchWalOldestOffset = oldest == null || !trow ? null : encodeOffset(trow.epoch, oldest);
-      } catch {
-        touchWalOldestOffset = null;
-      }
-      const waitActive =
-        storage === "memory" ? (journal?.meta.activeWaiters ?? 0) : this.routingKeyNotifier ? this.routingKeyNotifier.getActiveWaiters(derived) : 0;
+      const journal = this.getTouchJournal?.(stream) ?? null;
+      const waitActive = journal?.meta.activeWaiters ?? 0;
       const tailSeq = regRow.next_offset > 0n ? regRow.next_offset - 1n : -1n;
       const interpretedThrough = st.interpreted_through;
       const gcThrough = interpretedThrough < regRow.uploaded_through ? interpretedThrough : regRow.uploaded_through;
@@ -571,7 +546,6 @@ export class LiveMetricsV2 {
         instanceId: this.instanceId,
         region: this.region,
         touch: {
-          storage,
           coarseIntervalMs: c.touch.coarseIntervalMs,
           coalesceWindowMs: c.touch.coalesceWindowMs,
           mode: c.touch.mode,
@@ -596,16 +570,11 @@ export class LiveMetricsV2 {
           fineTouchesSuppressedBatchesDueToLag: c.touch.fineTouchesSuppressedBatchesDueToLag,
           fineTouchesSuppressedSecondsDueToLag: c.touch.fineTouchesSuppressedMsDueToLag / 1000,
           fineTouchesSuppressedBatchesDueToBudget: c.touch.fineTouchesSuppressedBatchesDueToBudget,
-          cursor: storage === "memory" ? (journal?.meta.cursor ?? null) : null,
-          epoch: storage === "memory" ? (journal?.meta.epoch ?? null) : null,
-          generation: storage === "memory" ? (journal?.meta.generation ?? null) : null,
-          pendingKeys: storage === "memory" ? (journal?.meta.pendingKeys ?? 0) : 0,
-          overflowBuckets: storage === "memory" ? (journal?.meta.overflowBuckets ?? 0) : 0,
-          walTailOffset: trow ? encodeOffset(trow.epoch, touchTailSeq) : null,
-          walNextOffset: trow ? encodeOffset(trow.epoch, trow.next_offset) : null,
-          walOldestOffset: touchWalOldestOffset,
-          walRetainedRows: trow ? clampBigInt(trow.wal_rows) : 0,
-          walRetainedBytes: trow ? clampBigInt(trow.wal_bytes) : 0,
+          cursor: journal?.meta.cursor ?? null,
+          epoch: journal?.meta.epoch ?? null,
+          generation: journal?.meta.generation ?? null,
+          pendingKeys: journal?.meta.pendingKeys ?? 0,
+          overflowBuckets: journal?.meta.overflowBuckets ?? 0,
         },
         templates: {
           active: activeTemplates,
@@ -624,15 +593,15 @@ export class LiveMetricsV2 {
           avgLatencyMs: c.wait.calls > 0 ? c.wait.latencySumMs / c.wait.calls : 0,
           p95LatencyMs: c.wait.latencyHist.p95(),
           activeWaiters: waitActive,
-          timeoutsFired: storage === "memory" ? (journal?.interval.timeoutsFired ?? 0) : rkInterval?.timeoutsFired ?? 0,
-          timeoutSweeps: storage === "memory" ? (journal?.interval.timeoutSweeps ?? 0) : rkInterval?.timeoutSweeps ?? 0,
-          timeoutSweepMsSum: storage === "memory" ? (journal?.interval.timeoutSweepMsSum ?? 0) : rkInterval?.timeoutSweepMsSum ?? 0,
-          timeoutSweepMsMax: storage === "memory" ? (journal?.interval.timeoutSweepMsMax ?? 0) : rkInterval?.timeoutSweepMsMax ?? 0,
-          notifyWakeups: storage === "memory" ? (journal?.interval.notifyWakeups ?? 0) : 0,
-          notifyFlushes: storage === "memory" ? (journal?.interval.notifyFlushes ?? 0) : 0,
-          notifyWakeMsSum: storage === "memory" ? (journal?.interval.notifyWakeMsSum ?? 0) : 0,
-          notifyWakeMsMax: storage === "memory" ? (journal?.interval.notifyWakeMsMax ?? 0) : 0,
-          timeoutHeapSize: storage === "memory" ? (journal?.interval.heapSize ?? 0) : rkInterval?.heapSize ?? 0,
+          timeoutsFired: journal?.interval.timeoutsFired ?? 0,
+          timeoutSweeps: journal?.interval.timeoutSweeps ?? 0,
+          timeoutSweepMsSum: journal?.interval.timeoutSweepMsSum ?? 0,
+          timeoutSweepMsMax: journal?.interval.timeoutSweepMsMax ?? 0,
+          notifyWakeups: journal?.interval.notifyWakeups ?? 0,
+          notifyFlushes: journal?.interval.notifyFlushes ?? 0,
+          notifyWakeMsSum: journal?.interval.notifyWakeMsSum ?? 0,
+          notifyWakeMsMax: journal?.interval.notifyWakeMsMax ?? 0,
+          timeoutHeapSize: journal?.interval.heapSize ?? 0,
         },
         interpreter: {
           eventsIn: c.interpreter.eventsIn,