@prisma/streams-server 0.1.0 → 0.1.2

package/src/touch/manager.ts

@@ -1,6 +1,5 @@
 import type { Config } from "../config";
 import type { SqliteDurableStore } from "../db/db";
-import { STREAM_FLAG_TOUCH } from "../db/db";
 import type { IngestQueue } from "../ingest";
 import type { StreamNotifier } from "../notifier";
 import type { SchemaRegistryStore } from "../schema/registry";
@@ -8,11 +7,9 @@ import { isTouchEnabled } from "./spec";
 import { TouchInterpreterWorkerPool } from "./worker_pool";
 import { LruCache } from "../util/lru";
 import type { BackpressureGate } from "../backpressure";
-import { resolveTouchStreamName } from "./naming";
 import { LiveTemplateRegistry, type TemplateDecl } from "./live_templates";
 import { LiveMetricsV2 } from "./live_metrics";
 import type { TouchConfig } from "./spec";
-import type { RoutingKeyNotifier } from "./routing_key_notifier";
 import { TouchJournal } from "./touch_journal";
 import { Result } from "better-result";
 
@@ -74,18 +71,12 @@ const HOT_INTEREST_MAX_KEYS = 64;
 
 type TouchRecord = {
   keyId: number;
-  key?: string;
   watermark: string;
   entity: string;
   kind: "table" | "template";
   templateId?: string;
 };
 
-type EnsureTouchStreamError = {
-  kind: "touch_stream_content_type_mismatch";
-  message: string;
-};
-
 type RestrictedTemplateBucketState = {
   bucketId: number;
   templateKeyIds: Set<number>;
@@ -111,22 +102,17 @@ type StreamRuntimeTotals = {
 export class TouchInterpreterManager {
   private readonly cfg: Config;
   private readonly db: SqliteDurableStore;
-  private readonly ingest: IngestQueue;
-  private readonly notifier: StreamNotifier;
   private readonly registry: SchemaRegistryStore;
-  private readonly backpressure?: BackpressureGate;
   private readonly pool: TouchInterpreterWorkerPool;
   private timer: any | null = null;
   private running = false;
   private stopping = false;
   private readonly dirty = new Set<string>();
   private readonly failures = new FailureTracker(1024);
-  private readonly lastTrimMs = new LruCache<string, number>(1024);
   private readonly lastBaseWalGc = new LruCache<string, { atMs: number; through: bigint }>(1024);
   private readonly templates: LiveTemplateRegistry;
   private readonly liveMetrics: LiveMetricsV2;
   private readonly lastTemplateGcMsByStream = new LruCache<string, number>(1024);
-  private readonly routingKeyNotifier?: RoutingKeyNotifier;
   private readonly journals = new Map<string, TouchJournal>();
   private readonly fineLagCoarseOnlyByStream = new Map<string, boolean>();
   private readonly touchModeByStream = new Map<string, "idle" | "fine" | "restricted" | "coarseOnly">();
@@ -146,26 +132,20 @@ export class TouchInterpreterManager {
     ingest: IngestQueue,
     notifier: StreamNotifier,
     registry: SchemaRegistryStore,
-    backpressure?: BackpressureGate,
-    routingKeyNotifier?: RoutingKeyNotifier
+    backpressure?: BackpressureGate
   ) {
     this.cfg = cfg;
     this.db = db;
-    this.ingest = ingest;
-    this.notifier = notifier;
     this.registry = registry;
-    this.backpressure = backpressure;
     this.pool = new TouchInterpreterWorkerPool(cfg, cfg.interpreterWorkers);
     this.templates = new LiveTemplateRegistry(db);
     this.liveMetrics = new LiveMetricsV2(db, ingest, {
-      routingKeyNotifier,
-      getTouchJournal: (derivedStream) => {
-        const j = this.journals.get(derivedStream);
+      getTouchJournal: (stream) => {
+        const j = this.journals.get(stream);
         if (!j) return null;
         return { meta: j.getMeta(), interval: j.snapshotAndResetIntervalStats() };
       },
     });
-    this.routingKeyNotifier = routingKeyNotifier;
   }
 
   start(): void {
@@ -261,33 +241,6 @@ export class TouchInterpreterManager {
         }
       }
 
-      // Opportunistically enforce touch retention. This keeps internal touch
-      // WAL bounded even if the touch stream isn't being actively read.
-      for (const stream of stateByStream.keys()) {
-        if (this.stopping) break;
-        const srow = this.db.getStream(stream);
-        if (!srow || this.db.isDeleted(srow)) continue;
-        const regRes = this.registry.getRegistryResult(stream);
-        if (Result.isError(regRes)) {
-          // eslint-disable-next-line no-console
-          console.error("touch registry read failed", stream, regRes.error.message);
-          continue;
-        }
-        const reg = regRes.value;
-        if (!isTouchEnabled(reg.interpreter)) continue;
-        const derived = resolveTouchStreamName(stream, reg.interpreter.touch);
-        if ((reg.interpreter.touch.storage ?? "memory") === "sqlite") {
-          const ensureRes = this.ensureTouchStream(derived);
-          if (Result.isError(ensureRes)) {
-            // eslint-disable-next-line no-console
-            console.error("touch retention stream validation failed", stream, ensureRes.error.message);
-            continue;
-          }
-          const retentionMs = reg.interpreter.touch.retention?.maxAgeMs;
-          if (retentionMs != null) this.maybeTrimTouchStream(derived, retentionMs);
-        }
-      }
-
       // Opportunistically GC base WAL beyond the interpreter checkpoint.
       //
       // commitManifest() already GC's on upload, but it can't retroactively GC
@@ -369,7 +322,6 @@ export class TouchInterpreterManager {
       return;
     }
     const touchCfg = reg.interpreter.touch;
-    const touchStorage = touchCfg.storage ?? "memory";
     const failProcessing = (message: string): void => {
       this.failures.recordFailure(stream);
       this.liveMetrics.recordInterpreterError(stream, touchCfg);
@@ -378,24 +330,19 @@ export class TouchInterpreterManager {
     };
 
     const nowMs = Date.now();
-    const hotFine = touchStorage === "memory" ? this.getHotFineSnapshot(stream, touchCfg, nowMs) : null;
+    const hotFine = this.getHotFineSnapshot(stream, touchCfg, nowMs);
     const fineWaitersActive = hotFine?.fineWaitersActive ?? 0;
     const coarseWaitersActive = hotFine?.coarseWaitersActive ?? 0;
-    const hasAnyWaiters = touchStorage === "memory" ? fineWaitersActive + coarseWaitersActive > 0 : true;
+    const hasAnyWaiters = fineWaitersActive + coarseWaitersActive > 0;
     const hasFineDemand =
-      touchStorage === "memory"
-        ? fineWaitersActive > 0 || (hotFine?.broadFineWaitersActive ?? 0) > 0 || (hotFine?.hotKeyCount ?? 0) > 0 || (hotFine?.hotTemplateCount ?? 0) > 0
-        : true;
+      fineWaitersActive > 0 || (hotFine?.broadFineWaitersActive ?? 0) > 0 || (hotFine?.hotKeyCount ?? 0) > 0 || (hotFine?.hotTemplateCount ?? 0) > 0;
 
     // Guardrail: when lag/backlog grows too large, temporarily suppress
     // fine/template touches (coarse table touches are still emitted).
     const lagAtStart = toOffset >= interpretedThrough ? toOffset - interpretedThrough : 0n;
     const suppressFineDueToLag = this.computeSuppressFineDueToLag(stream, touchCfg, lagAtStart, hasFineDemand);
-    if (touchStorage === "memory") {
-      const derived = resolveTouchStreamName(stream, touchCfg);
-      const j = this.getOrCreateJournal(derived, touchCfg);
-      j.setCoalesceMs(this.computeAdaptiveCoalesceMs(touchCfg, lagAtStart, hasAnyWaiters));
-    }
+    const j = this.getOrCreateJournal(stream, touchCfg);
+    j.setCoalesceMs(this.computeAdaptiveCoalesceMs(touchCfg, lagAtStart, hasAnyWaiters));
 
     const fineBudgetPerBatch = Math.max(0, Math.floor(touchCfg.fineTouchBudgetPerBatch ?? 2000));
     const lagReservedFineBudgetPerBatch = Math.max(0, Math.floor(touchCfg.lagReservedFineTouchBudgetPerBatch ?? 200));
@@ -480,7 +427,7 @@ export class TouchInterpreterManager {
     let fineSkippedColdKey = 0;
     let fineSkippedTemplateBucket = 0;
 
-    if (touchStorage === "memory" && hotFine && hotFine.keyFilteringEnabled && fineGranularity !== "template") {
+    if (hotFine && hotFine.keyFilteringEnabled && fineGranularity !== "template") {
       const keyActiveSet = hotFine.hotKeyActiveSet;
       const keyGraceSet = hotFine.hotKeyGraceSet;
       const keyCount = (keyActiveSet?.size ?? 0) + (keyGraceSet?.size ?? 0);
@@ -503,76 +450,22 @@ export class TouchInterpreterManager {
       }
     }
 
-    if (touchStorage === "memory" && fineGranularity === "template" && touches.length > 0) {
+    if (fineGranularity === "template" && touches.length > 0) {
       const coalesced = this.coalesceRestrictedTemplateTouches(stream, touchCfg, touches);
       touches = coalesced.touches;
       fineSkippedTemplateBucket = coalesced.dropped;
     }
 
     if (touches.length > 0) {
-      const derived = res.derivedStream;
-      if (touchStorage === "sqlite") {
-        const ensureRes = this.ensureTouchStream(derived);
-        if (Result.isError(ensureRes)) {
-          failProcessing(ensureRes.error.message);
-          return;
-        }
-        const encoder = new TextEncoder();
-        for (const t of touches) {
-          if (!t.key) {
-            failProcessing("sqlite touch storage requires routing key strings");
-            return;
-          }
-        }
-        const rows = touches.map((t) => ({
-          routingKey: encoder.encode(t.key!),
-          contentType: "application/json",
-          payload: encoder.encode(
-            JSON.stringify({
-              sourceOffset: t.watermark,
-              entity: t.entity,
-              kind: t.kind,
-              ...(t.kind === "template" && t.templateId ? { templateId: t.templateId } : {}),
-            })
-          ),
-        }));
-        const appendRes = await this.ingest.appendInternal({
-          stream: derived,
-          baseAppendMs: this.db.nowMs(),
-          rows,
-          contentType: "application/json",
-        });
-        if (Result.isError(appendRes)) {
-          failProcessing(`touch append failed: ${appendRes.error.kind}`);
-          return;
-        }
-        if (appendRes.value.appendedRows > 0) {
-          this.notifier.notify(derived, appendRes.value.lastOffset);
-          if (this.routingKeyNotifier) {
-            const appended = appendRes.value.appendedRows;
-            const start = appendRes.value.lastOffset - BigInt(appended) + 1n;
-            let keys = touches.map((t) => t.key ?? "");
-            // Be defensive: only notify for the rows we actually appended.
-            if (keys.length !== appended) keys = keys.slice(Math.max(0, keys.length - appended));
-            let seq = start;
-            for (const k of keys) {
-              if (!k) continue;
-              this.routingKeyNotifier.notify(derived, k, seq);
-              seq += 1n;
-            }
-          }
-        }
-      } else {
-        const j = this.getOrCreateJournal(derived, touchCfg);
-        for (const t of touches) {
-          let sourceOffsetSeq: bigint | undefined;
-          try {
-            sourceOffsetSeq = BigInt(t.watermark);
-          } catch {
-            sourceOffsetSeq = undefined;
-          }
-          j.touch(t.keyId >>> 0, sourceOffsetSeq);
+      const j = this.getOrCreateJournal(stream, touchCfg);
+      for (const t of touches) {
+        let sourceOffsetSeq: bigint | undefined;
+        try {
+          sourceOffsetSeq = BigInt(t.watermark);
+        } catch {
+          sourceOffsetSeq = undefined;
         }
+        j.touch(t.keyId >>> 0, sourceOffsetSeq);
       }
     }
 
@@ -649,38 +542,6 @@ export class TouchInterpreterManager {
     this.failures.recordSuccess(stream);
   }
 
-  private ensureTouchStream(stream: string): Result<void, EnsureTouchStreamError> {
-    const existing = this.db.getStream(stream);
-    if (existing) {
-      if (String(existing.content_type) !== "application/json") {
-        return Result.err({
-          kind: "touch_stream_content_type_mismatch",
-          message: `touch stream content-type mismatch: ${existing.content_type}`,
-        });
-      }
-      if ((existing.stream_flags & STREAM_FLAG_TOUCH) === 0) this.db.addStreamFlags(stream, STREAM_FLAG_TOUCH);
-      return Result.ok(undefined);
-    }
-    this.db.ensureStream(stream, { contentType: "application/json", streamFlags: STREAM_FLAG_TOUCH });
-    return Result.ok(undefined);
-  }
-
-  private maybeTrimTouchStream(stream: string, maxAgeMs: number): void {
-    const now = Date.now();
-    const last = this.lastTrimMs.get(stream) ?? 0;
-    // Throttle trims; tick can run frequently.
-    if (now - last < 10_000) return;
-    this.lastTrimMs.set(stream, now);
-
-    try {
-      const res = this.db.trimWalByAge(stream, maxAgeMs);
-      if (res.trimmedBytes > 0 && this.backpressure) this.backpressure.adjustOnWalTrim(res.trimmedBytes);
-    } catch (e) {
-      // eslint-disable-next-line no-console
-      console.error("touch retention trim failed", stream, e);
-    }
-  }
-
   private maybeGcBaseWal(stream: string, uploadedThrough: bigint, interpretedThrough: bigint): void {
     const gcTargetThrough = interpretedThrough < uploadedThrough ? interpretedThrough : uploadedThrough;
     if (gcTargetThrough < 0n) return;
@@ -784,8 +645,6 @@ export class TouchInterpreterManager {
     templateIdsUsed: string[];
     interestMode: "fine" | "coarse";
   }): () => void {
-    if ((args.touchCfg.storage ?? "memory") !== "memory") return () => {};
-
     const nowMs = Date.now();
     const limits = this.getHotFineLimits(args.touchCfg);
     const state = this.getOrCreateHotFineState(args.stream);
@@ -878,10 +737,9 @@ export class TouchInterpreterManager {
     journalTimeoutSweepMsTotal: number;
   } {
     const nowMs = Date.now();
-    const hot = (args.touchCfg.storage ?? "memory") === "memory" ? this.getHotFineSnapshot(args.stream, args.touchCfg, nowMs) : null;
+    const hot = this.getHotFineSnapshot(args.stream, args.touchCfg, nowMs);
     const totals = this.getOrCreateRuntimeTotals(args.stream);
-    const derived = resolveTouchStreamName(args.stream, args.touchCfg);
-    const journal = (args.touchCfg.storage ?? "memory") === "memory" ? this.journals.get(derived) ?? null : null;
+    const journal = this.journals.get(args.stream) ?? null;
     const journalTotals = journal?.getTotalStats();
     return {
       lagSourceOffsets: this.lagSourceOffsetsByStream.get(args.stream) ?? 0,
@@ -953,8 +811,8 @@ export class TouchInterpreterManager {
     return Array.from(entities);
   }
 
-  getOrCreateJournal(derivedStream: string, touchCfg: TouchConfig): TouchJournal {
-    const existing = this.journals.get(derivedStream);
+  getOrCreateJournal(stream: string, touchCfg: TouchConfig): TouchJournal {
+    const existing = this.journals.get(stream);
     if (existing) return existing;
     const mem = touchCfg.memory ?? {};
     const j = new TouchJournal({
@@ -964,7 +822,7 @@ export class TouchInterpreterManager {
       pendingMaxKeys: mem.pendingMaxKeys ?? 100_000,
       keyIndexMaxKeys: mem.keyIndexMaxKeys ?? 32,
     });
-    this.journals.set(derivedStream, j);
+    this.journals.set(stream, j);
     return j;
   }
 
@@ -978,8 +836,8 @@ export class TouchInterpreterManager {
     return maxCoalesceMs;
   }
 
-  getJournalIfExists(derivedStream: string): TouchJournal | null {
-    return this.journals.get(derivedStream) ?? null;
+  getJournalIfExists(stream: string): TouchJournal | null {
+    return this.journals.get(stream) ?? null;
   }
 
   private computeSuppressFineDueToLag(stream: string, touchCfg: TouchConfig, lagAtStart: bigint, hasFineDemand: boolean): boolean {
@@ -1014,7 +872,7 @@ export class TouchInterpreterManager {
         hasActiveWaiters = false;
       } else {
         const reg = regRes.value;
-        if (isTouchEnabled(reg.interpreter) && (reg.interpreter.touch.storage ?? "memory") === "memory") {
+        if (isTouchEnabled(reg.interpreter)) {
           const snap = this.getHotFineSnapshot(stream, reg.interpreter.touch, nowMs);
           hasActiveWaiters = snap.fineWaitersActive + snap.coarseWaitersActive > 0;
         }

package/src/touch/spec.ts

@@ -1,5 +1,4 @@
 import { Result } from "better-result";
-import { parseDurationMsResult } from "../util/duration.ts";
 import { dsError } from "../util/ds_error.ts";
 
 export type StreamInterpreterConfig = {
@@ -15,22 +14,6 @@ export type StreamInterpreterConfigValidationError = {
 
 export type TouchConfig = {
   enabled: boolean;
-  /**
-   * Touch storage backend.
-   *
-   * - memory: in-process, lossy (false positives allowed), epoch-scoped journal.
-   * - sqlite: derived companion stream in SQLite WAL (legacy).
-   *
-   * Default: memory.
-   */
-  storage?: "memory" | "sqlite";
-  /**
-   * Advanced override. When unset, the server exposes touches as a companion of
-   * the source stream at `/v1/stream/<name>/touch` and uses an internal derived
-   * stream name.
-   */
-  derivedStream?: string;
-  retention?: { maxAgeMs: number };
   /**
    * Coarse invalidation interval. The server emits at most one table-touch per
    * entity per interval.
@@ -209,72 +192,22 @@ function parseIntegerField(
   return Result.ok(n);
 }
 
-function validateRetentionResult(
-  raw: any
-): Result<{ maxAgeMs: number } | undefined, StreamInterpreterConfigValidationError> {
-  if (raw == null) return Result.ok(undefined);
-  if (!raw || typeof raw !== "object") return invalidInterpreter("interpreter.touch.retention must be an object");
-  const maxAge = raw.maxAge;
-  const maxAgeMsRaw = raw.maxAgeMs;
-  if (maxAge !== undefined && maxAgeMsRaw !== undefined) {
-    return invalidInterpreter("interpreter.touch.retention must specify only one of maxAge|maxAgeMs");
-  }
-  let ms: number | null = null;
-  if (maxAgeMsRaw !== undefined) {
-    if (typeof maxAgeMsRaw !== "number" || !Number.isFinite(maxAgeMsRaw) || maxAgeMsRaw < 0) {
-      return invalidInterpreter("interpreter.touch.retention.maxAgeMs must be a non-negative number");
-    }
-    ms = maxAgeMsRaw;
-  } else if (maxAge !== undefined) {
-    if (typeof maxAge !== "string" || maxAge.trim() === "") {
-      return invalidInterpreter("interpreter.touch.retention.maxAge must be a non-empty duration string");
-    }
-    const durationRes = parseDurationMsResult(maxAge);
-    if (Result.isError(durationRes)) {
-      return invalidInterpreter(`interpreter.touch.retention.maxAge ${durationRes.error.message}`);
-    }
-    ms = durationRes.value;
-    if (ms < 0) ms = 0;
-  } else {
-    return invalidInterpreter("interpreter.touch.retention must include maxAge or maxAgeMs");
-  }
-  return Result.ok({ maxAgeMs: ms });
-}
-
 function validateTouchConfigResult(raw: any): Result<TouchConfig, StreamInterpreterConfigValidationError> {
   if (!raw || typeof raw !== "object") return invalidInterpreter("interpreter.touch must be an object");
   const enabled = !!raw.enabled;
   if (!enabled) {
-    return Result.ok({
-      enabled: false,
-      storage: undefined,
-      derivedStream: undefined,
-      retention: undefined,
-    });
+    return Result.ok({ enabled: false });
   }
 
-  const storageRaw = raw.storage === undefined ? "memory" : String(raw.storage).trim();
-  if (storageRaw !== "memory" && storageRaw !== "sqlite") {
-    return invalidInterpreter("interpreter.touch.storage must be memory|sqlite");
+  if (raw.storage !== undefined) {
+    return invalidInterpreter("interpreter.touch.storage is no longer supported; touch always uses the in-memory journal");
   }
-  const storage = storageRaw as "memory" | "sqlite";
-
-  const derivedStream =
-    raw.derivedStream === undefined
-      ? undefined
-      : typeof raw.derivedStream === "string" && raw.derivedStream.trim() !== ""
-        ? raw.derivedStream
-        : null;
-  if (derivedStream === null) {
-    return invalidInterpreter("interpreter.touch.derivedStream must be a non-empty string when provided");
+  if (raw.derivedStream !== undefined) {
+    return invalidInterpreter("interpreter.touch.derivedStream is no longer supported");
+  }
+  if (raw.retention !== undefined) {
+    return invalidInterpreter("interpreter.touch.retention is no longer supported");
   }
-
-  // Touch companions are intended as short-retention invalidation journals.
-  // Default to 24h to prevent unbounded growth if retention is omitted.
-  const retentionRaw = raw.retention;
-  const retentionRes = retentionRaw === undefined ? Result.ok({ maxAgeMs: 24 * 60 * 60 * 1000 }) : validateRetentionResult(retentionRaw);
-  if (Result.isError(retentionRes)) return invalidInterpreter(retentionRes.error.message);
-  const retention = retentionRes.value;
 
   const coarseIntervalMsRes = parseNumberField(
     raw.coarseIntervalMs,
@@ -454,9 +387,6 @@ function validateTouchConfigResult(raw: any): Result<TouchConfig, StreamInterpre
 
   return Result.ok({
     enabled: true,
-    storage,
-    derivedStream,
-    retention,
     coarseIntervalMs: coarseIntervalMsRes.value,
     touchCoalesceWindowMs: touchCoalesceWindowMsRes.value,
     onMissingBefore,

package/src/touch/worker_pool.ts

@@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url";
 import { Worker } from "node:worker_threads";
 import { Result } from "better-result";
 import type { Config } from "../config";
+import { detectHostRuntime } from "../runtime/host_runtime.ts";
 import type { ProcessRequest, ProcessResult, WorkerMessage } from "./worker_protocol";
 import { dsError } from "../util/ds_error.ts";
 
@@ -123,7 +124,7 @@ export class TouchInterpreterWorkerPool {
     }
 
     const worker = new Worker(workerSpec, {
-      workerData: { config: this.cfg },
+      workerData: { config: this.cfg, hostRuntime: detectHostRuntime() },
       type: "module",
       smol: true,
     } as any);

package/src/touch/worker_protocol.ts

@@ -2,7 +2,6 @@ import type { StreamInterpreterConfig } from "./spec.ts";
 
 export type TouchRow = {
   keyId: number;
-  key?: string;
   watermark: string; // source stream offset (base-10 string)
   entity: string;
   kind: "table" | "template";
@@ -30,7 +29,6 @@ export type ProcessResult = {
   type: "result";
   id: number;
   stream: string;
-  derivedStream: string;
   processedThrough: bigint;
   touches: TouchRow[];
   stats: {

package/src/touch/naming.ts

@@ -1,13 +0,0 @@
-import type { TouchConfig } from "./spec.ts";
-
-const DEFAULT_TOUCH_SUFFIX = ".__touch";
-
-export function defaultTouchStreamName(sourceStream: string): string {
-  return `${sourceStream}${DEFAULT_TOUCH_SUFFIX}`;
-}
-
-export function resolveTouchStreamName(sourceStream: string, touch: TouchConfig): string {
-  const override = touch.derivedStream;
-  if (override && override.trim() !== "") return override;
-  return defaultTouchStreamName(sourceStream);
-}