@prisma/streams-server 0.0.1 → 0.1.0

package/src/touch/worker_pool.ts

@@ -0,0 +1,189 @@
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Worker } from "node:worker_threads";
+import { Result } from "better-result";
+import type { Config } from "../config";
+import type { ProcessRequest, ProcessResult, WorkerMessage } from "./worker_protocol";
+import { dsError } from "../util/ds_error.ts";
+
+type Pending = {
+  resolve: (r: Result<ProcessResult, WorkerPoolProcessError>) => void;
+};
+
+export type WorkerPoolProcessError = {
+  kind: "worker_pool_unavailable" | "worker_pool_failure";
+  message: string;
+};
+
+export class TouchInterpreterWorkerPool {
+  private readonly cfg: Config;
+  private readonly workerCount: number;
+  private readonly workers: Array<{ worker: Worker; busy: boolean; currentId: number | null }> = [];
+  private started = false;
+  private generation = 0;
+  private nextId = 1;
+  private readonly pending = new Map<number, Pending>();
+  private readonly queue: Array<Omit<ProcessRequest, "type" | "id"> & { id: number }> = [];
+
+  constructor(cfg: Config, workerCount: number) {
+    this.cfg = cfg;
+    this.workerCount = Math.max(0, Math.floor(workerCount));
+  }
+
+  start(): void {
+    if (this.started) return;
+    this.started = true;
+    this.generation += 1;
+    const generation = this.generation;
+    for (let i = 0; i < this.workerCount; i++) this.spawnWorker(i, generation);
+  }
+
+  stop(): void {
+    if (!this.started) return;
+    this.started = false;
+    this.generation += 1;
+    for (const w of this.workers) {
+      try {
+        w.worker.postMessage({ type: "stop" });
+      } catch {
+        // ignore
+      }
+      void w.worker.terminate();
+    }
+    this.workers.length = 0;
+    this.queue.length = 0;
+    for (const [id, p] of this.pending.entries()) {
+      p.resolve(Result.err({ kind: "worker_pool_failure", message: "worker pool stopped" }));
+      this.pending.delete(id);
+    }
+  }
+
+  restart(): void {
+    this.stop();
+    this.start();
+  }
+
+  async processResult(req: Omit<ProcessRequest, "type" | "id">): Promise<Result<ProcessResult, WorkerPoolProcessError>> {
+    if (!this.started) {
+      return Result.err({ kind: "worker_pool_unavailable", message: "worker pool not started" });
+    }
+    if (this.workerCount === 0) {
+      return Result.err({ kind: "worker_pool_unavailable", message: "worker pool disabled" });
+    }
+    const id = this.nextId++;
+    const queued = { ...req, id };
+    const value = await new Promise<Result<ProcessResult, WorkerPoolProcessError>>((resolve) => {
+      this.pending.set(id, { resolve });
+      this.queue.push(queued);
+      this.pump();
+    });
+    return value;
+  }
+
+  async process(req: Omit<ProcessRequest, "type" | "id">): Promise<ProcessResult> {
+    const res = await this.processResult(req);
+    if (Result.isError(res)) throw dsError(res.error.message);
+    return res.value;
+  }
+
+  private pump(): void {
+    if (!this.started) return;
+    if (this.queue.length === 0) return;
+    const slot = this.workers.find((w) => !w.busy);
+    if (!slot) return;
+    const next = this.queue.shift();
+    if (!next) return;
+    slot.busy = true;
+    slot.currentId = next.id;
+    slot.worker.postMessage({
+      type: "process",
+      id: next.id,
+      stream: next.stream,
+      fromOffset: next.fromOffset,
+      toOffset: next.toOffset,
+      interpreter: next.interpreter,
+      maxRows: next.maxRows,
+      maxBytes: next.maxBytes,
+      emitFineTouches: next.emitFineTouches,
+      fineTouchBudget: next.fineTouchBudget,
+      fineGranularity: next.fineGranularity,
+      interpretMode: next.interpretMode,
+      filterHotTemplates: next.filterHotTemplates,
+      hotTemplateIds: next.hotTemplateIds,
+    } satisfies ProcessRequest);
+  }
+
+  private spawnWorker(idx: number, generation: number = this.generation): void {
+    const workerUrl = new URL("./interpreter_worker.ts", import.meta.url);
+    let workerSpec = fileURLToPath(workerUrl);
+    if (!existsSync(workerSpec)) {
+      const fallback = resolve(process.cwd(), "src/touch/interpreter_worker.ts");
+      if (existsSync(fallback)) workerSpec = fallback;
+    }
+
+    const worker = new Worker(workerSpec, {
+      workerData: { config: this.cfg },
+      type: "module",
+      smol: true,
+    } as any);
+
+    const slot = { worker, busy: false, currentId: null };
+    this.workers.push(slot);
+
+    worker.on("message", (msg: WorkerMessage) => {
+      if (generation !== this.generation) return;
+      if (!msg || typeof msg !== "object") return;
+      if (msg.type === "result") {
+        const p = this.pending.get(msg.id);
+        if (p) {
+          this.pending.delete(msg.id);
+          slot.busy = false;
+          slot.currentId = null;
+          p.resolve(Result.ok(msg));
+        }
+        this.pump();
+        return;
+      }
+      if (msg.type === "error") {
+        const p = this.pending.get(msg.id);
+        if (p) {
+          this.pending.delete(msg.id);
+          slot.busy = false;
+          slot.currentId = null;
+          p.resolve(Result.err({ kind: "worker_pool_failure", message: msg.message }));
+        }
+        this.pump();
+      }
+    });
+
+    worker.on("error", (err) => {
+      if (generation !== this.generation) return;
+      // eslint-disable-next-line no-console
+      console.error(`touch interpreter worker ${idx} error`, err);
+    });
+
+    worker.on("exit", (code) => {
+      if (generation !== this.generation || !this.started) return;
+      // eslint-disable-next-line no-console
+      console.error(`touch interpreter worker ${idx} exited with code ${code}, respawning`);
+      if (slot.currentId != null) {
+        const p = this.pending.get(slot.currentId);
+        if (p) {
+          this.pending.delete(slot.currentId);
+          p.resolve(Result.err({ kind: "worker_pool_failure", message: "worker exited" }));
+        }
+      }
+      slot.busy = false;
+      slot.currentId = null;
+      try {
+        const widx = this.workers.indexOf(slot);
+        if (widx >= 0) this.workers.splice(widx, 1);
+      } catch {
+        // ignore
+      }
+      this.spawnWorker(idx, generation);
+      this.pump();
+    });
+  }
+}

package/src/touch/worker_protocol.ts

@@ -0,0 +1,58 @@
+import type { StreamInterpreterConfig } from "./spec.ts";
+
+export type TouchRow = {
+  keyId: number;
+  key?: string;
+  watermark: string; // source stream offset (base-10 string)
+  entity: string;
+  kind: "table" | "template";
+  templateId?: string;
+};
+
+export type ProcessRequest = {
+  type: "process";
+  id: number;
+  stream: string;
+  fromOffset: bigint;
+  toOffset: bigint;
+  interpreter: StreamInterpreterConfig;
+  maxRows: number;
+  maxBytes: number;
+  emitFineTouches?: boolean;
+  fineTouchBudget?: number | null;
+  fineGranularity?: "key" | "template";
+  interpretMode?: "full" | "hotTemplatesOnly";
+  filterHotTemplates?: boolean;
+  hotTemplateIds?: string[] | null;
+};
+
+export type ProcessResult = {
+  type: "result";
+  id: number;
+  stream: string;
+  derivedStream: string;
+  processedThrough: bigint;
+  touches: TouchRow[];
+  stats: {
+    rowsRead: number;
+    bytesRead: number;
+    changes: number;
+    touchesEmitted: number;
+    tableTouchesEmitted: number;
+    templateTouchesEmitted: number;
+    maxSourceTsMs?: number;
+    fineTouchesDroppedDueToBudget?: number;
+    fineTouchesSuppressedDueToBudget?: boolean;
+    fineTouchesSkippedColdTemplate?: number;
+  };
+};
+
+export type ProcessError = {
+  type: "error";
+  id: number;
+  stream: string;
+  message: string;
+  stack?: string;
+};
+
+export type WorkerMessage = ProcessResult | ProcessError;

package/src/types/proper-lockfile.d.ts

@@ -0,0 +1 @@
+declare module "proper-lockfile";

package/src/uploader.ts

@@ -0,0 +1,317 @@
+import { unlinkSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { Result } from "better-result";
+import type { Config } from "./config";
+import type { SqliteDurableStore, SegmentRow } from "./db/db";
+import type { ObjectStore } from "./objectstore/interface";
+import { buildManifestResult } from "./manifest";
+import { manifestObjectKey, segmentObjectKey, streamHash16Hex } from "./util/stream_paths";
+import { readU64LE } from "./util/endian";
+import { SegmentDiskCache } from "./segment/cache";
+import { retry } from "./util/retry";
+import { LruCache } from "./util/lru";
+import type { StatsCollector } from "./stats";
+import type { BackpressureGate } from "./backpressure";
+import { dsError } from "./util/ds_error.ts";
+
+export type UploaderController = {
+  start(): void;
+  stop(hard?: boolean): void;
+  countSegmentsWaiting(): number;
+  setHooks(hooks: { onSegmentsUploaded?: (stream: string) => void } | undefined): void;
+  publishManifest(stream: string): Promise<void>;
+};
+
+export class Uploader {
+  private readonly config: Config;
+  private readonly db: SqliteDurableStore;
+  private readonly os: ObjectStore;
+  private readonly diskCache?: SegmentDiskCache;
+  private readonly stats?: StatsCollector;
+  private readonly gate?: BackpressureGate;
+  private timer: any | null = null;
+  private running = false;
+  private stopping = false;
+  private readonly inflight = new Set<string>();
+  private readonly failures = new FailureTracker(1024);
+  private hooks?: { onSegmentsUploaded?: (stream: string) => void };
+  private readonly manifestInflight = new Set<string>();
+
+  constructor(
+    config: Config,
+    db: SqliteDurableStore,
+    os: ObjectStore,
+    diskCache?: SegmentDiskCache,
+    stats?: StatsCollector,
+    gate?: BackpressureGate,
+    hooks?: { onSegmentsUploaded?: (stream: string) => void }
+  ) {
+    this.config = config;
+    this.db = db;
+    this.os = os;
+    this.diskCache = diskCache;
+    this.stats = stats;
+    this.gate = gate;
+    this.hooks = hooks;
+  }
+
+  setHooks(hooks: { onSegmentsUploaded?: (stream: string) => void } | undefined): void {
+    this.hooks = hooks;
+  }
+
+  start(): void {
+    this.stopping = false;
+    if (this.timer) return;
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, this.config.uploadIntervalMs);
+  }
+
+  stop(hard = false): void {
+    if (hard) this.stopping = true;
+    else this.stopping = false;
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+  }
+
+  countSegmentsWaiting(): number {
+    return this.db.countPendingSegments();
+  }
+
+  private async tick(): Promise<void> {
+    if (this.stopping) return;
+    if (this.running) return;
+    this.running = true;
+    try {
+      const pending = this.db.pendingUploadSegments(1000);
+      if (pending.length === 0) return;
+
+      // Upload with bounded concurrency.
+      const queue = pending.filter((s) => !this.inflight.has(s.segment_id) && !this.failures.shouldSkip(s.stream));
+      if (queue.length === 0) return;
+      const streams = new Set(queue.map((s) => s.stream));
+
+      const workers: Promise<void>[] = [];
+      for (let i = 0; i < this.config.uploadConcurrency; i++) {
+        workers.push(this.uploadWorker(queue));
+      }
+      await Promise.all(workers);
+
+      // Notify indexer / listeners.
+      if (this.hooks?.onSegmentsUploaded) {
+        for (const stream of streams) {
+          try {
+            this.hooks.onSegmentsUploaded(stream);
+          } catch {
+            // ignore
+          }
+        }
+      }
+
+      // Publish manifests for affected streams.
+      for (const stream of streams) {
+        if (this.failures.shouldSkip(stream)) continue;
+        try {
+          await this.publishManifest(stream);
+        } catch (e) {
+          const msg = String((e as any)?.message ?? e);
+          const lower = msg.toLowerCase();
+          if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+            // eslint-disable-next-line no-console
+            console.error("manifest publish failed", stream, e);
+          }
+        }
+      }
+    } catch (e) {
+      const msg = String((e as any)?.message ?? e);
+      const lower = msg.toLowerCase();
+      if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+        // eslint-disable-next-line no-console
+        console.error("uploader tick error", e);
+      }
+    } finally {
+      this.running = false;
+    }
+  }
+
+  private async uploadWorker(queue: SegmentRow[]): Promise<void> {
+    while (queue.length > 0) {
+      if (this.stopping) return;
+      const seg = queue.shift();
+      if (!seg) return;
+      if (this.inflight.has(seg.segment_id)) continue;
+      this.inflight.add(seg.segment_id);
+      try {
+        try {
+          await this.uploadOne(seg);
+          this.failures.recordSuccess(seg.stream);
+        } catch (e) {
+          const msg = String((e as any)?.message ?? e);
+          const lower = msg.toLowerCase();
+          if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+            // eslint-disable-next-line no-console
+            console.error("segment upload failed", seg.segment_id, e);
+          }
+        }
+      } finally {
+        this.inflight.delete(seg.segment_id);
+      }
+    }
+  }
+
+  private async uploadOne(seg: SegmentRow): Promise<void> {
+    if (this.stopping) return;
+    const shash = streamHash16Hex(seg.stream);
+    const objectKey = segmentObjectKey(shash, seg.segment_index);
+    try {
+      const res = await retry(
+        async () => {
+          if (this.os.putFile) {
+            return this.os.putFile(objectKey, seg.local_path, seg.size_bytes);
+          }
+          const bytes = new Uint8Array(await readFile(seg.local_path));
+          return this.os.put(objectKey, bytes, { contentLength: seg.size_bytes });
+        },
+        {
+          retries: this.config.objectStoreRetries,
+          baseDelayMs: this.config.objectStoreBaseDelayMs,
+          maxDelayMs: this.config.objectStoreMaxDelayMs,
+          timeoutMs: this.config.objectStoreTimeoutMs,
+        }
+      );
+      this.db.markSegmentUploaded(seg.segment_id, res.etag, this.db.nowMs());
+      if (this.stats) this.stats.recordUploadedBytes(seg.size_bytes);
+      if (this.gate) this.gate.adjustOnUpload(seg.size_bytes);
+    } catch (e) {
+      this.failures.recordFailure(seg.stream);
+      throw e;
+    }
+  }
+
+  async publishManifest(stream: string): Promise<void> {
+    if (this.stopping) return;
+    if (this.manifestInflight.has(stream)) return;
+    this.manifestInflight.add(stream);
+    try {
+      const srow = this.db.getStream(stream);
+      if (!srow) return;
+
+      const prevPrefix = srow.uploaded_segment_count ?? 0;
+      let uploadedPrefix = this.db.advanceUploadedSegmentCount(stream);
+
+      const segCount = this.db.countSegmentsForStream(stream);
+      let meta = this.db.getSegmentMeta(stream);
+      const needsRebuild =
+        !meta ||
+        meta.segment_count !== segCount ||
+        meta.segment_offsets.byteLength !== segCount * 8 ||
+        meta.segment_blocks.byteLength !== segCount * 4 ||
+        meta.segment_last_ts.byteLength !== segCount * 8;
+      if (needsRebuild) {
+        meta = this.db.rebuildSegmentMeta(stream);
+      }
+      if (!meta) return;
+      if (uploadedPrefix > meta.segment_count) {
+        uploadedPrefix = meta.segment_count;
+        this.db.setUploadedSegmentCount(stream, uploadedPrefix);
+      }
+
+      const uploadedThrough =
+        uploadedPrefix === 0 ? -1n : readU64LE(meta.segment_offsets, (uploadedPrefix - 1) * 8) - 1n;
+
+      const manifestRow = this.db.getManifestRow(stream);
+      const generation = manifestRow.generation + 1;
+
+      const indexState = this.db.getIndexState(stream);
+      const indexRuns = this.db.listIndexRuns(stream);
+      const retiredRuns = this.db.listRetiredIndexRuns(stream);
+      const manifestRes = buildManifestResult({
+        streamName: stream,
+        streamRow: srow,
+        segmentMeta: meta,
+        uploadedPrefixCount: uploadedPrefix,
+        generation,
+        indexState,
+        indexRuns,
+        retiredRuns,
+      });
+      if (Result.isError(manifestRes)) {
+        this.failures.recordFailure(stream);
+        throw dsError(manifestRes.error.message);
+      }
+      const manifest = manifestRes.value;
+
+      const shash = streamHash16Hex(stream);
+      const mKey = manifestObjectKey(shash);
+      const body = new TextEncoder().encode(JSON.stringify(manifest));
+      let putRes;
+      try {
+        putRes = await retry(
+          () => this.os.put(mKey, body),
+          {
+            retries: this.config.objectStoreRetries,
+            baseDelayMs: this.config.objectStoreBaseDelayMs,
+            maxDelayMs: this.config.objectStoreMaxDelayMs,
+            timeoutMs: this.config.objectStoreTimeoutMs,
+          }
+        );
+      } catch (e) {
+        this.failures.recordFailure(stream);
+        throw e;
+      }
+
+      // Commit point: advance uploaded_through and delete WAL prefix.
+      this.db.commitManifest(stream, generation, putRes.etag, this.db.nowMs(), uploadedThrough);
+
+      // Local disk cleanup: delete newly uploaded segment files.
+      if (uploadedPrefix > prevPrefix) {
+        for (let i = prevPrefix; i < uploadedPrefix; i++) {
+          const seg = this.db.getSegmentByIndex(stream, i);
+          if (!seg) continue;
+          try {
+            const objectKey = segmentObjectKey(shash, seg.segment_index);
+            if (this.diskCache && this.diskCache.putFromLocal(objectKey, seg.local_path, seg.size_bytes)) {
+              continue;
+            }
+            unlinkSync(seg.local_path);
+          } catch {
+            // ignore
+          }
+        }
+      }
+    } finally {
+      this.manifestInflight.delete(stream);
+    }
+  }
+}
+
+class FailureTracker {
+  private readonly cache: LruCache<string, { attempts: number; untilMs: number }>;
+
+  constructor(maxEntries: number) {
+    this.cache = new LruCache(maxEntries);
+  }
+
+  shouldSkip(stream: string): boolean {
+    const item = this.cache.get(stream);
+    if (!item) return false;
+    if (Date.now() >= item.untilMs) {
+      this.cache.delete(stream);
+      return false;
+    }
+    return true;
+  }
+
+  recordFailure(stream: string): void {
+    const now = Date.now();
+    const item = this.cache.get(stream) ?? { attempts: 0, untilMs: now };
+    item.attempts += 1;
+    const backoff = Math.min(60_000, 500 * 2 ** (item.attempts - 1));
+    item.untilMs = now + backoff;
+    this.cache.set(stream, item);
+  }
+
+  recordSuccess(stream: string): void {
+    this.cache.delete(stream);
+  }
+}

package/src/util/base32_crockford.ts

@@ -0,0 +1,81 @@
+import { Result } from "better-result";
+import { dsError } from "./ds_error.ts";
+/**
+ * Crockford Base32 encoding/decoding.
+ *
+ * We use this for canonical 128-bit offsets encoded into a fixed 26-character string.
+ */
+
+const ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+const DECODE_MAP: Record<string, number> = (() => {
+  const m: Record<string, number> = {};
+  for (let i = 0; i < ALPHABET.length; i++) {
+    m[ALPHABET[i]] = i;
+    m[ALPHABET[i].toLowerCase()] = i;
+  }
+  // Crockford aliases
+  m["O"] = m["o"] = 0;
+  m["I"] = m["i"] = 1;
+  m["L"] = m["l"] = 1;
+  return m;
+})();
+
+export type CrockfordBase32Error = {
+  kind: "invalid_base32";
+  message: string;
+};
+
+function invalidBase32<T = never>(message: string): Result<T, CrockfordBase32Error> {
+  return Result.err({ kind: "invalid_base32", message });
+}
+
+export function encodeCrockfordBase32Fixed26Result(bytes16: Uint8Array): Result<string, CrockfordBase32Error> {
+  if (bytes16.byteLength !== 16) return invalidBase32(`expected 16 bytes, got ${bytes16.byteLength}`);
+  // Interpret as big-endian 128-bit unsigned integer.
+  let n = 0n;
+  for (const b of bytes16) n = (n << 8n) | BigInt(b);
+  // Pad with 2 zero bits on the left: 128 -> 130 bits.
+  n = n << 2n;
+
+  let out = "";
+  // Extract 26 groups of 5 bits from MSB to LSB.
+  // Highest bit position is 129.
+  for (let i = 0; i < 26; i++) {
+    const shift = 5n * BigInt(25 - i);
+    const idx = Number((n >> shift) & 31n);
+    out += ALPHABET[idx];
+  }
+  return Result.ok(out);
+}
+
+export function encodeCrockfordBase32Fixed26(bytes16: Uint8Array): string {
+  const res = encodeCrockfordBase32Fixed26Result(bytes16);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function decodeCrockfordBase32Fixed26Result(s: string): Result<Uint8Array, CrockfordBase32Error> {
+  if (s === "-1") return invalidBase32("-1 is a sentinel offset and cannot be decoded as base32");
+  if (s.length !== 26) return invalidBase32(`expected 26 chars, got ${s.length}`);
+  let n = 0n;
+  for (const ch of s) {
+    const v = DECODE_MAP[ch];
+    if (v === undefined) return invalidBase32(`invalid base32 char: ${ch}`);
+    n = (n << 5n) | BigInt(v);
+  }
+  // Remove the 2 padding bits.
+  n = n >> 2n;
+
+  const out = new Uint8Array(16);
+  for (let i = 15; i >= 0; i--) {
+    out[i] = Number(n & 0xffn);
+    n = n >> 8n;
+  }
+  return Result.ok(out);
+}
+
+export function decodeCrockfordBase32Fixed26(s: string): Uint8Array {
+  const res = decodeCrockfordBase32Fixed26Result(s);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}