@prisma/streams-server 0.0.1 → 0.1.1

package/src/util/bloom256.ts

@@ -0,0 +1,67 @@
+import { dsError } from "./ds_error.ts";
+/**
+ * A tiny 256-bit bloom filter (2048 bits) with 3 hash probes.
+ *
+ * This is used per DSB3 block to cheaply skip blocks during key-filtered reads.
+ *
+ * Correctness rule: bloom filter may have false positives but MUST NOT have false negatives.
+ */
+
+const BITS = 2048n;
+const MASK64 = (1n << 64n) - 1n;
+
+function fnv1a64(data: Uint8Array): bigint {
+  let h = 14695981039346656037n;
+  for (const b of data) {
+    h ^= BigInt(b);
+    h = (h * 1099511628211n) & MASK64;
+  }
+  return h;
+}
+
+function mix64(x: bigint): bigint {
+  // SplitMix64-esque mix.
+  x = (x + 0x9e3779b97f4a7c15n) & MASK64;
+  x = (x ^ (x >> 30n)) * 0xbf58476d1ce4e5b9n & MASK64;
+  x = (x ^ (x >> 27n)) * 0x94d049bb133111ebn & MASK64;
+  x = x ^ (x >> 31n);
+  return x & MASK64;
+}
+
+export class Bloom256 {
+  private bits: Uint8Array;
+
+  constructor(bits?: Uint8Array) {
+    if (bits && bits.byteLength !== 32) throw dsError("bloom must be 32 bytes");
+    this.bits = bits ? new Uint8Array(bits) : new Uint8Array(32);
+  }
+
+  toBytes(): Uint8Array {
+    return new Uint8Array(this.bits);
+  }
+
+  add(keyUtf8: Uint8Array): void {
+    if (keyUtf8.byteLength === 0) return;
+    const h1 = fnv1a64(keyUtf8);
+    const h2 = mix64(h1 ^ 0xa0761d6478bd642fn);
+    for (let i = 0; i < 3; i++) {
+      const idx = Number((h1 + BigInt(i) * h2) % BITS); // 0..2047
+      const byte = idx >> 3;
+      const bit = idx & 7;
+      this.bits[byte] |= 1 << bit;
+    }
+  }
+
+  maybeHas(keyUtf8: Uint8Array): boolean {
+    if (keyUtf8.byteLength === 0) return true;
+    const h1 = fnv1a64(keyUtf8);
+    const h2 = mix64(h1 ^ 0xa0761d6478bd642fn);
+    for (let i = 0; i < 3; i++) {
+      const idx = Number((h1 + BigInt(i) * h2) % BITS);
+      const byte = idx >> 3;
+      const bit = idx & 7;
+      if ((this.bits[byte] & (1 << bit)) === 0) return false;
+    }
+    return true;
+  }
+}

package/src/util/cleanup.ts

@@ -0,0 +1,22 @@
+import { existsSync, readdirSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+
+export function cleanupTempSegments(rootDir: string): void {
+  const base = join(rootDir, "local");
+  if (!existsSync(base)) return;
+  const walk = (dir: string) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(full);
+      } else if (entry.isFile() && entry.name.endsWith(".tmp")) {
+        try {
+          unlinkSync(full);
+        } catch {
+          // ignore
+        }
+      }
+    }
+  };
+  walk(base);
+}

package/src/util/crc32c.ts

@@ -0,0 +1,29 @@
+/**
+ * CRC32C (Castagnoli) implementation in pure TypeScript.
+ *
+ * Used for DSB3 block headers.
+ */
+
+const POLY = 0x82f63b78;
+let TABLE: Uint32Array | null = null;
+
+function makeTable(): Uint32Array {
+  const t = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let k = 0; k < 8; k++) {
+      c = (c & 1) ? (POLY ^ (c >>> 1)) : (c >>> 1);
+    }
+    t[i] = c >>> 0;
+  }
+  return t;
+}
+
+export function crc32c(buf: Uint8Array): number {
+  if (!TABLE) TABLE = makeTable();
+  let c = 0xffffffff;
+  for (const b of buf) {
+    c = TABLE[(c ^ b) & 0xff] ^ (c >>> 8);
+  }
+  return (c ^ 0xffffffff) >>> 0;
+}

package/src/util/ds_error.ts

@@ -0,0 +1,15 @@
+import { TaggedError } from "better-result";
+
+export class DurableStreamsError extends TaggedError("DurableStreamsError")<{
+  message: string;
+  cause?: unknown;
+  code?: string;
+}>() {}
+
+export function dsError(message: string, opts?: { cause?: unknown; code?: string }): DurableStreamsError {
+  return new DurableStreamsError({
+    message,
+    ...(opts?.cause !== undefined ? { cause: opts.cause } : {}),
+    ...(opts?.code !== undefined ? { code: opts.code } : {}),
+  });
+}

package/src/util/duration.ts

@@ -0,0 +1,17 @@
+import { Result } from "better-result";
+
+/** Parse a duration like 15s, 30m, 24h into milliseconds. */
+export function parseDurationMsResult(s: string): Result<number, { kind: "invalid_duration"; message: string }> {
+  const m = /^([0-9]+)(ms|s|m|h|d)$/.exec(s.trim());
+  if (!m) return Result.err({ kind: "invalid_duration", message: `invalid duration: ${s}` });
+  const n = Number(m[1]);
+  const unit = m[2];
+  switch (unit) {
+    case "ms": return Result.ok(n);
+    case "s": return Result.ok(n * 1000);
+    case "m": return Result.ok(n * 60 * 1000);
+    case "h": return Result.ok(n * 60 * 60 * 1000);
+    case "d": return Result.ok(n * 24 * 60 * 60 * 1000);
+    default: return Result.err({ kind: "invalid_duration", message: `invalid unit: ${unit}` });
+  }
+}

package/src/util/endian.ts

@@ -0,0 +1,53 @@
+/**
+ * Endian helpers.
+ *
+ * The TieredStore spec uses big-endian for on-disk encodings.
+ */
+
+export function writeU32BE(dst: Uint8Array, offset: number, value: number): void {
+  const dv = new DataView(dst.buffer, dst.byteOffset, dst.byteLength);
+  dv.setUint32(offset, value >>> 0, false);
+}
+
+export function writeU16BE(dst: Uint8Array, offset: number, value: number): void {
+  const dv = new DataView(dst.buffer, dst.byteOffset, dst.byteLength);
+  dv.setUint16(offset, value & 0xffff, false);
+}
+
+export function writeU64BE(dst: Uint8Array, offset: number, value: bigint): void {
+  const dv = new DataView(dst.buffer, dst.byteOffset, dst.byteLength);
+  dv.setBigUint64(offset, value, false);
+}
+
+export function readU32BE(src: Uint8Array, offset: number): number {
+  const dv = new DataView(src.buffer, src.byteOffset, src.byteLength);
+  return dv.getUint32(offset, false);
+}
+
+export function readU16BE(src: Uint8Array, offset: number): number {
+  const dv = new DataView(src.buffer, src.byteOffset, src.byteLength);
+  return dv.getUint16(offset, false);
+}
+
+export function readU64BE(src: Uint8Array, offset: number): bigint {
+  const dv = new DataView(src.buffer, src.byteOffset, src.byteLength);
+  return dv.getBigUint64(offset, false);
+}
+
+// Little-endian helpers for metadata arrays (not on-disk segment format).
+export function readU64LE(src: Uint8Array, offset: number): bigint {
+  const dv = new DataView(src.buffer, src.byteOffset, src.byteLength);
+  return dv.getBigUint64(offset, true);
+}
+
+export function concatBytes(parts: Uint8Array[]): Uint8Array {
+  let total = 0;
+  for (const p of parts) total += p.byteLength;
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const p of parts) {
+    out.set(p, off);
+    off += p.byteLength;
+  }
+  return out;
+}

package/src/util/json_pointer.ts

@@ -0,0 +1,148 @@
+import { Result } from "better-result";
+import { dsError } from "./ds_error.ts";
+
+type JsonPointerError = { kind: "invalid_json_pointer" | "invalid_json_pointer_operation"; message: string };
+
+function invalidPointerOperation<T = never>(message: string): Result<T, JsonPointerError> {
+  return Result.err({ kind: "invalid_json_pointer_operation", message });
+}
+
+export function parseJsonPointerResult(ptr: string): Result<string[], JsonPointerError> {
+  if (ptr === "") return Result.ok([]);
+  if (!ptr.startsWith("/")) return Result.err({ kind: "invalid_json_pointer", message: "invalid json pointer" });
+  return Result.ok(
+    ptr
+      .split("/")
+      .slice(1)
+      .map((seg) => seg.replace(/~1/g, "/").replace(/~0/g, "~"))
+  );
+}
+
+export function parseJsonPointer(ptr: string): string[] {
+  const res = parseJsonPointerResult(ptr);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+function isArrayIndex(seg: string): boolean {
+  return seg !== "" && /^[0-9]+$/.test(seg);
+}
+
+function getChild(container: any, seg: string): any {
+  if (Array.isArray(container) && isArrayIndex(seg)) {
+    return container[Number(seg)];
+  }
+  if (container && typeof container === "object") {
+    return (container as any)[seg];
+  }
+  return undefined;
+}
+
+function setChildResult(container: any, seg: string, value: any): Result<void, JsonPointerError> {
+  if (Array.isArray(container) && isArrayIndex(seg)) {
+    const idx = Number(seg);
+    if (idx < 0 || idx >= container.length) return invalidPointerOperation("array index out of bounds");
+    container[idx] = value;
+    return Result.ok(undefined);
+  }
+  if (container && typeof container === "object") {
+    (container as any)[seg] = value;
+    return Result.ok(undefined);
+  }
+  return invalidPointerOperation("invalid parent for set");
+}
+
+function deleteChildResult(container: any, seg: string): Result<void, JsonPointerError> {
+  if (Array.isArray(container) && isArrayIndex(seg)) {
+    const idx = Number(seg);
+    if (idx < 0 || idx >= container.length) return invalidPointerOperation("array index out of bounds");
+    container.splice(idx, 1);
+    return Result.ok(undefined);
+  }
+  if (container && typeof container === "object") {
+    delete (container as any)[seg];
+    return Result.ok(undefined);
+  }
+  return invalidPointerOperation("invalid parent for delete");
+}
+
+export type PointerResolution = {
+  parent: any;
+  key: string | null;
+  value: any;
+  exists: boolean;
+};
+
+export function resolvePointerResult(doc: any, ptr: string): Result<PointerResolution, JsonPointerError> {
+  const segmentsRes = parseJsonPointerResult(ptr);
+  if (Result.isError(segmentsRes)) return Result.err(segmentsRes.error);
+  const segments = segmentsRes.value;
+  if (segments.length === 0) {
+    return Result.ok({ parent: null, key: null, value: doc, exists: true });
+  }
+  let cur: any = doc;
+  for (let i = 0; i < segments.length - 1; i++) {
+    cur = getChild(cur, segments[i]);
+    if (cur === undefined) return Result.ok({ parent: null, key: null, value: undefined, exists: false });
+  }
+  const key = segments[segments.length - 1];
+  const value = cur === undefined ? undefined : getChild(cur, key);
+  return Result.ok({ parent: cur, key, value, exists: value !== undefined });
+}
+
+export function resolvePointer(doc: any, ptr: string): PointerResolution {
+  const res = resolvePointerResult(doc, ptr);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function setPointerValueResult(doc: any, ptr: string, value: any, opts?: { createParents?: boolean }): Result<any, JsonPointerError> {
+  const segmentsRes = parseJsonPointerResult(ptr);
+  if (Result.isError(segmentsRes)) return Result.err(segmentsRes.error);
+  const segments = segmentsRes.value;
+  if (segments.length === 0) {
+    return Result.ok(value);
+  }
+  let cur: any = doc;
+  for (let i = 0; i < segments.length - 1; i++) {
+    const seg = segments[i];
+    let next = getChild(cur, seg);
+    if (next === undefined) {
+      if (!opts?.createParents) return invalidPointerOperation("missing parent");
+      next = {};
+      const setRes = setChildResult(cur, seg, next);
+      if (Result.isError(setRes)) return setRes;
+    }
+    cur = next;
+  }
+  const leafRes = setChildResult(cur, segments[segments.length - 1], value);
+  if (Result.isError(leafRes)) return leafRes;
+  return Result.ok(doc);
+}
+
+export function setPointerValue(doc: any, ptr: string, value: any, opts?: { createParents?: boolean }): any {
+  const res = setPointerValueResult(doc, ptr, value, opts);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function deletePointerValueResult(doc: any, ptr: string): Result<any, JsonPointerError> {
+  const segmentsRes = parseJsonPointerResult(ptr);
+  if (Result.isError(segmentsRes)) return Result.err(segmentsRes.error);
+  const segments = segmentsRes.value;
+  if (segments.length === 0) return invalidPointerOperation("cannot delete document root");
+  let cur: any = doc;
+  for (let i = 0; i < segments.length - 1; i++) {
+    cur = getChild(cur, segments[i]);
+    if (cur === undefined) return invalidPointerOperation("missing parent");
+  }
+  const deleteRes = deleteChildResult(cur, segments[segments.length - 1]);
+  if (Result.isError(deleteRes)) return deleteRes;
+  return Result.ok(doc);
+}
+
+export function deletePointerValue(doc: any, ptr: string): any {
+  const res = deletePointerValueResult(doc, ptr);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}

package/src/util/log.ts

@@ -0,0 +1,25 @@
+type ConsoleFn = (...args: any[]) => void;
+
+let patched = false;
+
+function wrapConsole(orig: ConsoleFn, level: string): ConsoleFn {
+  return (...args: any[]) => {
+    const prefix = `[${new Date().toISOString()}] [${level}]`;
+    if (args.length === 0) return orig(prefix);
+    return orig(prefix, ...args);
+  };
+}
+
+export function initConsoleLogging(): void {
+  if (patched) return;
+  patched = true;
+  const globalAny = globalThis as any;
+  if (globalAny.__ds_console_patched) return;
+  globalAny.__ds_console_patched = true;
+
+  console.log = wrapConsole(console.log.bind(console), "INFO");
+  console.info = wrapConsole(console.info.bind(console), "INFO");
+  console.warn = wrapConsole(console.warn.bind(console), "WARN");
+  console.error = wrapConsole(console.error.bind(console), "ERROR");
+  if (console.debug) console.debug = wrapConsole(console.debug.bind(console), "DEBUG");
+}

package/src/util/lru.ts

@@ -0,0 +1,45 @@
+import { dsError } from "./ds_error.ts";
+export class LruCache<K, V> {
+  private readonly maxEntries: number;
+  private readonly map = new Map<K, V>();
+
+  constructor(maxEntries: number) {
+    if (maxEntries <= 0) throw dsError("maxEntries must be > 0");
+    this.maxEntries = maxEntries;
+  }
+
+  get size(): number {
+    return this.map.size;
+  }
+
+  get(key: K): V | undefined {
+    const value = this.map.get(key);
+    if (value === undefined) return undefined;
+    // Refresh LRU order.
+    this.map.delete(key);
+    this.map.set(key, value);
+    return value;
+  }
+
+  set(key: K, value: V): void {
+    if (this.map.has(key)) this.map.delete(key);
+    this.map.set(key, value);
+    while (this.map.size > this.maxEntries) {
+      const oldest = this.map.keys().next();
+      if (oldest.done) break;
+      this.map.delete(oldest.value);
+    }
+  }
+
+  has(key: K): boolean {
+    return this.map.has(key);
+  }
+
+  delete(key: K): boolean {
+    return this.map.delete(key);
+  }
+
+  clear(): void {
+    this.map.clear();
+  }
+}

package/src/util/retry.ts

@@ -0,0 +1,35 @@
+import { dsError } from "./ds_error.ts";
+export type RetryOptions = {
+  retries: number;
+  baseDelayMs: number;
+  maxDelayMs: number;
+  timeoutMs: number;
+};
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
+  if (ms <= 0) return p;
+  return Promise.race([
+    p,
+    new Promise<T>((_, reject) => setTimeout(() => reject(dsError("timeout")), ms)),
+  ]);
+}
+
+export async function retry<T>(fn: () => Promise<T>, opts: RetryOptions): Promise<T> {
+  let attempt = 0;
+  let delay = opts.baseDelayMs;
+  for (;;) {
+    try {
+      return await withTimeout(fn(), opts.timeoutMs);
+    } catch (e) {
+      attempt++;
+      if (attempt > opts.retries) throw e;
+      const jitter = Math.random() * 0.2 + 0.9; // 0.9-1.1
+      await sleep(Math.min(opts.maxDelayMs, Math.floor(delay * jitter)));
+      delay = Math.min(opts.maxDelayMs, delay * 2);
+    }
+  }
+}

package/src/util/siphash.ts

@@ -0,0 +1,71 @@
+import { dsError } from "./ds_error.ts";
+const MASK_64 = 0xffffffffffffffffn;
+
+function rotl(x: bigint, b: number): bigint {
+  return ((x << BigInt(b)) | (x >> BigInt(64 - b))) & MASK_64;
+}
+
+function readU64LE(bytes: Uint8Array, offset: number): bigint {
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  return dv.getBigUint64(offset, true);
+}
+
+export function siphash24(key: Uint8Array, msg: Uint8Array): bigint {
+  if (key.byteLength !== 16) throw dsError("siphash24 requires 16-byte key");
+  const k0 = readU64LE(key, 0);
+  const k1 = readU64LE(key, 8);
+
+  let v0 = 0x736f6d6570736575n ^ k0;
+  let v1 = 0x646f72616e646f6dn ^ k1;
+  let v2 = 0x6c7967656e657261n ^ k0;
+  let v3 = 0x7465646279746573n ^ k1;
+
+  const msgLen = msg.byteLength;
+  let off = 0;
+  while (off + 8 <= msgLen) {
+    const m = readU64LE(msg, off);
+    v3 ^= m;
+    sipRound();
+    sipRound();
+    v0 ^= m;
+    off += 8;
+  }
+
+  let b = BigInt(msgLen) << 56n;
+  for (let i = 0; i < msgLen - off; i++) {
+    b |= BigInt(msg[off + i]) << BigInt(8 * i);
+  }
+
+  v3 ^= b;
+  sipRound();
+  sipRound();
+  v0 ^= b;
+
+  v2 ^= 0xffn;
+  sipRound();
+  sipRound();
+  sipRound();
+  sipRound();
+
+  return (v0 ^ v1 ^ v2 ^ v3) & MASK_64;
+
+  function sipRound(): void {
+    v0 = (v0 + v1) & MASK_64;
+    v1 = rotl(v1, 13);
+    v1 ^= v0;
+    v0 = rotl(v0, 32);
+
+    v2 = (v2 + v3) & MASK_64;
+    v3 = rotl(v3, 16);
+    v3 ^= v2;
+
+    v0 = (v0 + v3) & MASK_64;
+    v3 = rotl(v3, 21);
+    v3 ^= v0;
+
+    v2 = (v2 + v1) & MASK_64;
+    v1 = rotl(v1, 17);
+    v1 ^= v2;
+    v2 = rotl(v2, 32);
+  }
+}

package/src/util/stream_paths.ts

@@ -0,0 +1,31 @@
+import { createHash } from "node:crypto";
+
+export function streamHash16Hex(stream: string): string {
+  const full = createHash("sha256").update(stream).digest();
+  return full.subarray(0, 16).toString("hex"); // 32 hex chars
+}
+
+export function pad16(n: number): string {
+  const s = String(n);
+  return s.padStart(16, "0");
+}
+
+export function segmentObjectKey(streamHash: string, segmentIndex: number): string {
+  return `streams/${streamHash}/segments/${pad16(segmentIndex)}.bin`;
+}
+
+export function manifestObjectKey(streamHash: string): string {
+  return `streams/${streamHash}/manifest.json`;
+}
+
+export function schemaObjectKey(streamHash: string): string {
+  return `streams/${streamHash}/schema-registry.json`;
+}
+
+export function indexRunObjectKey(streamHash: string, runId: string): string {
+  return `streams/${streamHash}/index/${runId}.idx`;
+}
+
+export function localSegmentPath(rootDir: string, streamHash: string, segmentIndex: number): string {
+  return `${rootDir}/local/streams/${streamHash}/segments/${pad16(segmentIndex)}.bin`;
+}

package/src/util/time.ts

@@ -0,0 +1,14 @@
+import { Result } from "better-result";
+
+export function parseTimestampMsResult(input: string): Result<bigint, { kind: "empty_timestamp" | "invalid_timestamp"; message: string }> {
+  const s = input.trim();
+  if (s === "") return Result.err({ kind: "empty_timestamp", message: "empty timestamp" });
+  // unix nanos
+  if (/^[0-9]+$/.test(s)) {
+    return Result.ok(BigInt(s) / 1_000_000n);
+  }
+  const d = new Date(s);
+  const ms = d.getTime();
+  if (Number.isNaN(ms)) return Result.err({ kind: "invalid_timestamp", message: `invalid timestamp: ${input}` });
+  return Result.ok(BigInt(ms));
+}

package/src/util/yield.ts

@@ -0,0 +1,3 @@
+export async function yieldToEventLoop(): Promise<void> {
+  await new Promise<void>((resolve) => setTimeout(resolve, 0));
+}

package/build/index.d.mts

@@ -1 +0,0 @@
-export { };

package/build/index.d.ts

@@ -1 +0,0 @@
-export { };

package/build/index.mjs

@@ -1 +0,0 @@
-export {  };