@prisma/streams-server 0.0.1 → 0.1.0

package/src/schema/registry.ts

@@ -0,0 +1,405 @@
+import Ajv from "ajv";
+import { createHash } from "node:crypto";
+import type { SqliteDurableStore, StreamRow } from "../db/db";
+import { Result } from "better-result";
+import { LruCache } from "../util/lru";
+import { DURABLE_LENS_V1_SCHEMA } from "./lens_schema";
+import { compileLensResult, lensFromJson, type CompiledLens, type Lens } from "../lens/lens";
+import { validateLensAgainstSchemasResult, fillLensDefaultsResult } from "./proof";
+import { parseJsonPointerResult } from "../util/json_pointer";
+import {
+  isTouchEnabled,
+  validateStreamInterpreterConfigResult,
+  type StreamInterpreterConfig,
+} from "../touch/spec";
+import { dsError } from "../util/ds_error.ts";
+
+export type RoutingKeyConfig = {
+  jsonPointer: string;
+  required: boolean;
+};
+
+export type SchemaRegistry = {
+  apiVersion: "durable.streams/schema-registry/v1";
+  schema: string;
+  currentVersion: number;
+  routingKey?: RoutingKeyConfig;
+  interpreter?: StreamInterpreterConfig;
+  boundaries: Array<{ offset: number; version: number }>;
+  schemas: Record<string, any>;
+  lenses: Record<string, any>;
+};
+
+export type SchemaRegistryMutationError = {
+  kind: "version_mismatch" | "bad_request";
+  message: string;
+  code?: string;
+};
+
+export type SchemaRegistryReadError = {
+  kind: "invalid_registry" | "invalid_lens_chain";
+  message: string;
+  code?: string;
+};
+
+type RegistryRow = { stream: string; registry_json: string; updated_at_ms: bigint };
+
+type Validator = ReturnType<Ajv["compile"]>;
+
+const AJV = new Ajv({
+  allErrors: true,
+  strict: false,
+  allowUnionTypes: true,
+  validateSchema: false,
+});
+
+const LENS_VALIDATOR = AJV.compile(DURABLE_LENS_V1_SCHEMA);
+
+function sha256Hex(input: string): string {
+  return createHash("sha256").update(input).digest("hex");
+}
+
+function defaultRegistry(stream: string): SchemaRegistry {
+  return {
+    apiVersion: "durable.streams/schema-registry/v1",
+    schema: stream,
+    currentVersion: 0,
+    boundaries: [],
+    schemas: {},
+    lenses: {},
+  };
+}
+
+function ensureNoRefResult(schema: any): Result<void, { message: string }> {
+  const stack: any[] = [schema];
+  while (stack.length > 0) {
+    const cur = stack.pop();
+    if (!cur || typeof cur !== "object") continue;
+    if (Object.prototype.hasOwnProperty.call(cur, "$ref")) {
+      return Result.err({ message: "external $ref is not supported" });
+    }
+    for (const v of Object.values(cur)) {
+      if (v && typeof v === "object") stack.push(v);
+    }
+  }
+  return Result.ok(undefined);
+}
+
+function validateJsonSchemaResult(schema: any): Result<void, { message: string }> {
+  const noRefRes = ensureNoRefResult(schema);
+  if (Result.isError(noRefRes)) return noRefRes;
+  try {
+    const validate = AJV.compile(schema);
+    if (!validate) return Result.err({ message: "schema validation failed" });
+  } catch (e: any) {
+    return Result.err({ message: String(e?.message ?? e) });
+  }
+  return Result.ok(undefined);
+}
+
+function parseRegistryResult(stream: string, json: string): Result<SchemaRegistry, { message: string }> {
+  let raw: any;
+  try {
+    raw = JSON.parse(json);
+  } catch (e: any) {
+    return Result.err({ message: String(e?.message ?? e) });
+  }
+  if (!raw || typeof raw !== "object") return Result.err({ message: "invalid schema registry" });
+  const reg = raw as SchemaRegistry;
+  if (reg.apiVersion !== "durable.streams/schema-registry/v1") return Result.err({ message: "invalid registry apiVersion" });
+  if (!reg.schema) reg.schema = stream;
+  if (!Array.isArray(reg.boundaries)) reg.boundaries = [];
+  if (!reg.schemas || typeof reg.schemas !== "object") reg.schemas = {};
+  if (!reg.lenses || typeof reg.lenses !== "object") reg.lenses = {};
+  if (typeof reg.currentVersion !== "number") reg.currentVersion = 0;
+  if ((reg as any).interpreter === null) delete (reg as any).interpreter;
+  return Result.ok(reg);
+}
+
+function serializeRegistry(reg: SchemaRegistry): string {
+  return JSON.stringify(reg);
+}
+
+function validateLensResult(raw: any): Result<Lens, { message: string }> {
+  const ok = LENS_VALIDATOR(raw);
+  if (!ok) {
+    const msg = AJV.errorsText(LENS_VALIDATOR.errors || undefined);
+    return Result.err({ message: `invalid lens: ${msg}` });
+  }
+  return Result.ok(raw as Lens);
+}
+
+function bigintToNumberSafeResult(v: bigint): Result<number, { message: string }> {
+  const max = BigInt(Number.MAX_SAFE_INTEGER);
+  if (v > max) return Result.err({ message: "offset exceeds MAX_SAFE_INTEGER" });
+  return Result.ok(Number(v));
+}
+
+export class SchemaRegistryStore {
+  private readonly db: SqliteDurableStore;
+  private readonly registryCache: LruCache<string, { reg: SchemaRegistry; updatedAtMs: bigint }>;
+  private readonly validatorCache: LruCache<string, Validator>;
+  private readonly lensCache: LruCache<string, CompiledLens>;
+  private readonly lensChainCache: LruCache<string, CompiledLens[]>;
+
+  constructor(db: SqliteDurableStore, opts?: { registryCacheEntries?: number; validatorCacheEntries?: number; lensCacheEntries?: number }) {
+    this.db = db;
+    this.registryCache = new LruCache(opts?.registryCacheEntries ?? 1024);
+    this.validatorCache = new LruCache(opts?.validatorCacheEntries ?? 256);
+    this.lensCache = new LruCache(opts?.lensCacheEntries ?? 256);
+    this.lensChainCache = new LruCache(opts?.lensCacheEntries ?? 256);
+  }
+
+  private loadRow(stream: string): RegistryRow | null {
+    return this.db.getSchemaRegistry(stream);
+  }
+
+  getRegistry(stream: string): SchemaRegistry {
+    const res = this.getRegistryResult(stream);
+    if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
+    return res.value;
+  }
+
+  getRegistryResult(stream: string): Result<SchemaRegistry, SchemaRegistryReadError> {
+    const row = this.loadRow(stream);
+    if (!row) return Result.ok(defaultRegistry(stream));
+    const cached = this.registryCache.get(stream);
+    if (cached && cached.updatedAtMs === row.updated_at_ms) return Result.ok(cached.reg);
+    const parseRes = parseRegistryResult(stream, row.registry_json);
+    if (Result.isError(parseRes)) {
+      return Result.err({ kind: "invalid_registry", message: parseRes.error.message });
+    }
+    const reg = parseRes.value;
+    this.registryCache.set(stream, { reg, updatedAtMs: row.updated_at_ms });
+    return Result.ok(reg);
+  }
+
+  updateRegistry(
+    stream: string,
+    streamRow: StreamRow,
+    update: { schema: any; lens?: any; routingKey?: RoutingKeyConfig; interpreter?: StreamInterpreterConfig | null }
+  ): SchemaRegistry {
+    const res = this.updateRegistryResult(stream, streamRow, update);
+    if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
+    return res.value;
+  }
+
+  updateRegistryResult(
+    stream: string,
+    streamRow: StreamRow,
+    update: { schema: any; lens?: any; routingKey?: RoutingKeyConfig; interpreter?: StreamInterpreterConfig | null }
+  ): Result<SchemaRegistry, SchemaRegistryMutationError> {
+    let validatedInterpreter: StreamInterpreterConfig | null | undefined = undefined;
+    if (update.routingKey) {
+      const pointerRes = parseJsonPointerResult(update.routingKey.jsonPointer);
+      if (Result.isError(pointerRes)) {
+        return Result.err({ kind: "bad_request", message: pointerRes.error.message });
+      }
+      if (typeof update.routingKey.required !== "boolean") {
+        return Result.err({ kind: "bad_request", message: "routingKey.required must be boolean" });
+      }
+    }
+    if (update.interpreter !== undefined) {
+      if (update.interpreter === null) {
+        validatedInterpreter = null;
+      } else {
+        const interpreterRes = validateStreamInterpreterConfigResult(update.interpreter);
+        if (Result.isError(interpreterRes)) {
+          return Result.err({ kind: "bad_request", message: interpreterRes.error.message });
+        }
+        validatedInterpreter = interpreterRes.value;
+      }
+    }
+    if (update.schema === undefined) return Result.err({ kind: "bad_request", message: "missing schema" });
+    const schemaRes = validateJsonSchemaResult(update.schema);
+    if (Result.isError(schemaRes)) return Result.err({ kind: "bad_request", message: schemaRes.error.message });
+
+    const regRes = this.getRegistryResult(stream);
+    if (Result.isError(regRes)) return Result.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code });
+    const reg = regRes.value;
+    const currentVersion = reg.currentVersion ?? 0;
+    const streamEmpty = streamRow.next_offset === 0n;
+
+    if (currentVersion === 0) {
+      if (!streamEmpty) return Result.err({ kind: "bad_request", message: "first schema requires empty stream" });
+      if (update.lens) {
+        const lensRes = validateLensResult(update.lens);
+        if (Result.isError(lensRes)) return Result.err({ kind: "bad_request", message: lensRes.error.message });
+        if (lensRes.value.from !== 0 || lensRes.value.to !== 1) {
+          return Result.err({
+            kind: "version_mismatch",
+            message: "lens version mismatch",
+            code: "schema_lens_version_mismatch",
+          });
+        }
+      }
+      const nextReg: SchemaRegistry = {
+        apiVersion: "durable.streams/schema-registry/v1",
+        schema: stream,
+        currentVersion: 1,
+        routingKey: update.routingKey,
+        interpreter: update.interpreter === undefined ? reg.interpreter : validatedInterpreter ?? undefined,
+        boundaries: [{ offset: 0, version: 1 }],
+        schemas: { ...reg.schemas, ["1"]: update.schema },
+        lenses: { ...reg.lenses },
+      };
+      this.persist(stream, nextReg);
+      this.syncInterpreterState(stream, nextReg);
+      return Result.ok(nextReg);
+    }
+
+    if (!update.lens) return Result.err({ kind: "bad_request", message: "lens required" });
+    const lensRes = validateLensResult(update.lens);
+    if (Result.isError(lensRes)) return Result.err({ kind: "bad_request", message: lensRes.error.message });
+    const lens = lensRes.value;
+    if (lens.from !== currentVersion || lens.to !== currentVersion + 1) {
+      return Result.err({
+        kind: "version_mismatch",
+        message: "lens version mismatch",
+        code: "schema_lens_version_mismatch",
+      });
+    }
+    if (lens.schema && lens.schema !== reg.schema) return Result.err({ kind: "bad_request", message: "lens schema mismatch" });
+
+    const oldSchema = reg.schemas[String(currentVersion)];
+    if (!oldSchema) return Result.err({ kind: "bad_request", message: "missing current schema" });
+    const proofRes = validateLensAgainstSchemasResult(oldSchema, update.schema, lens);
+    if (Result.isError(proofRes)) return Result.err({ kind: "bad_request", message: proofRes.error.message });
+    const defaultsRes = fillLensDefaultsResult(lens, update.schema);
+    if (Result.isError(defaultsRes)) return Result.err({ kind: "bad_request", message: defaultsRes.error.message });
+
+    const boundaryRes = bigintToNumberSafeResult(streamRow.next_offset);
+    if (Result.isError(boundaryRes)) return Result.err({ kind: "bad_request", message: boundaryRes.error.message });
+
+    const nextVersion = currentVersion + 1;
+    const nextReg: SchemaRegistry = {
+      apiVersion: "durable.streams/schema-registry/v1",
+      schema: reg.schema ?? stream,
+      currentVersion: nextVersion,
+      routingKey: update.routingKey ?? reg.routingKey,
+      interpreter: update.interpreter === undefined ? reg.interpreter : validatedInterpreter ?? undefined,
+      boundaries: [...reg.boundaries, { offset: boundaryRes.value, version: nextVersion }],
+      schemas: { ...reg.schemas, [String(nextVersion)]: update.schema },
+      lenses: { ...reg.lenses, [String(currentVersion)]: defaultsRes.value },
+    };
+    this.persist(stream, nextReg);
+    this.syncInterpreterState(stream, nextReg);
+    return Result.ok(nextReg);
+  }
+
+  updateRoutingKey(stream: string, routingKey: RoutingKeyConfig | null): SchemaRegistry {
+    const res = this.updateRoutingKeyResult(stream, routingKey);
+    if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
+    return res.value;
+  }
+
+  updateRoutingKeyResult(stream: string, routingKey: RoutingKeyConfig | null): Result<SchemaRegistry, SchemaRegistryMutationError> {
+    if (routingKey) {
+      const pointerRes = parseJsonPointerResult(routingKey.jsonPointer);
+      if (Result.isError(pointerRes)) {
+        return Result.err({ kind: "bad_request", message: pointerRes.error.message });
+      }
+      if (typeof routingKey.required !== "boolean") {
+        return Result.err({ kind: "bad_request", message: "routingKey.required must be boolean" });
+      }
+    }
+    const regRes = this.getRegistryResult(stream);
+    if (Result.isError(regRes)) return Result.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code });
+    const nextReg: SchemaRegistry = {
+      ...regRes.value,
+      routingKey: routingKey ?? undefined,
+    };
+    this.persist(stream, nextReg);
+    this.syncInterpreterState(stream, nextReg);
+    return Result.ok(nextReg);
+  }
+
+  updateInterpreter(stream: string, interpreter: StreamInterpreterConfig | null): SchemaRegistry {
+    const res = this.updateInterpreterResult(stream, interpreter);
+    if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
+    return res.value;
+  }
+
+  updateInterpreterResult(stream: string, interpreter: StreamInterpreterConfig | null): Result<SchemaRegistry, SchemaRegistryMutationError> {
+    let validatedInterpreter: StreamInterpreterConfig | null = interpreter;
+    if (interpreter) {
+      const interpreterRes = validateStreamInterpreterConfigResult(interpreter);
+      if (Result.isError(interpreterRes)) {
+        return Result.err({ kind: "bad_request", message: interpreterRes.error.message });
+      }
+      validatedInterpreter = interpreterRes.value;
+    }
+    const regRes = this.getRegistryResult(stream);
+    if (Result.isError(regRes)) return Result.err({ kind: "bad_request", message: regRes.error.message, code: regRes.error.code });
+    const nextReg: SchemaRegistry = {
+      ...regRes.value,
+      interpreter: validatedInterpreter ?? undefined,
+    };
+    this.persist(stream, nextReg);
+    this.syncInterpreterState(stream, nextReg);
+    return Result.ok(nextReg);
+  }
+
+  private persist(stream: string, reg: SchemaRegistry): void {
+    const json = serializeRegistry(reg);
+    this.db.upsertSchemaRegistry(stream, json);
+    this.registryCache.set(stream, { reg, updatedAtMs: this.db.nowMs() });
+  }
+
+  private syncInterpreterState(stream: string, reg: SchemaRegistry): void {
+    if (isTouchEnabled(reg.interpreter)) {
+      this.db.ensureStreamInterpreter(stream);
+    } else {
+      this.db.deleteStreamInterpreter(stream);
+    }
+  }
+
+  getValidatorForVersion(reg: SchemaRegistry, version: number): Validator | null {
+    const schema = reg.schemas[String(version)];
+    if (!schema) return null;
+    const hash = sha256Hex(JSON.stringify(schema));
+    const cached = this.validatorCache.get(hash);
+    if (cached) return cached;
+    const validate = AJV.compile(schema);
+    this.validatorCache.set(hash, validate);
+    return validate;
+  }
+
+  getLensChain(reg: SchemaRegistry, fromVersion: number, toVersion: number): CompiledLens[] {
+    const res = this.getLensChainResult(reg, fromVersion, toVersion);
+    if (Result.isError(res)) throw dsError(res.error.message, { code: res.error.code });
+    return res.value;
+  }
+
+  getLensChainResult(reg: SchemaRegistry, fromVersion: number, toVersion: number): Result<CompiledLens[], SchemaRegistryReadError> {
+    const key = `${reg.schema}:${fromVersion}->${toVersion}`;
+    const cached = this.lensChainCache.get(key);
+    if (cached) return Result.ok(cached);
+    const chain: CompiledLens[] = [];
+    for (let v = fromVersion; v < toVersion; v++) {
+      const lensRaw = reg.lenses[String(v)];
+      if (!lensRaw) {
+        return Result.err({
+          kind: "invalid_lens_chain",
+          message: `missing lens v${v}->v${v + 1}`,
+        });
+      }
+      const hash = sha256Hex(JSON.stringify(lensRaw));
+      let compiled = this.lensCache.get(hash);
+      if (!compiled) {
+        const compiledRes = compileLensResult(lensFromJson(lensRaw));
+        if (Result.isError(compiledRes)) {
+          return Result.err({
+            kind: "invalid_lens_chain",
+            message: compiledRes.error.message,
+          });
+        }
+        compiled = compiledRes.value;
+        this.lensCache.set(hash, compiled);
+      }
+      chain.push(compiled);
+    }
+    this.lensChainCache.set(key, chain);
+    return Result.ok(chain);
+  }
+}

package/src/segment/cache.ts

@@ -0,0 +1,179 @@
+import { mkdirSync, readdirSync, statSync, unlinkSync, renameSync, existsSync, writeFileSync, readFileSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
+
+export type SegmentCacheStats = {
+  hits: number;
+  misses: number;
+  evictions: number;
+  bytesAdded: number;
+  usedBytes: number;
+  maxBytes: number;
+  entryCount: number;
+};
+
+export class SegmentDiskCache {
+  private readonly rootDir: string;
+  private readonly maxBytes: number;
+  private readonly entries = new Map<string, { path: string; size: number }>();
+  private totalBytes = 0;
+  private hits = 0;
+  private misses = 0;
+  private evictions = 0;
+  private bytesAdded = 0;
+
+  constructor(rootDir: string, maxBytes: number) {
+    this.rootDir = rootDir;
+    this.maxBytes = maxBytes;
+    if (this.maxBytes > 0) {
+      mkdirSync(this.rootDir, { recursive: true });
+      this.loadIndex();
+    }
+  }
+
+  private loadIndex(): void {
+    if (!existsSync(this.rootDir)) return;
+    const files: Array<{ key: string; path: string; size: number; mtimeMs: number }> = [];
+    const walk = (dir: string) => {
+      for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+          walk(full);
+        } else if (entry.isFile()) {
+          const stat = statSync(full);
+          const key = relative(this.rootDir, full);
+          files.push({ key, path: full, size: stat.size, mtimeMs: stat.mtimeMs });
+        }
+      }
+    };
+    walk(this.rootDir);
+    files.sort((a, b) => a.mtimeMs - b.mtimeMs);
+    for (const f of files) {
+      this.entries.set(f.key, { path: f.path, size: f.size });
+      this.totalBytes += f.size;
+    }
+    this.evictIfNeeded(0);
+  }
+
+  getPath(objectKey: string): string {
+    return join(this.rootDir, objectKey);
+  }
+
+  has(objectKey: string): boolean {
+    const exists = this.entries.has(objectKey) && existsSync(this.getPath(objectKey));
+    if (!exists) this.entries.delete(objectKey);
+    return exists;
+  }
+
+  touch(objectKey: string): void {
+    const entry = this.entries.get(objectKey);
+    if (!entry) return;
+    this.entries.delete(objectKey);
+    this.entries.set(objectKey, entry);
+  }
+
+  recordHit(): void {
+    this.hits += 1;
+  }
+
+  recordMiss(): void {
+    this.misses += 1;
+  }
+
+  get(objectKey: string): Uint8Array | null {
+    if (!this.has(objectKey)) {
+      this.recordMiss();
+      return null;
+    }
+    this.recordHit();
+    this.touch(objectKey);
+    const path = this.getPath(objectKey);
+    return new Uint8Array(readFileSync(path));
+  }
+
+  put(objectKey: string, bytes: Uint8Array): boolean {
+    if (this.maxBytes <= 0) return false;
+    const sizeBytes = bytes.byteLength;
+    if (sizeBytes > this.maxBytes) return false;
+    this.evictIfNeeded(sizeBytes);
+    const dest = this.getPath(objectKey);
+    mkdirSync(dirname(dest), { recursive: true });
+    const tmp = `${dest}.tmp-${Date.now()}`;
+    try {
+      writeFileSync(tmp, bytes);
+      renameSync(tmp, dest);
+    } catch {
+      try {
+        unlinkSync(tmp);
+      } catch {
+        // ignore
+      }
+      return false;
+    }
+    const existing = this.entries.get(objectKey);
+    if (existing) this.totalBytes = Math.max(0, this.totalBytes - existing.size);
+    this.entries.set(objectKey, { path: dest, size: sizeBytes });
+    this.totalBytes += sizeBytes;
+    this.bytesAdded += sizeBytes;
+    return true;
+  }
+
+  putFromLocal(objectKey: string, localPath: string, sizeBytes: number): boolean {
+    if (this.maxBytes <= 0) return false;
+    if (sizeBytes > this.maxBytes) return false;
+    this.evictIfNeeded(sizeBytes);
+    const dest = this.getPath(objectKey);
+    mkdirSync(dirname(dest), { recursive: true });
+    try {
+      renameSync(localPath, dest);
+    } catch {
+      return false;
+    }
+    const existing = this.entries.get(objectKey);
+    if (existing) this.totalBytes = Math.max(0, this.totalBytes - existing.size);
+    this.entries.set(objectKey, { path: dest, size: sizeBytes });
+    this.totalBytes += sizeBytes;
+    this.bytesAdded += sizeBytes;
+    return true;
+  }
+
+  remove(objectKey: string): void {
+    const entry = this.entries.get(objectKey);
+    if (!entry) return;
+    try {
+      unlinkSync(entry.path);
+    } catch {
+      // ignore
+    }
+    this.totalBytes = Math.max(0, this.totalBytes - entry.size);
+    this.entries.delete(objectKey);
+  }
+
+  private evictIfNeeded(incomingBytes: number): void {
+    while (this.totalBytes + incomingBytes > this.maxBytes && this.entries.size > 0) {
+      const oldestKey = this.entries.keys().next().value as string;
+      const entry = this.entries.get(oldestKey);
+      if (entry) {
+        try {
+          unlinkSync(entry.path);
+        } catch {
+          // ignore
+        }
+        this.totalBytes = Math.max(0, this.totalBytes - entry.size);
+        this.evictions += 1;
+      }
+      this.entries.delete(oldestKey);
+    }
+  }
+
+  stats(): SegmentCacheStats {
+    return {
+      hits: this.hits,
+      misses: this.misses,
+      evictions: this.evictions,
+      bytesAdded: this.bytesAdded,
+      usedBytes: this.totalBytes,
+      maxBytes: this.maxBytes,
+      entryCount: this.entries.size,
+    };
+  }
+}