@prisma/streams-server 0.1.1 → 0.1.3

package/src/runtime/host_runtime.ts

@@ -0,0 +1,5 @@
+export type HostRuntime = "bun" | "node";
+
+export function detectHostRuntime(): HostRuntime {
+  return typeof (globalThis as any).Bun !== "undefined" || Boolean(process.versions?.bun) ? "bun" : "node";
+}

package/src/runtime_memory.ts

@@ -0,0 +1,200 @@
+import { readFileSync } from "node:fs";
+
+export type RuntimeMemorySubsystemGroups = {
+  heap_estimates: Record<string, number>;
+  mapped_files: Record<string, number>;
+  disk_caches: Record<string, number>;
+  configured_budgets: Record<string, number>;
+  pipeline_buffers: Record<string, number>;
+  sqlite_runtime: Record<string, number>;
+  counts: Record<string, number>;
+  [kind: string]: Record<string, number>;
+};
+
+export type RuntimeMemoryProcessUsage = {
+  rss_bytes: number;
+  heap_total_bytes: number;
+  heap_used_bytes: number;
+  external_bytes: number;
+  array_buffers_bytes: number;
+};
+
+export type RuntimeMemoryProcessBreakdown = {
+  source: "linux_proc_status" | "process_memory_usage_only";
+  rss_anon_bytes: number | null;
+  rss_file_bytes: number | null;
+  rss_shmem_bytes: number | null;
+  js_managed_bytes: number;
+  js_external_non_array_buffers_bytes: number;
+  mapped_file_bytes: number;
+  sqlite_runtime_bytes: number;
+  unattributed_anon_bytes: number | null;
+  unattributed_rss_bytes: number;
+};
+
+export type SqliteRuntimeMemoryStats = {
+  available: boolean;
+  source: "sqlite3_status64" | "unavailable";
+  memory_used_bytes: number;
+  memory_highwater_bytes: number;
+  pagecache_used_slots: number;
+  pagecache_used_slots_highwater: number;
+  pagecache_overflow_bytes: number;
+  pagecache_overflow_highwater_bytes: number;
+  malloc_count: number;
+  malloc_count_highwater: number;
+  open_connections: number;
+  prepared_statements: number;
+};
+
+export type RuntimeGcStats = {
+  forced_gc_count: number;
+  forced_gc_reclaimed_bytes_total: number;
+  last_forced_gc_at_ms: number | null;
+  last_forced_gc_before_bytes: number | null;
+  last_forced_gc_after_bytes: number | null;
+  last_forced_gc_reclaimed_bytes: number | null;
+  heap_snapshots_written: number;
+  last_heap_snapshot_at_ms: number | null;
+};
+
+export type RuntimeMemoryTotals = {
+  heap_estimate_bytes: number;
+  mapped_file_bytes: number;
+  disk_cache_bytes: number;
+  configured_budget_bytes: number;
+  pipeline_buffer_bytes: number;
+  sqlite_runtime_bytes: number;
+};
+
+export type RuntimeMemorySubsystemSnapshot = {
+  subsystems: RuntimeMemorySubsystemGroups;
+  totals: RuntimeMemoryTotals;
+};
+
+export type RuntimeMemorySnapshot = {
+  process: RuntimeMemoryProcessUsage;
+  process_breakdown: RuntimeMemoryProcessBreakdown;
+  sqlite: SqliteRuntimeMemoryStats;
+  gc: RuntimeGcStats;
+} & RuntimeMemorySubsystemSnapshot;
+
+export type RuntimeHighWaterMark = {
+  value: number;
+  at: string | null;
+};
+
+export type RuntimeMemoryHighWaterSnapshot = {
+  process: Record<string, RuntimeHighWaterMark>;
+  process_breakdown: Record<string, RuntimeHighWaterMark>;
+  sqlite: Record<string, RuntimeHighWaterMark>;
+  runtime_bytes: Record<string, Record<string, RuntimeHighWaterMark>>;
+  runtime_totals: Record<string, RuntimeHighWaterMark>;
+};
+
+export type LinuxStatusRssBreakdown = {
+  rss_anon_bytes: number;
+  rss_file_bytes: number;
+  rss_shmem_bytes: number;
+};
+
+type CachedLinuxStatusBreakdown = {
+  at_ms: number;
+  pid: number;
+  value: LinuxStatusRssBreakdown | null;
+};
+
+let cachedLinuxStatusBreakdown: CachedLinuxStatusBreakdown | null = null;
+
+export function sumRuntimeMemoryValues(values: Record<string, number>): number {
+  let total = 0;
+  for (const value of Object.values(values)) total += Math.max(0, Math.floor(value));
+  return total;
+}
+
+export function parseLinuxStatusRssBreakdown(status: string): LinuxStatusRssBreakdown | null {
+  let anon: number | null = null;
+  let file: number | null = null;
+  let shmem: number | null = null;
+  for (const line of status.split(/\r?\n/)) {
+    let match = line.match(/^RssAnon:\s+([0-9]+)\s+kB$/i);
+    if (match) {
+      anon = Number(match[1]) * 1024;
+      continue;
+    }
+    match = line.match(/^RssFile:\s+([0-9]+)\s+kB$/i);
+    if (match) {
+      file = Number(match[1]) * 1024;
+      continue;
+    }
+    match = line.match(/^RssShmem:\s+([0-9]+)\s+kB$/i);
+    if (match) {
+      shmem = Number(match[1]) * 1024;
+    }
+  }
+  if (anon == null || file == null || shmem == null) return null;
+  return {
+    rss_anon_bytes: Math.max(0, Math.floor(anon)),
+    rss_file_bytes: Math.max(0, Math.floor(file)),
+    rss_shmem_bytes: Math.max(0, Math.floor(shmem)),
+  };
+}
+
+export function readLinuxStatusRssBreakdown(ttlMs = 1_000): LinuxStatusRssBreakdown | null {
+  if (process.platform !== "linux") return null;
+  const now = Date.now();
+  if (
+    cachedLinuxStatusBreakdown &&
+    cachedLinuxStatusBreakdown.pid === process.pid &&
+    now - cachedLinuxStatusBreakdown.at_ms < Math.max(0, ttlMs)
+  ) {
+    return cachedLinuxStatusBreakdown.value;
+  }
+  let value: LinuxStatusRssBreakdown | null = null;
+  try {
+    value = parseLinuxStatusRssBreakdown(readFileSync("/proc/self/status", "utf8"));
+  } catch {
+    value = null;
+  }
+  cachedLinuxStatusBreakdown = {
+    at_ms: now,
+    pid: process.pid,
+    value,
+  };
+  return value;
+}
+
+export function buildProcessMemoryBreakdown(args: {
+  process: RuntimeMemoryProcessUsage;
+  mappedFileBytes: number;
+  sqliteRuntimeBytes: number;
+}): RuntimeMemoryProcessBreakdown {
+  const rssBreakdown = readLinuxStatusRssBreakdown();
+  const jsManagedBytes = Math.max(0, Math.floor(args.process.heap_used_bytes + args.process.external_bytes));
+  const jsExternalNonArrayBuffersBytes = Math.max(
+    0,
+    Math.floor(args.process.external_bytes - args.process.array_buffers_bytes)
+  );
+  const mappedFileBytes = Math.max(0, Math.floor(args.mappedFileBytes));
+  const sqliteRuntimeBytes = Math.max(0, Math.floor(args.sqliteRuntimeBytes));
+  const unattributedRssBytes = Math.max(
+    0,
+    Math.floor(args.process.rss_bytes - jsManagedBytes - mappedFileBytes - sqliteRuntimeBytes)
+  );
+  const unattributedAnonBytes =
+    rssBreakdown == null
+      ? null
+      : Math.max(0, Math.floor(rssBreakdown.rss_anon_bytes - jsManagedBytes - sqliteRuntimeBytes));
+  return {
+    source: rssBreakdown ? "linux_proc_status" : "process_memory_usage_only",
+    rss_anon_bytes: rssBreakdown?.rss_anon_bytes ?? null,
+    rss_file_bytes: rssBreakdown?.rss_file_bytes ?? null,
+    rss_shmem_bytes: rssBreakdown?.rss_shmem_bytes ?? null,
+    js_managed_bytes: jsManagedBytes,
+    js_external_non_array_buffers_bytes: jsExternalNonArrayBuffersBytes,
+    mapped_file_bytes: mappedFileBytes,
+    sqlite_runtime_bytes: sqliteRuntimeBytes,
+    unattributed_anon_bytes: unattributedAnonBytes,
+    unattributed_rss_bytes: unattributedRssBytes,
+  };
+}

package/src/runtime_memory_sampler.ts

@@ -0,0 +1,235 @@
+import { mkdirSync, createWriteStream, type WriteStream } from "node:fs";
+import { dirname, extname } from "node:path";
+import { isMainThread, threadId } from "node:worker_threads";
+
+type BunJscModule = {
+  heapStats?: () => unknown;
+  memoryUsage?: () => unknown;
+};
+
+type PhaseMeta = Record<string, unknown>;
+
+type ActivePhase = {
+  key: number;
+  label: string;
+  startedAtMs: number;
+  meta: PhaseMeta;
+};
+
+function isPromiseLike<T>(value: unknown): value is PromiseLike<T> {
+  return typeof value === "object" && value != null && "then" in value && typeof (value as { then?: unknown }).then === "function";
+}
+
+function insertScopeSuffix(path: string, scope: string): string {
+  if (scope === "main") return path;
+  const extension = extname(path);
+  if (extension === "") return `${path}.${scope}`;
+  return `${path.slice(0, -extension.length)}.${scope}${extension}`;
+}
+
+export class RuntimeMemorySampler {
+  private readonly outputPath: string;
+  private readonly intervalMs: number;
+  private readonly scope: string;
+  private stream: WriteStream | null = null;
+  private timer: any | null = null;
+  private activePhases = new Map<number, ActivePhase>();
+  private nextPhaseKey = 0;
+  private jscModule: BunJscModule | null | undefined;
+  private sampling = false;
+  private subsystemProvider: (() => unknown) | null = null;
+
+  constructor(path: string, opts: { intervalMs?: number; scope?: string } = {}) {
+    this.scope = opts.scope?.trim() || "main";
+    this.outputPath = insertScopeSuffix(path, this.scope);
+    this.intervalMs = Math.max(100, opts.intervalMs ?? 1_000);
+  }
+
+  start(): void {
+    if (this.stream) return;
+    mkdirSync(dirname(this.outputPath), { recursive: true });
+    this.stream = createWriteStream(this.outputPath, { flags: "a" });
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "sampler_start",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      is_main_thread: isMainThread,
+    });
+    void this.sample("startup");
+    this.timer = setInterval(() => {
+      void this.sample("interval");
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    void this.sample("shutdown");
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "sampler_stop",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      is_main_thread: isMainThread,
+    });
+    this.stream?.end();
+    this.stream = null;
+  }
+
+  enter(label: string, meta: PhaseMeta = {}): () => void {
+    if (!this.stream) return () => {};
+    const key = ++this.nextPhaseKey;
+    const phase: ActivePhase = {
+      key,
+      label,
+      startedAtMs: Date.now(),
+      meta,
+    };
+    this.activePhases.set(key, phase);
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "phase_enter",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      label,
+      meta,
+      active_phase_count: this.activePhases.size,
+    });
+    void this.sample("phase_enter", { label, meta });
+    return () => {
+      const current = this.activePhases.get(key);
+      if (!current) return;
+      this.activePhases.delete(key);
+      this.writeLine({
+        ts: new Date().toISOString(),
+        kind: "phase_exit",
+        scope: this.scope,
+        pid: process.pid,
+        thread_id: threadId,
+        label: current.label,
+        meta: current.meta,
+        duration_ms: Date.now() - current.startedAtMs,
+        active_phase_count: this.activePhases.size,
+      });
+      void this.sample("phase_exit", { label: current.label, meta: current.meta });
+    };
+  }
+
+  capture(reason: string, data: PhaseMeta = {}): void {
+    if (!this.stream) return;
+    this.writeLine({
+      ts: new Date().toISOString(),
+      kind: "marker",
+      scope: this.scope,
+      pid: process.pid,
+      thread_id: threadId,
+      reason,
+      data,
+    });
+    void this.sample(reason, data);
+  }
+
+  setSubsystemProvider(provider: (() => unknown) | null): void {
+    this.subsystemProvider = provider;
+  }
+
+  track<T>(label: string, meta: PhaseMeta, fn: () => T): T {
+    const leave = this.enter(label, meta);
+    try {
+      const result = fn();
+      if (isPromiseLike(result)) {
+        return (Promise.resolve(result).finally(() => leave()) as unknown) as T;
+      }
+      leave();
+      return result;
+    } catch (error) {
+      leave();
+      throw error;
+    }
+  }
+
+  private async sample(reason: string, data: PhaseMeta = {}): Promise<void> {
+    if (!this.stream || this.sampling) return;
+    this.sampling = true;
+    try {
+      const activePhases = Array.from(this.activePhases.values()).map((phase) => ({
+        key: phase.key,
+        label: phase.label,
+        started_at: new Date(phase.startedAtMs).toISOString(),
+        duration_ms: Date.now() - phase.startedAtMs,
+        meta: phase.meta,
+      }));
+      const primary = activePhases.length > 0 ? activePhases[activePhases.length - 1]! : null;
+      this.writeLine({
+        ts: new Date().toISOString(),
+        kind: "sample",
+        scope: this.scope,
+        pid: process.pid,
+        thread_id: threadId,
+        is_main_thread: isMainThread,
+        reason,
+        data,
+        process_memory_usage: process.memoryUsage(),
+        jsc_heap_stats: await this.readJscHeapStats(),
+        jsc_memory_usage: await this.readJscMemoryUsage(),
+        memory_subsystems: this.readSubsystems(),
+        active_phases: activePhases,
+        primary_phase: primary ? { label: primary.label, duration_ms: primary.duration_ms, meta: primary.meta } : null,
+      });
+    } finally {
+      this.sampling = false;
+    }
+  }
+
+  private async readJscHeapStats(): Promise<unknown> {
+    const jsc = await this.loadJscModule();
+    if (!jsc || typeof jsc.heapStats !== "function") return null;
+    try {
+      return jsc.heapStats();
+    } catch {
+      return null;
+    }
+  }
+
+  private async readJscMemoryUsage(): Promise<unknown> {
+    const jsc = await this.loadJscModule();
+    if (!jsc || typeof jsc.memoryUsage !== "function") return null;
+    try {
+      return jsc.memoryUsage();
+    } catch {
+      return null;
+    }
+  }
+
+  private async loadJscModule(): Promise<BunJscModule | null> {
+    if (this.jscModule !== undefined) return this.jscModule;
+    if (typeof (globalThis as { Bun?: unknown }).Bun === "undefined") {
+      this.jscModule = null;
+      return this.jscModule;
+    }
+    try {
+      this.jscModule = (await import("bun:jsc")) as BunJscModule;
+    } catch {
+      this.jscModule = null;
+    }
+    return this.jscModule;
+  }
+
+  private writeLine(payload: Record<string, unknown>): void {
+    if (!this.stream) return;
+    this.stream.write(`${JSON.stringify(payload)}\n`);
+  }
+
+  private readSubsystems(): unknown {
+    if (!this.subsystemProvider) return null;
+    try {
+      return this.subsystemProvider();
+    } catch {
+      return null;
+    }
+  }
+}

package/src/schema/read_json.ts

@@ -0,0 +1,43 @@
+import { Result } from "better-result";
+import { applyLensChainResult } from "../lens/lens";
+import type { SchemaRegistry } from "./registry";
+import { SchemaRegistryStore } from "./registry";
+
+export function schemaVersionForOffset(reg: SchemaRegistry, offset: bigint): number {
+  if (!reg.boundaries || reg.boundaries.length === 0) return 0;
+  const off = Number(offset);
+  let version = 0;
+  for (const boundary of reg.boundaries) {
+    if (boundary.offset <= off) version = boundary.version;
+    else break;
+  }
+  return version;
+}
+
+export function decodeJsonPayloadResult(
+  registry: SchemaRegistryStore,
+  stream: string,
+  offset: bigint,
+  payload: Uint8Array
+): Result<any, { status: 400 | 500; message: string }> {
+  const regRes = registry.getRegistryResult(stream);
+  if (Result.isError(regRes)) return Result.err({ status: 500, message: regRes.error.message });
+  const reg = regRes.value;
+  try {
+    const text = new TextDecoder().decode(payload);
+    let value: any = JSON.parse(text);
+    if (reg.currentVersion > 0) {
+      const version = schemaVersionForOffset(reg, offset);
+      if (version < reg.currentVersion) {
+        const chainRes = registry.getLensChainResult(reg, version, reg.currentVersion);
+        if (Result.isError(chainRes)) return Result.err({ status: 500, message: chainRes.error.message });
+        const transformedRes = applyLensChainResult(chainRes.value, value);
+        if (Result.isError(transformedRes)) return Result.err({ status: 400, message: transformedRes.error.message });
+        value = transformedRes.value;
+      }
+    }
+    return Result.ok(value);
+  } catch (e: unknown) {
+    return Result.err({ status: 400, message: String((e as any)?.message ?? e) });
+  }
+}