@prisma/streams-server 0.1.1 → 0.1.3

package/src/metrics.ts

@@ -1,26 +1,6 @@
-type Tags = Record<string, string>;
+import { buildInternalMetricsRecord } from "./profiles/metrics/normalize";
 
-type MetricEvent = {
-  apiVersion: "durable.streams/metrics/v1";
-  kind: "interval";
-  metric: string;
-  unit: "ns" | "bytes" | "count";
-  windowStart: number;
-  windowEnd: number;
-  intervalMs: number;
-  instance: string;
-  stream?: string;
-  tags?: Tags;
-  count: number;
-  sum: number;
-  min: number;
-  max: number;
-  avg: number;
-  p50: number;
-  p95: number;
-  p99: number;
-  buckets: Record<string, number>;
-};
+type Tags = Record<string, string>;
 
 class Histogram {
   private readonly maxSamples: number;
@@ -41,7 +21,7 @@ class Histogram {
     if (value < this.min) this.min = value;
     if (value > this.max) this.max = value;
     const bucket = Math.floor(Math.log2(Math.max(1, value)));
-    const key = String(1 << bucket);
+    const key = String(2 ** bucket);
     this.buckets[key] = (this.buckets[key] ?? 0) + 1;
     if (this.samples.length < this.maxSamples) {
       this.samples.push(value);
@@ -75,6 +55,10 @@ class Histogram {
 
 type SeriesKey = string;
 
+export type MetricsMemoryStats = {
+  seriesCount: number;
+};
+
 class MetricSeries {
   readonly metric: string;
   readonly unit: "ns" | "bytes" | "count";
@@ -134,16 +118,18 @@ export class Metrics {
     };
   }
 
-  flushInterval(): MetricEvent[] {
+  getMemoryStats(): MetricsMemoryStats {
+    return { seriesCount: this.series.size };
+  }
+
+  flushInterval(): Record<string, unknown>[] {
     const windowEnd = Date.now();
     const intervalMs = windowEnd - this.windowStartMs;
-    const events: MetricEvent[] = [];
+    const events: Record<string, unknown>[] = [];
     for (const s of this.series.values()) {
       const snap = s.hist.snapshotAndReset();
       if (snap.count === 0) continue;
-      events.push({
-        apiVersion: "durable.streams/metrics/v1",
-        kind: "interval",
+      events.push(buildInternalMetricsRecord({
         metric: s.metric,
         unit: s.unit,
         windowStart: this.windowStartMs,
@@ -153,7 +139,7 @@ export class Metrics {
         stream: s.stream,
         tags: s.tags,
         ...snap,
-      });
+      }));
     }
     this.windowStartMs = windowEnd;
     return events;

package/src/metrics_emitter.ts

@@ -1,3 +1,4 @@
+import { Result } from "better-result";
 import type { IngestQueue } from "./ingest";
 import type { Metrics } from "./metrics";
 
@@ -5,12 +6,27 @@ export class MetricsEmitter {
   private readonly metrics: Metrics;
   private readonly ingest: IngestQueue;
   private readonly intervalMs: number;
+  private readonly onAppended?: (args: {
+    lastOffset: bigint;
+    stream: string;
+  }) => void;
+  private readonly collectRuntimeMetrics?: () => void;
   private timer: any | null = null;
 
-  constructor(metrics: Metrics, ingest: IngestQueue, intervalMs: number) {
+  constructor(
+    metrics: Metrics,
+    ingest: IngestQueue,
+    intervalMs: number,
+    opts?: {
+      onAppended?: (args: { lastOffset: bigint; stream: string }) => void;
+      collectRuntimeMetrics?: () => void;
+    },
+  ) {
     this.metrics = metrics;
     this.ingest = ingest;
     this.intervalMs = intervalMs;
+    this.onAppended = opts?.onAppended;
+    this.collectRuntimeMetrics = opts?.collectRuntimeMetrics;
   }
 
   start(): void {
@@ -29,20 +45,27 @@ export class MetricsEmitter {
     const queue = this.ingest.getQueueStats();
     this.metrics.record("tieredstore.ingest.queue.bytes", queue.bytes, "bytes");
     this.metrics.record("tieredstore.ingest.queue.requests", queue.requests, "count");
+    this.collectRuntimeMetrics?.();
     const events = this.metrics.flushInterval();
     if (events.length === 0) return;
     const rows = events.map((e) => ({
-      routingKey: e.stream ? new TextEncoder().encode(e.stream) : null,
+      routingKey: typeof e.seriesKey === "string" ? new TextEncoder().encode(e.seriesKey) : null,
       contentType: "application/json",
       payload: new TextEncoder().encode(JSON.stringify(e)),
     }));
     try {
-      await this.ingest.appendInternal({
+      const appendRes = await this.ingest.appendInternal({
         stream: "__stream_metrics__",
         baseAppendMs: BigInt(Date.now()),
         rows,
         contentType: "application/json",
       });
+      if (!Result.isError(appendRes)) {
+        this.onAppended?.({
+          lastOffset: appendRes.value.lastOffset,
+          stream: "__stream_metrics__",
+        });
+      }
     } catch {
       // best-effort; drop on failure
     }

package/src/notifier.ts

@@ -1,8 +1,27 @@
 type Waiter = { afterSeq: bigint; resolve: () => void };
+type DetailsWaiter = { afterVersion: bigint; resolve: () => void };
+
+export type StreamNotifierMemoryStats = {
+  waiterStreams: number;
+  waiters: number;
+  latestSeqStreams: number;
+  detailsWaiterStreams: number;
+  detailsWaiters: number;
+  detailsVersionStreams: number;
+};
+
+export type StreamNotifierTopStreamEntry = {
+  stream: string;
+  waiters: number;
+  details_waiters: number;
+  total_waiters: number;
+};
 
 export class StreamNotifier {
   private readonly waiters = new Map<string, Set<Waiter>>();
   private readonly latestSeq = new Map<string, bigint>();
+  private readonly detailsWaiters = new Map<string, Set<DetailsWaiter>>();
+  private readonly detailsVersion = new Map<string, bigint>();
 
   notify(stream: string, newEndSeq: bigint): void {
     this.latestSeq.set(stream, newEndSeq);
@@ -52,13 +71,110 @@ export class StreamNotifier {
     });
   }
 
-  notifyClose(stream: string): void {
-    const set = this.waiters.get(stream);
+  currentDetailsVersion(stream: string): bigint {
+    return this.detailsVersion.get(stream) ?? 0n;
+  }
+
+  notifyDetailsChanged(stream: string): void {
+    const nextVersion = (this.detailsVersion.get(stream) ?? 0n) + 1n;
+    this.detailsVersion.set(stream, nextVersion);
+    const set = this.detailsWaiters.get(stream);
     if (!set || set.size === 0) return;
     for (const w of Array.from(set)) {
-      set.delete(w);
-      w.resolve();
+      if (nextVersion > w.afterVersion) {
+        set.delete(w);
+        w.resolve();
+      }
     }
-    if (set.size === 0) this.waiters.delete(stream);
+    if (set.size === 0) this.detailsWaiters.delete(stream);
+  }
+
+  waitForDetailsChange(stream: string, afterVersion: bigint, timeoutMs: number, signal?: AbortSignal): Promise<void> {
+    if (signal?.aborted) return Promise.resolve();
+    const latest = this.detailsVersion.get(stream);
+    if (latest != null && latest > afterVersion) return Promise.resolve();
+    return new Promise((resolve) => {
+      let done = false;
+      const set = this.detailsWaiters.get(stream) ?? new Set();
+      const cleanup = () => {
+        if (done) return;
+        done = true;
+        const s = this.detailsWaiters.get(stream);
+        if (s) {
+          s.delete(waiter);
+          if (s.size === 0) this.detailsWaiters.delete(stream);
+        }
+        if (timeoutId) clearTimeout(timeoutId);
+        if (signal) signal.removeEventListener("abort", onAbort);
+        resolve();
+      };
+      const waiter: DetailsWaiter = { afterVersion, resolve: cleanup };
+      set.add(waiter);
+      this.detailsWaiters.set(stream, set);
+
+      const onAbort = () => cleanup();
+      if (signal) signal.addEventListener("abort", onAbort, { once: true });
+
+      let timeoutId: any | null = null;
+      if (timeoutMs > 0) {
+        timeoutId = setTimeout(() => {
+          cleanup();
+        }, timeoutMs);
+      }
+    });
+  }
+
+  notifyClose(stream: string): void {
+    const set = this.waiters.get(stream);
+    if (set && set.size > 0) {
+      for (const w of Array.from(set)) {
+        set.delete(w);
+        w.resolve();
+      }
+      if (set.size === 0) this.waiters.delete(stream);
+    }
+
+    const detailsSet = this.detailsWaiters.get(stream);
+    if (detailsSet && detailsSet.size > 0) {
+      for (const w of Array.from(detailsSet)) {
+        detailsSet.delete(w);
+        w.resolve();
+      }
+      if (detailsSet.size === 0) this.detailsWaiters.delete(stream);
+    }
+  }
+
+  getMemoryStats(): StreamNotifierMemoryStats {
+    let waiters = 0;
+    for (const set of this.waiters.values()) waiters += set.size;
+    let detailsWaiters = 0;
+    for (const set of this.detailsWaiters.values()) detailsWaiters += set.size;
+    return {
+      waiterStreams: this.waiters.size,
+      waiters,
+      latestSeqStreams: this.latestSeq.size,
+      detailsWaiterStreams: this.detailsWaiters.size,
+      detailsWaiters,
+      detailsVersionStreams: this.detailsVersion.size,
+    };
+  }
+
+  getTopStreams(limit = 5): StreamNotifierTopStreamEntry[] {
+    const totals = new Map<string, StreamNotifierTopStreamEntry>();
+    for (const [stream, waiters] of this.waiters) {
+      const row = totals.get(stream) ?? { stream, waiters: 0, details_waiters: 0, total_waiters: 0 };
+      row.waiters = waiters.size;
+      row.total_waiters = row.waiters + row.details_waiters;
+      totals.set(stream, row);
+    }
+    for (const [stream, detailsWaiters] of this.detailsWaiters) {
+      const row = totals.get(stream) ?? { stream, waiters: 0, details_waiters: 0, total_waiters: 0 };
+      row.details_waiters = detailsWaiters.size;
+      row.total_waiters = row.waiters + row.details_waiters;
+      totals.set(stream, row);
+    }
+    return Array.from(totals.values())
+      .sort((a, b) => b.total_waiters - a.total_waiters || a.stream.localeCompare(b.stream))
+      .slice(0, Math.max(0, Math.floor(limit)));
   }
 }

package/src/objectstore/accounting.ts

@@ -0,0 +1,92 @@
+import type { SqliteDurableStore } from "../db/db";
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+
+type ClassifiedRequest = {
+  streamHash: string;
+  artifact: string;
+};
+
+function classifyKey(key: string): ClassifiedRequest | null {
+  const match = /^streams\/([0-9a-f]{32})\/(.+)$/.exec(key);
+  if (!match) return null;
+  const [, streamHash, rest] = match;
+  if (rest === "manifest.json") return { streamHash, artifact: "manifest" };
+  if (rest === "schema-registry.json") return { streamHash, artifact: "schema_registry" };
+  if (rest.startsWith("index/")) return { streamHash, artifact: "routing_index" };
+  if (rest.startsWith("lexicon/")) return { streamHash, artifact: "routing_key_lexicon" };
+  if (rest.startsWith("secondary-index/")) return { streamHash, artifact: "exact_index" };
+  if (rest.startsWith("segments/") && rest.endsWith(".bin")) return { streamHash, artifact: "segment" };
+  if (rest.startsWith("segments/") && rest.endsWith(".cix")) return { streamHash, artifact: "bundled_companion" };
+  return { streamHash, artifact: "meta" };
+}
+
+function classifyListPrefix(prefix: string): ClassifiedRequest | null {
+  const exact = classifyKey(prefix.replace(/\/+$/, ""));
+  if (exact) return exact;
+  const match = /^streams\/([0-9a-f]{32})(?:\/(.+))?\/?$/.exec(prefix);
+  if (!match) return null;
+  const [, streamHash, rest = ""] = match;
+  if (rest === "" || rest === "segments") return { streamHash, artifact: "segment" };
+  if (rest === "index") return { streamHash, artifact: "routing_index" };
+  if (rest.startsWith("lexicon")) return { streamHash, artifact: "routing_key_lexicon" };
+  if (rest.startsWith("secondary-index")) return { streamHash, artifact: "exact_index" };
+  return { streamHash, artifact: "meta" };
+}
+
+export class AccountingObjectStore implements ObjectStore {
+  constructor(
+    private readonly inner: ObjectStore,
+    private readonly db: SqliteDurableStore
+  ) {}
+
+  async put(key: string, data: Uint8Array, opts?: { contentType?: string; contentLength?: number }): Promise<PutResult> {
+    const res = await this.inner.put(key, data, opts);
+    const classified = classifyKey(key);
+    if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "put", data.byteLength);
+    return res;
+  }
+
+  async putFile(key: string, path: string, size: number, opts?: { contentType?: string }): Promise<PutResult> {
+    if (!this.inner.putFile) {
+      const bytes = await Bun.file(path).bytes();
+      const res = await this.inner.put(key, bytes, {
+        contentType: opts?.contentType,
+        contentLength: size,
+      });
+      const classified = classifyKey(key);
+      if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "put", size);
+      return res;
+    }
+    const res = await this.inner.putFile(key, path, size, opts);
+    const classified = classifyKey(key);
+    if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "put", size);
+    return res;
+  }
+
+  async get(key: string, opts?: GetOptions): Promise<Uint8Array | null> {
+    const res = await this.inner.get(key, opts);
+    const classified = classifyKey(key);
+    if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "get", res?.byteLength ?? 0);
+    return res;
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    const res = await this.inner.head(key);
+    const classified = classifyKey(key);
+    if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "head", res?.size ?? 0);
+    return res;
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.inner.delete(key);
+    const classified = classifyKey(key);
+    if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "delete", 0);
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    const res = await this.inner.list(prefix);
+    const classified = classifyListPrefix(prefix);
+    if (classified) this.db.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "list", 0);
+    return res;
+  }
+}

package/src/objectstore/mock_r2.ts

@@ -180,7 +180,7 @@ export class MockR2Store implements ObjectStore {
       out = entry.bytes.slice(start, start + length);
     } else if (entry.path) {
       if (length === total) {
-        out = new Uint8Array(readFileSync(entry.path));
+        out = readFileSync(entry.path);
       } else {
         const fd = openSync(entry.path, "r");
         try {

package/src/objectstore/r2.ts

@@ -31,6 +31,22 @@ function stripQuotes(value: string | null): string {
   return value.replace(/^\"|\"$/g, "");
 }
 
+function isMissingObjectError(err: unknown): boolean {
+  const record = err as Record<string, unknown> | null | undefined;
+  const status = record?.status;
+  const statusCode = record?.statusCode;
+  const code = String(record?.code ?? "");
+  const message = String(record?.message ?? err ?? "").toLowerCase();
+  if (status === 404 || statusCode === 404) return true;
+  if (code === "NoSuchKey" || code === "NotFound") return true;
+  return (
+    message.includes("not found") ||
+    message.includes("no such key") ||
+    message.includes("does not exist") ||
+    message === "missing"
+  );
+}
+
 export class R2ObjectStore implements ObjectStore {
   private readonly client: Bun.S3Client;
 
@@ -76,13 +92,13 @@ export class R2ObjectStore implements ObjectStore {
   async get(key: string, opts: GetOptions = {}): Promise<Uint8Array | null> {
     try {
       const file = this.file(key);
-      if (!(await file.exists())) return null;
       const body =
         opts.range == null
           ? file
           : file.slice(opts.range.start, opts.range.end == null ? undefined : opts.range.end + 1);
       return new Uint8Array(await body.arrayBuffer());
     } catch (err) {
+      if (isMissingObjectError(err)) return null;
       this.wrapError("GET", key, err);
     }
   }

package/src/profiles/evlog/schema.ts

@@ -0,0 +1,234 @@
+import {
+  SCHEMA_REGISTRY_API_VERSION,
+  type SchemaRegistry,
+  type SearchConfig,
+} from "../../schema/registry";
+
+export const EVLOG_CANONICAL_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    timestamp: { type: "string" },
+    level: { type: "string", enum: ["debug", "info", "warn", "error"] },
+    service: { type: ["string", "null"] },
+    environment: { type: ["string", "null"] },
+    version: { type: ["string", "null"] },
+    region: { type: ["string", "null"] },
+    requestId: { type: ["string", "null"] },
+    traceId: { type: ["string", "null"] },
+    spanId: { type: ["string", "null"] },
+    method: { type: ["string", "null"] },
+    path: { type: ["string", "null"] },
+    status: { type: ["integer", "null"] },
+    duration: { type: ["number", "null"] },
+    message: { type: ["string", "null"] },
+    why: { type: ["string", "null"] },
+    fix: { type: ["string", "null"] },
+    link: { type: ["string", "null"] },
+    sampling: {
+      type: ["object", "null"],
+      additionalProperties: true,
+    },
+    redaction: {
+      type: "object",
+      additionalProperties: false,
+      properties: {
+        keys: {
+          type: "array",
+          items: { type: "string" },
+        },
+      },
+      required: ["keys"],
+    },
+    context: {
+      type: "object",
+      additionalProperties: true,
+    },
+  },
+  required: [
+    "timestamp",
+    "level",
+    "service",
+    "environment",
+    "version",
+    "region",
+    "requestId",
+    "traceId",
+    "spanId",
+    "method",
+    "path",
+    "status",
+    "duration",
+    "message",
+    "why",
+    "fix",
+    "link",
+    "sampling",
+    "redaction",
+    "context",
+  ],
+} as const;
+
+export const EVLOG_DEFAULT_SEARCH_CONFIG: SearchConfig = {
+  profile: "evlog",
+  primaryTimestampField: "timestamp",
+  aliases: {
+    env: "environment",
+    msg: "message",
+    req: "requestId",
+    span: "spanId",
+    time: "timestamp",
+    trace: "traceId",
+    ts: "timestamp",
+  },
+  defaultFields: [
+    { field: "message", boost: 2 },
+    { field: "why", boost: 1.5 },
+    { field: "fix", boost: 1.25 },
+    { field: "error.message", boost: 2 },
+  ],
+  fields: {
+    timestamp: {
+      kind: "date",
+      bindings: [{ version: 1, jsonPointer: "/timestamp" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    level: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/level" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    service: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/service" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    environment: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/environment" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    requestId: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/requestId" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+    },
+    traceId: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/traceId" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+    },
+    spanId: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/spanId" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+    },
+    path: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/path" }],
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    method: {
+      kind: "keyword",
+      bindings: [{ version: 1, jsonPointer: "/method" }],
+      normalizer: "lowercase_v1",
+      exact: true,
+      prefix: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    status: {
+      kind: "integer",
+      bindings: [{ version: 1, jsonPointer: "/status" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    duration: {
+      kind: "float",
+      bindings: [{ version: 1, jsonPointer: "/duration" }],
+      exact: true,
+      column: true,
+      exists: true,
+      sortable: true,
+      aggregatable: true,
+    },
+    message: {
+      kind: "text",
+      bindings: [{ version: 1, jsonPointer: "/message" }],
+      analyzer: "unicode_word_v1",
+      exists: true,
+      positions: true,
+    },
+    why: {
+      kind: "text",
+      bindings: [{ version: 1, jsonPointer: "/why" }],
+      analyzer: "unicode_word_v1",
+      exists: true,
+      positions: true,
+    },
+    fix: {
+      kind: "text",
+      bindings: [{ version: 1, jsonPointer: "/fix" }],
+      analyzer: "unicode_word_v1",
+      exists: true,
+      positions: true,
+    },
+    "error.message": {
+      kind: "text",
+      bindings: [{ version: 1, jsonPointer: "/context/error/message" }],
+      analyzer: "unicode_word_v1",
+      exists: true,
+      positions: true,
+    },
+  },
+};
+
+export function buildEvlogDefaultRegistry(stream: string): SchemaRegistry {
+  return {
+    apiVersion: SCHEMA_REGISTRY_API_VERSION,
+    schema: stream,
+    currentVersion: 1,
+    search: structuredClone(EVLOG_DEFAULT_SEARCH_CONFIG),
+    boundaries: [{ offset: 0, version: 1 }],
+    schemas: {
+      "1": structuredClone(EVLOG_CANONICAL_SCHEMA),
+    },
+    lenses: {},
+  };
+}