@tungthedev/streams-server 0.2.0

package/src/metrics.ts

@@ -0,0 +1,147 @@
+import { buildInternalMetricsRecord } from "./profiles/metrics/normalize";
+
+type Tags = Record<string, string>;
+
+class Histogram {
+  private readonly maxSamples: number;
+  private samples: number[] = [];
+  count = 0;
+  sum = 0;
+  min = Number.POSITIVE_INFINITY;
+  max = Number.NEGATIVE_INFINITY;
+  buckets: Record<string, number> = {};
+
+  constructor(maxSamples = 1024) {
+    this.maxSamples = maxSamples;
+  }
+
+  add(value: number): void {
+    this.count++;
+    this.sum += value;
+    if (value < this.min) this.min = value;
+    if (value > this.max) this.max = value;
+    const bucket = Math.floor(Math.log2(Math.max(1, value)));
+    const key = String(2 ** bucket);
+    this.buckets[key] = (this.buckets[key] ?? 0) + 1;
+    if (this.samples.length < this.maxSamples) {
+      this.samples.push(value);
+    } else {
+      const idx = Math.floor(Math.random() * this.count);
+      if (idx < this.maxSamples) this.samples[idx] = value;
+    }
+  }
+
+  snapshotAndReset(): { count: number; sum: number; min: number; max: number; avg: number; p50: number; p95: number; p99: number; buckets: Record<string, number> } {
+    const count = this.count;
+    const sum = this.sum;
+    const min = count === 0 ? 0 : this.min;
+    const max = count === 0 ? 0 : this.max;
+    const avg = count === 0 ? 0 : sum / count;
+    const sorted = this.samples.slice().sort((a, b) => a - b);
+    const p = (q: number) => (sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)))]);
+    const p50 = p(0.5);
+    const p95 = p(0.95);
+    const p99 = p(0.99);
+    const buckets = { ...this.buckets };
+    this.samples = [];
+    this.count = 0;
+    this.sum = 0;
+    this.min = Number.POSITIVE_INFINITY;
+    this.max = Number.NEGATIVE_INFINITY;
+    this.buckets = {};
+    return { count, sum, min, max, avg, p50, p95, p99, buckets };
+  }
+}
+
+type SeriesKey = string;
+
+export type MetricsMemoryStats = {
+  seriesCount: number;
+};
+
+class MetricSeries {
+  readonly metric: string;
+  readonly unit: "ns" | "bytes" | "count";
+  readonly stream?: string;
+  readonly tags?: Tags;
+  readonly hist = new Histogram();
+
+  constructor(metric: string, unit: "ns" | "bytes" | "count", stream?: string, tags?: Tags) {
+    this.metric = metric;
+    this.unit = unit;
+    this.stream = stream;
+    this.tags = tags;
+  }
+}
+
+function keyFor(metric: string, unit: string, stream?: string, tags?: Tags): SeriesKey {
+  const tagStr = tags ? JSON.stringify(tags) : "";
+  return `${metric}|${unit}|${stream ?? ""}|${tagStr}`;
+}
+
+function instanceId(): string {
+  const host = typeof process !== "undefined" ? process.pid.toString() : "node";
+  const rand = Math.random().toString(16).slice(2, 8);
+  return `${host}-${rand}`;
+}
+
+export class Metrics {
+  private readonly startMs = Date.now();
+  private windowStartMs = Date.now();
+  private readonly series = new Map<SeriesKey, MetricSeries>();
+  private readonly instance = instanceId();
+
+  record(metric: string, value: number, unit: "ns" | "bytes" | "count", tags?: Tags, stream?: string): void {
+    const key = keyFor(metric, unit, stream, tags);
+    let s = this.series.get(key);
+    if (!s) {
+      s = new MetricSeries(metric, unit, stream, tags);
+      this.series.set(key, s);
+    }
+    s.hist.add(value);
+  }
+
+  recordAppend(bytes: number, entries: number): void {
+    this.record("tieredstore.append.bytes", bytes, "bytes");
+    this.record("tieredstore.append.entries", entries, "count");
+  }
+
+  recordRead(bytes: number, entries: number): void {
+    this.record("tieredstore.read.bytes", bytes, "bytes");
+    this.record("tieredstore.read.entries", entries, "count");
+  }
+
+  snapshot(): any {
+    return {
+      uptime_ms: Date.now() - this.startMs,
+      series: this.series.size,
+    };
+  }
+
+  getMemoryStats(): MetricsMemoryStats {
+    return { seriesCount: this.series.size };
+  }
+
+  flushInterval(): Record<string, unknown>[] {
+    const windowEnd = Date.now();
+    const intervalMs = windowEnd - this.windowStartMs;
+    const events: Record<string, unknown>[] = [];
+    for (const s of this.series.values()) {
+      const snap = s.hist.snapshotAndReset();
+      if (snap.count === 0) continue;
+      events.push(buildInternalMetricsRecord({
+        metric: s.metric,
+        unit: s.unit,
+        windowStart: this.windowStartMs,
+        windowEnd,
+        intervalMs,
+        instance: this.instance,
+        stream: s.stream,
+        tags: s.tags,
+        ...snap,
+      }));
+    }
+    this.windowStartMs = windowEnd;
+    return events;
+  }
+}

package/src/metrics_emitter.ts

@@ -0,0 +1,83 @@
+import { Result } from "better-result";
+import type { IngestQueue } from "./ingest";
+import type { Metrics } from "./metrics";
+
+export class MetricsEmitter {
+  private readonly metrics: Metrics;
+  private readonly ingest: IngestQueue;
+  private readonly intervalMs: number;
+  private readonly onAppended?: (args: {
+    lastOffset: bigint;
+    stream: string;
+  }) => void;
+  private readonly collectRuntimeMetrics?: () => void;
+  private timer: any | null = null;
+  private flushPromise: Promise<void> | null = null;
+
+  constructor(
+    metrics: Metrics,
+    ingest: IngestQueue,
+    intervalMs: number,
+    opts?: {
+      onAppended?: (args: { lastOffset: bigint; stream: string }) => void;
+      collectRuntimeMetrics?: () => void;
+    },
+  ) {
+    this.metrics = metrics;
+    this.ingest = ingest;
+    this.intervalMs = intervalMs;
+    this.onAppended = opts?.onAppended;
+    this.collectRuntimeMetrics = opts?.collectRuntimeMetrics;
+  }
+
+  start(): void {
+    if (this.intervalMs <= 0 || this.timer) return;
+    this.timer = setInterval(() => {
+      void this.flush();
+    }, this.intervalMs);
+  }
+
+  async stop(): Promise<void> {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    await this.flushPromise;
+  }
+
+  private async flush(): Promise<void> {
+    if (this.flushPromise) return this.flushPromise;
+    this.flushPromise = this.runFlush().finally(() => {
+      this.flushPromise = null;
+    });
+    return this.flushPromise;
+  }
+
+  private async runFlush(): Promise<void> {
+    const queue = this.ingest.getQueueStats();
+    this.metrics.record("tieredstore.ingest.queue.bytes", queue.bytes, "bytes");
+    this.metrics.record("tieredstore.ingest.queue.requests", queue.requests, "count");
+    this.collectRuntimeMetrics?.();
+    const events = this.metrics.flushInterval();
+    if (events.length === 0) return;
+    const rows = events.map((e) => ({
+      routingKey: typeof e.seriesKey === "string" ? new TextEncoder().encode(e.seriesKey) : null,
+      contentType: "application/json",
+      payload: new TextEncoder().encode(JSON.stringify(e)),
+    }));
+    try {
+      const appendRes = await this.ingest.appendInternal({
+        stream: "__stream_metrics__",
+        baseAppendMs: BigInt(Date.now()),
+        rows,
+        contentType: "application/json",
+      });
+      if (!Result.isError(appendRes)) {
+        this.onAppended?.({
+          lastOffset: appendRes.value.lastOffset,
+          stream: "__stream_metrics__",
+        });
+      }
+    } catch {
+      // best-effort; drop on failure
+    }
+  }
+}

package/src/notifier.ts

@@ -0,0 +1,180 @@
+type Waiter = { afterSeq: bigint; resolve: () => void };
+type DetailsWaiter = { afterVersion: bigint; resolve: () => void };
+
+export type StreamNotifierMemoryStats = {
+  waiterStreams: number;
+  waiters: number;
+  latestSeqStreams: number;
+  detailsWaiterStreams: number;
+  detailsWaiters: number;
+  detailsVersionStreams: number;
+};
+
+export type StreamNotifierTopStreamEntry = {
+  stream: string;
+  waiters: number;
+  details_waiters: number;
+  total_waiters: number;
+};
+
+export class StreamNotifier {
+  private readonly waiters = new Map<string, Set<Waiter>>();
+  private readonly latestSeq = new Map<string, bigint>();
+  private readonly detailsWaiters = new Map<string, Set<DetailsWaiter>>();
+  private readonly detailsVersion = new Map<string, bigint>();
+
+  notify(stream: string, newEndSeq: bigint): void {
+    this.latestSeq.set(stream, newEndSeq);
+    const set = this.waiters.get(stream);
+    if (!set || set.size === 0) return;
+    for (const w of Array.from(set)) {
+      if (newEndSeq > w.afterSeq) {
+        set.delete(w);
+        w.resolve();
+      }
+    }
+    if (set.size === 0) this.waiters.delete(stream);
+  }
+
+  waitFor(stream: string, afterSeq: bigint, timeoutMs: number, signal?: AbortSignal): Promise<void> {
+    if (signal?.aborted) return Promise.resolve();
+    const latest = this.latestSeq.get(stream);
+    if (latest != null && latest > afterSeq) return Promise.resolve();
+    return new Promise((resolve) => {
+      let done = false;
+      const set = this.waiters.get(stream) ?? new Set();
+      const cleanup = () => {
+        if (done) return;
+        done = true;
+        const s = this.waiters.get(stream);
+        if (s) {
+          s.delete(waiter);
+          if (s.size === 0) this.waiters.delete(stream);
+        }
+        if (timeoutId) clearTimeout(timeoutId);
+        if (signal) signal.removeEventListener("abort", onAbort);
+        resolve();
+      };
+      const waiter: Waiter = { afterSeq, resolve: cleanup };
+      set.add(waiter);
+      this.waiters.set(stream, set);
+
+      const onAbort = () => cleanup();
+      if (signal) signal.addEventListener("abort", onAbort, { once: true });
+
+      let timeoutId: any | null = null;
+      if (timeoutMs > 0) {
+        timeoutId = setTimeout(() => {
+          cleanup();
+        }, timeoutMs);
+      }
+    });
+  }
+
+  currentDetailsVersion(stream: string): bigint {
+    return this.detailsVersion.get(stream) ?? 0n;
+  }
+
+  notifyDetailsChanged(stream: string): void {
+    const nextVersion = (this.detailsVersion.get(stream) ?? 0n) + 1n;
+    this.detailsVersion.set(stream, nextVersion);
+    const set = this.detailsWaiters.get(stream);
+    if (!set || set.size === 0) return;
+    for (const w of Array.from(set)) {
+      if (nextVersion > w.afterVersion) {
+        set.delete(w);
+        w.resolve();
+      }
+    }
+    if (set.size === 0) this.detailsWaiters.delete(stream);
+  }
+
+  waitForDetailsChange(stream: string, afterVersion: bigint, timeoutMs: number, signal?: AbortSignal): Promise<void> {
+    if (signal?.aborted) return Promise.resolve();
+    const latest = this.detailsVersion.get(stream);
+    if (latest != null && latest > afterVersion) return Promise.resolve();
+    return new Promise((resolve) => {
+      let done = false;
+      const set = this.detailsWaiters.get(stream) ?? new Set();
+      const cleanup = () => {
+        if (done) return;
+        done = true;
+        const s = this.detailsWaiters.get(stream);
+        if (s) {
+          s.delete(waiter);
+          if (s.size === 0) this.detailsWaiters.delete(stream);
+        }
+        if (timeoutId) clearTimeout(timeoutId);
+        if (signal) signal.removeEventListener("abort", onAbort);
+        resolve();
+      };
+      const waiter: DetailsWaiter = { afterVersion, resolve: cleanup };
+      set.add(waiter);
+      this.detailsWaiters.set(stream, set);
+
+      const onAbort = () => cleanup();
+      if (signal) signal.addEventListener("abort", onAbort, { once: true });
+
+      let timeoutId: any | null = null;
+      if (timeoutMs > 0) {
+        timeoutId = setTimeout(() => {
+          cleanup();
+        }, timeoutMs);
+      }
+    });
+  }
+
+  notifyClose(stream: string): void {
+    const set = this.waiters.get(stream);
+    if (set && set.size > 0) {
+      for (const w of Array.from(set)) {
+        set.delete(w);
+        w.resolve();
+      }
+      if (set.size === 0) this.waiters.delete(stream);
+    }
+
+    const detailsSet = this.detailsWaiters.get(stream);
+    if (detailsSet && detailsSet.size > 0) {
+      for (const w of Array.from(detailsSet)) {
+        detailsSet.delete(w);
+        w.resolve();
+      }
+      if (detailsSet.size === 0) this.detailsWaiters.delete(stream);
+    }
+  }
+
+  getMemoryStats(): StreamNotifierMemoryStats {
+    let waiters = 0;
+    for (const set of this.waiters.values()) waiters += set.size;
+    let detailsWaiters = 0;
+    for (const set of this.detailsWaiters.values()) detailsWaiters += set.size;
+    return {
+      waiterStreams: this.waiters.size,
+      waiters,
+      latestSeqStreams: this.latestSeq.size,
+      detailsWaiterStreams: this.detailsWaiters.size,
+      detailsWaiters,
+      detailsVersionStreams: this.detailsVersion.size,
+    };
+  }
+
+  getTopStreams(limit = 5): StreamNotifierTopStreamEntry[] {
+    const totals = new Map<string, StreamNotifierTopStreamEntry>();
+    for (const [stream, waiters] of this.waiters) {
+      const row = totals.get(stream) ?? { stream, waiters: 0, details_waiters: 0, total_waiters: 0 };
+      row.waiters = waiters.size;
+      row.total_waiters = row.waiters + row.details_waiters;
+      totals.set(stream, row);
+    }
+    for (const [stream, detailsWaiters] of this.detailsWaiters) {
+      const row = totals.get(stream) ?? { stream, waiters: 0, details_waiters: 0, total_waiters: 0 };
+      row.details_waiters = detailsWaiters.size;
+      row.total_waiters = row.waiters + row.details_waiters;
+      totals.set(stream, row);
+    }
+    return Array.from(totals.values())
+      .sort((a, b) => b.total_waiters - a.total_waiters || a.stream.localeCompare(b.stream))
+      .slice(0, Math.max(0, Math.floor(limit)));
+  }
+}

package/src/objectstore/accounting.ts

@@ -0,0 +1,151 @@
+import type { Metrics } from "../metrics";
+import type { ObjectStoreAccountingRecorder } from "../store/stats_accounting_store";
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+
+type ClassifiedRequest = {
+  streamHash: string;
+  artifact: string;
+};
+
+function classifyKey(key: string): ClassifiedRequest | null {
+  const match = /^streams\/([0-9a-f]{32})\/(.+)$/.exec(key);
+  if (!match) return null;
+  const [, streamHash, rest] = match;
+  if (rest === "manifest.json") return { streamHash, artifact: "manifest" };
+  if (rest === "schema-registry.json") return { streamHash, artifact: "schema_registry" };
+  if (rest.startsWith("index/")) return { streamHash, artifact: "routing_index" };
+  if (rest.startsWith("lexicon/")) return { streamHash, artifact: "routing_key_lexicon" };
+  if (rest.startsWith("secondary-index/")) return { streamHash, artifact: "exact_index" };
+  if (rest.startsWith("segments/") && rest.endsWith(".bin")) return { streamHash, artifact: "segment" };
+  if (rest.startsWith("segments/") && rest.endsWith(".cix")) return { streamHash, artifact: "bundled_companion" };
+  return { streamHash, artifact: "meta" };
+}
+
+function classifyListPrefix(prefix: string): ClassifiedRequest | null {
+  const exact = classifyKey(prefix.replace(/\/+$/, ""));
+  if (exact) return exact;
+  const match = /^streams\/([0-9a-f]{32})(?:\/(.+))?\/?$/.exec(prefix);
+  if (!match) return null;
+  const [, streamHash, rest = ""] = match;
+  if (rest === "" || rest === "segments") return { streamHash, artifact: "segment" };
+  if (rest === "index") return { streamHash, artifact: "routing_index" };
+  if (rest.startsWith("lexicon")) return { streamHash, artifact: "routing_key_lexicon" };
+  if (rest.startsWith("secondary-index")) return { streamHash, artifact: "exact_index" };
+  return { streamHash, artifact: "meta" };
+}
+
+export class AccountingObjectStore implements ObjectStore {
+  constructor(
+    private readonly inner: ObjectStore,
+    private readonly accounting: ObjectStoreAccountingRecorder,
+    private readonly metrics?: Metrics
+  ) {}
+
+  private recordLatency(op: "put" | "get" | "head" | "delete" | "list", artifact: string, startedNs: bigint, outcome: "ok" | "miss" | "error"): void {
+    if (!this.metrics) return;
+    const elapsedNs = Number(process.hrtime.bigint() - startedNs);
+    this.metrics.record(`tieredstore.objectstore.${op}.latency`, elapsedNs, "ns", {
+      artifact,
+      outcome,
+    });
+  }
+
+  async put(key: string, data: Uint8Array, opts?: { contentType?: string; contentLength?: number }): Promise<PutResult> {
+    const startedNs = process.hrtime.bigint();
+    const classified = classifyKey(key);
+    const artifact = classified?.artifact ?? "unknown";
+    try {
+      const res = await this.inner.put(key, data, opts);
+      if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "put", data.byteLength);
+      this.recordLatency("put", artifact, startedNs, "ok");
+      return res;
+    } catch (error) {
+      this.recordLatency("put", artifact, startedNs, "error");
+      throw error;
+    }
+  }
+
+  async putFile(key: string, path: string, size: number, opts?: { contentType?: string }): Promise<PutResult> {
+    const startedNs = process.hrtime.bigint();
+    const classified = classifyKey(key);
+    const artifact = classified?.artifact ?? "unknown";
+    try {
+      if (!this.inner.putFile) {
+        const bytes = await Bun.file(path).bytes();
+        const res = await this.inner.put(key, bytes, {
+          contentType: opts?.contentType,
+          contentLength: size,
+        });
+        if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "put", size);
+        this.recordLatency("put", artifact, startedNs, "ok");
+        return res;
+      }
+      const res = await this.inner.putFile(key, path, size, opts);
+      if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "put", size);
+      this.recordLatency("put", artifact, startedNs, "ok");
+      return res;
+    } catch (error) {
+      this.recordLatency("put", artifact, startedNs, "error");
+      throw error;
+    }
+  }
+
+  async get(key: string, opts?: GetOptions): Promise<Uint8Array | null> {
+    const startedNs = process.hrtime.bigint();
+    const classified = classifyKey(key);
+    const artifact = classified?.artifact ?? "unknown";
+    try {
+      const res = await this.inner.get(key, opts);
+      if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "get", res?.byteLength ?? 0);
+      this.recordLatency("get", artifact, startedNs, res == null ? "miss" : "ok");
+      return res;
+    } catch (error) {
+      this.recordLatency("get", artifact, startedNs, "error");
+      throw error;
+    }
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    const startedNs = process.hrtime.bigint();
+    const classified = classifyKey(key);
+    const artifact = classified?.artifact ?? "unknown";
+    try {
+      const res = await this.inner.head(key);
+      if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "head", res?.size ?? 0);
+      this.recordLatency("head", artifact, startedNs, res == null ? "miss" : "ok");
+      return res;
+    } catch (error) {
+      this.recordLatency("head", artifact, startedNs, "error");
+      throw error;
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    const startedNs = process.hrtime.bigint();
+    const classified = classifyKey(key);
+    const artifact = classified?.artifact ?? "unknown";
+    try {
+      await this.inner.delete(key);
+      if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "delete", 0);
+      this.recordLatency("delete", artifact, startedNs, "ok");
+    } catch (error) {
+      this.recordLatency("delete", artifact, startedNs, "error");
+      throw error;
+    }
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    const startedNs = process.hrtime.bigint();
+    const classified = classifyListPrefix(prefix);
+    const artifact = classified?.artifact ?? (prefix.replace(/\/+$/, "") === "streams" ? "stream_catalog" : "unknown");
+    try {
+      const res = await this.inner.list(prefix);
+      if (classified) await this.accounting.recordObjectStoreRequestByHash(classified.streamHash, classified.artifact, "list", 0);
+      this.recordLatency("list", artifact, startedNs, "ok");
+      return res;
+    } catch (error) {
+      this.recordLatency("list", artifact, startedNs, "error");
+      throw error;
+    }
+  }
+}

package/src/objectstore/interface.ts

@@ -0,0 +1,13 @@
+export type PutResult = { etag: string };
+
+export type GetRange = { start: number; end?: number }; // end is inclusive; omit for EOF
+export type GetOptions = { range?: GetRange };
+
+export interface ObjectStore {
+  put(key: string, data: Uint8Array, opts?: { contentType?: string; contentLength?: number }): Promise<PutResult>;
+  putFile?(key: string, path: string, size: number, opts?: { contentType?: string }): Promise<PutResult>;
+  get(key: string, opts?: GetOptions): Promise<Uint8Array | null>;
+  head(key: string): Promise<{ etag: string; size: number } | null>;
+  delete(key: string): Promise<void>;
+  list(prefix: string): Promise<string[]>;
+}