@tungthedev/streams-server 0.2.0

package/src/index/secondary_schema.ts

@@ -0,0 +1,105 @@
+import { createHash } from "node:crypto";
+import { Result } from "better-result";
+import type { SchemaRegistry, SearchFieldConfig } from "../schema/registry";
+import { canonicalizeExactValue, extractSearchExactTermsResult, extractSearchExactValuesResult, getSearchFieldBinding } from "../search/schema";
+import { schemaVersionForOffset } from "../schema/read_json";
+import { resolvePointerResult } from "../util/json_pointer";
+
+export type SecondaryIndexField = {
+  name: string;
+  config: SearchFieldConfig;
+};
+
+export type SecondaryIndexTerm = {
+  index: SecondaryIndexField;
+  canonical: string;
+  bytes: Uint8Array;
+};
+
+function addRawValues(out: unknown[], value: unknown): void {
+  if (Array.isArray(value)) {
+    for (const item of value) addRawValues(out, item);
+    return;
+  }
+  out.push(value);
+}
+
+export function getConfiguredSecondaryIndexes(registry: SchemaRegistry): SecondaryIndexField[] {
+  const search = registry.search;
+  if (!search) return [];
+  return Object.entries(search.fields)
+    .filter(([, config]) => config.exact === true)
+    .map(([name, config]) => ({ name, config }));
+}
+
+export function hashSecondaryIndexField(index: SecondaryIndexField): string {
+  return createHash("sha256")
+    .update(
+      JSON.stringify({
+        name: index.name,
+        kind: index.config.kind,
+        bindings: index.config.bindings,
+        normalizer: index.config.normalizer ?? null,
+        analyzer: index.config.analyzer ?? null,
+        exact: index.config.exact === true,
+      })
+    )
+    .digest("hex");
+}
+
+export function canonicalizeSecondaryIndexValue(config: SearchFieldConfig, value: unknown): string | null {
+  return canonicalizeExactValue(config, value);
+}
+
+export function extractSecondaryIndexTermsResult(
+  registry: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<SecondaryIndexTerm[], { message: string }> {
+  const termsRes = extractSearchExactTermsResult(registry, offset, value);
+  if (Result.isError(termsRes)) return termsRes;
+  return Result.ok(
+    termsRes.value.map((term) => ({
+      index: { name: term.field, config: term.config },
+      canonical: term.canonical,
+      bytes: term.bytes,
+    }))
+  );
+}
+
+export function extractSecondaryIndexValuesResult(
+  registry: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, string[]>, { message: string }> {
+  return extractSearchExactValuesResult(registry, offset, value);
+}
+
+export function extractSecondaryIndexValuesForFieldResult(
+  registry: SchemaRegistry,
+  offset: bigint,
+  value: unknown,
+  index: SecondaryIndexField
+): Result<string[], { message: string }> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return Result.err({ message: "search fields require JSON object records" });
+  }
+  const version = schemaVersionForOffset(registry, offset);
+  const binding = getSearchFieldBinding(index.config, version);
+  if (!binding) return Result.ok([]);
+  const resolvedRes = resolvePointerResult(value, binding.jsonPointer);
+  if (Result.isError(resolvedRes)) return Result.err({ message: resolvedRes.error.message });
+  if (!resolvedRes.value.exists) return Result.ok([]);
+
+  const rawValues: unknown[] = [];
+  addRawValues(rawValues, resolvedRes.value.value);
+  const out: string[] = [];
+  const seen = new Set<string>();
+  for (const rawValue of rawValues) {
+    const canonical = canonicalizeExactValue(index.config, rawValue);
+    if (canonical == null || seen.has(canonical)) continue;
+    seen.add(canonical);
+    out.push(canonical);
+  }
+  return Result.ok(out);
+}

package/src/ingest.ts

@@ -0,0 +1,309 @@
+import type { Config } from "./config";
+import type { StatsCollector } from "./stats";
+import type { BackpressureGate } from "./backpressure";
+import type { Metrics } from "./metrics";
+import { Result } from "better-result";
+import {
+  type AppendRow,
+  type AppendSuccess,
+  type ProducerInfo,
+  type StoreAppendBatch,
+  type StoreAppendError,
+  type StoreAppendResult,
+  type StoreAppendTask,
+} from "./store/append";
+import type { WalAppendStore } from "./store/wal_store";
+
+export type AppendError = StoreAppendError | { kind: "overloaded" | "internal" };
+export type AppendResult = Result<AppendSuccess, AppendError>;
+export type { AppendRow, AppendSuccess, ProducerInfo } from "./store/append";
+
+type AppendTask = {
+  stream: string;
+  baseAppendMs: bigint;
+  rows: AppendRow[];
+  contentType: string | null;
+  streamSeq: string | null;
+  producer: ProducerInfo | null;
+  close: boolean;
+  reservedBytes: number;
+  enqueuedAtMs?: number;
+  resolve: (r: AppendResult) => void;
+};
+
+export class IngestQueue {
+  private readonly cfg: Config;
+  private readonly wal: WalAppendStore;
+  private readonly stats?: StatsCollector;
+  private readonly gate?: BackpressureGate;
+  private readonly metrics?: Metrics;
+  private readonly q: AppendTask[] = [];
+  private timer: any | null = null;
+  private scheduled = false;
+  private queuedBytes = 0;
+  private lastBacklogWarnMs = 0;
+  private flushPromise: Promise<void> | null = null;
+  private flushRequested = false;
+
+  constructor(cfg: Config, wal: WalAppendStore, stats?: StatsCollector, gate?: BackpressureGate, metrics?: Metrics) {
+    this.cfg = cfg;
+    this.wal = wal;
+    this.stats = stats;
+    this.gate = gate;
+    this.metrics = metrics;
+  }
+
+  start(): void {
+    if (this.timer) return;
+    this.timer = setInterval(() => {
+      void this.flush();
+    }, this.cfg.ingestFlushIntervalMs);
+  }
+
+  async stop(): Promise<void> {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    await this.flushPromise;
+    if (this.q.length > 0) await this.flush();
+  }
+
+  /**
+   * Enqueue an append. This returns after the batch containing it has committed.
+   */
+  append(args: {
+    stream: string;
+    baseAppendMs: bigint;
+    rows: AppendRow[];
+    contentType: string | null;
+    streamSeq?: string | null;
+    producer?: ProducerInfo | null;
+    close?: boolean;
+  }, opts?: { bypassBackpressure?: boolean; priority?: "high" | "normal" }): Promise<AppendResult> {
+    const bytes = args.rows.reduce((acc, r) => acc + r.payload.byteLength, 0);
+    if (!opts?.bypassBackpressure) {
+      if (this.q.length >= this.cfg.ingestMaxQueueRequests || this.queuedBytes + bytes > this.cfg.ingestMaxQueueBytes) {
+        if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "queue" });
+        return Promise.resolve(Result.err({ kind: "overloaded" }));
+      }
+      if (this.gate && !this.gate.reserve(bytes)) {
+        if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "backlog" });
+        this.warnBacklog();
+        return Promise.resolve(Result.err({ kind: "overloaded" }));
+      }
+    }
+    this.queuedBytes += bytes;
+    return new Promise((resolve) => {
+      const task: AppendTask = {
+        stream: args.stream,
+        baseAppendMs: args.baseAppendMs,
+        rows: args.rows,
+        contentType: args.contentType ?? null,
+        streamSeq: args.streamSeq ?? null,
+        producer: args.producer ?? null,
+        close: args.close ?? false,
+        reservedBytes: opts?.bypassBackpressure ? 0 : bytes,
+        enqueuedAtMs: this.stats ? Date.now() : undefined,
+        resolve,
+      };
+      if (opts?.priority === "high") this.q.unshift(task);
+      else this.q.push(task);
+      // Opportunistic flush if the queue gets large.
+      if (!this.scheduled && this.q.length >= this.cfg.ingestMaxBatchRequests) {
+        this.scheduled = true;
+        setTimeout(() => {
+          this.scheduled = false;
+          void this.flush();
+        }, 0);
+      }
+    });
+  }
+
+  appendInternal(args: {
+    stream: string;
+    baseAppendMs: bigint;
+    rows: AppendRow[];
+    contentType: string | null;
+  }): Promise<AppendResult> {
+    return this.append(args, { bypassBackpressure: true, priority: "high" });
+  }
+
+  getQueueStats(): { requests: number; bytes: number } {
+    return { requests: this.q.length, bytes: this.queuedBytes };
+  }
+
+  getMemoryStats(): { queuedPayloadBytes: number; queuedRequests: number } {
+    return {
+      queuedPayloadBytes: this.queuedBytes,
+      queuedRequests: this.q.length,
+    };
+  }
+
+  isQueueFull(): boolean {
+    return this.q.length >= this.cfg.ingestMaxQueueRequests || this.queuedBytes >= this.cfg.ingestMaxQueueBytes;
+  }
+
+  private warnBacklog(): void {
+    if (!this.gate) return;
+    const now = Date.now();
+    if (now - this.lastBacklogWarnMs < 10_000) return;
+    this.lastBacklogWarnMs = now;
+    const current = this.gate.getCurrentBytes();
+    const max = this.gate.getMaxBytes();
+    const msg =
+      `[backpressure] local backlog ${formatBytes(current)} exceeds limit ${formatBytes(max)}; rejecting appends (DS_LOCAL_BACKLOG_MAX_BYTES)`;
+    // eslint-disable-next-line no-console
+    console.warn(msg);
+  }
+
+  async flush(): Promise<void> {
+    this.flushRequested = true;
+    if (this.flushPromise) return this.flushPromise;
+    this.flushPromise = this.runFlushLoop().finally(() => {
+      this.flushPromise = null;
+    });
+    return this.flushPromise;
+  }
+
+  private async runFlushLoop(): Promise<void> {
+    while (this.flushRequested) {
+      this.flushRequested = false;
+      await this.flushOnce();
+    }
+  }
+
+  private async flushOnce(): Promise<void> {
+    if (this.q.length === 0) return;
+    const flushStartMs = Date.now();
+    let busyWaitMs = 0;
+
+    // Drain up to limits.
+    const batch: AppendTask[] = [];
+    let batchBytes = 0;
+    let batchReservedBytes = 0;
+    let drainCount = 0;
+    while (drainCount < this.q.length && batch.length < this.cfg.ingestMaxBatchRequests && batchBytes < this.cfg.ingestMaxBatchBytes) {
+      const t = this.q[drainCount]!;
+      batch.push(t);
+      drainCount += 1;
+      for (const r of t.rows) batchBytes += r.payload.byteLength;
+      batchReservedBytes += t.reservedBytes;
+    }
+    if (drainCount > 0) {
+      this.q.splice(0, drainCount);
+    }
+    this.queuedBytes = Math.max(0, this.queuedBytes - batchBytes);
+
+    // Compute queue wait/backpressure stats before executing the batch.
+    let bpOverMs = 0;
+    if (this.stats) {
+      const budgetMs = this.stats.getBackpressureBudgetMs();
+      const nowMs = Date.now();
+      for (const t of batch) {
+        if (t.enqueuedAtMs == null) continue;
+        const waitMs = Math.max(0, nowMs - t.enqueuedAtMs);
+        if (waitMs > budgetMs) {
+          bpOverMs += waitMs - budgetMs;
+        }
+      }
+    }
+
+    let walBytesCommitted = 0;
+    let results: StoreAppendResult[] = [];
+    const storeBatch: StoreAppendTask[] = batch.map(({ stream, baseAppendMs, rows, contentType, streamSeq, producer, close }) => ({
+      stream,
+      baseAppendMs,
+      rows,
+      contentType,
+      streamSeq,
+      producer,
+      close,
+    }));
+
+    const sleep = (ms: number): Promise<void> => new Promise((res) => setTimeout(res, ms));
+
+    try {
+      const maxBusyMs = Math.max(0, this.cfg.ingestBusyTimeoutMs);
+      const startMs = Date.now();
+      let attempt = 0;
+      let retryLimitExceeded = false;
+      while (true) {
+        const batchResult = await this.wal.appendBatch(storeBatch);
+        if (Result.isOk(batchResult)) {
+          results = batchResult.value.results;
+          walBytesCommitted = batchResult.value.walBytesCommitted;
+          break;
+        }
+        if (!isRetryableBatchResult(batchResult) || maxBusyMs <= 0) {
+          retryLimitExceeded = true;
+          break;
+        }
+        const elapsed = Date.now() - startMs;
+        if (elapsed >= maxBusyMs) {
+          retryLimitExceeded = true;
+          break;
+        }
+        const delay = Math.min(200, 5 * 2 ** attempt);
+        attempt += 1;
+        busyWaitMs += delay;
+        await sleep(delay);
+      }
+      if (retryLimitExceeded) {
+        if (this.gate && batchReservedBytes > 0) this.gate.release(batchReservedBytes);
+        for (const t of batch) t.resolve(Result.err({ kind: "internal" }));
+        const elapsedNs = (Date.now() - flushStartMs) * 1_000_000;
+        if (this.metrics) {
+          this.metrics.record("tieredstore.ingest.flush.latency", elapsedNs, "ns");
+          if (busyWaitMs > 0) this.metrics.record("tieredstore.ingest.store_retry.wait", busyWaitMs * 1_000_000, "ns");
+        }
+        return;
+      }
+      if (this.gate) {
+        const reservedCommitted = Math.min(batchReservedBytes, walBytesCommitted);
+        this.gate.commit(walBytesCommitted, reservedCommitted);
+        const extra = batchReservedBytes - walBytesCommitted;
+        if (extra > 0) this.gate.release(extra);
+      }
+      if (this.stats && walBytesCommitted > 0) this.stats.recordWalCommitBytes(walBytesCommitted);
+      if (this.stats && bpOverMs > 0) this.stats.recordBackpressureOverMs(bpOverMs);
+      for (let i = 0; i < batch.length; i++) batch[i].resolve(toAppendResult(results[i]));
+      const elapsedNs = (Date.now() - flushStartMs) * 1_000_000;
+      if (this.metrics) {
+        this.metrics.record("tieredstore.ingest.flush.latency", elapsedNs, "ns");
+        if (busyWaitMs > 0) this.metrics.record("tieredstore.ingest.store_retry.wait", busyWaitMs * 1_000_000, "ns");
+      }
+    } catch (e) {
+      // If the whole transaction failed, all tasks are treated as internal errors.
+      // eslint-disable-next-line no-console
+      console.error("ingest tx failed", e);
+      if (this.gate && batchReservedBytes > 0) this.gate.release(batchReservedBytes);
+      for (const t of batch) t.resolve(Result.err({ kind: "internal" }));
+      const elapsedNs = (Date.now() - flushStartMs) * 1_000_000;
+      if (this.metrics) {
+        this.metrics.record("tieredstore.ingest.flush.latency", elapsedNs, "ns");
+        if (busyWaitMs > 0) this.metrics.record("tieredstore.ingest.store_retry.wait", busyWaitMs * 1_000_000, "ns");
+      }
+    }
+  }
+}
+
+function isRetryableBatchResult(result: StoreAppendBatch): boolean {
+  return Result.isError(result) && result.error.kind === "retryable";
+}
+
+function toAppendResult(result: StoreAppendResult | undefined): AppendResult {
+  if (!result) return Result.err({ kind: "internal" });
+  if (Result.isError(result)) return Result.err(result.error);
+  return Result.ok(result.value);
+}
+
+function formatBytes(bytes: number): string {
+  const units = ["b", "kb", "mb", "gb"];
+  let value = Math.max(0, bytes);
+  let idx = 0;
+  while (value >= 1024 && idx < units.length - 1) {
+    value /= 1024;
+    idx += 1;
+  }
+  const digits = idx === 0 ? 0 : 1;
+  return `${value.toFixed(digits)}${units[idx]}`;
+}