@tungthedev/streams-server 0.2.0

package/src/search/schema.ts

@@ -0,0 +1,245 @@
+import { Result } from "better-result";
+import type { SchemaRegistry, SearchConfig, SearchFieldBinding, SearchFieldConfig } from "../schema/registry";
+import { resolvePointerResult } from "../util/json_pointer";
+import { schemaVersionForOffset } from "../schema/read_json";
+
+export type SearchExactTerm = {
+  field: string;
+  config: SearchFieldConfig;
+  canonical: string;
+  bytes: Uint8Array;
+};
+
+export function resolveSearchAlias(search: SearchConfig | undefined, fieldName: string): string {
+  return search?.aliases?.[fieldName] ?? fieldName;
+}
+
+export function getSearchFieldConfig(search: SearchConfig | undefined, fieldName: string): SearchFieldConfig | null {
+  const resolved = resolveSearchAlias(search, fieldName);
+  return search?.fields?.[resolved] ?? null;
+}
+
+export function getSearchFieldBinding(config: SearchFieldConfig, version: number): SearchFieldBinding | null {
+  let selected: SearchFieldBinding | null = null;
+  for (const binding of config.bindings) {
+    if (binding.version <= version && (!selected || binding.version > selected.version)) {
+      selected = binding;
+    }
+  }
+  return selected;
+}
+
+export function normalizeKeywordValue(value: unknown, normalizer: SearchFieldConfig["normalizer"]): string | null {
+  if (typeof value !== "string") return null;
+  return normalizer === "lowercase_v1" ? value.toLowerCase() : value;
+}
+
+export function canonicalizeExactValue(config: SearchFieldConfig, value: unknown): string | null {
+  switch (config.kind) {
+    case "keyword":
+      return normalizeKeywordValue(value, config.normalizer);
+    case "integer":
+      if (typeof value === "bigint") return value.toString();
+      if (typeof value === "number" && Number.isFinite(value) && Number.isInteger(value)) return String(value);
+      if (typeof value === "string" && /^-?(0|[1-9][0-9]*)$/.test(value.trim())) return String(BigInt(value.trim()));
+      return null;
+    case "float":
+      if (typeof value === "bigint") return value.toString();
+      if (typeof value === "number" && Number.isFinite(value)) return String(value);
+      if (typeof value === "string" && value.trim() !== "") {
+        const n = Number(value);
+        if (Number.isFinite(n)) return String(n);
+      }
+      return null;
+    case "date":
+      if (typeof value === "number" && Number.isFinite(value)) return String(Math.trunc(value));
+      if (typeof value === "bigint") return value.toString();
+      if (typeof value === "string" && value.trim() !== "") {
+        const parsed = Date.parse(value);
+        if (Number.isFinite(parsed)) return String(Math.trunc(parsed));
+        if (/^-?(0|[1-9][0-9]*)$/.test(value.trim())) return String(BigInt(value.trim()));
+      }
+      return null;
+    case "bool":
+      if (typeof value === "boolean") return value ? "true" : "false";
+      if (typeof value === "string") {
+        const lowered = value.trim().toLowerCase();
+        if (lowered === "true" || lowered === "false") return lowered;
+      }
+      return null;
+    default:
+      return null;
+  }
+}
+
+export function canonicalizeColumnValue(config: SearchFieldConfig, value: unknown): bigint | number | boolean | null {
+  switch (config.kind) {
+    case "integer": {
+      const canonical = canonicalizeExactValue(config, value);
+      return canonical == null ? null : BigInt(canonical);
+    }
+    case "date": {
+      const canonical = canonicalizeExactValue(config, value);
+      return canonical == null ? null : BigInt(canonical);
+    }
+    case "float": {
+      const canonical = canonicalizeExactValue(config, value);
+      if (canonical == null) return null;
+      const parsed = Number(canonical);
+      return Number.isFinite(parsed) ? parsed : null;
+    }
+    case "bool":
+      return canonicalizeExactValue(config, value) === "true"
+        ? true
+        : canonicalizeExactValue(config, value) === "false"
+          ? false
+          : null;
+    default:
+      return null;
+  }
+}
+
+export function analyzeTextValue(value: string, analyzer: SearchFieldConfig["analyzer"]): string[] {
+  if (analyzer !== "unicode_word_v1") return [];
+  const matches = value.toLowerCase().match(/[\p{L}\p{N}]+/gu);
+  return matches ? matches.filter((token) => token.length > 0) : [];
+}
+
+function addRawValues(out: unknown[], value: unknown): void {
+  if (Array.isArray(value)) {
+    for (const item of value) addRawValues(out, item);
+    return;
+  }
+  out.push(value);
+}
+
+export function extractRawSearchValuesForFieldsResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown,
+  fieldNames: Iterable<string>
+): Result<Map<string, unknown[]>, { message: string }> {
+  if (!reg.search) return Result.ok(new Map());
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return Result.err({ message: "search fields require JSON object records" });
+  }
+  const version = schemaVersionForOffset(reg, offset);
+  const out = new Map<string, unknown[]>();
+  for (const fieldName of fieldNames) {
+    const config = reg.search.fields[fieldName];
+    if (!config) continue;
+    const binding = getSearchFieldBinding(config, version);
+    if (!binding) continue;
+    const resolvedRes = resolvePointerResult(value, binding.jsonPointer);
+    if (Result.isError(resolvedRes)) return Result.err({ message: resolvedRes.error.message });
+    if (!resolvedRes.value.exists) continue;
+    const values: unknown[] = [];
+    addRawValues(values, resolvedRes.value.value);
+    if (values.length > 0) out.set(fieldName, values);
+  }
+  return Result.ok(out);
+}
+
+export function extractRawSearchValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, unknown[]>, { message: string }> {
+  return extractRawSearchValuesForFieldsResult(reg, offset, value, Object.keys(reg.search?.fields ?? {}));
+}
+
+export function extractSearchExactTermsResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<SearchExactTerm[], { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out: SearchExactTerm[] = [];
+  const seen = new Set<string>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config?.exact) continue;
+    for (const rawValue of values) {
+      const canonical = canonicalizeExactValue(config, rawValue);
+      if (canonical == null) continue;
+      const dedupeKey = `${fieldName}\u0000${canonical}`;
+      if (seen.has(dedupeKey)) continue;
+      seen.add(dedupeKey);
+      out.push({
+        field: fieldName,
+        config,
+        canonical,
+        bytes: new TextEncoder().encode(canonical),
+      });
+    }
+  }
+  return Result.ok(out);
+}
+
+export function extractSearchExactValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, string[]>, { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out = new Map<string, string[]>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config) continue;
+    const exactValues: string[] = [];
+    for (const rawValue of values) {
+      const canonical = canonicalizeExactValue(config, rawValue);
+      if (canonical != null) exactValues.push(canonical);
+    }
+    if (exactValues.length > 0) out.set(fieldName, exactValues);
+  }
+  return Result.ok(out);
+}
+
+export function extractSearchColumnValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, Array<bigint | number | boolean>>, { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out = new Map<string, Array<bigint | number | boolean>>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config?.column) continue;
+    const colValues: Array<bigint | number | boolean> = [];
+    for (const rawValue of values) {
+      const normalized = canonicalizeColumnValue(config, rawValue);
+      if (normalized != null) colValues.push(normalized);
+    }
+    if (colValues.length > 0) out.set(fieldName, colValues);
+  }
+  return Result.ok(out);
+}
+
+export function extractSearchTextValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, string[]>, { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out = new Map<string, string[]>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config) continue;
+    const textValues: string[] = [];
+    for (const rawValue of values) {
+      if (config.kind === "keyword") {
+        const normalized = normalizeKeywordValue(rawValue, config.normalizer);
+        if (normalized != null) textValues.push(normalized);
+      } else if (config.kind === "text" && typeof rawValue === "string") {
+        textValues.push(rawValue);
+      }
+    }
+    if (textValues.length > 0) out.set(fieldName, textValues);
+  }
+  return Result.ok(out);
+}

package/src/segment/cache.ts

@@ -0,0 +1,270 @@
+import { mkdirSync, readdirSync, statSync, unlinkSync, renameSync, existsSync, writeFileSync, readFileSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
+import { LruCache } from "../util/lru";
+
+export type MappedSegmentFile = {
+  objectKey: string;
+  path: string;
+  bytes: Uint8Array;
+  sizeBytes: number;
+};
+
+export type SegmentCacheStats = {
+  hits: number;
+  misses: number;
+  evictions: number;
+  bytesAdded: number;
+  usedBytes: number;
+  maxBytes: number;
+  entryCount: number;
+  mappedBytes: number;
+  mappedEntryCount: number;
+  pinnedEntryCount: number;
+};
+
+export class SegmentDiskCache {
+  private readonly rootDir: string;
+  private readonly maxBytes: number;
+  private readonly entries = new Map<string, { path: string; size: number }>();
+  private readonly pinnedKeys = new Set<string>();
+  private readonly mappedFiles: LruCache<string, MappedSegmentFile>;
+  private totalBytes = 0;
+  private hits = 0;
+  private misses = 0;
+  private evictions = 0;
+  private bytesAdded = 0;
+
+  constructor(rootDir: string, maxBytes: number, mappedEntries = 64) {
+    this.rootDir = rootDir;
+    this.maxBytes = maxBytes;
+    this.mappedFiles = new LruCache(Math.max(1, mappedEntries));
+    if (this.maxBytes > 0) {
+      mkdirSync(this.rootDir, { recursive: true });
+      this.loadIndex();
+    }
+  }
+
+  private loadIndex(): void {
+    if (!existsSync(this.rootDir)) return;
+    const files: Array<{ key: string; path: string; size: number; mtimeMs: number }> = [];
+    const walk = (dir: string) => {
+      for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+          walk(full);
+        } else if (entry.isFile()) {
+          const stat = statSync(full);
+          const key = relative(this.rootDir, full);
+          files.push({ key, path: full, size: stat.size, mtimeMs: stat.mtimeMs });
+        }
+      }
+    };
+    walk(this.rootDir);
+    files.sort((a, b) => a.mtimeMs - b.mtimeMs);
+    for (const f of files) {
+      this.entries.set(f.key, { path: f.path, size: f.size });
+      this.totalBytes += f.size;
+    }
+    this.evictIfNeeded(0);
+  }
+
+  getPath(objectKey: string): string {
+    return join(this.rootDir, objectKey);
+  }
+
+  has(objectKey: string): boolean {
+    const exists = this.entries.has(objectKey) && existsSync(this.getPath(objectKey));
+    if (!exists) this.entries.delete(objectKey);
+    return exists;
+  }
+
+  touch(objectKey: string): void {
+    const entry = this.entries.get(objectKey);
+    if (!entry) return;
+    this.entries.delete(objectKey);
+    this.entries.set(objectKey, entry);
+  }
+
+  recordHit(): void {
+    this.hits += 1;
+  }
+
+  recordMiss(): void {
+    this.misses += 1;
+  }
+
+  get(objectKey: string): Uint8Array | null {
+    if (!this.has(objectKey)) {
+      this.recordMiss();
+      return null;
+    }
+    this.recordHit();
+    this.touch(objectKey);
+    const mapped = this.getMapped(objectKey);
+    if (mapped) return mapped.bytes;
+    const path = this.getPath(objectKey);
+    return readFileSync(path);
+  }
+
+  getMapped(objectKey: string): MappedSegmentFile | null {
+    if (!this.has(objectKey)) return null;
+    const cached = this.mappedFiles.get(objectKey);
+    if (cached) {
+      this.pinnedKeys.add(objectKey);
+      this.touch(objectKey);
+      return cached;
+    }
+
+    const path = this.getPath(objectKey);
+    let sizeBytes: number;
+    try {
+      sizeBytes = statSync(path).size;
+    } catch {
+      this.entries.delete(objectKey);
+      return null;
+    }
+
+    let bytes: Uint8Array;
+    try {
+      bytes = (Bun as any).mmap(path, { shared: true }) as Uint8Array;
+    } catch {
+      return null;
+    }
+    if (bytes.byteLength !== sizeBytes) return null;
+
+    const mapped = { objectKey, path, bytes, sizeBytes };
+    this.mappedFiles.set(objectKey, mapped);
+    this.pinnedKeys.add(objectKey);
+    this.touch(objectKey);
+    return mapped;
+  }
+
+  put(objectKey: string, bytes: Uint8Array): boolean {
+    if (this.maxBytes <= 0) return false;
+    const sizeBytes = bytes.byteLength;
+    if (sizeBytes > this.maxBytes) return false;
+    this.evictIfNeeded(sizeBytes);
+    const dest = this.getPath(objectKey);
+    mkdirSync(dirname(dest), { recursive: true });
+    const tmp = `${dest}.tmp-${Date.now()}`;
+    try {
+      writeFileSync(tmp, bytes);
+      renameSync(tmp, dest);
+    } catch {
+      try {
+        unlinkSync(tmp);
+      } catch {
+        // ignore
+      }
+      return false;
+    }
+    const existing = this.entries.get(objectKey);
+    if (existing) this.totalBytes = Math.max(0, this.totalBytes - existing.size);
+    this.mappedFiles.delete(objectKey);
+    this.entries.set(objectKey, { path: dest, size: sizeBytes });
+    this.totalBytes += sizeBytes;
+    this.bytesAdded += sizeBytes;
+    return true;
+  }
+
+  putFromLocal(objectKey: string, localPath: string, sizeBytes: number): boolean {
+    if (this.maxBytes <= 0) return false;
+    if (sizeBytes > this.maxBytes) return false;
+    this.evictIfNeeded(sizeBytes);
+    const dest = this.getPath(objectKey);
+    mkdirSync(dirname(dest), { recursive: true });
+    try {
+      renameSync(localPath, dest);
+    } catch {
+      return false;
+    }
+    const existing = this.entries.get(objectKey);
+    if (existing) this.totalBytes = Math.max(0, this.totalBytes - existing.size);
+    this.mappedFiles.delete(objectKey);
+    this.entries.set(objectKey, { path: dest, size: sizeBytes });
+    this.totalBytes += sizeBytes;
+    this.bytesAdded += sizeBytes;
+    return true;
+  }
+
+  remove(objectKey: string): void {
+    if (this.pinnedKeys.has(objectKey)) return;
+    const entry = this.entries.get(objectKey);
+    if (!entry) return;
+    try {
+      unlinkSync(entry.path);
+    } catch {
+      // ignore
+    }
+    this.totalBytes = Math.max(0, this.totalBytes - entry.size);
+    this.entries.delete(objectKey);
+  }
+
+  private evictIfNeeded(incomingBytes: number): void {
+    while (this.totalBytes + incomingBytes > this.maxBytes && this.entries.size > 0) {
+      const oldestKey = this.entries.keys().next().value as string;
+      if (this.pinnedKeys.has(oldestKey)) {
+        let removed = false;
+        for (const candidateKey of this.entries.keys()) {
+          if (this.pinnedKeys.has(candidateKey)) continue;
+          const candidate = this.entries.get(candidateKey);
+          if (!candidate) continue;
+          try {
+            unlinkSync(candidate.path);
+          } catch {
+            // ignore
+          }
+          this.totalBytes = Math.max(0, this.totalBytes - candidate.size);
+          this.entries.delete(candidateKey);
+          this.mappedFiles.delete(candidateKey);
+          this.evictions += 1;
+          removed = true;
+          break;
+        }
+        if (!removed) break;
+        continue;
+      }
+      const entry = this.entries.get(oldestKey);
+      if (entry) {
+        try {
+          unlinkSync(entry.path);
+        } catch {
+          // ignore
+        }
+        this.totalBytes = Math.max(0, this.totalBytes - entry.size);
+        this.mappedFiles.delete(oldestKey);
+        this.evictions += 1;
+      }
+      this.entries.delete(oldestKey);
+    }
+  }
+
+  stats(): SegmentCacheStats {
+    let mappedBytes = 0;
+    let mappedEntryCount = 0;
+    for (const mapped of this.mappedFiles.values()) {
+      mappedBytes += mapped.sizeBytes;
+      mappedEntryCount += 1;
+    }
+    return {
+      hits: this.hits,
+      misses: this.misses,
+      evictions: this.evictions,
+      bytesAdded: this.bytesAdded,
+      usedBytes: this.totalBytes,
+      maxBytes: this.maxBytes,
+      entryCount: this.entries.size,
+      mappedBytes,
+      mappedEntryCount,
+      pinnedEntryCount: this.pinnedKeys.size,
+    };
+  }
+
+  bytesForObjectKeyPrefix(prefix: string): number {
+    let total = 0;
+    for (const [objectKey, entry] of this.entries.entries()) {
+      if (objectKey.startsWith(prefix)) total += entry.size;
+    }
+    return total;
+  }
+}

package/src/segment/cached_segment.ts

@@ -0,0 +1,89 @@
+import { existsSync, readFileSync } from "node:fs";
+import type { SegmentRow } from "../db/db";
+import type { ObjectStore } from "../objectstore/interface";
+import type { SegmentDiskCache } from "./cache";
+import { dsError } from "../util/ds_error";
+import type { RetryOptions } from "../util/retry";
+import { retry } from "../util/retry";
+import { segmentObjectKey, streamHash16Hex } from "../util/stream_paths";
+
+export type SegmentReadSource =
+  | { kind: "mapped"; path: string; bytes: Uint8Array }
+  | { kind: "bytes"; bytes: Uint8Array };
+
+function readRangeFromBytes(bytes: Uint8Array, start: number, end: number): Uint8Array {
+  const boundedStart = Math.max(0, Math.min(start, bytes.byteLength));
+  const boundedEnd = Math.max(boundedStart, Math.min(end + 1, bytes.byteLength));
+  return bytes.subarray(boundedStart, boundedEnd);
+}
+
+export function readRangeFromSource(source: SegmentReadSource, start: number, end: number): Uint8Array {
+  return readRangeFromBytes(source.bytes, start, end);
+}
+
+export async function loadSegmentSource(
+  os: ObjectStore,
+  seg: SegmentRow,
+  diskCache?: SegmentDiskCache,
+  retryOpts?: RetryOptions
+): Promise<SegmentReadSource> {
+  if (seg.local_path && seg.local_path.length > 0 && existsSync(seg.local_path)) {
+    try {
+      const bytes = (Bun as any).mmap(seg.local_path, { shared: true }) as Uint8Array;
+      return { kind: "mapped", path: seg.local_path, bytes };
+    } catch {
+      return { kind: "bytes", bytes: readFileSync(seg.local_path) };
+    }
+  }
+
+  const objectKey = segmentObjectKey(streamHash16Hex(seg.stream), seg.segment_index);
+  if (diskCache && diskCache.has(objectKey)) {
+    diskCache.recordHit();
+    diskCache.touch(objectKey);
+    const mapped = diskCache.getMapped(objectKey);
+    if (mapped) return { kind: "mapped", path: mapped.path, bytes: mapped.bytes };
+    const cachedPath = diskCache.getPath(objectKey);
+    if (existsSync(cachedPath)) return { kind: "bytes", bytes: readFileSync(cachedPath) };
+    diskCache.remove(objectKey);
+  }
+
+  if (diskCache) diskCache.recordMiss();
+
+  const bytes = await retry(
+    async () => {
+      const res = await os.get(objectKey);
+      if (!res) throw dsError(`object store missing segment: ${objectKey}`);
+      return res;
+    },
+    retryOpts ?? { retries: 0, baseDelayMs: 0, maxDelayMs: 0, timeoutMs: 0 }
+  );
+
+  if (diskCache?.put(objectKey, bytes)) {
+    const mapped = diskCache.getMapped(objectKey);
+    if (mapped) return { kind: "mapped", path: mapped.path, bytes: mapped.bytes };
+    return { kind: "bytes", bytes: readFileSync(diskCache.getPath(objectKey)) };
+  }
+  return { kind: "bytes", bytes };
+}
+
+export async function loadSegmentBytesCached(
+  os: ObjectStore,
+  seg: SegmentRow,
+  diskCache?: SegmentDiskCache,
+  retryOpts?: RetryOptions
+): Promise<Uint8Array> {
+  const source = await loadSegmentSource(os, seg, diskCache, retryOpts);
+  return source.bytes;
+}
+
+export async function readSegmentRangeCached(
+  os: ObjectStore,
+  seg: SegmentRow,
+  start: number,
+  end: number,
+  diskCache?: SegmentDiskCache,
+  retryOpts?: RetryOptions
+): Promise<Uint8Array> {
+  const source = await loadSegmentSource(os, seg, diskCache, retryOpts);
+  return readRangeFromSource(source, start, end);
+}