@tungthedev/streams-server 0.2.0

package/src/segment/segmenter.ts

@@ -0,0 +1,412 @@
+import { mkdirSync, openSync, closeSync, writeSync, fsyncSync, renameSync, existsSync, unlinkSync } from "node:fs";
+import { dirname } from "node:path";
+import type { Config } from "../config";
+import type { SegmentStore } from "../store/segment_manifest_store";
+import { encodeBlock, encodeFooter, type BlockIndexEntry, type SegmentRecord } from "./format";
+import { readU32BE } from "../util/endian";
+import { localSegmentPath, streamHash16Hex } from "../util/stream_paths";
+import { LruCache } from "../util/lru";
+import { RuntimeMemorySampler } from "../runtime_memory_sampler";
+import { yieldToEventLoop } from "../util/yield";
+
+export type SegmenterOptions = {
+  minCandidateBytes?: number; // default: segmentMaxBytes
+  minCandidateRows?: number; // default: segmentTargetRows
+  maxIntervalMs?: number; // default: segmentMaxIntervalMs
+  candidatesPerTick?: number;
+  maxRowsPerSegment?: number;
+};
+
+export type SegmenterHooks = {
+  onSegmentSealed?: (stream: string, payloadBytes: number, segmentBytes: number) => void;
+};
+
+export type SegmenterMemoryStats = {
+  active_builds: number;
+  active_streams: number;
+  active_payload_bytes: number;
+  active_segment_bytes_estimate: number;
+  active_rows: number;
+};
+
+const SEGMENT_COMPRESSION_WINDOW = 8;
+const MIN_COMPRESSED_FILL_RATIO = 0.5;
+const MAX_COMPRESSION_BOOST_MULTIPLIER = 5;
+
+export class Segmenter {
+  private readonly config: Config;
+  private readonly db: SegmentStore;
+  private readonly opts: Required<SegmenterOptions>;
+  private readonly hooks?: SegmenterHooks;
+  private readonly memorySampler?: RuntimeMemorySampler;
+  private timer: any | null = null;
+  private running = false;
+  private stopping = false;
+  private readonly failures = new FailureTracker(1024);
+  private activeBuildStream: string | null = null;
+  private activePayloadBytes = 0;
+  private activeSegmentBytesEstimate = 0;
+  private activeRows = 0;
+
+  constructor(
+    config: Config,
+    db: SegmentStore,
+    opts: SegmenterOptions = {},
+    hooks?: SegmenterHooks,
+    memorySampler?: RuntimeMemorySampler
+  ) {
+    this.config = config;
+    this.db = db;
+    this.opts = {
+      minCandidateBytes: opts.minCandidateBytes ?? config.segmentMaxBytes,
+      minCandidateRows: opts.minCandidateRows ?? config.segmentTargetRows,
+      maxIntervalMs: opts.maxIntervalMs ?? config.segmentMaxIntervalMs,
+      candidatesPerTick: opts.candidatesPerTick ?? 8,
+      maxRowsPerSegment: opts.maxRowsPerSegment ?? 250_000,
+    };
+    this.hooks = hooks;
+    this.memorySampler = memorySampler;
+  }
+
+  start(): void {
+    this.stopping = false;
+    if (this.timer) return;
+    if (this.config.segmentCheckIntervalMs <= 0) return;
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, this.config.segmentCheckIntervalMs);
+  }
+
+  async stop(hard = false): Promise<void> {
+    if (hard) this.stopping = true;
+    else this.stopping = false;
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    while (this.running) {
+      await new Promise((resolve) => setTimeout(resolve, 5));
+    }
+  }
+
+  getMemoryStats(): SegmenterMemoryStats {
+    return {
+      active_builds: this.activeBuildStream ? 1 : 0,
+      active_streams: this.activeBuildStream ? 1 : 0,
+      active_payload_bytes: this.activePayloadBytes,
+      active_segment_bytes_estimate: this.activeSegmentBytesEstimate,
+      active_rows: this.activeRows,
+    };
+  }
+
+  async tick(): Promise<void> {
+    if (this.stopping) return;
+    if (this.running) return;
+    this.running = true;
+    try {
+      const candidates = await this.db.candidates(
+        BigInt(this.opts.minCandidateBytes),
+        BigInt(this.opts.minCandidateRows),
+        BigInt(this.opts.maxIntervalMs),
+        this.opts.candidatesPerTick
+      );
+      for (const c of candidates) {
+        if (this.failures.shouldSkip(c.stream)) continue;
+        try {
+          await this.buildOne(c.stream);
+          this.failures.recordSuccess(c.stream);
+        } catch (e) {
+          this.failures.recordFailure(c.stream);
+          const msg = String((e as any)?.message ?? e);
+          const lower = msg.toLowerCase();
+          if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+            // eslint-disable-next-line no-console
+            console.error("segment build failed", c.stream, e);
+          }
+        }
+      }
+    } catch (e) {
+      const msg = String((e as any)?.message ?? e);
+      const lower = msg.toLowerCase();
+      if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+        // eslint-disable-next-line no-console
+        console.error("segmenter tick error", e);
+      }
+    } finally {
+      this.running = false;
+    }
+  }
+
+  private isSqliteBusy(err: any): boolean {
+    const code = String(err?.code ?? "");
+    const errno = Number(err?.errno ?? -1);
+    return code === "SQLITE_BUSY" || code === "SQLITE_BUSY_SNAPSHOT" || errno === 5 || errno === 517;
+  }
+
+  private async runWithBusyRetry<T>(fn: () => T | Promise<T>): Promise<T> {
+    const maxBusyMs = Math.max(0, this.config.ingestBusyTimeoutMs);
+    if (maxBusyMs <= 0) return await fn();
+    const startMs = Date.now();
+    let attempt = 0;
+    for (;;) {
+      try {
+        return await fn();
+      } catch (e) {
+        if (!this.isSqliteBusy(e)) throw e;
+        const elapsed = Date.now() - startMs;
+        if (elapsed >= maxBusyMs) throw e;
+        const delay = Math.min(200, 5 * 2 ** attempt);
+        attempt += 1;
+        await new Promise((res) => setTimeout(res, delay));
+      }
+    }
+  }
+
+  private cleanupTmp(tmpPath: string): void {
+    try {
+      if (existsSync(tmpPath)) unlinkSync(tmpPath);
+    } catch {
+      // ignore
+    }
+  }
+
+  private async resolvePayloadSealTargetBytes(stream: string): Promise<bigint> {
+    const baseTarget = BigInt(this.config.segmentMaxBytes);
+    const ratio = await this.db.recentSegmentCompressionRatio(stream, SEGMENT_COMPRESSION_WINDOW);
+    if (ratio == null || !Number.isFinite(ratio) || ratio <= 0 || ratio >= MIN_COMPRESSED_FILL_RATIO) {
+      return baseTarget;
+    }
+    const desiredCompressedBytes = Math.ceil(this.config.segmentMaxBytes * MIN_COMPRESSED_FILL_RATIO);
+    const boosted = BigInt(Math.ceil(desiredCompressedBytes / ratio));
+    const maxBoosted = baseTarget * BigInt(MAX_COMPRESSION_BOOST_MULTIPLIER);
+    if (boosted > maxBoosted) return maxBoosted;
+    return boosted > baseTarget ? boosted : baseTarget;
+  }
+
+  private async shouldSealStream(row: { stream: string; pending_bytes: bigint; pending_rows: bigint; last_segment_cut_ms: bigint }): Promise<boolean> {
+    const payloadSealTargetBytes = await this.resolvePayloadSealTargetBytes(row.stream);
+    if (row.pending_bytes >= payloadSealTargetBytes) return true;
+    if (row.pending_rows >= BigInt(this.opts.minCandidateRows)) return true;
+    if (this.opts.maxIntervalMs > 0 && BigInt(Date.now()) - row.last_segment_cut_ms >= BigInt(this.opts.maxIntervalMs)) return true;
+    return false;
+  }
+
+  private async buildOne(stream: string): Promise<void> {
+    if (this.stopping) return;
+    let row = await this.db.getSegmentStreamState(stream);
+    if (!row || this.db.isDeleted(row)) return;
+    if (!(await this.shouldSealStream(row))) return;
+
+    // Claim.
+    const claim = await this.db.tryClaimSegment(stream);
+    if (!claim) return;
+
+    try {
+      const claimedRow = await this.db.getSegmentStreamState(stream);
+      if (!claimedRow || this.db.isDeleted(claimedRow)) return;
+      row = claimedRow;
+      if (!(await this.shouldSealStream(row))) return;
+      const startOffset = row.sealed_through + 1n;
+      const maxOffset = row.next_offset - 1n;
+      if (startOffset > maxOffset) return;
+
+      this.activeBuildStream = stream;
+      this.activePayloadBytes = 0;
+      this.activeSegmentBytesEstimate = 0;
+      this.activeRows = 0;
+      const segmentIndex = await this.db.nextSegmentIndexForStream(stream);
+      const shash = streamHash16Hex(stream);
+      const localPath = localSegmentPath(this.config.rootDir, shash, segmentIndex);
+      const tmpPath = `${localPath}.tmp`;
+      const leaveCutPhase = this.memorySampler?.enter("cut", {
+        stream,
+        segment_index: segmentIndex,
+      });
+      mkdirSync(dirname(localPath), { recursive: true });
+
+      // Build blocks and stream-write to temp file.
+      const fd = openSync(tmpPath, "w");
+      try {
+        let blockRecords: SegmentRecord[] = [];
+        let blockBytesApprox = 0;
+        let fileBytes = 0;
+        let blockCount = 0;
+        let blockFirstOffset = startOffset;
+        const blockIndex: BlockIndexEntry[] = [];
+
+        // Decide endOffset by scanning WAL rows until threshold.
+        // IMPORTANT: pending_bytes tracks WAL payload bytes only (not record/block overhead).
+        const payloadSealTargetBytes = await this.resolvePayloadSealTargetBytes(stream);
+        const rowSealTarget = BigInt(this.opts.minCandidateRows);
+        let payloadBytes = 0n;
+        let rowsSealed = 0n;
+        let endOffset = startOffset - 1n;
+        let lastAppendMs = 0n;
+
+        let lastYieldMs = Date.now();
+        let recordsSinceYield = 0;
+        for await (const rec of this.db.readWalRange(stream, startOffset, maxOffset)) {
+          const offset = BigInt(rec.offset);
+          const payload: Uint8Array = rec.payload;
+          const routingKey: Uint8Array | null = rec.routingKey ?? null;
+          const appendMs = BigInt(rec.tsMs);
+          lastAppendMs = appendMs;
+
+          const keyBytes = routingKey ?? new Uint8Array(0);
+          const segRec: SegmentRecord = {
+            appendNs: appendMs * 1_000_000n,
+            routingKey: keyBytes,
+            payload,
+          };
+          const recSize = 8 + 4 + keyBytes.byteLength + 4 + payload.byteLength;
+
+          if (blockRecords.length > 0 && blockBytesApprox + recSize > this.config.blockMaxBytes) {
+            const blockOffset = fileBytes;
+            const block = encodeBlock(blockRecords);
+            const compressedLen = readU32BE(block, 8);
+            blockIndex.push({
+              blockOffset,
+              firstOffset: blockFirstOffset,
+              recordCount: blockRecords.length,
+              compressedLen,
+              firstAppendNs: blockRecords[0].appendNs,
+              lastAppendNs: blockRecords[blockRecords.length - 1].appendNs,
+            });
+            writeSync(fd, block);
+            fileBytes += block.byteLength;
+            blockCount += 1;
+            blockRecords = [];
+            blockBytesApprox = 0;
+            await yieldToEventLoop();
+          }
+
+          if (blockRecords.length === 0) blockFirstOffset = offset;
+          blockRecords.push(segRec);
+          blockBytesApprox += recSize;
+
+          payloadBytes += BigInt(payload.byteLength);
+          rowsSealed += 1n;
+          endOffset = offset;
+          this.activePayloadBytes = Number(payloadBytes);
+          this.activeRows = Number(rowsSealed);
+          this.activeSegmentBytesEstimate = fileBytes + blockBytesApprox;
+
+          recordsSinceYield += 1;
+          if (recordsSinceYield >= 512 || Date.now() - lastYieldMs >= 10) {
+            await yieldToEventLoop();
+            lastYieldMs = Date.now();
+            recordsSinceYield = 0;
+          }
+
+          if (payloadBytes >= payloadSealTargetBytes) break;
+          if (rowsSealed >= rowSealTarget) break;
+          if (rowsSealed >= BigInt(this.opts.maxRowsPerSegment)) break;
+        }
+
+        if (rowsSealed === 0n) return;
+
+        if (blockRecords.length > 0) {
+          const blockOffset = fileBytes;
+          const block = encodeBlock(blockRecords);
+          const compressedLen = readU32BE(block, 8);
+          blockIndex.push({
+            blockOffset,
+            firstOffset: blockFirstOffset,
+            recordCount: blockRecords.length,
+            compressedLen,
+            firstAppendNs: blockRecords[0].appendNs,
+            lastAppendNs: blockRecords[blockRecords.length - 1].appendNs,
+          });
+          writeSync(fd, block);
+          fileBytes += block.byteLength;
+          blockCount += 1;
+        }
+
+        const footer = encodeFooter(blockIndex);
+        writeSync(fd, footer);
+        fileBytes += footer.byteLength;
+        this.activeSegmentBytesEstimate = fileBytes;
+
+        fsyncSync(fd);
+
+        const segmentId = `${shash}-${segmentIndex}-${startOffset.toString()}-${endOffset.toString()}`;
+        renameSync(tmpPath, localPath);
+
+        if (!this.stopping) {
+          try {
+            await this.runWithBusyRetry(async () => {
+              await this.db.commitSealedSegment({
+                segmentId,
+                stream,
+                segmentIndex,
+                startOffset,
+                endOffset,
+                blockCount,
+                lastAppendMs,
+                sizeBytes: fileBytes,
+                localPath,
+                payloadBytes,
+                rowsSealed,
+                claimToken: claim.token,
+              });
+            });
+            if (this.hooks?.onSegmentSealed) this.hooks.onSegmentSealed(stream, Number(payloadBytes), fileBytes);
+          } catch (e) {
+            try {
+              if (existsSync(localPath)) unlinkSync(localPath);
+            } catch {
+              // ignore
+            }
+            throw e;
+          }
+        }
+      } finally {
+        closeSync(fd);
+        this.cleanupTmp(tmpPath);
+        leaveCutPhase?.();
+      }
+    } finally {
+      this.activeBuildStream = null;
+      this.activePayloadBytes = 0;
+      this.activeSegmentBytesEstimate = 0;
+      this.activeRows = 0;
+      // Release claim.
+      if (!this.stopping) {
+        try {
+          await this.db.setSegmentInProgress(stream, 0, claim);
+        } catch {
+          // ignore
+        }
+      }
+    }
+  }
+}
+
+class FailureTracker {
+  private readonly cache: LruCache<string, { attempts: number; untilMs: number }>;
+
+  constructor(maxEntries: number) {
+    this.cache = new LruCache(maxEntries);
+  }
+
+  shouldSkip(stream: string): boolean {
+    const item = this.cache.get(stream);
+    if (!item) return false;
+    if (Date.now() >= item.untilMs) {
+      this.cache.delete(stream);
+      return false;
+    }
+    return true;
+  }
+
+  recordFailure(stream: string): void {
+    const now = Date.now();
+    const item = this.cache.get(stream) ?? { attempts: 0, untilMs: now };
+    item.attempts += 1;
+    const backoff = Math.min(60_000, 500 * 2 ** (item.attempts - 1));
+    item.untilMs = now + backoff;
+    this.cache.set(stream, item);
+  }
+
+  recordSuccess(stream: string): void {
+    this.cache.delete(stream);
+  }
+}

package/src/segment/segmenter_worker.ts

@@ -0,0 +1,72 @@
+import { parentPort, workerData, threadId } from "node:worker_threads";
+import type { Config } from "../config.ts";
+import { SqliteDurableStore } from "../db/db.ts";
+import type { HostRuntime } from "../runtime/host_runtime.ts";
+import { RuntimeMemorySampler } from "../runtime_memory_sampler.ts";
+import { setSqliteRuntimeOverride } from "../sqlite/adapter.ts";
+import { Segmenter, type SegmenterHooks, type SegmenterOptions } from "./segmenter.ts";
+import { initConsoleLogging } from "../util/log.ts";
+
+initConsoleLogging();
+
+const data = workerData as { config: Config; hostRuntime?: HostRuntime; opts?: SegmenterOptions };
+const cfg = data.config;
+setSqliteRuntimeOverride(data.hostRuntime ?? null);
+// The main server process initializes/migrates schema; workers should avoid
+// concurrent migrations on the same sqlite file.
+const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.workerSqliteCacheBytes, skipMigrations: true });
+const memorySampler =
+  cfg.memorySamplerPath != null
+    ? new RuntimeMemorySampler(cfg.memorySamplerPath, {
+        intervalMs: cfg.memorySamplerIntervalMs,
+        scope: `segmenter-worker-${threadId}`,
+      })
+    : undefined;
+memorySampler?.start();
+
+const hooks: SegmenterHooks = {
+  onSegmentSealed: (stream, payloadBytes, segmentBytes) => {
+    parentPort?.postMessage({ type: "sealed", stream, payloadBytes, segmentBytes });
+  },
+};
+
+const segmenter = new Segmenter(cfg, db, data.opts ?? {}, hooks, memorySampler);
+segmenter.start();
+const memoryTimer = setInterval(() => {
+  try {
+    parentPort?.postMessage({ type: "memory", workerId: threadId, stats: segmenter.getMemoryStats() });
+  } catch {
+    // ignore
+  }
+}, 1_000);
+
+parentPort?.on("message", (msg: any) => {
+  if (!msg || typeof msg !== "object") return;
+  if (msg.type === "stop") {
+    try {
+      clearInterval(memoryTimer);
+    } catch {
+      // ignore
+    }
+    try {
+      segmenter.stop();
+    } catch {
+      // ignore
+    }
+    try {
+      db.close();
+    } catch {
+      // ignore
+    }
+    try {
+      memorySampler?.stop();
+    } catch {
+      // ignore
+    }
+    try {
+      parentPort?.postMessage({ type: "stopped" });
+    } catch {
+      // ignore
+    }
+  }
+});

package/src/segment/segmenter_workers.ts

@@ -0,0 +1,130 @@
+import { Worker } from "node:worker_threads";
+import type { Config } from "../config";
+import { detectHostRuntime } from "../runtime/host_runtime.ts";
+import { resolveWorkerModuleUrl } from "../compute/worker_module_url";
+import type { SegmenterHooks, SegmenterMemoryStats, SegmenterOptions } from "./segmenter";
+
+export type SegmenterController = {
+  start: () => void;
+  stop: (hard?: boolean) => void | Promise<void>;
+  getMemoryStats?: () => SegmenterMemoryStats;
+};
+
+type WorkerMessage =
+  | { type: "sealed"; stream: string; payloadBytes: number; segmentBytes: number }
+  | { type: "memory"; workerId: number; stats: SegmenterMemoryStats }
+  | { type: "stopped" };
+
+export class SegmenterWorkerPool implements SegmenterController {
+  private readonly config: Config;
+  private readonly workerCount: number;
+  private readonly opts: SegmenterOptions;
+  private readonly hooks?: SegmenterHooks;
+  private readonly workers: Worker[] = [];
+  private readonly workerMemory = new Map<number, { stats: SegmenterMemoryStats; reportedAtMs: number }>();
+  private started = false;
+
+  constructor(config: Config, workerCount: number, opts: SegmenterOptions = {}, hooks?: SegmenterHooks) {
+    this.config = config;
+    this.workerCount = Math.max(0, Math.floor(workerCount));
+    this.opts = opts;
+    this.hooks = hooks;
+  }
+
+  start(): void {
+    if (this.started) return;
+    this.started = true;
+    for (let i = 0; i < this.workerCount; i++) {
+      this.spawnWorker(i);
+    }
+  }
+
+  async stop(_hard?: boolean): Promise<void> {
+    if (!this.started) return;
+    this.started = false;
+    const workers = this.workers.slice();
+    this.workers.length = 0;
+    this.workerMemory.clear();
+    // Await termination so the worker threads are gone before stop() resolves;
+    // see the note in TouchProcessorWorkerPool.stop -- a lingering worker thread
+    // racing the host process's WASM teardown can abort the process on Linux.
+    await Promise.all(
+      workers.map((w) => {
+        try {
+          w.postMessage({ type: "stop" });
+        } catch {
+          // ignore
+        }
+        return w.terminate();
+      }),
+    );
+  }
+
+  getMemoryStats(): SegmenterMemoryStats {
+    const now = Date.now();
+    let activeBuilds = 0;
+    let activeStreams = 0;
+    let activePayloadBytes = 0;
+    let activeSegmentBytesEstimate = 0;
+    let activeRows = 0;
+    for (const [workerId, entry] of this.workerMemory) {
+      if (now - entry.reportedAtMs > 5_000) {
+        this.workerMemory.delete(workerId);
+        continue;
+      }
+      activeBuilds += Math.max(0, entry.stats.active_builds);
+      activeStreams += Math.max(0, entry.stats.active_streams);
+      activePayloadBytes += Math.max(0, entry.stats.active_payload_bytes);
+      activeSegmentBytesEstimate += Math.max(0, entry.stats.active_segment_bytes_estimate);
+      activeRows += Math.max(0, entry.stats.active_rows);
+    }
+    return {
+      active_builds: activeBuilds,
+      active_streams: activeStreams,
+      active_payload_bytes: activePayloadBytes,
+      active_segment_bytes_estimate: activeSegmentBytesEstimate,
+      active_rows: activeRows,
+    };
+  }
+
+  private spawnWorker(idx: number): void {
+    const workerSpec = resolveWorkerModuleUrl(import.meta.url, "./segmenter_worker.ts", "../segment/segmenter_worker.js");
+    const worker = new Worker(workerSpec, {
+      workerData: {
+        config: this.config,
+        hostRuntime: detectHostRuntime(),
+        opts: this.opts,
+      },
+      type: "module",
+      smol: true,
+    } as any);
+
+    worker.on("message", (msg: WorkerMessage) => {
+      if (msg?.type === "sealed") {
+        this.hooks?.onSegmentSealed?.(msg.stream, msg.payloadBytes, msg.segmentBytes);
+      } else if (msg?.type === "memory") {
+        this.workerMemory.set(msg.workerId, {
+          stats: msg.stats,
+          reportedAtMs: Date.now(),
+        });
+      }
+    });
+
+    worker.on("error", (err) => {
+      // eslint-disable-next-line no-console
+      console.error(`segmenter worker ${idx} error`, err);
+    });
+
+    worker.on("exit", (code) => {
+      this.workerMemory.delete(worker.threadId);
+      if (!this.started) return;
+      if (code !== 0) {
+        // eslint-disable-next-line no-console
+        console.error(`segmenter worker ${idx} exited with code ${code}, respawning`);
+        this.spawnWorker(idx);
+      }
+    });
+
+    this.workers.push(worker);
+  }
+}