@prisma/streams-server 0.0.1 → 0.1.1

package/src/segment/format.ts

@@ -0,0 +1,331 @@
+import { Result } from "better-result";
+import { zstdCompressSync, zstdDecompressSync } from "node:zlib";
+import { Bloom256 } from "../util/bloom256";
+import { crc32c } from "../util/crc32c";
+import { concatBytes, readU32BE, readU64BE, writeU32BE, writeU64BE } from "../util/endian";
+import { dsError } from "../util/ds_error.ts";
+
+export type SegmentRecord = {
+  appendNs: bigint;
+  routingKey: Uint8Array; // UTF8 bytes (may be empty)
+  payload: Uint8Array;
+};
+
+export type DecodedBlock = {
+  recordCount: number;
+  firstAppendNs: bigint;
+  lastAppendNs: bigint;
+  bloom: Uint8Array; // 32 bytes
+  records: SegmentRecord[];
+};
+
+export type BlockIndexEntry = {
+  blockOffset: number;
+  firstOffset: bigint;
+  recordCount: number;
+  compressedLen: number;
+  firstAppendNs: bigint;
+  lastAppendNs: bigint;
+};
+
+export type SegmentFooter = {
+  version: number;
+  blocks: BlockIndexEntry[];
+};
+
+export type ParsedFooter = {
+  footer: SegmentFooter | null;
+  footerStart: number;
+};
+
+export type BlockHeader = {
+  uncompressedLen: number;
+  compressedLen: number;
+  recordCount: number;
+  bloom: Uint8Array;
+  firstAppendNs: bigint;
+  lastAppendNs: bigint;
+  crc32c: number;
+};
+
+export type SegmentFormatError = {
+  kind: "invalid_segment_format";
+  message: string;
+};
+
+export type IterateBlockEntry = {
+  blockOffset: number;
+  blockBytes: Uint8Array;
+  decoded: DecodedBlock;
+};
+
+function invalidSegment<T = never>(message: string): Result<T, SegmentFormatError> {
+  return Result.err({ kind: "invalid_segment_format", message });
+}
+
+export const DSB3_HEADER_BYTES = 68;
+
+const FOOTER_MAGIC = "DSF1";
+const FOOTER_VERSION = 1;
+const FOOTER_ENTRY_BYTES = 40; // 8+8+4+4+8+8
+const FOOTER_TRAILER_BYTES = 8; // u32 len + 4-byte magic
+
+export function encodeRecord(rec: SegmentRecord): Uint8Array {
+  const keyLen = rec.routingKey.byteLength;
+  const dataLen = rec.payload.byteLength;
+  const out = new Uint8Array(8 + 4 + keyLen + 4 + dataLen);
+  writeU64BE(out, 0, rec.appendNs);
+  writeU32BE(out, 8, keyLen);
+  out.set(rec.routingKey, 12);
+  writeU32BE(out, 12 + keyLen, dataLen);
+  out.set(rec.payload, 16 + keyLen);
+  return out;
+}
+
+export function encodeBlock(records: SegmentRecord[]): Uint8Array {
+  const res = encodeBlockResult(records);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function encodeBlockResult(records: SegmentRecord[]): Result<Uint8Array, SegmentFormatError> {
+  if (records.length === 0) return invalidSegment("empty block");
+
+  const bloom = new Bloom256();
+  for (const r of records) bloom.add(r.routingKey);
+
+  const recBytes = records.map(encodeRecord);
+  const uncompressed = concatBytes(recBytes);
+  const compressed = new Uint8Array(zstdCompressSync(uncompressed));
+  const crc = crc32c(compressed);
+
+  const header = new Uint8Array(DSB3_HEADER_BYTES);
+  header[0] = "D".charCodeAt(0);
+  header[1] = "S".charCodeAt(0);
+  header[2] = "B".charCodeAt(0);
+  header[3] = "3".charCodeAt(0);
+  writeU32BE(header, 4, uncompressed.byteLength);
+  writeU32BE(header, 8, compressed.byteLength);
+  writeU32BE(header, 12, records.length);
+  header.set(bloom.toBytes(), 16);
+
+  const firstTs = records[0].appendNs;
+  const lastTs = records[records.length - 1].appendNs;
+  writeU64BE(header, 48, firstTs);
+  writeU64BE(header, 56, lastTs);
+  writeU32BE(header, 64, crc);
+
+  return Result.ok(concatBytes([header, compressed]));
+}
+
+export function decodeBlock(blockBytes: Uint8Array): DecodedBlock {
+  const res = decodeBlockResult(blockBytes);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function decodeBlockResult(blockBytes: Uint8Array): Result<DecodedBlock, SegmentFormatError> {
+  if (blockBytes.byteLength < DSB3_HEADER_BYTES) return invalidSegment("block too small");
+  if (
+    blockBytes[0] !== "D".charCodeAt(0) ||
+    blockBytes[1] !== "S".charCodeAt(0) ||
+    blockBytes[2] !== "B".charCodeAt(0) ||
+    blockBytes[3] !== "3".charCodeAt(0)
+  ) {
+    return invalidSegment("bad block magic");
+  }
+
+  const uncompressedLen = readU32BE(blockBytes, 4);
+  const compressedLen = readU32BE(blockBytes, 8);
+  const recordCount = readU32BE(blockBytes, 12);
+  const bloom = blockBytes.slice(16, 48);
+  const firstAppendNs = readU64BE(blockBytes, 48);
+  const lastAppendNs = readU64BE(blockBytes, 56);
+  const expectedCrc = readU32BE(blockBytes, 64);
+
+  const payload = blockBytes.slice(DSB3_HEADER_BYTES, DSB3_HEADER_BYTES + compressedLen);
+  if (payload.byteLength !== compressedLen) return invalidSegment("truncated block");
+  const actualCrc = crc32c(payload);
+  if (actualCrc !== expectedCrc) return invalidSegment("crc mismatch");
+
+  let uncompressed: Uint8Array;
+  try {
+    uncompressed = new Uint8Array(zstdDecompressSync(payload));
+  } catch (e: any) {
+    return invalidSegment(String(e?.message ?? e));
+  }
+  if (uncompressed.byteLength !== uncompressedLen) {
+    return invalidSegment(`bad uncompressed len: got=${uncompressed.byteLength} expected=${uncompressedLen}`);
+  }
+
+  const records: SegmentRecord[] = [];
+  let off = 0;
+  for (let i = 0; i < recordCount; i++) {
+    if (off + 8 + 4 > uncompressed.byteLength) return invalidSegment("truncated record");
+    const appendNs = readU64BE(uncompressed, off);
+    off += 8;
+    const keyLen = readU32BE(uncompressed, off);
+    off += 4;
+    if (off + keyLen + 4 > uncompressed.byteLength) return invalidSegment("truncated key");
+    const routingKey = uncompressed.slice(off, off + keyLen);
+    off += keyLen;
+    const dataLen = readU32BE(uncompressed, off);
+    off += 4;
+    if (off + dataLen > uncompressed.byteLength) return invalidSegment("truncated payload");
+    const payload = uncompressed.slice(off, off + dataLen);
+    off += dataLen;
+    records.push({ appendNs, routingKey, payload });
+  }
+
+  return Result.ok({ recordCount, firstAppendNs, lastAppendNs, bloom, records });
+}
+
+export function encodeFooter(entries: BlockIndexEntry[]): Uint8Array {
+  const footerLen = 12 + entries.length * FOOTER_ENTRY_BYTES;
+  const footer = new Uint8Array(footerLen + FOOTER_TRAILER_BYTES);
+  footer[0] = FOOTER_MAGIC.charCodeAt(0);
+  footer[1] = FOOTER_MAGIC.charCodeAt(1);
+  footer[2] = FOOTER_MAGIC.charCodeAt(2);
+  footer[3] = FOOTER_MAGIC.charCodeAt(3);
+  writeU32BE(footer, 4, FOOTER_VERSION);
+  writeU32BE(footer, 8, entries.length);
+
+  let off = 12;
+  for (const e of entries) {
+    writeU64BE(footer, off, BigInt(e.blockOffset));
+    off += 8;
+    writeU64BE(footer, off, e.firstOffset);
+    off += 8;
+    writeU32BE(footer, off, e.recordCount);
+    off += 4;
+    writeU32BE(footer, off, e.compressedLen);
+    off += 4;
+    writeU64BE(footer, off, e.firstAppendNs);
+    off += 8;
+    writeU64BE(footer, off, e.lastAppendNs);
+    off += 8;
+  }
+
+  // Trailer: footer length + magic
+  writeU32BE(footer, footerLen, footerLen);
+  footer[footerLen + 4] = FOOTER_MAGIC.charCodeAt(0);
+  footer[footerLen + 5] = FOOTER_MAGIC.charCodeAt(1);
+  footer[footerLen + 6] = FOOTER_MAGIC.charCodeAt(2);
+  footer[footerLen + 7] = FOOTER_MAGIC.charCodeAt(3);
+
+  return footer;
+}
+
+export function parseFooter(segmentBytes: Uint8Array): ParsedFooter | null {
+  if (segmentBytes.byteLength < FOOTER_TRAILER_BYTES) return null;
+  const tail = segmentBytes.slice(segmentBytes.byteLength - 4);
+  const tailMagic = String.fromCharCode(tail[0], tail[1], tail[2], tail[3]);
+  if (tailMagic !== FOOTER_MAGIC) return null;
+
+  const footerLen = readU32BE(segmentBytes, segmentBytes.byteLength - 8);
+  if (footerLen <= 0 || footerLen + FOOTER_TRAILER_BYTES > segmentBytes.byteLength) return null;
+
+  const footerStart = segmentBytes.byteLength - FOOTER_TRAILER_BYTES - footerLen;
+  if (footerStart < 0) return null;
+  const footer = segmentBytes.slice(footerStart, footerStart + footerLen);
+  const parsed = parseFooterBytes(footer);
+  return { footer: parsed, footerStart };
+}
+
+export function* iterateBlocksResult(
+  segmentBytes: Uint8Array
+): Generator<Result<IterateBlockEntry, SegmentFormatError>, void, void> {
+  const parsed = parseFooter(segmentBytes);
+  const limit = parsed ? parsed.footerStart : segmentBytes.byteLength;
+  let off = 0;
+  while (off < limit) {
+    if (off + DSB3_HEADER_BYTES > limit) {
+      yield invalidSegment("truncated segment (block header)");
+      return;
+    }
+    const header = segmentBytes.slice(off, off + DSB3_HEADER_BYTES);
+    const compressedLen = readU32BE(header, 8);
+    const totalLen = DSB3_HEADER_BYTES + compressedLen;
+    if (off + totalLen > limit) {
+      yield invalidSegment("truncated segment (block payload)");
+      return;
+    }
+    const blockBytes = segmentBytes.slice(off, off + totalLen);
+    const decodedRes = decodeBlockResult(blockBytes);
+    if (Result.isError(decodedRes)) {
+      yield decodedRes;
+      return;
+    }
+    yield Result.ok({ blockOffset: off, blockBytes, decoded: decodedRes.value });
+    off += totalLen;
+  }
+}
+
+export function* iterateBlocks(segmentBytes: Uint8Array): Generator<IterateBlockEntry, void, void> {
+  for (const itemRes of iterateBlocksResult(segmentBytes)) {
+    if (Result.isError(itemRes)) throw dsError(itemRes.error.message);
+    yield itemRes.value;
+  }
+}
+
+export function parseFooterBytes(footer: Uint8Array): SegmentFooter | null {
+  if (footer.byteLength < 12) return null;
+  const magic = String.fromCharCode(footer[0], footer[1], footer[2], footer[3]);
+  if (magic !== FOOTER_MAGIC) return null;
+  const version = readU32BE(footer, 4);
+  const blockCount = readU32BE(footer, 8);
+  const expectedLen = 12 + blockCount * FOOTER_ENTRY_BYTES;
+  if (footer.byteLength !== expectedLen) return null;
+  const blocks: BlockIndexEntry[] = [];
+  let off = 12;
+  for (let i = 0; i < blockCount; i++) {
+    const blockOffset = Number(readU64BE(footer, off));
+    off += 8;
+    const firstOffset = readU64BE(footer, off);
+    off += 8;
+    const recordCount = readU32BE(footer, off);
+    off += 4;
+    const compressedLen = readU32BE(footer, off);
+    off += 4;
+    const firstAppendNs = readU64BE(footer, off);
+    off += 8;
+    const lastAppendNs = readU64BE(footer, off);
+    off += 8;
+    blocks.push({ blockOffset, firstOffset, recordCount, compressedLen, firstAppendNs, lastAppendNs });
+  }
+  return { version, blocks };
+}
+
+export function parseBlockHeader(header: Uint8Array): BlockHeader {
+  const res = parseBlockHeaderResult(header);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function parseBlockHeaderResult(header: Uint8Array): Result<BlockHeader, SegmentFormatError> {
+  if (header.byteLength < DSB3_HEADER_BYTES) return invalidSegment("block header too small");
+  if (
+    header[0] !== "D".charCodeAt(0) ||
+    header[1] !== "S".charCodeAt(0) ||
+    header[2] !== "B".charCodeAt(0) ||
+    header[3] !== "3".charCodeAt(0)
+  ) {
+    return invalidSegment("bad block magic");
+  }
+  const uncompressedLen = readU32BE(header, 4);
+  const compressedLen = readU32BE(header, 8);
+  const recordCount = readU32BE(header, 12);
+  const bloom = header.slice(16, 48);
+  const firstAppendNs = readU64BE(header, 48);
+  const lastAppendNs = readU64BE(header, 56);
+  const crc32cVal = readU32BE(header, 64);
+  return Result.ok({
+    uncompressedLen,
+    compressedLen,
+    recordCount,
+    bloom,
+    firstAppendNs,
+    lastAppendNs,
+    crc32c: crc32cVal,
+  });
+}

package/src/segment/segmenter.ts

@@ -0,0 +1,326 @@
+import { mkdirSync, openSync, closeSync, writeSync, fsyncSync, renameSync, existsSync, unlinkSync } from "node:fs";
+import { dirname } from "node:path";
+import type { Config } from "../config";
+import type { SqliteDurableStore } from "../db/db";
+import { encodeBlock, encodeFooter, type BlockIndexEntry, type SegmentRecord } from "./format";
+import { readU32BE } from "../util/endian";
+import { localSegmentPath, streamHash16Hex } from "../util/stream_paths";
+import { LruCache } from "../util/lru";
+import { yieldToEventLoop } from "../util/yield";
+
+export type SegmenterOptions = {
+  minCandidateBytes?: number; // default: segmentMaxBytes
+  minCandidateRows?: number; // default: segmentTargetRows
+  maxIntervalMs?: number; // default: segmentMaxIntervalMs
+  candidatesPerTick?: number;
+  maxRowsPerSegment?: number;
+};
+
+export type SegmenterHooks = {
+  onSegmentSealed?: (payloadBytes: number, segmentBytes: number) => void;
+};
+
+export class Segmenter {
+  private readonly config: Config;
+  private readonly db: SqliteDurableStore;
+  private readonly opts: Required<SegmenterOptions>;
+  private readonly hooks?: SegmenterHooks;
+  private timer: any | null = null;
+  private running = false;
+  private stopping = false;
+  private readonly failures = new FailureTracker(1024);
+
+  constructor(config: Config, db: SqliteDurableStore, opts: SegmenterOptions = {}, hooks?: SegmenterHooks) {
+    this.config = config;
+    this.db = db;
+    this.opts = {
+      minCandidateBytes: opts.minCandidateBytes ?? config.segmentMaxBytes,
+      minCandidateRows: opts.minCandidateRows ?? config.segmentTargetRows,
+      maxIntervalMs: opts.maxIntervalMs ?? config.segmentMaxIntervalMs,
+      candidatesPerTick: opts.candidatesPerTick ?? 8,
+      maxRowsPerSegment: opts.maxRowsPerSegment ?? 250_000,
+    };
+    this.hooks = hooks;
+  }
+
+  start(): void {
+    this.stopping = false;
+    if (this.timer) return;
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, this.config.segmentCheckIntervalMs);
+  }
+
+  stop(hard = false): void {
+    if (hard) this.stopping = true;
+    else this.stopping = false;
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+  }
+
+  private async tick(): Promise<void> {
+    if (this.stopping) return;
+    if (this.running) return;
+    this.running = true;
+    try {
+      const candidates = this.db.candidates(
+        BigInt(this.opts.minCandidateBytes),
+        BigInt(this.opts.minCandidateRows),
+        BigInt(this.opts.maxIntervalMs),
+        this.opts.candidatesPerTick
+      );
+      for (const c of candidates) {
+        if (this.failures.shouldSkip(c.stream)) continue;
+        try {
+          await this.buildOne(c.stream);
+          this.failures.recordSuccess(c.stream);
+        } catch (e) {
+          this.failures.recordFailure(c.stream);
+          const msg = String((e as any)?.message ?? e);
+          const lower = msg.toLowerCase();
+          if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+            // eslint-disable-next-line no-console
+            console.error("segment build failed", c.stream, e);
+          }
+        }
+      }
+    } catch (e) {
+      const msg = String((e as any)?.message ?? e);
+      const lower = msg.toLowerCase();
+      if (!this.stopping && !lower.includes("database has closed") && !lower.includes("closed database") && !lower.includes("statement has finalized")) {
+        // eslint-disable-next-line no-console
+        console.error("segmenter tick error", e);
+      }
+    } finally {
+      this.running = false;
+    }
+  }
+
+  private isSqliteBusy(err: any): boolean {
+    const code = String(err?.code ?? "");
+    const errno = Number(err?.errno ?? -1);
+    return code === "SQLITE_BUSY" || code === "SQLITE_BUSY_SNAPSHOT" || errno === 5 || errno === 517;
+  }
+
+  private async runWithBusyRetry<T>(fn: () => T): Promise<T> {
+    const maxBusyMs = Math.max(0, this.config.ingestBusyTimeoutMs);
+    if (maxBusyMs <= 0) return fn();
+    const startMs = Date.now();
+    let attempt = 0;
+    for (;;) {
+      try {
+        return fn();
+      } catch (e) {
+        if (!this.isSqliteBusy(e)) throw e;
+        const elapsed = Date.now() - startMs;
+        if (elapsed >= maxBusyMs) throw e;
+        const delay = Math.min(200, 5 * 2 ** attempt);
+        attempt += 1;
+        await new Promise((res) => setTimeout(res, delay));
+      }
+    }
+  }
+
+  private cleanupTmp(tmpPath: string): void {
+    try {
+      if (existsSync(tmpPath)) unlinkSync(tmpPath);
+    } catch {
+      // ignore
+    }
+  }
+
+  private async buildOne(stream: string): Promise<void> {
+    if (this.stopping) return;
+    const row = this.db.getStream(stream);
+    if (!row || this.db.isDeleted(row)) return;
+    if (row.segment_in_progress) return;
+
+    const startOffset = row.sealed_through + 1n;
+    const maxOffset = row.next_offset - 1n;
+    if (startOffset > maxOffset) return;
+
+    // Claim.
+    if (!this.db.tryClaimSegment(stream)) return;
+
+    try {
+      const segmentIndex = this.db.nextSegmentIndexForStream(stream);
+      const shash = streamHash16Hex(stream);
+      const localPath = localSegmentPath(this.config.rootDir, shash, segmentIndex);
+      const tmpPath = `${localPath}.tmp`;
+      mkdirSync(dirname(localPath), { recursive: true });
+
+      // Build blocks and stream-write to temp file.
+      const fd = openSync(tmpPath, "w");
+      try {
+        let blockRecords: SegmentRecord[] = [];
+        let blockBytesApprox = 0;
+        let fileBytes = 0;
+        let blockCount = 0;
+        let blockFirstOffset = startOffset;
+        const blockIndex: BlockIndexEntry[] = [];
+
+        // Decide endOffset by scanning WAL rows until threshold.
+        // IMPORTANT: pending_bytes tracks WAL payload bytes only (not record/block overhead).
+        let payloadBytes = 0n;
+        let rowsSealed = 0n;
+        let endOffset = startOffset - 1n;
+        let lastAppendMs = 0n;
+
+        let lastYieldMs = Date.now();
+        let recordsSinceYield = 0;
+        for (const rec of this.db.iterWalRange(stream, startOffset, maxOffset)) {
+          const offset = BigInt(rec.offset);
+          const payload: Uint8Array = rec.payload;
+          const routingKey: Uint8Array | null = rec.routing_key ?? null;
+          const appendMs = BigInt(rec.ts_ms);
+          lastAppendMs = appendMs;
+
+          const keyBytes = routingKey ?? new Uint8Array(0);
+          const segRec: SegmentRecord = {
+            appendNs: appendMs * 1_000_000n,
+            routingKey: keyBytes,
+            payload,
+          };
+          const recSize = 8 + 4 + keyBytes.byteLength + 4 + payload.byteLength;
+
+          if (blockRecords.length > 0 && blockBytesApprox + recSize > this.config.blockMaxBytes) {
+            const blockOffset = fileBytes;
+            const block = encodeBlock(blockRecords);
+            const compressedLen = readU32BE(block, 8);
+            blockIndex.push({
+              blockOffset,
+              firstOffset: blockFirstOffset,
+              recordCount: blockRecords.length,
+              compressedLen,
+              firstAppendNs: blockRecords[0].appendNs,
+              lastAppendNs: blockRecords[blockRecords.length - 1].appendNs,
+            });
+            writeSync(fd, block);
+            fileBytes += block.byteLength;
+            blockCount += 1;
+            blockRecords = [];
+            blockBytesApprox = 0;
+            await yieldToEventLoop();
+          }
+
+          if (blockRecords.length === 0) blockFirstOffset = offset;
+          blockRecords.push(segRec);
+          blockBytesApprox += recSize;
+
+          payloadBytes += BigInt(payload.byteLength);
+          rowsSealed += 1n;
+          endOffset = offset;
+
+          recordsSinceYield += 1;
+          if (recordsSinceYield >= 512 || Date.now() - lastYieldMs >= 10) {
+            await yieldToEventLoop();
+            lastYieldMs = Date.now();
+            recordsSinceYield = 0;
+          }
+
+          if (payloadBytes >= BigInt(this.config.segmentMaxBytes)) break;
+          if (rowsSealed >= BigInt(this.opts.maxRowsPerSegment)) break;
+        }
+
+        if (rowsSealed === 0n) return;
+
+        if (blockRecords.length > 0) {
+          const blockOffset = fileBytes;
+          const block = encodeBlock(blockRecords);
+          const compressedLen = readU32BE(block, 8);
+          blockIndex.push({
+            blockOffset,
+            firstOffset: blockFirstOffset,
+            recordCount: blockRecords.length,
+            compressedLen,
+            firstAppendNs: blockRecords[0].appendNs,
+            lastAppendNs: blockRecords[blockRecords.length - 1].appendNs,
+          });
+          writeSync(fd, block);
+          fileBytes += block.byteLength;
+          blockCount += 1;
+        }
+
+        const footer = encodeFooter(blockIndex);
+        writeSync(fd, footer);
+        fileBytes += footer.byteLength;
+
+        fsyncSync(fd);
+
+        const segmentId = `${shash}-${segmentIndex}-${startOffset.toString()}-${endOffset.toString()}`;
+        renameSync(tmpPath, localPath);
+
+        if (!this.stopping) {
+          try {
+            await this.runWithBusyRetry(() => {
+              this.db.commitSealedSegment({
+                segmentId,
+                stream,
+                segmentIndex,
+                startOffset,
+                endOffset,
+                blockCount,
+                lastAppendMs,
+                sizeBytes: fileBytes,
+                localPath,
+                payloadBytes,
+                rowsSealed,
+              });
+            });
+            if (this.hooks?.onSegmentSealed) this.hooks.onSegmentSealed(Number(payloadBytes), fileBytes);
+          } catch (e) {
+            try {
+              if (existsSync(localPath)) unlinkSync(localPath);
+            } catch {
+              // ignore
+            }
+            throw e;
+          }
+        }
+      } finally {
+        closeSync(fd);
+        this.cleanupTmp(tmpPath);
+      }
+    } finally {
+      // Release claim.
+      if (!this.stopping) {
+        try {
+          this.db.setSegmentInProgress(stream, 0);
+        } catch {
+          // ignore
+        }
+      }
+    }
+  }
+}
+
+class FailureTracker {
+  private readonly cache: LruCache<string, { attempts: number; untilMs: number }>;
+
+  constructor(maxEntries: number) {
+    this.cache = new LruCache(maxEntries);
+  }
+
+  shouldSkip(stream: string): boolean {
+    const item = this.cache.get(stream);
+    if (!item) return false;
+    if (Date.now() >= item.untilMs) {
+      this.cache.delete(stream);
+      return false;
+    }
+    return true;
+  }
+
+  recordFailure(stream: string): void {
+    const now = Date.now();
+    const item = this.cache.get(stream) ?? { attempts: 0, untilMs: now };
+    item.attempts += 1;
+    const backoff = Math.min(60_000, 500 * 2 ** (item.attempts - 1));
+    item.untilMs = now + backoff;
+    this.cache.set(stream, item);
+  }
+
+  recordSuccess(stream: string): void {
+    this.cache.delete(stream);
+  }
+}

package/src/segment/segmenter_worker.ts

@@ -0,0 +1,43 @@
+import { parentPort, workerData } from "node:worker_threads";
+import type { Config } from "../config.ts";
+import { SqliteDurableStore } from "../db/db.ts";
+import { Segmenter, type SegmenterHooks, type SegmenterOptions } from "./segmenter.ts";
+import { initConsoleLogging } from "../util/log.ts";
+
+initConsoleLogging();
+
+const data = workerData as { config: Config; opts?: SegmenterOptions };
+const cfg = data.config;
+// The main server process initializes/migrates schema; workers should avoid
+// concurrent migrations on the same sqlite file.
+const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes, skipMigrations: true });
+
+const hooks: SegmenterHooks = {
+  onSegmentSealed: (payloadBytes, segmentBytes) => {
+    parentPort?.postMessage({ type: "sealed", payloadBytes, segmentBytes });
+  },
+};
+
+const segmenter = new Segmenter(cfg, db, data.opts ?? {}, hooks);
+segmenter.start();
+
+parentPort?.on("message", (msg: any) => {
+  if (!msg || typeof msg !== "object") return;
+  if (msg.type === "stop") {
+    try {
+      segmenter.stop();
+    } catch {
+      // ignore
+    }
+    try {
+      db.close();
+    } catch {
+      // ignore
+    }
+    try {
+      parentPort?.postMessage({ type: "stopped" });
+    } catch {
+      // ignore
+    }
+  }
+});