@yeaft/webchat-agent 0.1.519 → 0.1.521

package/unify/storage/atomic.js

@@ -0,0 +1,107 @@
+/**
+ * atomic.js — Crash-safe single-file writes.
+ *
+ * task-334o §Δ15 / §Δ22 §Δ23 shard-store foundation.
+ *
+ * Contract: writeAtomic(path, data) never leaves a half-written file at `path`.
+ * If the process crashes at any point, `path` is either the pre-existing content
+ * or the new content — never a torn mix. Leftover `*.tmp.<pid>.<n>` files are
+ * the only debris; they are safe to delete on boot (see sweepTmp()).
+ *
+ * Implementation:
+ *   1. Write bytes to `path.tmp.<pid>.<counter>` via writeFileSync.
+ *   2. fsync the tmp file (force bytes to disk before rename).
+ *   3. rename(tmp, path) — POSIX-atomic on same filesystem.
+ *   4. fsync the parent dir (persist the rename itself).
+ *
+ * Step 4 is what most naive "atomic write" implementations skip. Without it,
+ * a crash after the rename call returns can still lose the rename on ext4
+ * with data=ordered. We do the dir fsync on Linux/macOS; on Windows we skip
+ * (fsync on a directory is an error there) and accept the minor risk window.
+ *
+ * This module has no knowledge of VP/task/message — it's a pure primitive.
+ */
+
+import {
+  writeFileSync,
+  renameSync,
+  openSync,
+  fsyncSync,
+  closeSync,
+  existsSync,
+  unlinkSync,
+  readdirSync,
+} from 'fs';
+import { dirname, basename, join } from 'path';
+
+let tmpCounter = 0;
+
+/**
+ * Atomically write `data` (string | Buffer) to `path`.
+ * Throws on failure; never leaves `path` in a half-written state.
+ */
+export function writeAtomic(path, data) {
+  const dir = dirname(path);
+  const tmpPath = `${path}.tmp.${process.pid}.${++tmpCounter}`;
+
+  writeFileSync(tmpPath, data);
+
+  // fsync the tmp file so the bytes hit disk before we swap.
+  try {
+    const fd = openSync(tmpPath, 'r+');
+    try {
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+  } catch {
+    // Best-effort; some filesystems / platforms don't support fsync on a file
+    // opened r+. The rename below is still the atomic boundary.
+  }
+
+  renameSync(tmpPath, path);
+
+  // fsync the parent directory so the rename is durable.
+  // Windows: cannot fsync a directory; skip.
+  if (process.platform !== 'win32') {
+    try {
+      const dfd = openSync(dir, 'r');
+      try {
+        fsyncSync(dfd);
+      } finally {
+        closeSync(dfd);
+      }
+    } catch {
+      // Directory fsync is best-effort. A failure here does not invalidate
+      // the rename itself; it only weakens durability on power loss.
+    }
+  }
+}
+
+/**
+ * Remove any leftover `*.tmp.*` files in `dir` from a previous crashed write.
+ * Safe to call on boot. Returns the count removed.
+ *
+ * Only matches the specific `<basename>.tmp.<pid>.<counter>` shape — won't
+ * touch user files that happen to end in `.tmp`.
+ */
+export function sweepTmp(dir) {
+  if (!existsSync(dir)) return 0;
+  let removed = 0;
+  for (const name of readdirSync(dir)) {
+    if (/\.tmp\.\d+\.\d+$/.test(name)) {
+      try {
+        unlinkSync(join(dir, name));
+        removed++;
+      } catch {
+        // Ignore — another process may have beaten us to it.
+      }
+    }
+  }
+  return removed;
+}
+
+/** Check whether a given file path looks like our tmp sidecar. Test helper. */
+export function isTmpPath(path) {
+  return /\.tmp\.\d+\.\d+$/.test(basename(path));
+}

package/unify/storage/compact.js

@@ -0,0 +1,80 @@
+/**
+ * compact.js — External compaction entry point for 334g (dream).
+ *
+ * `runCompact({ dir, threshold, schema })` opens a shard store at `dir`,
+ * inspects each shard for softCap breach, and rewrites shard files that
+ * need it. Unlike the internal `compactShard` used by `remove()`, this is
+ * the call dream uses to periodically reclaim space across the whole store.
+ *
+ * This module does NOT:
+ *   - decide what's stale enough to delete (that's dream's job, using
+ *     `supersededBy` chains)
+ *   - re-score or re-rank entries
+ *   - touch memory files belonging to other stores (it only touches `dir`)
+ *
+ * Parameters:
+ *   dir       : shard-store directory
+ *   schema    : shard schema (shards[], softCap)
+ *   threshold : optional override — compact any shard where
+ *               entries >= threshold.entries || bytes >= threshold.bytes
+ *               (if omitted, uses the schema's softCap)
+ *   deleteIds : optional array of ids that dream has decided to purge
+ *               (lets dream do "compact + delete" in one pass)
+ *
+ * Returns:
+ *   { compacted: [shardName, ...], deleted: [id, ...], stillOver: [shardName, ...] }
+ */
+
+import { openShardStore } from './shard-store.js';
+
+export async function runCompact({ dir, schema = {}, threshold, deleteIds = [] } = {}) {
+  if (!dir) throw new Error('runCompact: dir required');
+  const store = openShardStore(dir, schema);
+  const stats = store.stats();
+  const compacted = [];
+  const deleted = [];
+  const stillOver = [];
+
+  // First pass: honour explicit deletions (dream hands us a hitlist).
+  const byShard = new Map();
+  for (const id of deleteIds) {
+    const entry = store.getIndex().entries.find((e) => e.id === id);
+    if (!entry) continue;
+    const list = byShard.get(entry.shard) || [];
+    list.push(id);
+    byShard.set(entry.shard, list);
+  }
+
+  for (const [shardName] of byShard) {
+    // Use public remove() which already calls the internal compacter.
+    for (const id of byShard.get(shardName)) {
+      if (store.remove(id)) deleted.push(id);
+    }
+    compacted.push(shardName);
+  }
+
+  // Second pass: defrag shards whose size still exceeds the threshold.
+  const thr = threshold || {};
+  for (const shardName of Object.keys(stats.shards)) {
+    if (compacted.includes(shardName)) continue;
+    const bucket = store.stats().shards[shardName];
+    if (!bucket) continue;
+    const cap = thr.entries != null || thr.bytes != null
+      ? thr
+      : (schema.softCap?.[shardName] || schema.defaultSoftCap);
+    if (!cap) continue;
+    const overEntries = cap.entries != null && bucket.entries > cap.entries;
+    const overBytes = cap.bytes != null && bucket.bytes > cap.bytes;
+    if (overEntries || overBytes) {
+      store.compact(shardName);
+      compacted.push(shardName);
+      // Re-read stats after compaction; if still over, surface to caller.
+      const newBucket = store.stats().shards[shardName];
+      if (newBucket.entries > (cap.entries ?? Infinity) || newBucket.bytes > (cap.bytes ?? Infinity)) {
+        stillOver.push(shardName);
+      }
+    }
+  }
+
+  return { compacted, deleted, stillOver };
+}

package/unify/storage/index.js

@@ -0,0 +1,44 @@
+/**
+ * storage/ — task-334o Storage Layer v1.
+ *
+ * Business-semantics-free primitives shared by 334b (group messages),
+ * 334f (VP memory), 334l (user memory), 334i (migration), 334n (summaries).
+ *
+ * Modules:
+ *   - atomic      : writeAtomic (tmp → rename + fsync)
+ *   - jsonl-log   : append-only log with size/line rotation
+ *   - jsonl-index : segment manifest for jsonl-log
+ *   - shard-store : schema-aware shard storage (get/put/query/remove/compact)
+ *   - shard-index : manifest for shard-store
+ *   - compact     : external compaction entry point for dream
+ *
+ * API stability: 334o freezes its public API. Downstream slices must not
+ * request new fields — extensions go through `shard-schema.js` versioning
+ * (see slice spec §framework §2).
+ */
+
+export { writeAtomic, sweepTmp, isTmpPath } from './atomic.js';
+export { openLog } from './jsonl-log.js';
+export {
+  emptyIndex,
+  loadIndex,
+  saveIndex,
+  listSegmentFiles,
+  statSegmentFromDisk,
+  nextSegmentName,
+  INDEX_FILE,
+  INDEX_VERSION,
+} from './jsonl-index.js';
+export { openShardStore } from './shard-store.js';
+export {
+  emptyShardIndex,
+  loadShardIndex,
+  saveShardIndex,
+  rebuildShardIndexFromDisk,
+  shardFileName,
+  START_MARK,
+  END_MARK,
+  SHARD_INDEX_FILE,
+  SHARD_INDEX_VERSION,
+} from './shard-index.js';
+export { runCompact } from './compact.js';

package/unify/storage/jsonl-index.js

@@ -0,0 +1,122 @@
+/**
+ * jsonl-index.js — Manifest for a jsonl-log directory.
+ *
+ * The index file tracks which segment files cover which ID/timestamp ranges
+ * so `jsonl-log.readRange` can do O(1) segment selection without scanning
+ * the whole log. It's rewritten atomically on every rotation.
+ *
+ * Schema (index.json):
+ *   {
+ *     version:  1,
+ *     nextId:   <number | null>,     // optional — caller-managed id counter
+ *     segments: [
+ *       { file:"000001.jsonl", firstId, lastId, firstTs, lastTs, count, bytes }
+ *     ]
+ *   }
+ *
+ * This module owns ONLY the manifest — it does not read or write the jsonl
+ * segments themselves. jsonl-log.js drives rotation and hands us updated
+ * segment metadata.
+ *
+ * No business semantics. "id" and "ts" are opaque; we don't care if they're
+ * msg_xxx, mem_xxx, numeric, or empty.
+ */
+
+import { readFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'fs';
+import { join } from 'path';
+import { writeAtomic } from './atomic.js';
+
+export const INDEX_FILE = 'index.json';
+export const INDEX_VERSION = 1;
+
+/** Build an empty manifest. */
+export function emptyIndex() {
+  return { version: INDEX_VERSION, nextId: null, segments: [] };
+}
+
+/**
+ * Load the index from `dir/index.json`. Returns `null` if the file is missing,
+ * unreadable, or corrupt (caller should rebuild). Does NOT auto-rebuild —
+ * the caller decides whether a missing / corrupt index is fatal.
+ */
+export function loadIndex(dir) {
+  const path = join(dir, INDEX_FILE);
+  if (!existsSync(path)) return null;
+  try {
+    const raw = readFileSync(path, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== 'object') return null;
+    if (!Array.isArray(parsed.segments)) return null;
+    return {
+      version: parsed.version || INDEX_VERSION,
+      nextId: parsed.nextId ?? null,
+      segments: parsed.segments,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Atomically persist an index. Creates `dir` if missing. */
+export function saveIndex(dir, index) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const payload = JSON.stringify({
+    version: INDEX_VERSION,
+    nextId: index.nextId ?? null,
+    segments: index.segments,
+  }, null, 2);
+  writeAtomic(join(dir, INDEX_FILE), payload);
+}
+
+/**
+ * List *.jsonl segment files in `dir`, sorted lexicographically.
+ * Returns names only, not full paths.
+ */
+export function listSegmentFiles(dir) {
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((n) => /^\d+\.jsonl$/.test(n))
+    .sort();
+}
+
+/**
+ * Produce an index entry by reading a segment file off disk. Used when
+ * recovering from a missing/corrupt index.json.
+ *
+ * Caller provides `parseLine(line) -> { id, ts }` so we can populate the
+ * firstId/lastId/firstTs/lastTs fields without this module knowing the schema.
+ * Malformed lines are skipped (best-effort rebuild).
+ */
+export function statSegmentFromDisk(dir, fileName, parseLine) {
+  const path = join(dir, fileName);
+  const bytes = statSync(path).size;
+  const raw = readFileSync(path, 'utf8');
+  const lines = raw.split('\n').filter((l) => l.length > 0);
+  let firstId = null, lastId = null, firstTs = null, lastTs = null;
+  let count = 0;
+  for (const line of lines) {
+    let parsed;
+    try {
+      parsed = parseLine(line);
+    } catch {
+      continue;
+    }
+    if (!parsed) continue;
+    count++;
+    if (firstId === null) firstId = parsed.id ?? null;
+    lastId = parsed.id ?? lastId;
+    if (firstTs === null) firstTs = parsed.ts ?? null;
+    lastTs = parsed.ts ?? lastTs;
+  }
+  return { file: fileName, firstId, lastId, firstTs, lastTs, count, bytes };
+}
+
+/** Pick the highest segment number in `segments` (caller decides next filename). */
+export function nextSegmentName(segments) {
+  let max = 0;
+  for (const seg of segments) {
+    const m = /^(\d+)\.jsonl$/.exec(seg.file);
+    if (m) max = Math.max(max, parseInt(m[1], 10));
+  }
+  return String(max + 1).padStart(6, '0') + '.jsonl';
+}

package/unify/storage/jsonl-log.js

@@ -0,0 +1,222 @@
+/**
+ * jsonl-log.js — Append-only JSONL log with size-based rotation.
+ *
+ * task-334o acceptance criterion 1:
+ *   - append single-line write < 2ms on local SSD
+ *   - rotate at maxSegmentBytes=1 MiB OR maxSegmentLines=5000 (first to hit)
+ *   - index.json is atomically updated on every rotation
+ *
+ * Layout (managed here):
+ *   <dir>/000001.jsonl
+ *   <dir>/000002.jsonl
+ *   <dir>/index.json
+ *
+ * API:
+ *   const log = openLog(dir, { maxSegmentBytes, maxSegmentLines, parseLine });
+ *   log.append(obj)              // writes JSON.stringify(obj) + '\n'
+ *   log.readRange(firstId, lastId)  // iterable of records in [firstId, lastId]
+ *   log.streamAll()              // iterable of all records, oldest -> newest
+ *   log.rotate()                 // force rotate (test hook)
+ *   log.close()                  // close fd, flush index
+ *
+ * No business semantics: the log does not know what an id means; it only
+ * needs the caller to provide a `parseLine(line) -> {id, ts}` so the index
+ * metadata can be rebuilt from disk if index.json is lost or corrupt.
+ */
+
+import {
+  openSync,
+  closeSync,
+  writeSync,
+  readFileSync,
+  existsSync,
+  mkdirSync,
+  statSync,
+} from 'fs';
+import { join } from 'path';
+import {
+  loadIndex,
+  saveIndex,
+  listSegmentFiles,
+  statSegmentFromDisk,
+  nextSegmentName,
+  emptyIndex,
+} from './jsonl-index.js';
+
+const DEFAULT_MAX_BYTES = 1 * 1024 * 1024; // 1 MiB
+const DEFAULT_MAX_LINES = 5000;
+
+const defaultParseLine = (line) => {
+  const obj = JSON.parse(line);
+  return { id: obj.id ?? null, ts: obj.ts ?? null };
+};
+
+/**
+ * Open (or create) an append-only JSONL log rooted at `dir`.
+ * On startup, verifies / rebuilds index.json against files on disk.
+ */
+export function openLog(dir, opts = {}) {
+  const maxSegmentBytes = opts.maxSegmentBytes ?? DEFAULT_MAX_BYTES;
+  const maxSegmentLines = opts.maxSegmentLines ?? DEFAULT_MAX_LINES;
+  const parseLine = opts.parseLine ?? defaultParseLine;
+
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+  let index = loadIndex(dir);
+  const filesOnDisk = listSegmentFiles(dir);
+
+  // Rebuild index if missing, corrupt, or out-of-sync with actual segment files.
+  if (!index || !indexMatchesDisk(index, filesOnDisk)) {
+    index = rebuildIndexFromDisk(dir, filesOnDisk, parseLine);
+    saveIndex(dir, index);
+  }
+
+  // Current segment = last one in the index; create 000001.jsonl if empty.
+  let current = index.segments[index.segments.length - 1];
+  if (!current) {
+    current = {
+      file: '000001.jsonl',
+      firstId: null, lastId: null,
+      firstTs: null, lastTs: null,
+      count: 0, bytes: 0,
+    };
+    index.segments.push(current);
+    // Touch the file so the fd is openable.
+    const path = join(dir, current.file);
+    if (!existsSync(path)) closeSync(openSync(path, 'a'));
+  }
+
+  let fd = openSync(join(dir, current.file), 'a');
+
+  /** Detect & perform rotation when current segment is full. */
+  function maybeRotate() {
+    if (current.bytes >= maxSegmentBytes || current.count >= maxSegmentLines) {
+      rotate();
+    }
+  }
+
+  function rotate() {
+    // Close current fd.
+    closeSync(fd);
+    // Open new segment.
+    const newFile = nextSegmentName(index.segments);
+    const newSeg = {
+      file: newFile,
+      firstId: null, lastId: null,
+      firstTs: null, lastTs: null,
+      count: 0, bytes: 0,
+    };
+    index.segments.push(newSeg);
+    current = newSeg;
+    // Persist the rotation atomically before we start writing to the new file
+    // so a crash mid-rotation doesn't lose the boundary.
+    saveIndex(dir, index);
+    fd = openSync(join(dir, current.file), 'a');
+  }
+
+  function append(obj) {
+    const line = JSON.stringify(obj) + '\n';
+    const buf = Buffer.from(line, 'utf8');
+    writeSync(fd, buf, 0, buf.length);
+    current.count += 1;
+    current.bytes += buf.length;
+    const id = obj.id ?? null;
+    const ts = obj.ts ?? null;
+    if (current.firstId === null) current.firstId = id;
+    current.lastId = id;
+    if (current.firstTs === null) current.firstTs = ts;
+    current.lastTs = ts;
+    maybeRotate();
+  }
+
+  function* streamAll() {
+    for (const seg of index.segments) {
+      const path = join(dir, seg.file);
+      if (!existsSync(path)) continue;
+      const raw = readFileSync(path, 'utf8');
+      for (const line of raw.split('\n')) {
+        if (!line) continue;
+        try {
+          yield JSON.parse(line);
+        } catch {
+          // Skip malformed line — don't crash the read pipeline.
+        }
+      }
+    }
+  }
+
+  /** Read records whose id falls in `[firstId, lastId]` inclusive (string or number). */
+  function* readRange(firstId, lastId) {
+    for (const seg of index.segments) {
+      if (!segmentOverlaps(seg, firstId, lastId)) continue;
+      const path = join(dir, seg.file);
+      if (!existsSync(path)) continue;
+      const raw = readFileSync(path, 'utf8');
+      for (const line of raw.split('\n')) {
+        if (!line) continue;
+        let obj;
+        try { obj = JSON.parse(line); } catch { continue; }
+        const id = obj.id ?? null;
+        if (id === null) continue;
+        if (idBetween(id, firstId, lastId)) yield obj;
+      }
+    }
+  }
+
+  function setNextId(n) { index.nextId = n; saveIndex(dir, index); }
+  function getNextId() { return index.nextId; }
+  function getIndex() { return index; }
+  function flushIndex() { saveIndex(dir, index); }
+
+  function close() {
+    try { closeSync(fd); } catch { /* already closed */ }
+    saveIndex(dir, index);
+  }
+
+  return {
+    append, readRange, streamAll, rotate, close,
+    setNextId, getNextId, getIndex, flushIndex,
+  };
+}
+
+/** True when index's segment list matches exactly the files on disk. */
+function indexMatchesDisk(index, filesOnDisk) {
+  const indexFiles = index.segments.map((s) => s.file);
+  if (indexFiles.length !== filesOnDisk.length) return false;
+  for (let i = 0; i < indexFiles.length; i++) {
+    if (indexFiles[i] !== filesOnDisk[i]) return false;
+  }
+  return true;
+}
+
+/**
+ * Rebuild the index by scanning every segment file on disk. Expensive but
+ * only happens on first open or after index.json loss.
+ */
+function rebuildIndexFromDisk(dir, files, parseLine) {
+  const idx = emptyIndex();
+  for (const file of files) {
+    idx.segments.push(statSegmentFromDisk(dir, file, parseLine));
+  }
+  return idx;
+}
+
+/**
+ * Cheap overlap test. If a segment's firstId/lastId are null (empty segment),
+ * we treat it as non-overlapping.
+ */
+function segmentOverlaps(seg, first, last) {
+  if (seg.firstId == null || seg.lastId == null) return false;
+  // Ordering on strings is lexicographic which matches ULID-style ids used
+  // by the product (msg_01HW...). For numeric ids, JS > / < works too.
+  return !(compareId(seg.lastId, first) < 0 || compareId(seg.firstId, last) > 0);
+}
+
+function idBetween(id, first, last) {
+  return compareId(id, first) >= 0 && compareId(id, last) <= 0;
+}
+
+function compareId(a, b) {
+  if (a === b) return 0;
+  return a < b ? -1 : 1;
+}