@yeaft/webchat-agent 0.1.519 → 0.1.520

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.519",
+  "version": "0.1.520",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/storage/atomic.js

@@ -0,0 +1,107 @@
+/**
+ * atomic.js — Crash-safe single-file writes.
+ *
+ * task-334o §Δ15 / §Δ22 §Δ23 shard-store foundation.
+ *
+ * Contract: writeAtomic(path, data) never leaves a half-written file at `path`.
+ * If the process crashes at any point, `path` is either the pre-existing content
+ * or the new content — never a torn mix. Leftover `*.tmp.<pid>.<n>` files are
+ * the only debris; they are safe to delete on boot (see sweepTmp()).
+ *
+ * Implementation:
+ *   1. Write bytes to `path.tmp.<pid>.<counter>` via writeFileSync.
+ *   2. fsync the tmp file (force bytes to disk before rename).
+ *   3. rename(tmp, path) — POSIX-atomic on same filesystem.
+ *   4. fsync the parent dir (persist the rename itself).
+ *
+ * Step 4 is what most naive "atomic write" implementations skip. Without it,
+ * a crash after the rename call returns can still lose the rename on ext4
+ * with data=ordered. We do the dir fsync on Linux/macOS; on Windows we skip
+ * (fsync on a directory is an error there) and accept the minor risk window.
+ *
+ * This module has no knowledge of VP/task/message — it's a pure primitive.
+ */
+
+import {
+  writeFileSync,
+  renameSync,
+  openSync,
+  fsyncSync,
+  closeSync,
+  existsSync,
+  unlinkSync,
+  readdirSync,
+} from 'fs';
+import { dirname, basename, join } from 'path';
+
+let tmpCounter = 0;
+
+/**
+ * Atomically write `data` (string | Buffer) to `path`.
+ * Throws on failure; never leaves `path` in a half-written state.
+ */
+export function writeAtomic(path, data) {
+  const dir = dirname(path);
+  const tmpPath = `${path}.tmp.${process.pid}.${++tmpCounter}`;
+
+  writeFileSync(tmpPath, data);
+
+  // fsync the tmp file so the bytes hit disk before we swap.
+  try {
+    const fd = openSync(tmpPath, 'r+');
+    try {
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+  } catch {
+    // Best-effort; some filesystems / platforms don't support fsync on a file
+    // opened r+. The rename below is still the atomic boundary.
+  }
+
+  renameSync(tmpPath, path);
+
+  // fsync the parent directory so the rename is durable.
+  // Windows: cannot fsync a directory; skip.
+  if (process.platform !== 'win32') {
+    try {
+      const dfd = openSync(dir, 'r');
+      try {
+        fsyncSync(dfd);
+      } finally {
+        closeSync(dfd);
+      }
+    } catch {
+      // Directory fsync is best-effort. A failure here does not invalidate
+      // the rename itself; it only weakens durability on power loss.
+    }
+  }
+}
+
+/**
+ * Remove any leftover `*.tmp.*` files in `dir` from a previous crashed write.
+ * Safe to call on boot. Returns the count removed.
+ *
+ * Only matches the specific `<basename>.tmp.<pid>.<counter>` shape — won't
+ * touch user files that happen to end in `.tmp`.
+ */
+export function sweepTmp(dir) {
+  if (!existsSync(dir)) return 0;
+  let removed = 0;
+  for (const name of readdirSync(dir)) {
+    if (/\.tmp\.\d+\.\d+$/.test(name)) {
+      try {
+        unlinkSync(join(dir, name));
+        removed++;
+      } catch {
+        // Ignore — another process may have beaten us to it.
+      }
+    }
+  }
+  return removed;
+}
+
+/** Check whether a given file path looks like our tmp sidecar. Test helper. */
+export function isTmpPath(path) {
+  return /\.tmp\.\d+\.\d+$/.test(basename(path));
+}

package/unify/storage/compact.js

@@ -0,0 +1,80 @@
+/**
+ * compact.js — External compaction entry point for 334g (dream).
+ *
+ * `runCompact({ dir, threshold, schema })` opens a shard store at `dir`,
+ * inspects each shard for softCap breach, and rewrites shard files that
+ * need it. Unlike the internal `compactShard` used by `remove()`, this is
+ * the call dream uses to periodically reclaim space across the whole store.
+ *
+ * This module does NOT:
+ *   - decide what's stale enough to delete (that's dream's job, using
+ *     `supersededBy` chains)
+ *   - re-score or re-rank entries
+ *   - touch memory files belonging to other stores (it only touches `dir`)
+ *
+ * Parameters:
+ *   dir       : shard-store directory
+ *   schema    : shard schema (shards[], softCap)
+ *   threshold : optional override — compact any shard where
+ *               entries >= threshold.entries || bytes >= threshold.bytes
+ *               (if omitted, uses the schema's softCap)
+ *   deleteIds : optional array of ids that dream has decided to purge
+ *               (lets dream do "compact + delete" in one pass)
+ *
+ * Returns:
+ *   { compacted: [shardName, ...], deleted: [id, ...], stillOver: [shardName, ...] }
+ */
+
+import { openShardStore } from './shard-store.js';
+
+export async function runCompact({ dir, schema = {}, threshold, deleteIds = [] } = {}) {
+  if (!dir) throw new Error('runCompact: dir required');
+  const store = openShardStore(dir, schema);
+  const stats = store.stats();
+  const compacted = [];
+  const deleted = [];
+  const stillOver = [];
+
+  // First pass: honour explicit deletions (dream hands us a hitlist).
+  const byShard = new Map();
+  for (const id of deleteIds) {
+    const entry = store.getIndex().entries.find((e) => e.id === id);
+    if (!entry) continue;
+    const list = byShard.get(entry.shard) || [];
+    list.push(id);
+    byShard.set(entry.shard, list);
+  }
+
+  for (const [shardName] of byShard) {
+    // Use public remove() which already calls the internal compacter.
+    for (const id of byShard.get(shardName)) {
+      if (store.remove(id)) deleted.push(id);
+    }
+    compacted.push(shardName);
+  }
+
+  // Second pass: defrag shards whose size still exceeds the threshold.
+  const thr = threshold || {};
+  for (const shardName of Object.keys(stats.shards)) {
+    if (compacted.includes(shardName)) continue;
+    const bucket = store.stats().shards[shardName];
+    if (!bucket) continue;
+    const cap = thr.entries != null || thr.bytes != null
+      ? thr
+      : (schema.softCap?.[shardName] || schema.defaultSoftCap);
+    if (!cap) continue;
+    const overEntries = cap.entries != null && bucket.entries > cap.entries;
+    const overBytes = cap.bytes != null && bucket.bytes > cap.bytes;
+    if (overEntries || overBytes) {
+      store.compact(shardName);
+      compacted.push(shardName);
+      // Re-read stats after compaction; if still over, surface to caller.
+      const newBucket = store.stats().shards[shardName];
+      if (newBucket.entries > (cap.entries ?? Infinity) || newBucket.bytes > (cap.bytes ?? Infinity)) {
+        stillOver.push(shardName);
+      }
+    }
+  }
+
+  return { compacted, deleted, stillOver };
+}

package/unify/storage/index.js

@@ -0,0 +1,44 @@
+/**
+ * storage/ — task-334o Storage Layer v1.
+ *
+ * Business-semantics-free primitives shared by 334b (group messages),
+ * 334f (VP memory), 334l (user memory), 334i (migration), 334n (summaries).
+ *
+ * Modules:
+ *   - atomic      : writeAtomic (tmp → rename + fsync)
+ *   - jsonl-log   : append-only log with size/line rotation
+ *   - jsonl-index : segment manifest for jsonl-log
+ *   - shard-store : schema-aware shard storage (get/put/query/remove/compact)
+ *   - shard-index : manifest for shard-store
+ *   - compact     : external compaction entry point for dream
+ *
+ * API stability: 334o freezes its public API. Downstream slices must not
+ * request new fields — extensions go through `shard-schema.js` versioning
+ * (see slice spec §framework §2).
+ */
+
+export { writeAtomic, sweepTmp, isTmpPath } from './atomic.js';
+export { openLog } from './jsonl-log.js';
+export {
+  emptyIndex,
+  loadIndex,
+  saveIndex,
+  listSegmentFiles,
+  statSegmentFromDisk,
+  nextSegmentName,
+  INDEX_FILE,
+  INDEX_VERSION,
+} from './jsonl-index.js';
+export { openShardStore } from './shard-store.js';
+export {
+  emptyShardIndex,
+  loadShardIndex,
+  saveShardIndex,
+  rebuildShardIndexFromDisk,
+  shardFileName,
+  START_MARK,
+  END_MARK,
+  SHARD_INDEX_FILE,
+  SHARD_INDEX_VERSION,
+} from './shard-index.js';
+export { runCompact } from './compact.js';

package/unify/storage/jsonl-index.js

@@ -0,0 +1,122 @@
+/**
+ * jsonl-index.js — Manifest for a jsonl-log directory.
+ *
+ * The index file tracks which segment files cover which ID/timestamp ranges
+ * so `jsonl-log.readRange` can do O(1) segment selection without scanning
+ * the whole log. It's rewritten atomically on every rotation.
+ *
+ * Schema (index.json):
+ *   {
+ *     version:  1,
+ *     nextId:   <number | null>,     // optional — caller-managed id counter
+ *     segments: [
+ *       { file:"000001.jsonl", firstId, lastId, firstTs, lastTs, count, bytes }
+ *     ]
+ *   }
+ *
+ * This module owns ONLY the manifest — it does not read or write the jsonl
+ * segments themselves. jsonl-log.js drives rotation and hands us updated
+ * segment metadata.
+ *
+ * No business semantics. "id" and "ts" are opaque; we don't care if they're
+ * msg_xxx, mem_xxx, numeric, or empty.
+ */
+
+import { readFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'fs';
+import { join } from 'path';
+import { writeAtomic } from './atomic.js';
+
+export const INDEX_FILE = 'index.json';
+export const INDEX_VERSION = 1;
+
+/** Build an empty manifest. */
+export function emptyIndex() {
+  return { version: INDEX_VERSION, nextId: null, segments: [] };
+}
+
+/**
+ * Load the index from `dir/index.json`. Returns `null` if the file is missing,
+ * unreadable, or corrupt (caller should rebuild). Does NOT auto-rebuild —
+ * the caller decides whether a missing / corrupt index is fatal.
+ */
+export function loadIndex(dir) {
+  const path = join(dir, INDEX_FILE);
+  if (!existsSync(path)) return null;
+  try {
+    const raw = readFileSync(path, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== 'object') return null;
+    if (!Array.isArray(parsed.segments)) return null;
+    return {
+      version: parsed.version || INDEX_VERSION,
+      nextId: parsed.nextId ?? null,
+      segments: parsed.segments,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Atomically persist an index. Creates `dir` if missing. */
+export function saveIndex(dir, index) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const payload = JSON.stringify({
+    version: INDEX_VERSION,
+    nextId: index.nextId ?? null,
+    segments: index.segments,
+  }, null, 2);
+  writeAtomic(join(dir, INDEX_FILE), payload);
+}
+
+/**
+ * List *.jsonl segment files in `dir`, sorted lexicographically.
+ * Returns names only, not full paths.
+ */
+export function listSegmentFiles(dir) {
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((n) => /^\d+\.jsonl$/.test(n))
+    .sort();
+}
+
+/**
+ * Produce an index entry by reading a segment file off disk. Used when
+ * recovering from a missing/corrupt index.json.
+ *
+ * Caller provides `parseLine(line) -> { id, ts }` so we can populate the
+ * firstId/lastId/firstTs/lastTs fields without this module knowing the schema.
+ * Malformed lines are skipped (best-effort rebuild).
+ */
+export function statSegmentFromDisk(dir, fileName, parseLine) {
+  const path = join(dir, fileName);
+  const bytes = statSync(path).size;
+  const raw = readFileSync(path, 'utf8');
+  const lines = raw.split('\n').filter((l) => l.length > 0);
+  let firstId = null, lastId = null, firstTs = null, lastTs = null;
+  let count = 0;
+  for (const line of lines) {
+    let parsed;
+    try {
+      parsed = parseLine(line);
+    } catch {
+      continue;
+    }
+    if (!parsed) continue;
+    count++;
+    if (firstId === null) firstId = parsed.id ?? null;
+    lastId = parsed.id ?? lastId;
+    if (firstTs === null) firstTs = parsed.ts ?? null;
+    lastTs = parsed.ts ?? lastTs;
+  }
+  return { file: fileName, firstId, lastId, firstTs, lastTs, count, bytes };
+}
+
+/** Pick the highest segment number in `segments` (caller decides next filename). */
+export function nextSegmentName(segments) {
+  let max = 0;
+  for (const seg of segments) {
+    const m = /^(\d+)\.jsonl$/.exec(seg.file);
+    if (m) max = Math.max(max, parseInt(m[1], 10));
+  }
+  return String(max + 1).padStart(6, '0') + '.jsonl';
+}

package/unify/storage/jsonl-log.js

@@ -0,0 +1,222 @@
+/**
+ * jsonl-log.js — Append-only JSONL log with size-based rotation.
+ *
+ * task-334o acceptance criterion 1:
+ *   - append single-line write < 2ms on local SSD
+ *   - rotate at maxSegmentBytes=1 MiB OR maxSegmentLines=5000 (first to hit)
+ *   - index.json is atomically updated on every rotation
+ *
+ * Layout (managed here):
+ *   <dir>/000001.jsonl
+ *   <dir>/000002.jsonl
+ *   <dir>/index.json
+ *
+ * API:
+ *   const log = openLog(dir, { maxSegmentBytes, maxSegmentLines, parseLine });
+ *   log.append(obj)              // writes JSON.stringify(obj) + '\n'
+ *   log.readRange(firstId, lastId)  // iterable of records in [firstId, lastId]
+ *   log.streamAll()              // iterable of all records, oldest -> newest
+ *   log.rotate()                 // force rotate (test hook)
+ *   log.close()                  // close fd, flush index
+ *
+ * No business semantics: the log does not know what an id means; it only
+ * needs the caller to provide a `parseLine(line) -> {id, ts}` so the index
+ * metadata can be rebuilt from disk if index.json is lost or corrupt.
+ */
+
+import {
+  openSync,
+  closeSync,
+  writeSync,
+  readFileSync,
+  existsSync,
+  mkdirSync,
+  statSync,
+} from 'fs';
+import { join } from 'path';
+import {
+  loadIndex,
+  saveIndex,
+  listSegmentFiles,
+  statSegmentFromDisk,
+  nextSegmentName,
+  emptyIndex,
+} from './jsonl-index.js';
+
+const DEFAULT_MAX_BYTES = 1 * 1024 * 1024; // 1 MiB
+const DEFAULT_MAX_LINES = 5000;
+
+const defaultParseLine = (line) => {
+  const obj = JSON.parse(line);
+  return { id: obj.id ?? null, ts: obj.ts ?? null };
+};
+
+/**
+ * Open (or create) an append-only JSONL log rooted at `dir`.
+ * On startup, verifies / rebuilds index.json against files on disk.
+ */
+export function openLog(dir, opts = {}) {
+  const maxSegmentBytes = opts.maxSegmentBytes ?? DEFAULT_MAX_BYTES;
+  const maxSegmentLines = opts.maxSegmentLines ?? DEFAULT_MAX_LINES;
+  const parseLine = opts.parseLine ?? defaultParseLine;
+
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+  let index = loadIndex(dir);
+  const filesOnDisk = listSegmentFiles(dir);
+
+  // Rebuild index if missing, corrupt, or out-of-sync with actual segment files.
+  if (!index || !indexMatchesDisk(index, filesOnDisk)) {
+    index = rebuildIndexFromDisk(dir, filesOnDisk, parseLine);
+    saveIndex(dir, index);
+  }
+
+  // Current segment = last one in the index; create 000001.jsonl if empty.
+  let current = index.segments[index.segments.length - 1];
+  if (!current) {
+    current = {
+      file: '000001.jsonl',
+      firstId: null, lastId: null,
+      firstTs: null, lastTs: null,
+      count: 0, bytes: 0,
+    };
+    index.segments.push(current);
+    // Touch the file so the fd is openable.
+    const path = join(dir, current.file);
+    if (!existsSync(path)) closeSync(openSync(path, 'a'));
+  }
+
+  let fd = openSync(join(dir, current.file), 'a');
+
+  /** Detect & perform rotation when current segment is full. */
+  function maybeRotate() {
+    if (current.bytes >= maxSegmentBytes || current.count >= maxSegmentLines) {
+      rotate();
+    }
+  }
+
+  function rotate() {
+    // Close current fd.
+    closeSync(fd);
+    // Open new segment.
+    const newFile = nextSegmentName(index.segments);
+    const newSeg = {
+      file: newFile,
+      firstId: null, lastId: null,
+      firstTs: null, lastTs: null,
+      count: 0, bytes: 0,
+    };
+    index.segments.push(newSeg);
+    current = newSeg;
+    // Persist the rotation atomically before we start writing to the new file
+    // so a crash mid-rotation doesn't lose the boundary.
+    saveIndex(dir, index);
+    fd = openSync(join(dir, current.file), 'a');
+  }
+
+  function append(obj) {
+    const line = JSON.stringify(obj) + '\n';
+    const buf = Buffer.from(line, 'utf8');
+    writeSync(fd, buf, 0, buf.length);
+    current.count += 1;
+    current.bytes += buf.length;
+    const id = obj.id ?? null;
+    const ts = obj.ts ?? null;
+    if (current.firstId === null) current.firstId = id;
+    current.lastId = id;
+    if (current.firstTs === null) current.firstTs = ts;
+    current.lastTs = ts;
+    maybeRotate();
+  }
+
+  function* streamAll() {
+    for (const seg of index.segments) {
+      const path = join(dir, seg.file);
+      if (!existsSync(path)) continue;
+      const raw = readFileSync(path, 'utf8');
+      for (const line of raw.split('\n')) {
+        if (!line) continue;
+        try {
+          yield JSON.parse(line);
+        } catch {
+          // Skip malformed line — don't crash the read pipeline.
+        }
+      }
+    }
+  }
+
+  /** Read records whose id falls in `[firstId, lastId]` inclusive (string or number). */
+  function* readRange(firstId, lastId) {
+    for (const seg of index.segments) {
+      if (!segmentOverlaps(seg, firstId, lastId)) continue;
+      const path = join(dir, seg.file);
+      if (!existsSync(path)) continue;
+      const raw = readFileSync(path, 'utf8');
+      for (const line of raw.split('\n')) {
+        if (!line) continue;
+        let obj;
+        try { obj = JSON.parse(line); } catch { continue; }
+        const id = obj.id ?? null;
+        if (id === null) continue;
+        if (idBetween(id, firstId, lastId)) yield obj;
+      }
+    }
+  }
+
+  function setNextId(n) { index.nextId = n; saveIndex(dir, index); }
+  function getNextId() { return index.nextId; }
+  function getIndex() { return index; }
+  function flushIndex() { saveIndex(dir, index); }
+
+  function close() {
+    try { closeSync(fd); } catch { /* already closed */ }
+    saveIndex(dir, index);
+  }
+
+  return {
+    append, readRange, streamAll, rotate, close,
+    setNextId, getNextId, getIndex, flushIndex,
+  };
+}
+
+/** True when index's segment list matches exactly the files on disk. */
+function indexMatchesDisk(index, filesOnDisk) {
+  const indexFiles = index.segments.map((s) => s.file);
+  if (indexFiles.length !== filesOnDisk.length) return false;
+  for (let i = 0; i < indexFiles.length; i++) {
+    if (indexFiles[i] !== filesOnDisk[i]) return false;
+  }
+  return true;
+}
+
+/**
+ * Rebuild the index by scanning every segment file on disk. Expensive but
+ * only happens on first open or after index.json loss.
+ */
+function rebuildIndexFromDisk(dir, files, parseLine) {
+  const idx = emptyIndex();
+  for (const file of files) {
+    idx.segments.push(statSegmentFromDisk(dir, file, parseLine));
+  }
+  return idx;
+}
+
+/**
+ * Cheap overlap test. If a segment's firstId/lastId are null (empty segment),
+ * we treat it as non-overlapping.
+ */
+function segmentOverlaps(seg, first, last) {
+  if (seg.firstId == null || seg.lastId == null) return false;
+  // Ordering on strings is lexicographic which matches ULID-style ids used
+  // by the product (msg_01HW...). For numeric ids, JS > / < works too.
+  return !(compareId(seg.lastId, first) < 0 || compareId(seg.firstId, last) > 0);
+}
+
+function idBetween(id, first, last) {
+  return compareId(id, first) >= 0 && compareId(id, last) <= 0;
+}
+
+function compareId(a, b) {
+  if (a === b) return 0;
+  return a < b ? -1 : 1;
+}

package/unify/storage/shard-index.js

@@ -0,0 +1,158 @@
+/**
+ * shard-index.js — Manifest for shard-store.
+ *
+ * shard-store keeps N shard files (one per schema.shard value) and one
+ * `index.json` that maps entry ids → their shard + byte range. This module
+ * owns only the manifest; shard-store.js drives writes.
+ *
+ * Schema (index.json):
+ *   {
+ *     version: 1,
+ *     entries: [{
+ *       id, shard,
+ *       byteOffset, byteLen,         // byte range inside shard file
+ *       meta: { kind?, tags?, pinned?, ...caller-chosen }
+ *     }],
+ *     shards: {
+ *       <name>: { entries: <count>, bytes: <size>, softCap: { entries, bytes } }
+ *     }
+ *   }
+ *
+ * entries[] is append-style but we rewrite it atomically on every mutation.
+ * At ~thousands of entries this is still cheap (<10 KB JSON) and keeps the
+ * read path O(1) — the full index loads into memory on open.
+ *
+ * If index.json is lost or corrupt, shard-store rebuilds it by scanning the
+ * shard markdown files for <!--entry:<id>:START/END--> delimiters.
+ */
+
+import { existsSync, readFileSync, mkdirSync, readdirSync, statSync } from 'fs';
+import { join } from 'path';
+import { writeAtomic } from './atomic.js';
+
+export const SHARD_INDEX_FILE = 'index.json';
+export const SHARD_INDEX_VERSION = 1;
+
+export const START_MARK = (id) => `<!--entry:${id}:START-->`;
+export const END_MARK = (id) => `<!--entry:${id}:END-->`;
+
+/** Regex that matches any start or end delimiter. */
+const ENTRY_MARK_RE = /<!--entry:([A-Za-z0-9_\-]+):(START|END)-->/g;
+
+export function emptyShardIndex() {
+  return { version: SHARD_INDEX_VERSION, entries: [], shards: {} };
+}
+
+export function loadShardIndex(dir) {
+  const path = join(dir, SHARD_INDEX_FILE);
+  if (!existsSync(path)) return null;
+  try {
+    const raw = readFileSync(path, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== 'object') return null;
+    if (!Array.isArray(parsed.entries)) return null;
+    if (!parsed.shards || typeof parsed.shards !== 'object') return null;
+    return {
+      version: parsed.version || SHARD_INDEX_VERSION,
+      entries: parsed.entries,
+      shards: parsed.shards,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function saveShardIndex(dir, index) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const payload = JSON.stringify({
+    version: SHARD_INDEX_VERSION,
+    entries: index.entries,
+    shards: index.shards,
+  }, null, 2);
+  writeAtomic(join(dir, SHARD_INDEX_FILE), payload);
+}
+
+/** Compose the on-disk filename for a shard (schema may customise). */
+export function shardFileName(shardName) {
+  return `memory-${shardName}.md`;
+}
+
+/**
+ * Scan shard files in `dir` and rebuild the index entirely from disk.
+ * Relies only on the <!--entry:<id>:START/END--> delimiters. Returns a
+ * fresh index object. Caller is responsible for populating `meta` again
+ * by reading each entry's frontmatter if they need it — this module does
+ * not parse the entry body (keeping the store schema-agnostic).
+ */
+export function rebuildShardIndexFromDisk(dir, schema) {
+  const index = emptyShardIndex();
+  if (!existsSync(dir)) return index;
+
+  // Preseed shard buckets from the schema so even empty shards show up.
+  for (const shardName of schema.shards || []) {
+    index.shards[shardName] = {
+      entries: 0,
+      bytes: 0,
+      softCap: schema.softCap?.[shardName] || schema.defaultSoftCap || null,
+    };
+  }
+
+  for (const name of readdirSync(dir)) {
+    if (!name.startsWith('memory-') || !name.endsWith('.md')) continue;
+    const shardName = name.slice('memory-'.length, -'.md'.length);
+    const path = join(dir, name);
+    const body = readFileSync(path, 'utf8');
+    const bytes = statSync(path).size;
+
+    // Ensure bucket exists even if schema didn't preseed this shard.
+    if (!index.shards[shardName]) {
+      index.shards[shardName] = {
+        entries: 0,
+        bytes,
+        softCap: schema.defaultSoftCap || null,
+      };
+    } else {
+      index.shards[shardName].bytes = bytes;
+    }
+
+    // Walk START/END pairs. Tolerate out-of-order markers by matching by id.
+    const starts = new Map();
+    ENTRY_MARK_RE.lastIndex = 0;
+    let m;
+    while ((m = ENTRY_MARK_RE.exec(body))) {
+      const id = m[1];
+      const kind = m[2];
+      if (kind === 'START') {
+        starts.set(id, m.index);
+      } else if (kind === 'END' && starts.has(id)) {
+        const startIdx = starts.get(id);
+        const endIdx = m.index + m[0].length;
+        index.entries.push({
+          id,
+          shard: shardName,
+          byteOffset: startIdx,
+          byteLen: endIdx - startIdx,
+          meta: {},
+        });
+        index.shards[shardName].entries += 1;
+        starts.delete(id);
+      }
+    }
+  }
+  return index;
+}
+
+/** Upsert (or insert) a single entry record. Mutates `index` in place. */
+export function putEntryRecord(index, record) {
+  const i = index.entries.findIndex((e) => e.id === record.id);
+  if (i >= 0) index.entries[i] = record;
+  else index.entries.push(record);
+}
+
+/** Remove an entry record by id. Returns the removed record or null. */
+export function removeEntryRecord(index, id) {
+  const i = index.entries.findIndex((e) => e.id === id);
+  if (i < 0) return null;
+  const [removed] = index.entries.splice(i, 1);
+  return removed;
+}

package/unify/storage/shard-store.js

@@ -0,0 +1,317 @@
+/**
+ * shard-store.js — Schema-aware shard storage on top of shard-index.
+ *
+ * Stores opaque "entries" across a small number of shard files. Each entry
+ * is a chunk of text (typically the serialised body the caller supplies),
+ * bracketed by `<!--entry:<id>:START-->` / `<!--entry:<id>:END-->` delimiters.
+ *
+ * Caller provides a `schema` describing:
+ *   - shards      : allowed shard names (open set if undefined)
+ *   - softCap     : per-shard { entries, bytes } soft limit
+ *   - defaultSoftCap : fallback for shards not explicitly listed
+ *
+ * API surface (§10 acceptance):
+ *   put(entry)            → { id, shard, needsRecompression }
+ *   get(id)               → { id, shard, body, meta } | null
+ *   query(filter)         → { results: [...], needsRecompression: [shard names] }
+ *   remove(id)            → boolean
+ *   compact(shardName?)   → rewrites shard(s) to strip tombstone gaps
+ *
+ * What this module does NOT know:
+ *   - What an entry body means (kind, sourceRef, superseded chains...).
+ *     It only reads meta fields the caller surfaces through `entry.meta`
+ *     for query filtering.
+ *   - What a VP, task, group, or message is.
+ *   - When to compact. Compaction is a separate primitive called by 334g
+ *     (dream). This module only surfaces `needsRecompression` advisory.
+ *
+ * Soft-cap semantics (acceptance #4):
+ *   When a shard exceeds its softCap, operations succeed normally but the
+ *   return value carries `needsRecompression: true` (put) or the shard
+ *   name is listed in `result.needsRecompression` (query). The store never
+ *   auto-compacts in response.
+ */
+
+import {
+  existsSync,
+  readFileSync,
+  mkdirSync,
+  appendFileSync,
+  statSync,
+} from 'fs';
+import { join } from 'path';
+import { writeAtomic } from './atomic.js';
+import {
+  loadShardIndex,
+  saveShardIndex,
+  rebuildShardIndexFromDisk,
+  putEntryRecord,
+  removeEntryRecord,
+  shardFileName,
+  START_MARK,
+  END_MARK,
+  emptyShardIndex,
+} from './shard-index.js';
+
+/**
+ * Open (or create) a shard store rooted at `dir`.
+ * `schema` example:
+ *   {
+ *     shards: ['skill', 'lessons', 'preferences', 'relations'],
+ *     softCap: {
+ *       skill:       { entries: 80, bytes: 64 * 1024 },
+ *       lessons:     { entries: 80, bytes: 64 * 1024 },
+ *     },
+ *     defaultSoftCap: { entries: 150, bytes: 128 * 1024 },
+ *   }
+ */
+export function openShardStore(dir, schema = {}) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const shardSchema = normaliseSchema(schema);
+
+  let index = loadShardIndex(dir);
+  if (!index || !indexLooksConsistent(dir, index)) {
+    index = rebuildShardIndexFromDisk(dir, shardSchema);
+    // Preserve meta from the old index if rebuild lost it and we have a
+    // readable on-disk frontmatter strategy — out of scope for 334o; callers
+    // re-hydrate meta through `refreshMeta()` below if they care.
+    saveShardIndex(dir, index);
+  }
+
+  /** Write a fresh entry to a shard file, append-style. */
+  function put(entry) {
+    validateEntry(entry, shardSchema);
+
+    // Remove old copy if same id exists (keeps "put" upsert-like).
+    const existing = index.entries.find((e) => e.id === entry.id);
+    if (existing) {
+      compactShard(existing.shard, [entry.id]);
+    }
+
+    const shard = entry.shard;
+    const path = join(dir, shardFileName(shard));
+    const payload = formatEntry(entry);
+
+    // byteOffset is the size of the file BEFORE we append.
+    const byteOffset = existsSync(path) ? statSync(path).size : 0;
+    appendFileSync(path, payload);
+    const byteLen = Buffer.byteLength(payload, 'utf8');
+
+    putEntryRecord(index, {
+      id: entry.id,
+      shard,
+      byteOffset,
+      byteLen,
+      meta: sanitiseMeta(entry.meta),
+    });
+    updateShardStats(index, shard, path);
+    saveShardIndex(dir, index);
+
+    return {
+      id: entry.id,
+      shard,
+      needsRecompression: isOverSoftCap(index, shard, shardSchema),
+    };
+  }
+
+  /** Read one entry by id. Returns null if absent. */
+  function get(id) {
+    const rec = index.entries.find((e) => e.id === id);
+    if (!rec) return null;
+    const path = join(dir, shardFileName(rec.shard));
+    if (!existsSync(path)) return null;
+    const raw = readFileSync(path, 'utf8');
+    // Slice by byte range is approximate for multi-byte UTF-8 — we use the
+    // delimiter as the authoritative boundary to stay safe on emoji etc.
+    const body = extractBody(raw, id);
+    if (body === null) return null;
+    return { id, shard: rec.shard, body, meta: rec.meta || {} };
+  }
+
+  /**
+   * Filter entries in-memory. Filter fields:
+   *   shard : string | string[]     exact shard match
+   *   kind  : string | string[]     matches meta.kind
+   *   tags  : string[]              entry must contain ALL listed tags
+   *   pinned: boolean               exact match on meta.pinned
+   *   where : (rec) => boolean      escape hatch
+   */
+  function query(filter = {}) {
+    const { shard, kind, tags, pinned, where } = filter;
+    const results = [];
+    for (const rec of index.entries) {
+      if (shard && !matchesOneOf(rec.shard, shard)) continue;
+      if (kind && !matchesOneOf(rec.meta?.kind, kind)) continue;
+      if (pinned !== undefined && Boolean(rec.meta?.pinned) !== Boolean(pinned)) continue;
+      if (tags && tags.length > 0) {
+        const recTags = rec.meta?.tags || [];
+        if (!tags.every((t) => recTags.includes(t))) continue;
+      }
+      if (where && !where(rec)) continue;
+      results.push(rec);
+    }
+    // Surface which shards need re-compression so dream can schedule work.
+    const over = [];
+    for (const name of Object.keys(index.shards)) {
+      if (isOverSoftCap(index, name, shardSchema)) over.push(name);
+    }
+    return { results, needsRecompression: over };
+  }
+
+  /** Delete one entry; compacts the shard to reclaim space immediately. */
+  function remove(id) {
+    const rec = index.entries.find((e) => e.id === id);
+    if (!rec) return false;
+    compactShard(rec.shard, [id]);
+    return true;
+  }
+
+  /**
+   * Rewrite a shard file, omitting the entries listed in `deleteIds`.
+   * Exposed as both the implementation of `remove` and the public compact
+   * primitive used by `compact()` (no deletions, just defrag).
+   */
+  function compactShard(shardName, deleteIds = []) {
+    const path = join(dir, shardFileName(shardName));
+    if (!existsSync(path)) return;
+    const raw = readFileSync(path, 'utf8');
+    const keepIds = index.entries
+      .filter((e) => e.shard === shardName && !deleteIds.includes(e.id))
+      .map((e) => e.id);
+    const parts = [];
+    for (const id of keepIds) {
+      const body = extractBody(raw, id);
+      if (body === null) continue;
+      parts.push(formatEntry({ id, shard: shardName, body, meta: null }));
+    }
+    writeAtomic(path, parts.join(''));
+
+    // Update in-memory records with their new byte offsets.
+    let cursor = 0;
+    for (let i = 0; i < keepIds.length; i++) {
+      const id = keepIds[i];
+      const rec = index.entries.find((e) => e.id === id);
+      const part = parts[i];
+      const len = Buffer.byteLength(part, 'utf8');
+      rec.byteOffset = cursor;
+      rec.byteLen = len;
+      cursor += len;
+    }
+
+    // Drop removed ids from the index entirely.
+    for (const id of deleteIds) removeEntryRecord(index, id);
+
+    updateShardStats(index, shardName, path);
+    saveShardIndex(dir, index);
+  }
+
+  /** Public compact: rewrite one shard (or all) with no deletions. */
+  function compact(shardName) {
+    if (shardName) return compactShard(shardName, []);
+    for (const name of Object.keys(index.shards)) compactShard(name, []);
+  }
+
+  /** Allow caller (memory-family) to re-hydrate meta after bulk rebuild. */
+  function setMeta(id, meta) {
+    const rec = index.entries.find((e) => e.id === id);
+    if (!rec) return false;
+    rec.meta = sanitiseMeta(meta);
+    saveShardIndex(dir, index);
+    return true;
+  }
+
+  function stats() {
+    return structuredClone({ shards: index.shards, count: index.entries.length });
+  }
+
+  function getIndex() { return index; }
+
+  return { put, get, query, remove, compact, setMeta, stats, getIndex };
+}
+
+// ─── Helpers ────────────────────────────────────────────────────
+
+function normaliseSchema(schema) {
+  return {
+    shards: Array.isArray(schema.shards) ? schema.shards.slice() : [],
+    softCap: schema.softCap || {},
+    defaultSoftCap: schema.defaultSoftCap || { entries: 1000, bytes: 10 * 1024 * 1024 },
+  };
+}
+
+function validateEntry(entry, schema) {
+  if (!entry || typeof entry !== 'object') throw new Error('entry must be an object');
+  if (!entry.id || typeof entry.id !== 'string') throw new Error('entry.id required (string)');
+  if (!/^[A-Za-z0-9_\-]+$/.test(entry.id)) throw new Error('entry.id must be [A-Za-z0-9_-]+');
+  if (!entry.shard || typeof entry.shard !== 'string') throw new Error('entry.shard required');
+  if (schema.shards.length > 0 && !schema.shards.includes(entry.shard)) {
+    // Open shard extension allowed by returning a warning? Spec says shards
+    // are fixed — so reject unknown ones. Caller can extend schema.shards[].
+    throw new Error(`entry.shard "${entry.shard}" not in schema.shards`);
+  }
+  if (typeof entry.body !== 'string') throw new Error('entry.body required (string)');
+}
+
+function formatEntry({ id, body }) {
+  // Leading \n so successive appends stay visually separated even if the
+  // previous entry's body didn't end in a newline.
+  return `\n${START_MARK(id)}\n${body.replace(/\n+$/, '')}\n${END_MARK(id)}\n`;
+}
+
+function extractBody(raw, id) {
+  const start = raw.indexOf(START_MARK(id));
+  const end = raw.indexOf(END_MARK(id));
+  if (start < 0 || end < 0 || end < start) return null;
+  const bodyStart = start + START_MARK(id).length;
+  return raw.slice(bodyStart, end).replace(/^\n+/, '').replace(/\n+$/, '');
+}
+
+function updateShardStats(index, shardName, path) {
+  const bucket = index.shards[shardName] || (index.shards[shardName] = {
+    entries: 0, bytes: 0, softCap: null,
+  });
+  bucket.bytes = existsSync(path) ? statSync(path).size : 0;
+  bucket.entries = index.entries.filter((e) => e.shard === shardName).length;
+}
+
+function isOverSoftCap(index, shardName, schema) {
+  const bucket = index.shards[shardName];
+  if (!bucket) return false;
+  const cap = schema.softCap?.[shardName] || schema.defaultSoftCap;
+  if (!cap) return false;
+  if (cap.entries != null && bucket.entries > cap.entries) return true;
+  if (cap.bytes != null && bucket.bytes > cap.bytes) return true;
+  return false;
+}
+
+function matchesOneOf(value, needle) {
+  if (Array.isArray(needle)) return needle.includes(value);
+  return value === needle;
+}
+
+function sanitiseMeta(meta) {
+  if (!meta || typeof meta !== 'object') return {};
+  // Only allow JSON-safe fields (number/string/boolean/array of those).
+  // Anything weird silently dropped so a bad call can't corrupt the index.
+  const out = {};
+  for (const [k, v] of Object.entries(meta)) {
+    if (v === null || ['string', 'number', 'boolean'].includes(typeof v)) {
+      out[k] = v;
+    } else if (Array.isArray(v) && v.every((x) => typeof x === 'string')) {
+      out[k] = v.slice();
+    }
+  }
+  return out;
+}
+
+function indexLooksConsistent(dir, index) {
+  if (!index || !Array.isArray(index.entries)) return false;
+  // Cheap sanity: each shard listed in index has a file on disk, OR the shard
+  // is empty (no entries yet). Caller recomputes sizes next op.
+  for (const name of Object.keys(index.shards)) {
+    const path = join(dir, shardFileName(name));
+    const bucket = index.shards[name];
+    if (bucket.entries > 0 && !existsSync(path)) return false;
+  }
+  return true;
+}