@ngockhoale/ukit 3.0.6 → 3.0.8

package/src/core/observability/segments/retention.js

@@ -0,0 +1,381 @@
+/**
+ * retention.js (TASK-006, SPEC §5 DF-FR05 / §8)
+ *
+ * Write path, rotation, and retention for the append-only JSONL segment
+ * store. Layout under `root`:
+ *
+ *   active.jsonl                  shared append target (O_APPEND — concurrent
+ *                                 writers interleave whole rows, never bytes)
+ *   seg-<seq>-<boot>-<writer>.jsonl   sealed segment
+ *   seg-<...>.meta.json           sidecar: checksum, record/seq/importance
+ *                                 stats, sealed_at_ms, recovered flag
+ *   quarantine/                   corrupt sealed segments (never falsified)
+ *   _state.json                   next_segment_seq, expired/quarantined
+ *                                 counters, disabled flag, active_since_ms
+ *
+ * Durability contract: appends are buffered writes — there is NO per-event
+ * fsync. A hard kill can lose the unflushed tail (bounded by the OS page
+ * cache, typically the last few KB); recoverTail() drops the partial final
+ * row and preserves the complete prefix. Never claim zero critical drops
+ * without fault evidence.
+ *
+ * Ordering contract: records are ordered by (boot_id, writer_id, sequence)
+ * and sealed segments by sealed_seq — never by wall clock, so clock jumps
+ * cannot reorder stored facts.
+ *
+ * Public surface (consumed by TASK-007 recorder):
+ *   appendRecord(root, record, policy?)   → { ok: true, written: true }
+ *                                         | { ok: false, reason, errors? }
+ *   rotateIfNeeded(root, policy?)         → { ok: true, rotated, name? }
+ *                                         | { ok: false, reason }
+ *   enforceRetention(root, policy?)       → { ok: true, expired, disabled }
+ *                                         | { ok: false, reason }
+ *
+ * policy: { maxSegmentBytes, maxSegmentAgeMs, maxTotalBytes, ioTimeoutMs,
+ *           now } — `now` is injectable for deterministic tests.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { validateSemanticRecord } from '../schema/validate.js';
+import { IMPORTANCE_LEVELS } from '../schema/constants.js';
+import {
+  ACTIVE_FILE,
+  DEFAULT_IO_TIMEOUT_MS,
+  ioReason,
+  ioTimeoutMsOf,
+  listSegments,
+  metaPathFor,
+  nowOf,
+  policyOf,
+  readState,
+  resolveRoot,
+  sanitizeNameComponent,
+  sha256Buffer,
+  totalSegmentBytes,
+  withTimeout,
+  writeState,
+} from './internal.js';
+
+function importanceRank(level) {
+  const idx = IMPORTANCE_LEVELS.indexOf(level);
+  return idx >= 0 ? idx : 0; // unknown → lowest → evicted first under cap
+}
+
+/**
+ * Seal the current active segment: parse stats, checksum the exact bytes,
+ * move it aside via link+unlink (atomic, never overwrites an existing seal),
+ * and write its meta sidecar. Concurrent rotators lose the race on ENOENT
+ * and simply report not-sealed.
+ */
+async function sealActive(root, state, policy) {
+  const timeoutMs = ioTimeoutMsOf(policy);
+  const activePath = path.join(root, ACTIVE_FILE);
+
+  let lst;
+  try {
+    lst = await withTimeout(fs.promises.lstat(activePath), timeoutMs);
+  } catch (err) {
+    if (err && err.code === 'ENOENT') return { ok: true, sealed: false };
+    return { ok: false, reason: ioReason(err) };
+  }
+  if (lst.isSymbolicLink()) return { ok: false, reason: 'unsafe_path' };
+  if (!lst.isFile() || lst.size === 0) {
+    state.active_since_ms = null;
+    await writeState(root, state, timeoutMs);
+    return { ok: true, sealed: false };
+  }
+
+  let buf;
+  try {
+    buf = await withTimeout(fs.promises.readFile(activePath), timeoutMs);
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+
+  // Stats from the sealed bytes themselves — never trusted from memory.
+  let recordCount = 0;
+  let corruptLines = 0;
+  let seqMin = null;
+  let seqMax = null;
+  let importanceMax = 'low';
+  let maxRank = -1;
+  let bootId = 'unknown';
+  let writerId = 'unknown';
+  const text = buf.toString('utf8');
+  const complete = text.endsWith('\n') ? text.slice(0, -1) : text;
+  for (const line of complete.split('\n')) {
+    if (line.length === 0) continue;
+    let parsed = null;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      corruptLines += 1;
+      continue;
+    }
+    recordCount += 1;
+    if (bootId === 'unknown' && typeof parsed.boot_id === 'string') bootId = parsed.boot_id;
+    if (writerId === 'unknown' && typeof parsed.writer_id === 'string') writerId = parsed.writer_id;
+    if (Number.isInteger(parsed.sequence)) {
+      seqMin = seqMin === null ? parsed.sequence : Math.min(seqMin, parsed.sequence);
+      seqMax = seqMax === null ? parsed.sequence : Math.max(seqMax, parsed.sequence);
+    }
+    const rank = importanceRank(parsed.importance);
+    if (rank > maxRank) {
+      maxRank = rank;
+      importanceMax = parsed.importance;
+    }
+  }
+
+  // link+unlink: fails atomically if the target seal already exists (another
+  // writer won the seq) — rename would silently overwrite it.
+  let seq = state.next_segment_seq;
+  let name = null;
+  let linked = false;
+  for (let attempt = 0; attempt < 64; attempt++) {
+    name = `seg-${String(seq).padStart(6, '0')}-${sanitizeNameComponent(bootId)}-${sanitizeNameComponent(writerId)}.jsonl`;
+    const target = path.join(root, name);
+    try {
+      await withTimeout(fs.promises.link(activePath, target), timeoutMs);
+      await withTimeout(fs.promises.unlink(activePath), timeoutMs);
+      linked = true;
+      break;
+    } catch (err) {
+      if (err && err.code === 'ENOENT') return { ok: true, sealed: false }; // lost the race
+      if (err && err.code === 'EEXIST') {
+        seq += 1;
+        continue;
+      }
+      return { ok: false, reason: ioReason(err) };
+    }
+  }
+  // Every attempt collided — never write meta for a name that was not sealed.
+  if (!linked) return { ok: false, reason: 'seal_conflict' };
+
+  const meta = {
+    sealed_seq: seq,
+    boot_id: bootId,
+    writer_id: writerId,
+    sealed_at_ms: nowOf(policy),
+    record_count: recordCount,
+    seq_min: seqMin,
+    seq_max: seqMax,
+    importance_max: importanceMax,
+    corrupt_lines: corruptLines,
+    recovered: false,
+    checksum: sha256Buffer(buf),
+  };
+  try {
+    await withTimeout(
+      fs.promises.writeFile(metaPathFor(path.join(root, name)), JSON.stringify(meta, null, 2)),
+      timeoutMs,
+    );
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+
+  state.next_segment_seq = Math.max(state.next_segment_seq, seq + 1);
+  state.active_since_ms = null;
+  await writeState(root, state, timeoutMs);
+  return { ok: true, sealed: true, name };
+}
+
+/**
+ * Append one validated record to the active segment. Validates the envelope
+ * (records reaching this layer are already sanitizeObserved output — this is
+ * the integrity gate, not the privacy gate). Rotates BEFORE writing when the
+ * line would push the active segment past maxSegmentBytes, then enforces
+ * retention. When the hard cap cannot be satisfied even after eviction the
+ * store flips to `disabled` and every subsequent append returns
+ * { ok: false, reason: 'disabled' } — never unbounded growth.
+ */
+export async function appendRecord(root, record, policy) {
+  const p = policyOf(policy);
+  const timeoutMs = ioTimeoutMsOf(p);
+
+  const validation = validateSemanticRecord(record);
+  if (!validation.ok) {
+    return { ok: false, reason: 'invalid_record', errors: validation.errors };
+  }
+  const line = JSON.stringify(record) + '\n';
+  const lineBytes = Buffer.byteLength(line, 'utf8');
+
+  const resolved = await resolveRoot(root, timeoutMs);
+  if (!resolved.ok) return { ok: false, reason: resolved.reason };
+  const realRoot = resolved.root;
+
+  try {
+    await withTimeout(fs.promises.mkdir(realRoot, { recursive: true }), timeoutMs);
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+
+  let state = await readState(realRoot, timeoutMs);
+  if (state.disabled) return { ok: false, reason: 'disabled' };
+
+  // Rotate before writing when this line would overflow the active segment.
+  // A symlinked active.jsonl is rejected outright — appending through it
+  // would write outside the storage root (same rule as rotateIfNeeded).
+  try {
+    const st = await withTimeout(fs.promises.lstat(path.join(realRoot, ACTIVE_FILE)), timeoutMs);
+    if (st.isSymbolicLink()) return { ok: false, reason: 'unsafe_path' };
+    if (st.isFile() && st.size > 0 && st.size + lineBytes > p.maxSegmentBytes) {
+      const sealed = await sealActive(realRoot, state, p);
+      if (!sealed.ok) return { ok: false, reason: sealed.reason };
+      state = await readState(realRoot, timeoutMs);
+    }
+  } catch (err) {
+    if (!(err && err.code === 'ENOENT')) return { ok: false, reason: ioReason(err) };
+  }
+
+  // O_NOFOLLOW closes the lstat→append race: if active.jsonl was swapped to
+  // a symlink in between, open fails with ELOOP instead of writing through.
+  const appendFlags =
+    fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_APPEND |
+    (fs.constants.O_NOFOLLOW || 0);
+  try {
+    await withTimeout(
+      fs.promises.appendFile(path.join(realRoot, ACTIVE_FILE), line, { flag: appendFlags }),
+      timeoutMs,
+    );
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+
+  if (state.active_since_ms === null || state.active_since_ms === undefined) {
+    state.active_since_ms = nowOf(p);
+    await writeState(realRoot, state, timeoutMs);
+  }
+
+  const retention = await enforceRetention(realRoot, p);
+  if (!retention.ok) return { ok: false, reason: retention.reason };
+  if (retention.disabled) return { ok: false, reason: 'disabled' };
+  return { ok: true, written: true };
+}
+
+/**
+ * Seal the active segment when it has reached the size threshold or its age
+ * exceeds maxSegmentAgeMs. Safe under concurrent callers — losers of the
+ * seal race observe ENOENT and report rotated:false.
+ */
+export async function rotateIfNeeded(root, policy) {
+  const p = policyOf(policy);
+  const timeoutMs = ioTimeoutMsOf(p);
+  const resolved = await resolveRoot(root, timeoutMs);
+  if (!resolved.ok) return { ok: false, reason: resolved.reason };
+  if (!resolved.exists) return { ok: true, rotated: false };
+
+  const state = await readState(resolved.root, timeoutMs);
+  const activePath = path.join(resolved.root, ACTIVE_FILE);
+  let st;
+  try {
+    st = await withTimeout(fs.promises.lstat(activePath), timeoutMs);
+  } catch (err) {
+    if (err && err.code === 'ENOENT') return { ok: true, rotated: false };
+    return { ok: false, reason: ioReason(err) };
+  }
+  if (st.isSymbolicLink()) return { ok: false, reason: 'unsafe_path' };
+  if (!st.isFile() || st.size === 0) return { ok: true, rotated: false };
+
+  const oversized = st.size >= p.maxSegmentBytes;
+  const aged =
+    state.active_since_ms !== null &&
+    state.active_since_ms !== undefined &&
+    nowOf(p) - state.active_since_ms >= p.maxSegmentAgeMs;
+  if (!oversized && !aged) return { ok: true, rotated: false };
+
+  const sealed = await sealActive(resolved.root, state, p);
+  if (!sealed.ok) return { ok: false, reason: sealed.reason };
+  return { ok: true, rotated: sealed.sealed, name: sealed.name };
+}
+
+async function readMeta(segmentPath, timeoutMs) {
+  try {
+    const raw = await withTimeout(fs.promises.readFile(metaPathFor(segmentPath), 'utf8'), timeoutMs);
+    const meta = JSON.parse(raw);
+    return meta && typeof meta === 'object' ? meta : null;
+  } catch {
+    return null;
+  }
+}
+
+async function removeSegment(root, name, timeoutMs) {
+  try {
+    await withTimeout(fs.promises.unlink(path.join(root, name)), timeoutMs);
+  } catch {
+    // Already gone (concurrent eviction) — fine.
+  }
+  try {
+    await withTimeout(fs.promises.unlink(metaPathFor(path.join(root, name))), timeoutMs);
+  } catch {
+    // No sidecar — fine.
+  }
+}
+
+/**
+ * Retention = age + size + importance under a hard total cap.
+ *  1. Age: sealed segments older than maxSegmentAgeMs expire.
+ *  2. Cap: while live .jsonl bytes exceed maxTotalBytes, evict the lowest
+ *     importance_max segment first (ties → lowest sealed_seq = oldest).
+ *  3. If the cap is still unhittable (e.g. the active segment alone exceeds
+ *     it), the store reports disabled — bounded, never silent growth.
+ * The active segment is never evicted; only sealed facts expire.
+ */
+export async function enforceRetention(root, policy) {
+  const p = policyOf(policy);
+  const timeoutMs = ioTimeoutMsOf(p);
+  const resolved = await resolveRoot(root, timeoutMs);
+  if (!resolved.ok) return { ok: false, reason: resolved.reason };
+  if (!resolved.exists) return { ok: true, expired: 0, disabled: false };
+
+  const state = await readState(resolved.root, timeoutMs);
+  const listing = await listSegments(resolved.root, timeoutMs);
+  if (!listing.ok) return { ok: false, reason: listing.reason };
+
+  const now = nowOf(p);
+  let expired = 0;
+  const live = [];
+
+  for (const seg of listing.sealed) {
+    const meta = await readMeta(seg.path, timeoutMs);
+    const sealedAt = meta && Number.isFinite(meta.sealed_at_ms) ? meta.sealed_at_ms : null;
+    let ageBasis = sealedAt;
+    if (ageBasis === null) {
+      try {
+        ageBasis = (await withTimeout(fs.promises.stat(seg.path), timeoutMs)).mtimeMs;
+      } catch {
+        ageBasis = now; // unknown age → treat as fresh, never evict blindly
+      }
+    }
+    if (now - ageBasis >= p.maxSegmentAgeMs) {
+      await removeSegment(resolved.root, seg.name, timeoutMs);
+      expired += 1;
+    } else {
+      live.push({
+        name: seg.name,
+        seq: seg.seq,
+        size: seg.size,
+        rank: meta ? importanceRank(meta.importance_max) : 0,
+      });
+    }
+  }
+
+  let usage = (listing.active ? listing.active.size : 0) + live.reduce((s, seg) => s + seg.size, 0);
+
+  // Hard cap: evict lowest-importance first, oldest (lowest seq) on ties.
+  live.sort((a, b) => a.rank - b.rank || a.seq - b.seq);
+  while (usage > p.maxTotalBytes && live.length > 0) {
+    const victim = live.shift();
+    await removeSegment(resolved.root, victim.name, timeoutMs);
+    usage -= victim.size;
+    expired += 1;
+  }
+
+  const disabled = usage > p.maxTotalBytes;
+  state.expired_segments = (state.expired_segments || 0) + expired;
+  if (disabled) state.disabled = true;
+  await writeState(resolved.root, state, timeoutMs);
+
+  return { ok: true, expired, disabled };
+}