@ngockhoale/ukit 3.0.6 → 3.0.7

package/src/core/observability/segments/readSegments.js

@@ -0,0 +1,215 @@
+/**
+ * readSegments.js (TASK-006, SPEC §5 DF-FR05 / §8)
+ *
+ * Typed reader over the append-only JSONL segment store.
+ *
+ *   readSegments(root, { fromInclusive?, limit?, ioTimeoutMs? })
+ *     → AsyncIterable<record> with attached:
+ *         .coverage → { expired_segments, quarantined_segments, corrupt_lines,
+ *                       partial_tail_bytes, cursor_missed, degraded[] }
+ *         .ok       → boolean (false when any degraded entry was recorded)
+ *
+ * Read order is deterministic and never wall-clock based: sealed segments in
+ * sealed_seq order, then the active segment. `fromInclusive` is a record_id
+ * cursor — iteration starts at the first retained record with that id; when
+ * the cursor is not found in retained data, all retained records are yielded
+ * and coverage.cursor_missed = true (no claim of reproducibility over expired
+ * data). `limit` caps yielded records.
+ *
+ * Integrity: every sealed segment's sha256 is verified against its meta
+ * sidecar before a single row is served; a mismatch or missing meta sends the
+ * segment to quarantine/ — corrupt data is never falsified into the stream.
+ * A partial tail row in the active segment (hard-kill artifact) is skipped
+ * and counted, never parsed. Symlinked or foreign-named files are never
+ * followed or read.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { validateSemanticRecord } from '../schema/validate.js';
+import { quarantineSegment } from './recovery.js';
+import {
+  DEFAULT_IO_TIMEOUT_MS,
+  ioReason,
+  listSegments,
+  metaPathFor,
+  readState,
+  resolveRoot,
+  sha256Buffer,
+  withTimeout,
+} from './internal.js';
+
+export function readSegments(root, opts = {}) {
+  const timeoutMs = Number.isFinite(opts.ioTimeoutMs) ? opts.ioTimeoutMs : DEFAULT_IO_TIMEOUT_MS;
+  const fromInclusive = typeof opts.fromInclusive === 'string' ? opts.fromInclusive : null;
+  const limit = Number.isInteger(opts.limit) && opts.limit >= 0 ? opts.limit : null;
+
+  const coverage = {
+    expired_segments: 0,
+    quarantined_segments: 0,
+    corrupt_lines: 0,
+    partial_tail_bytes: 0,
+    cursor_missed: false,
+    degraded: [],
+  };
+  const degrade = (reason, file) => {
+    coverage.degraded.push({ reason, file });
+  };
+
+  async function* iterate() {
+    const resolved = await resolveRoot(root, timeoutMs);
+    if (!resolved.ok) {
+      degrade(resolved.reason, String(root));
+      return;
+    }
+    if (!resolved.exists) return; // empty store → empty stream, still ok
+
+    const state = await readState(resolved.root, timeoutMs);
+    coverage.expired_segments = state.expired_segments || 0;
+
+    const listing = await listSegments(resolved.root, timeoutMs);
+    if (!listing.ok) {
+      degrade(listing.reason, resolved.root);
+      return;
+    }
+    for (const skipped of listing.skipped) {
+      degrade(skipped.reason, skipped.name);
+    }
+
+    let yielded = 0;
+    let cursorPending = fromInclusive !== null;
+    // Records scanned while the cursor is still pending. If the cursor is
+    // found the buffer is discarded (records before it are not yielded); if
+    // the scan ends with the cursor unfound, the buffer IS the retained set
+    // and is flushed so consumers still get every retained record.
+    let pending = cursorPending ? [] : null;
+
+    const emit = (record) => {
+      if (cursorPending) {
+        if (record.record_id !== fromInclusive) {
+          pending.push(record);
+          return null;
+        }
+        cursorPending = false;
+        pending = null;
+      }
+      return record;
+    };
+
+    // Sealed segments: verify checksum BEFORE serving a single row.
+    for (const seg of listing.sealed) {
+      if (limit !== null && yielded >= limit) return;
+      let meta = null;
+      try {
+        meta = JSON.parse(
+          await withTimeout(fs.promises.readFile(metaPathFor(seg.path), 'utf8'), timeoutMs),
+        );
+      } catch {
+        meta = null;
+      }
+      let buf = null;
+      try {
+        buf = await withTimeout(fs.promises.readFile(seg.path), timeoutMs);
+      } catch (err) {
+        degrade(ioReason(err), seg.name);
+        continue;
+      }
+      const checksumOk =
+        meta && typeof meta.checksum === 'string' && meta.checksum === sha256Buffer(buf);
+      if (!checksumOk) {
+        const q = await quarantineSegment(resolved.root, seg.name, 'checksum_mismatch', {
+          ioTimeoutMs: timeoutMs,
+        });
+        if (q.ok) {
+          coverage.quarantined_segments += 1;
+          degrade('checksum_mismatch_quarantined', seg.name);
+        } else {
+          degrade(`quarantine_failed:${q.reason}`, seg.name);
+        }
+        continue;
+      }
+      const text = buf.toString('utf8');
+      const complete = text.endsWith('\n') ? text.slice(0, -1) : text;
+      for (const line of complete.split('\n')) {
+        if (limit !== null && yielded >= limit) return;
+        if (line.length === 0) continue;
+        let record;
+        try {
+          record = JSON.parse(line);
+        } catch {
+          coverage.corrupt_lines += 1;
+          continue;
+        }
+        if (!validateSemanticRecord(record).ok) {
+          coverage.corrupt_lines += 1;
+          continue;
+        }
+        const out = emit(record);
+        if (out) {
+          yielded += 1;
+          yield out;
+        }
+      }
+    }
+
+    // Active segment: no checksum (still open); a partial tail row is a
+    // hard-kill artifact — skipped and counted, never parsed.
+    if (listing.active && (limit === null || yielded < limit)) {
+      let buf;
+      try {
+        buf = await withTimeout(fs.promises.readFile(listing.active.path), timeoutMs);
+      } catch (err) {
+        degrade(ioReason(err), 'active.jsonl');
+        buf = null;
+      }
+      if (buf) {
+        const text = buf.toString('utf8');
+        const endsComplete = text.length === 0 || text.endsWith('\n');
+        const body = endsComplete ? text.slice(0, -1) : text.slice(0, text.lastIndexOf('\n') + 1);
+        if (!endsComplete) {
+          coverage.partial_tail_bytes += buf.length - (text.lastIndexOf('\n') + 1);
+        }
+        for (const line of body.split('\n')) {
+          if (limit !== null && yielded >= limit) return;
+          if (line.length === 0) continue;
+          let record;
+          try {
+            record = JSON.parse(line);
+          } catch {
+            coverage.corrupt_lines += 1;
+            continue;
+          }
+          if (!validateSemanticRecord(record).ok) {
+            coverage.corrupt_lines += 1;
+            continue;
+          }
+          const out = emit(record);
+          if (out) {
+            yielded += 1;
+            yield out;
+          }
+        }
+      }
+    }
+    if (cursorPending) {
+      coverage.cursor_missed = true;
+      for (const record of pending) {
+        if (limit !== null && yielded >= limit) break;
+        yielded += 1;
+        yield record;
+      }
+    }
+  }
+
+  const iterable = {
+    coverage,
+    get ok() {
+      return coverage.degraded.length === 0;
+    },
+    [Symbol.asyncIterator]() {
+      return iterate();
+    },
+  };
+  return iterable;
+}

package/src/core/observability/segments/recovery.js

@@ -0,0 +1,123 @@
+/**
+ * recovery.js (TASK-006, SPEC §5 DF-FR05)
+ *
+ * Partial-tail recovery and corrupt-segment quarantine for the append-only
+ * JSONL segment store.
+ *
+ *   recoverTail(segmentPath, opts?)
+ *     → { ok: true, recovered, droppedBytes, droppedLines }
+ *     | { ok: false, reason }
+ *
+ *   quarantineSegment(root, segmentName, reason)
+ *     → { ok: true } | { ok: false, reason }
+ *
+ * Recovery never fabricates data: a truncated tail is dropped, the complete
+ * prefix is preserved byte-for-byte, the segment is marked `recovered: true`
+ * in its sidecar meta, and any existing checksum is recomputed over the
+ * truncated bytes (never carried over from the pre-truncation file).
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {
+  DEFAULT_IO_TIMEOUT_MS,
+  QUARANTINE_DIR,
+  ioReason,
+  metaPathFor,
+  resolveFileInDir,
+  sha256Buffer,
+  withTimeout,
+} from './internal.js';
+
+/**
+ * Move a corrupt sealed segment (and its meta sidecar) into `quarantine/`.
+ * The segment is never deleted and never silently repaired — it is preserved
+ * for inspection outside the live read path.
+ */
+export async function quarantineSegment(root, segmentName, reason, opts = {}) {
+  const timeoutMs = Number.isFinite(opts.ioTimeoutMs) ? opts.ioTimeoutMs : DEFAULT_IO_TIMEOUT_MS;
+  const base = path.basename(String(segmentName));
+  if (base !== segmentName || !base.endsWith('.jsonl')) {
+    return { ok: false, reason: 'unsafe_path' };
+  }
+  const quarantineDir = path.join(root, QUARANTINE_DIR);
+  try {
+    await withTimeout(fs.promises.mkdir(quarantineDir, { recursive: true }), timeoutMs);
+    const from = path.join(root, base);
+    let to = path.join(quarantineDir, base);
+    try {
+      await withTimeout(fs.promises.lstat(to), timeoutMs);
+      to = path.join(quarantineDir, `${base}.${Date.now()}.corrupt`);
+    } catch {
+      // Free name — keep the original filename for inspectability.
+    }
+    await withTimeout(fs.promises.rename(from, to), timeoutMs);
+    const metaFrom = metaPathFor(from);
+    try {
+      await withTimeout(fs.promises.rename(metaFrom, `${to}.meta.json`), timeoutMs);
+    } catch {
+      // Missing meta is expected for some corruption modes — not fatal.
+    }
+    return { ok: true };
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+}
+
+/**
+ * Drop a partial tail row (hard-kill artifact) from a JSONL segment.
+ * The file must end in '\n' to be considered complete; anything after the
+ * last newline is uncommitted and removed. The complete prefix is preserved
+ * exactly — recovery never rewrites surviving rows.
+ */
+export async function recoverTail(segmentPath, opts = {}) {
+  const timeoutMs = Number.isFinite(opts.ioTimeoutMs) ? opts.ioTimeoutMs : DEFAULT_IO_TIMEOUT_MS;
+  const resolved = await resolveFileInDir(segmentPath, timeoutMs);
+  if (!resolved.ok) return { ok: false, reason: resolved.reason };
+
+  let buf;
+  try {
+    buf = await withTimeout(fs.promises.readFile(resolved.path), timeoutMs);
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+  if (buf.length === 0) {
+    return { ok: true, recovered: false, droppedBytes: 0, droppedLines: 0 };
+  }
+  if (buf[buf.length - 1] === 0x0a) {
+    return { ok: true, recovered: false, droppedBytes: 0, droppedLines: 0 };
+  }
+
+  const lastNl = buf.lastIndexOf(0x0a);
+  const keep = lastNl + 1; // 0 when the whole file is one partial row
+  const dropped = buf.subarray(keep);
+  const droppedLines = dropped.toString('utf8').split('\n').filter((s) => s.length > 0).length;
+  try {
+    await withTimeout(fs.promises.truncate(resolved.path, keep), timeoutMs);
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+
+  // Mark recovery in the sidecar meta. If a checksum exists it is recomputed
+  // over the truncated bytes — a stale checksum would falsify the segment.
+  const metaPath = metaPathFor(resolved.path);
+  let meta = {};
+  try {
+    meta = JSON.parse(await withTimeout(fs.promises.readFile(metaPath, 'utf8'), timeoutMs));
+  } catch {
+    meta = {};
+  }
+  meta.recovered = true;
+  meta.recovered_dropped_bytes = dropped.length;
+  if (typeof meta.checksum === 'string') {
+    meta.checksum = sha256Buffer(buf.subarray(0, keep));
+  }
+  try {
+    await withTimeout(fs.promises.writeFile(metaPath, JSON.stringify(meta, null, 2)), timeoutMs);
+  } catch (err) {
+    return { ok: false, reason: ioReason(err) };
+  }
+
+  return { ok: true, recovered: true, droppedBytes: dropped.length, droppedLines };
+}