@playola-radio/slipstream 0.1.0

package/dist/transcript/fs-io.js

@@ -0,0 +1,287 @@
+import { lstat as fsLstat, open, readdir, readlink as fsReadlink, realpath as fsRealpath, } from 'node:fs/promises';
+import { join } from 'node:path';
+/** The real filesystem IO for transcript files. Missing → `missing`; any other
+ * open/read error → `inaccessible` (disclosed, never a silent empty read). The
+ * identity, size, and bytes all come from ONE open handle so a concurrent replace
+ * cannot supply a different generation's bytes than the id reports. */
+export const nodeTranscriptFileIO = {
+    async readFrom(path, start, maxBytes) {
+        let handle;
+        try {
+            handle = await open(path, 'r');
+        }
+        catch (err) {
+            if (err.code === 'ENOENT')
+                return { ok: false, reason: 'missing' };
+            return { ok: false, reason: 'inaccessible' };
+        }
+        try {
+            const s = await handle.stat({ bigint: true });
+            const id = { dev: s.dev, ino: s.ino };
+            const size = Number(s.size);
+            if (size <= start)
+                return { ok: true, id, size, bytes: Buffer.alloc(0) };
+            const length = Math.min(size - start, maxBytes);
+            const buf = Buffer.allocUnsafe(length);
+            const { bytesRead } = await handle.read(buf, 0, length, start);
+            return { ok: true, id, size, bytes: buf.subarray(0, bytesRead) };
+        }
+        catch {
+            return { ok: false, reason: 'inaccessible' };
+        }
+        finally {
+            await handle.close();
+        }
+    },
+};
+const FATAL_UTF8 = new TextDecoder('utf8', { fatal: true });
+const FIRST_LINE_CHUNK = 64 * 1024;
+/** A first line longer than this is not line-delimited JSONL; disclose it
+ * malformed rather than accumulate unbounded memory. */
+const MAX_FIRST_LINE = 16 * 1024 * 1024;
+/** Head-scan bounds for finding a Claude transcript's first cwd-bearing record.
+ * Enough lines to clear a cwd-less preamble; a per-line cap skips a large
+ * attachment (its cwd, if any, recurs on the smaller records around it); a total
+ * cap bounds the read even when large lines are skipped. */
+export const HEAD_SCAN_LINES = 64;
+export const HEAD_LINE_CAP = 256 * 1024;
+export const HEAD_SCAN_BYTES = 4 * 1024 * 1024;
+function classifyDirError(err) {
+    return err.code === 'ENOENT' ? 'missing' : 'inaccessible';
+}
+/** The real filesystem IO for transcript discovery. */
+export const nodeDiscoveryIO = {
+    async listDir(dir) {
+        try {
+            const entries = await readdir(dir, { withFileTypes: true });
+            const paths = entries
+                .filter((e) => e.isFile() && e.name.endsWith('.jsonl'))
+                .map((e) => join(dir, e.name));
+            return { ok: true, paths };
+        }
+        catch (err) {
+            return { ok: false, reason: classifyDirError(err) };
+        }
+    },
+    async listTreeJsonl(dir, limit) {
+        const paths = [];
+        let truncated = false;
+        let incomplete = false;
+        const stack = [dir];
+        let rootErrored;
+        let first = true;
+        while (stack.length > 0) {
+            const current = stack.pop();
+            let entries;
+            try {
+                entries = await readdir(current, { withFileTypes: true });
+            }
+            catch (err) {
+                if (first)
+                    rootErrored = classifyDirError(err);
+                else
+                    incomplete = true; // a subdirectory we could not read: disclose the gap
+                continue;
+            }
+            finally {
+                first = false;
+            }
+            for (const e of entries) {
+                const full = join(current, e.name);
+                if (e.isDirectory()) {
+                    stack.push(full);
+                }
+                else if (e.isFile() && e.name.endsWith('.jsonl')) {
+                    if (paths.length >= limit) {
+                        truncated = true;
+                        continue;
+                    }
+                    paths.push(full);
+                }
+            }
+        }
+        if (rootErrored)
+            return { ok: false, reason: rootErrored };
+        return { paths, truncated, incomplete };
+    },
+    async readFirstLine(path) {
+        let handle;
+        try {
+            handle = await open(path, 'r');
+        }
+        catch {
+            return { ok: false, reason: 'inaccessible' };
+        }
+        try {
+            // Identify the exact bytes about to be read, from the SAME open handle, so the
+            // generation stamped on the result is the file whose content derived the ctx —
+            // not a replacement swapped in between this read and the reader's later stat.
+            const idStat = await handle.stat({ bigint: true });
+            const id = { dev: idStat.dev, ino: idStat.ino };
+            // Read the COMPLETE first line: a Claude first record can exceed one chunk,
+            // and a truncated prefix fails to parse, which would silently fall through to
+            // slug-trust (mis-binding) or falsely split a multibyte char (false malformed).
+            const chunks = [];
+            let total = 0;
+            let pos = 0;
+            let sawByte = false;
+            for (;;) {
+                const buf = Buffer.allocUnsafe(FIRST_LINE_CHUNK);
+                const { bytesRead } = await handle.read(buf, 0, FIRST_LINE_CHUNK, pos);
+                if (bytesRead === 0)
+                    break; // EOF before any newline
+                sawByte = true;
+                pos += bytesRead;
+                const nl = buf.subarray(0, bytesRead).indexOf(0x0a);
+                const lineLen = nl >= 0 ? nl : bytesRead;
+                // A single line beyond this bound is pathological (not line-delimited
+                // JSONL); disclose it malformed rather than read unbounded memory. Checked
+                // whether or not the chunk ends the line, so a huge line terminated by a
+                // newline cannot slip past the bound an unterminated one is held to.
+                if (total + lineLen > MAX_FIRST_LINE)
+                    return { ok: false, reason: 'malformed' };
+                chunks.push(buf.subarray(0, lineLen));
+                total += lineLen;
+                if (nl >= 0)
+                    break;
+            }
+            if (!sawByte)
+                return { ok: false, reason: 'empty' };
+            let line;
+            try {
+                // Fatal decode over the COMPLETE line bytes: a lossily-decoded id/cwd would
+                // coin a fabricated session identity or membership. Invalid UTF-8 is
+                // malformed, not a clean read; decoding the whole line at once keeps a
+                // multibyte char that straddles a chunk boundary intact.
+                line = FATAL_UTF8.decode(Buffer.concat(chunks, total));
+            }
+            catch {
+                return { ok: false, reason: 'malformed' };
+            }
+            return { ok: true, line, id };
+        }
+        catch {
+            return { ok: false, reason: 'inaccessible' };
+        }
+        finally {
+            await handle.close();
+        }
+    },
+    async readHeadLines(path) {
+        let handle;
+        try {
+            handle = await open(path, 'r');
+        }
+        catch {
+            return { ok: false, reason: 'inaccessible' };
+        }
+        try {
+            // The generation of the exact bytes scanned, from the same open handle (see
+            // readFirstLine): the id stamped on the binding is the file this head derived.
+            const idStat = await handle.stat({ bigint: true });
+            const id = { dev: idStat.dev, ino: idStat.ino };
+            const lines = [];
+            let pos = 0;
+            let sawByte = false;
+            let truncated = false; // stopped at a line/byte budget, not EOF: more may lie beyond
+            let skipped = false; // at least one line was dropped (oversized/undecodable): a gap
+            let cur = []; // the current line's bytes, across chunk boundaries
+            let curLen = 0;
+            let skip = false; // the current line exceeded the per-line cap: discard it
+            const finish = (seg) => {
+                // Finalize the current line [.. seg]; return true when the line budget is hit.
+                if (!skip && curLen + seg.length <= HEAD_LINE_CAP) {
+                    cur.push(Buffer.from(seg));
+                    try {
+                        // Fatal-decode the whole line at once so a multibyte char split across a
+                        // chunk survives; an undecodable line is skipped, not fabricated.
+                        lines.push(FATAL_UTF8.decode(Buffer.concat(cur)));
+                    }
+                    catch {
+                        skipped = true; // an undecodable line dropped: a cwd could have been here
+                    }
+                }
+                else {
+                    skipped = true; // an oversized line dropped: a cwd could have been here
+                }
+                cur = [];
+                curLen = 0;
+                skip = false;
+                return lines.length >= HEAD_SCAN_LINES;
+            };
+            outer: for (;;) {
+                const buf = Buffer.allocUnsafe(FIRST_LINE_CHUNK);
+                const { bytesRead } = await handle.read(buf, 0, FIRST_LINE_CHUNK, pos);
+                if (bytesRead === 0)
+                    break; // EOF: the whole file was read
+                sawByte = true;
+                pos += bytesRead;
+                let start = 0;
+                for (let i = 0; i < bytesRead; i += 1) {
+                    if (buf[i] !== 0x0a)
+                        continue;
+                    if (finish(buf.subarray(start, i))) {
+                        truncated = true; // hit the line budget with more file to read
+                        break outer;
+                    }
+                    start = i + 1;
+                }
+                if (start < bytesRead) {
+                    const seg = buf.subarray(start, bytesRead);
+                    curLen += seg.length;
+                    if (curLen > HEAD_LINE_CAP) {
+                        skip = true;
+                        cur = [];
+                    }
+                    else {
+                        cur.push(Buffer.from(seg));
+                    }
+                }
+                if (pos >= HEAD_SCAN_BYTES) {
+                    truncated = true; // hit the byte budget
+                    break;
+                }
+            }
+            if (!sawByte)
+                return { ok: false, reason: 'empty' };
+            return { ok: true, lines, truncated, skipped, id };
+        }
+        catch {
+            return { ok: false, reason: 'inaccessible' };
+        }
+        finally {
+            await handle.close();
+        }
+    },
+    async realpath(path) {
+        try {
+            return await fsRealpath(path);
+        }
+        catch {
+            return undefined;
+        }
+    },
+    async probe(path) {
+        let stats;
+        try {
+            stats = await fsLstat(path);
+        }
+        catch (err) {
+            // ENOENT (nothing there) and ENOTDIR (a parent component is not a directory,
+            // so nothing can exist below it) are both confirmed absence. Anything else
+            // (e.g. EACCES) is a failure to inspect, which must NOT masquerade as absence.
+            const code = err.code;
+            return code === 'ENOENT' || code === 'ENOTDIR' ? { kind: 'absent' } : { kind: 'error' };
+        }
+        if (!stats.isSymbolicLink())
+            return { kind: 'present' };
+        try {
+            // The RAW literal target: callers resolve it against the filesystem so an
+            // intervening symlink is followed before any `..` in the target.
+            return { kind: 'symlink', target: await fsReadlink(path) };
+        }
+        catch {
+            return { kind: 'error' };
+        }
+    },
+};

package/dist/transcript/runner.js

@@ -0,0 +1,62 @@
+import { createTranscriptWatcher } from "./watcher.js";
+export function createCoverageRunner(opts) {
+    const watchers = opts.harnesses.map((harness) => createTranscriptWatcher({
+        harness,
+        home: opts.homes[harness],
+        root: opts.root,
+        codexScanLimit: opts.codexScanLimit,
+        discoveryIO: opts.discoveryIO,
+        fileIO: opts.fileIO,
+        sink: opts.sink,
+        publish: opts.publish,
+    }));
+    const setTimer = opts.setTimer ??
+        ((ms, fn) => {
+            const t = setTimeout(fn, ms);
+            t.unref?.();
+            return t;
+        });
+    const clearTimer = opts.clearTimer ?? ((h) => clearTimeout(h));
+    let stopped = false;
+    let timer;
+    let inflight;
+    const tick = async () => {
+        for (const w of watchers) {
+            try {
+                await w.tick();
+            }
+            catch (err) {
+                opts.onError?.(err);
+            }
+        }
+    };
+    const schedule = () => {
+        if (stopped)
+            return;
+        timer = setTimer(opts.intervalMs, run);
+    };
+    const run = () => {
+        if (stopped)
+            return;
+        timer = undefined;
+        inflight = tick().finally(() => {
+            inflight = undefined;
+            schedule();
+        });
+    };
+    const start = () => {
+        if (stopped || timer !== undefined || inflight !== undefined)
+            return;
+        run();
+    };
+    const stop = async () => {
+        stopped = true;
+        if (timer !== undefined) {
+            clearTimer(timer);
+            timer = undefined;
+        }
+        if (inflight)
+            await inflight.catch(() => { });
+    };
+    return { tick, start, stop };
+}

package/dist/transcript/steppers.js

@@ -0,0 +1,30 @@
+import { claudeStep, initialClaudeState } from "./claude.js";
+import { codexStep, initialCodexState } from "./codex.js";
+/** Bind the pure Claude core to one file's mutable join state. */
+export function claudeStepper(ctx) {
+    let state = initialClaudeState();
+    return {
+        reset: () => {
+            state = initialClaudeState();
+        },
+        step: (record) => {
+            const out = claudeStep(state, record, ctx);
+            state = out.state;
+            return { evidence: out.evidence, diagnostics: out.diagnostics };
+        },
+    };
+}
+/** Bind the pure Codex core to one file's mutable join state. */
+export function codexStepper(ctx) {
+    let state = initialCodexState();
+    return {
+        reset: () => {
+            state = initialCodexState();
+        },
+        step: (record) => {
+            const out = codexStep(state, record, ctx);
+            state = out.state;
+            return { evidence: out.evidence, diagnostics: out.diagnostics };
+        },
+    };
+}

package/dist/transcript/types.js

@@ -0,0 +1,30 @@
+import { isAbsolute, relative, resolve, sep } from 'node:path';
+/**
+ * Relativize native paths into the capture root's space, dropping anything that
+ * escapes the root (a different worktree, a parent dir) — such a write cannot
+ * match any change this session captures. Returns a `paths` scope with the
+ * in-root paths, or `undefined` when nothing remains in scope.
+ *
+ * Pure and string-only: it assumes `root`/`cwd` are already canonical (the
+ * plumbing realpaths them once at bind time). An absolute record path in a
+ * confirmed alias namespace (`ctx.rootAliases`) relativizes to the same in-root
+ * path as capture's canonical form, so it is kept rather than silently dropped.
+ */
+export function scopeFromPaths(rawPaths, ctx) {
+    const roots = [ctx.root, ...(ctx.rootAliases ?? [])];
+    const kept = [];
+    for (const raw of rawPaths) {
+        if (typeof raw !== 'string' || raw.length === 0)
+            continue;
+        const abs = isAbsolute(raw) ? raw : resolve(ctx.cwd, raw);
+        for (const root of roots) {
+            const rel = relative(root, abs);
+            if (rel === '' || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel))
+                continue;
+            if (!kept.includes(rel))
+                kept.push(rel);
+            break;
+        }
+    }
+    return kept.length > 0 ? { kind: 'paths', paths: kept } : undefined;
+}

package/dist/transcript/watcher.js

@@ -0,0 +1,133 @@
+import { discover } from "./discovery.js";
+import { createTranscriptFileReader, } from "./file-reader.js";
+import { claudeStepper, codexStepper } from "./steppers.js";
+/**
+ * Fold discovery issues and per-file read results into one honest coverage
+ * verdict. A file that was actually read (`readable`/`degraded`) counts as
+ * coverage; a `missing`/`inaccessible` file did not. Any issue at all — or a file
+ * whose evidence is still held back by ingestion backpressure — downgrades a
+ * successful read to `degraded`, so one readable transcript never conceals an
+ * unreadable sibling and `readable` never claims a scope whose evidence has not
+ * yet been recorded. With nothing read, a hard failure is `unavailable` while a
+ * merely-not-yet-written home stays `pending`. An unconfirmed replacement degrades
+ * a readable aggregate but stays pending when alone.
+ */
+export function aggregateCoverage(discoveryIssues, files) {
+    const issues = [...discoveryIssues];
+    let readCount = 0;
+    let backpressured = false;
+    let withheld = false;
+    for (const f of files) {
+        if (f.state === 'readable' || f.state === 'degraded') {
+            readCount += 1;
+            if (f.backpressured)
+                backpressured = true;
+            for (const d of f.issues)
+                issues.push({ kind: d.kind, detail: d.detail });
+        }
+        else if (f.state === 'unconfirmed') {
+            withheld = true;
+            // The file at this path was replaced (a new inode) and discovery has not yet
+            // re-confirmed the replacement's membership/scope. It is neither read nor a
+            // failure: withhold it (pending-compatible, no issue) so a one-tick self-healing
+            // replacement race never flaps coverage to unavailable. Next tick's discovery
+            // re-derives the binding for the new generation (recreating this reader if it is
+            // a confirmed in-root member, or dropping it if it is out-of-root); nothing is
+            // dropped, enrichment is delayed one tick.
+            continue;
+        }
+        else if (f.state === 'missing') {
+            issues.push({ kind: 'missing', detail: `transcript ${f.path ?? 'transcript'} disappeared` });
+        }
+        else {
+            issues.push({ kind: 'inaccessible', detail: `transcript ${f.path ?? 'transcript'} unreadable` });
+        }
+    }
+    let state;
+    if (readCount > 0) {
+        state = issues.length > 0 || backpressured || withheld ? 'degraded' : 'readable';
+    }
+    else {
+        const hardFailure = issues.some((i) => i.kind !== 'missing');
+        state = hardFailure ? 'unavailable' : 'pending';
+    }
+    return { state, issues };
+}
+function coverageKey(data) {
+    const issues = (data.issues ?? [])
+        .map((i) => `${i.kind}:${i.detail}`)
+        .sort()
+        .join('|');
+    return `${data.state}#${issues}`;
+}
+function makeStepper(harness, ctx) {
+    return harness === 'claude-code' ? claudeStepper(ctx) : codexStepper(ctx);
+}
+/** The identity that gates reader reuse: the harness session, the resolved scope
+ * (canonical cwd + root aliases), AND the file generation. A change in the ctx means
+ * the prior reader was reading under a scope discovery has since corrected — most
+ * often a provisional binding, made while the transcript was empty, whose real
+ * cwd/aliases discovery only learned once records (or a cwd-bearing record past a
+ * leading summary) appeared. A change in the generation means the file at this path
+ * was replaced (a new inode) and discovery has re-confirmed the replacement in-root:
+ * the old reader is pinned to the prior inode and would refuse the new one, so it is
+ * dropped and a fresh reader is created against the confirmed new generation. */
+function bindingReaderKey(binding) {
+    const { ctx, generation } = binding;
+    return `${ctx.harnessSessionId}\0${ctx.cwd}\0${(ctx.rootAliases ?? []).join(',')}\0${generation.dev}:${generation.ino}`;
+}
+export function createTranscriptWatcher(opts) {
+    const { harness, home, root, codexScanLimit, discoveryIO, fileIO, sink, publish } = opts;
+    const readers = new Map();
+    let lastKey;
+    const tick = async () => {
+        const { bindings, issues: discoveryIssues } = await discover(harness, discoveryIO, home, root, codexScanLimit);
+        const perFile = [];
+        for (const binding of bindings) {
+            // Recreate the reader whenever its bound context or file generation changes: a
+            // changed session id is a rotated transcript (the same file name now belongs to
+            // a different harness session), a changed cwd/aliases is a provisional binding
+            // (made while the transcript was empty, or before a cwd-bearing record appeared
+            // past a leading summary) that discovery has since resolved to the real scope,
+            // and a changed generation is a replacement (new inode) discovery has re-confirmed
+            // in-root. In every case the old reader read under a scope/generation that no
+            // longer applies, so drop it and reread from zero; the ingestor's log-derived
+            // dedup absorbs the re-read. This never conflicts an invocation: a record already
+            // read under the prior binding implies its cwd was known then (aliases are derived
+            // from the transcript's own cwd), so its scope does not change on the reread;
+            // only records first seen after the resolution get the corrected scope.
+            const ctxKey = bindingReaderKey(binding);
+            let entry = readers.get(binding.path);
+            if (!entry || entry.ctxKey !== ctxKey) {
+                entry = {
+                    reader: createTranscriptFileReader({
+                        path: binding.path,
+                        io: fileIO,
+                        sink,
+                        stepper: makeStepper(harness, binding.ctx),
+                        generation: binding.generation,
+                    }),
+                    ctxKey,
+                };
+                readers.set(binding.path, entry);
+            }
+            const result = await entry.reader.poll();
+            perFile.push({
+                state: result.state,
+                issues: result.issues,
+                backpressured: result.backpressured,
+                path: binding.path,
+            });
+        }
+        const coverage = aggregateCoverage(discoveryIssues, perFile);
+        const data = coverage.issues.length > 0
+            ? { harness, state: coverage.state, issues: coverage.issues }
+            : { harness, state: coverage.state };
+        const key = coverageKey(data);
+        if (key !== lastKey) {
+            await publish(data);
+            lastKey = key;
+        }
+    };
+    return { tick };
+}