@oortmem/claude-code 0.0.0-stage → 0.1.1

package/dist/queue.js

@@ -0,0 +1,351 @@
+import { createHash, randomBytes, randomUUID } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { z } from 'zod';
+import { appendPrivateFile, ensurePrivateDir, observe, scrubObservation, } from '@oortmem/core';
+import { unlinkQuietly, withShortLock, writeFileAtomic } from './fsutil.js';
+/**
+ * The write-path queue between the Stop hook and the drain.
+ *
+ * - `$OORT_HOME/queue/<session>.jsonl`: one JSON line per scrubbed observation,
+ *   appended by the Stop hook (0600, directory 0700).
+ * - `$OORT_HOME/queue/.lock`: the drain lock (`wx`); one drain at a time. A lock
+ *   older than ten minutes is from a crashed drain and is broken.
+ * - `<file>.lock`: a short per-file lock held by an append, and by the drain
+ *   while it rewrites the file, so no concurrent append is lost.
+ * - `$OORT_HOME/queue/dead/<session>.jsonl`: lines that failed three times, or
+ *   could not be parsed.
+ */
+export const DRAIN_LOCK_STALE_MS = 10 * 60 * 1000;
+export const DRAIN_MAX_PER_RUN = 50;
+export const MAX_ATTEMPTS = 3;
+/** Consecutive adapter failures after which a drain run stops (the adapter is likely down). */
+const CIRCUIT_BREAKER = 3;
+const APPEND_LOCK_WAIT_MS = 500;
+const COMMIT_LOCK_WAIT_MS = 5_000;
+const ScopeSchema = z.object({
+    orgId: z.string(),
+    customerId: z.string(),
+    agentId: z.string(),
+    projectId: z.string().optional(),
+});
+const ObservationSchema = z.object({
+    userText: z.string(),
+    assistantText: z.string(),
+    toolSummaries: z.array(z.string()),
+    sessionId: z.string().optional(),
+    turnRef: z.string().optional(),
+    projectId: z.string().optional(),
+    at: z.string(),
+});
+const QueueLineSchema = z.object({
+    v: z.literal(1),
+    id: z.string(),
+    scope: ScopeSchema,
+    obs: ObservationSchema,
+    attempts: z.number().int().min(0),
+    queuedAt: z.string(),
+});
+export const queueDir = (home) => path.join(home, 'queue');
+/** A file-name-safe form of a session id; anything unusual is hashed. */
+export function queueName(session) {
+    return /^[A-Za-z0-9_-]{1,128}$/.test(session)
+        ? session
+        : `h-${createHash('sha256').update(session).digest('hex').slice(0, 40)}`;
+}
+/**
+ * Scrubs and appends observations to the session's queue file; observations
+ * that are mostly secret are dropped. Never holds a lock across anything but
+ * the append; if the file lock is busy, the batch goes to a unique segment
+ * file instead (the drain reads every `*.jsonl`), so nothing is lost.
+ */
+export function enqueueObservations(home, session, scope, observations, now = new Date()) {
+    const lines = [];
+    let dropped = 0;
+    for (const raw of observations) {
+        const s = scrubObservation(raw);
+        if (s.dropped) {
+            dropped++;
+            continue;
+        }
+        const line = {
+            v: 1,
+            id: randomUUID(),
+            scope,
+            obs: s.obs,
+            attempts: 0,
+            queuedAt: now.toISOString(),
+        };
+        lines.push(`${JSON.stringify(line)}\n`);
+    }
+    if (lines.length === 0)
+        return { queued: 0, dropped };
+    const dir = queueDir(home);
+    ensurePrivateDir(dir);
+    const file = path.join(dir, `${queueName(session)}.jsonl`);
+    const data = lines.join('');
+    const r = withShortLock(`${file}.lock`, APPEND_LOCK_WAIT_MS, () => appendPrivateFile(file, data));
+    if (!r.ok) {
+        const segment = path.join(dir, `${queueName(session)}.${process.pid}-${randomBytes(4).toString('hex')}.jsonl`);
+        appendPrivateFile(segment, data);
+    }
+    return { queued: lines.length, dropped };
+}
+function queueFiles(dir) {
+    let names;
+    try {
+        names = fs.readdirSync(dir);
+    }
+    catch {
+        return [];
+    }
+    const files = [];
+    for (const name of names) {
+        if (name.startsWith('.') || !name.endsWith('.jsonl'))
+            continue;
+        try {
+            const st = fs.statSync(path.join(dir, name));
+            if (st.isFile())
+                files.push({ name, mtime: st.mtimeMs });
+        }
+        catch {
+            // Removed by a concurrent drain.
+        }
+    }
+    files.sort((a, b) => a.mtime - b.mtime || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    return files.map((f) => f.name);
+}
+function countLines(dir) {
+    let n = 0;
+    for (const name of queueFiles(dir)) {
+        try {
+            const text = fs.readFileSync(path.join(dir, name), 'utf8');
+            for (const l of text.split('\n'))
+                if (l.length > 0)
+                    n++;
+        }
+        catch {
+            // Gone.
+        }
+    }
+    return n;
+}
+/** True when some queue file is waiting to be drained. */
+export function hasPendingQueue(home) {
+    return queueFiles(queueDir(home)).length > 0;
+}
+function parseLine(raw) {
+    try {
+        const r = QueueLineSchema.safeParse(JSON.parse(raw));
+        return r.success ? r.data : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Takes the drain lock, breaking one older than ten minutes; null when another drain holds it. */
+function acquireDrainLock(lock) {
+    try {
+        return fs.openSync(lock, 'wx', 0o600);
+    }
+    catch (e) {
+        if (e.code !== 'EEXIST')
+            throw e;
+    }
+    try {
+        if (Date.now() - fs.statSync(lock).mtimeMs <= DRAIN_LOCK_STALE_MS)
+            return null;
+        unlinkQuietly(lock);
+        return fs.openSync(lock, 'wx', 0o600);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * A stable id for a queued turn: `<sessionId>:<transcript turn>` when both are known, else the
+ * queue line's own id (kept across retries, since a failed line stays in its file).
+ */
+export function turnRefOf(line) {
+    return line.obs.sessionId !== undefined && line.obs.turnRef !== undefined
+        ? `${line.obs.sessionId}:${line.obs.turnRef}`
+        : line.id;
+}
+/**
+ * Observes queued lines, oldest file first, at most 50 per run, under the
+ * drain lock. A line that observed is removed; one whose observe threw keeps
+ * its place with `attempts + 1`, and moves to `dead/` at the third failure.
+ * Without embed, decide and write adapters the queue is left untouched.
+ */
+export async function drainQueue(deps) {
+    const t0 = performance.now();
+    const dir = queueDir(deps.config.home);
+    const logBase = { event: 'drain', session: null, project: null };
+    const { embed, decide, write, remote } = deps;
+    if (remote === undefined &&
+        (embed === undefined || decide === undefined || write === undefined)) {
+        const remaining = countLines(dir);
+        deps.log.write({
+            ...logBase,
+            outcome: 'drain-skipped:missing-adapters',
+            counts: { remaining },
+            timings: { totalMs: performance.now() - t0 },
+        });
+        return { processed: 0, failed: 0, remaining };
+    }
+    if (queueFiles(dir).length === 0) {
+        deps.log.write({
+            ...logBase,
+            outcome: 'ok',
+            counts: { processed: 0, failed: 0, remaining: 0 },
+            timings: { totalMs: performance.now() - t0 },
+        });
+        return { processed: 0, failed: 0, remaining: 0 };
+    }
+    const lockPath = path.join(dir, '.lock');
+    const lock = acquireDrainLock(lockPath);
+    if (lock === null) {
+        const remaining = countLines(dir);
+        deps.log.write({
+            ...logBase,
+            outcome: 'locked',
+            counts: { remaining },
+            timings: { totalMs: performance.now() - t0 },
+        });
+        return { processed: 0, failed: 0, remaining };
+    }
+    let processed = 0;
+    let failed = 0;
+    let budget = DRAIN_MAX_PER_RUN;
+    let consecutive = 0;
+    let deadCount = 0;
+    let held = false;
+    const deadIds = [];
+    try {
+        fs.writeSync(lock, `${process.pid}\n`);
+        for (const name of queueFiles(dir)) {
+            if (budget === 0 || consecutive >= CIRCUIT_BREAKER)
+                break;
+            const file = path.join(dir, name);
+            let buf;
+            try {
+                buf = fs.readFileSync(file);
+            }
+            catch {
+                continue;
+            }
+            // Only complete lines; a trailing partial line is left for later.
+            const end = buf.lastIndexOf(0x0a) + 1;
+            const lines = buf.subarray(0, end).toString('utf8').split('\n');
+            lines.pop();
+            const keep = [];
+            const dead = [];
+            for (const raw of lines) {
+                if (raw.length === 0)
+                    continue;
+                if (budget === 0 || consecutive >= CIRCUIT_BREAKER) {
+                    keep.push(raw);
+                    continue;
+                }
+                const line = parseLine(raw);
+                if (line === undefined) {
+                    dead.push(raw);
+                    failed++;
+                    continue;
+                }
+                budget--;
+                try {
+                    if (remote !== undefined) {
+                        await remote.observe(line.obs, line.scope.projectId, turnRefOf(line));
+                    }
+                    else {
+                        await observe(line.obs, line.scope, {
+                            store: deps.store,
+                            embed: embed,
+                            decide: decide,
+                            write: write,
+                            config: deps.config,
+                            now: deps.now?.() ?? new Date(),
+                            harness: 'claude-code',
+                        });
+                    }
+                    processed++;
+                    consecutive = 0;
+                }
+                catch (e) {
+                    if (e?.holdQueue === true) {
+                        // The account cannot save right now (signed out, no plan, a limit, an outage): keep
+                        // this line and the rest exactly as they are and try again on a later drain.
+                        held = true;
+                        keep.push(raw);
+                        budget = 0;
+                        continue;
+                    }
+                    failed++;
+                    consecutive++;
+                    const next = JSON.stringify({ ...line, attempts: line.attempts + 1 });
+                    if (line.attempts + 1 >= MAX_ATTEMPTS) {
+                        dead.push(next);
+                        deadIds.push(line.id);
+                    }
+                    else {
+                        keep.push(next);
+                    }
+                }
+                try {
+                    const now = new Date();
+                    fs.futimesSync(lock, now, now);
+                }
+                catch {
+                    // Best effort: keeps a long run from looking stale.
+                }
+            }
+            if (dead.length > 0) {
+                deadCount += dead.length;
+                appendPrivateFile(path.join(dir, 'dead', name), dead.map((l) => `${l}\n`).join(''));
+            }
+            const committed = withShortLock(`${file}.lock`, COMMIT_LOCK_WAIT_MS, () => {
+                let current;
+                try {
+                    current = fs.readFileSync(file);
+                }
+                catch {
+                    current = Buffer.alloc(0);
+                }
+                // Appends only ever grow the file; everything past `end` arrived during this run.
+                const tail = current.subarray(Math.min(end, current.length));
+                const out = Buffer.concat([Buffer.from(keep.map((l) => `${l}\n`).join(''), 'utf8'), tail]);
+                if (out.length === 0)
+                    unlinkQuietly(file);
+                else
+                    writeFileAtomic(file, out, 0o600);
+            });
+            if (!committed.ok) {
+                // Lines stay queued and are observed again next run; a duplicate is reconciled.
+                deps.log.write({ ...logBase, outcome: 'commit-lock-busy', counts: {}, timings: {} });
+            }
+        }
+    }
+    finally {
+        fs.closeSync(lock);
+        unlinkQuietly(lockPath);
+    }
+    if (deadCount > 0) {
+        deps.log.write({
+            ...logBase,
+            outcome: 'dead-lettered',
+            counts: { lines: deadCount },
+            timings: {},
+            ids: deadIds,
+        });
+    }
+    const remaining = countLines(dir);
+    deps.log.write({
+        ...logBase,
+        outcome: held ? 'held' : 'ok',
+        counts: { processed, failed, remaining },
+        timings: { totalMs: performance.now() - t0 },
+    });
+    return { processed, failed, remaining };
+}
+//# sourceMappingURL=queue.js.map

package/dist/transcript.d.ts

@@ -0,0 +1,88 @@
+import { type Observation } from '@oortmem/core';
+/**
+ * Claude Code session transcripts (`~/.claude/projects/<project>/<session>.jsonl`).
+ * Only `user` and `assistant` rows carry conversation; every other row type
+ * (`attachment`, `system`, `api-request`, `file-history-snapshot`,
+ * `queue-operation`, titles, cost state, ...) is skipped.
+ */
+export interface Turn {
+    /** The last row of the turn (for an assistant turn: its last assistant row). */
+    uuid: string;
+    /** The first row's parent. */
+    parentUuid: string | null;
+    role: 'user' | 'assistant';
+    text: string;
+    toolSummaries: string[];
+    /** The last row's timestamp. */
+    timestamp: string;
+    sessionId: string;
+    cwd: string;
+}
+/** A turn plus the uuids of every message row merged into it, in order. */
+export interface DetailedTurn {
+    turn: Turn;
+    rowUuids: string[];
+}
+/**
+ * XML-style blocks Claude Code writes into user rows on the harness's behalf:
+ * background-task reports, `!` shell mode, slash commands and their local
+ * output, and system reminders (which carry hook context). They are not the
+ * user's words and are stripped, tag and content, before a row becomes text.
+ */
+export declare const HARNESS_TAGS: readonly ['task-notification', 'bash-input', 'bash-stdout', 'bash-stderr', 'command-name', 'command-message', 'command-args', 'local-command-stdout', 'local-command-caveat', 'system-reminder'];
+/**
+ * Oort's own injected block (see `formatForInjection`). Stripped from user and
+ * assistant text alike, so recalled memories are never observed back into memory.
+ */
+export declare const OORT_MEMORY_TAG = "oort-memory";
+/**
+ * Scrubs `s` before cutting it to `n` chars: cutting first would leave a
+ * secret's prefix that the scrubber no longer recognises. A cut that would
+ * fall inside a `[REDACTED:…]` marker moves to just before it.
+ */
+export declare function scrubThenCut(s: string, n: number): string;
+/**
+ * Parses a transcript into alternating user and assistant turns, keeping the
+ * uuids of the rows behind each turn. Rules:
+ * - only `user`/`assistant` rows; sidechain, meta and malformed rows are skipped;
+ * - consecutive assistant rows (Claude Code writes one row per content block)
+ *   merge into one assistant turn; `thinking` blocks are ignored;
+ * - `tool_use` blocks become `"<name>(<first 120 chars of input>)"` summaries;
+ * - a user row holding only `tool_result` blocks merges into the previous
+ *   turn's summaries as `"result: <first 200 chars>"`; mixed rows contribute
+ *   their results to the previous turn and their text to a new user turn;
+ * - `HARNESS_TAGS` blocks and `<oort-memory>` are stripped from user text, and
+ *   `<oort-memory>` from assistant text; a user row holding only harness output
+ *   becomes a user turn with empty text, so the reply to it is a turn of its own.
+ */
+export declare function parseTranscriptDetailed(jsonl: string): DetailedTurn[];
+export declare function parseTranscript(jsonl: string): Turn[];
+/** An observation and the index (into the turn list) of the assistant turn it came from. */
+export interface PairedObservation {
+    obs: Observation;
+    assistantIndex: number;
+}
+/**
+ * One observation per user→assistant pair, at the assistant turn's timestamp
+ * (or the nearest known one when it has none, see `nearestKnownTimestamps`).
+ * Consecutive user turns (an interrupted prompt and its retry) are joined; an
+ * assistant turn without a user turn before it, and a trailing unanswered user
+ * turn, yield nothing. A reply to harness-only rows (a background task report,
+ * say) is observed with `userText: ""`: no text is invented for the user. When
+ * both sides are empty after stripping, the pair yields nothing.
+ */
+export declare function pairTurns(turns: Turn[]): PairedObservation[];
+export declare function turnsToObservations(turns: Turn[]): Observation[];
+export interface CursorSelection {
+    /** Observations after the cursor, oldest first. */
+    fresh: Observation[];
+    /** False when a cursor was set but no row of this transcript carries it. */
+    cursorFound: boolean;
+}
+/**
+ * Observations not yet processed, given `cursor` (the uuid of the last
+ * processed assistant row). When the cursor sits inside a turn rather than at
+ * its end, the turn grew after it was processed (the transcript lagged, or a
+ * stop hook made Claude continue), so it is taken again.
+ */
+export declare function selectAfterCursor(detailed: DetailedTurn[], cursor: string | null): CursorSelection;