@manyoars/oars 0.1.1 → 0.3.0

package/dist/sessions/cells.js

@@ -0,0 +1,181 @@
+// The cell (recall spec Section 2.5): counts for one motif in one frame bucket in one UTC week (or, for a live cell, one UTC day), order 0. Cells are computed from the capture records, which
+// stay the primary record, so a cell file can be dropped and rebuilt and any later derivation can run over old data. Nothing here reads raw.
+import { closedFrame, utcDay } from './frame.js';
+import { classMotif, motifsOf, stepsOfCall } from './motif.js';
+import { bucketOf } from './pool-wire.js';
+import { CALL_SHAPE_METHOD, cellRef, contentHash, METHODS } from './read.js';
+import { NO_RECALLS } from './recall-consequence.js';
+import { EXTRACTOR_VERSION, harnessOf } from './schema.js';
+export const UNKNOWN = 'unknown';
+/** The class name an effort cell carries: a class motif with no prompt class in it, so a read of prompt classes skips it by this prefix. */
+export const EFFORT_CLASS = 'effort:';
+/**
+ * Rule 3: effort is not in a cell's key. The one cell that holds it pairs it with the bucket of the prompt's thinking tokens, which is the fact an effort level explains, and it
+ * carries `week` and `exposure` but no frame axis: rule 3 bars joining `effort` to `model` or `harnessVersion`, and the pair would be four dimensions over `q_dims`.
+ */
+export const effortClass = (effort, thinking) => `${EFFORT_CLASS}${effort}/thinking:${String(bucketOf('tokens', thinking))}`;
+/** The frame bucket of a cell that joins no axis. */
+export const NO_AXES = '-';
+const ONE_OPERATOR = 1;
+/**
+ * A harness version's bucket: its patch by tens. 70.5 percent of one operator's later prompts ran a version not yet seen, so an
+ * exact-version cell goes stale in days; a reader compares buckets by distance instead. A version with no patch stands for itself.
+ */
+export const harnessBucket = (version) => {
+    if (version === null)
+        return UNKNOWN;
+    const found = /^(\d+\.\d+)\.(\d+)/.exec(version);
+    return found === null ? version : `${found[1] ?? ''}.${String(Math.floor(Number(found[2]) / 10))}x`;
+};
+/** Only what most comparisons cross: platform, shell, and stack are on the frame and in the records, so a later extractor can add them without new capture. */
+export const frameBucketOf = (frame) => [frame.harness, harnessBucket(frame.harnessVersion), frame.model ?? UNKNOWN].join('/');
+export const bucketFields = (bucket) => {
+    const [harness = UNKNOWN, version = UNKNOWN, model = UNKNOWN] = bucket.split('/');
+    return { harness, harnessBucket: version, model };
+};
+/** The frame bucket of a (name, `platform`, `shell`) cell: fit's third pool form. Rule 3 lets the two join each other and a name within `q_dims`, but not a version or a model beside them, so the cell is version-free and model-free. */
+export const ENVIRONMENT_BUCKET = 'env:';
+export const environmentBucketOf = (frame) => `${ENVIRONMENT_BUCKET}${frame.platform}:${frame.shell ?? UNKNOWN}`;
+export const isEnvironmentCell = (cell) => cell.frameBucket.startsWith(ENVIRONMENT_BUCKET);
+export const environmentFieldsOf = (bucket) => {
+    const held = bucket.slice(ENVIRONMENT_BUCKET.length);
+    const at = held.indexOf(':');
+    return { platform: held.slice(0, at), shell: held.slice(at + 1) };
+};
+/** A one-step named call with no flag and no pull: the name that is one dimension. A call with no identity (a file read, an edit) is its action alone, so it counts. A flag or a pull is a second dimension of its own, so its cell would be four with the pair. */
+const isPlainCall = (motif) => {
+    const [only] = motif.steps;
+    if (motif.level !== 'named' || motif.steps.length !== 1 || only === undefined)
+        return false;
+    return only.identity === null || !('program' in only.identity) || (only.identity.flags.length === 0 && (only.identity.pulls?.length ?? 0) === 0);
+};
+const DAY_MS = 86_400_000;
+/** The Monday of the UTC week a UTC day falls in, as a day: the week's name in a cell. An unknown day is an unknown week. */
+export const weekStartOf = (day) => {
+    if (day === UNKNOWN)
+        return UNKNOWN;
+    const millis = Date.parse(day);
+    return new Date(millis - ((new Date(millis).getUTCDay() + 6) % 7) * DAY_MS).toISOString().slice(0, 10);
+};
+/** The last UTC day a cell can hold: its day, or the Sunday that closes its week. A check that evidence predates a day reads this, since a week's cell may hold any of its seven days. */
+export const lastDayOfCell = (cell) => (cell.grain === 'week' && cell.day !== UNKNOWN ? new Date(Date.parse(cell.day) + 6 * DAY_MS).toISOString().slice(0, 10) : cell.day);
+const cellIds = new WeakMap();
+/** `cell:<contentHash>`: a cell that counts differently is a different cell, so a read's inputs pin what it read. */
+export const cellIdOf = (cell) => {
+    const known = cellIds.get(cell);
+    if (known !== undefined)
+        return known;
+    const id = cellRef(contentHash(cell));
+    cellIds.set(cell, id);
+    return id;
+};
+const newTally = (motifId, frameBucket, grain, day, exposed) => ({ motifId, frameBucket, grain, day, sessions: new Set(), ok: 0, failed: 0, tokens: 0, durationMs: 0, exposed });
+const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+/** A frame for a record captured before frames existed, from the fields it does hold, held to the same public vocabulary. */
+const oldFrame = (parts) => closedFrame({
+    harness: parts.harness,
+    harnessVersion: parts.version,
+    model: parts.model === '' ? null : parts.model,
+    effort: parts.effort,
+    day: utcDay(parts.ts),
+    platform: parts.platform,
+    shell: null,
+    stack: { languages: [], frameworks: [], size: null },
+});
+const mostUsedModel = (models) => [...Object.entries(models)].sort(([a, one], [b, two]) => two - one || compare(a, b))[0]?.[0] ?? null;
+const framedPrompt = (prompt, harness, version, ts, platform) => prompt.frame ?? oldFrame({ harness, version, model: prompt.model, effort: prompt.efforts[0] ?? null, ts: prompt.ts ?? ts, platform });
+/** One line of work (spec Section 2.3): a window never crosses a prompt, a sidechain, or a transcript. */
+const linesOf = (tools) => {
+    const lines = new Map();
+    for (const tool of tools) {
+        const key = `${tool.session}\n${String(tool.side)}\n${String(tool.prompt)}`;
+        const line = lines.get(key);
+        if (line === undefined)
+            lines.set(key, [tool]);
+        else
+            line.push(tool);
+    }
+    return [...lines.values()].map((line) => line.sort((a, b) => a.n - b.n));
+};
+export const deriveCells = (store, graph = NO_RECALLS) => {
+    const tallies = new Map();
+    const motifs = new Map();
+    const tallyAt = (motif, frameBucket, grain, day, exposed) => {
+        const key = `${motif.motifId}/${frameBucket}/${grain}/${day}/${String(exposed)}`;
+        const found = tallies.get(key);
+        if (found !== undefined)
+            return found;
+        const fresh = newTally(motif.motifId, frameBucket, grain, day, exposed);
+        tallies.set(key, fresh);
+        return fresh;
+    };
+    /** The tallies one occurrence counts into: its week cell, and for a one-step shape motif with a known day its live cell too (rule 6). A window of two or more steps, a named motif, and a class motif have no day cell. */
+    const talliesOf = (motif, frame, exposed, frameBucket = frameBucketOf(frame)) => {
+        motifs.set(motif.motifId, motif);
+        const day = frame.day ?? UNKNOWN;
+        const weekly = tallyAt(motif, frameBucket, 'week', weekStartOf(day), exposed);
+        const tallies = motif.level === 'shape' && motif.steps.length === 1 && day !== UNKNOWN ? [weekly, tallyAt(motif, frameBucket, 'day', day, exposed)] : [weekly];
+        // Fit's (name, platform, shell) cell: a week cell beside the versioned one, counting the same call.
+        return isPlainCall(motif) ? [...tallies, tallyAt(motif, environmentBucketOf(frame), 'week', weekStartOf(day), exposed)] : tallies;
+    };
+    const sessions = new Map([...store.sessions, ...store.subagents].map((session) => [session.session, session]));
+    const prompts = new Map(store.prompts.map((prompt) => [`${prompt.session}#${String(prompt.index)}`, prompt]));
+    for (const line of linesOf(store.tools)) {
+        const [first] = line;
+        const session = first === undefined ? undefined : sessions.get(first.session);
+        if (first === undefined || session === undefined)
+            continue;
+        const harness = harnessOf(session);
+        const version = first.harnessVersion ?? session.env.harness;
+        const prompt = prompts.get(`${first.session}#${String(first.prompt)}`);
+        const frame = prompt === undefined
+            ? (session.frame ?? oldFrame({ harness, version, model: mostUsedModel(session.models), effort: null, ts: first.ts ?? session.tsFirst, platform: session.env.os }))
+            : framedPrompt(prompt, harness, version, first.ts, session.env.os);
+        const placed = line.flatMap((tool) => graph.place(tool, stepsOfCall(tool, harness)));
+        const steps = placed.map(({ step }) => step);
+        for (const { motif, start } of motifsOf(steps)) {
+            const window = steps.slice(start, start + motif.steps.length);
+            const status = window.at(-1)?.status;
+            for (const tally of talliesOf(motif, frame, placed.slice(start, start + motif.steps.length).some(({ exposed }) => exposed))) {
+                tally.sessions.add(first.parent);
+                if (status === 'ok')
+                    tally.ok += 1;
+                if (status === 'error')
+                    tally.failed += 1;
+                for (const step of window) {
+                    tally.tokens += step.tokens;
+                    tally.durationMs += step.durationMs;
+                }
+            }
+        }
+    }
+    for (const prompt of store.prompts) {
+        const session = sessions.get(prompt.session);
+        const frame = framedPrompt(prompt, harnessOf(session ?? {}), session?.env.harness ?? null, null, session?.env.os ?? UNKNOWN);
+        const exposed = graph.exposedPrompt(prompt.session, prompt.index);
+        const effort = frame.effort === null ? [] : talliesOf(classMotif(effortClass(frame.effort, prompt.usage.thinking)), frame, exposed, NO_AXES);
+        for (const tally of [...talliesOf(classMotif(prompt.cls), frame, exposed), ...effort]) {
+            tally.sessions.add(prompt.session);
+            if (prompt.errors > 0)
+                tally.failed += 1;
+            else
+                tally.ok += 1;
+            const { input, cacheWrite, cacheRead, output } = prompt.usage;
+            tally.tokens += input + cacheWrite + cacheRead + output;
+            tally.durationMs += (prompt.wallSec ?? 0) * 1000;
+        }
+    }
+    const cells = [...tallies.values()]
+        .sort((a, b) => compare(a.motifId, b.motifId) || compare(a.frameBucket, b.frameBucket) || compare(a.grain, b.grain) || compare(a.day, b.day) || Number(a.exposed) - Number(b.exposed))
+        .map((tally) => ({ motifId: tally.motifId, frameBucket: tally.frameBucket, grain: tally.grain, day: tally.day, sessions: tally.sessions.size, clusters: ONE_OPERATOR, ok: tally.ok, failed: tally.failed, tokens: tally.tokens, durationMs: tally.durationMs, exposed: tally.exposed }));
+    return { cells, motifs: [...motifs.values()].sort((a, b) => compare(a.motifId, b.motifId)) };
+};
+export const serializeCells = (cells) => cells.map((cell) => `${JSON.stringify(cell)}\n`).join('');
+/** The fields of an error row a recovery read depends on, as text: only those, so a change elsewhere is not a new pool, and sorted, so record order is not one either. */
+const errorRowsRead = (errors) => errors.map(({ session, agent, toolUseId, prompt, tool, sig, ts, next, editFollowed }) => JSON.stringify([session, agent ?? null, toolUseId ?? null, prompt, tool, sig, ts, next, editFollowed])).sort();
+export const poolOf = (set, errors) => ({
+    cells: set.cells,
+    motifs: new Map(set.motifs.map((motif) => [motif.motifId, motif])),
+    version: contentHash([EXTRACTOR_VERSION, METHODS, CALL_SHAPE_METHOD, set.cells.map(cellIdOf), errorRowsRead(errors)]),
+    lastDay: set.cells.reduce((last, cell) => (cell.day !== UNKNOWN && (last === null || cell.day > last) ? cell.day : last), null),
+});

package/dist/sessions/codex.d.ts

@@ -0,0 +1,53 @@
+/** The Codex home for an environment; a test passes its own so it never reads the real one. */
+export declare const codexRoot: (env: {
+    [name: string]: string | undefined;
+}, home?: string) => string;
+type Obj = {
+    [key: string]: unknown;
+};
+/** A rollout file found in the store, before its lines are read. */
+export interface CodexSession {
+    /** The rollout's key: the thread uuid in the file name (Codex's own session id), `<thread>_<rollout>` for a reverted thread's file when the thread itself is not in the store (else its file is folded into the thread's, `revisions`), else the file's stem. */
+    id: string;
+    /** `session_meta.cwd`; null when the first line names none. */
+    folder: string | null;
+    /** `session_meta.parent_thread_id`: set for a spawned agent's rollout, which is deposited as a subagent of that thread. */
+    parent: string | null;
+    /** `session_meta.agent_role` (protocol.rs:3486), when the spawned agent has one. */
+    agentRole: string | null;
+    /** `session_meta.forked_from_id`: the thread whose history a fork copied, to match its copied records against. */
+    forkedFrom: string | null;
+    /**
+     * The rollouts of the thread's reverts, folded into this one (thread-store/src/local/revert_thread.rs:15-18): each starts at `from`, the ordinal it
+     * takes from `history_base.end_ordinal_exclusive`, and supersedes the records at or after it in every earlier file. Empty for a thread never reverted.
+     */
+    revisions: {
+        file: string;
+        from: number;
+        at: number;
+    }[];
+    /** `session_meta.timestamp` in milliseconds (a fresh `started_at` for each new rollout, rollout/src/recorder.rs:953-966); 0 when absent. A fork is matched against the lineage as of this time. */
+    started: number;
+    file: string;
+    size: number;
+    mtimeMs: number;
+}
+export interface CodexStore {
+    sessions: CodexSession[];
+    /** The session as transcript text for `captureSession`; throws when the file cannot be read. */
+    transcript: (session: CodexSession) => Promise<string>;
+    /** Files skipped because their id is not a safe file name (it names a skeleton), counted so the skip is never silent. */
+    rejected: number;
+    /** Files that could not be inspected, reported once each and never fatal; the reason names no path, so a log line built from it leaks none. */
+    unreadable: {
+        file: string;
+        reason: string;
+    }[];
+}
+/** The non-blank lines of a rollout that are not JSON (a torn write), kept so `captureSession` counts them and the deposit line reports them. */
+export declare const rolloutBadLines: (text: string) => string[];
+/** One rollout as the lines `captureSession` reads. */
+export declare const rolloutLines: (text: string, fallbackId: string, parentText?: string) => Obj[];
+/** Opens the Codex store under `root`; undefined when it holds no `sessions` folder, so a machine without Codex is skipped silently. */
+export declare const openCodexStore: (root: string) => Promise<CodexStore | undefined>;
+export {};