@manyoars/oars 0.5.4 → 0.5.5

package/dist/sessions/recall-reads.js

@@ -109,55 +109,71 @@ export const FIT_FLOORS = { minFailedSessions: 3, minFailedShare: 0.1, minCalls:
  * `FIT_FLOORS`: a base rate is still a fact to read on request.
  */
 export const PUSH_FLOORS = { ...FIT_FLOORS, minFailedShare: 0.5 };
-/** `keep` narrows which calls are tallied, not what can come next: the call after a kept one is what followed it, wherever the slice ends. */
-export const fitReads = (store, floors = FIT_FLOORS, keep = () => true) => {
-    const models = modelsByPrompt(store);
-    const harnesses = harnessesOf(store);
+/**
+ * `fitReads` as a fold: each `add` tallies one run of sessions into the same shapes, and `finish` applies the floors. Runs come in the whole store's row order, so the tie order of `followed` matches.
+ * `keep` narrows which calls are tallied, not what can come next: the call after a kept one is what followed it, wherever the slice ends.
+ */
+export const fitFold = (floors = FIT_FLOORS, keep = () => true) => {
+    const harnesses = new Map();
     const shapes = new Map();
-    for (const calls of callsBySession(store).values()) {
-        for (const [position, call] of calls.entries()) {
-            if (!keep(call))
-                continue;
-            const id = toolShapeId(call.tool, call.keys);
-            const shape = shapes.get(id) ?? { tool: call.tool, keys: [...call.keys].sort(), calls: 0, failed: 0, sessions: new Set(), following: new Map(), days: [], versions: new Set(), models: new Set() };
-            shapes.set(id, shape);
-            shape.calls += 1;
-            if (call.status !== 'error')
+    const add = (store) => {
+        const models = modelsByPrompt(store);
+        for (const [session, harness] of harnessesOf(store))
+            harnesses.set(session, harness);
+        for (const calls of callsBySession(store).values()) {
+            for (const [position, call] of calls.entries()) {
+                if (!keep(call))
+                    continue;
+                const id = toolShapeId(call.tool, call.keys);
+                const shape = shapes.get(id) ?? { tool: call.tool, keys: [...call.keys].sort(), calls: 0, failed: 0, sessions: new Set(), following: new Map(), days: [], versions: new Set(), models: new Set() };
+                shapes.set(id, shape);
+                shape.calls += 1;
+                if (call.status !== 'error')
+                    continue;
+                shape.failed += 1;
+                shape.sessions.add(call.session);
+                shape.days.push(utcDay(call.ts));
+                if (call.harnessVersion !== null)
+                    shape.versions.add(call.harnessVersion);
+                const model = models.get(`${call.session}#${String(call.prompt)}`);
+                if (model !== undefined)
+                    shape.models.add(model);
+                const next = calls[position + 1];
+                if (next !== undefined)
+                    bump(shape.following, next.tool);
+            }
+        }
+    };
+    const finish = () => {
+        const reads = [];
+        for (const [readId, shape] of shapes) {
+            if (shape.sessions.size < floors.minFailedSessions || shape.calls < floors.minCalls || shape.failed / shape.calls < floors.minFailedShare)
                 continue;
-            shape.failed += 1;
-            shape.sessions.add(call.session);
-            shape.days.push(utcDay(call.ts));
-            if (call.harnessVersion !== null)
-                shape.versions.add(call.harnessVersion);
-            const model = models.get(`${call.session}#${String(call.prompt)}`);
-            if (model !== undefined)
-                shape.models.add(model);
-            const next = calls[position + 1];
-            if (next !== undefined)
-                bump(shape.following, next.tool);
+            reads.push({
+                readId,
+                family: 'fit',
+                tool: shape.tool,
+                keys: shape.keys,
+                n: shape.sessions.size,
+                count: shape.failed,
+                calls: shape.calls,
+                clusters: 1,
+                ...dayRange(shape.days),
+                harnesses: harnessNames(shape.sessions, harnesses),
+                harnessVersions: [...shape.versions].sort(),
+                models: [...shape.models].sort(),
+                followed: top(shape.following),
+            });
         }
-    }
-    const reads = [];
-    for (const [readId, shape] of shapes) {
-        if (shape.sessions.size < floors.minFailedSessions || shape.calls < floors.minCalls || shape.failed / shape.calls < floors.minFailedShare)
-            continue;
-        reads.push({
-            readId,
-            family: 'fit',
-            tool: shape.tool,
-            keys: shape.keys,
-            n: shape.sessions.size,
-            count: shape.failed,
-            calls: shape.calls,
-            clusters: 1,
-            ...dayRange(shape.days),
-            harnesses: harnessNames(shape.sessions, harnesses),
-            harnessVersions: [...shape.versions].sort(),
-            models: [...shape.models].sort(),
-            followed: top(shape.following),
-        });
-    }
-    return reads.sort((a, b) => b.count - a.count || a.readId.localeCompare(b.readId));
+        return reads.sort((x, y) => y.count - x.count || x.readId.localeCompare(y.readId));
+    };
+    return { add, finish };
+};
+/** `fitFold` over the whole store at once. `keep` narrows the calls tallied, not what follows them. */
+export const fitReads = (store, floors = FIT_FLOORS, keep = () => true) => {
+    const fold = fitFold(floors, keep);
+    fold.add(store);
+    return fold.finish();
 };
 /** The action of a tool under the first of its harnesses whose table lists it, else under the first harness (the default one when a read names none). */
 export const actionOfShape = (tool, harnesses) => {
@@ -170,28 +186,40 @@ export const actionOfShape = (tool, harnesses) => {
  * 79 of 79 pushes, at 4.5x and 8.6x their action's share: a real signal, not a defect. Open parameter, like the floors.
  */
 export const PUSH_MIN_LIFT = 2;
+/** The per-action call tally `aboveBaseRate` compares against, as a fold: each `add` tallies one run of sessions, so a refresh never holds every call. */
+export const actionFold = () => {
+    const byAction = new Map();
+    return {
+        add: (store) => {
+            const harnesses = harnessesOf(store);
+            for (const calls of callsBySession(store).values()) {
+                for (const call of calls) {
+                    const action = actionOfShape(call.tool, [harnesses.get(call.session) ?? harnessOf({})]);
+                    const tally = byAction.get(action) ?? { calls: 0, failed: 0 };
+                    byAction.set(action, tally);
+                    tally.calls += 1;
+                    if (call.status === 'error')
+                        tally.failed += 1;
+                }
+            }
+        },
+        finish: () => byAction,
+    };
+};
+/** `aboveBaseRate` over a finished action tally. */
+export const aboveActionRate = (byAction, reads, lift = PUSH_MIN_LIFT) => reads.filter((read) => {
+    const base = byAction.get(actionOfShape(read.tool, read.harnesses));
+    // A fit read always counts its calls (`calls` is null only for entity reads), and its own calls are in the tally, so `base` is there; the fallbacks keep the types honest.
+    return base !== undefined && read.calls !== null && read.count * base.calls >= lift * base.failed * read.calls;
+});
 /**
  * The reads whose shape fails at `lift` times the share of its whole action or more, where the action's share counts every call of every shape of it, those under the floors too.
  * A tool no table lists is the action `other` (or `mcp.call`), so it is held to that action's share like any other and can enter the index and fire. Compared in integers, so a tie is exact.
  */
 export const aboveBaseRate = (store, reads, lift = PUSH_MIN_LIFT) => {
-    const harnesses = harnessesOf(store);
-    const byAction = new Map();
-    for (const calls of callsBySession(store).values()) {
-        for (const call of calls) {
-            const action = actionOfShape(call.tool, [harnesses.get(call.session) ?? harnessOf({})]);
-            const tally = byAction.get(action) ?? { calls: 0, failed: 0 };
-            byAction.set(action, tally);
-            tally.calls += 1;
-            if (call.status === 'error')
-                tally.failed += 1;
-        }
-    }
-    return reads.filter((read) => {
-        const base = byAction.get(actionOfShape(read.tool, read.harnesses));
-        // A fit read always counts its calls (`calls` is null only for entity reads), and its own calls are in the tally, so `base` is there; the fallbacks keep the types honest.
-        return base !== undefined && read.calls !== null && read.count * base.calls >= lift * base.failed * read.calls;
-    });
+    const fold = actionFold();
+    fold.add(store);
+    return aboveActionRate(fold.finish(), reads, lift);
 };
 /** The snapshot a store answers from: its latest deposit. A deposit time compares with a recall's `at`, which is what "older than the recall" needs. */
 export const poolVersionOf = (store) => store.sessions.reduce((latest, session) => (session.at > latest ? session.at : latest), '');

package/dist/sessions/reversion.d.ts

@@ -1,8 +1,47 @@
 import type { Store } from './store.js';
 /** Where the nearest later reversion ran: in the prompt that did the work, in a later prompt of that session, or in a later session of the same repository. */
 export type Reversion = 'same-prompt' | 'later-prompt' | 'later-session';
+/** Where a call sits: its session, its position there, the position of a pair inside it (a MultiEdit holds several edits in order), its time, and its prompt. */
+interface Place {
+    session: string;
+    n: number;
+    k: number;
+    ts: string | null;
+    prompt: number;
+}
+interface Placed extends Place {
+    sha: string;
+}
+/** One digest pair of an edit call on a path: `old` is null for a Write, which does not show the text it replaced. */
+interface Written extends Place {
+    old: string | null;
+    now: string;
+}
+/** A commit call of a session: its position there and its time. */
+interface Commit {
+    n: number;
+    ts: string | null;
+}
+/** What the calls of the runs folded so far hold: every main-line call's commits, revert words, restored paths, and edit digests, and each session's repository. */
+export interface ReversionIndex {
+    repos: Map<string, string | null>;
+    commits: Placed[];
+    reverts: Placed[];
+    committed: Map<string, Commit[]>;
+    edited: Map<string, Place[]>;
+    written: Map<string, Written[]>;
+    restored: Array<Place & {
+        target: string;
+    }>;
+}
+export declare const newReversionIndex: () => ReversionIndex;
 /**
- * Reads every main-line call once: the clean shell calls' commits, revert words (read segments only, since a segment whose result the call's status does not report (`read` false) could have failed behind another command), and
- * restored paths; and every edit that did not fail, with its digests. A subagent's call is not on the main line, so it is not read. Returns the nearest reversion of a prompt's work, or null for none found.
+ * Reads every main-line call of one run once: the clean shell calls' commits, revert words (read segments only, since a segment whose result the call's status does not report (`read` false) could have failed behind another command), and
+ * restored paths; and every edit that did not fail, with its digests. A subagent's call is not on the main line, so it is not read. Give it the run's own sessions only: a context parent passed here is folded again in its own run. Runs fold in the whole store's order, and the matching waits for `finishReversions`.
  */
+export declare const foldReversions: (index: ReversionIndex, store: Store) => void;
+/** Matches the folded calls once. Returns the nearest reversion of a prompt's work, or null for none found. */
+export declare const finishReversions: (index: ReversionIndex) => ((session: string, index: number | null) => Reversion | null);
+/** Every main-line call of the store matched at once: the fold of one run, then the finish. */
 export declare const reversionsOf: (store: Store) => ((session: string, index: number | null) => Reversion | null);
+export {};

package/dist/sessions/reversion.js

@@ -41,18 +41,15 @@ const keptBy = (committed, edit, restore) => {
         return before.some((commit) => commit.n > edit.n && commit.n <= restore.n);
     return before.some((commit) => commit.n > edit.n) || (committed.get(restore.session) ?? []).some((commit) => commit.n <= restore.n && !(commit.ts !== null && edit.ts !== null && commit.ts < edit.ts));
 };
+export const newReversionIndex = () => ({ repos: new Map(), commits: [], reverts: [], committed: new Map(), edited: new Map(), written: new Map(), restored: [] });
 /**
- * Reads every main-line call once: the clean shell calls' commits, revert words (read segments only, since a segment whose result the call's status does not report (`read` false) could have failed behind another command), and
- * restored paths; and every edit that did not fail, with its digests. A subagent's call is not on the main line, so it is not read. Returns the nearest reversion of a prompt's work, or null for none found.
+ * Reads every main-line call of one run once: the clean shell calls' commits, revert words (read segments only, since a segment whose result the call's status does not report (`read` false) could have failed behind another command), and
+ * restored paths; and every edit that did not fail, with its digests. A subagent's call is not on the main line, so it is not read. Give it the run's own sessions only: a context parent passed here is folded again in its own run. Runs fold in the whole store's order, and the matching waits for `finishReversions`.
  */
-export const reversionsOf = (store) => {
-    const repos = new Map(store.sessions.map((session) => [session.session, session.repo]));
-    const commits = [];
-    const reverts = [];
-    const committed = new Map();
-    const edited = new Map();
-    const written = new Map();
-    const restored = [];
+export const foldReversions = (index, store) => {
+    const { repos, commits, reverts, committed, edited, written, restored } = index;
+    for (const session of store.sessions)
+        repos.set(session.session, session.repo);
     for (const tool of store.tools) {
         if (tool.side || tool.prompt === null)
             continue;
@@ -75,6 +72,10 @@ export const reversionsOf = (store) => {
                 for (const word of revertWords(segment.run))
                     reverts.push({ ...at, sha: word });
     }
+};
+/** Matches the folded calls once. Returns the nearest reversion of a prompt's work, or null for none found. */
+export const finishReversions = (index) => {
+    const { repos, commits, reverts, committed, edited, written, restored } = index;
     const found = new Map();
     const note = (earlier, later) => {
         const where = whereOf(earlier, later, repos);
@@ -112,3 +113,9 @@ export const reversionsOf = (store) => {
     }
     return (session, index) => found.get(`${session}#${String(index)}`) ?? null;
 };
+/** Every main-line call of the store matched at once: the fold of one run, then the finish. */
+export const reversionsOf = (store) => {
+    const index = newReversionIndex();
+    foldReversions(index, store);
+    return finishReversions(index);
+};

package/dist/sessions/store.d.ts

@@ -34,6 +34,17 @@ export interface SessionsRuntime {
     stderr?: {
         isTTY?: boolean | undefined;
     } | undefined;
+    /** A seam for tests: awaited right after a build's scan of the log and before any session is loaded, so a test can change the log in exactly that window. */
+    afterScan?: () => Promise<void>;
+    /** How many sessions (with their subagents) `refresh` loads and folds at a time; absent means `REFRESH_RUN`. */
+    refreshRun?: number;
+    /** A seam for tests and proofs: told once what a refresh folded, after its last run. `reversions` is the size of the matching's index. */
+    refreshed?: (folded: {
+        runs: number;
+        reversions: number;
+    }) => void;
+    /** A seam for proofs: told as `refresh` passes the end of its scan, its last run, the pool fold's finish, and the pool step, so a proof can read the heap at each. */
+    refreshStep?: (step: 'scan' | 'runs' | 'finish' | 'pool') => void;
 }
 interface FileState {
     size: number;
@@ -74,6 +85,8 @@ export interface PoolChoices {
     folders: {
         [entry: string]: FolderSend;
     };
+    /** The answer to the join's region question (in the EU, the EEA, or the UK), kept so a run stopped between that question and the covenant screen does not ask it again. Null until asked. */
+    euEeaUk: 'yes' | 'no' | null;
 }
 export type FolderSend = 'skeleton' | 'off';
 export declare const paths: (storeDir: string) => {
@@ -304,19 +317,52 @@ export interface Store {
     newerCaptures: number;
 }
 /** What one pass over the log keeps: a number or two per session, never the records. */
-interface LogScan {
+export interface LogScan {
     /** Each session's highest rev, over every kind of record. */
     latest: Map<string, number>;
     /** True when some session has a record below its highest rev, which a compaction would drop. */
     superseded: boolean;
     /** Sessions whose newest capture record was written by a newer oars, counted as `loadStore` counts them. */
     newerCaptures: number;
+    /** Each subagent's parent, from its session record at its highest rev (a main session has none). */
+    parents: Map<string, string>;
+    /** Each session's `cv` at its highest rev. */
+    cvs: Map<string, string>;
+    /** Set when asked for: each session's record lines at its highest rev as flat `[start, length, ...]` byte pairs (the newline included). Only lines this scan saw are in it, so a row appended after the scan is never read through it. */
+    spans?: Map<string, number[]>;
+    /** With `spans`: the file the scan read (device, inode, and size from `stat` on the handle it read), so a load uses the spans only on that same file. */
+    identity?: LogIdentity;
 }
-export declare const scanLog: (file: string, held?: FileHandle) => Promise<LogScan>;
+/** Which file a scan read: a compaction renames a new file into place, which changes the inode; an append only grows the size. */
+interface LogIdentity {
+    dev: bigint;
+    ino: bigint;
+    size: bigint;
+}
+export declare const scanLog: (file: string, held?: FileHandle, withSpans?: boolean) => Promise<LogScan>;
+/**
+ * The order `loadStore` visits sessions in, as one list: the sessions whose capture version cannot be read (version 2, whose prompts and errors the log holds as rows) first, then the rest, each group by where
+ * the session's first line at its highest rev starts in the log, subagents and mains alike. Needs a scan made with spans. A run is a contiguous slice of this list.
+ */
+export declare const runOrder: (scan: LogScan) => string[];
+/** True when the line carries a `"session":"<id>"` field of one of `ids`. The writer emits compact JSON, so the field of the line's own session is always found; a nested match only keeps the line for the parse to decide. */
+export declare const namesAny: (line: string, ids: ReadonlySet<string>) => boolean;
 /** What a read keeps beyond the default. */
 export type LoadOptions = {
     /** Keep each session's `messageUsage` entries. Nothing in the store reads them yet, and on a large store they are a third more heap, so a read holds them only when asked. */
     messageUsage?: boolean;
+    /** Read only these sessions' rows (main and subagent ids): a build that handles a chunk at a time holds that chunk, not the history. */
+    only?: ReadonlySet<string>;
+    /** Keep the session records alone, with no prompt, tool, or error rows: enough to gate and place a session without holding its content. */
+    sessionsOnly?: boolean;
+    /** The revs of an earlier `scanLog`: the read skips its own first pass. A build that reads chunk after chunk scans once; a row written after that scan is not seen, like a session that began after the build did. */
+    latest?: ReadonlyMap<string, number>;
+    /** With `only`: the byte spans of an earlier `scanLog(file, held, true)`. The read seeks to the lines of the wanted sessions instead of streaming the whole log; a log that is no longer the scanned file is streamed instead, and a torn or short line is skipped, never misread. */
+    spans?: ReadonlyMap<string, readonly number[]>;
+    /** With `spans`: the scan's `identity`. The spans are used only when the log opened for this read is that file, grown or not; any other file (a compaction renamed one into place) is streamed, as without spans. */
+    identity?: LogIdentity;
+    /** A seam: called once for every log line the read looks at. */
+    lineRead?: () => void;
 };
 /**
  * The store as a reader sees it: for each session only the highest `rev`. A version-3 or later session is derived here from its capture records
@@ -343,6 +389,13 @@ export declare const readSkeletons: (storeDir: string, only?: string, lean?: boo
     skeletons: Skeleton[];
     unreadable: number;
 }>;
+/** `derive`'s prompt sums, folded run by run: the prompts of each run in turn give the same sums, in the same key order, as the prompts of the whole store. */
+export interface DeriveFold {
+    add: (prompts: readonly PromptObservation[]) => void;
+    /** The error and session parts, which need every error and session record at once, then the whole record. */
+    finish: (store: Pick<Store, 'sessions' | 'errors'>, at: string) => DerivedRecords;
+}
+export declare const deriveFold: () => DeriveFold;
 /** The derived records, a pure function of the store. */
 export declare const derive: (store: Store, at: string) => DerivedRecords;
 export declare const signatureRecord: (sig: string, list: readonly ErrorObservation[]) => SignatureRecord;