@manyoars/oars 0.5.4 → 0.5.5

package/dist/sessions/store.js

@@ -85,14 +85,14 @@ export const readState = async (storeDir) => {
                         folders[entry] = mode;
             const yesVersion = typeof pool.rawConsentVersion === 'string' && pool.rawConsentVersion !== '' ? pool.rawConsentVersion : null;
             const granted = pool.rawConsent === 'granted' && yesVersion !== null;
-            return { files, redactDomains: domains, allow, pool: { rawConsent: granted ? 'granted' : pool.rawConsent === 'declined' ? 'declined' : null, rawConsentVersion: granted ? yesVersion : null, withdrawalPending: pool.withdrawalPending === true && !granted, folders } };
+            return { files, redactDomains: domains, allow, pool: { rawConsent: granted ? 'granted' : pool.rawConsent === 'declined' ? 'declined' : null, rawConsentVersion: granted ? yesVersion : null, withdrawalPending: pool.withdrawalPending === true && !granted, folders, euEeaUk: pool.euEeaUk === 'yes' || pool.euEeaUk === 'no' ? pool.euEeaUk : null } };
         }
     }
     catch (error) {
         if (!isMissing(error) && !(error instanceof SyntaxError))
             throw error;
     }
-    return { files: {}, redactDomains: [], allow: [], pool: { rawConsent: null, rawConsentVersion: null, withdrawalPending: false, folders: {} } };
+    return { files: {}, redactDomains: [], allow: [], pool: { rawConsent: null, rawConsentVersion: null, withdrawalPending: false, folders: {}, euEeaUk: null } };
 };
 /** Changes the pool choices under the store lock, so a deposit running at the same time keeps them (it writes the state it read). */
 export const updatePoolChoices = (storeDir, change) => withLock(storeDir, async (held) => {
@@ -372,6 +372,115 @@ const tally = (trace, extract, subagent) => {
     add('error-line', extract.errorLines.length);
 };
 export const digest = (parts) => createHash('sha256').update(parts.join('\n')).digest('hex').slice(0, 16);
+/** Sessions a refresh loads and folds at a time. Peak memory is the sessions of a run plus what the folds keep (spec 3 option A), so this is the knob between the heap and the number of seeks. */
+const REFRESH_RUN = 100;
+const asError = (error) => (error instanceof Error ? error : new Error(String(error)));
+/** The one place a pool fold's failure is caught: its result, or the error, so a step of the pool never stops a deposit. */
+const attempt = (step) => {
+    try {
+        return step();
+    }
+    catch (error) {
+        return asError(error);
+    }
+};
+/** The pool fold settled after the last run, or the error that stopped it. */
+const finished = (pool, runtime, runs) => {
+    if (pool instanceof Error)
+        return pool;
+    return attempt(() => {
+        const folds = pool.finish();
+        runtime.refreshStep?.('finish');
+        runtime.refreshed?.({ runs, reversions: reversionEntries(pool.index) });
+        return folds;
+    });
+};
+const reversionEntries = (index) => {
+    const lists = [...index.committed.values(), ...index.edited.values(), ...index.written.values()];
+    return index.commits.length + index.reverts.length + index.restored.length + lists.reduce((sum, list) => sum + list.length, 0);
+};
+/**
+ * `loadStore` and the folds over the whole store, one run of sessions at a time (t1243): the log is scanned once with spans, then each run of `runOrder` is read by seeking, folded, and let go. A subagent's parent
+ * outside its run is read as context for the agent's errors and folded in its own run only. Sessions and errors are held with their place in the whole read and put back in that order, which is the only order anything
+ * written depends on.
+ */
+const foldRuns = async (runtime) => {
+    const scan = await scanLog(paths(runtime.storeDir).observations, undefined, true);
+    await runtime.afterScan?.();
+    runtime.refreshStep?.('scan');
+    const order = runOrder(scan);
+    const size = Math.max(1, Math.floor(runtime.refreshRun ?? REFRESH_RUN));
+    // Loaded here, not at the top: the pool imports every read family, and a lookup hook never reaches this step. A failure to load or fold costs the index only its fit reads' read IDs (the shapes come from the always-on read fold), as a failure of `currentPool` does.
+    let pool;
+    try {
+        const { poolFold } = await import('./pool-fold.js');
+        const { readRecalls } = await import('./recall-object.js');
+        pool = poolFold(await readRecalls(runtime.storeDir));
+    }
+    catch (error) {
+        pool = asError(error);
+    }
+    const derive = deriveFold();
+    // Also loaded here: it reaches the read families a lookup hook must not load.
+    const readFolding = (await import('./read-folds.js')).readFold();
+    const counts = { prompts: 0, tools: 0 };
+    const start = (session) => scan.spans?.get(session)?.[0] ?? Number.MAX_SAFE_INTEGER;
+    const sessions = [];
+    const subagents = [];
+    // Version 2 error rows come first in a whole read, then each readable session's in its place, then the agents' last.
+    const errors = [];
+    let runs = 0;
+    for (let from = 0; from < order.length; from += size) {
+        const own = new Set(order.slice(from, from + size));
+        const wanted = new Set(own);
+        for (const id of own) {
+            const parent = scan.parents.get(id);
+            if (parent !== undefined)
+                wanted.add(parent);
+        }
+        const loaded = await loadStore(runtime.storeDir, undefined, { only: wanted, latest: scan.latest, spans: scan.spans, identity: scan.identity });
+        const run = {
+            sessions: loaded.sessions.filter(({ session }) => own.has(session)),
+            subagents: loaded.subagents.filter(({ session }) => own.has(session)),
+            prompts: loaded.prompts.filter(({ session }) => own.has(session)),
+            tools: loaded.tools.filter(({ session }) => own.has(session)),
+            errors: loaded.errors.filter(({ session, agent }) => own.has(agent ?? session)),
+            newerCaptures: 0,
+        };
+        runs += 1;
+        derive.add(run.prompts);
+        readFolding.add(run);
+        counts.prompts += run.prompts.length;
+        counts.tools += run.tools.length;
+        for (const record of run.sessions)
+            sessions.push({ at: start(record.session), record });
+        for (const record of run.subagents)
+            subagents.push({ at: start(record.session), record });
+        for (const record of run.errors) {
+            const unreadable = !isReadable(scan.cvs.get(record.session) ?? '0');
+            errors.push(record.agent === undefined ? { group: unreadable ? 0 : 1, at: start(record.session), record } : { group: 2, at: start(record.agent), record });
+        }
+        if (!(pool instanceof Error)) {
+            const open = pool;
+            const failed = attempt(() => {
+                open.add(run);
+            });
+            if (failed instanceof Error)
+                pool = failed;
+        }
+    }
+    runtime.refreshStep?.('runs');
+    const byPlace = (list) => list.sort((a, b) => a.at - b.at);
+    const store = {
+        sessions: byPlace(sessions).map(({ record }) => record),
+        subagents: byPlace(subagents).map(({ record }) => record),
+        errors: errors.sort((a, b) => a.group - b.group || a.at - b.at).map(({ record }) => record),
+        prompts: [],
+        tools: [],
+        newerCaptures: scan.newerCaptures,
+    };
+    return { store, folds: finished(pool, runtime, runs), derive, reads: readFolding.finish(), counts };
+};
 /**
  * Rewrites `derived.json` (the derive step), then the hook index (the index step), from the whole store. With a trace, a failure of
  * either is recorded on it instead of thrown, so the caller can log the deposit that already wrote its records, then the failed step
@@ -382,18 +491,21 @@ export const refresh = async (runtime, redactDomains, at, trace, held = () => Pr
     // The extractor version alone until the store is read: a derive that fails before the high-water mark is known still has a key (spec 4.4).
     let key = digest([EXTRACTOR_VERSION]);
     let store;
+    let folds;
+    let readFolds;
     try {
-        store = await loadStore(runtime.storeDir);
+        const folded = await foldRuns(runtime);
+        ({ store, folds, reads: readFolds } = folded);
         // The key is the account's capture high-water mark (the highest rev of each session) and `ev` (spec 4.2); the index shares it.
         const highWater = [...store.sessions, ...store.subagents].map((session) => `${session.session}:${String(session.rev)}`).sort();
         key = digest([EXTRACTOR_VERSION, ...highWater]);
-        const derived = derive(store, at);
+        const derived = folded.derive.finish(store, at);
         await requireHold(held);
         await writeAtomic(paths(runtime.storeDir).derived, JSON.stringify(derived));
         if (trace !== undefined) {
             trace.derive = {
                 ms: Math.round(performance.now() - started),
-                in: { session: store.sessions.length, subagent: store.subagents.length, 'user-prompt': store.prompts.length, tool: store.tools.length },
+                in: { session: store.sessions.length, subagent: store.subagents.length, 'user-prompt': folded.counts.prompts, tool: folded.counts.tools },
                 out: { signature: derived.signatures.length, summary: derived.summaries.length },
                 key,
             };
@@ -407,17 +519,18 @@ export const refresh = async (runtime, redactDomains, at, trace, held = () => Pr
     }
     // The pool is derived for the index's fit reads only: one that cannot be read or written costs the index those, never the index itself (`currentPool` says so on stderr). Loaded here, not at the top: the pool imports every read family, and a lookup hook never reaches this step. It runs before the clock starts, so the index's `ms` is the index and not the derive (the first pool run takes 12 to 17 s on a store copy).
     // A module that will not load costs the index the same fit reads, with one line on stderr, as `currentPool` does when the pool cannot be built.
-    const current = await import('./pool.js').then(async ({ currentPool }) => currentPool(runtime.storeDir, store), (error) => {
+    const current = await import('./pool.js').then(async ({ currentPool }) => currentPool(runtime.storeDir, store, folds), (error) => {
         process.stderr.write(`oars: the read pool module could not be loaded, and the index keeps no fit reads: ${error instanceof Error ? error.message : String(error)}
 `);
         return undefined;
     });
+    runtime.refreshStep?.('pool');
     const reads = current?.reads ?? [];
     const indexStarted = performance.now();
     try {
         // The next-node reads for each harness and version, derived over the same pool; a deposit that has no pool leaves the index without them.
         const next = current === undefined ? undefined : (await import('./recall-frames.js')).nextByFrame(current.pool, store);
-        const index = buildIndex(store, redactDomains, at, reads, next);
+        const index = buildIndex(store, redactDomains, at, reads, next, readFolds);
         await requireHold(held);
         await writeAtomic(indexPath(runtime.storeDir), JSON.stringify(index));
         if (trace !== undefined) {
@@ -1085,26 +1198,103 @@ const depositSessionLocked = async (runtime, transcript, now, held, trace) => {
     await removePending(runtime.storeDir, drained.settled);
     return own.result;
 };
-export const scanLog = async (file, held) => {
+/** The log's lines with the byte range each takes (its newline included), read as bytes so an offset is a file position. The same split as `logLines`: on 0x0a alone. */
+const logSpans = async function* (file, held) {
+    let rest = Buffer.alloc(0);
+    let at = 0;
+    try {
+        const stream = held === undefined ? createReadStream(file, { highWaterMark: 1 << 20 }) : held.createReadStream({ highWaterMark: 1 << 20, start: 0, autoClose: false });
+        const chunks = stream;
+        for await (const chunk of chunks) {
+            let buffer = Buffer.concat([rest, chunk]);
+            let from = 0;
+            for (let end = buffer.indexOf(10, from); end !== -1; end = buffer.indexOf(10, from)) {
+                yield { line: buffer.toString('utf8', from, end), start: at + from, length: end + 1 - from };
+                from = end + 1;
+            }
+            buffer = buffer.subarray(from);
+            at += from;
+            rest = buffer;
+        }
+    }
+    catch (error) {
+        if (!isMissing(error))
+            throw error;
+    }
+    yield { line: rest.toString('utf8'), start: at, length: rest.length };
+};
+export const scanLog = async (file, held, withSpans = false) => {
+    // Spans are byte offsets into one file: read it through a handle, and stat that handle, so the identity is the file the offsets belong to.
+    const own = withSpans && held === undefined ? await openLog(file) : undefined;
+    try {
+        return await scanHandle(file, held ?? own, withSpans);
+    }
+    finally {
+        await own?.close();
+    }
+};
+const scanHandle = async (file, held, withSpans) => {
     const latest = new Map();
     const cvAt = new Map();
+    // Every rev's lines are noted as `[start, length, rev, ...]` triples, then thinned to the highest rev once it is known.
+    const seen = withSpans ? new Map() : undefined;
+    const info = withSpans && held !== undefined ? await held.stat({ bigint: true }) : undefined;
     let superseded = false;
-    for await (const line of logLines(file, held)) {
+    for await (const { line, start, length } of withSpans ? logSpans(file, held) : plainLines(file, held)) {
         const observation = observationOf(line);
         if (observation === undefined)
             continue;
-        const seen = latest.get(observation.session);
-        superseded ||= seen !== undefined && seen !== observation.rev;
-        latest.set(observation.session, Math.max(seen ?? 0, observation.rev));
+        if (seen !== undefined) {
+            const own = seen.get(observation.session) ?? [];
+            own.push(start, length, observation.rev);
+            seen.set(observation.session, own);
+        }
+        const before = latest.get(observation.session);
+        superseded ||= before !== undefined && before !== observation.rev;
+        latest.set(observation.session, Math.max(before ?? 0, observation.rev));
         // The last session record at the highest rev is the one `loadStore` reads.
         if (observation.kind === 'session' && observation.rev >= (cvAt.get(observation.session)?.rev ?? 0))
-            cvAt.set(observation.session, { rev: observation.rev, cv: observation.cv });
+            cvAt.set(observation.session, { rev: observation.rev, cv: observation.cv, parent: observation.parent });
     }
     let newerCaptures = 0;
-    for (const [session, held] of cvAt)
-        if (held.rev === latest.get(session) && Number(held.cv) > Number(CAPTURE_VERSION))
+    const parents = new Map();
+    const cvs = new Map();
+    for (const [session, held] of cvAt) {
+        if (held.rev !== latest.get(session))
+            continue;
+        cvs.set(session, held.cv);
+        if (held.parent !== null)
+            parents.set(session, held.parent);
+        if (Number(held.cv) > Number(CAPTURE_VERSION))
             newerCaptures += 1;
-    return { latest, superseded, newerCaptures };
+    }
+    if (seen === undefined)
+        return { latest, superseded, newerCaptures, parents, cvs };
+    // A superseded line is never read by a chunk load, so the index keeps only each session's lines at its highest rev.
+    const spans = new Map();
+    for (const [session, triples] of seen) {
+        const kept = [];
+        for (let n = 0; n < triples.length; n += 3)
+            if (triples[n + 2] === latest.get(session))
+                kept.push(triples[n] ?? 0, triples[n + 1] ?? 0);
+        spans.set(session, kept);
+    }
+    return info === undefined ? { latest, superseded, newerCaptures, parents, cvs, spans } : { latest, superseded, newerCaptures, parents, cvs, spans, identity: { dev: info.dev, ino: info.ino, size: info.size } };
+};
+/**
+ * The order `loadStore` visits sessions in, as one list: the sessions whose capture version cannot be read (version 2, whose prompts and errors the log holds as rows) first, then the rest, each group by where
+ * the session's first line at its highest rev starts in the log, subagents and mains alike. Needs a scan made with spans. A run is a contiguous slice of this list.
+ */
+export const runOrder = (scan) => {
+    // Every key of `spans` holds at least one pair, so the fallback is never taken; it only satisfies the indexed read's type.
+    const start = (session) => scan.spans?.get(session)?.[0] ?? Number.MAX_SAFE_INTEGER;
+    const sessions = [...(scan.spans?.keys() ?? [])];
+    const unreadable = (session) => !isReadable(scan.cvs.get(session) ?? '0');
+    return sessions.sort((a, b) => Number(unreadable(b)) - Number(unreadable(a)) || start(a) - start(b));
+};
+const plainLines = async function* (file, held) {
+    for await (const line of logLines(file, held))
+        yield { line, start: 0, length: 0 };
 };
 /** Returns the first copy it was given of each string, so the strings a log repeats are held once. */
 const interner = () => {
@@ -1134,6 +1324,47 @@ const internTool = (tool, intern) => ({
     target: tool.target === null ? null : intern(tool.target),
     ...(tool.segments === undefined ? {} : { segments: tool.segments.map((segment) => ({ ...segment, run: intern(segment.run) })) }),
 });
+const SESSION_FIELD = /"session":"((?:[^"\\]|\\.)*)"/g;
+/** True when the line carries a `"session":"<id>"` field of one of `ids`. The writer emits compact JSON, so the field of the line's own session is always found; a nested match only keeps the line for the parse to decide. */
+export const namesAny = (line, ids) => {
+    for (const match of line.matchAll(SESSION_FIELD)) {
+        const value = match[1] ?? '';
+        // An id that JSON escaped (a quote or a backslash in it) is not the raw id: keep the line and let the parse decide.
+        if (value.includes('\\') || ids.has(value))
+            return true;
+    }
+    return false;
+};
+/** True when `held` is the file `identity` describes and has not shrunk: only an append can have happened since. */
+const sameFile = async (held, identity) => {
+    const now = await held.stat({ bigint: true });
+    return now.dev === identity.dev && now.ino === identity.ino && now.size >= identity.size;
+};
+/** The wanted sessions' lines, in file order, read by seeking: adjacent spans are one read. */
+const spanLines = async function* (held, spans, ids) {
+    const wanted = [];
+    for (const id of ids) {
+        const own = spans.get(id) ?? [];
+        for (let n = 0; n < own.length; n += 2)
+            wanted.push({ start: own[n] ?? 0, length: own[n + 1] ?? 0 });
+    }
+    wanted.sort((a, b) => a.start - b.start);
+    for (let n = 0; n < wanted.length;) {
+        const first = wanted[n] ?? { start: 0, length: 0 };
+        let end = first.start + first.length;
+        n += 1;
+        for (let next = wanted[n]; next !== undefined && next.start === end; next = wanted[n]) {
+            end += next.length;
+            n += 1;
+        }
+        const buffer = Buffer.alloc(end - first.start);
+        const { bytesRead } = await held.read(buffer, 0, buffer.length, first.start);
+        // A short read (the log shrank) yields what is there; a torn last line fails its parse and is skipped.
+        for (const line of buffer.toString('utf8', 0, bytesRead).split('\n'))
+            if (line !== '')
+                yield line;
+    }
+};
 const withoutMessageUsage = (session) => {
     if (session.messageUsage === undefined)
         return session;
@@ -1163,12 +1394,22 @@ export const loadStore = async (storeDir, afterFirstPass, options = {}) => {
     };
     const held = await openLog(file);
     try {
-        const { latest } = await scanLog(file, held);
+        const latest = options.latest ?? (await scanLog(file, held)).latest;
         await afterFirstPass?.();
-        for await (const line of logLines(file, held)) {
+        const { only, spans, identity } = options;
+        const seek = held !== undefined && only !== undefined && spans !== undefined && identity !== undefined && (await sameFile(held, identity));
+        for await (const line of seek ? spanLines(held, spans, only) : logLines(file, held)) {
+            options.lineRead?.();
+            // A read of a few sessions skips the JSON parse of every other line: the cheap check can only keep a line, never drop one that names a wanted session.
+            if (options.only !== undefined && !namesAny(line, options.only))
+                continue;
             const observation = observationOf(line);
             if (observation === undefined || observation.rev !== latest.get(observation.session))
                 continue;
+            if (options.only !== undefined && !options.only.has(observation.session))
+                continue;
+            if (options.sessionsOnly === true && observation.kind !== 'session')
+                continue;
             if (observation.kind === 'session')
                 of(observation.session).session = options.messageUsage === true ? observation : withoutMessageUsage(observation);
             else if (observation.kind === 'user-prompt')
@@ -1201,12 +1442,12 @@ export const loadStore = async (storeDir, afterFirstPass, options = {}) => {
         store.tools.push(...tools);
         if (session.parent !== null) {
             store.subagents.push(session);
-            if (isReadable(session.cv))
+            if (isReadable(session.cv) && options.sessionsOnly !== true)
                 agents.push({ session, prompts, tools, errorLines, parent: session.parent });
             continue;
         }
         store.sessions.push(session);
-        if (!isReadable(session.cv))
+        if (!isReadable(session.cv) || options.sessionsOnly === true)
             continue;
         const derived = deriveSession({ session, prompts, tools, errorLines }, redact);
         store.prompts.push(...derived.prompts);
@@ -1481,52 +1722,62 @@ export const readSkeletons = async (storeDir, only, lean = false) => {
     }
     return { skeletons, unreadable };
 };
-/** The derived records, a pure function of the store. */
-export const derive = (store, at) => {
-    const bySig = new Map();
-    for (const error of store.errors)
-        bySig.set(error.sig, [...(bySig.get(error.sig) ?? []), error]);
-    const signatures = [...bySig.entries()].map(([sig, list]) => signatureRecord(sig, list));
-    signatures.sort((a, b) => b.sessions - a.sessions || b.count - a.count || a.sig.localeCompare(b.sig));
+export const deriveFold = () => {
     const bigrams = {};
     const recovery = {};
     const classes = {};
     const ratio = new Map();
-    for (const prompt of store.prompts) {
-        for (const [pair, count] of Object.entries(prompt.bigrams))
-            bigrams[pair] = (bigrams[pair] ?? 0) + count;
-        if (prompt.recovery.length > 0) {
-            const pair = prompt.recovery.slice(0, 2).join('>');
-            recovery[pair] = (recovery[pair] ?? 0) + 1;
-        }
-        const entry = ratio.get(prompt.cls) ?? { explore: 0, act: 0 };
-        entry.explore += prompt.explore;
-        entry.act += prompt.act;
-        ratio.set(prompt.cls, entry);
-        classes[prompt.cls] = { prompts: (classes[prompt.cls]?.prompts ?? 0) + 1, exploreToAct: 0 };
-    }
-    for (const [cls, entry] of ratio) {
-        const row = classes[cls];
-        if (row !== undefined)
-            row.exploreToAct = Math.round((entry.explore / Math.max(entry.act, 1)) * 100) / 100;
-    }
     const promptCount = new Map();
-    for (const prompt of store.prompts)
-        promptCount.set(prompt.session, (promptCount.get(prompt.session) ?? 0) + 1);
-    const errorCount = new Map();
-    for (const error of store.errors)
-        errorCount.set(error.session, (errorCount.get(error.session) ?? 0) + 1);
-    const summaries = store.sessions.map((session) => ({
-        session: session.session,
-        project: session.project,
-        prompts: promptCount.get(session.session) ?? 0,
-        errors: errorCount.get(session.session) ?? 0,
-        tools: Object.values(session.tools).reduce((sum, count) => sum + count, 0),
-        tsFirst: session.tsFirst,
-        tsLast: session.tsLast,
-        tldrs: session.tldrs.length,
-    }));
-    return { v: EXTRACTOR_VERSION, at, sessions: store.sessions.length, signatures, bigrams, recovery, classes, summaries };
+    return {
+        add: (prompts) => {
+            for (const prompt of prompts) {
+                for (const [pair, count] of Object.entries(prompt.bigrams))
+                    bigrams[pair] = (bigrams[pair] ?? 0) + count;
+                if (prompt.recovery.length > 0) {
+                    const pair = prompt.recovery.slice(0, 2).join('>');
+                    recovery[pair] = (recovery[pair] ?? 0) + 1;
+                }
+                const entry = ratio.get(prompt.cls) ?? { explore: 0, act: 0 };
+                entry.explore += prompt.explore;
+                entry.act += prompt.act;
+                ratio.set(prompt.cls, entry);
+                classes[prompt.cls] = { prompts: (classes[prompt.cls]?.prompts ?? 0) + 1, exploreToAct: 0 };
+                promptCount.set(prompt.session, (promptCount.get(prompt.session) ?? 0) + 1);
+            }
+        },
+        finish: (store, at) => {
+            const bySig = new Map();
+            for (const error of store.errors)
+                bySig.set(error.sig, [...(bySig.get(error.sig) ?? []), error]);
+            const signatures = [...bySig.entries()].map(([sig, list]) => signatureRecord(sig, list));
+            signatures.sort((a, b) => b.sessions - a.sessions || b.count - a.count || a.sig.localeCompare(b.sig));
+            for (const [cls, entry] of ratio) {
+                const row = classes[cls];
+                if (row !== undefined)
+                    row.exploreToAct = Math.round((entry.explore / Math.max(entry.act, 1)) * 100) / 100;
+            }
+            const errorCount = new Map();
+            for (const error of store.errors)
+                errorCount.set(error.session, (errorCount.get(error.session) ?? 0) + 1);
+            const summaries = store.sessions.map((session) => ({
+                session: session.session,
+                project: session.project,
+                prompts: promptCount.get(session.session) ?? 0,
+                errors: errorCount.get(session.session) ?? 0,
+                tools: Object.values(session.tools).reduce((sum, count) => sum + count, 0),
+                tsFirst: session.tsFirst,
+                tsLast: session.tsLast,
+                tldrs: session.tldrs.length,
+            }));
+            return { v: EXTRACTOR_VERSION, at, sessions: store.sessions.length, signatures, bigrams, recovery, classes, summaries };
+        },
+    };
+};
+/** The derived records, a pure function of the store. */
+export const derive = (store, at) => {
+    const fold = deriveFold();
+    fold.add(store.prompts);
+    return fold.finish(store, at);
 };
 export const signatureRecord = (sig, list) => {
     const stamps = list.map((error) => error.ts).filter((ts) => ts !== null).sort();

package/dist/telemetry/measures.d.ts

@@ -27,6 +27,11 @@ export declare const firstRunMeasures: (event: unknown) => Measure[];
  * for the operator's query, not a measure of the program. A version that is not a word reads `other`, and a missing one `unrecorded`; the show is still counted.
  */
 export declare const covenantShownMeasures: (event: unknown) => Measure[];
+/**
+ * The cell of a `join-step` event: one yes under the oars version, in a metric named for the step and what it did (`join-step.send.done`), so the pool's rollup holds, for each install and day, which steps of the join it reached.
+ * Like the first-run and covenant-shown cells it is for the operator's query (`queries/join-funnel.sql`), so `measuresOf` leaves it out and the fleet read skips it. A word off the closed lists reads as `other`.
+ */
+export declare const joinStepMeasures: (event: unknown) => Measure[];
 export declare const isRecordOnlyMetric: (metric: string) => boolean;
 /** What one event measures; an event of a type nothing measures, or that is not an object, measures nothing. */
 export declare const measuresOf: (event: unknown) => Measure[];

package/dist/telemetry/measures.js

@@ -2,7 +2,7 @@
 // A measure is one observation in a cell: a latency in milliseconds, or a yes/no that rates are made from. The cell is (metric, dimension, value): `hook.pre-tool.latency` at harness
 // `claude-code`, version `2.1.5`. The reader is tolerant on purpose: the pool reads batches kept whole from any client, so a value that does not fit is skipped, never trusted.
 import { isHarness } from '../sessions/pool-wire.js';
-import { CI_WORDS, SURFACE_WORDS } from './words.js';
+import { CI_WORDS, JOIN_STATES, JOIN_STEPS, SURFACE_WORDS } from './words.js';
 /** The windows the report shows and the pool serves, in days. */
 export const WINDOWS = [7, 30];
 const DAY_MS = 86_400_000;
@@ -100,8 +100,22 @@ export const covenantShownMeasures = (event) => {
     const version = valueOf(fieldOf(fieldOf(event, 'data'), 'version'));
     return [rate(`covenant-shown.${version}`, 'cli', valueOf(fieldOf(event, 'cli')), true)];
 };
+/**
+ * The cell of a `join-step` event: one yes under the oars version, in a metric named for the step and what it did (`join-step.send.done`), so the pool's rollup holds, for each install and day, which steps of the join it reached.
+ * Like the first-run and covenant-shown cells it is for the operator's query (`queries/join-funnel.sql`), so `measuresOf` leaves it out and the fleet read skips it. A word off the closed lists reads as `other`.
+ */
+export const joinStepMeasures = (event) => {
+    if (fieldOf(event, 'type') !== 'join-step')
+        return [];
+    const data = fieldOf(event, 'data');
+    const step = fieldOf(data, 'step');
+    const state = fieldOf(data, 'state');
+    const stepWord = typeof step === 'string' && JOIN_STEPS.some((word) => word === step) ? step : 'other';
+    const stateWord = typeof state === 'string' && JOIN_STATES.some((word) => word === state) ? state : 'other';
+    return [rate(`join-step.${stepWord}.${stateWord}`, 'cli', valueOf(fieldOf(event, 'cli')), true)];
+};
 /** The metrics the rollup records for the operator's queries and the fleet read never serves: a cell of one event or one count per install measures nothing across installs. */
-const RECORD_ONLY_PREFIXES = ['first-run.', 'covenant-shown.'];
+const RECORD_ONLY_PREFIXES = ['first-run.', 'covenant-shown.', 'join-step.'];
 export const isRecordOnlyMetric = (metric) => RECORD_ONLY_PREFIXES.some((prefix) => metric.startsWith(prefix));
 /** What one event measures; an event of a type nothing measures, or that is not an object, measures nothing. */
 export const measuresOf = (event) => {

package/dist/telemetry/schema.js

@@ -6,7 +6,7 @@ import { ExitCode } from '../errors.js';
 import { FIELD_ROWS } from '../sessions/pool-outbox.js';
 import { OTHER, HARNESSES } from '../sessions/pool-wire.js';
 import { LITERAL, publicModel, publicVersion } from '../sessions/public-vocabulary.js';
-import { CI_WORDS, SURFACE_WORDS } from './words.js';
+import { CI_WORDS, JOIN_STATES, JOIN_STEPS, SURFACE_WORDS } from './words.js';
 /** Bumped when an event type or field changes meaning; every spooled event carries it. */
 export const SCHEMA_VERSION = 1;
 /** A command or option name as the program defines it: letters, digits, and hyphens. No space, slash, dot, colon, or quote. */
@@ -138,6 +138,8 @@ const EVENTS = new Map([
     ['first-run', rules({ ci: oneOf(CI_WORDS), surface: oneOf(SURFACE_WORDS) })],
     // The CLI told the operator which covenant to read and accept: the version and nothing else.
     ['covenant-shown', rules({ version: covenantVersion })],
+    // One step of `oars pool join` in one run: which of the closed steps and what it did, and nothing else (docs/telemetry.md).
+    ['join-step', rules({ step: oneOf(JOIN_STEPS), state: oneOf(JOIN_STATES) })],
 ]);
 export const EVENT_TYPES = [...EVENTS.keys()];
 /** The field names a type admits, so a test can feed every one. */

package/dist/telemetry/words.d.ts

@@ -4,3 +4,9 @@ export type CiWord = (typeof CI_WORDS)[number];
 /** How this copy of oars arrived, from the folder it runs in and the package manager that started it (`surfaceWord` in `fields.ts`); the path itself never reaches the spool. A folder it cannot place is `unknown`. */
 export declare const SURFACE_WORDS: readonly ["npm-global", "npx", "claude-plugin", "codex-plugin", "gemini-extension", "unknown"];
 export type SurfaceWord = (typeof SURFACE_WORDS)[number];
+/** The steps of `oars pool join`, as the `join-step` event and the join plan name them. A name outside this list never reaches the spool. */
+export declare const JOIN_STEPS: readonly ["signin", "region", "terms", "batch", "outbox", "covenant", "send"];
+export type JoinStep = (typeof JOIN_STEPS)[number];
+/** What a step did in one run: finished now, found already done and skipped, stopped the run, or had nothing to act on (no session ready to deposit; never counted as reached). */
+export declare const JOIN_STATES: readonly ["done", "skipped", "stopped", "none"];
+export type JoinState = (typeof JOIN_STATES)[number];

package/dist/telemetry/words.js

@@ -4,3 +4,7 @@
 export const CI_WORDS = ['none', 'github-actions', 'gitlab-ci', 'circleci', 'travis-ci', 'jenkins', 'buildkite', 'azure-pipelines', 'bitbucket', 'teamcity', 'appveyor', 'codebuild', 'other'];
 /** How this copy of oars arrived, from the folder it runs in and the package manager that started it (`surfaceWord` in `fields.ts`); the path itself never reaches the spool. A folder it cannot place is `unknown`. */
 export const SURFACE_WORDS = ['npm-global', 'npx', 'claude-plugin', 'codex-plugin', 'gemini-extension', 'unknown'];
+/** The steps of `oars pool join`, as the `join-step` event and the join plan name them. A name outside this list never reaches the spool. */
+export const JOIN_STEPS = ['signin', 'region', 'terms', 'batch', 'outbox', 'covenant', 'send'];
+/** What a step did in one run: finished now, found already done and skipped, stopped the run, or had nothing to act on (no session ready to deposit; never counted as reached). */
+export const JOIN_STATES = ['done', 'skipped', 'stopped', 'none'];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@manyoars/oars",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "mcpName": "io.github.manyoars/oars",
   "description": "A local, searchable record of your coding agent sessions. Reads Claude Code, Cursor, Codex, Copilot CLI, Gemini CLI, and more, and keeps the record on your machine.",
   "homepage": "https://manyoars.com/",