@mikode13/harness 9.2.0 → 9.3.1

package/dist/recovery/infrastructure/fileRecoveryStore.js

@@ -2,6 +2,8 @@ import { createHash, randomBytes } from 'node:crypto';
 import { link, mkdir, open, readdir, readFile, realpath, rename, rm, stat, truncate, unlink, } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { dirname, isAbsolute, join } from 'node:path';
+import { classifyHostFailure, describeFailure, treatErrors, } from '#src/shared/domain/providerFailure';
+import { Logger } from '#src/shared/infrastructure/logger';
 import { fileChanges, sameState } from "../domain/fileChanges.js";
 import { historyStart, NothingToMoveError, WorkspaceBusyError, WorkspaceMovedError, } from "../domain/recoveryStore.js";
 import { chainEntries, nextPruneStep } from "../domain/retention.js";
@@ -146,15 +148,19 @@ async function storeContent(contentDirectory, content) {
  * same instant could both believe they hold it.
  */
 export class FileRecoveryStore {
+    workspaceId;
     root;
     directory;
     keepRuns;
-    constructor(root, directory, keepRuns) {
+    logger;
+    constructor(root, directory, keepRuns, logger, workspaceId) {
         this.root = root;
         this.directory = directory;
         this.keepRuns = keepRuns;
+        this.logger = logger;
+        this.workspaceId = workspaceId;
     }
-    static async open({ root, directory = defaultStateDirectory(), keepRuns = defaultKeepRuns, }) {
+    static async open({ root, directory = defaultStateDirectory(), keepRuns = defaultKeepRuns, logger = new Logger(), }) {
         if (!Number.isInteger(keepRuns) || keepRuns < 3) {
             throw new RangeError(`keepRuns must be an integer of at least 3; got ${String(keepRuns)}`);
         }
@@ -164,7 +170,8 @@ export class FileRecoveryStore {
         await makeDirectory(join(workspace, 'runs'));
         await makeDirectory(join(workspace, 'content'));
         await writeDurably(join(workspace, 'workspace.json'), JSON.stringify({ root: realRoot }));
-        return new FileRecoveryStore(realRoot, workspace, keepRuns);
+        // Real, so a state directory reached through a link is the history it holds.
+        return new FileRecoveryStore(realRoot, workspace, keepRuns, logger, await realpath(workspace));
     }
     async startRun() {
         const runId = newRunId();
@@ -201,6 +208,7 @@ export class FileRecoveryStore {
                 contentDirectory: join(this.directory, 'content'),
                 journal,
                 discard: () => this.discardRun(record),
+                settle: finished => this.settleAtFinish(finished),
                 release: () => this.unlock(runId),
             });
         }
@@ -236,8 +244,8 @@ export class FileRecoveryStore {
             if (run.status !== 'interrupted')
                 continue;
             // A journal it cannot read leaves the run listed: whether it changed anything is unknown.
-            const entries = await this.readEntries(run.runId, run.journal).catch(() => undefined);
-            if (entries && changedNothing(entries))
+            const journal = await this.readJournal(run.runId, run.journal).catch(() => undefined);
+            if (journal && changedNothing(journal))
                 gone.add(run.runId);
         }
         const parents = new Map(runs.map(run => [run.runId, run.parentRunId]));
@@ -262,9 +270,11 @@ export class FileRecoveryStore {
             await this.assertNotAbsorbed(runId);
             throw new UnknownRunError(runId);
         });
+        const { entries, unrecorded } = await this.readJournal(runId, record.journal);
         return {
             record: withLiveStatus(record, holder),
-            entries: await this.readEntries(runId, record.journal),
+            entries,
+            unrecorded: unrecorded.map(({ path, by }) => ({ path, by })),
         };
     }
     async readRecord(runId) {
@@ -433,16 +443,16 @@ export class FileRecoveryStore {
     }
     /** Takes one run's changes back to its parent, or forward again, file by file. */
     async step(runId, direction, announce) {
-        const { entries } = await this.readRun(runId);
+        const { entries, unrecorded } = await this.readRun(runId);
         const changes = fileChanges(entries, true);
-        const conflicts = [];
+        // What the history could not copy is left as it is, in either direction.
+        const conflicting = new Set(unrecorded.map(change => change.path));
         // Undone newest first, so a folder the run created is empty by the time its file goes.
         for (const change of direction === 'undo' ? changes.reverse() : changes) {
-            if (!(await this.moveFile(change, direction, announce))) {
-                conflicts.push({ path: change.path, runId });
-            }
+            if (!(await this.moveFile(change, direction, announce)))
+                conflicting.add(change.path);
         }
-        return conflicts;
+        return [...conflicting].map((path) => ({ path, runId }));
     }
     /**
      * Writes the state the step leads to, only if the file holds the state it leaves. A file
@@ -536,12 +546,21 @@ export class FileRecoveryStore {
     async chain(older, newer) {
         const olderRecord = await this.readRecord(older);
         const newerRecord = await this.readRecord(newer);
-        const entries = chainEntries(await this.readEntries(older, olderRecord.journal), await this.readEntries(newer, newerRecord.journal));
+        const olderJournal = await this.readJournal(older, olderRecord.journal);
+        const newerJournal = await this.readJournal(newer, newerRecord.journal);
+        const entries = chainEntries(olderJournal.entries, newerJournal.entries);
         const journal = `journal-${randomBytes(4).toString('hex')}.jsonl`;
         const lines = entries.flatMap(({ status, sequence, ...change }) => [
             { type: 'prepared', sequence, ...change },
             ...(status === 'prepared' ? [] : [{ type: status, sequence }]),
         ]);
+        // What either run could not copy stays uncopied: undoing the chain cannot take it back.
+        for (const [index, { path, by }] of [
+            ...olderJournal.unrecorded,
+            ...newerJournal.unrecorded,
+        ].entries()) {
+            lines.push({ type: 'unrecorded', sequence: entries.length + index + 1, path, by });
+        }
         await writeDurably(join(this.runDirectory(newer), journal), lines.map(line => `${JSON.stringify(line)}\n`).join(''));
         const record = {
             ...newerRecord,
@@ -607,16 +626,36 @@ export class FileRecoveryStore {
         await syncDirectory(this.runDirectory(record.runId));
         await rm(this.runDirectory(record.runId), { recursive: true, force: true });
     }
-    /** Removes the stored contents no kept run refers to. */
+    /**
+     * Removes the stored contents no kept run refers to. An observation still open refers to its
+     * manifest and to every content in it, which settling it needs.
+     */
     async collectContent() {
         const referenced = new Set();
         for (const run of (await this.listRuns()).runs) {
-            for (const entry of await this.readEntries(run.runId, run.journal)) {
+            const { entries, observations } = await this.readJournal(run.runId, run.journal);
+            for (const entry of entries) {
                 for (const state of [entry.before, entry.after]) {
                     if (state.exists)
                         referenced.add(state.hash);
                 }
             }
+            for (const observation of observations) {
+                referenced.add(observation.manifest);
+                const stored = await this.readContent(observation.manifest).catch((error) => {
+                    // Never stored whole: settling it will find nothing to compare.
+                    if (isNotFound(error))
+                        return undefined;
+                    throw error;
+                });
+                if (!stored)
+                    continue;
+                const manifest = JSON.parse(stored.toString('utf8'));
+                for (const state of Object.values(manifest.files)) {
+                    if (state.exists)
+                        referenced.add(state.hash);
+                }
+            }
         }
         const contentDirectory = join(this.directory, 'content');
         for (const prefix of await readdir(contentDirectory)) {
@@ -632,10 +671,9 @@ export class FileRecoveryStore {
         await syncDirectory(this.directory);
     }
     /**
-     * Marks every other run still recorded as `running` as `interrupted`: this run holds the lock,
-     * so their processes are gone. Each change one left `prepared` is settled from what its file
-     * holds now: its `after` means it was made, its `before` that it was not. A file that holds
-     * neither stays `prepared`, because nothing tells whether the change reached it.
+     * Settles every other run left unfinished: one still recorded as `running` is `interrupted`,
+     * since this run holds the lock and its process is gone, and one that finished `unsettled`
+     * keeps its status.
      */
     async settleInterrupted(ownRunId) {
         const ids = (await readdir(join(this.directory, 'runs'))).filter(id => runIdPattern.test(id) && id !== ownRunId);
@@ -645,43 +683,80 @@ export class FileRecoveryStore {
                     return undefined;
                 throw error;
             });
-            if (record?.status !== 'running')
+            if (record?.status === 'running' || record?.unsettled)
+                await this.settleRun(record);
+        }
+    }
+    /**
+     * Settles a run that finished with an observation open, under its own lock, before it releases
+     * the workspace. If that fails, its record stays `unsettled`, and the next run or move settles
+     * it, so the failure is reported and not thrown.
+     */
+    async settleAtFinish(record) {
+        try {
+            return await this.settleRun(record, { finishing: true });
+        }
+        catch (error) {
+            treatErrors(() => {
+                this.logger.warn(`The history could not settle what run ${record.runId} was observing when it finished; the next run or move settles it: ${describeFailure(error)}`);
+            }, classifyHostFailure, 'Recovery store logger failed while reporting an unsettled run');
+            return true;
+        }
+    }
+    /**
+     * Settles what a run left unfinished. Each change it left `prepared` is settled from what its
+     * file holds now: its `after` means it was made, its `before` that it was not. A file that holds
+     * neither stays `prepared`, because nothing tells whether the change reached it. Each
+     * observation it left open is settled from its manifest. A run still `running` becomes
+     * `interrupted`; any other keeps its status. Whether the run is still in the history: one that
+     * changed nothing leaves it if it was still `running`, or if it is `finishing`, which unties
+     * its turns from it. One that finished is kept otherwise, since the turns of a conversation may
+     * name it.
+     */
+    async settleRun(record, { finishing = false } = {}) {
+        const { runId } = record;
+        const journal = await this.readJournal(runId, record.journal);
+        const lines = [];
+        for (const entry of journal.entries) {
+            if (entry.status !== 'prepared' || !isAbsolute(entry.path))
                 continue;
-            const journal = await this.readJournal(runId, record.journal);
-            const settled = [];
-            for (const entry of journal.entries) {
-                if (entry.status !== 'prepared' || !isAbsolute(entry.path))
-                    continue;
-                const state = await currentState(entry.path);
-                if (state && sameState(state, entry.after)) {
-                    settled.push({ type: 'applied', sequence: entry.sequence });
-                }
-                else if (state && sameState(state, entry.before)) {
-                    settled.push({ type: 'abandoned', sequence: entry.sequence });
-                }
+            const state = await currentState(entry.path);
+            if (state && sameState(state, entry.after)) {
+                lines.push({ type: 'applied', sequence: entry.sequence });
             }
-            let sequence = journal.lastSequence;
-            for (const observation of journal.observations) {
-                const lines = await this.settleObservation(observation, journal.entries, sequence);
-                sequence += lines.length;
-                settled.push(...lines);
-            }
-            await appendWhole(join(this.runDirectory(runId), record.journal ?? firstJournal), settled);
-            if (changedNothing(await this.readEntries(runId, record.journal))) {
-                await this.discardRun(record);
-                continue;
+            else if (state && sameState(state, entry.before)) {
+                lines.push({ type: 'abandoned', sequence: entry.sequence });
             }
-            // Last, so a crash before it leaves a run that is settled again, and settling is idempotent.
-            await writeDurably(join(this.runDirectory(runId), 'run.json'), JSON.stringify({ ...record, status: 'interrupted' }));
         }
+        let sequence = journal.lastSequence;
+        for (const observation of journal.observations) {
+            const observed = await this.settleObservation(observation, journal, sequence);
+            sequence += observed.length;
+            lines.push(...observed);
+        }
+        await appendWhole(join(this.runDirectory(runId), record.journal ?? firstJournal), lines);
+        const leaves = finishing || record.status === 'running';
+        if (leaves && changedNothing(await this.readJournal(runId, record.journal))) {
+            await this.discardRun(record);
+            return false;
+        }
+        const settled = {
+            ...record,
+            status: record.status === 'running' ? 'interrupted' : record.status,
+        };
+        delete settled.unsettled;
+        // Last, so a crash before it leaves a run that is settled again, and settling is idempotent.
+        await writeDurably(join(this.runDirectory(runId), 'run.json'), JSON.stringify(settled));
+        return true;
     }
     /**
      * The lines that record what a command an interrupted run was observing changed: every file of
      * its manifest that holds something else now, as changed by that command, and then the end of
      * the observation. A file recorded after the observation began is left alone: the run
-     * recorded it itself. One that is now a folder, a link or too large to record is left out.
+     * recorded it itself. One that is now a folder, a link or too large to record, or that was one
+     * and is no longer, or that cannot be read, is recorded as a change the history could not copy.
      */
-    async settleObservation(observation, entries, lastSequence) {
+    async settleObservation(observation, { entries, unrecorded: uncopied, }, lastSequence) {
         const stored = await this.readContent(observation.manifest).catch((error) => {
             if (isNotFound(error))
                 return undefined;
@@ -691,31 +766,74 @@ export class FileRecoveryStore {
         if (!stored)
             return [{ type: 'observed', sequence: observation.sequence }];
         const manifest = JSON.parse(stored.toString('utf8'));
-        const recorded = new Set(entries.filter(entry => entry.sequence > observation.sequence).map(entry => entry.path));
+        // What the run recorded itself after the observation began, its changes and those it could
+        // not copy, such as when a command could not be confirmed stopped.
+        const recorded = new Set([...entries, ...uncopied]
+            .filter(line => line.sequence > observation.sequence)
+            .map(line => line.path));
         const lines = [];
         let sequence = lastSequence;
+        const unrecorded = (path) => ({
+            type: 'unrecorded',
+            sequence: ++sequence,
+            path,
+            by: observation.by,
+        });
         for (const [path, before] of Object.entries(manifest.files)) {
             if (!isAbsolute(path) || recorded.has(path))
                 continue;
             const after = await this.saveState(path);
-            if (!after || sameState(after, before))
+            if (after === undefined || after === 'unreadable') {
+                lines.push(unrecorded(path));
+                continue;
+            }
+            if (sameState(after, before))
                 continue;
             sequence++;
             lines.push({ type: 'prepared', sequence, path, before, after, observedBy: observation.by }, { type: 'applied', sequence });
         }
+        // What it held before is unknown, so only a path that is no longer uncopyable has changed.
+        for (const path of manifest.uncopyable ?? []) {
+            if (!isAbsolute(path) || recorded.has(path))
+                continue;
+            const now = await currentState(path).catch((error) => {
+                this.warnUnreadable(path, error);
+                return 'unreadable';
+            });
+            if (now !== undefined)
+                lines.push(unrecorded(path));
+        }
         lines.push({ type: 'observed', sequence: observation.sequence });
         return lines;
     }
-    /** What `path` holds now, with its content stored, or `undefined` for what no entry records. */
+    warnUnreadable(path, error) {
+        treatErrors(() => {
+            this.logger.warn(`The history could not read ${path} to settle what a command changed, and records it as a change undo cannot take back: ${describeFailure(error)}`);
+        }, classifyHostFailure, 'Recovery store logger failed while reporting an unreadable file');
+    }
+    /**
+     * What `path` holds now, with its content stored, or `undefined` for what no entry records.
+     * A file it cannot read is `unreadable`, and reported: it cannot be undone either, and settling
+     * never waits on it, or every later run would fail to start. A failure to store its content is
+     * thrown, so the observation stays open and settling it is tried again.
+     */
     async saveState(path) {
-        const state = await currentState(path);
-        if (!state?.exists)
-            return state;
-        const content = await readFile(path).catch((error) => {
-            if (isNotFound(error))
-                return undefined;
-            throw error;
-        });
+        let state;
+        let content;
+        try {
+            state = await currentState(path);
+            if (!state?.exists)
+                return state;
+            content = await readFile(path).catch((error) => {
+                if (isNotFound(error))
+                    return undefined;
+                throw error;
+            });
+        }
+        catch (error) {
+            this.warnUnreadable(path, error);
+            return 'unreadable';
+        }
         if (!content)
             return { exists: false };
         const { hash } = await storeContent(join(this.directory, 'content'), content);
@@ -736,13 +854,15 @@ export class FileRecoveryStore {
         }
         catch (error) {
             // A run that stopped before opening its journal never changed anything.
-            if (isNotFound(error))
-                return { entries: [], observations: [], lastSequence: 0 };
+            if (isNotFound(error)) {
+                return { entries: [], observations: [], unrecorded: [], lastSequence: 0 };
+            }
             throw error;
         }
         const lines = text.split('\n');
         const entries = new Map();
         const observations = new Map();
+        const unrecorded = [];
         let lastSequence = 0;
         lines.forEach((text, index) => {
             if (text === '')
@@ -783,6 +903,10 @@ export class FileRecoveryStore {
                 observations.delete(line.sequence);
                 return;
             }
+            if (line.type === 'unrecorded') {
+                unrecorded.push({ sequence: line.sequence, path: line.path, by: line.by });
+                return;
+            }
             const entry = entries.get(line.sequence);
             if (entry)
                 entry.status = line.type;
@@ -790,6 +914,7 @@ export class FileRecoveryStore {
         return {
             entries: [...entries.values()],
             observations: [...observations.values()],
+            unrecorded,
             lastSequence,
         };
     }
@@ -839,9 +964,12 @@ export class FileRecoveryStore {
             await unlink(path);
     }
 }
-/** No change it recorded may have reached a file: it recorded none, or abandoned each one. */
-function changedNothing(entries) {
-    return entries.every(entry => entry.status === 'abandoned');
+/**
+ * No change it recorded may have reached a file: it recorded none, or abandoned each one, and
+ * none it could not copy.
+ */
+function changedNothing({ entries, unrecorded, }) {
+    return unrecorded.length === 0 && entries.every(entry => entry.status === 'abandoned');
 }
 /**
  * A run recorded as `running` only runs while it holds the lock, its process lives, and, in this
@@ -932,6 +1060,7 @@ class FileRunJournal {
     contentDirectory;
     journal;
     discard;
+    settle;
     release;
     sequence = 0;
     stored = 0;
@@ -945,19 +1074,26 @@ class FileRunJournal {
     // The changes recorded as prepared and not abandoned: those that may have reached a file. One
     // whose `prepared` line failed is not here, since nothing writes a file before that line.
     effects = new Set();
+    // The observations begun and not yet observed: what they saw is not all recorded.
+    observing = new Set();
+    // A change it could not copy reached a file, and stays in the history.
+    lost = false;
+    // Settling what it left open as it finished found that it changed nothing.
+    settledEmpty = false;
     // Appends run one after another, so the lines never interleave.
     pending = Promise.resolve();
     finished = false;
     // Set when an append fails: it may have left part of a line, and anything after it would be
     // joined to that part and unreadable.
     broken;
-    constructor({ record, runDirectory, contentDirectory, journal, discard, release, }) {
+    constructor({ record, runDirectory, contentDirectory, journal, discard, settle, release, }) {
         this.runId = record.runId;
         this.record = record;
         this.runDirectory = runDirectory;
         this.contentDirectory = contentDirectory;
         this.journal = journal;
         this.discard = discard;
+        this.settle = settle;
         this.release = release;
     }
     async saveContent(content) {
@@ -1000,6 +1136,7 @@ class FileRunJournal {
         await this.append({ type: 'observing', sequence, manifest, by });
         // Until it is observed, whatever ran may have changed a file.
         this.effects.add(sequence);
+        this.observing.add(sequence);
         return sequence;
     }
     async observed(sequence, measure) {
@@ -1007,6 +1144,12 @@ class FileRunJournal {
         await this.append({ type: 'observed', sequence, ...(measure ? { measure } : {}) });
         // Its changes, if any, are entries of their own now.
         this.effects.delete(sequence);
+        this.observing.delete(sequence);
+    }
+    async unrecorded(path, by) {
+        this.assertOpen();
+        await this.append({ type: 'unrecorded', sequence: ++this.sequence, path, by });
+        this.lost = true;
     }
     get storedBytes() {
         return this.stored;
@@ -1031,7 +1174,7 @@ class FileRunJournal {
         return this.state.hash;
     }
     get changed() {
-        return this.effects.size > 0;
+        return !this.settledEmpty && (this.lost || this.effects.size > 0);
     }
     async finish(status) {
         this.assertOpen();
@@ -1044,8 +1187,17 @@ class FileRunJournal {
                 await this.discard();
                 return;
             }
-            const record = { ...this.record, status, finishedAt: new Date().toISOString() };
+            const unsettled = this.observing.size > 0;
+            const record = {
+                ...this.record,
+                status,
+                finishedAt: new Date().toISOString(),
+                ...(unsettled ? { unsettled } : {}),
+            };
             await writeDurably(join(this.runDirectory, 'run.json'), JSON.stringify(record));
+            // Recorded first as unsettled, so a failure or a crash here leaves it to the next run.
+            if (unsettled)
+                this.settledEmpty = !(await this.settle(record));
         }
         finally {
             await this.release();