@mikode13/harness 4.0.1 → 5.0.0

package/dist/recovery/infrastructure/fileRecoveryStore.js

@@ -0,0 +1,1043 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { link, mkdir, open, readdir, readFile, realpath, rename, rm, stat, truncate, unlink, } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, isAbsolute, join } from 'node:path';
+import { fileChanges, sameState } from "../domain/fileChanges.js";
+import { historyStart, NothingToMoveError, WorkspaceBusyError, WorkspaceMovedError, } from "../domain/recoveryStore.js";
+import { chainEntries, nextPruneStep } from "../domain/retention.js";
+import { HistoryExpiredError, redoTarget, route, UnknownRunError } from "../domain/runTree.js";
+import { currentState, hasRealParents, removeEmptyFolders, removeFile, removeTemporary, temporaryPath, writeFileState, } from "./workspaceFiles.js";
+const appName = 'mikode-harness';
+// The store holds source code the agent touched: only its owner may read it.
+const privateDirectory = 0o700;
+const privateFile = 0o600;
+const firstJournal = 'journal.jsonl';
+// A run is roughly one prompt: enough to go back through a working session, not a project's life.
+const defaultKeepRuns = 25;
+const runIdPattern = /^\d{8}T\d{9}Z-[0-9a-f]{6}$/;
+const hashPattern = /^[0-9a-f]{64}$/;
+/** Where an application keeps state that is neither configuration nor cache, per platform. */
+export function defaultStateDirectory(env = process.env, platform = process.platform, home = homedir()) {
+    // The XDG spec says to ignore a relative path, which here could land inside the repository.
+    const xdg = env.XDG_STATE_HOME;
+    if (xdg && isAbsolute(xdg))
+        return join(xdg, appName);
+    if (platform === 'darwin')
+        return join(home, 'Library', 'Application Support', appName);
+    return join(home, '.local', 'state', appName);
+}
+/** Sorts by start time as text, so the latest run is the last one listed. */
+function newRunId() {
+    return `${new Date().toISOString().replace(/[-:.]/g, '')}-${randomBytes(3).toString('hex')}`;
+}
+function hashOf(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+function isNotFound(error) {
+    return error.code === 'ENOENT';
+}
+/**
+ * The runs and moves this process holds a workspace lock for and has not ended. A lock naming
+ * this process and none of these was left by one whose release failed: it is as stale as a dead
+ * process's, which a process that lives on would otherwise keep until it exits.
+ */
+const heldHere = new Set();
+/** Whether the holder of a lock is still at work. */
+function isLiveHolder(holder) {
+    // A holder that cannot be named cannot be shown to be gone.
+    if (holder.pid === undefined)
+        return true;
+    if (holder.pid === process.pid)
+        return heldHere.has(holder.runId);
+    return isAlive(holder.pid);
+}
+/** Whether a process still exists. One we may not signal exists too. */
+function isAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}
+async function syncDirectory(path) {
+    const directory = await open(path, 'r');
+    try {
+        await directory.sync();
+    }
+    finally {
+        await directory.close();
+    }
+}
+/**
+ * Creates a private folder and any missing parents. A new folder only survives a crash once
+ * its parent's entry for it is synced too, so each parent that gained one is.
+ */
+async function makeDirectory(path) {
+    const first = await mkdir(path, { recursive: true, mode: privateDirectory });
+    if (first === undefined)
+        return;
+    for (let created = path;; created = dirname(created)) {
+        await syncDirectory(dirname(created));
+        if (created === first)
+            return;
+    }
+}
+/**
+ * Puts `content` at `path` all at once: a crash leaves either the old file or the new one, never
+ * half of the new one. It returns only once the new file is on disk. With `exclusive`, it fails
+ * with `EEXIST` instead of replacing a file already there.
+ */
+async function writeDurably(path, content, { exclusive = false } = {}) {
+    const temporary = `${path}.${randomBytes(6).toString('hex')}.tmp`;
+    try {
+        const handle = await open(temporary, 'wx', privateFile);
+        try {
+            await handle.writeFile(content);
+            await handle.sync();
+        }
+        finally {
+            await handle.close();
+        }
+        // A link fails if the path is taken; a rename would replace it.
+        await (exclusive ? link : rename)(temporary, path);
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+    await syncDirectory(dirname(path));
+}
+/**
+ * Stores `content` under its hash, unless it is stored already, and returns the hash and whether
+ * it was new. It returns only once the content is on disk.
+ */
+async function storeContent(contentDirectory, content) {
+    const hash = hashOf(content);
+    const path = contentPath(contentDirectory, hash);
+    const stored = await stat(path).then(() => true, (error) => {
+        if (isNotFound(error))
+            return false;
+        throw error;
+    });
+    if (!stored) {
+        await makeDirectory(dirname(path));
+        await writeDurably(path, content);
+    }
+    return { hash, written: !stored };
+}
+/**
+ * The recovery store on disk, in a private directory per workspace under the platform's state
+ * directory:
+ *
+ * ```text
+ * <directory>/workspaces/<hash of the root>/
+ *   workspace.json            the root it belongs to
+ *   lock                      the run writing now, if any
+ *   head.json                 the run the workspace is at, and a move in progress
+ *   revisions.jsonl           every move of the workspace through its history
+ *   content/<ab>/<hash>       file contents, named by their SHA-256
+ *   runs/<runId>/run.json     the run's record
+ *   runs/<runId>/journal.jsonl  its changes, one line per step, appended and synced
+ * ```
+ *
+ * The lock only keeps a second run of this harness from writing at the same time. It does not
+ * stop an editor or another program, and two processes taking over a dead run's lock in the
+ * same instant could both believe they hold it.
+ */
+export class FileRecoveryStore {
+    root;
+    directory;
+    keepRuns;
+    constructor(root, directory, keepRuns) {
+        this.root = root;
+        this.directory = directory;
+        this.keepRuns = keepRuns;
+    }
+    static async open({ root, directory = defaultStateDirectory(), keepRuns = defaultKeepRuns, }) {
+        if (!Number.isInteger(keepRuns) || keepRuns < 3) {
+            throw new RangeError(`keepRuns must be an integer of at least 3; got ${String(keepRuns)}`);
+        }
+        // Two spellings of one folder, such as a symlinked path, are one workspace.
+        const realRoot = await realpath(root);
+        const workspace = join(directory, 'workspaces', createHash('sha256').update(realRoot).digest('hex').slice(0, 16));
+        await makeDirectory(join(workspace, 'runs'));
+        await makeDirectory(join(workspace, 'content'));
+        await writeDurably(join(workspace, 'workspace.json'), JSON.stringify({ root: realRoot }));
+        return new FileRecoveryStore(realRoot, workspace, keepRuns);
+    }
+    async startRun() {
+        const runId = newRunId();
+        try {
+            // Inside, because taking the lock can fail after publishing it. Unlocking removes only a
+            // lock naming this run, never another run's.
+            await this.lock(runId);
+            await this.settleInterrupted(runId);
+            // A run starts where the user last sent the workspace.
+            await this.finishPendingMove();
+            // Before the new run exists, so it is never pruned, and after the move, so no run on its
+            // route is.
+            await this.prune();
+            const head = await this.readHead();
+            const parentRunId = head.runId;
+            const runDirectory = join(this.directory, 'runs', runId);
+            await makeDirectory(runDirectory);
+            const record = {
+                runId,
+                root: this.root,
+                ...(parentRunId ? { parentRunId } : {}),
+                startedAt: new Date().toISOString(),
+                status: 'running',
+                pid: process.pid,
+            };
+            await writeDurably(join(runDirectory, 'run.json'), JSON.stringify(record));
+            // After the record, so the head never names a run that does not exist.
+            await this.writeHead({ runId, revision: head.revision });
+            const journal = await open(join(runDirectory, firstJournal), 'a', privateFile);
+            await syncDirectory(runDirectory);
+            return new FileRunJournal({
+                record,
+                runDirectory,
+                contentDirectory: join(this.directory, 'content'),
+                journal,
+                discard: () => this.discardRun(record),
+                release: () => this.unlock(runId),
+            });
+        }
+        catch (error) {
+            await this.unlock(runId);
+            throw error;
+        }
+    }
+    async listRuns() {
+        const ids = (await readdir(join(this.directory, 'runs'))).filter(id => runIdPattern.test(id));
+        const holder = await readLock(join(this.directory, 'lock'));
+        const runs = [];
+        for (const runId of ids.sort()) {
+            const record = await this.readRecord(runId).catch((error) => {
+                // A crash between creating a run's folder and writing its record leaves the folder
+                // empty: that run never changed anything.
+                if (isNotFound(error))
+                    return undefined;
+                throw error;
+            });
+            if (record)
+                runs.push(withLiveStatus(record, holder));
+        }
+        // A run retention removed or chained into a later one, whose folder a crash left behind, is
+        // not in the history.
+        const gone = new Set([
+            ...runs.flatMap(run => run.absorbed ?? []),
+            ...(await this.listPruned()),
+        ]);
+        // A dead run that changed nothing is not in the history either, though its removal did not
+        // finish. No run follows it: the next run or move removes it before starting.
+        for (const run of runs) {
+            if (run.status !== 'interrupted')
+                continue;
+            // A journal it cannot read leaves the run listed: whether it changed anything is unknown.
+            const entries = await this.readEntries(run.runId, run.journal).catch(() => undefined);
+            if (entries && changedNothing(entries))
+                gone.add(run.runId);
+        }
+        const parents = new Map(runs.map(run => [run.runId, run.parentRunId]));
+        let { runId: head } = await this.readHead();
+        while (head !== undefined && gone.has(head))
+            head = parents.get(head);
+        return { ...(head ? { head } : {}), runs: runs.filter(run => !gone.has(run.runId)) };
+    }
+    async readRun(runId) {
+        // An id from outside is checked before it names a folder.
+        if (!runIdPattern.test(runId))
+            throw new UnknownRunError(runId);
+        // The lock first: a run that finishes between the two reads then shows as finished, not as
+        // a `running` record nobody holds.
+        const holder = await readLock(join(this.directory, 'lock'));
+        // Before the record: a crash can leave the folder of a run retention already took.
+        await this.assertNotAbsorbed(runId);
+        const record = await this.readRecord(runId).catch(async (error) => {
+            if (!isNotFound(error))
+                throw error;
+            // Retention may have taken it since the check above.
+            await this.assertNotAbsorbed(runId);
+            throw new UnknownRunError(runId);
+        });
+        return {
+            record: withLiveStatus(record, holder),
+            entries: await this.readEntries(runId, record.journal),
+        };
+    }
+    async readRecord(runId) {
+        return JSON.parse(await readFile(join(this.runDirectory(runId), 'run.json'), 'utf8'));
+    }
+    runDirectory(runId) {
+        // The id names a folder, so it must never be a path of its own.
+        if (!runIdPattern.test(runId))
+            throw new Error(`Not a run id: ${runId}`);
+        return join(this.directory, 'runs', runId);
+    }
+    async readHead() {
+        try {
+            const head = JSON.parse(await readFile(join(this.directory, 'head.json'), 'utf8'));
+            return { ...head, revision: head.revision ?? 0 };
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return { revision: 0 };
+            throw error;
+        }
+    }
+    async writeHead(head) {
+        await writeDurably(join(this.directory, 'head.json'), JSON.stringify(head));
+    }
+    goTo(target, options = {}) {
+        return this.move(() => (target === historyStart ? undefined : target), options);
+    }
+    undo(options = {}) {
+        return this.move(({ runs, head }) => {
+            if (head === undefined)
+                throw new NothingToMoveError('undo');
+            return runs.find(run => run.runId === head)?.parentRunId;
+        }, options);
+    }
+    redo(options = {}) {
+        return this.move(({ runs, head, revisions }) => {
+            const target = redoTarget(runs, revisions, head);
+            if (target === undefined)
+                throw new NothingToMoveError('redo');
+            return target;
+        }, options);
+    }
+    async listRevisions() {
+        let text;
+        try {
+            text = await readFile(join(this.directory, 'revisions.jsonl'), 'utf8');
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return [];
+            throw error;
+        }
+        const lines = text.split('\n');
+        return lines.flatMap((line, index) => {
+            if (line === '')
+                return [];
+            try {
+                return [JSON.parse(line)];
+            }
+            catch (error) {
+                // A crash in the middle of an append leaves half a line, always the last one. Its
+                // move is still pending, and finishing it records it again.
+                if (index === lines.length - 1)
+                    return [];
+                throw new Error(`The history of this workspace is corrupt at line ${String(index + 1)}`, {
+                    cause: error,
+                });
+            }
+        });
+    }
+    /** Holds the workspace like a run does, then moves it to the run `resolve` picks. */
+    async move(resolve, { reason, from }) {
+        const moveId = `move-${newRunId()}`;
+        try {
+            await this.lock(moveId);
+            await this.settleInterrupted(moveId);
+            await this.finishPendingMove();
+            const head = await this.readHead();
+            // Only now, under the lock and once nothing is left unsettled, is the head the one the
+            // move would start from.
+            const at = head.runId ?? historyStart;
+            if (from !== undefined && from !== at)
+                throw new WorkspaceMovedError(from, at);
+            const { runs } = await this.listRuns();
+            const to = resolve({ runs, head: head.runId, revisions: await this.listRevisions() });
+            if (to !== undefined && !runs.some(run => run.runId === to)) {
+                await this.assertNotAbsorbed(to);
+            }
+            return await this.takeSteps(head, runs, {
+                revision: head.revision + 1,
+                ...(head.runId ? { from: head.runId } : {}),
+                ...(to ? { to } : {}),
+                ...(reason === undefined ? {} : { reason }),
+                conflicts: [],
+            });
+        }
+        finally {
+            await this.unlock(moveId);
+        }
+    }
+    /** Finishes a move whose process stopped halfway, from the run its last step reached. */
+    async finishPendingMove() {
+        const head = await this.readHead();
+        const { pending } = head;
+        if (!pending)
+            return;
+        // It was recorded, and only clearing it from the head was missing.
+        if ((await this.listRevisions()).some(revision => revision.revision === pending.revision)) {
+            await this.writeHead({
+                ...(pending.to ? { runId: pending.to } : {}),
+                revision: pending.revision,
+            });
+            return;
+        }
+        await this.takeSteps(head, (await this.listRuns()).runs, pending);
+    }
+    /**
+     * Undoes, then redoes, one run at a time from where the head is to `pending.to`. After each
+     * step the head names the run reached, with the move still pending, so a crash loses at
+     * most the step in progress, and that step is safe to take again.
+     */
+    async takeSteps(head, runs, pending) {
+        const { undo, redo } = route(runs, head.runId, pending.to);
+        const parents = new Map(runs.map(run => [run.runId, run.parentRunId]));
+        let reached = head.runId;
+        const save = () => this.writeHead({ ...(reached ? { runId: reached } : {}), revision: head.revision, pending });
+        // What a write the crash interrupted left beside its file.
+        if (pending.temporary) {
+            await removeTemporary(pending.temporary);
+            delete pending.temporary;
+        }
+        await save();
+        const announce = async (temporary) => {
+            if (temporary === undefined)
+                delete pending.temporary;
+            else
+                pending.temporary = temporary;
+            await save();
+        };
+        for (const runId of undo) {
+            pending.conflicts.push(...(await this.step(runId, 'undo', announce)));
+            reached = parents.get(runId);
+            await save();
+        }
+        for (const runId of redo) {
+            pending.conflicts.push(...(await this.step(runId, 'redo', announce)));
+            reached = runId;
+            await save();
+        }
+        // Every write that started has finished, so no temporary is left to name.
+        delete pending.temporary;
+        const { conflicts, ...rest } = pending;
+        const revision = {
+            ...rest,
+            at: new Date().toISOString(),
+            complete: conflicts.length === 0,
+            conflicts,
+        };
+        await this.appendRevision(revision);
+        await this.writeHead({
+            ...(pending.to ? { runId: pending.to } : {}),
+            revision: pending.revision,
+        });
+        return revision;
+    }
+    /** Takes one run's changes back to its parent, or forward again, file by file. */
+    async step(runId, direction, announce) {
+        const { entries } = await this.readRun(runId);
+        const changes = fileChanges(entries, true);
+        const conflicts = [];
+        // Undone newest first, so a folder the run created is empty by the time its file goes.
+        for (const change of direction === 'undo' ? changes.reverse() : changes) {
+            if (!(await this.moveFile(change, direction, announce))) {
+                conflicts.push({ path: change.path, runId });
+            }
+        }
+        return conflicts;
+    }
+    /**
+     * Writes the state the step leads to, only if the file holds the state it leaves. A file
+     * already there is left alone, so taking a step again is safe. Returns whether the file
+     * ends where the step leads.
+     */
+    async moveFile(change, direction, announce) {
+        const [source, destination] = direction === 'undo' ? [change.after, change.before] : [change.before, change.after];
+        // First, so a file reached through a folder that became a link is never taken for the
+        // destination, and nothing is read or removed through it.
+        if (!(await hasRealParents(change.path)))
+            return false;
+        const current = await currentState(change.path);
+        if (current && sameState(current, destination)) {
+            if (direction === 'undo' && !destination.exists) {
+                await removeEmptyFolders(change.createdFolders);
+            }
+            return true;
+        }
+        if (!current || !sameState(current, source))
+            return false;
+        // Someone wrote to the file between two of the run's changes, and going back to before
+        // the first would discard that.
+        if (direction === 'undo' && !change.continuous)
+            return false;
+        if (destination.exists) {
+            const temporary = temporaryPath(change.path);
+            await announce(temporary);
+            await writeFileState(change.path, await this.readContent(destination.hash), destination.mode, temporary);
+            await announce(undefined);
+        }
+        else {
+            await removeFile(change.path);
+            if (direction === 'undo')
+                await removeEmptyFolders(change.createdFolders);
+        }
+        return true;
+    }
+    /** Throws `HistoryExpiredError` if retention removed `runId` or chained it into a later run. */
+    async assertNotAbsorbed(runId) {
+        const { runs } = await this.listRuns();
+        const keptIn = runs.find(run => run.absorbed?.includes(runId));
+        if (keptIn)
+            throw new HistoryExpiredError(runId, keptIn.runId);
+        if ((await this.listPruned()).includes(runId))
+            throw new HistoryExpiredError(runId);
+    }
+    async listPruned() {
+        try {
+            const text = await readFile(join(this.directory, 'pruned.jsonl'), 'utf8');
+            return text.split('\n').flatMap(line => {
+                try {
+                    return [JSON.parse(line)];
+                }
+                catch {
+                    // Half a line a crash left, whose removal never happened.
+                    return [];
+                }
+            });
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return [];
+            throw error;
+        }
+    }
+    /**
+     * Brings the history down to `keepRuns - 1`, so the run about to start makes `keepRuns`, then
+     * frees the contents no run kept refers to.
+     */
+    async prune() {
+        await this.finishChains();
+        const revisions = await this.listRevisions();
+        for (;;) {
+            const { head, runs } = await this.listRuns();
+            const step = nextPruneStep(runs, revisions, head, this.keepRuns - 1);
+            if (!step)
+                break;
+            if (step.kind === 'remove')
+                await this.removeRun(step.runId);
+            else
+                await this.chain(step.runId, step.into);
+        }
+        await this.collectContent();
+    }
+    /**
+     * Chains the changes of `older` into `newer`, its child: `newer` then hangs from `older`'s
+     * parent and goes back to where `older` started. The new record is the commit point. Before
+     * it, the history is as it was; after it, `finishChains` removes what is left of `older`.
+     */
+    async chain(older, newer) {
+        const olderRecord = await this.readRecord(older);
+        const newerRecord = await this.readRecord(newer);
+        const entries = chainEntries(await this.readEntries(older, olderRecord.journal), await this.readEntries(newer, newerRecord.journal));
+        const journal = `journal-${randomBytes(4).toString('hex')}.jsonl`;
+        const lines = entries.flatMap(({ status, sequence, ...change }) => [
+            { type: 'prepared', sequence, ...change },
+            ...(status === 'prepared' ? [] : [{ type: status, sequence }]),
+        ]);
+        await writeDurably(join(this.runDirectory(newer), journal), lines.map(line => `${JSON.stringify(line)}\n`).join(''));
+        const record = {
+            ...newerRecord,
+            absorbed: [...(olderRecord.absorbed ?? []), older, ...(newerRecord.absorbed ?? [])],
+            journal,
+        };
+        // It now hangs where `older` did: from its parent, or from the start.
+        if (olderRecord.parentRunId)
+            record.parentRunId = olderRecord.parentRunId;
+        else
+            delete record.parentRunId;
+        await writeDurably(join(this.runDirectory(newer), 'run.json'), JSON.stringify(record));
+        await this.finishChains();
+    }
+    /**
+     * Removes what a chain or a removal a crash interrupted left behind: the folders of runs
+     * pruned or absorbed by a kept run, and journals a run no longer reads.
+     */
+    async finishChains() {
+        for (const runId of await this.listPruned()) {
+            await rm(this.runDirectory(runId), { recursive: true, force: true });
+        }
+        const { runs } = await this.listRuns();
+        for (const run of runs) {
+            for (const absorbed of run.absorbed ?? []) {
+                await rm(this.runDirectory(absorbed), { recursive: true, force: true });
+            }
+            const current = run.journal ?? firstJournal;
+            for (const name of await readdir(this.runDirectory(run.runId))) {
+                if (name.startsWith('journal') && name.endsWith('.jsonl') && name !== current) {
+                    await rm(join(this.runDirectory(run.runId), name), { force: true });
+                }
+            }
+        }
+    }
+    /**
+     * Removes a run whole, recording its id first so a move to it reports an expired history.
+     * Its record goes next, so a crash halfway leaves no run behind.
+     */
+    async removeRun(runId) {
+        await appendWhole(join(this.directory, 'pruned.jsonl'), [runId]);
+        // The log may be new: its entry in the folder must last before the run it names goes.
+        await syncDirectory(this.directory);
+        await rm(join(this.runDirectory(runId), 'run.json'), { force: true });
+        await syncDirectory(this.runDirectory(runId));
+        await rm(this.runDirectory(runId), { recursive: true, force: true });
+    }
+    /**
+     * Takes a run that changed nothing out of the history, under the lock: the workspace goes back
+     * to its parent first, so the head never names a run that is gone. Until the run is removed,
+     * `listRuns` already hides it, and the next run or move finishes removing it. The contents it
+     * saved are freed by the next prune.
+     */
+    async discardRun(record) {
+        const { runId: headRunId, ...head } = await this.readHead();
+        if (headRunId === record.runId) {
+            await this.writeHead({
+                ...head,
+                ...(record.parentRunId ? { runId: record.parentRunId } : {}),
+            });
+        }
+        await rm(join(this.runDirectory(record.runId), 'run.json'), { force: true });
+        await syncDirectory(this.runDirectory(record.runId));
+        await rm(this.runDirectory(record.runId), { recursive: true, force: true });
+    }
+    /** Removes the stored contents no kept run refers to. */
+    async collectContent() {
+        const referenced = new Set();
+        for (const run of (await this.listRuns()).runs) {
+            for (const entry of await this.readEntries(run.runId, run.journal)) {
+                for (const state of [entry.before, entry.after]) {
+                    if (state.exists)
+                        referenced.add(state.hash);
+                }
+            }
+        }
+        const contentDirectory = join(this.directory, 'content');
+        for (const prefix of await readdir(contentDirectory)) {
+            for (const hash of await readdir(join(contentDirectory, prefix))) {
+                if (hashPattern.test(hash) && !referenced.has(hash)) {
+                    await rm(join(contentDirectory, prefix, hash), { force: true });
+                }
+            }
+        }
+    }
+    async appendRevision(revision) {
+        await appendWhole(join(this.directory, 'revisions.jsonl'), [revision]);
+        await syncDirectory(this.directory);
+    }
+    /**
+     * Marks every other run still recorded as `running` as `interrupted`: this run holds the lock,
+     * so their processes are gone. Each change one left `prepared` is settled from what its file
+     * holds now: its `after` means it was made, its `before` that it was not. A file that holds
+     * neither stays `prepared`, because nothing tells whether the change reached it.
+     */
+    async settleInterrupted(ownRunId) {
+        const ids = (await readdir(join(this.directory, 'runs'))).filter(id => runIdPattern.test(id) && id !== ownRunId);
+        for (const runId of ids) {
+            const record = await this.readRecord(runId).catch((error) => {
+                if (isNotFound(error))
+                    return undefined;
+                throw error;
+            });
+            if (record?.status !== 'running')
+                continue;
+            const journal = await this.readJournal(runId, record.journal);
+            const settled = [];
+            for (const entry of journal.entries) {
+                if (entry.status !== 'prepared' || !isAbsolute(entry.path))
+                    continue;
+                const state = await currentState(entry.path);
+                if (state && sameState(state, entry.after)) {
+                    settled.push({ type: 'applied', sequence: entry.sequence });
+                }
+                else if (state && sameState(state, entry.before)) {
+                    settled.push({ type: 'abandoned', sequence: entry.sequence });
+                }
+            }
+            let sequence = journal.lastSequence;
+            for (const observation of journal.observations) {
+                const lines = await this.settleObservation(observation, journal.entries, sequence);
+                sequence += lines.length;
+                settled.push(...lines);
+            }
+            await appendWhole(join(this.runDirectory(runId), record.journal ?? firstJournal), settled);
+            if (changedNothing(await this.readEntries(runId, record.journal))) {
+                await this.discardRun(record);
+                continue;
+            }
+            // Last, so a crash before it leaves a run that is settled again, and settling is idempotent.
+            await writeDurably(join(this.runDirectory(runId), 'run.json'), JSON.stringify({ ...record, status: 'interrupted' }));
+        }
+    }
+    /**
+     * The lines that record what a command an interrupted run was observing changed: every file of
+     * its manifest that holds something else now, as changed by that command, and then the end of
+     * the observation. A file recorded after the observation began is left alone: the run
+     * recorded it itself. One that is now a folder, a link or too large to record is left out.
+     */
+    async settleObservation(observation, entries, lastSequence) {
+        const stored = await this.readContent(observation.manifest).catch((error) => {
+            if (isNotFound(error))
+                return undefined;
+            throw error;
+        });
+        // Its manifest was never stored whole: the command it was saved for never started.
+        if (!stored)
+            return [{ type: 'observed', sequence: observation.sequence }];
+        const manifest = JSON.parse(stored.toString('utf8'));
+        const recorded = new Set(entries.filter(entry => entry.sequence > observation.sequence).map(entry => entry.path));
+        const lines = [];
+        let sequence = lastSequence;
+        for (const [path, before] of Object.entries(manifest.files)) {
+            if (!isAbsolute(path) || recorded.has(path))
+                continue;
+            const after = await this.saveState(path);
+            if (!after || sameState(after, before))
+                continue;
+            sequence++;
+            lines.push({ type: 'prepared', sequence, path, before, after, observedBy: observation.by }, { type: 'applied', sequence });
+        }
+        lines.push({ type: 'observed', sequence: observation.sequence });
+        return lines;
+    }
+    /** What `path` holds now, with its content stored, or `undefined` for what no entry records. */
+    async saveState(path) {
+        const state = await currentState(path);
+        if (!state?.exists)
+            return state;
+        const content = await readFile(path).catch((error) => {
+            if (isNotFound(error))
+                return undefined;
+            throw error;
+        });
+        if (!content)
+            return { exists: false };
+        const { hash } = await storeContent(join(this.directory, 'content'), content);
+        // Read again: the file may have changed between hashing it and storing it.
+        return hash === state.hash ? state : this.saveState(path);
+    }
+    async readEntries(runId, journal = firstJournal) {
+        return (await this.readJournal(runId, journal)).entries;
+    }
+    /**
+     * A run's changes, the observations it began and never finished, and the highest sequence
+     * number its journal uses, which the next line it gets must exceed.
+     */
+    async readJournal(runId, journal = firstJournal) {
+        let text;
+        try {
+            text = await readFile(join(this.runDirectory(runId), journal), 'utf8');
+        }
+        catch (error) {
+            // A run that stopped before opening its journal never changed anything.
+            if (isNotFound(error))
+                return { entries: [], observations: [], lastSequence: 0 };
+            throw error;
+        }
+        const lines = text.split('\n');
+        const entries = new Map();
+        const observations = new Map();
+        let lastSequence = 0;
+        lines.forEach((text, index) => {
+            if (text === '')
+                return;
+            let line;
+            try {
+                line = JSON.parse(text);
+            }
+            catch (error) {
+                // A crash in the middle of an append leaves half a line, always the last one. Its
+                // step never returned, so no change was made after it.
+                if (index === lines.length - 1)
+                    return;
+                throw new Error(`The journal of run ${runId} is corrupt at line ${String(index + 1)}`, {
+                    cause: error,
+                });
+            }
+            lastSequence = Math.max(lastSequence, line.sequence);
+            if (line.type === 'prepared') {
+                const { sequence, path, before, after, createdFolders, observedBy } = line;
+                entries.set(sequence, {
+                    sequence,
+                    path,
+                    before,
+                    after,
+                    ...(createdFolders ? { createdFolders } : {}),
+                    ...(observedBy === undefined ? {} : { observedBy }),
+                    status: 'prepared',
+                });
+                return;
+            }
+            if (line.type === 'observing') {
+                const { sequence, manifest, by } = line;
+                observations.set(sequence, { sequence, manifest, by });
+                return;
+            }
+            if (line.type === 'observed') {
+                observations.delete(line.sequence);
+                return;
+            }
+            const entry = entries.get(line.sequence);
+            if (entry)
+                entry.status = line.type;
+        });
+        return {
+            entries: [...entries.values()],
+            observations: [...observations.values()],
+            lastSequence,
+        };
+    }
+    async readContent(hash) {
+        if (!hashPattern.test(hash))
+            throw new Error(`Not a content hash: ${hash}`);
+        return readFile(contentPath(join(this.directory, 'content'), hash));
+    }
+    async lock(runId) {
+        const path = join(this.directory, 'lock');
+        // Twice at most: once as it is, and once more after clearing a dead run's lock.
+        for (let attempt = 0; attempt < 2; attempt++) {
+            try {
+                // Written whole before it appears, so a lock that exists always names its holder.
+                await writeDurably(path, JSON.stringify({ runId, pid: process.pid }), { exclusive: true });
+                heldHere.add(runId);
+                return;
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST')
+                    throw error;
+            }
+            const holder = await readLock(path);
+            // Released meanwhile: try again.
+            if (!holder)
+                continue;
+            if (isLiveHolder(holder))
+                throw new WorkspaceBusyError(holder.runId);
+            // Its process died, or its run ended, without releasing it. Read again just before
+            // removing, so a lock another run has just taken is left alone.
+            const again = await readLock(path);
+            if (again !== undefined && again.pid === holder.pid && again.runId === holder.runId) {
+                await unlink(path).catch((error) => {
+                    if (!isNotFound(error))
+                        throw error;
+                });
+            }
+        }
+        throw new WorkspaceBusyError((await readLock(path))?.runId ?? 'unknown');
+    }
+    async unlock(runId) {
+        // First: the run has ended even if removing its lock fails, so that lock is stale.
+        heldHere.delete(runId);
+        const path = join(this.directory, 'lock');
+        // Only its own lock: one cleared as stale and taken by another run is not this run's.
+        if ((await readLock(path))?.runId === runId)
+            await unlink(path);
+    }
+}
+/** No change it recorded may have reached a file: it recorded none, or abandoned each one. */
+function changedNothing(entries) {
+    return entries.every(entry => entry.status === 'abandoned');
+}
+/**
+ * A run recorded as `running` only runs while it holds the lock, its process lives, and, in this
+ * process, it has not ended. Any other is reported as `interrupted`, before the next run settles
+ * it.
+ */
+function withLiveStatus(record, holder) {
+    if (record.status !== 'running')
+        return record;
+    const live = holder?.runId === record.runId && holder.pid !== undefined && isLiveHolder(holder);
+    return live ? record : { ...record, status: 'interrupted' };
+}
+/**
+ * Appends `lines`, as JSON, to a file of one line each, created private if missing. A crash may
+ * have left its last line without a newline. A whole line, which the readers already count, is
+ * kept and ended. Half a line would join the first new line, so it is cut off: whatever was
+ * writing it never returned.
+ */
+async function appendWhole(path, lines) {
+    if (lines.length === 0)
+        return;
+    let text = '';
+    try {
+        text = await readFile(path, 'utf8');
+    }
+    catch (error) {
+        if (!isNotFound(error))
+            throw error;
+    }
+    const last = text.slice(text.lastIndexOf('\n') + 1);
+    let separator = '';
+    if (last !== '') {
+        if (isWholeLine(last))
+            separator = '\n';
+        else
+            await truncate(path, Buffer.byteLength(text) - Buffer.byteLength(last));
+    }
+    const journal = await open(path, 'a', privateFile);
+    try {
+        await journal.appendFile(separator + lines.map(line => `${JSON.stringify(line)}\n`).join(''));
+        await journal.sync();
+    }
+    finally {
+        await journal.close();
+    }
+}
+function isWholeLine(text) {
+    try {
+        JSON.parse(text);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Who holds the lock, or `undefined` when nobody does. A lock this store did not write whole
+ * has no `pid`, so it is never taken for a dead run's.
+ */
+async function readLock(path) {
+    let text;
+    try {
+        text = await readFile(path, 'utf8');
+    }
+    catch (error) {
+        if (isNotFound(error))
+            return undefined;
+        throw error;
+    }
+    try {
+        const { runId, pid } = JSON.parse(text);
+        if (typeof runId === 'string' && Number.isInteger(pid))
+            return { runId, pid: pid };
+    }
+    catch {
+        // Not JSON: named below as unknown.
+    }
+    return { runId: 'unknown' };
+}
+function contentPath(contentDirectory, hash) {
+    // Two levels, so no folder holds every file a workspace ever stored.
+    return join(contentDirectory, hash.slice(0, 2), hash);
+}
+class FileRunJournal {
+    runId;
+    record;
+    runDirectory;
+    contentDirectory;
+    journal;
+    discard;
+    release;
+    sequence = 0;
+    stored = 0;
+    // The changes recorded as prepared and not abandoned: those that may have reached a file. One
+    // whose `prepared` line failed is not here, since nothing writes a file before that line.
+    effects = new Set();
+    // Appends run one after another, so the lines never interleave.
+    pending = Promise.resolve();
+    finished = false;
+    // Set when an append fails: it may have left part of a line, and anything after it would be
+    // joined to that part and unreadable.
+    broken;
+    constructor({ record, runDirectory, contentDirectory, journal, discard, release, }) {
+        this.runId = record.runId;
+        this.record = record;
+        this.runDirectory = runDirectory;
+        this.contentDirectory = contentDirectory;
+        this.journal = journal;
+        this.discard = discard;
+        this.release = release;
+    }
+    async saveContent(content) {
+        this.assertOpen();
+        const { hash, written } = await storeContent(this.contentDirectory, content);
+        if (written)
+            this.stored += content.length;
+        return hash;
+    }
+    async prepare(change) {
+        this.assertOpen();
+        const sequence = ++this.sequence;
+        await this.append({ type: 'prepared', sequence, ...change });
+        this.effects.add(sequence);
+        return sequence;
+    }
+    applied(sequence) {
+        this.assertOpen();
+        return this.append({ type: 'applied', sequence });
+    }
+    async abandoned(sequence) {
+        this.assertOpen();
+        await this.append({ type: 'abandoned', sequence });
+        // Only once recorded: a change whose abandonment was not may still count as made.
+        this.effects.delete(sequence);
+    }
+    async observe(manifest, by) {
+        this.assertOpen();
+        const sequence = ++this.sequence;
+        await this.append({ type: 'observing', sequence, manifest, by });
+        // Until it is observed, whatever ran may have changed a file.
+        this.effects.add(sequence);
+        return sequence;
+    }
+    async observed(sequence, measure) {
+        this.assertOpen();
+        await this.append({ type: 'observed', sequence, ...(measure ? { measure } : {}) });
+        // Its changes, if any, are entries of their own now.
+        this.effects.delete(sequence);
+    }
+    get storedBytes() {
+        return this.stored;
+    }
+    get changed() {
+        return this.effects.size > 0;
+    }
+    async finish(status) {
+        this.assertOpen();
+        this.finished = true;
+        try {
+            await this.pending;
+            await this.journal.close();
+            // Its record is left `running`: if removing it fails, readers already take it as gone.
+            if (!this.changed) {
+                await this.discard();
+                return;
+            }
+            const record = { ...this.record, status, finishedAt: new Date().toISOString() };
+            await writeDurably(join(this.runDirectory, 'run.json'), JSON.stringify(record));
+        }
+        finally {
+            await this.release();
+        }
+    }
+    append(line) {
+        const write = this.pending.then(async () => {
+            if (this.broken !== undefined) {
+                throw new Error(`The journal of run ${this.runId} stopped after a failed write`, {
+                    cause: this.broken,
+                });
+            }
+            try {
+                await this.journal.appendFile(`${JSON.stringify(line)}\n`);
+                await this.journal.sync();
+            }
+            catch (error) {
+                this.broken = error ?? new Error('Unknown write failure');
+                throw error;
+            }
+        });
+        // The chain goes on, so each later append reports the broken journal to its own caller.
+        this.pending = write.catch(() => undefined);
+        return write;
+    }
+    assertOpen() {
+        if (this.finished)
+            throw new Error(`Run ${this.runId} has already finished`);
+    }
+}
+//# sourceMappingURL=fileRecoveryStore.js.map