@deksden-com/dd-flow-cli 0.9.0-beta.144 → 0.9.0-beta.149

package/dist/services/memory-upgrade-publication.js

@@ -0,0 +1,413 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { isScalar, parseDocument } from "yaml";
+import { AppError } from "../shared/errors.js";
+import { publishImmutableInput } from "../storage/immutable-input.js";
+import { acquireInstallationLock } from "./installation-lock.js";
+import { stageRetainedMemoryRepairFile, verifyRetainedMemoryRepairFile } from "./memory-repair-publication-files.js";
+import { PreparedMemoryLayer } from "./memory-layer.js";
+import { hybridDigest } from "./hybrid-contracts.js";
+import { readUpgradePreservationBaseline, upgradeAuthoringPaths } from "./memory-upgrade-inventory.js";
+const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
+function refuse(reason) {
+    throw new AppError("memory_upgrade_publication_invalid", "Canonical publication requires exact retained state", 1, { reason, effect: "no_effect" });
+}
+function logical(value) {
+    if (!value || path.isAbsolute(value) || value.includes("\\") || value.includes("\0")
+        || value.split("/").some(part => !part || part === "." || part === ".."))
+        refuse("unsafe_publication_path");
+}
+function directory(value) {
+    if (!fs.lstatSync(value).isDirectory())
+        refuse("linked_or_missing_publication_directory");
+}
+function parent(root, value, create = false, mode = 0o755) {
+    logical(value);
+    directory(root);
+    let current = root;
+    for (const segment of value.split("/").slice(0, -1)) {
+        current = path.join(current, segment);
+        if (create && !fs.lstatSync(current, { throwIfNoEntry: false }))
+            fs.mkdirSync(current, { mode });
+        directory(current);
+    }
+    return path.join(root, value);
+}
+function tree(root) {
+    directory(root);
+    const entries = [];
+    let total = 0;
+    const walk = (prefix) => {
+        for (const name of fs.readdirSync(path.join(root, prefix)).sort()) {
+            const relative = prefix + name;
+            logical(relative);
+            const file = path.join(root, relative), before = fs.lstatSync(file);
+            if (before.isDirectory()) {
+                if (relative.split("/").length > 64)
+                    refuse("unbounded_tree");
+                walk(`${relative}/`);
+                continue;
+            }
+            if (!before.isFile() && !before.isSymbolicLink() || before.isFile() && before.nlink !== 1
+                || entries.length >= 20_000 || before.size > 64 * 1024 * 1024)
+                refuse("unsupported_tree_entry");
+            const bytes = before.isSymbolicLink() ? fs.readlinkSync(file, { encoding: "buffer" }) : fs.readFileSync(file);
+            const after = fs.lstatSync(file);
+            if ((total += bytes.length) > 64 * 1024 * 1024 || before.dev !== after.dev || before.ino !== after.ino
+                || before.mode !== after.mode || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs)
+                refuse("tree_changed_during_read");
+            entries.push({ path: relative, kind: before.isSymbolicLink() ? "link" : "file", mode: before.mode & 0o777, sha256: hash(bytes) });
+        }
+    };
+    walk("");
+    return entries.sort((a, b) => a.path.localeCompare(b.path, "en"));
+}
+function syncDirectory(value) {
+    const fd = fs.openSync(value, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
+    try {
+        fs.fsyncSync(fd);
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+function syncParents(root, logical) {
+    let current = path.dirname(parent(root, logical));
+    while (current !== root) {
+        syncDirectory(current);
+        current = path.dirname(current);
+    }
+    syncDirectory(root);
+}
+/** Finite filesystem effects only, not Stage/Work/SQL authority. The caller
+ * must retain this exact plan under its accepted diff and settled Work before
+ * applying it. Restoring bytes does not admit a RUN or accept semantic edits.
+ */
+export class PreparedMemoryUpgradePublication {
+    sha256;
+    #plan;
+    constructor(plan) { this.#plan = structuredClone(plan); this.sha256 = hybridDigest(plan); Object.freeze(this); }
+    static prepare(input) {
+        const { inventory: selected, layer, changes } = input;
+        if (!/^[a-f0-9]{64}$/.test(input.selectionSha256) || !(layer instanceof PreparedMemoryLayer)
+            || selected.inventory.target_layer_sha256 !== layer.sha256)
+            refuse("foreign_publication_selection");
+        const old = readUpgradePreservationBaseline(input.oldTree, selected);
+        const authoring = new Set([...upgradeAuthoringPaths(selected.inventory, changes).values()].flat().map(value => value.slice(13)));
+        const workspace = selected.inventory.workspace_root;
+        if (fs.realpathSync(workspace) !== workspace)
+            refuse("noncanonical_workspace");
+        const root = path.join(workspace, ".memory-bank"), before = tree(root), current = new Map(before.map(entry => [entry.path, entry]));
+        const frozen = new Map(selected.inventory.entries.map(entry => [entry.path, entry]));
+        for (const entry of before) {
+            const expected = frozen.get(entry.path)?.current;
+            if (!authoring.has(entry.path) && (!expected || entry.kind !== expected.kind || entry.mode !== expected.mode || entry.sha256 !== expected.sha256))
+                refuse("unassigned_project_change");
+        }
+        for (const [name, entry] of frozen)
+            if (entry.current && !current.has(name) && !authoring.has(name))
+                refuse("unassigned_project_removal");
+        const actions = [], archived = [];
+        const target = layer.files();
+        const add = (name, bytes, mode = 0o644) => {
+            logical(name);
+            const source = current.get(name) ?? null;
+            const after = bytes === null ? null : { path: name, kind: "file", mode, sha256: hash(bytes) };
+            if (hybridDigest(source) !== hybridDigest(after))
+                actions.push({ path: name, before: source, after, base64: bytes?.toString("base64") ?? null });
+        };
+        const content = [];
+        const archiveSource = (name, original, independent) => {
+            const bytes = old.get(name), entry = frozen.get(name);
+            if (!bytes || bytes.kind !== "file" || independent && bytes.sha256 !== entry?.baseline_sha256
+                || bytes.sha256 !== original.sha256)
+                refuse("unverified_archive_bytes");
+            // Historical documents are recovery data, not another active Markdown
+            // catalog. The record retains their original paths and raw byte hashes.
+            const archive = `archive/mb-upgrade/${input.selectionSha256}/files/${name}.bin`;
+            if (current.has(archive))
+                refuse("archive_destination_exists");
+            add(archive, bytes.bytes, bytes.mode);
+            archived.push({ path: name, archive, mode: bytes.mode, sha256: bytes.sha256 });
+        };
+        const mechanicalChanges = changes.flatMap(change => change.ownership === "canonical" && change.action === "move"
+            ? [{ ...change, to: null, action: "archive" }, { ...change, from: null, action: "add" }] : [change]);
+        for (const change of mechanicalChanges) {
+            if (change.action === "retain" || change.action === "semantic-work" || !["canonical", "generated"].includes(change.ownership))
+                continue;
+            const name = (change.to ?? change.from).slice(13), entry = frozen.get(name);
+            if (!entry || entry.ownership !== change.ownership || entry.action !== change.action && entry.ownership !== "generated")
+                refuse("changed_mechanical_action");
+            const original = current.get(name) ?? null;
+            if (original && (original.kind !== "file" || original.sha256 !== entry.current?.sha256 || original.mode !== entry.current.mode))
+                refuse("changed_canonical_source");
+            if (original)
+                archiveSource(name, original, entry.ownership === "canonical");
+            const bytes = change.action === "archive" ? null : target.get(name);
+            if (bytes === undefined)
+                refuse("missing_canonical_target");
+            const start = actions.length;
+            add(name, bytes, original?.mode ?? 0o644);
+            content.push(...actions.splice(start));
+        }
+        if (input.versionSource !== undefined) {
+            const name = input.versionSource;
+            if (!["index.md", "mbb/index.md"].includes(name) || content.some(action => action.path === name))
+                refuse("invalid_version_source");
+            const source = current.get(name), original = old.get(name);
+            const targetManifest = target.get("dd-flow/manifest.json"), oldManifest = old.get("dd-flow/manifest.json");
+            if (!source || source.kind !== "file" || !original || !targetManifest || !oldManifest)
+                refuse("missing_version_context");
+            const from = JSON.parse(oldManifest.bytes.toString("utf8")).canon_version_at_source_commit;
+            const to = JSON.parse(targetManifest.toString("utf8")).canon_version_at_source_commit;
+            if (typeof from !== "string" || typeof to !== "string" || !/^\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.-]+)?$/.test(to))
+                refuse("invalid_target_version");
+            const text = new TextDecoder("utf-8", { fatal: true }).decode(fs.readFileSync(parent(root, name)));
+            const frontmatter = text.match(/^---\n([\s\S]*?)\n---(?:\n|$)/);
+            if (!frontmatter)
+                refuse("version_frontmatter_missing");
+            const document = parseDocument(frontmatter[1]), scalar = document.get("memory_bank_version", true);
+            if (document.errors.length)
+                refuse("version_marker_changed");
+            const oldText = new TextDecoder("utf-8", { fatal: true }).decode(original.bytes);
+            const oldFrontmatter = oldText.match(/^---\n([\s\S]*?)\n---(?:\n|$)/);
+            const oldDocument = oldFrontmatter ? parseDocument(oldFrontmatter[1]) : null;
+            const oldMarker = oldDocument && !oldDocument.errors.length ? oldDocument.get("memory_bank_version", true) : null;
+            // The admitted project marker may legitimately lag its installed pack.
+            // Compare with its frozen document, not a different version context.
+            const markerFrom = isScalar(oldMarker) && typeof oldMarker.value === "string" ? oldMarker.value : from;
+            let updated;
+            if (!document.has("memory_bank_version")) {
+                if (!oldDocument || oldDocument.errors.length || oldDocument.has("memory_bank_version"))
+                    refuse("version_marker_changed");
+                const offset = 4 + frontmatter[1].length;
+                updated = text.slice(0, offset) + "\nmemory_bank_version: " + JSON.stringify(to) + text.slice(offset);
+            }
+            else {
+                if (!isScalar(scalar) || !scalar.range || scalar.value !== markerFrom && scalar.value !== to)
+                    refuse("version_marker_changed");
+                const offset = 4, start = offset + scalar.range[0], end = offset + scalar.range[1];
+                updated = scalar.value === to ? text : text.slice(0, start) + JSON.stringify(to) + text.slice(end);
+            }
+            const bytes = Buffer.from(updated);
+            if (hash(bytes) !== source.sha256) {
+                // Preserve the historical marker document, not merely its version.
+                // If semantic Work changed it, its old bytes remain in the frozen tree;
+                // this archive records that original rather than inventing a template.
+                const archive = `archive/mb-upgrade/${input.selectionSha256}/files/${name}.bin`;
+                if (current.has(archive))
+                    refuse("archive_destination_exists");
+                add(archive, original.bytes, original.mode);
+                archived.push({ path: name, archive, mode: original.mode, sha256: original.sha256 });
+                const count = actions.length;
+                add(name, bytes, source.mode);
+                content.push(...actions.splice(count));
+            }
+        }
+        // Recovery material and its project-owned migration record precede every
+        // destructive effect. They survive worktree cleanup through integration.
+        const record = `archive/mb-upgrade/${input.selectionSha256}/migration.json`;
+        if (current.has(record))
+            refuse("migration_record_exists");
+        add(record, Buffer.from(`${JSON.stringify({ schema_id: "dd-flow/memory-upgrade-content-migration@1",
+            selection_sha256: input.selectionSha256, inventory_sha256: selected.sha256, target_layer_sha256: layer.sha256,
+            path_map: changes.map(({ unit_id, from, to, action, ownership }) => ({ unit_id, from, to, action, ownership })),
+            archived, changes: content.map(action => ({ path: action.path, before: action.before, after: action.after })) }, null, 2)}\n`));
+        actions.push(...content);
+        let finalBytes = before.reduce((size, entry) => size + fs.lstatSync(path.join(root, entry.path)).size, 0);
+        let finalCount = before.length;
+        for (const action of actions) {
+            if (!action.before && fs.lstatSync(path.join(root, action.path), { throwIfNoEntry: false }))
+                refuse("publication_destination_exists");
+            if (action.before) {
+                finalBytes -= fs.lstatSync(path.join(root, action.path)).size;
+                finalCount--;
+            }
+            if (action.after) {
+                finalBytes += Buffer.from(action.base64, "base64").length;
+                finalCount++;
+            }
+        }
+        if (finalBytes > 64 * 1024 * 1024 || finalCount > 20_000)
+            refuse("publication_exceeds_inventory_capacity");
+        const plan = { schema_id: "dd-flow/memory-upgrade-publication@1", workspace, selection_sha256: input.selectionSha256,
+            inventory_sha256: selected.sha256, layer_sha256: layer.sha256, before, actions };
+        return PreparedMemoryUpgradePublication.restore(Buffer.from(JSON.stringify(plan)), hybridDigest(plan));
+    }
+    static restore(bytes, expectedSha256) {
+        try {
+            if (!(bytes instanceof Uint8Array) || bytes.byteLength > 160 * 1024 * 1024 || !/^[a-f0-9]{64}$/.test(expectedSha256))
+                refuse("unbounded_plan");
+            const plan = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+            if (!plan || Object.keys(plan).sort().join() !== "actions,before,inventory_sha256,layer_sha256,schema_id,selection_sha256,workspace"
+                || plan.schema_id !== "dd-flow/memory-upgrade-publication@1" || hybridDigest(plan) !== expectedSha256
+                || [plan.selection_sha256, plan.inventory_sha256, plan.layer_sha256].some(value => typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value))
+                || !path.isAbsolute(plan.workspace) || !Array.isArray(plan.before) || !Array.isArray(plan.actions)
+                || plan.before.length > 20_000 || plan.actions.length > 40_001)
+                refuse("invalid_retained_plan");
+            const state = new Map();
+            let total = 0;
+            const entry = (value) => {
+                if (!value || Object.keys(value).sort().join() !== "kind,mode,path,sha256" || typeof value.path !== "string")
+                    refuse("invalid_retained_entry");
+                logical(value.path);
+                if (!["file", "link"].includes(value.kind) || !Number.isInteger(value.mode) || value.mode < 0 || value.mode > 0o777
+                    || !/^[a-f0-9]{64}$/.test(value.sha256))
+                    refuse("invalid_retained_entry");
+            };
+            for (const value of plan.before) {
+                entry(value);
+                if (state.has(value.path))
+                    refuse("duplicate_retained_entry");
+                state.set(value.path, value);
+            }
+            const paths = new Set();
+            for (const action of plan.actions) {
+                if (!action || Object.keys(action).sort().join() !== "after,base64,before,path" || typeof action.path !== "string")
+                    refuse("invalid_retained_action");
+                logical(action.path);
+                if (paths.has(action.path) || hybridDigest(action.before) !== hybridDigest(state.get(action.path) ?? null))
+                    refuse("invalid_retained_action");
+                if (hybridDigest(action.before) === hybridDigest(action.after))
+                    refuse("unchanged_retained_action");
+                paths.add(action.path);
+                if (action.before && action.before.kind !== "file")
+                    refuse("linked_publication_source");
+                if (action.after) {
+                    entry(action.after);
+                    if (action.after.path !== action.path || action.after.kind !== "file" || typeof action.base64 !== "string"
+                        || action.base64.length > 90 * 1024 * 1024)
+                        refuse("invalid_retained_output");
+                    const content = Buffer.from(action.base64, "base64");
+                    total += content.length;
+                    if (total > 128 * 1024 * 1024 || content.toString("base64") !== action.base64 || hash(content) !== action.after.sha256)
+                        refuse("retained_output_changed");
+                    const physical = action.path.normalize("NFC").toLowerCase();
+                    for (const name of state.keys()) {
+                        const prior = name.normalize("NFC").toLowerCase();
+                        if (name !== action.path && (prior === physical || prior.startsWith(`${physical}/`) || physical.startsWith(`${prior}/`)))
+                            refuse("conflicting_publication_paths");
+                    }
+                    state.set(action.path, action.after);
+                }
+                else {
+                    if (!action.before || action.base64 !== null)
+                        refuse("invalid_retained_removal");
+                    state.delete(action.path);
+                }
+            }
+            return new PreparedMemoryUpgradePublication(plan);
+        }
+        catch (error) {
+            if (error instanceof AppError)
+                throw error;
+            return refuse("invalid_retained_plan");
+        }
+    }
+    bytes() { return Buffer.from(JSON.stringify(this.#plan)); }
+    beforeFiles() { return structuredClone(this.#plan.before); }
+    actions() { return structuredClone(this.#plan.actions); }
+    outputs() { return this.#plan.actions.flatMap(action => action.after ? [structuredClone(action.after)] : []); }
+    classify() {
+        if (fs.realpathSync(this.#plan.workspace) !== this.#plan.workspace)
+            refuse("workspace_changed");
+        const actual = tree(path.join(this.#plan.workspace, ".memory-bank")), observed = new Map(actual.map(entry => [entry.path, entry]));
+        const state = new Map(this.#plan.before.map(entry => [entry.path, entry]));
+        let completed = 0, pending = false;
+        for (const action of this.#plan.actions) {
+            const value = observed.get(action.path) ?? null;
+            if (hybridDigest(value) === hybridDigest(action.after)) {
+                if (pending)
+                    refuse("unknown_publication_prefix");
+                completed++;
+                if (action.after)
+                    state.set(action.path, action.after);
+                else
+                    state.delete(action.path);
+            }
+            else if (hybridDigest(value) === hybridDigest(action.before))
+                pending = true;
+            else
+                refuse("unknown_publication_state");
+        }
+        const expected = [...state.values()].sort((a, b) => a.path.localeCompare(b.path, "en"));
+        if (hybridDigest(actual) !== hybridDigest(expected))
+            refuse("unknown_publication_state");
+        for (const action of this.#plan.actions) {
+            const entry = state.get(action.path), stat = fs.lstatSync(path.join(this.#plan.workspace, ".memory-bank", action.path), { throwIfNoEntry: false });
+            if (entry ? !stat?.isFile() : stat !== undefined)
+                refuse("unexpected_publication_destination_kind");
+        }
+        return { completed, total: this.#plan.actions.length };
+    }
+    applyNextAction() {
+        const workspace = this.#plan.workspace;
+        if (fs.realpathSync(workspace) !== workspace)
+            refuse("workspace_changed");
+        const namespace = path.dirname(parent(workspace, ".dd-flow/memory-upgrade-publications/owner", true));
+        const lock = acquireInstallationLock(path.join(namespace, "workspace.lock"), this.sha256);
+        try {
+            lock.assertOwnership();
+            const journal = path.dirname(parent(namespace, `${this.sha256}/plan.json`, true, 0o700));
+            if ((fs.lstatSync(journal).mode & 0o7777) !== 0o700)
+                refuse("unsafe_journal_mode");
+            const planFile = path.join(journal, "plan.json");
+            if (!fs.lstatSync(planFile, { throwIfNoEntry: false }) && this.classify().completed !== 0)
+                refuse("publication_journal_missing");
+            publishImmutableInput(planFile, this.bytes(), "memory_upgrade_publication_invalid");
+            // A crash between link and unlink leaves two names for our complete
+            // staged addition. Remove only that exact journal alias, never user data.
+            for (const [index, item] of this.#plan.actions.entries())
+                if (item.after && !item.before) {
+                    const slot = path.join(journal, `action-${index}.tmp`), destination = path.join(workspace, ".memory-bank", item.path);
+                    const staged = fs.lstatSync(slot, { throwIfNoEntry: false }), published = fs.lstatSync(destination, { throwIfNoEntry: false });
+                    if (staged && published && staged.dev === published.dev && staged.ino === published.ino) {
+                        if (!staged.isFile() || staged.nlink !== 2 || (staged.mode & 0o777) !== item.after.mode
+                            || hash(fs.readFileSync(slot)) !== item.after.sha256)
+                            refuse("unknown_linked_staging_file");
+                        fs.unlinkSync(slot);
+                        syncDirectory(journal);
+                    }
+                }
+            const view = this.classify();
+            for (const item of this.#plan.actions.slice(0, view.completed))
+                syncParents(path.join(workspace, ".memory-bank"), item.path);
+            if (view.completed === view.total)
+                return { ...view, status: "applied" };
+            const action = this.#plan.actions[view.completed], root = path.join(workspace, ".memory-bank");
+            const destination = parent(root, action.path, true);
+            let staged;
+            if (action.after) {
+                const bytes = Buffer.from(action.base64, "base64");
+                staged = stageRetainedMemoryRepairFile(journal, `action-${view.completed}.tmp`, bytes, action.after.mode).path;
+                verifyRetainedMemoryRepairFile(staged, bytes, action.after.mode);
+            }
+            if (this.classify().completed !== view.completed)
+                refuse("publication_cursor_changed");
+            lock.assertOwnership();
+            if (staged) {
+                if (action.before)
+                    fs.renameSync(staged, destination);
+                else {
+                    fs.linkSync(staged, destination);
+                    fs.unlinkSync(staged);
+                }
+            }
+            else
+                fs.unlinkSync(destination);
+            syncParents(root, action.path);
+            syncDirectory(journal);
+            const after = this.classify();
+            if (after.completed !== view.completed + 1)
+                refuse("publication_readback_failed");
+            return { ...after, status: after.completed === after.total ? "applied" : "pending" };
+        }
+        catch (error) {
+            throw new AppError("memory_upgrade_publication_pending", "Preserve the retained plan and reconcile its exact filesystem prefix", 1, { effect: "publication_pending", publication_sha256: this.sha256, cause: error instanceof Error ? error.message : String(error) });
+        }
+        finally {
+            lock.release();
+        }
+    }
+}