@deksden-com/dd-flow-cli 0.9.0-beta.144 → 0.9.0-beta.149

package/dist/services/memory-upgrade-bootstrap-readiness.js

@@ -0,0 +1,251 @@
+import { createHash } from "node:crypto";
+import { AppError } from "../shared/errors.js";
+import { appendAudit } from "./audit.js";
+import { hybridDigest, validateHybridContract } from "./hybrid-contracts.js";
+import { prepareMemoryInitBootstrapSchema } from "./memory-init-bootstrap-readiness.js";
+import { restoreMemoryUpgradeSelectionData } from "./memory-upgrade-selection.js";
+import { PreparedUpgradeStorageReadiness, readUpgradeStorageReadiness, upgradeStorageMigration, resolveUpgradeHistoricalScope, assertUpgradeExecutionStorage } from "./memory-upgrade-storage-readiness.js";
+import { RetainedExecutionBundle } from "./execution-bundles.js";
+import { assertPreparedRunBundleSqlAuthority } from "./runs.js";
+import { runContextRelocations } from "./stage-context.js";
+import semver from "semver";
+import { PreparedMemoryInitLintCapability, isMemoryInitLintCapabilityFacts } from "./memory-init-lint-capability.js";
+import { inspectMemoryLintExecutable } from "./memory-init-preflight.js";
+import { readBoundedReleaseFile } from "./release-source.js";
+const eventType = "memory.bootstrap_ready";
+const bytes = (value) => Buffer.from(JSON.stringify(value));
+const hash = (value) => createHash("sha256").update(value).digest("hex");
+function refuse(reason) {
+    throw new AppError("memory_upgrade_bootstrap_readiness_invalid", "Upgrade bootstrap requires its retained selection and accepted storage", 1, { reason, effect: "no_effect", next_action: "reconcile_exact_request" });
+}
+/** SQL-only construction. This receipt authorizes execution bootstrap, not
+ * semantic acceptance, canonical publication or a substitute handler registry. */
+function material(context, binding) {
+    const retained = readUpgradeStorageReadiness(context, binding.scope);
+    if (!retained?.selection || retained.sha256 !== binding.storage_sha256
+        || retained.selection.sha256 !== binding.selection_sha256)
+        refuse("storage_selection_changed");
+    const p = retained.selection.payload, release = retained.selection.release;
+    if (binding.flow !== undefined) {
+        const facts = { schema_id: "dd-flow/memory-upgrade-engine-flow@1", flow_id: "mb-upgrade", source_sha256: p.source.sha256,
+            descriptor_sha256: p.execution.snapshot.settings.descriptor_sha256,
+            stages: ["preflight", "diff-analysis", "upgrade", "verify", "review", "finish"] };
+        if (hybridDigest(binding.flow) !== hybridDigest({ facts, sha256: hybridDigest(facts),
+            engine_manifest_sha256: p.engine.manifest_sha256, engine_sha256: p.engine.identity.integrity_checksum }))
+            refuse("selected_engine_flow_changed");
+    }
+    if (binding.lint !== undefined) {
+        if (!isMemoryInitLintCapabilityFacts(binding.lint))
+            refuse("invalid_lint_capability");
+        const compatibility = p.layer.files.find(file => file.path === "dd-flow/compatibility.json");
+        const minimum = compatibility ? JSON.parse(Buffer.from(compatibility.base64, "base64").toString("utf8")).mb_lint?.min_version : null;
+        if (typeof minimum !== "string" || !semver.valid(minimum) || semver.lt(binding.lint.version, minimum))
+            refuse("selected_linter_below_target_minimum");
+    }
+    const target = retained.receipt.proof.target;
+    if (!target || typeof target.pack_sha256 !== "string" || !/^[a-f0-9]{64}$/.test(target.pack_sha256))
+        refuse("invalid_target_pack");
+    if (binding.execution_sha256 !== p.execution.sha256 || typeof binding.engine_manifest_base64 !== "string"
+        || binding.engine_manifest_base64.length > 2 * 1024 * 1024)
+        refuse("execution_binding_changed");
+    const manifestBytes = Buffer.from(binding.engine_manifest_base64, "base64");
+    if (manifestBytes.toString("base64") !== binding.engine_manifest_base64 || hash(manifestBytes) !== p.engine.manifest_sha256)
+        refuse("engine_manifest_changed");
+    const manifest = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(manifestBytes));
+    if (manifest.integrity?.checksum !== p.engine.identity.integrity_checksum
+        || release.engine.package !== manifest.package_name || release.engine.version !== manifest.package_version)
+        refuse("foreign_engine");
+    const scope = binding.scope, fact = { requestId: scope.requestId, projectRoot: scope.projectRoot, workspace: scope.workspaceRoot };
+    const baseline = hybridDigest({ inventory_sha256: p.current.inventory.sha256, git: p.workspace });
+    const evidenceValue = { schema_id: "dd-flow/memory-upgrade-storage-evidence@1", scope,
+        storage_sha256: retained.sha256, proof: retained.receipt.proof };
+    const evidence = [{ kind: "storage-compatibility", path: "receipts/storage-compatibility.json",
+            sha256: hash(bytes(evidenceValue)), base64: bytes(evidenceValue).toString("base64") }];
+    const migration = retained.receipt.proof.route === "migration" ? upgradeStorageMigration(retained.receipt.proof.migration, retained.selection.sha256) : undefined;
+    if (migration)
+        evidence.push(...migration.proof.evidence);
+    const reference = (kind) => { const entry = evidence.find(entry => entry.kind === kind); return entry ? { path: entry.path, sha256: entry.sha256 } : null; };
+    const receipt = { schema_id: "dd-flow/memory-bootstrap-receipt@2", request_id: scope.requestId, flow_id: "mb-upgrade",
+        project_root: scope.projectRoot, workspace: { path: scope.workspaceRoot, repository: scope.projectRoot,
+            branch: p.workspace.branch, commit: p.workspace.head, git_state: p.workspace.head === null ? "unborn" : "committed" },
+        baseline_sha256: baseline,
+        project_current: { canon_version: p.current.canon_version, pack_sha256: hybridDigest(p.current.inventory.inventory.entries
+                .filter(entry => entry.path.startsWith("dd-flow/") && entry.current)
+                .map(entry => ({ path: entry.path, sha256: entry.current.sha256 }))) },
+        upgrade_target: { canon_version: release.canon.version, pack_sha256: target.pack_sha256 },
+        effective_execution: { engine_sha256: p.engine.identity.integrity_checksum, canon_sha256: release.canon.archive.sha256 },
+        readiness: "ready", storage: { route: migration ? "migration" : "verified-no-op", outcome: migration ? "applied" : "no-op",
+            compatibility_receipt: { path: evidence[0].path, sha256: evidence[0].sha256 },
+            drain_receipt: reference("writer-drain"), backup_receipt: reference("storage-backup"), plan_receipt: reference("storage-plan"),
+            apply_receipt: reference("storage-apply"), verification_receipt: reference("storage-verification") },
+        effects: migration ? [{ kind: "storage-apply", receipt: reference("storage-apply") }] : [] };
+    const admission = { receipts: new Map(evidence.map(entry => [entry.sha256, { kind: entry.kind, ...fact }])),
+        bootstrapIdentity: { ...fact, baselineSha256: baseline, engineSha256: p.engine.identity.integrity_checksum,
+            canonSha256: release.canon.archive.sha256,
+            workspaceGit: { repository: scope.projectRoot, branch: p.workspace.branch, commit: p.workspace.head } } };
+    const authority = { schema_id: "dd-flow/memory-bootstrap-authority@1", request_id: scope.requestId,
+        project_root: scope.projectRoot, workspace_root: scope.workspaceRoot, flow_id: "mb-upgrade", readiness: "ready",
+        model_access: "unknown", bootstrap_sha256: hash(bytes(receipt)), binding, receipt, evidence };
+    return { authority, admission, projectId: retained.receipt.project_id, hasPreservationBaseline: p.current.old_tree !== undefined,
+        hasLintCapability: binding.lint !== undefined, hasGitPolicy: p.git !== undefined, hasFlowCapability: binding.flow !== undefined };
+}
+/** Accepted storage is the independent input record. A lone bootstrap marker
+ * cannot manufacture its selection, execution or receipt bytes. No filesystem. */
+export function readMemoryUpgradeBootstrapReadiness(context, scope) {
+    try {
+        return context.db.readSnapshot(() => {
+            const query = (logical) => context.db.all("SELECT project_id,payload_json,created_at FROM audit_events WHERE event_type=? AND json_extract(payload_json,'$.request_id')=? AND json_extract(payload_json,'$.project_root')=? AND json_extract(payload_json,'$.workspace_root')=? LIMIT 2", [eventType, logical.requestId, logical.projectRoot, logical.workspaceRoot]);
+            let logical = scope, rows = query(scope);
+            if (!rows.length) {
+                try {
+                    logical = resolveUpgradeHistoricalScope(context, scope).scope;
+                }
+                catch (error) {
+                    if (error instanceof AppError && error.details?.reason === 'project_not_active')
+                        return null;
+                    throw error;
+                }
+                rows = query(logical);
+            }
+            if (!rows.length)
+                return null;
+            if (rows.length !== 1 || Buffer.byteLength(rows[0].payload_json) > 4 * 1024 * 1024)
+                refuse("ambiguous_bootstrap_authority");
+            const value = JSON.parse(rows[0].payload_json), binding = value.binding;
+            if (!binding || Object.keys(binding).filter(key => key !== "lint" && key !== "flow").sort().join() !== "engine_manifest_base64,execution_sha256,scope,selection_sha256,storage_sha256"
+                || hybridDigest(binding.scope) !== hybridDigest(logical))
+                refuse("invalid_bootstrap_binding");
+            const selected = material(context, binding);
+            if (rows[0].project_id !== selected.projectId || hybridDigest(value) !== hybridDigest(selected.authority))
+                refuse("bootstrap_authority_changed");
+            return { ...selected, createdAt: rows[0].created_at };
+        });
+    }
+    catch (error) {
+        if (error instanceof AppError)
+            throw error;
+        return refuse("invalid_bootstrap_authority");
+    }
+}
+export class PreparedMemoryUpgradeBootstrapReadiness {
+    #binding;
+    #material;
+    #lint;
+    constructor(binding, selected, lint) {
+        this.#binding = structuredClone(binding);
+        this.#material = selected;
+        this.#lint = lint;
+        Object.freeze(this);
+    }
+    /** A relocated, already allocated RUN keeps original bootstrap/migration
+     * bytes. Admit its current stores without replaying the old cutover in the
+     * new Home. This never allocates a RUN or creates a new readiness receipt. */
+    static assertForRetainedRun(context, bundle) {
+        if (!(bundle instanceof RetainedExecutionBundle) || bundle.manifest.flow_id !== "mb-upgrade")
+            refuse("retained_upgrade_bundle_required");
+        const frozen = bundle.execution.snapshot().scope;
+        const scope = { requestId: frozen.requestId, projectRoot: frozen.projectRoot, workspaceRoot: frozen.workspace };
+        const { projectId } = resolveUpgradeHistoricalScope(context, scope);
+        assertPreparedRunBundleSqlAuthority(context, { projectId, runId: bundle.manifest.run_id }, bundle);
+        if (!runContextRelocations(context, projectId, bundle.manifest.run_id).length) {
+            this.prepare(context, scope);
+            return;
+        }
+        const ready = readMemoryUpgradeBootstrapReadiness(context, scope), reference = bundle.manifest.bootstrap_receipts[0];
+        if (!ready?.hasPreservationBaseline || !ready.hasLintCapability || !ready.hasFlowCapability
+            || ready.projectId !== projectId || ready.authority.binding.execution_sha256 !== bundle.execution.sha256
+            || bundle.manifest.bootstrap_receipts.length !== 1 || reference?.sha256 !== ready.authority.bootstrap_sha256
+            || !bundle.blob(reference.path).equals(bytes(ready.authority.receipt)))
+            refuse("retained_bootstrap_changed");
+        assertUpgradeExecutionStorage(context, bundle.engine());
+    }
+    /** Full productive prerequisite path; never install or repair global tools.
+     * Exact replay retains the first functional qualification instead of probing
+     * again after effects or choosing a different executable from today's PATH. */
+    static prepareForRun(context, scope) {
+        if (context.db.isOwnedWriteTransaction?.())
+            refuse("preparation_outside_writer_required");
+        const storage = readUpgradeStorageReadiness(context, scope), git = storage?.selection?.payload.git;
+        if (!git)
+            refuse("historical_selection_requires_new_request");
+        if (hash(readBoundedReleaseFile(git.executable, 64 * 1024 * 1024)) !== git.probe.executable_sha256)
+            refuse("selected_git_changed");
+        const prior = readMemoryUpgradeBootstrapReadiness(context, scope);
+        if (prior) {
+            const lint = prior.authority.binding.lint;
+            if (!lint || !prior.hasFlowCapability)
+                refuse("historical_bootstrap_requires_new_request");
+            if (hash(readBoundedReleaseFile(lint.executable, 64 * 1024 * 1024)) !== lint.executable_sha256)
+                refuse("selected_linter_changed");
+            return this.prepare(context, scope);
+        }
+        if (!storage?.receipt.selection)
+            refuse("accepted_storage_required");
+        const retained = storage.receipt.selection;
+        const restored = restoreMemoryUpgradeSelectionData(Buffer.from(retained.payload_json), retained.sha256, retained.request_input, scope);
+        const flow = restored.engine.admitMemoryUpgradeFlow(restored.source);
+        const selected = inspectMemoryLintExecutable(context.env);
+        return this.#prepare(context, scope, PreparedMemoryInitLintCapability.prepareExecutable(selected.executable, selected.probe, context.env, { temporaryConfiguration: true }), flow);
+    }
+    static prepare(context, scope, lint) {
+        if (context.db.isOwnedWriteTransaction?.())
+            refuse("preparation_outside_writer_required");
+        if (lint !== undefined && !(lint instanceof PreparedMemoryInitLintCapability))
+            refuse("prepared_lint_capability_required");
+        return this.#prepare(context, scope, lint, readMemoryUpgradeBootstrapReadiness(context, scope)?.authority.binding.flow);
+    }
+    static #prepare(context, scope, lint, flow) {
+        if (context.db.isOwnedWriteTransaction?.())
+            refuse("preparation_outside_writer_required");
+        if (lint !== undefined && !(lint instanceof PreparedMemoryInitLintCapability))
+            refuse("prepared_lint_capability_required");
+        lint?.revalidateExecutable();
+        const lintFacts = lint?.facts() ?? readMemoryUpgradeBootstrapReadiness(context, scope)?.authority.binding.lint;
+        // Re-observe physical admission and the original old-tree before issuing
+        // fresh execution authority, even if a historical storage receipt exists.
+        PreparedUpgradeStorageReadiness.prepareFromSelection(context, scope);
+        const storage = readUpgradeStorageReadiness(context, scope);
+        if (!storage?.selection || !storage.receipt.selection)
+            refuse("accepted_storage_required");
+        const retained = storage.receipt.selection;
+        const selected = restoreMemoryUpgradeSelectionData(Buffer.from(retained.payload_json, "utf8"), retained.sha256, retained.request_input, scope);
+        if (!selected.oldTree)
+            refuse("preservation_baseline_required");
+        const validate = prepareMemoryInitBootstrapSchema(selected.engine);
+        const binding = { scope: { ...scope }, storage_sha256: storage.sha256, selection_sha256: selected.sha256,
+            execution_sha256: selected.execution.sha256, engine_manifest_base64: selected.engine.manifestBytes().toString("base64"), ...(lintFacts ? { lint: lintFacts } : {}),
+            ...(flow ? { flow } : {}) };
+        const prepared = context.db.readSnapshot(() => material(context, binding));
+        if (!validate(prepared.authority.receipt)
+            || validateHybridContract("memory-bootstrap-receipt", prepared.authority.receipt, prepared.admission).length)
+            refuse("selected_engine_bootstrap_contract_invalid");
+        return new PreparedMemoryUpgradeBootstrapReadiness(binding, prepared, lint);
+    }
+    apply(context) {
+        const prior = readMemoryUpgradeBootstrapReadiness(context, this.#binding.scope);
+        if (prior) {
+            if (hybridDigest(prior.authority) !== hybridDigest(this.#material.authority))
+                refuse("bootstrap_request_changed");
+            return { ...prior, changed: false };
+        }
+        const current = PreparedMemoryUpgradeBootstrapReadiness.#prepare(context, this.#binding.scope, this.#lint, this.#binding.flow);
+        if (hybridDigest(current.#material.authority) !== hybridDigest(this.#material.authority))
+            refuse("bootstrap_changed_after_preparation");
+        return context.db.writeTransaction(() => {
+            const existing = readMemoryUpgradeBootstrapReadiness(context, this.#binding.scope);
+            if (existing) {
+                if (hybridDigest(existing.authority) !== hybridDigest(this.#material.authority))
+                    refuse("bootstrap_request_changed");
+                return { ...existing, changed: false };
+            }
+            const selected = material(context, this.#binding);
+            if (hybridDigest(selected.authority) !== hybridDigest(this.#material.authority))
+                refuse("bootstrap_dependencies_changed");
+            appendAudit(context, { projectId: selected.projectId, eventType, payload: selected.authority });
+            const accepted = readMemoryUpgradeBootstrapReadiness(context, this.#binding.scope);
+            if (!accepted)
+                refuse("bootstrap_readback_failed");
+            return { ...accepted, changed: true };
+        }, "memoryUpgrade.bootstrapReadiness");
+    }
+}

package/dist/services/memory-upgrade-evidence.js

@@ -0,0 +1,100 @@
+import { createHash } from "node:crypto";
+import { AppError } from "../shared/errors.js";
+import { readUpgradeStorageReadiness } from "./memory-upgrade-storage-readiness.js";
+import { readUpgradePreservationBaseline, readUpgradeProjectFiles } from "./memory-upgrade-inventory.js";
+import { localEnvironmentAllowed } from "./workspace-files.js";
+import { PreparedMemoryLayer } from "./memory-layer.js";
+import { readRunStartRequest } from "./run-start-custody.js";
+export const upgradeEvidenceIndexPath = "upgrade-baseline.json";
+const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
+export const upgradeEvidenceFilePath = (sha256) => `upgrade-baseline-${sha256}.bin`;
+export const upgradeTargetIndexPath = "upgrade-target.json";
+export const upgradeTargetFilePath = (sha256) => `upgrade-target-${sha256}.bin`;
+function refuse(reason) {
+    throw new AppError("memory_upgrade_evidence_invalid", "Old document evidence requires the retained upgrade selection", 2, { reason, effect: "no_effect" });
+}
+/** SQL-selected old bytes, not today's workspace or agent-proposed paths.
+ * RUN start publishes these using its existing retained artifact/replay owner.
+ * Equal content shares a copy; the index retains original paths and modes.
+ * Links are never followed; private/environment files are never materialized.
+ * Optional link identities are frozen literal metadata inside this regular index.
+ */
+export function readMemoryUpgradeEvidence(context, scope) {
+    const selected = readUpgradeStorageReadiness(context, scope)?.selection;
+    if (!selected?.payload.current.old_tree)
+        refuse("retained_old_tree_required");
+    const files = readUpgradePreservationBaseline(selected.payload.current.old_tree, selected.payload.current.inventory);
+    const artifacts = new Map();
+    const entries = [];
+    const projectFiles = readUpgradeProjectFiles(selected.payload.current.project_files ?? []);
+    const sources = [...files].map(([logical, file]) => [`.memory-bank/${logical}`, file]);
+    const excluded = sources.filter(([logical, file]) => file.kind !== "file" || logical.split("/").some(part => [".git", "node_modules", ".dd-flow"].includes(part) || !localEnvironmentAllowed(part))).length;
+    for (const file of projectFiles)
+        sources.push([file.path, { kind: "file", bytes: Buffer.from(file.base64, "base64"), mode: file.mode, sha256: file.sha256 }]);
+    for (const [logical, file] of sources) {
+        if (file.kind !== "file" || logical.split("/").some(part => [".git", "node_modules", ".dd-flow"].includes(part) || !localEnvironmentAllowed(part)))
+            continue;
+        if (file.bytes.length > 16 * 1024 * 1024)
+            refuse("old_document_exceeds_read_bound");
+        const retained = { root: "run", path: upgradeEvidenceFilePath(file.sha256), sha256: file.sha256 };
+        artifacts.set(retained.path, file.bytes);
+        entries.push({ source: { root: "workspace", path: logical, sha256: file.sha256 }, retained, mode: file.mode });
+    }
+    const index = { schema_id: "dd-flow/memory-upgrade-evidence@1", selection_sha256: selected.sha256,
+        ...(selected.payload.current.project_links?.length ? { project_link_metadata: selected.payload.current.project_links } : {}),
+        entries, excluded_entries: excluded };
+    const bytes = Buffer.from(JSON.stringify(index));
+    if (bytes.length > 16 * 1024 * 1024)
+        refuse("evidence_index_exceeds_read_bound");
+    const reference = { root: "run", path: upgradeEvidenceIndexPath, sha256: hash(bytes) };
+    artifacts.set(reference.path, bytes);
+    // Custody readers need the index, not publication copies. Keep full source
+    // validation above, and encode the deduplicated artifacts only on demand.
+    let encoded;
+    return { index, reference, get artifacts() {
+            if (!encoded) {
+                encoded = [...artifacts].map(([path, bytes]) => ({ path, base64: bytes.toString("base64") }));
+                artifacts.clear();
+            }
+            return encoded;
+        } };
+}
+/** Target guidance is distinct from old project evidence. Only frozen canonical
+ * layer bytes are exported, never today's installation or a caller's files. */
+export function readMemoryUpgradeTargetEvidence(context, scope) {
+    const selected = readUpgradeStorageReadiness(context, scope)?.selection;
+    if (!selected)
+        refuse("retained_target_required");
+    const payload = selected.payload, frozen = payload.layer;
+    const files = PreparedMemoryLayer.restoreRetained(new Map(frozen.files.map(file => [file.path, Buffer.from(file.base64, "base64")])), frozen.sha256, frozen.installation_sha256).files();
+    const artifacts = new Map();
+    const entries = [...files].map(([logical, bytes]) => {
+        const sha256 = hash(bytes), retained = { root: "run", path: upgradeTargetFilePath(sha256), sha256 };
+        artifacts.set(retained.path, { path: retained.path, base64: bytes.toString("base64") });
+        return { path: `.memory-bank/${logical}`, retained };
+    });
+    const index = { schema_id: "dd-flow/memory-upgrade-target@1", selection_sha256: selected.sha256,
+        source_version: payload.current.canon_version, target_version: selected.release.canon.version,
+        ...(payload.current.flow_pack_canon_version ? { source_flow_pack_version: payload.current.flow_pack_canon_version } : {}),
+        ...(payload.current.version_source ? { version_marker_update: { path: `.memory-bank/${payload.current.version_source}`,
+                field: "memory_bank_version", target_version: selected.release.canon.version, operation: "cli_marker_only" } } : {}),
+        layer_sha256: frozen.sha256, impact: payload.impact, inventory: payload.current.inventory, entries };
+    const bytes = Buffer.from(JSON.stringify(index));
+    if (bytes.length > 16 * 1024 * 1024)
+        refuse("target_index_exceeds_read_bound");
+    const reference = { root: "run", path: upgradeTargetIndexPath, sha256: hash(bytes) };
+    artifacts.set(reference.path, { path: reference.path, base64: bytes.toString("base64") });
+    return { index, reference, artifacts: [...artifacts.values()] };
+}
+/** Older RUNs keep their original packets; presence comes from allocated SQL
+ * custody, not a stray file or today's target selection. */
+export function readIssuedMemoryUpgradeTarget(context, scope, projectId) {
+    const request = readRunStartRequest(context, projectId, scope.requestId);
+    const artifact = request?.payload.artifacts.find(file => file.path === upgradeTargetIndexPath);
+    if (!artifact)
+        return null;
+    const target = readMemoryUpgradeTargetEvidence(context, scope);
+    if (artifact.base64 !== target.artifacts.find(file => file.path === upgradeTargetIndexPath).base64)
+        refuse("target_allocation_changed");
+    return target;
+}