@timurproko/a1 0.1.8-dev.235df0e → 0.1.8-dev.259

package/dist/{src/foundation → foundation}/release/bootstrap.js

@@ -1,45 +1,90 @@
 import { spawn } from "node:child_process";
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { connect } from "node:net";
 import { platform } from "node:os";
 import { resolve } from "node:path";
 import { selectCohortLaunch } from "./cohort-selection.js";
 import { CohortStateStore } from "./cohort-state.js";
-import { assertLaunchProfileId, resolveProductPaths } from "../lifecycle/index.js";
+import { assertLaunchProfileId, createSupervisorStartupAttempt, readSupervisorStartupResult, resolveCohortEndpoint, resolveProductPaths, sessionSelectionArguments } from "../lifecycle/index.js";
 import { encodeFrame, LineFrameDecoder } from "../protocol/index.js";
 import { cleanupProvenIdleOwner, processIsAlive } from "./process-cleanup.js";
+import { sweepDeadEndpoints } from "./endpoints.js";
 import { consumeMaterializationProof, materializeRelease, readCertifiedReleaseManifest, readMaterializedRelease, resolveReleaseEntryPoint, verifyMaterializedRelease } from "./release-store.js";
+import { scheduleReleaseCleanup } from "./release-gc.js";
+import { createRestartSeal, readRestartCertifiedRelease, releaseCertificationDocument } from "./restart-certification.js";
 import { PRODUCT_IDENTITY, PRODUCT_TEXT } from "../../product-identity.js";
+import { markStartupPhase } from "../startup/index.js";
 export async function runBootstrap(options) {
     const environment = { ...(options.environment ?? process.env) };
-    const launchProfileId = options.launchIntent?.profile.id ?? "a1";
+    await markStartupPhase(environment, "bootstrap-start");
+    const launchProfileId = options.launchIntent?.profileId ?? "a1";
     assertLaunchProfileId(launchProfileId);
+    const sessionArgs = sessionSelectionArguments(options.launchIntent?.sessionSelection);
+    if (sessionArgs.length > 0 && launchProfileId !== "a1")
+        throw new Error("session selection requires the normal A1 profile");
     environment[PRODUCT_IDENTITY.environment.launchProfile] = launchProfileId;
     const output = options.output ?? process.stderr;
     const paths = resolveProductPaths(environment);
     await mkdir(paths.runtimeDir, { recursive: true, mode: 0o700 });
     const stateStore = new CohortStateStore(paths.dataDir);
     let state = await stateStore.read();
-    let endpoint = await readEndpointMetadata(paths.endpointMetadataPath);
+    // Invariant: records left by cohorts whose processes are gone say nothing about ownership, and there
+    // can now be several of them. Clearing them first keeps the decision below about what is
+    // actually running.
+    await sweepDeadEndpoints(paths).catch(() => []);
+    // Performance: reconciliation is serialized by cohort state, but launch never awaits a physical
+    // backlog. The scheduler commits a plan and starts a detached bounded worker.
+    const scheduleMaintenance = options.scheduleMaintenance ?? scheduleReleaseCleanup;
+    void scheduleMaintenance(paths.dataDir, paths).catch(() => { });
+    // Protocol: each cohort keeps its own endpoint, so a launch looks for the endpoint of the release it
+    // is about to run rather than for the one endpoint the runtime directory used to have.
+    const activeReleaseId = state.references.active;
+    let endpointPaths = activeReleaseId === null
+        ? { endpoint: paths.endpoint, endpointMetadataPath: paths.endpointMetadataPath }
+        : resolveCohortEndpoint(paths, activeReleaseId, environment);
+    let endpoint = await readEndpointMetadata(endpointPaths.endpointMetadataPath);
+    if (endpoint === null) {
+        // Compatibility: a release that predates cohort-scoped endpoints published one endpoint for the whole
+        // runtime directory. Recognizing it is what lets a session started by that release keep
+        // working through the first launch that knows about cohorts.
+        const legacy = await readEndpointMetadata(paths.endpointMetadataPath);
+        if (legacy !== null) {
+            endpoint = legacy;
+            endpointPaths = { endpoint: paths.endpoint, endpointMetadataPath: paths.endpointMetadataPath };
+        }
+    }
     let probe = endpoint ? await probeOwnership(endpoint) : "dead";
     const installedVersion = await readInstalledVersion(options.packageRoot);
     const activeId = state.references.active;
     const active = activeId === null ? undefined : state.releases[activeId];
-    if (active?.approval === "approved" && active.packageVersion === installedVersion) {
+    const activeIsLaunchable = active === undefined || (options.releaseIsLaunchable?.(active.releaseRoot) ?? true);
+    if (activeIsLaunchable && active?.approval === "approved" && active.packageVersion === installedVersion) {
         const endpointMatches = endpoint?.releaseId === active.releaseId
             && endpoint.releaseRoot === active.releaseRoot
             && endpoint.contentDigest === active.contentDigest;
         if (endpointMatches && probe === "live-verified") {
             const retained = await readCertifiedReleaseManifest(active, resolve(paths.dataDir, "releases"));
-            return await launchUi(retained, environment);
+            return await launchUi(retained, environment, sessionArgs);
         }
         if (endpoint === null || probe === "dead") {
             if (endpoint)
-                await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
-            const retained = await readMaterializedRelease(active.releaseRoot);
-            await startSupervisor(retained, environment);
-            await waitForVerifiedEndpoint(paths.endpointMetadataPath, retained, 8_000);
-            return await launchUi(retained, environment);
+                await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
+            await markStartupPhase(environment, "durable-validation-start");
+            const retained = await readRestartCertifiedRelease(active, paths.dataDir).catch(async () => {
+                const verified = await readMaterializedRelease(active.releaseRoot, resolve(paths.dataDir, "releases"));
+                if (verified.releaseId !== active.releaseId || verified.contentDigest !== active.contentDigest || verified.packageVersion !== active.packageVersion) {
+                    throw new Error("completely verified release differs from the approved active record");
+                }
+                await certifyMaterializedRelease(verified, paths.dataDir);
+                return verified;
+            });
+            await markStartupPhase(environment, "durable-validation-complete");
+            const retainedPaths = resolveCohortEndpoint(paths, retained.releaseId, environment);
+            await markStartupPhase(environment, "replacement-supervisor-start");
+            const startup = await startSupervisor(retained, environment);
+            await waitForVerifiedEndpoint(retainedPaths.endpointMetadataPath, retained, 8_000, startup);
+            await markStartupPhase(environment, "replacement-supervisor-ready");
+            return await launchUi(retained, environment, sessionArgs);
         }
     }
     const candidate = await materializeRelease(options.packageRoot, paths.dataDir);
@@ -51,7 +96,28 @@ export async function runBootstrap(options) {
         await stateStore.activate(candidate.releaseId);
         state = await stateStore.read();
     }
-    endpoint = await readEndpointMetadata(paths.endpointMetadataPath);
+    else if (!activeIsLaunchable && candidate.releaseId !== activeId
+        && state.releases[candidate.releaseId]?.approval !== "approved") {
+        // Invariant: the active reference points at a copy that cannot launch, so reusing it
+        // is off the table — but an unapproved candidate would lose the selection
+        // below to that same broken active (`start-active`). Approving the healed
+        // candidate here lets ordinary cohort selection activate it, while a live
+        // busy cohort still wins the endpoint checks and keeps its sessions.
+        const diagnosticsPath = await certifyMaterializedRelease(candidate, paths.dataDir);
+        await stateStore.approve(candidate.releaseId, diagnosticsPath);
+        state = await stateStore.read();
+    }
+    // Protocol: the candidate's own endpoint decides whether this launch attaches or starts a supervisor.
+    // A cohort other than this one is not in the way: it listens somewhere else.
+    endpointPaths = resolveCohortEndpoint(paths, candidate.releaseId, environment);
+    endpoint = await readEndpointMetadata(endpointPaths.endpointMetadataPath);
+    if (endpoint === null) {
+        const legacy = await readEndpointMetadata(paths.endpointMetadataPath);
+        if (legacy !== null) {
+            endpoint = legacy;
+            endpointPaths = { endpoint: paths.endpoint, endpointMetadataPath: paths.endpointMetadataPath };
+        }
+    }
     probe = endpoint ? await probeOwnership(endpoint) : "dead";
     let decision = selectCohortLaunch(candidate, state, endpoint, probe);
     if (decision.action === "blocked") {
@@ -64,18 +130,18 @@ export async function runBootstrap(options) {
         await writeFile(resolve(paths.dataDir, `cleanup-${Date.now()}.json`), JSON.stringify(diagnostics, null, 2));
         if (!diagnostics.terminated)
             throw new Error(`could not safely clean stale ${PRODUCT_TEXT.displayName} supervisor ${endpoint.pid}`);
-        await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
+        await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
         decision = selectCohortLaunch(candidate, await stateStore.read(), null, "dead");
     }
     else if (endpoint && probe === "dead") {
-        await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
+        await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
     }
     if (decision.action === "launch-retained-ui") {
         const retained = await verifyMaterializedRelease(decision.releaseRoot, undefined, resolve(paths.dataDir, "releases"));
         if (decision.recordPending && endpoint) {
             await stateStore.blockPending("candidate activation deferred by live non-resumable instances", endpoint.ownership.nonResumableInstanceIds);
         }
-        const code = await launchUi(retained, environment);
+        const code = await launchUi(retained, environment, sessionArgs);
         if (decision.recordPending)
             await activatePendingAfterBlockerExit(candidate, stateStore, paths, environment);
         return code;
@@ -94,7 +160,7 @@ export async function runBootstrap(options) {
             output.write(`${PRODUCT_TEXT.diagnostic("could not complete bounded idle cohort shutdown; the candidate remains pending.")}\n`);
             return 1;
         }
-        await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
+        await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
         await stateStore.activate(candidate.releaseId);
         decision = { action: "activate-candidate", releaseId: candidate.releaseId, releaseRoot: candidate.releaseRoot, reason: "idle cohort released ownership" };
     }
@@ -112,9 +178,9 @@ export async function runBootstrap(options) {
     else {
         selected = await readMaterializedRelease(decision.releaseRoot);
     }
-    await startSupervisor(selected, environment);
-    await waitForVerifiedEndpoint(paths.endpointMetadataPath, selected, 8_000);
-    return await launchUi(selected, environment);
+    const startup = await startSupervisor(selected, environment);
+    await waitForVerifiedEndpoint(resolveCohortEndpoint(paths, selected.releaseId, environment).endpointMetadataPath, selected, 8_000, startup);
+    return await launchUi(selected, environment, sessionArgs);
 }
 async function readInstalledVersion(packageRoot) {
     const manifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
@@ -128,33 +194,37 @@ export async function certifyMaterializedRelease(release, dataDir, verification
         await verifyMaterializedRelease(release.releaseRoot, release, resolve(dataDir, "releases"), verification);
     }
     const path = resolve(dataDir, `certification-${release.releaseId}.json`);
-    await writeFile(path, JSON.stringify({
-        schema: PRODUCT_IDENTITY.evidence.releaseCertificationSchema,
-        releaseId: release.releaseId,
-        contentDigest: release.contentDigest,
-        verifiedAt: new Date().toISOString(),
-        checks: [{ id: "immutable-content", passed: true }],
-    }, null, 2), { mode: 0o600 });
+    const restartSeal = await createRestartSeal(release, dataDir);
+    await chmod(path, 0o600).catch(() => { });
+    await writeFile(path, JSON.stringify(releaseCertificationDocument(release, restartSeal), null, 2), { mode: 0o600 });
+    await chmod(path, 0o400);
     return path;
 }
 export async function startSupervisor(release, environment) {
     const entry = await resolveReleaseEntryPoint(release, "bin/supervisor.js");
-    const child = spawn(process.execPath, [entry], {
+    const paths = resolveProductPaths(environment);
+    const attempt = await createSupervisorStartupAttempt(paths.runtimeDir, release.releaseId);
+    const child = spawn(process.execPath, [entry, "--startup-attempt", attempt.attemptId], {
         detached: true,
         env: releaseEnvironment(environment, release),
         stdio: "ignore",
         windowsHide: true,
     });
+    const childOutcome = new Promise(resolvePromise => {
+        child.once("close", (exitCode, signal) => resolvePromise({ exitCode, signal }));
+    });
     await new Promise((resolvePromise, rejectPromise) => {
         child.once("spawn", resolvePromise);
         child.once("error", rejectPromise);
     });
     child.unref();
+    return { ...attempt, childOutcome };
 }
-async function launchUi(release, environment) {
+async function launchUi(release, environment, sessionArgs) {
+    await markStartupPhase(environment, "bootstrap-selected");
     const entry = await resolveReleaseEntryPoint(release, "bin/guardian.js");
     return await new Promise((resolvePromise, rejectPromise) => {
-        const child = spawn(process.execPath, [entry], {
+        const child = spawn(process.execPath, [entry, ...sessionArgs], {
             env: releaseEnvironment(environment, release),
             stdio: "inherit",
             windowsHide: false,
@@ -167,16 +237,32 @@ export function releaseEnvironment(environment, release) {
     return {
         ...environment,
         [PRODUCT_IDENTITY.environment.releaseId]: release.releaseId,
+        [PRODUCT_IDENTITY.environment.releaseLayers]: (release.dependencyLayers ?? []).map(layer => layer.layerId).join(","),
         [PRODUCT_IDENTITY.environment.releaseRoot]: release.releaseRoot,
         [PRODUCT_IDENTITY.environment.releaseDigest]: release.contentDigest,
     };
 }
-export async function waitForVerifiedEndpoint(path, release, timeoutMs) {
+export async function waitForVerifiedEndpoint(path, release, timeoutMs, startup) {
     const deadline = Date.now() + timeoutMs;
+    let childOutcome = null;
+    void startup?.childOutcome.then(outcome => { childOutcome = outcome; });
     while (Date.now() < deadline) {
         const metadata = await readEndpointMetadata(path);
-        if (metadata && metadata.releaseId === release.releaseId && await probeOwnership(metadata) === "live-verified")
+        if (metadata && metadata.releaseId === release.releaseId && await probeOwnership(metadata) === "live-verified") {
+            if (startup)
+                await rm(startup.resultPath, { force: true });
             return;
+        }
+        if (startup) {
+            const result = await readSupervisorStartupResult(startup.resultPath, startup.attemptId, startup.releaseId);
+            if (result?.outcome === "failure") {
+                throw Object.assign(new Error(PRODUCT_TEXT.diagnostic(`supervisor startup failed at ${result.stage}: ${result.message}`)), {
+                    code: result.code ?? "SUPERVISOR_STARTUP_FAILED",
+                });
+            }
+        }
+        if (childOutcome)
+            throw new Error(PRODUCT_TEXT.diagnostic(`supervisor exited before readiness: ${JSON.stringify(childOutcome)}`));
         await new Promise(resolvePromise => setTimeout(resolvePromise, 40));
     }
     throw new Error(PRODUCT_TEXT.diagnostic(`supervisor did not publish verified endpoint metadata within ${timeoutMs}ms`));
@@ -273,8 +359,8 @@ async function activatePendingAfterBlockerExit(candidate, stateStore, paths, env
         return;
     await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
     await stateStore.activate(candidate.releaseId);
-    await startSupervisor(candidate, environment);
-    await waitForVerifiedEndpoint(paths.endpointMetadataPath, candidate, 8_000);
+    const startup = await startSupervisor(candidate, environment);
+    await waitForVerifiedEndpoint(resolveCohortEndpoint(paths, candidate.releaseId, environment).endpointMetadataPath, candidate, 8_000, startup);
 }
 export async function releaseVerifiedIdleOwner(metadata, dataDir, operations = {}) {
     try {
@@ -282,7 +368,7 @@ export async function releaseVerifiedIdleOwner(metadata, dataDir, operations = {
         return true;
     }
     catch {
-        // The authenticated idle-release handshake authorizes bounded cleanup of
+        // Security: the authenticated idle-release handshake authorizes bounded cleanup of
         // this exact boot when a native handle outlives graceful shutdown.
         const diagnostics = await (operations.cleanup ?? cleanupProvenIdleOwner)(metadata);
         await writeFile(resolve(dataDir, `cleanup-${Date.now()}.json`), JSON.stringify(diagnostics, null, 2));

package/dist/foundation/release/cohort-state.d.ts

@@ -0,0 +1,156 @@
+import type { MaterializedRelease } from "./release-store.js";
+export declare const RELEASE_COHORT_SCHEMA: string;
+export type ReleaseApproval = "candidate" | "approved" | "rejected";
+export interface ReleaseRecord {
+    readonly releaseId: string;
+    readonly releaseRoot: string;
+    readonly packageVersion: string;
+    readonly contentDigest: string;
+    readonly approval: ReleaseApproval;
+    readonly materializedAt: string;
+    readonly certifiedAt: string | null;
+    readonly diagnosticsPath: string | null;
+}
+export interface ReleaseReferences {
+    readonly active: string | null;
+    readonly pending: string | null;
+    readonly approved: string | null;
+    readonly rollback: string | null;
+    readonly retention: readonly string[];
+}
+export type ExternalReleaseHoldAuthority = "agent" | "migration";
+/** A current hold supplied by an authority that must participate in every reconciliation. */
+export interface ExternalReleaseHold {
+    readonly authority: ExternalReleaseHoldAuthority;
+    readonly releaseId: string;
+}
+export interface ActiveReleaseTransactionReference {
+    readonly status: "active" | "completed" | "rolled-back" | "failed";
+    readonly priorActiveReleaseId: string | null;
+}
+export type ReleaseCleanupStage = "detached" | "trash";
+export interface ReleaseCleanupDisposition {
+    readonly release: ReleaseRecord;
+    readonly stage: ReleaseCleanupStage;
+    readonly trashPath: string | null;
+    readonly attempts: number;
+    readonly lastAttemptAt: string | null;
+    readonly lastError: string | null;
+}
+export interface ReleaseCleanupDiagnostic {
+    readonly releaseId: string;
+    readonly stage: string;
+    readonly attemptedAt: string;
+    readonly error: string;
+}
+export type ReleaseCleanupWorkerStatus = "scheduled" | "running" | "completed" | "blocked" | "continued" | "failed" | "skipped";
+export interface ReleaseCleanupWorkerRun {
+    readonly runId: string;
+    readonly status: ReleaseCleanupWorkerStatus;
+    readonly scheduledAt: string;
+    readonly startedAt: string | null;
+    readonly completedAt: string | null;
+    readonly pid: number | null;
+    readonly batches: number;
+    readonly attempted: number;
+    readonly completed: number;
+    readonly remaining: number;
+    readonly error: string | null;
+}
+export interface ReleaseCleanupWorkerSummary {
+    readonly batches: number;
+    readonly attempted: number;
+    readonly completed: number;
+    readonly remaining: number;
+}
+export interface ReleaseCleanupState {
+    readonly pending: Readonly<Record<string, ReleaseCleanupDisposition>>;
+    readonly diagnostics: readonly ReleaseCleanupDiagnostic[];
+    readonly workerRuns: readonly ReleaseCleanupWorkerRun[];
+}
+export interface CohortState {
+    readonly schema: typeof RELEASE_COHORT_SCHEMA;
+    readonly revision: number;
+    readonly releases: Readonly<Record<string, ReleaseRecord>>;
+    readonly references: ReleaseReferences;
+    readonly cleanup: ReleaseCleanupState;
+    readonly activation: {
+        readonly state: "idle" | "pending" | "draining" | "blocked" | "failed";
+        readonly reason: string | null;
+        readonly blockerGenerationIds: readonly string[];
+        readonly updatedAt: string;
+    };
+}
+export interface EndpointOwnership {
+    readonly state: "idle" | "busy" | "draining" | "released" | "blocked";
+    readonly liveInstanceIds: readonly string[];
+    readonly nonResumableInstanceIds: readonly string[];
+    readonly uncertainInstanceIds: readonly string[];
+}
+export interface SupervisorEndpointMetadata {
+    readonly schema: string;
+    readonly supervisorId: string;
+    readonly endpoint: string;
+    readonly pid: number;
+    readonly pidStartIdentity: string;
+    readonly bootNonce: string;
+    readonly startedAt: string;
+    readonly releaseId: string;
+    readonly releaseRoot: string;
+    readonly contentDigest: string;
+    readonly ownership: EndpointOwnership;
+    readonly envelope: string;
+    readonly envelopeRevision: number;
+    readonly requiredFeatures: readonly string[];
+    readonly optionalFeatures: readonly string[];
+    readonly contractDigest: string;
+}
+export interface ProtectedReleaseInputs {
+    readonly liveReleaseIds?: readonly string[];
+    readonly externalHolds?: readonly ExternalReleaseHold[];
+    readonly transaction?: ActiveReleaseTransactionReference | null;
+}
+export interface ProtectedReleasePlan {
+    readonly protectedReleaseIds: readonly string[];
+    readonly retainedReleaseIds: readonly string[];
+    readonly collectibleReleaseIds: readonly string[];
+}
+export interface OrphanReleaseDisposition {
+    readonly release: ReleaseRecord;
+    readonly stage: ReleaseCleanupStage;
+    readonly trashPath?: string | null;
+}
+export interface ReleaseRetentionReconciliation {
+    readonly state: CohortState;
+    readonly plan: ProtectedReleasePlan;
+    readonly detachedReleaseIds: readonly string[];
+}
+/**
+ * Derive protection exclusively from current selectors and ownership authorities.
+ * The historical retention snapshot is deliberately not an input to this plan.
+ */
+export declare function planProtectedReleases(state: CohortState, inputs?: ProtectedReleaseInputs): ProtectedReleasePlan;
+/** Serializes validated cohort-state revisions under a lock and commits each update atomically. */
+export declare class CohortStateStore {
+    readonly path: string;
+    constructor(dataDir: string);
+    read(): Promise<CohortState>;
+    update(operation: (current: CohortState) => CohortState | Promise<CohortState>): Promise<CohortState>;
+    recordCandidate(release: MaterializedRelease): Promise<CohortState>;
+    approve(releaseId: string, diagnosticsPath: string): Promise<CohortState>;
+    activate(releaseId: string): Promise<CohortState>;
+    blockPending(reason: string, blockerGenerationIds: readonly string[]): Promise<CohortState>;
+    rollback(ownershipReleased: boolean): Promise<CohortState>;
+    setRetention(releaseIds: readonly string[]): Promise<CohortState>;
+    /** Atomically replace legacy retention history and detach every currently collectible record. */
+    reconcileRetention(resolveInputs: (current: CohortState) => ProtectedReleaseInputs | Promise<ProtectedReleaseInputs>, orphans?: readonly OrphanReleaseDisposition[]): Promise<ReleaseRetentionReconciliation>;
+    markCleanupTrash(releaseId: string, trashPath: string): Promise<CohortState>;
+    recordCleanupFailure(releaseId: string, stage: string, error: unknown): Promise<CohortState>;
+    completeCleanup(releaseId: string): Promise<CohortState>;
+    recordCleanupWorkerScheduled(runId: string): Promise<CohortState>;
+    recordCleanupWorkerStarted(runId: string, pid: number): Promise<CohortState>;
+    recordCleanupWorkerProgress(runId: string, summary: ReleaseCleanupWorkerSummary): Promise<CohortState>;
+    recordCleanupWorkerFinished(runId: string, status: Exclude<ReleaseCleanupWorkerStatus, "scheduled" | "running">, summary: ReleaseCleanupWorkerSummary, error?: unknown): Promise<CohortState>;
+    removeUnreferencedRelease(releaseId: string, externalReferences: readonly string[]): Promise<CohortState>;
+}
+export declare function emptyState(): CohortState;