@timurproko/a1 0.2.7-dev.1244 → 0.2.7-dev.1249
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/features/owned-ui/settings-app.js +2 -2
- package/dist/foundation/process-containment/addon.d.ts +5 -0
- package/dist/foundation/release/bootstrap.js +34 -31
- package/dist/foundation/release/cohort-selection.d.ts +0 -6
- package/dist/foundation/release/cohort-selection.js +5 -13
- package/dist/foundation/release/cohort-state.d.ts +2 -0
- package/dist/foundation/release/endpoint-lock.d.ts +2 -0
- package/dist/foundation/release/endpoint-lock.js +37 -0
- package/dist/foundation/release/endpoint-recovery.d.ts +10 -0
- package/dist/foundation/release/endpoint-recovery.js +45 -0
- package/dist/foundation/release/endpoints.d.ts +10 -2
- package/dist/foundation/release/endpoints.js +42 -24
- package/dist/foundation/release/index.d.ts +2 -0
- package/dist/foundation/release/index.js +2 -0
- package/dist/foundation/release/process-cleanup.js +12 -2
- package/dist/foundation/release/supervisor-identity.d.ts +15 -0
- package/dist/foundation/release/supervisor-identity.js +63 -0
- package/dist/foundation/release/update.js +2 -2
- package/dist/foundation/storage/control-store.d.ts +5 -3
- package/dist/foundation/storage/control-store.js +8 -7
- package/dist/foundation/supervision/main.js +3 -1
- package/dist/foundation/supervision/server.js +80 -53
- package/dist/integrations/pi/engine/compaction-progress.js +13 -5
- package/dist/native/darwin-arm64/manifest.json +3 -3
- package/dist/native/darwin-arm64/process-guardian.node +0 -0
- package/dist/native/darwin-arm64/terminal-host.manifest.json +3 -3
- package/dist/native/darwin-arm64/terminal-host.node +0 -0
- package/dist/native/darwin-x64/manifest.json +3 -3
- package/dist/native/darwin-x64/process-guardian.node +0 -0
- package/dist/native/darwin-x64/terminal-host.manifest.json +3 -3
- package/dist/native/darwin-x64/terminal-host.node +0 -0
- package/dist/native/linux-x64/manifest.json +3 -3
- package/dist/native/linux-x64/process-guardian.node +0 -0
- package/dist/native/linux-x64/terminal-host.manifest.json +3 -3
- package/dist/native/linux-x64/terminal-host.node +0 -0
- package/dist/native/win32-x64/manifest.json +3 -3
- package/dist/native/win32-x64/process-guardian.node +0 -0
- package/dist/native/win32-x64/terminal-host.manifest.json +3 -3
- package/dist/native/win32-x64/terminal-host.node +0 -0
- package/dist/ui/settings/declarations.d.ts +1 -1
- package/dist/ui/settings/declarations.js +1 -1
- package/docs/architecture/project-structure.md +24 -16
- package/docs/architecture/toolchain.md +1 -1
- package/docs/ci-release-runbook.md +2 -0
- package/docs/launch-recovery.md +23 -0
- package/docs/local-worktree-cleanup.md +4 -0
- package/docs/openspec-archive-automation.md +11 -3
- package/docs/validation.md +3 -1
- package/package.json +3 -3
|
@@ -543,8 +543,8 @@ export class SettingsApp {
|
|
|
543
543
|
flagDialog: entry.backend !== "a1" ? null
|
|
544
544
|
: entry.id === PLUS_MENU_SETTING_ID ? new PlusMenuDialog(entry.flags)
|
|
545
545
|
: entry.id === TAB_BAR_RIGHT_ITEMS_SETTING_ID ? new FlagToggleDialog(entry.flags, {
|
|
546
|
-
title: "
|
|
547
|
-
description: "Choose
|
|
546
|
+
title: "Widgets",
|
|
547
|
+
description: "Choose which widgets the tab bar shows.",
|
|
548
548
|
empty: "No items match",
|
|
549
549
|
}) : null,
|
|
550
550
|
filter: modelDialog === null ? null : new LineInput(""),
|
|
@@ -25,6 +25,11 @@ export interface SpawnContainedOptions {
|
|
|
25
25
|
export interface ProcessGuardianAddon {
|
|
26
26
|
version(): ProcessGuardianVersion;
|
|
27
27
|
inspectProcessStart(pid: number): string | null;
|
|
28
|
+
/** Additive capability; older retained addons remain usable for containment, not recovery gates. */
|
|
29
|
+
tryAcquireProcessGate?(path: string): {
|
|
30
|
+
readonly acquired: boolean;
|
|
31
|
+
release(): void;
|
|
32
|
+
};
|
|
28
33
|
captureTerminalModes(): string;
|
|
29
34
|
restoreTerminalModes(encoded: string): void;
|
|
30
35
|
deliverExitNotice(path: string, encodedModes: string): void;
|
|
@@ -4,6 +4,7 @@ import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
|
4
4
|
import { connect } from "node:net";
|
|
5
5
|
import { resolve } from "node:path";
|
|
6
6
|
import { selectCohortLaunch } from "./cohort-selection.js";
|
|
7
|
+
import { recoverSupervisorEndpoint } from "./endpoint-recovery.js";
|
|
7
8
|
import { selectOrdinaryLaunchReleaseId } from "./ordinary-launch-selection.js";
|
|
8
9
|
import { releaseChannelOf } from "./latest-release.js";
|
|
9
10
|
import { CohortStateStore } from "./cohort-state.js";
|
|
@@ -22,7 +23,16 @@ import { PRIVATE_ENVIRONMENT, isCurrentLaunchContract, launchContractTarget, rea
|
|
|
22
23
|
import { assertResidentContinuity, ensureTerminalHost, readResidentTabLaunchSettings, selectResidentHost } from "../resident-tabs/index.js";
|
|
23
24
|
const RELEASE_RESELECTION_MESSAGE = "a1-release-reselection";
|
|
24
25
|
export async function runBootstrap(options) {
|
|
25
|
-
|
|
26
|
+
try {
|
|
27
|
+
return await runBootstrapAttempt(options, 1);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (!(error instanceof Error && "code" in error && typeof error.code === "string" && error.code.startsWith("SUPERVISOR_RECOVERY_")))
|
|
31
|
+
throw error;
|
|
32
|
+
// Rationale: actionable recovery failures are a single line, not an uncaught Node stack trace.
|
|
33
|
+
(options.output ?? process.stderr).write(`${PRODUCT_TEXT.diagnostic(error.message)}\n`);
|
|
34
|
+
return 1;
|
|
35
|
+
}
|
|
26
36
|
}
|
|
27
37
|
async function runBootstrapAttempt(options, retriesRemaining) {
|
|
28
38
|
const launchProfileId = options.launchIntent?.profileId ?? "a1";
|
|
@@ -122,7 +132,7 @@ async function runBootstrapAttempt(options, retriesRemaining) {
|
|
|
122
132
|
}
|
|
123
133
|
if (endpoint === null || probe === "dead") {
|
|
124
134
|
if (endpoint)
|
|
125
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
135
|
+
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint, endpoint);
|
|
126
136
|
await markStartupPhase(environment, "durable-validation-start");
|
|
127
137
|
const retained = await readRestartCertifiedRelease(active, paths.dataDir).catch(async () => {
|
|
128
138
|
let verified;
|
|
@@ -217,20 +227,12 @@ async function runBootstrapAttempt(options, retriesRemaining) {
|
|
|
217
227
|
probe = endpoint ? await probeOwnership(endpoint) : "dead";
|
|
218
228
|
let decision = selectCohortLaunch(candidate, state, endpoint, probe);
|
|
219
229
|
if (decision.action === "blocked") {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
return
|
|
223
|
-
}
|
|
224
|
-
if (decision.action === "clean-stale-owner" && endpoint) {
|
|
225
|
-
const diagnostics = await cleanupProvenIdleOwner(endpoint);
|
|
226
|
-
await writeFile(resolve(paths.dataDir, `cleanup-${Date.now()}.json`), JSON.stringify(diagnostics, null, 2));
|
|
227
|
-
if (!diagnostics.terminated)
|
|
228
|
-
throw new Error(`could not safely clean stale ${PRODUCT_TEXT.displayName} supervisor ${endpoint.pid}`);
|
|
229
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
230
|
-
decision = selectCohortLaunch(candidate, await stateStore.read(), null, "dead");
|
|
230
|
+
// Concurrency: reread through the common recovery path rather than fail on an old snapshot.
|
|
231
|
+
await ensureSupervisor(candidate, environment);
|
|
232
|
+
return await launchSelected(candidate);
|
|
231
233
|
}
|
|
232
|
-
|
|
233
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
234
|
+
if (endpoint && probe === "dead") {
|
|
235
|
+
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint, endpoint);
|
|
234
236
|
}
|
|
235
237
|
if (decision.action === "launch-retained-ui") {
|
|
236
238
|
const retainedRecord = state.releases[decision.releaseId];
|
|
@@ -259,7 +261,7 @@ async function runBootstrapAttempt(options, retriesRemaining) {
|
|
|
259
261
|
output.write(`${PRODUCT_TEXT.diagnostic("could not complete bounded idle cohort shutdown; the candidate remains pending.")}\n`);
|
|
260
262
|
return 1;
|
|
261
263
|
}
|
|
262
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
264
|
+
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint, endpoint);
|
|
263
265
|
await stateStore.activate(candidate.releaseId);
|
|
264
266
|
decision = { action: "activate-candidate", releaseId: candidate.releaseId, releaseRoot: candidate.releaseRoot, reason: "idle cohort released ownership" };
|
|
265
267
|
}
|
|
@@ -377,17 +379,15 @@ export async function recordParentCertifiedRelease(release, dataDir) {
|
|
|
377
379
|
/** Reuse a verified owner, or join the winner if another launch starts the same cohort first. */
|
|
378
380
|
export async function ensureSupervisor(release, environment) {
|
|
379
381
|
const paths = resolveCohortEndpoint(resolveProductPaths(environment), release.releaseId, environment);
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
const startup = await startSupervisor(release, environment);
|
|
390
|
-
await waitForVerifiedEndpoint(paths.endpointMetadataPath, release, 8_000, startup);
|
|
382
|
+
await recoverSupervisorEndpoint({
|
|
383
|
+
metadataPath: paths.endpointMetadataPath,
|
|
384
|
+
endpoint: paths.endpoint,
|
|
385
|
+
matches: metadata => endpointMatchesRelease(metadata, release),
|
|
386
|
+
startAndWait: async () => {
|
|
387
|
+
const startup = await startSupervisor(release, environment);
|
|
388
|
+
await waitForVerifiedEndpoint(paths.endpointMetadataPath, release, 8_000, startup);
|
|
389
|
+
},
|
|
390
|
+
});
|
|
391
391
|
}
|
|
392
392
|
function endpointMatchesRelease(metadata, release) {
|
|
393
393
|
return metadata.releaseId === release.releaseId && metadata.releaseRoot === release.releaseRoot
|
|
@@ -463,7 +463,7 @@ export async function waitForVerifiedEndpoint(path, release, timeoutMs, startup)
|
|
|
463
463
|
void startup?.childOutcome.then(outcome => { childOutcome = outcome; });
|
|
464
464
|
while (Date.now() < deadline) {
|
|
465
465
|
const metadata = await readEndpointMetadata(path);
|
|
466
|
-
if (metadata && endpointMatchesRelease(metadata, release) && await probeOwnership(metadata) === "live-verified") {
|
|
466
|
+
if (metadata && endpointMatchesRelease(metadata, release) && await probeOwnership(metadata, { attempts: 1 }) === "live-verified") {
|
|
467
467
|
if (startup)
|
|
468
468
|
await rm(startup.resultPath, { force: true });
|
|
469
469
|
return;
|
|
@@ -507,18 +507,21 @@ async function activatePendingAfterBlockerExit(candidate, stateStore, paths, env
|
|
|
507
507
|
return;
|
|
508
508
|
if (!await releaseVerifiedIdleOwner(endpoint, paths.dataDir))
|
|
509
509
|
return;
|
|
510
|
-
await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
|
|
510
|
+
await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint, endpoint);
|
|
511
511
|
await stateStore.activate(candidate.releaseId);
|
|
512
512
|
await ensureSupervisor(candidate, environment);
|
|
513
513
|
}
|
|
514
514
|
export async function releaseVerifiedIdleOwner(metadata, dataDir, operations = {}) {
|
|
515
515
|
try {
|
|
516
|
+
// Protocol: the owner rechecks its boot nonce and live instances before cooperative exit.
|
|
517
|
+
// A durable idle snapshot alone never authorizes a signal to a still-live process.
|
|
518
|
+
await requestIdleOwnershipRelease(metadata, metadata.releaseId, 2_000);
|
|
516
519
|
await (operations.waitForExit ?? waitForProcessExit)(metadata.pid, 3_000);
|
|
517
520
|
return true;
|
|
518
521
|
}
|
|
519
522
|
catch {
|
|
520
|
-
// Security:
|
|
521
|
-
//
|
|
523
|
+
// Security: a lingering Windows handle can keep numeric PID probes positive after exit.
|
|
524
|
+
// Native observation may retire that exact dead boot, but never signals a still-live owner.
|
|
522
525
|
const diagnostics = await (operations.cleanup ?? cleanupProvenIdleOwner)(metadata);
|
|
523
526
|
await writeFile(resolve(dataDir, `cleanup-${Date.now()}.json`), JSON.stringify(diagnostics, null, 2));
|
|
524
527
|
return diagnostics.terminated;
|
|
@@ -23,12 +23,6 @@ export type CohortLaunchDecision = {
|
|
|
23
23
|
readonly releaseId: string;
|
|
24
24
|
readonly releaseRoot: string;
|
|
25
25
|
readonly reason: string;
|
|
26
|
-
} | {
|
|
27
|
-
readonly action: "clean-stale-owner";
|
|
28
|
-
readonly releaseId: string;
|
|
29
|
-
readonly releaseRoot: string;
|
|
30
|
-
readonly reason: string;
|
|
31
|
-
readonly pid: number;
|
|
32
26
|
} | {
|
|
33
27
|
readonly action: "blocked";
|
|
34
28
|
readonly releaseId: string;
|
|
@@ -30,20 +30,12 @@ export function selectCohortLaunch(candidate, state, endpoint, probe) {
|
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
32
|
if (endpoint && probe === "unresponsive") {
|
|
33
|
-
|
|
34
|
-
return {
|
|
35
|
-
action: "blocked",
|
|
36
|
-
releaseId: endpoint.releaseId,
|
|
37
|
-
releaseRoot: endpoint.releaseRoot,
|
|
38
|
-
reason: "ownership of live launch instances is uncertain; preserving the unresponsive supervisor",
|
|
39
|
-
pid: endpoint.pid,
|
|
40
|
-
};
|
|
41
|
-
}
|
|
33
|
+
// Security: even an empty persisted list cannot authorize killing an unresponsive process.
|
|
42
34
|
return {
|
|
43
|
-
action: "
|
|
44
|
-
releaseId:
|
|
45
|
-
releaseRoot:
|
|
46
|
-
reason: "
|
|
35
|
+
action: "blocked",
|
|
36
|
+
releaseId: endpoint.releaseId,
|
|
37
|
+
releaseRoot: endpoint.releaseRoot,
|
|
38
|
+
reason: "the background service did not respond after automatic recovery; retry shortly",
|
|
47
39
|
pid: endpoint.pid,
|
|
48
40
|
};
|
|
49
41
|
}
|
|
@@ -94,6 +94,8 @@ export interface SupervisorEndpointMetadata {
|
|
|
94
94
|
readonly endpoint: string;
|
|
95
95
|
readonly pid: number;
|
|
96
96
|
readonly pidStartIdentity: string;
|
|
97
|
+
/** Exact OS creation token; absent on historical supervisor metadata. */
|
|
98
|
+
readonly nativeStartIdentity?: string;
|
|
97
99
|
readonly bootNonce: string;
|
|
98
100
|
readonly startedAt: string;
|
|
99
101
|
readonly releaseId: string;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { mkdir, realpath } from "node:fs/promises";
|
|
2
|
+
import { basename, dirname, join } from "node:path";
|
|
3
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
4
|
+
import { acquireSupervisorGate } from "./supervisor-identity.js";
|
|
5
|
+
/** Serialize endpoint publication and retirement using a lock released by the OS on process death. */
|
|
6
|
+
export async function withEndpointLock(metadataPath, operation) {
|
|
7
|
+
let authority;
|
|
8
|
+
try {
|
|
9
|
+
await mkdir(dirname(metadataPath), { recursive: true, mode: 0o700 });
|
|
10
|
+
authority = join(await realpath(dirname(metadataPath)), `${basename(metadataPath)}.authority`);
|
|
11
|
+
}
|
|
12
|
+
catch (cause) {
|
|
13
|
+
throw Object.assign(new Error("cannot access A1 startup state; check runtime-directory permissions and free disk space", { cause }), { code: "SUPERVISOR_RECOVERY_UNAVAILABLE" });
|
|
14
|
+
}
|
|
15
|
+
const deadline = Date.now() + 3_000;
|
|
16
|
+
for (;;) {
|
|
17
|
+
let gate;
|
|
18
|
+
try {
|
|
19
|
+
gate = acquireSupervisorGate(authority);
|
|
20
|
+
}
|
|
21
|
+
catch (cause) {
|
|
22
|
+
throw Object.assign(new Error("cannot access A1 startup coordination; check installation and runtime-directory permissions", { cause }), { code: "SUPERVISOR_RECOVERY_UNAVAILABLE" });
|
|
23
|
+
}
|
|
24
|
+
if (gate.acquired) {
|
|
25
|
+
try {
|
|
26
|
+
return await operation();
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
gate.release();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
gate.release();
|
|
33
|
+
if (Date.now() >= deadline)
|
|
34
|
+
throw Object.assign(new Error("another A1 startup is still preparing; retry shortly"), { code: "SUPERVISOR_RECOVERY_BUSY" });
|
|
35
|
+
await delay(20);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SupervisorEndpointMetadata } from "./cohort-state.js";
|
|
2
|
+
interface SupervisorRecoveryOptions {
|
|
3
|
+
readonly metadataPath: string;
|
|
4
|
+
readonly endpoint: string;
|
|
5
|
+
readonly matches: (metadata: SupervisorEndpointMetadata) => boolean;
|
|
6
|
+
readonly startAndWait: () => Promise<void>;
|
|
7
|
+
}
|
|
8
|
+
/** Recover all launch routes identically; a retry never grants permission to kill a live process. */
|
|
9
|
+
export declare function recoverSupervisorEndpoint(options: SupervisorRecoveryOptions): Promise<void>;
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
import { probeOwnership, readEndpointMetadata, removeEndpointArtifacts } from "./endpoints.js";
|
|
3
|
+
/** Recover all launch routes identically; a retry never grants permission to kill a live process. */
|
|
4
|
+
export async function recoverSupervisorEndpoint(options) {
|
|
5
|
+
const { metadataPath, endpoint } = options;
|
|
6
|
+
let recovered = false;
|
|
7
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
8
|
+
const metadata = await readEndpointMetadata(metadataPath);
|
|
9
|
+
if (metadata) {
|
|
10
|
+
const probe = await probeOwnership(metadata);
|
|
11
|
+
if (probe === "live-verified" && options.matches(metadata)) {
|
|
12
|
+
if (recovered)
|
|
13
|
+
await diagnostic(metadataPath, "recovered", attempt + 1);
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
// Concurrency: another owner may have published while an earlier identity probe was pending.
|
|
17
|
+
if (JSON.stringify(await readEndpointMetadata(metadataPath)) !== JSON.stringify(metadata))
|
|
18
|
+
continue;
|
|
19
|
+
if (probe !== "dead") {
|
|
20
|
+
await diagnostic(metadataPath, probe, attempt + 1);
|
|
21
|
+
throw Object.assign(new Error(probe === "unresponsive"
|
|
22
|
+
? "A1's background service is not responding; retry shortly. Your sessions have been preserved."
|
|
23
|
+
: "A1's background service identity changed; retry startup. Your sessions have been preserved."), {
|
|
24
|
+
code: "SUPERVISOR_RECOVERY_BLOCKED",
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
recovered = true;
|
|
28
|
+
if (!await removeEndpointArtifacts(metadataPath, endpoint, metadata))
|
|
29
|
+
continue;
|
|
30
|
+
await diagnostic(metadataPath, "stale-owner-retired", attempt + 1);
|
|
31
|
+
}
|
|
32
|
+
// Concurrency: the child takes the same endpoint gate before binding/publication. If another launch wins,
|
|
33
|
+
// startAndWait joins its authenticated endpoint; it never unlinks that winner's socket.
|
|
34
|
+
await options.startAndWait();
|
|
35
|
+
if (recovered)
|
|
36
|
+
await diagnostic(metadataPath, "recovered", attempt + 1);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
await diagnostic(metadataPath, "ownership-changing", 3);
|
|
40
|
+
throw Object.assign(new Error("A1 startup changed repeatedly; retry shortly"), { code: "SUPERVISOR_RECOVERY_BUSY" });
|
|
41
|
+
}
|
|
42
|
+
async function diagnostic(path, reason, attempts) {
|
|
43
|
+
// Security: one bounded summary per endpoint, no raw metadata, user paths, or session content.
|
|
44
|
+
await writeFile(`${path}.recovery.json`, JSON.stringify({ reason, attempts, at: new Date().toISOString() }), { mode: 0o600 }).catch(() => undefined);
|
|
45
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { OwnershipProbe } from "./cohort-selection.js";
|
|
2
|
+
import { type ObserveSupervisorProcess } from "./supervisor-identity.js";
|
|
2
3
|
import type { SupervisorEndpointMetadata } from "./cohort-state.js";
|
|
3
4
|
import type { CohortEndpointPaths, ProductPaths } from "../lifecycle/index.js";
|
|
4
5
|
export interface RecordedEndpoint {
|
|
@@ -20,5 +21,12 @@ export declare function sweepDeadEndpoints(paths: ProductPaths): Promise<readonl
|
|
|
20
21
|
/** The releases that an identity-verified live cohort runs from and that must be retained. */
|
|
21
22
|
export declare function liveReleaseIds(paths: ProductPaths): Promise<readonly string[]>;
|
|
22
23
|
export declare function readEndpointMetadata(path: string): Promise<SupervisorEndpointMetadata | null>;
|
|
23
|
-
export
|
|
24
|
-
|
|
24
|
+
export interface OwnershipProbeOptions {
|
|
25
|
+
readonly observe?: ObserveSupervisorProcess;
|
|
26
|
+
readonly attempts?: number;
|
|
27
|
+
readonly timeoutMs?: number;
|
|
28
|
+
}
|
|
29
|
+
/** Reprobe transient failures; only native absence or a replaced generation establishes death. */
|
|
30
|
+
export declare function probeOwnership(metadata: SupervisorEndpointMetadata, options?: OwnershipProbeOptions): Promise<OwnershipProbe>;
|
|
31
|
+
/** Retire only a still-dead exact publication. Socket removal belongs to the gated listener. */
|
|
32
|
+
export declare function removeEndpointArtifacts(metadataPath: string, endpoint: string, expected: SupervisorEndpointMetadata): Promise<boolean>;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { readdir, readFile, rm } from "node:fs/promises";
|
|
2
2
|
import { connect } from "node:net";
|
|
3
|
-
import { platform } from "node:os";
|
|
4
3
|
import { resolve } from "node:path";
|
|
5
4
|
import { PRODUCT_IDENTITY } from "../../product-identity.js";
|
|
6
5
|
import { encodeFrame, LineFrameDecoder } from "../protocol/index.js";
|
|
7
|
-
import {
|
|
6
|
+
import { supervisorProcessState } from "./supervisor-identity.js";
|
|
7
|
+
import { withEndpointLock } from "./endpoint-lock.js";
|
|
8
8
|
/**
|
|
9
9
|
* Every endpoint recorded under this runtime directory: one per cohort, plus the single
|
|
10
10
|
* endpoint published by releases that predate cohort-scoped identity. More than one can be
|
|
@@ -12,9 +12,12 @@ import { processIsAlive } from "./process-cleanup.js";
|
|
|
12
12
|
*/
|
|
13
13
|
export async function listRecordedEndpoints(paths) {
|
|
14
14
|
const recorded = [];
|
|
15
|
-
|
|
15
|
+
// Compatibility: a cohort listener passes its scoped paths here; legacy discovery still
|
|
16
|
+
// names the runtime directory's fixed metadata path, not that listener's own record.
|
|
17
|
+
const legacyPath = resolve(paths.runtimeDir, PRODUCT_IDENTITY.endpoint.metadataFilename);
|
|
18
|
+
const legacy = await readEndpointMetadata(legacyPath);
|
|
16
19
|
if (legacy) {
|
|
17
|
-
recorded.push({ metadata: legacy, paths: { endpoint:
|
|
20
|
+
recorded.push({ metadata: legacy, paths: { endpoint: legacy.endpoint, endpointMetadataPath: legacyPath } });
|
|
18
21
|
}
|
|
19
22
|
const names = await readdir(paths.endpointsDir).catch(() => []);
|
|
20
23
|
for (const name of names) {
|
|
@@ -35,10 +38,11 @@ export async function listRecordedEndpoints(paths) {
|
|
|
35
38
|
export async function sweepDeadEndpoints(paths) {
|
|
36
39
|
const swept = [];
|
|
37
40
|
for (const recorded of await listRecordedEndpoints(paths)) {
|
|
38
|
-
if (
|
|
41
|
+
if (await supervisorProcessState(recorded.metadata) !== "gone")
|
|
39
42
|
continue;
|
|
40
|
-
await removeEndpointArtifacts(recorded.paths.endpointMetadataPath, recorded.paths.endpoint)
|
|
41
|
-
|
|
43
|
+
if (await removeEndpointArtifacts(recorded.paths.endpointMetadataPath, recorded.paths.endpoint, recorded.metadata)) {
|
|
44
|
+
swept.push(recorded.metadata.releaseId);
|
|
45
|
+
}
|
|
42
46
|
}
|
|
43
47
|
return swept;
|
|
44
48
|
}
|
|
@@ -82,18 +86,24 @@ export async function readEndpointMetadata(path) {
|
|
|
82
86
|
return null;
|
|
83
87
|
}
|
|
84
88
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
89
|
+
/** Reprobe transient failures; only native absence or a replaced generation establishes death. */
|
|
90
|
+
export async function probeOwnership(metadata, options = {}) {
|
|
91
|
+
const attempts = options.attempts ?? 3;
|
|
92
|
+
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
93
|
+
const identity = await requestIdentity(metadata.endpoint, options.timeoutMs ?? 500);
|
|
94
|
+
if (identity) {
|
|
95
|
+
return identity.supervisorId === metadata.supervisorId
|
|
96
|
+
&& identity.bootNonce === metadata.bootNonce
|
|
97
|
+
&& identity.pidStartIdentity === metadata.pidStartIdentity
|
|
98
|
+
&& identity.releaseId === metadata.releaseId
|
|
99
|
+
? "live-verified" : "identity-mismatch";
|
|
100
|
+
}
|
|
101
|
+
if (await supervisorProcessState(metadata, options.observe) === "gone")
|
|
102
|
+
return "dead";
|
|
103
|
+
if (attempt + 1 < attempts)
|
|
104
|
+
await new Promise(resolvePromise => setTimeout(resolvePromise, 50));
|
|
105
|
+
}
|
|
106
|
+
return "unresponsive";
|
|
97
107
|
}
|
|
98
108
|
async function requestIdentity(endpoint, timeoutMs) {
|
|
99
109
|
return await new Promise(resolvePromise => {
|
|
@@ -104,6 +114,7 @@ async function requestIdentity(endpoint, timeoutMs) {
|
|
|
104
114
|
if (settled)
|
|
105
115
|
return;
|
|
106
116
|
settled = true;
|
|
117
|
+
clearTimeout(timer);
|
|
107
118
|
socket.destroy();
|
|
108
119
|
resolvePromise(value);
|
|
109
120
|
};
|
|
@@ -121,14 +132,21 @@ async function requestIdentity(endpoint, timeoutMs) {
|
|
|
121
132
|
finish(null);
|
|
122
133
|
}
|
|
123
134
|
});
|
|
124
|
-
setTimeout(() => finish(null), timeoutMs)
|
|
135
|
+
const timer = setTimeout(() => finish(null), timeoutMs);
|
|
125
136
|
});
|
|
126
137
|
}
|
|
127
138
|
function isStringArray(value) {
|
|
128
139
|
return Array.isArray(value) && value.every(item => typeof item === "string" && item.length > 0);
|
|
129
140
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
141
|
+
/** Retire only a still-dead exact publication. Socket removal belongs to the gated listener. */
|
|
142
|
+
export async function removeEndpointArtifacts(metadataPath, endpoint, expected) {
|
|
143
|
+
return await withEndpointLock(metadataPath, async () => {
|
|
144
|
+
const current = await readEndpointMetadata(metadataPath);
|
|
145
|
+
if (!current || current.endpoint !== endpoint || JSON.stringify(current) !== JSON.stringify(expected))
|
|
146
|
+
return false;
|
|
147
|
+
if (await supervisorProcessState(current) !== "gone" || await probeOwnership(current, { attempts: 1 }) !== "dead")
|
|
148
|
+
return false;
|
|
149
|
+
await rm(metadataPath, { force: true });
|
|
150
|
+
return true;
|
|
151
|
+
});
|
|
134
152
|
}
|
|
@@ -8,6 +8,8 @@ export type { ActiveReleaseTransactionReference, CohortState, EndpointOwnership,
|
|
|
8
8
|
export { DEPENDENCY_LAYER_MANIFEST, DEPENDENCY_LAYER_SCHEMA, RUNTIME_PAYLOAD_INVENTORY, dependencyLayerCertificationPath, dependencyReference, generateDependencyRuntimePayload, legacyDependencyLayerCertificationPath, materializeDependencyLayer, readCertifiedDependencyLayer, selectDependencyRuntimePayload, selectPublishedDependencyRuntimePayload, verifyDependencyLayer, } from "./dependency-layer.js";
|
|
9
9
|
export type { DependencyLayerIdentity, DependencyLayerOperationEvent, DependencyLayerReference, GeneratedRuntimePayload, MaterializeDependencyLayerOptions, MaterializedDependencyLayer, PublishedRuntimeFileClassification, PublishedRuntimePayload, ReadCertifiedDependencyLayerOptions, RuntimePayloadExclusion, RuntimePayloadInventory, SelectedRuntimePayload, } from "./dependency-layer.js";
|
|
10
10
|
export { listRecordedEndpoints, liveReleaseIds, probeOwnership, readEndpointMetadata, removeEndpointArtifacts, sweepDeadEndpoints } from "./endpoints.js";
|
|
11
|
+
export { currentSupervisorStartIdentity, observeSupervisorProcess, supervisorProcessState } from "./supervisor-identity.js";
|
|
12
|
+
export { withEndpointLock } from "./endpoint-lock.js";
|
|
11
13
|
export type { RecordedEndpoint } from "./endpoints.js";
|
|
12
14
|
export { cleanupProvenIdleOwner, cleanupVerifiedOwner, processIsAlive } from "./process-cleanup.js";
|
|
13
15
|
export type { CleanupDiagnostics } from "./process-cleanup.js";
|
|
@@ -4,6 +4,8 @@ export { selectOrdinaryLaunchReleaseId } from "./ordinary-launch-selection.js";
|
|
|
4
4
|
export { CohortStateStore, RELEASE_COHORT_SCHEMA, emptyState, planProtectedReleases } from "./cohort-state.js";
|
|
5
5
|
export { DEPENDENCY_LAYER_MANIFEST, DEPENDENCY_LAYER_SCHEMA, RUNTIME_PAYLOAD_INVENTORY, dependencyLayerCertificationPath, dependencyReference, generateDependencyRuntimePayload, legacyDependencyLayerCertificationPath, materializeDependencyLayer, readCertifiedDependencyLayer, selectDependencyRuntimePayload, selectPublishedDependencyRuntimePayload, verifyDependencyLayer, } from "./dependency-layer.js";
|
|
6
6
|
export { listRecordedEndpoints, liveReleaseIds, probeOwnership, readEndpointMetadata, removeEndpointArtifacts, sweepDeadEndpoints } from "./endpoints.js";
|
|
7
|
+
export { currentSupervisorStartIdentity, observeSupervisorProcess, supervisorProcessState } from "./supervisor-identity.js";
|
|
8
|
+
export { withEndpointLock } from "./endpoint-lock.js";
|
|
7
9
|
export { cleanupProvenIdleOwner, cleanupVerifiedOwner, processIsAlive } from "./process-cleanup.js";
|
|
8
10
|
export { PRODUCT_PACKAGE_NAME, createReleaseIdentity, deriveReleaseIdentity, digestManifestFiles, discoverReleasePayload, packageRootFromModule, releaseFileIdentity, resolveWithin, } from "./release.js";
|
|
9
11
|
export { collectRelease, prepareReleaseCleanup, runBoundedReleaseCleanup, runReleaseCleanupWorker, scheduleReleaseCleanup } from "./release-gc.js";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import { supervisorProcessState } from "./supervisor-identity.js";
|
|
2
3
|
export function processIsAlive(pid) {
|
|
3
4
|
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
4
5
|
return false;
|
|
@@ -15,17 +16,26 @@ export async function cleanupProvenIdleOwner(metadata, graceMs = 1_500) {
|
|
|
15
16
|
}
|
|
16
17
|
export async function cleanupVerifiedOwner(metadata, options = {}) {
|
|
17
18
|
const graceMs = options.graceMs ?? 1_500;
|
|
18
|
-
if (!options.allowLiveInstances && (metadata.ownership.liveInstanceIds.length > 0 || metadata.ownership.nonResumableInstanceIds.length > 0)) {
|
|
19
|
+
if (!options.allowLiveInstances && (metadata.ownership.liveInstanceIds.length > 0 || metadata.ownership.nonResumableInstanceIds.length > 0 || metadata.ownership.uncertainInstanceIds.length > 0)) {
|
|
19
20
|
throw new Error("refusing to terminate an owner with recorded live generations");
|
|
20
21
|
}
|
|
21
22
|
const started = Date.now();
|
|
22
23
|
const attempted = [];
|
|
23
|
-
|
|
24
|
+
const state = await supervisorProcessState(metadata);
|
|
25
|
+
if (state === "gone")
|
|
24
26
|
return { pid: metadata.pid, attempted: ["owner-already-dead"], terminated: true, elapsedMs: Date.now() - started };
|
|
27
|
+
// Security: ordinary recovery never signals a live owner, even with an empty durable list.
|
|
28
|
+
// Explicit update cleanup also needs the exact native generation, not merely an old handshake.
|
|
29
|
+
if (!options.allowLiveInstances || state !== "same") {
|
|
30
|
+
return { pid: metadata.pid, attempted: ["live-owner-preserved"], terminated: false, elapsedMs: Date.now() - started };
|
|
31
|
+
}
|
|
25
32
|
attempted.push("graceful-termination");
|
|
26
33
|
await terminate(metadata.pid, false);
|
|
27
34
|
if (await waitUntilDead(metadata.pid, graceMs))
|
|
28
35
|
return { pid: metadata.pid, attempted, terminated: true, elapsedMs: Date.now() - started };
|
|
36
|
+
if (await supervisorProcessState(metadata) !== "same") {
|
|
37
|
+
return { pid: metadata.pid, attempted, terminated: await supervisorProcessState(metadata) === "gone", elapsedMs: Date.now() - started };
|
|
38
|
+
}
|
|
29
39
|
attempted.push("forced-process-tree-termination");
|
|
30
40
|
await terminate(metadata.pid, true);
|
|
31
41
|
const terminated = await waitUntilDead(metadata.pid, graceMs);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { NativeProcessIdentity } from "../lifecycle/index.js";
|
|
2
|
+
import type { SupervisorEndpointMetadata } from "./cohort-state.js";
|
|
3
|
+
export type SupervisorProcessState = "same" | "gone" | "unknown";
|
|
4
|
+
export type ObserveSupervisorProcess = (pid: number) => Promise<NativeProcessIdentity | null>;
|
|
5
|
+
/** Inspection code comes from this runtime, never from a path in a stale endpoint record. */
|
|
6
|
+
export declare function observeSupervisorProcess(pid: number): Promise<NativeProcessIdentity | null>;
|
|
7
|
+
/** New supervisors use a native creation token as their opaque handshake identity as well. */
|
|
8
|
+
export declare function currentSupervisorStartIdentity(): string;
|
|
9
|
+
/** The current runtime also owns the OS gate ABI; metadata cannot choose a different addon. */
|
|
10
|
+
export declare function acquireSupervisorGate(path: string): {
|
|
11
|
+
readonly acquired: boolean;
|
|
12
|
+
release(): void;
|
|
13
|
+
};
|
|
14
|
+
/** A failed inspection is uncertainty, never permission to erase a possibly live owner. */
|
|
15
|
+
export declare function supervisorProcessState(metadata: SupervisorEndpointMetadata, observe?: ObserveSupervisorProcess): Promise<SupervisorProcessState>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { fileURLToPath } from "node:url";
|
|
2
|
+
import { createNativeProcessInspector, loadProcessGuardianAddon, resolveProcessGuardianPath } from "../process-containment/index.js";
|
|
3
|
+
const ownRoot = fileURLToPath(new URL("../../../", import.meta.url));
|
|
4
|
+
/** Inspection code comes from this runtime, never from a path in a stale endpoint record. */
|
|
5
|
+
export async function observeSupervisorProcess(pid) {
|
|
6
|
+
return await (await createNativeProcessInspector(process.env, process.platform, ownRoot)).observe(pid);
|
|
7
|
+
}
|
|
8
|
+
/** New supervisors use a native creation token as their opaque handshake identity as well. */
|
|
9
|
+
export function currentSupervisorStartIdentity() {
|
|
10
|
+
try {
|
|
11
|
+
const token = loadProcessGuardianAddon(resolveProcessGuardianPath(ownRoot, process.env)).inspectProcessStart(process.pid);
|
|
12
|
+
if (!validNativeToken(token))
|
|
13
|
+
throw new Error("native process identity is unavailable");
|
|
14
|
+
return token;
|
|
15
|
+
}
|
|
16
|
+
catch (cause) {
|
|
17
|
+
throw Object.assign(new Error("cannot verify A1's background service process; check installation and process-inspection permissions", { cause }), { code: "SUPERVISOR_RECOVERY_UNAVAILABLE" });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** The current runtime also owns the OS gate ABI; metadata cannot choose a different addon. */
|
|
21
|
+
export function acquireSupervisorGate(path) {
|
|
22
|
+
const addon = loadProcessGuardianAddon(resolveProcessGuardianPath(ownRoot, process.env));
|
|
23
|
+
if (!addon.tryAcquireProcessGate)
|
|
24
|
+
throw new Error("this installation lacks startup recovery support; update or repair A1");
|
|
25
|
+
return addon.tryAcquireProcessGate(path);
|
|
26
|
+
}
|
|
27
|
+
/** A failed inspection is uncertainty, never permission to erase a possibly live owner. */
|
|
28
|
+
export async function supervisorProcessState(metadata, observe = observeSupervisorProcess) {
|
|
29
|
+
let identity;
|
|
30
|
+
try {
|
|
31
|
+
identity = await observe(metadata.pid);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return "unknown";
|
|
35
|
+
}
|
|
36
|
+
if (identity === null)
|
|
37
|
+
return "gone";
|
|
38
|
+
if (identity.pid !== metadata.pid)
|
|
39
|
+
return "unknown";
|
|
40
|
+
if (metadata.nativeStartIdentity !== undefined) {
|
|
41
|
+
// Compatibility: only compare tokens of the same native format, not an old Node timestamp.
|
|
42
|
+
if (!validNativeToken(metadata.nativeStartIdentity) || !validNativeToken(identity.startIdentity)
|
|
43
|
+
|| identity.startIdentity.split(":")[0] !== metadata.nativeStartIdentity.split(":")[0])
|
|
44
|
+
return "unknown";
|
|
45
|
+
return identity.startIdentity === metadata.nativeStartIdentity ? "same" : "gone";
|
|
46
|
+
}
|
|
47
|
+
// Compatibility: legacy publication happened after process creation. A native creation time
|
|
48
|
+
// strictly later than that publication (with a one-second clock margin) disproves ownership;
|
|
49
|
+
// a near timestamp is NOT proof of ownership. Linux tick-only tokens cannot prove this.
|
|
50
|
+
const published = Date.parse(metadata.startedAt);
|
|
51
|
+
const created = nativeCreationTime(identity.startIdentity);
|
|
52
|
+
return Number.isFinite(published) && created !== null && created > published + 1_000 ? "gone" : "unknown";
|
|
53
|
+
}
|
|
54
|
+
function validNativeToken(value) {
|
|
55
|
+
return typeof value === "string" && /^(?:windows-filetime:\d{1,20}|linux-proc-start:\d{1,20}|darwin-proc-start:\d{1,20}:\d{1,6})$/.test(value);
|
|
56
|
+
}
|
|
57
|
+
function nativeCreationTime(identity) {
|
|
58
|
+
const windows = /^windows-filetime:(\d+)$/.exec(identity);
|
|
59
|
+
if (windows)
|
|
60
|
+
return Number(BigInt(windows[1]) / 10000n - 11644473600000n);
|
|
61
|
+
const darwin = /^darwin-proc-start:(\d+):(\d+)$/.exec(identity);
|
|
62
|
+
return darwin ? Number(darwin[1]) * 1_000 + Number(darwin[2]) / 1_000 : null;
|
|
63
|
+
}
|
|
@@ -155,7 +155,7 @@ export function createUpdateLifecycleCoordinator(environment = process.env, file
|
|
|
155
155
|
}
|
|
156
156
|
const plan = planUpdateOwnership(ownership, await canonicalImmutableRoot(paths.dataDir, endpoint.releaseRoot));
|
|
157
157
|
if (plan === "clean-dead-record") {
|
|
158
|
-
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint);
|
|
158
|
+
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint, endpoint);
|
|
159
159
|
return { priorActiveVersion };
|
|
160
160
|
}
|
|
161
161
|
// Rationale: leaving sessions alone is the expected outcome, and saying so mid-update tears
|
|
@@ -182,7 +182,7 @@ export function createUpdateLifecycleCoordinator(environment = process.env, file
|
|
|
182
182
|
throw new Error(`verified ${PRODUCT_TEXT.displayName} owner ${endpoint.pid} did not terminate`);
|
|
183
183
|
});
|
|
184
184
|
}
|
|
185
|
-
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint);
|
|
185
|
+
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint, endpoint);
|
|
186
186
|
return { priorActiveVersion };
|
|
187
187
|
},
|
|
188
188
|
async verifyPackageUnlocked(packageRoot) {
|
|
@@ -5,14 +5,16 @@ export declare class ControlStore {
|
|
|
5
5
|
#private;
|
|
6
6
|
readonly database: DatabaseSync;
|
|
7
7
|
readonly bootNonce: string | null;
|
|
8
|
-
constructor(path: string, bootNonce?: string | null
|
|
8
|
+
constructor(path: string, bootNonce?: string | null, options?: {
|
|
9
|
+
readonly reconcileOnOpen?: boolean;
|
|
10
|
+
});
|
|
9
11
|
migrate(): void;
|
|
10
12
|
createLaunchInstance(instance: LaunchInstance): void;
|
|
11
13
|
activateLaunchInstance(instanceId: LaunchInstanceId, ownerClientId: string, rootIdentity: NativeProcessIdentity, containmentIdentity: ProcessContainmentIdentity, activatedAt: string): boolean;
|
|
12
14
|
beginLaunchInstanceStop(instanceId: LaunchInstanceId, ownerClientId: string, stoppingAt: string): boolean;
|
|
13
15
|
completeLaunchInstance(instanceId: LaunchInstanceId, ownerClientId: string, terminalState: "completed" | "interrupted", outcome: LaunchInstanceOutcome, completedAt: string): boolean;
|
|
14
|
-
loadActiveLaunchInstances(): LaunchInstance[];
|
|
16
|
+
loadActiveLaunchInstances(bootNonce?: string): LaunchInstance[];
|
|
15
17
|
loadLaunchInstance(instanceId: LaunchInstanceId): LaunchInstance | null;
|
|
16
|
-
reconcilePriorBootLaunchInstances(bootNonce: string, reconciledAt?: string): number;
|
|
18
|
+
reconcilePriorBootLaunchInstances(bootNonce: string, reconciledAt?: string, protectedBoots?: readonly string[]): number;
|
|
17
19
|
close(): void;
|
|
18
20
|
}
|