@timurproko/a1 0.2.7-dev.1244 → 0.2.7-dev.1245
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/foundation/process-containment/addon.d.ts +5 -0
- package/dist/foundation/release/bootstrap.js +34 -31
- package/dist/foundation/release/cohort-selection.d.ts +0 -6
- package/dist/foundation/release/cohort-selection.js +5 -13
- package/dist/foundation/release/cohort-state.d.ts +2 -0
- package/dist/foundation/release/endpoint-lock.d.ts +2 -0
- package/dist/foundation/release/endpoint-lock.js +37 -0
- package/dist/foundation/release/endpoint-recovery.d.ts +10 -0
- package/dist/foundation/release/endpoint-recovery.js +45 -0
- package/dist/foundation/release/endpoints.d.ts +10 -2
- package/dist/foundation/release/endpoints.js +42 -24
- package/dist/foundation/release/index.d.ts +2 -0
- package/dist/foundation/release/index.js +2 -0
- package/dist/foundation/release/process-cleanup.js +12 -2
- package/dist/foundation/release/supervisor-identity.d.ts +15 -0
- package/dist/foundation/release/supervisor-identity.js +63 -0
- package/dist/foundation/release/update.js +2 -2
- package/dist/foundation/storage/control-store.d.ts +5 -3
- package/dist/foundation/storage/control-store.js +8 -7
- package/dist/foundation/supervision/main.js +3 -1
- package/dist/foundation/supervision/server.js +75 -53
- package/dist/native/darwin-arm64/manifest.json +3 -3
- package/dist/native/darwin-arm64/process-guardian.node +0 -0
- package/dist/native/darwin-arm64/terminal-host.manifest.json +1 -1
- package/dist/native/darwin-x64/manifest.json +3 -3
- package/dist/native/darwin-x64/process-guardian.node +0 -0
- package/dist/native/darwin-x64/terminal-host.manifest.json +1 -1
- package/dist/native/linux-x64/manifest.json +3 -3
- package/dist/native/linux-x64/process-guardian.node +0 -0
- package/dist/native/linux-x64/terminal-host.manifest.json +1 -1
- package/dist/native/win32-x64/manifest.json +3 -3
- package/dist/native/win32-x64/process-guardian.node +0 -0
- package/dist/native/win32-x64/terminal-host.manifest.json +2 -2
- package/dist/native/win32-x64/terminal-host.node +0 -0
- package/docs/architecture/project-structure.md +2 -0
- package/docs/launch-recovery.md +23 -0
- package/package.json +3 -3
|
@@ -25,6 +25,11 @@ export interface SpawnContainedOptions {
|
|
|
25
25
|
export interface ProcessGuardianAddon {
|
|
26
26
|
version(): ProcessGuardianVersion;
|
|
27
27
|
inspectProcessStart(pid: number): string | null;
|
|
28
|
+
/** Additive capability; older retained addons remain usable for containment, not recovery gates. */
|
|
29
|
+
tryAcquireProcessGate?(path: string): {
|
|
30
|
+
readonly acquired: boolean;
|
|
31
|
+
release(): void;
|
|
32
|
+
};
|
|
28
33
|
captureTerminalModes(): string;
|
|
29
34
|
restoreTerminalModes(encoded: string): void;
|
|
30
35
|
deliverExitNotice(path: string, encodedModes: string): void;
|
|
@@ -4,6 +4,7 @@ import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
|
4
4
|
import { connect } from "node:net";
|
|
5
5
|
import { resolve } from "node:path";
|
|
6
6
|
import { selectCohortLaunch } from "./cohort-selection.js";
|
|
7
|
+
import { recoverSupervisorEndpoint } from "./endpoint-recovery.js";
|
|
7
8
|
import { selectOrdinaryLaunchReleaseId } from "./ordinary-launch-selection.js";
|
|
8
9
|
import { releaseChannelOf } from "./latest-release.js";
|
|
9
10
|
import { CohortStateStore } from "./cohort-state.js";
|
|
@@ -22,7 +23,16 @@ import { PRIVATE_ENVIRONMENT, isCurrentLaunchContract, launchContractTarget, rea
|
|
|
22
23
|
import { assertResidentContinuity, ensureTerminalHost, readResidentTabLaunchSettings, selectResidentHost } from "../resident-tabs/index.js";
|
|
23
24
|
const RELEASE_RESELECTION_MESSAGE = "a1-release-reselection";
|
|
24
25
|
export async function runBootstrap(options) {
|
|
25
|
-
|
|
26
|
+
try {
|
|
27
|
+
return await runBootstrapAttempt(options, 1);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (!(error instanceof Error && "code" in error && typeof error.code === "string" && error.code.startsWith("SUPERVISOR_RECOVERY_")))
|
|
31
|
+
throw error;
|
|
32
|
+
// Rationale: actionable recovery failures are a single line, not an uncaught Node stack trace.
|
|
33
|
+
(options.output ?? process.stderr).write(`${PRODUCT_TEXT.diagnostic(error.message)}\n`);
|
|
34
|
+
return 1;
|
|
35
|
+
}
|
|
26
36
|
}
|
|
27
37
|
async function runBootstrapAttempt(options, retriesRemaining) {
|
|
28
38
|
const launchProfileId = options.launchIntent?.profileId ?? "a1";
|
|
@@ -122,7 +132,7 @@ async function runBootstrapAttempt(options, retriesRemaining) {
|
|
|
122
132
|
}
|
|
123
133
|
if (endpoint === null || probe === "dead") {
|
|
124
134
|
if (endpoint)
|
|
125
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
135
|
+
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint, endpoint);
|
|
126
136
|
await markStartupPhase(environment, "durable-validation-start");
|
|
127
137
|
const retained = await readRestartCertifiedRelease(active, paths.dataDir).catch(async () => {
|
|
128
138
|
let verified;
|
|
@@ -217,20 +227,12 @@ async function runBootstrapAttempt(options, retriesRemaining) {
|
|
|
217
227
|
probe = endpoint ? await probeOwnership(endpoint) : "dead";
|
|
218
228
|
let decision = selectCohortLaunch(candidate, state, endpoint, probe);
|
|
219
229
|
if (decision.action === "blocked") {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
return
|
|
223
|
-
}
|
|
224
|
-
if (decision.action === "clean-stale-owner" && endpoint) {
|
|
225
|
-
const diagnostics = await cleanupProvenIdleOwner(endpoint);
|
|
226
|
-
await writeFile(resolve(paths.dataDir, `cleanup-${Date.now()}.json`), JSON.stringify(diagnostics, null, 2));
|
|
227
|
-
if (!diagnostics.terminated)
|
|
228
|
-
throw new Error(`could not safely clean stale ${PRODUCT_TEXT.displayName} supervisor ${endpoint.pid}`);
|
|
229
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
230
|
-
decision = selectCohortLaunch(candidate, await stateStore.read(), null, "dead");
|
|
230
|
+
// Concurrency: reread through the common recovery path rather than fail on an old snapshot.
|
|
231
|
+
await ensureSupervisor(candidate, environment);
|
|
232
|
+
return await launchSelected(candidate);
|
|
231
233
|
}
|
|
232
|
-
|
|
233
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
234
|
+
if (endpoint && probe === "dead") {
|
|
235
|
+
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint, endpoint);
|
|
234
236
|
}
|
|
235
237
|
if (decision.action === "launch-retained-ui") {
|
|
236
238
|
const retainedRecord = state.releases[decision.releaseId];
|
|
@@ -259,7 +261,7 @@ async function runBootstrapAttempt(options, retriesRemaining) {
|
|
|
259
261
|
output.write(`${PRODUCT_TEXT.diagnostic("could not complete bounded idle cohort shutdown; the candidate remains pending.")}\n`);
|
|
260
262
|
return 1;
|
|
261
263
|
}
|
|
262
|
-
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint);
|
|
264
|
+
await removeEndpointArtifacts(endpointPaths.endpointMetadataPath, endpointPaths.endpoint, endpoint);
|
|
263
265
|
await stateStore.activate(candidate.releaseId);
|
|
264
266
|
decision = { action: "activate-candidate", releaseId: candidate.releaseId, releaseRoot: candidate.releaseRoot, reason: "idle cohort released ownership" };
|
|
265
267
|
}
|
|
@@ -377,17 +379,15 @@ export async function recordParentCertifiedRelease(release, dataDir) {
|
|
|
377
379
|
/** Reuse a verified owner, or join the winner if another launch starts the same cohort first. */
|
|
378
380
|
export async function ensureSupervisor(release, environment) {
|
|
379
381
|
const paths = resolveCohortEndpoint(resolveProductPaths(environment), release.releaseId, environment);
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
const startup = await startSupervisor(release, environment);
|
|
390
|
-
await waitForVerifiedEndpoint(paths.endpointMetadataPath, release, 8_000, startup);
|
|
382
|
+
await recoverSupervisorEndpoint({
|
|
383
|
+
metadataPath: paths.endpointMetadataPath,
|
|
384
|
+
endpoint: paths.endpoint,
|
|
385
|
+
matches: metadata => endpointMatchesRelease(metadata, release),
|
|
386
|
+
startAndWait: async () => {
|
|
387
|
+
const startup = await startSupervisor(release, environment);
|
|
388
|
+
await waitForVerifiedEndpoint(paths.endpointMetadataPath, release, 8_000, startup);
|
|
389
|
+
},
|
|
390
|
+
});
|
|
391
391
|
}
|
|
392
392
|
function endpointMatchesRelease(metadata, release) {
|
|
393
393
|
return metadata.releaseId === release.releaseId && metadata.releaseRoot === release.releaseRoot
|
|
@@ -463,7 +463,7 @@ export async function waitForVerifiedEndpoint(path, release, timeoutMs, startup)
|
|
|
463
463
|
void startup?.childOutcome.then(outcome => { childOutcome = outcome; });
|
|
464
464
|
while (Date.now() < deadline) {
|
|
465
465
|
const metadata = await readEndpointMetadata(path);
|
|
466
|
-
if (metadata && endpointMatchesRelease(metadata, release) && await probeOwnership(metadata) === "live-verified") {
|
|
466
|
+
if (metadata && endpointMatchesRelease(metadata, release) && await probeOwnership(metadata, { attempts: 1 }) === "live-verified") {
|
|
467
467
|
if (startup)
|
|
468
468
|
await rm(startup.resultPath, { force: true });
|
|
469
469
|
return;
|
|
@@ -507,18 +507,21 @@ async function activatePendingAfterBlockerExit(candidate, stateStore, paths, env
|
|
|
507
507
|
return;
|
|
508
508
|
if (!await releaseVerifiedIdleOwner(endpoint, paths.dataDir))
|
|
509
509
|
return;
|
|
510
|
-
await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint);
|
|
510
|
+
await removeEndpointArtifacts(paths.endpointMetadataPath, paths.endpoint, endpoint);
|
|
511
511
|
await stateStore.activate(candidate.releaseId);
|
|
512
512
|
await ensureSupervisor(candidate, environment);
|
|
513
513
|
}
|
|
514
514
|
export async function releaseVerifiedIdleOwner(metadata, dataDir, operations = {}) {
|
|
515
515
|
try {
|
|
516
|
+
// Protocol: the owner rechecks its boot nonce and live instances before cooperative exit.
|
|
517
|
+
// A durable idle snapshot alone never authorizes a signal to a still-live process.
|
|
518
|
+
await requestIdleOwnershipRelease(metadata, metadata.releaseId, 2_000);
|
|
516
519
|
await (operations.waitForExit ?? waitForProcessExit)(metadata.pid, 3_000);
|
|
517
520
|
return true;
|
|
518
521
|
}
|
|
519
522
|
catch {
|
|
520
|
-
// Security:
|
|
521
|
-
//
|
|
523
|
+
// Security: a lingering Windows handle can keep numeric PID probes positive after exit.
|
|
524
|
+
// Native observation may retire that exact dead boot, but never signals a still-live owner.
|
|
522
525
|
const diagnostics = await (operations.cleanup ?? cleanupProvenIdleOwner)(metadata);
|
|
523
526
|
await writeFile(resolve(dataDir, `cleanup-${Date.now()}.json`), JSON.stringify(diagnostics, null, 2));
|
|
524
527
|
return diagnostics.terminated;
|
|
@@ -23,12 +23,6 @@ export type CohortLaunchDecision = {
|
|
|
23
23
|
readonly releaseId: string;
|
|
24
24
|
readonly releaseRoot: string;
|
|
25
25
|
readonly reason: string;
|
|
26
|
-
} | {
|
|
27
|
-
readonly action: "clean-stale-owner";
|
|
28
|
-
readonly releaseId: string;
|
|
29
|
-
readonly releaseRoot: string;
|
|
30
|
-
readonly reason: string;
|
|
31
|
-
readonly pid: number;
|
|
32
26
|
} | {
|
|
33
27
|
readonly action: "blocked";
|
|
34
28
|
readonly releaseId: string;
|
|
@@ -30,20 +30,12 @@ export function selectCohortLaunch(candidate, state, endpoint, probe) {
|
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
32
|
if (endpoint && probe === "unresponsive") {
|
|
33
|
-
|
|
34
|
-
return {
|
|
35
|
-
action: "blocked",
|
|
36
|
-
releaseId: endpoint.releaseId,
|
|
37
|
-
releaseRoot: endpoint.releaseRoot,
|
|
38
|
-
reason: "ownership of live launch instances is uncertain; preserving the unresponsive supervisor",
|
|
39
|
-
pid: endpoint.pid,
|
|
40
|
-
};
|
|
41
|
-
}
|
|
33
|
+
// Security: even an empty persisted list cannot authorize killing an unresponsive process.
|
|
42
34
|
return {
|
|
43
|
-
action: "
|
|
44
|
-
releaseId:
|
|
45
|
-
releaseRoot:
|
|
46
|
-
reason: "
|
|
35
|
+
action: "blocked",
|
|
36
|
+
releaseId: endpoint.releaseId,
|
|
37
|
+
releaseRoot: endpoint.releaseRoot,
|
|
38
|
+
reason: "the background service did not respond after automatic recovery; retry shortly",
|
|
47
39
|
pid: endpoint.pid,
|
|
48
40
|
};
|
|
49
41
|
}
|
|
@@ -94,6 +94,8 @@ export interface SupervisorEndpointMetadata {
|
|
|
94
94
|
readonly endpoint: string;
|
|
95
95
|
readonly pid: number;
|
|
96
96
|
readonly pidStartIdentity: string;
|
|
97
|
+
/** Exact OS creation token; absent on historical supervisor metadata. */
|
|
98
|
+
readonly nativeStartIdentity?: string;
|
|
97
99
|
readonly bootNonce: string;
|
|
98
100
|
readonly startedAt: string;
|
|
99
101
|
readonly releaseId: string;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { mkdir, realpath } from "node:fs/promises";
|
|
2
|
+
import { basename, dirname, join } from "node:path";
|
|
3
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
4
|
+
import { acquireSupervisorGate } from "./supervisor-identity.js";
|
|
5
|
+
/** Serialize endpoint publication and retirement using a lock released by the OS on process death. */
|
|
6
|
+
export async function withEndpointLock(metadataPath, operation) {
|
|
7
|
+
let authority;
|
|
8
|
+
try {
|
|
9
|
+
await mkdir(dirname(metadataPath), { recursive: true, mode: 0o700 });
|
|
10
|
+
authority = join(await realpath(dirname(metadataPath)), `${basename(metadataPath)}.authority`);
|
|
11
|
+
}
|
|
12
|
+
catch (cause) {
|
|
13
|
+
throw Object.assign(new Error("cannot access A1 startup state; check runtime-directory permissions and free disk space", { cause }), { code: "SUPERVISOR_RECOVERY_UNAVAILABLE" });
|
|
14
|
+
}
|
|
15
|
+
const deadline = Date.now() + 3_000;
|
|
16
|
+
for (;;) {
|
|
17
|
+
let gate;
|
|
18
|
+
try {
|
|
19
|
+
gate = acquireSupervisorGate(authority);
|
|
20
|
+
}
|
|
21
|
+
catch (cause) {
|
|
22
|
+
throw Object.assign(new Error("cannot access A1 startup coordination; check installation and runtime-directory permissions", { cause }), { code: "SUPERVISOR_RECOVERY_UNAVAILABLE" });
|
|
23
|
+
}
|
|
24
|
+
if (gate.acquired) {
|
|
25
|
+
try {
|
|
26
|
+
return await operation();
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
gate.release();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
gate.release();
|
|
33
|
+
if (Date.now() >= deadline)
|
|
34
|
+
throw Object.assign(new Error("another A1 startup is still preparing; retry shortly"), { code: "SUPERVISOR_RECOVERY_BUSY" });
|
|
35
|
+
await delay(20);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SupervisorEndpointMetadata } from "./cohort-state.js";
|
|
2
|
+
interface SupervisorRecoveryOptions {
|
|
3
|
+
readonly metadataPath: string;
|
|
4
|
+
readonly endpoint: string;
|
|
5
|
+
readonly matches: (metadata: SupervisorEndpointMetadata) => boolean;
|
|
6
|
+
readonly startAndWait: () => Promise<void>;
|
|
7
|
+
}
|
|
8
|
+
/** Recover all launch routes identically; a retry never grants permission to kill a live process. */
|
|
9
|
+
export declare function recoverSupervisorEndpoint(options: SupervisorRecoveryOptions): Promise<void>;
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
import { probeOwnership, readEndpointMetadata, removeEndpointArtifacts } from "./endpoints.js";
|
|
3
|
+
/** Recover all launch routes identically; a retry never grants permission to kill a live process. */
|
|
4
|
+
export async function recoverSupervisorEndpoint(options) {
|
|
5
|
+
const { metadataPath, endpoint } = options;
|
|
6
|
+
let recovered = false;
|
|
7
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
8
|
+
const metadata = await readEndpointMetadata(metadataPath);
|
|
9
|
+
if (metadata) {
|
|
10
|
+
const probe = await probeOwnership(metadata);
|
|
11
|
+
if (probe === "live-verified" && options.matches(metadata)) {
|
|
12
|
+
if (recovered)
|
|
13
|
+
await diagnostic(metadataPath, "recovered", attempt + 1);
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
// Concurrency: another owner may have published while an earlier identity probe was pending.
|
|
17
|
+
if (JSON.stringify(await readEndpointMetadata(metadataPath)) !== JSON.stringify(metadata))
|
|
18
|
+
continue;
|
|
19
|
+
if (probe !== "dead") {
|
|
20
|
+
await diagnostic(metadataPath, probe, attempt + 1);
|
|
21
|
+
throw Object.assign(new Error(probe === "unresponsive"
|
|
22
|
+
? "A1's background service is not responding; retry shortly. Your sessions have been preserved."
|
|
23
|
+
: "A1's background service identity changed; retry startup. Your sessions have been preserved."), {
|
|
24
|
+
code: "SUPERVISOR_RECOVERY_BLOCKED",
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
recovered = true;
|
|
28
|
+
if (!await removeEndpointArtifacts(metadataPath, endpoint, metadata))
|
|
29
|
+
continue;
|
|
30
|
+
await diagnostic(metadataPath, "stale-owner-retired", attempt + 1);
|
|
31
|
+
}
|
|
32
|
+
// Concurrency: the child takes the same endpoint gate before binding/publication. If another launch wins,
|
|
33
|
+
// startAndWait joins its authenticated endpoint; it never unlinks that winner's socket.
|
|
34
|
+
await options.startAndWait();
|
|
35
|
+
if (recovered)
|
|
36
|
+
await diagnostic(metadataPath, "recovered", attempt + 1);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
await diagnostic(metadataPath, "ownership-changing", 3);
|
|
40
|
+
throw Object.assign(new Error("A1 startup changed repeatedly; retry shortly"), { code: "SUPERVISOR_RECOVERY_BUSY" });
|
|
41
|
+
}
|
|
42
|
+
async function diagnostic(path, reason, attempts) {
|
|
43
|
+
// Security: one bounded summary per endpoint, no raw metadata, user paths, or session content.
|
|
44
|
+
await writeFile(`${path}.recovery.json`, JSON.stringify({ reason, attempts, at: new Date().toISOString() }), { mode: 0o600 }).catch(() => undefined);
|
|
45
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { OwnershipProbe } from "./cohort-selection.js";
|
|
2
|
+
import { type ObserveSupervisorProcess } from "./supervisor-identity.js";
|
|
2
3
|
import type { SupervisorEndpointMetadata } from "./cohort-state.js";
|
|
3
4
|
import type { CohortEndpointPaths, ProductPaths } from "../lifecycle/index.js";
|
|
4
5
|
export interface RecordedEndpoint {
|
|
@@ -20,5 +21,12 @@ export declare function sweepDeadEndpoints(paths: ProductPaths): Promise<readonl
|
|
|
20
21
|
/** The releases that an identity-verified live cohort runs from and that must be retained. */
|
|
21
22
|
export declare function liveReleaseIds(paths: ProductPaths): Promise<readonly string[]>;
|
|
22
23
|
export declare function readEndpointMetadata(path: string): Promise<SupervisorEndpointMetadata | null>;
|
|
23
|
-
export
|
|
24
|
-
|
|
24
|
+
export interface OwnershipProbeOptions {
|
|
25
|
+
readonly observe?: ObserveSupervisorProcess;
|
|
26
|
+
readonly attempts?: number;
|
|
27
|
+
readonly timeoutMs?: number;
|
|
28
|
+
}
|
|
29
|
+
/** Reprobe transient failures; only native absence or a replaced generation establishes death. */
|
|
30
|
+
export declare function probeOwnership(metadata: SupervisorEndpointMetadata, options?: OwnershipProbeOptions): Promise<OwnershipProbe>;
|
|
31
|
+
/** Retire only a still-dead exact publication. Socket removal belongs to the gated listener. */
|
|
32
|
+
export declare function removeEndpointArtifacts(metadataPath: string, endpoint: string, expected: SupervisorEndpointMetadata): Promise<boolean>;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { readdir, readFile, rm } from "node:fs/promises";
|
|
2
2
|
import { connect } from "node:net";
|
|
3
|
-
import { platform } from "node:os";
|
|
4
3
|
import { resolve } from "node:path";
|
|
5
4
|
import { PRODUCT_IDENTITY } from "../../product-identity.js";
|
|
6
5
|
import { encodeFrame, LineFrameDecoder } from "../protocol/index.js";
|
|
7
|
-
import {
|
|
6
|
+
import { supervisorProcessState } from "./supervisor-identity.js";
|
|
7
|
+
import { withEndpointLock } from "./endpoint-lock.js";
|
|
8
8
|
/**
|
|
9
9
|
* Every endpoint recorded under this runtime directory: one per cohort, plus the single
|
|
10
10
|
* endpoint published by releases that predate cohort-scoped identity. More than one can be
|
|
@@ -12,9 +12,12 @@ import { processIsAlive } from "./process-cleanup.js";
|
|
|
12
12
|
*/
|
|
13
13
|
export async function listRecordedEndpoints(paths) {
|
|
14
14
|
const recorded = [];
|
|
15
|
-
|
|
15
|
+
// Compatibility: a cohort listener passes its scoped paths here; legacy discovery still
|
|
16
|
+
// names the runtime directory's fixed metadata path, not that listener's own record.
|
|
17
|
+
const legacyPath = resolve(paths.runtimeDir, PRODUCT_IDENTITY.endpoint.metadataFilename);
|
|
18
|
+
const legacy = await readEndpointMetadata(legacyPath);
|
|
16
19
|
if (legacy) {
|
|
17
|
-
recorded.push({ metadata: legacy, paths: { endpoint:
|
|
20
|
+
recorded.push({ metadata: legacy, paths: { endpoint: legacy.endpoint, endpointMetadataPath: legacyPath } });
|
|
18
21
|
}
|
|
19
22
|
const names = await readdir(paths.endpointsDir).catch(() => []);
|
|
20
23
|
for (const name of names) {
|
|
@@ -35,10 +38,11 @@ export async function listRecordedEndpoints(paths) {
|
|
|
35
38
|
export async function sweepDeadEndpoints(paths) {
|
|
36
39
|
const swept = [];
|
|
37
40
|
for (const recorded of await listRecordedEndpoints(paths)) {
|
|
38
|
-
if (
|
|
41
|
+
if (await supervisorProcessState(recorded.metadata) !== "gone")
|
|
39
42
|
continue;
|
|
40
|
-
await removeEndpointArtifacts(recorded.paths.endpointMetadataPath, recorded.paths.endpoint)
|
|
41
|
-
|
|
43
|
+
if (await removeEndpointArtifacts(recorded.paths.endpointMetadataPath, recorded.paths.endpoint, recorded.metadata)) {
|
|
44
|
+
swept.push(recorded.metadata.releaseId);
|
|
45
|
+
}
|
|
42
46
|
}
|
|
43
47
|
return swept;
|
|
44
48
|
}
|
|
@@ -82,18 +86,24 @@ export async function readEndpointMetadata(path) {
|
|
|
82
86
|
return null;
|
|
83
87
|
}
|
|
84
88
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
89
|
+
/** Reprobe transient failures; only native absence or a replaced generation establishes death. */
|
|
90
|
+
export async function probeOwnership(metadata, options = {}) {
|
|
91
|
+
const attempts = options.attempts ?? 3;
|
|
92
|
+
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
93
|
+
const identity = await requestIdentity(metadata.endpoint, options.timeoutMs ?? 500);
|
|
94
|
+
if (identity) {
|
|
95
|
+
return identity.supervisorId === metadata.supervisorId
|
|
96
|
+
&& identity.bootNonce === metadata.bootNonce
|
|
97
|
+
&& identity.pidStartIdentity === metadata.pidStartIdentity
|
|
98
|
+
&& identity.releaseId === metadata.releaseId
|
|
99
|
+
? "live-verified" : "identity-mismatch";
|
|
100
|
+
}
|
|
101
|
+
if (await supervisorProcessState(metadata, options.observe) === "gone")
|
|
102
|
+
return "dead";
|
|
103
|
+
if (attempt + 1 < attempts)
|
|
104
|
+
await new Promise(resolvePromise => setTimeout(resolvePromise, 50));
|
|
105
|
+
}
|
|
106
|
+
return "unresponsive";
|
|
97
107
|
}
|
|
98
108
|
async function requestIdentity(endpoint, timeoutMs) {
|
|
99
109
|
return await new Promise(resolvePromise => {
|
|
@@ -104,6 +114,7 @@ async function requestIdentity(endpoint, timeoutMs) {
|
|
|
104
114
|
if (settled)
|
|
105
115
|
return;
|
|
106
116
|
settled = true;
|
|
117
|
+
clearTimeout(timer);
|
|
107
118
|
socket.destroy();
|
|
108
119
|
resolvePromise(value);
|
|
109
120
|
};
|
|
@@ -121,14 +132,21 @@ async function requestIdentity(endpoint, timeoutMs) {
|
|
|
121
132
|
finish(null);
|
|
122
133
|
}
|
|
123
134
|
});
|
|
124
|
-
setTimeout(() => finish(null), timeoutMs)
|
|
135
|
+
const timer = setTimeout(() => finish(null), timeoutMs);
|
|
125
136
|
});
|
|
126
137
|
}
|
|
127
138
|
function isStringArray(value) {
|
|
128
139
|
return Array.isArray(value) && value.every(item => typeof item === "string" && item.length > 0);
|
|
129
140
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
141
|
+
/** Retire only a still-dead exact publication. Socket removal belongs to the gated listener. */
|
|
142
|
+
export async function removeEndpointArtifacts(metadataPath, endpoint, expected) {
|
|
143
|
+
return await withEndpointLock(metadataPath, async () => {
|
|
144
|
+
const current = await readEndpointMetadata(metadataPath);
|
|
145
|
+
if (!current || current.endpoint !== endpoint || JSON.stringify(current) !== JSON.stringify(expected))
|
|
146
|
+
return false;
|
|
147
|
+
if (await supervisorProcessState(current) !== "gone" || await probeOwnership(current, { attempts: 1 }) !== "dead")
|
|
148
|
+
return false;
|
|
149
|
+
await rm(metadataPath, { force: true });
|
|
150
|
+
return true;
|
|
151
|
+
});
|
|
134
152
|
}
|
|
@@ -8,6 +8,8 @@ export type { ActiveReleaseTransactionReference, CohortState, EndpointOwnership,
|
|
|
8
8
|
export { DEPENDENCY_LAYER_MANIFEST, DEPENDENCY_LAYER_SCHEMA, RUNTIME_PAYLOAD_INVENTORY, dependencyLayerCertificationPath, dependencyReference, generateDependencyRuntimePayload, legacyDependencyLayerCertificationPath, materializeDependencyLayer, readCertifiedDependencyLayer, selectDependencyRuntimePayload, selectPublishedDependencyRuntimePayload, verifyDependencyLayer, } from "./dependency-layer.js";
|
|
9
9
|
export type { DependencyLayerIdentity, DependencyLayerOperationEvent, DependencyLayerReference, GeneratedRuntimePayload, MaterializeDependencyLayerOptions, MaterializedDependencyLayer, PublishedRuntimeFileClassification, PublishedRuntimePayload, ReadCertifiedDependencyLayerOptions, RuntimePayloadExclusion, RuntimePayloadInventory, SelectedRuntimePayload, } from "./dependency-layer.js";
|
|
10
10
|
export { listRecordedEndpoints, liveReleaseIds, probeOwnership, readEndpointMetadata, removeEndpointArtifacts, sweepDeadEndpoints } from "./endpoints.js";
|
|
11
|
+
export { currentSupervisorStartIdentity, observeSupervisorProcess, supervisorProcessState } from "./supervisor-identity.js";
|
|
12
|
+
export { withEndpointLock } from "./endpoint-lock.js";
|
|
11
13
|
export type { RecordedEndpoint } from "./endpoints.js";
|
|
12
14
|
export { cleanupProvenIdleOwner, cleanupVerifiedOwner, processIsAlive } from "./process-cleanup.js";
|
|
13
15
|
export type { CleanupDiagnostics } from "./process-cleanup.js";
|
|
@@ -4,6 +4,8 @@ export { selectOrdinaryLaunchReleaseId } from "./ordinary-launch-selection.js";
|
|
|
4
4
|
export { CohortStateStore, RELEASE_COHORT_SCHEMA, emptyState, planProtectedReleases } from "./cohort-state.js";
|
|
5
5
|
export { DEPENDENCY_LAYER_MANIFEST, DEPENDENCY_LAYER_SCHEMA, RUNTIME_PAYLOAD_INVENTORY, dependencyLayerCertificationPath, dependencyReference, generateDependencyRuntimePayload, legacyDependencyLayerCertificationPath, materializeDependencyLayer, readCertifiedDependencyLayer, selectDependencyRuntimePayload, selectPublishedDependencyRuntimePayload, verifyDependencyLayer, } from "./dependency-layer.js";
|
|
6
6
|
export { listRecordedEndpoints, liveReleaseIds, probeOwnership, readEndpointMetadata, removeEndpointArtifacts, sweepDeadEndpoints } from "./endpoints.js";
|
|
7
|
+
export { currentSupervisorStartIdentity, observeSupervisorProcess, supervisorProcessState } from "./supervisor-identity.js";
|
|
8
|
+
export { withEndpointLock } from "./endpoint-lock.js";
|
|
7
9
|
export { cleanupProvenIdleOwner, cleanupVerifiedOwner, processIsAlive } from "./process-cleanup.js";
|
|
8
10
|
export { PRODUCT_PACKAGE_NAME, createReleaseIdentity, deriveReleaseIdentity, digestManifestFiles, discoverReleasePayload, packageRootFromModule, releaseFileIdentity, resolveWithin, } from "./release.js";
|
|
9
11
|
export { collectRelease, prepareReleaseCleanup, runBoundedReleaseCleanup, runReleaseCleanupWorker, scheduleReleaseCleanup } from "./release-gc.js";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import { supervisorProcessState } from "./supervisor-identity.js";
|
|
2
3
|
export function processIsAlive(pid) {
|
|
3
4
|
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
4
5
|
return false;
|
|
@@ -15,17 +16,26 @@ export async function cleanupProvenIdleOwner(metadata, graceMs = 1_500) {
|
|
|
15
16
|
}
|
|
16
17
|
export async function cleanupVerifiedOwner(metadata, options = {}) {
|
|
17
18
|
const graceMs = options.graceMs ?? 1_500;
|
|
18
|
-
if (!options.allowLiveInstances && (metadata.ownership.liveInstanceIds.length > 0 || metadata.ownership.nonResumableInstanceIds.length > 0)) {
|
|
19
|
+
if (!options.allowLiveInstances && (metadata.ownership.liveInstanceIds.length > 0 || metadata.ownership.nonResumableInstanceIds.length > 0 || metadata.ownership.uncertainInstanceIds.length > 0)) {
|
|
19
20
|
throw new Error("refusing to terminate an owner with recorded live generations");
|
|
20
21
|
}
|
|
21
22
|
const started = Date.now();
|
|
22
23
|
const attempted = [];
|
|
23
|
-
|
|
24
|
+
const state = await supervisorProcessState(metadata);
|
|
25
|
+
if (state === "gone")
|
|
24
26
|
return { pid: metadata.pid, attempted: ["owner-already-dead"], terminated: true, elapsedMs: Date.now() - started };
|
|
27
|
+
// Security: ordinary recovery never signals a live owner, even with an empty durable list.
|
|
28
|
+
// Explicit update cleanup also needs the exact native generation, not merely an old handshake.
|
|
29
|
+
if (!options.allowLiveInstances || state !== "same") {
|
|
30
|
+
return { pid: metadata.pid, attempted: ["live-owner-preserved"], terminated: false, elapsedMs: Date.now() - started };
|
|
31
|
+
}
|
|
25
32
|
attempted.push("graceful-termination");
|
|
26
33
|
await terminate(metadata.pid, false);
|
|
27
34
|
if (await waitUntilDead(metadata.pid, graceMs))
|
|
28
35
|
return { pid: metadata.pid, attempted, terminated: true, elapsedMs: Date.now() - started };
|
|
36
|
+
if (await supervisorProcessState(metadata) !== "same") {
|
|
37
|
+
return { pid: metadata.pid, attempted, terminated: await supervisorProcessState(metadata) === "gone", elapsedMs: Date.now() - started };
|
|
38
|
+
}
|
|
29
39
|
attempted.push("forced-process-tree-termination");
|
|
30
40
|
await terminate(metadata.pid, true);
|
|
31
41
|
const terminated = await waitUntilDead(metadata.pid, graceMs);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { NativeProcessIdentity } from "../lifecycle/index.js";
|
|
2
|
+
import type { SupervisorEndpointMetadata } from "./cohort-state.js";
|
|
3
|
+
export type SupervisorProcessState = "same" | "gone" | "unknown";
|
|
4
|
+
export type ObserveSupervisorProcess = (pid: number) => Promise<NativeProcessIdentity | null>;
|
|
5
|
+
/** Inspection code comes from this runtime, never from a path in a stale endpoint record. */
|
|
6
|
+
export declare function observeSupervisorProcess(pid: number): Promise<NativeProcessIdentity | null>;
|
|
7
|
+
/** New supervisors use a native creation token as their opaque handshake identity as well. */
|
|
8
|
+
export declare function currentSupervisorStartIdentity(): string;
|
|
9
|
+
/** The current runtime also owns the OS gate ABI; metadata cannot choose a different addon. */
|
|
10
|
+
export declare function acquireSupervisorGate(path: string): {
|
|
11
|
+
readonly acquired: boolean;
|
|
12
|
+
release(): void;
|
|
13
|
+
};
|
|
14
|
+
/** A failed inspection is uncertainty, never permission to erase a possibly live owner. */
|
|
15
|
+
export declare function supervisorProcessState(metadata: SupervisorEndpointMetadata, observe?: ObserveSupervisorProcess): Promise<SupervisorProcessState>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { fileURLToPath } from "node:url";
|
|
2
|
+
import { createNativeProcessInspector, loadProcessGuardianAddon, resolveProcessGuardianPath } from "../process-containment/index.js";
|
|
3
|
+
const ownRoot = fileURLToPath(new URL("../../../", import.meta.url));
|
|
4
|
+
/** Inspection code comes from this runtime, never from a path in a stale endpoint record. */
|
|
5
|
+
export async function observeSupervisorProcess(pid) {
|
|
6
|
+
return await (await createNativeProcessInspector(process.env, process.platform, ownRoot)).observe(pid);
|
|
7
|
+
}
|
|
8
|
+
/** New supervisors use a native creation token as their opaque handshake identity as well. */
|
|
9
|
+
export function currentSupervisorStartIdentity() {
|
|
10
|
+
try {
|
|
11
|
+
const token = loadProcessGuardianAddon(resolveProcessGuardianPath(ownRoot, process.env)).inspectProcessStart(process.pid);
|
|
12
|
+
if (!validNativeToken(token))
|
|
13
|
+
throw new Error("native process identity is unavailable");
|
|
14
|
+
return token;
|
|
15
|
+
}
|
|
16
|
+
catch (cause) {
|
|
17
|
+
throw Object.assign(new Error("cannot verify A1's background service process; check installation and process-inspection permissions", { cause }), { code: "SUPERVISOR_RECOVERY_UNAVAILABLE" });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** The current runtime also owns the OS gate ABI; metadata cannot choose a different addon. */
|
|
21
|
+
export function acquireSupervisorGate(path) {
|
|
22
|
+
const addon = loadProcessGuardianAddon(resolveProcessGuardianPath(ownRoot, process.env));
|
|
23
|
+
if (!addon.tryAcquireProcessGate)
|
|
24
|
+
throw new Error("this installation lacks startup recovery support; update or repair A1");
|
|
25
|
+
return addon.tryAcquireProcessGate(path);
|
|
26
|
+
}
|
|
27
|
+
/** A failed inspection is uncertainty, never permission to erase a possibly live owner. */
|
|
28
|
+
export async function supervisorProcessState(metadata, observe = observeSupervisorProcess) {
|
|
29
|
+
let identity;
|
|
30
|
+
try {
|
|
31
|
+
identity = await observe(metadata.pid);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return "unknown";
|
|
35
|
+
}
|
|
36
|
+
if (identity === null)
|
|
37
|
+
return "gone";
|
|
38
|
+
if (identity.pid !== metadata.pid)
|
|
39
|
+
return "unknown";
|
|
40
|
+
if (metadata.nativeStartIdentity !== undefined) {
|
|
41
|
+
// Compatibility: only compare tokens of the same native format, not an old Node timestamp.
|
|
42
|
+
if (!validNativeToken(metadata.nativeStartIdentity) || !validNativeToken(identity.startIdentity)
|
|
43
|
+
|| identity.startIdentity.split(":")[0] !== metadata.nativeStartIdentity.split(":")[0])
|
|
44
|
+
return "unknown";
|
|
45
|
+
return identity.startIdentity === metadata.nativeStartIdentity ? "same" : "gone";
|
|
46
|
+
}
|
|
47
|
+
// Compatibility: legacy publication happened after process creation. A native creation time
|
|
48
|
+
// strictly later than that publication (with a one-second clock margin) disproves ownership;
|
|
49
|
+
// a near timestamp is NOT proof of ownership. Linux tick-only tokens cannot prove this.
|
|
50
|
+
const published = Date.parse(metadata.startedAt);
|
|
51
|
+
const created = nativeCreationTime(identity.startIdentity);
|
|
52
|
+
return Number.isFinite(published) && created !== null && created > published + 1_000 ? "gone" : "unknown";
|
|
53
|
+
}
|
|
54
|
+
function validNativeToken(value) {
|
|
55
|
+
return typeof value === "string" && /^(?:windows-filetime:\d{1,20}|linux-proc-start:\d{1,20}|darwin-proc-start:\d{1,20}:\d{1,6})$/.test(value);
|
|
56
|
+
}
|
|
57
|
+
function nativeCreationTime(identity) {
|
|
58
|
+
const windows = /^windows-filetime:(\d+)$/.exec(identity);
|
|
59
|
+
if (windows)
|
|
60
|
+
return Number(BigInt(windows[1]) / 10000n - 11644473600000n);
|
|
61
|
+
const darwin = /^darwin-proc-start:(\d+):(\d+)$/.exec(identity);
|
|
62
|
+
return darwin ? Number(darwin[1]) * 1_000 + Number(darwin[2]) / 1_000 : null;
|
|
63
|
+
}
|
|
@@ -155,7 +155,7 @@ export function createUpdateLifecycleCoordinator(environment = process.env, file
|
|
|
155
155
|
}
|
|
156
156
|
const plan = planUpdateOwnership(ownership, await canonicalImmutableRoot(paths.dataDir, endpoint.releaseRoot));
|
|
157
157
|
if (plan === "clean-dead-record") {
|
|
158
|
-
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint);
|
|
158
|
+
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint, endpoint);
|
|
159
159
|
return { priorActiveVersion };
|
|
160
160
|
}
|
|
161
161
|
// Rationale: leaving sessions alone is the expected outcome, and saying so mid-update tears
|
|
@@ -182,7 +182,7 @@ export function createUpdateLifecycleCoordinator(environment = process.env, file
|
|
|
182
182
|
throw new Error(`verified ${PRODUCT_TEXT.displayName} owner ${endpoint.pid} did not terminate`);
|
|
183
183
|
});
|
|
184
184
|
}
|
|
185
|
-
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint);
|
|
185
|
+
await removeEndpointArtifacts(owner.paths.endpointMetadataPath, owner.paths.endpoint, endpoint);
|
|
186
186
|
return { priorActiveVersion };
|
|
187
187
|
},
|
|
188
188
|
async verifyPackageUnlocked(packageRoot) {
|
|
@@ -5,14 +5,16 @@ export declare class ControlStore {
|
|
|
5
5
|
#private;
|
|
6
6
|
readonly database: DatabaseSync;
|
|
7
7
|
readonly bootNonce: string | null;
|
|
8
|
-
constructor(path: string, bootNonce?: string | null
|
|
8
|
+
constructor(path: string, bootNonce?: string | null, options?: {
|
|
9
|
+
readonly reconcileOnOpen?: boolean;
|
|
10
|
+
});
|
|
9
11
|
migrate(): void;
|
|
10
12
|
createLaunchInstance(instance: LaunchInstance): void;
|
|
11
13
|
activateLaunchInstance(instanceId: LaunchInstanceId, ownerClientId: string, rootIdentity: NativeProcessIdentity, containmentIdentity: ProcessContainmentIdentity, activatedAt: string): boolean;
|
|
12
14
|
beginLaunchInstanceStop(instanceId: LaunchInstanceId, ownerClientId: string, stoppingAt: string): boolean;
|
|
13
15
|
completeLaunchInstance(instanceId: LaunchInstanceId, ownerClientId: string, terminalState: "completed" | "interrupted", outcome: LaunchInstanceOutcome, completedAt: string): boolean;
|
|
14
|
-
loadActiveLaunchInstances(): LaunchInstance[];
|
|
16
|
+
loadActiveLaunchInstances(bootNonce?: string): LaunchInstance[];
|
|
15
17
|
loadLaunchInstance(instanceId: LaunchInstanceId): LaunchInstance | null;
|
|
16
|
-
reconcilePriorBootLaunchInstances(bootNonce: string, reconciledAt?: string): number;
|
|
18
|
+
reconcilePriorBootLaunchInstances(bootNonce: string, reconciledAt?: string, protectedBoots?: readonly string[]): number;
|
|
17
19
|
close(): void;
|
|
18
20
|
}
|
|
@@ -30,7 +30,7 @@ const LAUNCH_INSTANCES_TABLE = `
|
|
|
30
30
|
export class ControlStore {
|
|
31
31
|
database;
|
|
32
32
|
bootNonce;
|
|
33
|
-
constructor(path, bootNonce = null) {
|
|
33
|
+
constructor(path, bootNonce = null, options = {}) {
|
|
34
34
|
this.bootNonce = bootNonce;
|
|
35
35
|
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
36
36
|
this.database = new DatabaseSync(path);
|
|
@@ -38,7 +38,7 @@ export class ControlStore {
|
|
|
38
38
|
this.database.exec("PRAGMA journal_mode = WAL");
|
|
39
39
|
this.database.exec("PRAGMA foreign_keys = ON");
|
|
40
40
|
this.migrate();
|
|
41
|
-
if (bootNonce !== null)
|
|
41
|
+
if (bootNonce !== null && options.reconcileOnOpen !== false)
|
|
42
42
|
this.reconcilePriorBootLaunchInstances(bootNonce);
|
|
43
43
|
}
|
|
44
44
|
catch (error) {
|
|
@@ -284,16 +284,16 @@ export class ControlStore {
|
|
|
284
284
|
.run(terminalState, completedAt, JSON.stringify(outcome), instanceId, ownerClientId, this.bootNonce);
|
|
285
285
|
return Number(result.changes) === 1;
|
|
286
286
|
}
|
|
287
|
-
loadActiveLaunchInstances() {
|
|
287
|
+
loadActiveLaunchInstances(bootNonce) {
|
|
288
288
|
return this.database.prepare(`SELECT * FROM launch_instances
|
|
289
|
-
WHERE state IN ('requested', 'active', 'stopping') ORDER BY created_at, id`).all()
|
|
289
|
+
WHERE state IN ('requested', 'active', 'stopping') ${bootNonce === undefined ? "" : "AND owner_boot_nonce = ?"} ORDER BY created_at, id`).all(...(bootNonce === undefined ? [] : [bootNonce]))
|
|
290
290
|
.map(launchInstanceFromRow);
|
|
291
291
|
}
|
|
292
292
|
loadLaunchInstance(instanceId) {
|
|
293
293
|
const row = this.database.prepare("SELECT * FROM launch_instances WHERE id = ?").get(instanceId);
|
|
294
294
|
return row ? launchInstanceFromRow(row) : null;
|
|
295
295
|
}
|
|
296
|
-
reconcilePriorBootLaunchInstances(bootNonce, reconciledAt = new Date().toISOString()) {
|
|
296
|
+
reconcilePriorBootLaunchInstances(bootNonce, reconciledAt = new Date().toISOString(), protectedBoots = []) {
|
|
297
297
|
const outcome = {
|
|
298
298
|
kind: "interrupted",
|
|
299
299
|
reason: "supervisor-disconnect",
|
|
@@ -301,8 +301,9 @@ export class ControlStore {
|
|
|
301
301
|
};
|
|
302
302
|
const result = this.database.prepare(`UPDATE launch_instances
|
|
303
303
|
SET state = 'interrupted', completed_at = ?, outcome_json = ?
|
|
304
|
-
WHERE state IN ('requested', 'active', 'stopping') AND owner_boot_nonce <>
|
|
305
|
-
|
|
304
|
+
WHERE state IN ('requested', 'active', 'stopping') AND owner_boot_nonce <> ?
|
|
305
|
+
${protectedBoots.length === 0 ? "" : `AND owner_boot_nonce NOT IN (${protectedBoots.map(() => "?").join(",")})`}`)
|
|
306
|
+
.run(reconciledAt, JSON.stringify(outcome), bootNonce, ...protectedBoots);
|
|
306
307
|
return Number(result.changes);
|
|
307
308
|
}
|
|
308
309
|
close() {
|
|
@@ -36,7 +36,9 @@ export async function runSupervisor(arguments_ = []) {
|
|
|
36
36
|
paths = { ...productPaths, ...resolveCohortEndpoint(productPaths, release.releaseId) };
|
|
37
37
|
stage = "storage";
|
|
38
38
|
const bootNonce = randomUUID();
|
|
39
|
-
|
|
39
|
+
// Concurrency: opening SQLite does not establish endpoint authority. The winning listener
|
|
40
|
+
// reconciles old boots only after binding; losing startup attempts must not mutate live rows.
|
|
41
|
+
const store = new ControlStore(paths.databasePath, bootNonce, { reconcileOnOpen: false });
|
|
40
42
|
const cohortState = new CohortStateStore(paths.dataDir);
|
|
41
43
|
server = new SupervisorServer(store, paths, release, bootNonce, undefined, undefined, undefined, async () => {
|
|
42
44
|
const transaction = await new UpdateTransactionStore(paths.dataDir).read();
|
|
@@ -5,7 +5,7 @@ import { platform } from "node:os";
|
|
|
5
5
|
import { dirname } from "node:path";
|
|
6
6
|
import { assertLaunchInstanceOutcome, assertNativeProcessIdentity, assertProcessContainmentIdentity, } from "../lifecycle/index.js";
|
|
7
7
|
import { encodeFrame, isCommandMessage, isControlHello, LineFrameDecoder, localControlHello, MAX_CONTROL_FRAME_BYTES, negotiateControlFeatures } from "../protocol/index.js";
|
|
8
|
-
import { processIsAlive } from "../release/index.js";
|
|
8
|
+
import { listRecordedEndpoints, readEndpointMetadata, supervisorProcessState, currentSupervisorStartIdentity, withEndpointLock, processIsAlive } from "../release/index.js";
|
|
9
9
|
import { ControlStore } from "../storage/index.js";
|
|
10
10
|
import { resolveProductPaths } from "./paths.js";
|
|
11
11
|
import { PRODUCT_IDENTITY, PRODUCT_TEXT } from "../../product-identity.js";
|
|
@@ -43,7 +43,7 @@ export async function commitEndpointMetadata(path, source, operations = {}) {
|
|
|
43
43
|
export class SupervisorServer {
|
|
44
44
|
id = randomUUID();
|
|
45
45
|
bootNonce;
|
|
46
|
-
pidStartIdentity =
|
|
46
|
+
pidStartIdentity = currentSupervisorStartIdentity();
|
|
47
47
|
startedAt = new Date().toISOString();
|
|
48
48
|
paths;
|
|
49
49
|
#server = null;
|
|
@@ -84,7 +84,7 @@ export class SupervisorServer {
|
|
|
84
84
|
this.supersededPollMs = supersededPollMs;
|
|
85
85
|
this.bootNonce = bootNonce;
|
|
86
86
|
this.paths = paths;
|
|
87
|
-
for (const instance of store.loadActiveLaunchInstances())
|
|
87
|
+
for (const instance of store.loadActiveLaunchInstances(bootNonce))
|
|
88
88
|
this.#instances.set(instance.id, instance);
|
|
89
89
|
}
|
|
90
90
|
snapshot() {
|
|
@@ -98,41 +98,60 @@ export class SupervisorServer {
|
|
|
98
98
|
};
|
|
99
99
|
}
|
|
100
100
|
async listen() {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
101
|
+
await withEndpointLock(this.paths.endpointMetadataPath, async () => {
|
|
102
|
+
if (this.#server)
|
|
103
|
+
throw new Error("supervisor is already listening");
|
|
104
|
+
await mkdir(this.paths.runtimeDir, { recursive: true, mode: 0o700 });
|
|
105
|
+
await mkdir(this.paths.endpointsDir, { recursive: true, mode: 0o700 });
|
|
106
|
+
const recorded = await readEndpointMetadata(this.paths.endpointMetadataPath);
|
|
107
|
+
if (recorded && await supervisorProcessState(recorded) !== "gone") {
|
|
108
|
+
throw Object.assign(new Error("another supervisor still owns this endpoint"), { code: "EADDRINUSE" });
|
|
109
|
+
}
|
|
110
|
+
if (platform() !== "win32") {
|
|
111
|
+
if (this.paths.endpointDirectory)
|
|
112
|
+
await ensureManagedEndpointDirectory(dirname(this.paths.endpoint));
|
|
113
|
+
const existing = await lstat(this.paths.endpoint).catch(error => {
|
|
114
|
+
if (error.code === "ENOENT")
|
|
115
|
+
return null;
|
|
116
|
+
throw error;
|
|
117
|
+
});
|
|
118
|
+
if (existing) {
|
|
119
|
+
if (await endpointIsLive(this.paths.endpoint)) {
|
|
120
|
+
throw Object.assign(new Error(PRODUCT_TEXT.diagnostic(`supervisor already owns ${this.paths.endpoint}`)), { code: "EADDRINUSE" });
|
|
121
|
+
}
|
|
122
|
+
await rm(this.paths.endpoint, { force: true });
|
|
116
123
|
}
|
|
117
|
-
await rm(this.paths.endpoint, { force: true });
|
|
118
124
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
125
|
+
const server = createServer(socket => this.#attach(socket));
|
|
126
|
+
this.#server = server;
|
|
127
|
+
await new Promise((resolve, reject) => {
|
|
128
|
+
server.once("error", reject);
|
|
129
|
+
server.listen(this.paths.endpoint, () => {
|
|
130
|
+
this.#ownsEndpoint = true;
|
|
131
|
+
server.off("error", reject);
|
|
132
|
+
resolve();
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
// Concurrency: only the bound winner may reconcile shared control data. Other cohort boots
|
|
136
|
+
// remain protected even when temporarily unresponsive; their instances never become ours.
|
|
137
|
+
await withEndpointLock(this.paths.databasePath, async () => {
|
|
138
|
+
// Concurrency: all cohorts sharing SQLite serialize boot reconciliation plus publication, so a new
|
|
139
|
+
// cohort cannot insert instances between another cohort's inventory and stale-row update.
|
|
140
|
+
const protectedBoots = [];
|
|
141
|
+
for (const other of await listRecordedEndpoints(this.paths)) {
|
|
142
|
+
if (other.paths.endpointMetadataPath !== this.paths.endpointMetadataPath
|
|
143
|
+
&& await supervisorProcessState(other.metadata) !== "gone")
|
|
144
|
+
protectedBoots.push(other.metadata.bootNonce);
|
|
145
|
+
}
|
|
146
|
+
this.store.reconcilePriorBootLaunchInstances(this.bootNonce, new Date().toISOString(), protectedBoots);
|
|
147
|
+
await this.#writeEndpointMetadata();
|
|
128
148
|
});
|
|
149
|
+
if (this.readActiveReleaseId) {
|
|
150
|
+
const poll = setInterval(() => void this.#refreshCohortRole(), this.supersededPollMs);
|
|
151
|
+
poll.unref();
|
|
152
|
+
this.#supersededPoll = poll;
|
|
153
|
+
}
|
|
129
154
|
});
|
|
130
|
-
await this.#writeEndpointMetadata();
|
|
131
|
-
if (this.readActiveReleaseId) {
|
|
132
|
-
const poll = setInterval(() => void this.#refreshCohortRole(), this.supersededPollMs);
|
|
133
|
-
poll.unref();
|
|
134
|
-
this.#supersededPoll = poll;
|
|
135
|
-
}
|
|
136
155
|
}
|
|
137
156
|
// Invariant: superseding a cohort never interrupts its held instances; it retires only when empty.
|
|
138
157
|
async #refreshCohortRole() {
|
|
@@ -167,27 +186,29 @@ export class SupervisorServer {
|
|
|
167
186
|
this.#supersededPoll = null;
|
|
168
187
|
for (const client of this.#clients)
|
|
169
188
|
client.destroy();
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
189
|
+
await withEndpointLock(this.paths.endpointMetadataPath, async () => {
|
|
190
|
+
try {
|
|
191
|
+
await this.#metadataWrites.catch(() => undefined);
|
|
192
|
+
if (this.#ownsEndpoint) {
|
|
193
|
+
// Concurrency: remove only our publication, before releasing the listening endpoint.
|
|
194
|
+
// A contender that failed to bind must never erase the successful owner's discovery file.
|
|
195
|
+
const metadata = await readFile(this.paths.endpointMetadataPath, "utf8").then(source => JSON.parse(source)).catch(() => null);
|
|
196
|
+
if (metadata?.supervisorId === this.id && metadata.bootNonce === this.bootNonce) {
|
|
197
|
+
await rm(this.paths.endpointMetadataPath, { force: true });
|
|
198
|
+
}
|
|
199
|
+
// Platform: Node unlinks its Unix socket when the server closes; unlinking it early would
|
|
200
|
+
// let a replacement bind before the old server finishes releasing that path.
|
|
178
201
|
}
|
|
179
|
-
// Platform: Node unlinks its Unix socket when the server closes; unlinking it early would
|
|
180
|
-
// let a replacement bind before the old server finishes releasing that path.
|
|
181
202
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
}
|
|
203
|
+
finally {
|
|
204
|
+
// Platform: a metadata sharing error must not leave a bound but undiscoverable listener.
|
|
205
|
+
if (this.#server)
|
|
206
|
+
await new Promise(resolve => this.#server?.close(() => resolve()));
|
|
207
|
+
this.#server = null;
|
|
208
|
+
this.#ownsEndpoint = false;
|
|
209
|
+
this.store.close();
|
|
210
|
+
}
|
|
211
|
+
});
|
|
191
212
|
}
|
|
192
213
|
async closeForReleaseReplacement(stopAgents) {
|
|
193
214
|
await this.close(stopAgents);
|
|
@@ -518,6 +539,7 @@ export class SupervisorServer {
|
|
|
518
539
|
supervisorId: this.id,
|
|
519
540
|
pid: process.pid,
|
|
520
541
|
pidStartIdentity: this.pidStartIdentity,
|
|
542
|
+
nativeStartIdentity: this.pidStartIdentity,
|
|
521
543
|
bootNonce: this.bootNonce,
|
|
522
544
|
endpoint: this.paths.endpoint,
|
|
523
545
|
databasePath: this.paths.databasePath,
|
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
"platform": "darwin",
|
|
8
8
|
"architecture": "arm64",
|
|
9
9
|
"capability": "supported",
|
|
10
|
-
"builtAt": "2026-11-
|
|
10
|
+
"builtAt": "2026-11-06T08:23:55.616Z",
|
|
11
11
|
"artifact": {
|
|
12
12
|
"filename": "process-guardian.node",
|
|
13
|
-
"sha256": "
|
|
14
|
-
"size":
|
|
13
|
+
"sha256": "9ab579426034c752ef8ed53a789566f5b7cfe102149d30c28f4bdb60e73107c5",
|
|
14
|
+
"size": 456272
|
|
15
15
|
},
|
|
16
16
|
"provenance": {
|
|
17
17
|
"language": "Rust",
|
|
Binary file
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"architecture": "arm64",
|
|
10
10
|
"target": "aarch64-apple-darwin",
|
|
11
11
|
"capability": "supported",
|
|
12
|
-
"builtAt": "2026-11-
|
|
12
|
+
"builtAt": "2026-11-06T08:27:51.656Z",
|
|
13
13
|
"artifact": {
|
|
14
14
|
"filename": "terminal-host.node",
|
|
15
15
|
"sha256": "b2548f95b9660b11b6d28fb4b35f254cde9940d9d19233517bb70507ffcf76ab",
|
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
"platform": "darwin",
|
|
8
8
|
"architecture": "x64",
|
|
9
9
|
"capability": "supported",
|
|
10
|
-
"builtAt": "2026-11-
|
|
10
|
+
"builtAt": "2026-11-06T08:24:02.357Z",
|
|
11
11
|
"artifact": {
|
|
12
12
|
"filename": "process-guardian.node",
|
|
13
|
-
"sha256": "
|
|
14
|
-
"size":
|
|
13
|
+
"sha256": "65f3a469a6faa9b578bc095e7559012775ee85e5b93cbc9451c62604f769146e",
|
|
14
|
+
"size": 452648
|
|
15
15
|
},
|
|
16
16
|
"provenance": {
|
|
17
17
|
"language": "Rust",
|
|
Binary file
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"architecture": "x64",
|
|
10
10
|
"target": "x86_64-apple-darwin",
|
|
11
11
|
"capability": "supported",
|
|
12
|
-
"builtAt": "2026-11-
|
|
12
|
+
"builtAt": "2026-11-06T08:31:58.733Z",
|
|
13
13
|
"artifact": {
|
|
14
14
|
"filename": "terminal-host.node",
|
|
15
15
|
"sha256": "cd1103472889d53e2e039cf7673b2fe6e9e0937234591b1202234d0c1c357d2a",
|
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
"platform": "linux",
|
|
8
8
|
"architecture": "x64",
|
|
9
9
|
"capability": "supported",
|
|
10
|
-
"builtAt": "2026-11-
|
|
10
|
+
"builtAt": "2026-11-06T08:23:51.242Z",
|
|
11
11
|
"artifact": {
|
|
12
12
|
"filename": "process-guardian.node",
|
|
13
|
-
"sha256": "
|
|
14
|
-
"size":
|
|
13
|
+
"sha256": "64232d4c827bb7f69c14612986f9b6780a0593d8842cd039a411b6bfe29cd5da",
|
|
14
|
+
"size": 500264
|
|
15
15
|
},
|
|
16
16
|
"provenance": {
|
|
17
17
|
"language": "Rust",
|
|
Binary file
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"architecture": "x64",
|
|
10
10
|
"target": "x86_64-unknown-linux-gnu",
|
|
11
11
|
"capability": "supported",
|
|
12
|
-
"builtAt": "2026-11-
|
|
12
|
+
"builtAt": "2026-11-06T08:26:29.024Z",
|
|
13
13
|
"artifact": {
|
|
14
14
|
"filename": "terminal-host.node",
|
|
15
15
|
"sha256": "f5abfe9464780c9acdf85dfbf8a0f194bf0355188ab23feaa09cb7df84363088",
|
|
@@ -7,11 +7,11 @@
|
|
|
7
7
|
"platform": "win32",
|
|
8
8
|
"architecture": "x64",
|
|
9
9
|
"capability": "supported",
|
|
10
|
-
"builtAt": "2026-11-
|
|
10
|
+
"builtAt": "2026-11-06T08:24:41.351Z",
|
|
11
11
|
"artifact": {
|
|
12
12
|
"filename": "process-guardian.node",
|
|
13
|
-
"sha256": "
|
|
14
|
-
"size":
|
|
13
|
+
"sha256": "dab6b6aed5ab24435d7814fc5099f26707486ee187551ff1dfd939679682f32b",
|
|
14
|
+
"size": 414208
|
|
15
15
|
},
|
|
16
16
|
"provenance": {
|
|
17
17
|
"language": "Rust",
|
|
Binary file
|
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
"architecture": "x64",
|
|
10
10
|
"target": "x86_64-pc-windows-msvc",
|
|
11
11
|
"capability": "supported",
|
|
12
|
-
"builtAt": "2026-11-
|
|
12
|
+
"builtAt": "2026-11-06T08:29:00.280Z",
|
|
13
13
|
"artifact": {
|
|
14
14
|
"filename": "terminal-host.node",
|
|
15
|
-
"sha256": "
|
|
15
|
+
"sha256": "00dc90f70d76ff36418a301d36dd406d143a6bf1da6b56e24ae5341b4e0c7d8e",
|
|
16
16
|
"size": 7131136
|
|
17
17
|
},
|
|
18
18
|
"entry": {
|
|
Binary file
|
|
@@ -47,6 +47,8 @@ Three layer boundaries hold regardless of the DAG: `src/contracts/*` import noth
|
|
|
47
47
|
|
|
48
48
|
`src/cli` contains command policy but delegates runtime work. `src/composition` is the concrete dependency-injection boundary: it may know both neutral contracts and Pi implementations, while product features receive vendor-neutral ports. Foundation modules never import product features. Pi package knowledge remains inside the Pi adapter owners.
|
|
49
49
|
|
|
50
|
+
Release-owned endpoint observation and recovery coordinate with supervision-owned publication and storage-owned boot reconciliation through the process guardian's OS-released gates; see [startup recovery](../launch-recovery.md) for the identity, concurrency, and diagnostic boundaries.
|
|
51
|
+
|
|
50
52
|
Shared code moves to foundation only when it is product-agnostic, has a precise contract, and has more than one real consumer. Do not create `core`, `common`, `utils`, or `misc` dumping grounds.
|
|
51
53
|
|
|
52
54
|
## Feature ownership
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Startup recovery after process termination
|
|
2
|
+
|
|
3
|
+
A1 reconciles recoverable runtime ownership automatically when the user starts it again after a process crash or forced termination. Users should not need to find PIDs, delete session or control databases, restart a supervisor, or reboot to clear a stale record.
|
|
4
|
+
|
|
5
|
+
## Recovery boundary
|
|
6
|
+
|
|
7
|
+
Supervisor metadata carries an additive native process-creation token. Endpoint handshakes still use the existing supervisor, boot, release, and opaque start-identity fields for compatibility; new supervisors use their native token in that start field as well. A failed probe is retried; native process inspection distinguishes a terminated process object or reused PID from the recorded owner. Older records remain readable: an absent process is reclaimable, while a live legacy PID is not treated as dead merely because its timestamp format differs. Windows and macOS native creation times strictly later than a legacy publication, with a one-second margin, can disprove that recorded ownership. Tick-only Linux legacy evidence remains conservative until absence or a matching handshake establishes the outcome.
|
|
8
|
+
|
|
9
|
+
Endpoint retirement requires the same publication and a fresh dead-owner observation under a native file-descriptor gate. The gate is never deleted; the OS releases its lock on exit, including forced exit. Supervisor binding, Unix stale-socket removal, metadata publication, and closure use that gate. Losing startups cannot delete the winner's endpoint. A separate shared-database gate serializes boot reconciliation and publication across cohorts. Only the bound winner reconciles old rows, and recorded surviving cohort boots remain protected.
|
|
10
|
+
|
|
11
|
+
Successful recovery is silent and continues the original profile, launch mode, and session selection. Saved sessions are not deleted or rewritten by endpoint recovery. Resident holders and their session-writer locks retain their independent ownership; restarting a supervisor is not permission to stop a resident tab or create a second session writer.
|
|
12
|
+
|
|
13
|
+
## Exceptional failures
|
|
14
|
+
|
|
15
|
+
An uninspectable or genuinely unresponsive live owner is preserved after bounded retries. Empty saved instance lists are not authority to terminate it. The command returns a short recovery message instead of an uncaught stack or the old internal uncertain-live-ownership paragraph. Missing native recovery support or inaccessible runtime storage remains an installation or permission failure, not proof that an owner is dead.
|
|
16
|
+
|
|
17
|
+
A recovery attempt may replace one bounded `<endpoint-metadata>.recovery.json` summary containing only a reason, attempt count, and timestamp. It is diagnostic only, contains no session data or credentials, and never grants cleanup authority. The `.authority` gate files are also not user cleanup targets.
|
|
18
|
+
|
|
19
|
+
This restores launch availability, not killed process memory. Killing every Node process also kills the terminal-restoration watcher; no alternative executable is started to evade that limitation. Real OS permission failures, damaged installations, and persistently live unresponsive services cannot be converted into safe success by ignoring ownership.
|
|
20
|
+
|
|
21
|
+
## Regression boundaries
|
|
22
|
+
|
|
23
|
+
The focused fixtures use isolated runtime/data roots and recorded fixture process identities. They cover supervisor, guardian, root-runtime, recovery-gate, PID-reuse, concurrent-startup, and Windows all-recorded-Node loss, plus native resident server/holder/tab loss in the resident lane. They never terminate processes by image name. Another managed runtime, a separate cohort, saved session bytes, and an unrelated sentinel process establish the preservation boundaries.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@timurproko/a1",
|
|
3
|
-
"version": "0.2.7-dev.
|
|
3
|
+
"version": "0.2.7-dev.1245",
|
|
4
4
|
"description": "Standalone terminal workspace for supervised native and managed agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"privateLaunchContract": "neutral-launch-v1",
|
|
@@ -126,7 +126,7 @@
|
|
|
126
126
|
"@mariozechner/clipboard": "^0.3.9"
|
|
127
127
|
},
|
|
128
128
|
"publication": {
|
|
129
|
-
"source": "
|
|
130
|
-
"pullRequest":
|
|
129
|
+
"source": "fc4a2588e22b6d90034465d80d7923c8fede91d8",
|
|
130
|
+
"pullRequest": 870
|
|
131
131
|
}
|
|
132
132
|
}
|