@evomap/evolver-proxy 2.0.2 → 2.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/evolver-proxy.d.ts +12 -0
- package/dist/bin/evolver-proxy.js +192 -59
- package/dist/daemon/proxyDaemon.js +10 -0
- package/dist/daemon/systemdNotifier.d.ts +2 -0
- package/dist/daemon/systemdNotifier.js +11 -1
- package/dist/lifecycle/claimNudge.js +1 -1
- package/dist/lifecycle/deployGuard.js +1 -1
- package/dist/lifecycle/legacyNodeId.js +1 -1
- package/dist/lifecycle/manager.js +1 -1
- package/dist/llm/bodyCapture.js +1 -1
- package/dist/llm/index.js +1 -1
- package/dist/llm/server.js +1 -1
- package/dist/llm/traceBackfill.js +1 -1
- package/dist/llm/traceConfig.js +1 -1
- package/dist/llm/traceControl.js +1 -1
- package/dist/llm/traceEnvelope.js +1 -1
- package/dist/llm/traceSink.js +1 -1
- package/dist/llm/traceUploadPayload.js +1 -1
- package/dist/llm/upstream.js +1 -1
- package/dist/router/cachePassthrough.js +1 -1
- package/dist/router/features.js +1 -1
- package/dist/router/index.js +1 -1
- package/dist/router/messagesRoute.js +1 -1
- package/dist/router/modelRouter.js +1 -1
- package/dist/router/providerRoutes.js +1 -1
- package/dist/router/sseScan.js +1 -1
- package/dist/selfUpdate/bootstrap.d.ts +106 -13
- package/dist/selfUpdate/bootstrap.js +3418 -176
- package/dist/selfUpdate/bootstrapReadiness.d.ts +9 -0
- package/dist/selfUpdate/bootstrapReadiness.js +153 -0
- package/dist/selfUpdate/controllerLifecycleAuthority.d.ts +45 -0
- package/dist/selfUpdate/controllerLifecycleAuthority.js +61 -0
- package/dist/selfUpdate/executor.d.ts +17 -6
- package/dist/selfUpdate/executor.js +158 -58
- package/dist/selfUpdate/index.d.ts +2 -1
- package/dist/selfUpdate/index.js +2 -1
- package/dist/selfUpdate/migration.d.ts +80 -15
- package/dist/selfUpdate/migration.js +2513 -156
- package/dist/selfUpdate/policy.js +2 -8
- package/dist/selfUpdate/recoveryChildStartGate.d.ts +29 -0
- package/dist/selfUpdate/recoveryChildStartGate.js +319 -0
- package/dist/selfUpdate/releaseBinary.d.ts +3 -0
- package/dist/selfUpdate/releaseBinary.js +46 -3
- package/dist/selfUpdate/transaction.d.ts +8 -0
- package/dist/selfUpdate/transaction.js +166 -18
- package/dist/selfUpdate/unixController.d.ts +8 -0
- package/dist/selfUpdate/unixController.js +364 -38
- package/dist/selfUpdate/windowsController.d.ts +14 -2
- package/dist/selfUpdate/windowsController.js +482 -103
- package/dist/selfUpdate/windowsUpdater.d.ts +25 -0
- package/dist/selfUpdate/windowsUpdater.js +174 -7
- package/dist/sync/engine.js +1 -1
- package/package.json +3 -3
|
@@ -2,12 +2,18 @@ import { spawn } from 'node:child_process';
|
|
|
2
2
|
import { lstat } from 'node:fs/promises';
|
|
3
3
|
import { bindStableWindowsRecoveryController, inspectDurableSelfUpdate, markWindowsInstallApplied, provisionStableWindowsRecoveryController, recoverDurableSelfUpdate, rollbackDurableSelfUpdate, } from './transaction.js';
|
|
4
4
|
import { SELF_UPDATE_FAILURE_CODES } from './failureCodes.js';
|
|
5
|
-
import {
|
|
5
|
+
import { revalidatePendingWindowsUpdaterHelper, resolveWindowsUpdaterPaths, WINDOWS_UPDATER_WORKER_ARG, } from './windowsUpdater.js';
|
|
6
|
+
import { resolveRecoveryControllerAuthority, } from './controllerLifecycleAuthority.js';
|
|
7
|
+
import { DEFAULT_RECOVERY_CHILD_START_GATE_TIMEOUT_MS, deliverRecoveryChildStartGate, RECOVERY_CHILD_START_GATE_ENV, } from './recoveryChildStartGate.js';
|
|
6
8
|
export const WINDOWS_RECOVERY_CONTROLLER_ARG = '--evolver-windows-recovery-controller';
|
|
7
9
|
export const WINDOWS_RECOVERY_CONTROLLER_PROVISION_ARG = '--evolver-windows-recovery-controller-provision';
|
|
8
10
|
const DEFAULT_CONFIRMATION_TIMEOUT_MS = 30_000;
|
|
9
11
|
const DEFAULT_POLL_INTERVAL_MS = 100;
|
|
10
12
|
const DEFAULT_STOP_TIMEOUT_MS = 2_000;
|
|
13
|
+
const DEFAULT_WORKER_TIMEOUT_MS = 120_000;
|
|
14
|
+
// The child performs up to three native owner-identity probes before acknowledging. Each
|
|
15
|
+
// PowerShell-backed Windows probe has its own 15s budget, so keep enough headroom for CI hosts.
|
|
16
|
+
const DEFAULT_STARTUP_ATTESTATION_TIMEOUT_MS = 120_000;
|
|
11
17
|
const PENDING_CONTROLLER_STAGES = new Set(['installed', 'restarted', 'health_check_pending']);
|
|
12
18
|
const RECOVER_BEFORE_START_STAGES = new Set([
|
|
13
19
|
'preparing',
|
|
@@ -24,6 +30,15 @@ const PENDING_SWAP_JOURNAL_STAGES = new Set([
|
|
|
24
30
|
'rolling_back',
|
|
25
31
|
'rollback_pending',
|
|
26
32
|
]);
|
|
33
|
+
class NativeChildTerminationUnconfirmedError extends Error {
|
|
34
|
+
child;
|
|
35
|
+
exit;
|
|
36
|
+
constructor(child, exit) {
|
|
37
|
+
super('windows_updater_worker_termination_unconfirmed');
|
|
38
|
+
this.child = child;
|
|
39
|
+
this.exit = exit;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
27
42
|
export async function maybeRunWindowsRecoveryController(options = {}) {
|
|
28
43
|
const argv = options.argv ?? process.argv.slice(2);
|
|
29
44
|
const command = argv.length === 2 && argv[0] === 'proxy' ? argv[1] : undefined;
|
|
@@ -34,7 +49,10 @@ export async function maybeRunWindowsRecoveryController(options = {}) {
|
|
|
34
49
|
return 64;
|
|
35
50
|
try {
|
|
36
51
|
if (command === WINDOWS_RECOVERY_CONTROLLER_PROVISION_ARG) {
|
|
37
|
-
await provisionStableWindowsRecoveryController(
|
|
52
|
+
await provisionStableWindowsRecoveryController({
|
|
53
|
+
...options,
|
|
54
|
+
replaceExisting: options.env?.['EVOLVER_INTERNAL_BOOTSTRAP_EXCLUSIVE'] !== '1',
|
|
55
|
+
}, options.processExecPath ?? process.execPath);
|
|
38
56
|
return 0;
|
|
39
57
|
}
|
|
40
58
|
return await runWindowsRecoveryController(options);
|
|
@@ -51,9 +69,13 @@ export async function runWindowsRecoveryController(options = {}) {
|
|
|
51
69
|
const confirmationTimeoutMs = positiveDuration(options.confirmationTimeoutMs, DEFAULT_CONFIRMATION_TIMEOUT_MS);
|
|
52
70
|
const pollIntervalMs = positiveDuration(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS);
|
|
53
71
|
const stopTimeoutMs = positiveDuration(options.stopTimeoutMs, DEFAULT_STOP_TIMEOUT_MS);
|
|
72
|
+
const workerTimeoutMs = positiveDuration(options.workerTimeoutMs, DEFAULT_WORKER_TIMEOUT_MS);
|
|
73
|
+
const startupGateTimeoutMs = positiveDuration(options.startupGateTimeoutMs, DEFAULT_RECOVERY_CHILD_START_GATE_TIMEOUT_MS);
|
|
74
|
+
const startupAttestationTimeoutMs = positiveDuration(options.startupAttestationTimeoutMs, DEFAULT_STARTUP_ATTESTATION_TIMEOUT_MS);
|
|
54
75
|
const spawnBoundTarget = options.spawnTarget ?? spawnTarget;
|
|
55
|
-
|
|
56
|
-
const
|
|
76
|
+
let authority;
|
|
77
|
+
const bindingOptions = { ...options, env, platform };
|
|
78
|
+
const bound = await bindStableWindowsRecoveryController(bindingOptions, processExecPath);
|
|
57
79
|
const transactionOptions = {
|
|
58
80
|
...options,
|
|
59
81
|
env,
|
|
@@ -62,124 +84,321 @@ export async function runWindowsRecoveryController(options = {}) {
|
|
|
62
84
|
stateDir: bound.stateDir,
|
|
63
85
|
targetPath: bound.targetPath,
|
|
64
86
|
beforeJournalMutation: async () => {
|
|
65
|
-
|
|
87
|
+
assertOwnedRecoveryAuthority(authority);
|
|
88
|
+
await options.beforeJournalMutation?.();
|
|
89
|
+
await revalidateBoundWindowsController(bindingOptions, processExecPath, bound, authority);
|
|
90
|
+
assertOwnedRecoveryAuthority(authority);
|
|
66
91
|
},
|
|
67
92
|
};
|
|
68
|
-
|
|
69
|
-
if (
|
|
93
|
+
const observed = await inspectDurableSelfUpdate(transactionOptions);
|
|
94
|
+
if (observed.outcome === 'blocked') {
|
|
70
95
|
logger.write('[evolver-windows-controller] recovery_blocked_before_start\n');
|
|
71
96
|
return 1;
|
|
72
97
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
if (workerExitCode !== 0) {
|
|
80
|
-
let rollback;
|
|
98
|
+
let child;
|
|
99
|
+
let childExit;
|
|
100
|
+
let releaseAttempted = false;
|
|
101
|
+
const guardUnconfirmedChild = async (_target, exit) => {
|
|
102
|
+
logger.write('[evolver-windows-controller] child_termination_unconfirmed\n');
|
|
103
|
+
if (authority?.kind === 'owned') {
|
|
81
104
|
try {
|
|
82
|
-
|
|
83
|
-
|
|
105
|
+
authority.retainProcess();
|
|
106
|
+
logger.write('[evolver-windows-controller] child_guardian_active\n');
|
|
107
|
+
return;
|
|
84
108
|
}
|
|
85
109
|
catch {
|
|
110
|
+
logger.write('[evolver-windows-controller] child_guardian_retain_failed\n');
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// Without a verifiable PID generation there is no safe lock handoff. Keep this live owner
|
|
114
|
+
// until exact exit instead of returning a dead-owner lock that another process could reclaim.
|
|
115
|
+
await exit;
|
|
116
|
+
};
|
|
117
|
+
const armBoundChild = (target) => {
|
|
118
|
+
if (authority?.kind !== 'owned')
|
|
119
|
+
return;
|
|
120
|
+
if (target.pid === undefined) {
|
|
121
|
+
throw new Error('self_update_recovery_controller_child_pid_unavailable');
|
|
122
|
+
}
|
|
123
|
+
authority.armProcess(target.pid);
|
|
124
|
+
};
|
|
125
|
+
const disarmBoundChild = () => {
|
|
126
|
+
if (authority?.kind === 'owned')
|
|
127
|
+
authority.disarmProcess();
|
|
128
|
+
};
|
|
129
|
+
const stopBoundChild = async (target, exit) => {
|
|
130
|
+
const stopped = await stopNativeChild(target, exit, stopTimeoutMs);
|
|
131
|
+
if (!stopped) {
|
|
132
|
+
await guardUnconfirmedChild(target, exit);
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
disarmBoundChild();
|
|
136
|
+
}
|
|
137
|
+
return stopped;
|
|
138
|
+
};
|
|
139
|
+
const runBoundUpdaterWorker = options.runUpdaterWorker
|
|
140
|
+
?? ((workerPath, stateDir, workerEnv, startupGateToken) => runUpdaterWorker(workerPath, stateDir, workerEnv, startupGateToken, workerTimeoutMs, stopTimeoutMs, startupGateTimeoutMs, armBoundChild, disarmBoundChild, () => { assertOwnedRecoveryAuthority(authority); }, options.spawnUpdaterWorker ?? spawnUpdaterWorker));
|
|
141
|
+
const releaseAuthority = async () => {
|
|
142
|
+
if (!authority || releaseAttempted)
|
|
143
|
+
return true;
|
|
144
|
+
releaseAttempted = true;
|
|
145
|
+
try {
|
|
146
|
+
authority.release();
|
|
147
|
+
return true;
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
logger.write('[evolver-windows-controller] lifecycle_authority_release_failed\n');
|
|
151
|
+
if (child && childExit)
|
|
152
|
+
await stopBoundChild(child, childExit);
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
const launchTerminalTarget = async () => {
|
|
157
|
+
await revalidateBoundWindowsController(bindingOptions, processExecPath, bound, authority);
|
|
158
|
+
authority.assertAuthorized();
|
|
159
|
+
const prepared = authority.prepareTarget(env);
|
|
160
|
+
const launched = spawnBoundTarget(bound.targetPath, prepared.env, prepared.startupAckToken !== undefined);
|
|
161
|
+
child = launched;
|
|
162
|
+
const launchedExit = waitForNativeExit(launched);
|
|
163
|
+
childExit = launchedExit;
|
|
164
|
+
const spawned = waitForNativeSpawn(launched);
|
|
165
|
+
try {
|
|
166
|
+
// Node publishes the native PID synchronously when spawn succeeds. Arm the guardian before
|
|
167
|
+
// the first await so a hard controller crash cannot expose a stale parent-owner lock.
|
|
168
|
+
armBoundChild(launched);
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
logger.write('[evolver-windows-controller] child_guardian_arm_failed\n');
|
|
172
|
+
if (await spawned)
|
|
173
|
+
await stopBoundChild(launched, launchedExit);
|
|
174
|
+
await releaseAuthority();
|
|
175
|
+
return 1;
|
|
176
|
+
}
|
|
177
|
+
if (!await spawned) {
|
|
178
|
+
disarmBoundChild();
|
|
179
|
+
await releaseAuthority();
|
|
180
|
+
return 1;
|
|
181
|
+
}
|
|
182
|
+
authority.assertAuthorized();
|
|
183
|
+
if (!await deliverRecoveryChildStartGate(launched, prepared.startupGateToken, startupGateTimeoutMs)) {
|
|
184
|
+
logger.write('[evolver-windows-controller] startup_gate_failed\n');
|
|
185
|
+
await stopBoundChild(launched, launchedExit);
|
|
186
|
+
await releaseAuthority();
|
|
187
|
+
return 1;
|
|
188
|
+
}
|
|
189
|
+
if (prepared.startupAckToken
|
|
190
|
+
&& !await waitForStartupAttestation(launched, prepared.startupAckToken, startupAttestationTimeoutMs)) {
|
|
191
|
+
logger.write('[evolver-windows-controller] startup_attestation_failed\n');
|
|
192
|
+
await stopBoundChild(launched, launchedExit);
|
|
193
|
+
await releaseAuthority();
|
|
194
|
+
return 1;
|
|
195
|
+
}
|
|
196
|
+
if (!await releaseAuthority())
|
|
197
|
+
return 1;
|
|
198
|
+
return exitCode(await launchedExit);
|
|
199
|
+
};
|
|
200
|
+
try {
|
|
201
|
+
authority = await resolveRecoveryControllerAuthority(env, options.lifecycleAuthority);
|
|
202
|
+
await revalidateBoundWindowsController(bindingOptions, processExecPath, bound, authority);
|
|
203
|
+
let initial = await inspectDurableSelfUpdate(transactionOptions);
|
|
204
|
+
if (initial.outcome === 'blocked') {
|
|
205
|
+
logger.write('[evolver-windows-controller] recovery_blocked_before_start\n');
|
|
206
|
+
return 1;
|
|
207
|
+
}
|
|
208
|
+
if (authority.kind === 'delegated' && !stateAllowsDelegatedSpawn(initial)) {
|
|
209
|
+
logger.write('[evolver-windows-controller] lifecycle_authority_blocked\n');
|
|
210
|
+
return 1;
|
|
211
|
+
}
|
|
212
|
+
if (await pendingSwapExists(bound.stateDir)) {
|
|
213
|
+
if (initial.stage === undefined || !PENDING_SWAP_JOURNAL_STAGES.has(initial.stage)) {
|
|
86
214
|
logger.write('[evolver-windows-controller] pending_swap_blocked\n');
|
|
87
215
|
return 1;
|
|
88
216
|
}
|
|
89
|
-
|
|
90
|
-
|
|
217
|
+
assertOwnedRecoveryAuthority(authority);
|
|
218
|
+
const workerExitCode = await executeBoundUpdaterWorker(runBoundUpdaterWorker, bound.stateDir, env, authority, options.assertUpdaterHelperTrust);
|
|
219
|
+
assertOwnedRecoveryAuthority(authority);
|
|
220
|
+
if (workerExitCode !== 0) {
|
|
221
|
+
let rollback;
|
|
222
|
+
try {
|
|
223
|
+
rollback = await rollbackDurableSelfUpdate(transactionOptions, SELF_UPDATE_FAILURE_CODES.REPLACE_FAILED);
|
|
224
|
+
rollback = await completePreparedRollback(rollback, transactionOptions, bound.stateDir, env, runBoundUpdaterWorker, authority, options.assertUpdaterHelperTrust);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
logger.write('[evolver-windows-controller] pending_swap_blocked\n');
|
|
228
|
+
return 1;
|
|
229
|
+
}
|
|
230
|
+
if (rollback.outcome !== 'rolled_back') {
|
|
231
|
+
logger.write('[evolver-windows-controller] pending_swap_blocked\n');
|
|
232
|
+
return 1;
|
|
233
|
+
}
|
|
234
|
+
return await launchTerminalTarget();
|
|
235
|
+
}
|
|
236
|
+
initial = await inspectDurableSelfUpdate(transactionOptions);
|
|
237
|
+
}
|
|
238
|
+
if (initial.stage === 'install_pending') {
|
|
239
|
+
assertOwnedRecoveryAuthority(authority);
|
|
240
|
+
initial = await markWindowsInstallApplied(transactionOptions);
|
|
241
|
+
if (initial.outcome === 'blocked') {
|
|
242
|
+
logger.write('[evolver-windows-controller] install_not_applied\n');
|
|
91
243
|
return 1;
|
|
92
244
|
}
|
|
93
|
-
return launchRestoredTarget(bound.targetPath, env, spawnBoundTarget);
|
|
94
245
|
}
|
|
95
|
-
initial = await inspectDurableSelfUpdate(transactionOptions);
|
|
96
|
-
}
|
|
97
|
-
if (initial.stage === 'install_pending') {
|
|
98
|
-
initial = await markWindowsInstallApplied(transactionOptions);
|
|
99
246
|
if (initial.outcome === 'blocked') {
|
|
100
|
-
logger.write('[evolver-windows-controller] install_not_applied\n');
|
|
101
|
-
return 1;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
if (initial.outcome === 'blocked') {
|
|
105
|
-
logger.write('[evolver-windows-controller] recovery_blocked_before_start\n');
|
|
106
|
-
return 1;
|
|
107
|
-
}
|
|
108
|
-
if (initial.stage !== undefined && RECOVER_BEFORE_START_STAGES.has(initial.stage)) {
|
|
109
|
-
initial = await recoverAndApplyRollback(transactionOptions, bound.stateDir, env, runBoundUpdaterWorker);
|
|
110
|
-
if (initial.outcome === 'blocked' || initial.outcome === 'rollback_pending') {
|
|
111
247
|
logger.write('[evolver-windows-controller] recovery_blocked_before_start\n');
|
|
112
248
|
return 1;
|
|
113
249
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
250
|
+
if (initial.stage !== undefined && RECOVER_BEFORE_START_STAGES.has(initial.stage)) {
|
|
251
|
+
assertOwnedRecoveryAuthority(authority);
|
|
252
|
+
initial = await recoverAndApplyRollback(transactionOptions, bound.stateDir, env, runBoundUpdaterWorker, authority, options.assertUpdaterHelperTrust);
|
|
253
|
+
if (initial.outcome === 'blocked' || initial.outcome === 'rollback_pending') {
|
|
254
|
+
logger.write('[evolver-windows-controller] recovery_blocked_before_start\n');
|
|
255
|
+
return 1;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const pendingAtLaunch = isControllerPending(initial);
|
|
259
|
+
if (!pendingAtLaunch)
|
|
260
|
+
return await launchTerminalTarget();
|
|
261
|
+
await revalidateBoundWindowsController(bindingOptions, processExecPath, bound, authority);
|
|
262
|
+
assertOwnedRecoveryAuthority(authority);
|
|
263
|
+
const prepared = authority.prepareTarget(env);
|
|
264
|
+
const launched = spawnBoundTarget(bound.targetPath, prepared.env, prepared.startupAckToken !== undefined);
|
|
265
|
+
child = launched;
|
|
266
|
+
const launchedExit = waitForNativeExit(launched);
|
|
267
|
+
childExit = launchedExit;
|
|
268
|
+
const spawned = waitForNativeSpawn(launched);
|
|
128
269
|
try {
|
|
129
|
-
|
|
270
|
+
// Arm before yielding for the same hard-crash boundary as terminal launches.
|
|
271
|
+
armBoundChild(launched);
|
|
130
272
|
}
|
|
131
273
|
catch {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
}
|
|
274
|
+
logger.write('[evolver-windows-controller] child_guardian_arm_failed\n');
|
|
275
|
+
if (await spawned)
|
|
276
|
+
await stopBoundChild(launched, launchedExit);
|
|
277
|
+
await releaseAuthority();
|
|
278
|
+
return 1;
|
|
138
279
|
}
|
|
139
|
-
if (
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
await stopNativeChild(launched, childExit, stopTimeoutMs);
|
|
280
|
+
if (!await spawned) {
|
|
281
|
+
disarmBoundChild();
|
|
282
|
+
await releaseAuthority();
|
|
143
283
|
return 1;
|
|
144
284
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
285
|
+
authority.assertAuthorized();
|
|
286
|
+
const startupGateAccepted = await deliverRecoveryChildStartGate(launched, prepared.startupGateToken, startupGateTimeoutMs);
|
|
287
|
+
if (!startupGateAccepted) {
|
|
288
|
+
logger.write('[evolver-windows-controller] startup_gate_failed\n');
|
|
148
289
|
}
|
|
149
|
-
if (
|
|
150
|
-
await
|
|
151
|
-
|
|
290
|
+
if (!startupGateAccepted || (prepared.startupAckToken
|
|
291
|
+
&& !await waitForStartupAttestation(launched, prepared.startupAckToken, startupAttestationTimeoutMs))) {
|
|
292
|
+
if (!await stopBoundChild(launched, launchedExit))
|
|
293
|
+
return 1;
|
|
294
|
+
let rollback = await rollbackDurableSelfUpdate(transactionOptions, SELF_UPDATE_FAILURE_CODES.RESTART_FAILED);
|
|
295
|
+
rollback = await completePreparedRollback(rollback, transactionOptions, bound.stateDir, env, runBoundUpdaterWorker, authority, options.assertUpdaterHelperTrust);
|
|
296
|
+
if (rollback.outcome === 'confirmed') {
|
|
297
|
+
if (!await releaseAuthority())
|
|
298
|
+
return 1;
|
|
299
|
+
return exitCode(await launchedExit);
|
|
300
|
+
}
|
|
301
|
+
if (rollback.outcome !== 'rolled_back') {
|
|
302
|
+
logger.write('[evolver-windows-controller] startup_attestation_failed\n');
|
|
303
|
+
return 1;
|
|
304
|
+
}
|
|
305
|
+
child = undefined;
|
|
306
|
+
childExit = undefined;
|
|
307
|
+
return await launchTerminalTarget();
|
|
308
|
+
}
|
|
309
|
+
const deadline = Date.now() + confirmationTimeoutMs;
|
|
310
|
+
for (;;) {
|
|
311
|
+
const event = await Promise.race([
|
|
312
|
+
launchedExit.then((exit) => ({ type: 'exit', exit })),
|
|
313
|
+
delay(Math.min(pollIntervalMs, Math.max(1, deadline - Date.now())))
|
|
314
|
+
.then(() => ({ type: 'poll' })),
|
|
315
|
+
]);
|
|
316
|
+
assertOwnedRecoveryAuthority(authority);
|
|
317
|
+
let state;
|
|
152
318
|
try {
|
|
153
|
-
|
|
154
|
-
? SELF_UPDATE_FAILURE_CODES.RESTART_FAILED
|
|
155
|
-
: 'health_confirmation_timeout');
|
|
156
|
-
rollback = await completePreparedRollback(rollback, transactionOptions, bound.stateDir, env, runBoundUpdaterWorker);
|
|
319
|
+
state = await inspectDurableSelfUpdate(transactionOptions);
|
|
157
320
|
}
|
|
158
321
|
catch {
|
|
159
|
-
|
|
160
|
-
|
|
322
|
+
if (event.type === 'exit' || Date.now() >= deadline) {
|
|
323
|
+
logger.write('[evolver-windows-controller] recovery_state_unavailable\n');
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
161
328
|
}
|
|
162
|
-
if (
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
329
|
+
if (state?.outcome === 'confirmed') {
|
|
330
|
+
if (!await releaseAuthority())
|
|
331
|
+
return 1;
|
|
332
|
+
return exitCode(await launchedExit);
|
|
333
|
+
}
|
|
334
|
+
if (state?.outcome === 'blocked') {
|
|
335
|
+
await stopBoundChild(launched, launchedExit);
|
|
166
336
|
return 1;
|
|
167
337
|
}
|
|
168
|
-
|
|
338
|
+
if (state?.outcome === 'rolled_back') {
|
|
339
|
+
if (!await stopBoundChild(launched, launchedExit))
|
|
340
|
+
return 1;
|
|
341
|
+
child = undefined;
|
|
342
|
+
childExit = undefined;
|
|
343
|
+
return await launchTerminalTarget();
|
|
344
|
+
}
|
|
345
|
+
if (event.type === 'exit' || Date.now() >= deadline) {
|
|
346
|
+
if (!await stopBoundChild(launched, launchedExit))
|
|
347
|
+
return 1;
|
|
348
|
+
let rollback;
|
|
349
|
+
try {
|
|
350
|
+
rollback = await rollbackDurableSelfUpdate(transactionOptions, event.type === 'exit'
|
|
351
|
+
? SELF_UPDATE_FAILURE_CODES.RESTART_FAILED
|
|
352
|
+
: 'health_confirmation_timeout');
|
|
353
|
+
rollback = await completePreparedRollback(rollback, transactionOptions, bound.stateDir, env, runBoundUpdaterWorker, authority, options.assertUpdaterHelperTrust);
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
logger.write('[evolver-windows-controller] rollback_blocked\n');
|
|
357
|
+
return 1;
|
|
358
|
+
}
|
|
359
|
+
if (rollback.outcome === 'confirmed') {
|
|
360
|
+
if (!await releaseAuthority())
|
|
361
|
+
return 1;
|
|
362
|
+
return exitCode(await launchedExit);
|
|
363
|
+
}
|
|
364
|
+
if (rollback.outcome !== 'rolled_back') {
|
|
365
|
+
logger.write('[evolver-windows-controller] rollback_blocked\n');
|
|
366
|
+
return 1;
|
|
367
|
+
}
|
|
368
|
+
child = undefined;
|
|
369
|
+
childExit = undefined;
|
|
370
|
+
return await launchTerminalTarget();
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
catch (error) {
|
|
375
|
+
if (error instanceof NativeChildTerminationUnconfirmedError) {
|
|
376
|
+
await guardUnconfirmedChild(error.child, error.exit);
|
|
169
377
|
}
|
|
378
|
+
logger.write('[evolver-windows-controller] lifecycle_authority_blocked\n');
|
|
379
|
+
if (child && childExit)
|
|
380
|
+
await stopBoundChild(child, childExit);
|
|
381
|
+
await releaseAuthority();
|
|
382
|
+
return 1;
|
|
383
|
+
}
|
|
384
|
+
finally {
|
|
385
|
+
if (!releaseAttempted)
|
|
386
|
+
await releaseAuthority();
|
|
170
387
|
}
|
|
171
388
|
}
|
|
172
|
-
async function recoverAndApplyRollback(options, stateDir, env, runBoundUpdaterWorker) {
|
|
389
|
+
async function recoverAndApplyRollback(options, stateDir, env, runBoundUpdaterWorker, authority, assertUpdaterHelperTrust) {
|
|
390
|
+
assertOwnedRecoveryAuthority(authority);
|
|
173
391
|
const recovery = await recoverDurableSelfUpdate(options);
|
|
174
|
-
return completePreparedRollback(recovery, options, stateDir, env, runBoundUpdaterWorker);
|
|
392
|
+
return completePreparedRollback(recovery, options, stateDir, env, runBoundUpdaterWorker, authority, assertUpdaterHelperTrust);
|
|
175
393
|
}
|
|
176
|
-
async function completePreparedRollback(recovery, options, stateDir, env, runBoundUpdaterWorker) {
|
|
394
|
+
async function completePreparedRollback(recovery, options, stateDir, env, runBoundUpdaterWorker, authority, assertUpdaterHelperTrust) {
|
|
177
395
|
if (recovery.outcome !== 'rollback_pending')
|
|
178
396
|
return recovery;
|
|
179
|
-
const workerExitCode = await executeBoundUpdaterWorker(runBoundUpdaterWorker, stateDir, env);
|
|
397
|
+
const workerExitCode = await executeBoundUpdaterWorker(runBoundUpdaterWorker, stateDir, env, authority, assertUpdaterHelperTrust);
|
|
180
398
|
if (workerExitCode !== 0) {
|
|
181
399
|
return { ...recovery, outcome: 'blocked', failureCode: 'rollback_apply_failed' };
|
|
182
400
|
}
|
|
401
|
+
assertOwnedRecoveryAuthority(authority);
|
|
183
402
|
return recoverDurableSelfUpdate(options);
|
|
184
403
|
}
|
|
185
404
|
async function pendingSwapExists(stateDir) {
|
|
@@ -199,53 +418,213 @@ async function pendingSwapExists(stateDir) {
|
|
|
199
418
|
function isControllerPending(state) {
|
|
200
419
|
return state.stage !== undefined && PENDING_CONTROLLER_STAGES.has(state.stage);
|
|
201
420
|
}
|
|
202
|
-
function
|
|
421
|
+
function stateAllowsDelegatedSpawn(state) {
|
|
422
|
+
return state.outcome === 'none'
|
|
423
|
+
|| state.outcome === 'confirmed'
|
|
424
|
+
|| state.outcome === 'rolled_back';
|
|
425
|
+
}
|
|
426
|
+
function assertOwnedRecoveryAuthority(authority) {
|
|
427
|
+
if (authority?.kind !== 'owned') {
|
|
428
|
+
throw new Error('self_update_recovery_controller_owner_lease_required');
|
|
429
|
+
}
|
|
430
|
+
authority.assertAuthorized();
|
|
431
|
+
}
|
|
432
|
+
async function revalidateBoundWindowsController(options, processExecPath, expected, authority) {
|
|
433
|
+
authority.assertAuthorized();
|
|
434
|
+
const actual = await bindStableWindowsRecoveryController(options, processExecPath);
|
|
435
|
+
if (actual.controllerPath !== expected.controllerPath
|
|
436
|
+
|| actual.stateDir !== expected.stateDir
|
|
437
|
+
|| actual.targetPath !== expected.targetPath) {
|
|
438
|
+
throw new Error('self_update_windows_recovery_controller_binding_changed');
|
|
439
|
+
}
|
|
440
|
+
authority.assertAuthorized();
|
|
441
|
+
}
|
|
442
|
+
function spawnTarget(targetPath, env, startupAttestation) {
|
|
443
|
+
const startupGate = env[RECOVERY_CHILD_START_GATE_ENV] !== undefined;
|
|
203
444
|
return spawn(targetPath, ['proxy'], {
|
|
204
445
|
env,
|
|
205
|
-
stdio:
|
|
446
|
+
stdio: startupGate
|
|
447
|
+
? ['inherit', 'inherit', 'inherit', startupAttestation ? 'pipe' : 'ignore', 'pipe']
|
|
448
|
+
: startupAttestation
|
|
449
|
+
? ['inherit', 'inherit', 'inherit', 'pipe']
|
|
450
|
+
: 'inherit',
|
|
206
451
|
windowsHide: true,
|
|
207
452
|
});
|
|
208
453
|
}
|
|
209
|
-
async function runUpdaterWorker(workerPath, _stateDir, env) {
|
|
210
|
-
|
|
454
|
+
async function runUpdaterWorker(workerPath, _stateDir, env, startupGateToken, timeoutMs, stopTimeoutMs, startupGateTimeoutMs, armChild, disarmChild, assertChildAuthority, spawnWorker) {
|
|
455
|
+
const child = spawnWorker(workerPath, env);
|
|
456
|
+
const exit = waitForNativeExit(child);
|
|
457
|
+
const spawned = waitForNativeSpawn(child);
|
|
458
|
+
try {
|
|
459
|
+
// Protect the updater PID generation before yielding to its spawn event.
|
|
460
|
+
armChild(child);
|
|
461
|
+
}
|
|
462
|
+
catch {
|
|
463
|
+
if (await spawned && !await stopNativeChild(child, exit, stopTimeoutMs)) {
|
|
464
|
+
throw new NativeChildTerminationUnconfirmedError(child, exit);
|
|
465
|
+
}
|
|
466
|
+
throw new Error('windows_updater_worker_guardian_arm_failed');
|
|
467
|
+
}
|
|
468
|
+
if (!await spawned) {
|
|
469
|
+
disarmChild();
|
|
470
|
+
return 1;
|
|
471
|
+
}
|
|
472
|
+
try {
|
|
473
|
+
assertChildAuthority();
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
if (!await stopNativeChild(child, exit, stopTimeoutMs)) {
|
|
477
|
+
throw new NativeChildTerminationUnconfirmedError(child, exit);
|
|
478
|
+
}
|
|
479
|
+
try {
|
|
480
|
+
disarmChild();
|
|
481
|
+
}
|
|
482
|
+
catch {
|
|
483
|
+
// The lost authority remains the primary failure.
|
|
484
|
+
}
|
|
485
|
+
throw new Error('windows_updater_worker_authority_lost_before_start_gate');
|
|
486
|
+
}
|
|
487
|
+
if (!await deliverRecoveryChildStartGate(child, startupGateToken, startupGateTimeoutMs)) {
|
|
488
|
+
if (!await stopNativeChild(child, exit, stopTimeoutMs)) {
|
|
489
|
+
throw new NativeChildTerminationUnconfirmedError(child, exit);
|
|
490
|
+
}
|
|
491
|
+
disarmChild();
|
|
492
|
+
return 1;
|
|
493
|
+
}
|
|
494
|
+
const event = await Promise.race([
|
|
495
|
+
exit.then((result) => ({ type: 'exit', result })),
|
|
496
|
+
delay(timeoutMs).then(() => ({ type: 'timeout' })),
|
|
497
|
+
]);
|
|
498
|
+
if (event.type === 'exit') {
|
|
499
|
+
disarmChild();
|
|
500
|
+
return exitCode(event.result);
|
|
501
|
+
}
|
|
502
|
+
if (!await stopNativeChild(child, exit, stopTimeoutMs)) {
|
|
503
|
+
throw new NativeChildTerminationUnconfirmedError(child, exit);
|
|
504
|
+
}
|
|
505
|
+
disarmChild();
|
|
506
|
+
return 1;
|
|
507
|
+
}
|
|
508
|
+
function spawnUpdaterWorker(workerPath, env) {
|
|
509
|
+
return spawn(workerPath, ['proxy', WINDOWS_UPDATER_WORKER_ARG], {
|
|
211
510
|
env,
|
|
212
|
-
stdio: 'inherit',
|
|
511
|
+
stdio: ['inherit', 'inherit', 'inherit', 'ignore', 'pipe'],
|
|
213
512
|
windowsHide: true,
|
|
214
|
-
})
|
|
513
|
+
});
|
|
215
514
|
}
|
|
216
|
-
async function executeBoundUpdaterWorker(runner, stateDir, env) {
|
|
515
|
+
async function executeBoundUpdaterWorker(runner, stateDir, env, authority, assertUpdaterHelperTrust) {
|
|
516
|
+
assertOwnedRecoveryAuthority(authority);
|
|
517
|
+
const workerEnv = { ...env, EVOLVER_SELF_UPDATE_STATE_DIR: stateDir };
|
|
518
|
+
const prepared = authority.prepareWorker(workerEnv);
|
|
519
|
+
assertOwnedRecoveryAuthority(authority);
|
|
217
520
|
const paths = resolveWindowsUpdaterPaths(stateDir);
|
|
218
|
-
const bound = await
|
|
521
|
+
const bound = await revalidatePendingWindowsUpdaterHelper({
|
|
219
522
|
stateDir,
|
|
220
|
-
|
|
221
|
-
relativePath: ['windows-updater', 'updater.exe'],
|
|
222
|
-
label: 'controller_worker',
|
|
523
|
+
helperPath: paths.helperPath,
|
|
223
524
|
platform: 'win32',
|
|
525
|
+
...(assertUpdaterHelperTrust ? { assertHelperTrust: assertUpdaterHelperTrust } : {}),
|
|
224
526
|
});
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
return exitCode(await waitForNativeExit(spawnBoundTarget(targetPath, env)));
|
|
527
|
+
assertOwnedRecoveryAuthority(authority);
|
|
528
|
+
const result = await runner(bound.executablePath, bound.stateDir, prepared.env, prepared.startupGateToken);
|
|
529
|
+
assertOwnedRecoveryAuthority(authority);
|
|
530
|
+
return result;
|
|
230
531
|
}
|
|
231
532
|
function waitForNativeExit(child) {
|
|
232
533
|
return new Promise((resolve) => {
|
|
233
|
-
|
|
234
|
-
|
|
534
|
+
let spawned = child.pid !== undefined;
|
|
535
|
+
const cleanup = () => {
|
|
536
|
+
child.off('spawn', onSpawn);
|
|
537
|
+
child.off('error', onError);
|
|
538
|
+
};
|
|
539
|
+
const onSpawn = () => { spawned = true; };
|
|
540
|
+
const onError = (error) => {
|
|
541
|
+
// A post-spawn process error does not prove that the native child exited.
|
|
542
|
+
if (spawned)
|
|
543
|
+
return;
|
|
544
|
+
cleanup();
|
|
545
|
+
resolve({ code: null, signal: null, error });
|
|
546
|
+
};
|
|
547
|
+
child.once('spawn', onSpawn);
|
|
548
|
+
child.on('error', onError);
|
|
549
|
+
child.once('exit', (code, signal) => {
|
|
550
|
+
cleanup();
|
|
551
|
+
resolve({ code, signal });
|
|
552
|
+
});
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
function waitForNativeSpawn(child) {
|
|
556
|
+
if (child.pid !== undefined)
|
|
557
|
+
return Promise.resolve(true);
|
|
558
|
+
return new Promise((resolve) => {
|
|
559
|
+
const finish = (spawned) => {
|
|
560
|
+
child.off('spawn', onSpawn);
|
|
561
|
+
child.off('error', onError);
|
|
562
|
+
resolve(spawned);
|
|
563
|
+
};
|
|
564
|
+
const onSpawn = () => { finish(true); };
|
|
565
|
+
const onError = () => { finish(false); };
|
|
566
|
+
child.once('spawn', onSpawn);
|
|
567
|
+
child.once('error', onError);
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
function waitForStartupAttestation(child, token, timeoutMs) {
|
|
571
|
+
const stream = child.stdio[3];
|
|
572
|
+
if (!stream || typeof stream.on !== 'function')
|
|
573
|
+
return Promise.resolve(false);
|
|
574
|
+
const readable = stream;
|
|
575
|
+
const expected = `${token}\n`;
|
|
576
|
+
return new Promise((resolve) => {
|
|
577
|
+
let raw = '';
|
|
578
|
+
let settled = false;
|
|
579
|
+
const finish = (result) => {
|
|
580
|
+
if (settled)
|
|
581
|
+
return;
|
|
582
|
+
settled = true;
|
|
583
|
+
clearTimeout(timer);
|
|
584
|
+
readable.off('data', onData);
|
|
585
|
+
readable.off('end', onEnd);
|
|
586
|
+
readable.off('error', onError);
|
|
587
|
+
resolve(result);
|
|
588
|
+
};
|
|
589
|
+
const onData = (chunk) => {
|
|
590
|
+
raw += Buffer.isBuffer(chunk) ? chunk.toString('utf8') : chunk;
|
|
591
|
+
if (Buffer.byteLength(raw, 'utf8') > 128 || !expected.startsWith(raw))
|
|
592
|
+
finish(false);
|
|
593
|
+
};
|
|
594
|
+
const onEnd = () => { finish(raw === expected); };
|
|
595
|
+
const onError = () => { finish(false); };
|
|
596
|
+
const timer = setTimeout(() => { finish(false); }, timeoutMs);
|
|
597
|
+
readable.on('data', onData);
|
|
598
|
+
readable.once('end', onEnd);
|
|
599
|
+
readable.once('error', onError);
|
|
235
600
|
});
|
|
236
601
|
}
|
|
237
602
|
async function stopNativeChild(child, exit, timeoutMs) {
|
|
238
603
|
if (child.exitCode !== null || child.signalCode !== null)
|
|
239
|
-
return;
|
|
240
|
-
|
|
604
|
+
return true;
|
|
605
|
+
try {
|
|
606
|
+
child.kill('SIGTERM');
|
|
607
|
+
}
|
|
608
|
+
catch {
|
|
609
|
+
// A failed signal is not an exit; the bounded native-exit wait remains authoritative.
|
|
610
|
+
}
|
|
241
611
|
const stopped = await Promise.race([
|
|
242
612
|
exit.then(() => true),
|
|
243
613
|
delay(timeoutMs).then(() => false),
|
|
244
614
|
]);
|
|
245
615
|
if (!stopped) {
|
|
246
|
-
|
|
247
|
-
|
|
616
|
+
try {
|
|
617
|
+
child.kill('SIGKILL');
|
|
618
|
+
}
|
|
619
|
+
catch {
|
|
620
|
+
// A failed signal is not an exit; the second bounded wait still protects authority.
|
|
621
|
+
}
|
|
622
|
+
return Promise.race([
|
|
623
|
+
exit.then(() => true),
|
|
624
|
+
delay(timeoutMs).then(() => false),
|
|
625
|
+
]);
|
|
248
626
|
}
|
|
627
|
+
return true;
|
|
249
628
|
}
|
|
250
629
|
function exitCode(exit) {
|
|
251
630
|
if (exit.error)
|