@zq-silk/yui 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/agent/managedRuntimeEnvironment.js +2 -1
- package/dist/cli/commandCatalog.js +251 -13
- package/dist/cli/updateOrchestrator.js +8 -0
- package/dist/cli/updatePorts.js +76 -22
- package/dist/cli.js +264 -20
- package/dist/commands/configCommands.js +83 -9
- package/dist/commands/controllerCommands.js +103 -0
- package/dist/commands/deliveryGuardPreflight.js +30 -0
- package/dist/commands/durableJobCommands.js +231 -0
- package/dist/commands/executionAuditCommands.js +193 -0
- package/dist/commands/grantCommands.js +374 -0
- package/dist/commands/projectCommands.js +119 -81
- package/dist/commands/releaseCommands.js +444 -0
- package/dist/commands/resourcesCommands.js +274 -0
- package/dist/commands/sessionCommands.js +104 -0
- package/dist/commands/taskActor.js +117 -0
- package/dist/commands/taskChangeSetCommands.js +60 -0
- package/dist/commands/taskCommands.js +618 -202
- package/dist/commands/taskCompletionGate.js +78 -1
- package/dist/commands/taskContextCommand.js +33 -6
- package/dist/commands/taskInputCommands.js +1 -1
- package/dist/commands/taskIntegrationCommands.js +136 -33
- package/dist/commands/taskIntegrationQueueCommands.js +228 -0
- package/dist/commands/taskNextActionCommand.js +100 -0
- package/dist/commands/taskOverlapCommands.js +120 -0
- package/dist/commands/taskOverviewCommand.js +36 -8
- package/dist/commands/telemetryCommands.js +330 -0
- package/dist/commands/workflowCommands.js +415 -0
- package/dist/config/yuiConfig.js +62 -0
- package/dist/controller/clientRuntime.js +42 -1
- package/dist/controller/controller.js +402 -56
- package/dist/controller/controllerMain.js +25 -2
- package/dist/controller/domainIdentity.js +16 -8
- package/dist/controller/fileSchedulerStoreAdapter.js +423 -31
- package/dist/controller/handoverCandidate.js +168 -0
- package/dist/controller/jobClient.js +102 -0
- package/dist/controller/jobControl.js +613 -0
- package/dist/controller/jobSupervisor.js +498 -0
- package/dist/controller/providerHookRunFence.js +34 -5
- package/dist/controller/resourceCleanupLinux.js +18 -9
- package/dist/controller/resourceInventoryLinux.js +90 -39
- package/dist/controller/runtime.js +165 -15
- package/dist/controller/runtimeEventInbox.js +234 -57
- package/dist/controller/runtimeEventProcessor.js +297 -58
- package/dist/controller/sessionOwnerReconciliation.js +321 -0
- package/dist/core/controllerServer.js +416 -27
- package/dist/core/controllerTelemetry.js +167 -0
- package/dist/doctor/doctor.js +113 -16
- package/dist/domain/validation.js +9 -0
- package/dist/execution/executionGroup.js +40 -3
- package/dist/executor/agentExecutor.js +6 -3
- package/dist/executor/effectiveLaunch.js +52 -0
- package/dist/executor/executorRegistry.js +50 -0
- package/dist/executor/fileRoleLaunchPlanner.js +61 -6
- package/dist/grant/capabilityGrant.js +282 -0
- package/dist/integration/changeSet.js +16 -3
- package/dist/integration/changeSetManifest.js +46 -0
- package/dist/integration/gitIntegrationService.js +528 -147
- package/dist/integration/integrationAttempt.js +54 -5
- package/dist/integration/integrationQueueEntry.js +221 -0
- package/dist/integration/integrationQueueService.js +955 -0
- package/dist/integration/manifestTags.js +99 -0
- package/dist/integration/overlapDiagnostics.js +211 -0
- package/dist/job/durableJob.js +449 -0
- package/dist/job/jobRunner.js +350 -0
- package/dist/lifecycle/exactRunTerminalization.js +24 -2
- package/dist/lifecycle/providerErrorClass.js +126 -0
- package/dist/message/message.js +16 -3
- package/dist/observability/executionAudit.js +545 -0
- package/dist/observability/faultClassification.js +160 -0
- package/dist/observability/runtimeIdentity.js +367 -0
- package/dist/release/fakeReleasePorts.js +55 -0
- package/dist/release/releaseHandover.js +475 -0
- package/dist/release/releaseIdempotencyStore.js +165 -0
- package/dist/release/releaseWorkflow.js +459 -0
- package/dist/release/releaseWorkflowEngine.js +688 -0
- package/dist/release/releaseWorkflowPorts.js +1720 -0
- package/dist/release/runtimeRelease.js +495 -0
- package/dist/release/workflowFileLock.js +218 -0
- package/dist/repository/gitWorkspace.js +177 -1
- package/dist/repository/projectMaintenanceLock.js +315 -0
- package/dist/repository/taskWorkspaceCoordinator.js +87 -17
- package/dist/repository/taskWorkspacePreparer.js +1091 -517
- package/dist/resources/autoResourceGc.js +116 -0
- package/dist/resources/liveReferences.js +574 -0
- package/dist/resources/resourceDiscovery.js +477 -0
- package/dist/resources/resourceGc.js +645 -0
- package/dist/resources/resourceRegistrar.js +256 -0
- package/dist/resources/resourceRegistry.js +150 -0
- package/dist/resources/resourceRegistryStore.js +41 -0
- package/dist/resources/resourceTypes.js +42 -0
- package/dist/resources/sqliteResourceRegistry.js +111 -0
- package/dist/review/reviewConfig.js +10 -0
- package/dist/review/reviewFinding.js +240 -0
- package/dist/review/reviewFindingLedger.js +545 -0
- package/dist/review/reviewOutcomeClassifier.js +61 -0
- package/dist/review/reviewRound.js +56 -4
- package/dist/run/agentRun.js +80 -4
- package/dist/run/providerRetry.js +84 -0
- package/dist/run/providerRetryConfig.js +63 -0
- package/dist/run/yieldReceipt.js +65 -0
- package/dist/runtime/exactControlPlane.js +79 -2
- package/dist/runtime/index.js +4 -0
- package/dist/runtime/sessionOwnerIdentity.js +269 -0
- package/dist/runtime/sessionOwnerRegistry.js +132 -0
- package/dist/runtime/sessionReconciliation.js +93 -0
- package/dist/runtime/sessionTerminationGuard.js +211 -0
- package/dist/runtime/taskRuntimeIsolation.js +13 -0
- package/dist/runtime/tmuxAdapters.js +34 -1
- package/dist/scheduler/actionability.js +155 -0
- package/dist/scheduler/activeRoleRunDelivery.js +14 -5
- package/dist/scheduler/activeTaskProgress.js +60 -0
- package/dist/scheduler/leaderWakeupProcessor.js +22 -11
- package/dist/scheduler/roleRunStall.js +135 -29
- package/dist/scheduler/taskExecutionProjection.js +11 -0
- package/dist/storage/compatibleTaskStore.js +112 -5
- package/dist/storage/migration/productionRegistry.js +736 -1
- package/dist/storage/sqliteSchema.js +264 -3
- package/dist/storage/sqliteStore.js +487 -13
- package/dist/storage/storeRpc.js +21 -0
- package/dist/storage/taskStore.js +974 -21
- package/dist/storage/upgrade/homeClassification.js +120 -2
- package/dist/storage/upgrade/migrationReceipt.js +67 -0
- package/dist/storage/upgrade/pseudoLayoutRepair.js +241 -0
- package/dist/storage/upgrade/recordVersions.js +10 -1
- package/dist/storage/upgrade/sqliteMigrationTarget.js +58 -6
- package/dist/storage/upgrade/sqliteRecordMigrationTarget.js +290 -0
- package/dist/storage/upgrade/sqliteStateMigration.js +258 -2
- package/dist/storage/upgrade/upgradeOrchestrator.js +482 -16
- package/dist/task/deliveryGuard.js +226 -0
- package/dist/task/nextAction.js +738 -0
- package/dist/task/repairWave.js +137 -0
- package/dist/task/taskRecordReference.js +6 -1
- package/dist/telemetry/sqliteTelemetryStore.js +387 -0
- package/dist/telemetry/telemetryCompaction.js +251 -0
- package/dist/telemetry/telemetryConfig.js +64 -0
- package/dist/telemetry/telemetryRouter.js +32 -0
- package/dist/telemetry/telemetryStore.js +19 -0
- package/dist/telemetry/telemetryWiring.js +33 -0
- package/dist/tmux/tmuxManager.js +20 -1
- package/dist/tmux/tmuxSocketEndpoint.js +20 -0
- package/dist/verification/gateArtifact.js +216 -0
- package/dist/verification/gateArtifactStore.js +87 -0
- package/dist/verification/verificationGateService.js +414 -0
- package/dist/verification/verificationPlan.js +308 -0
- package/dist/workspace/gitChangeSetCapture.js +12 -2
- package/dist/workspace/workItemChangeSetManager.js +60 -3
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +8 -0
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic Controller handover orchestration (Issue 02).
|
|
3
|
+
*
|
|
4
|
+
* The activator (`yui release activate`) drives one versioned handover:
|
|
5
|
+
*
|
|
6
|
+
* preflight (read-only) -> handover lock -> fence the old Controller
|
|
7
|
+
* -> start the candidate -> read back its identity -> switch the active
|
|
8
|
+
* release pointer -> commit the old Controller -> wait for promotion
|
|
9
|
+
* -> handover receipt.
|
|
10
|
+
*
|
|
11
|
+
* Every phase is recoverable from the durable fence + pointer state. A failed
|
|
12
|
+
* candidate rolls back to the old Controller; a stuck old Controller leaves
|
|
13
|
+
* the candidate read-only and reports dual-owner; a crashed activator resumes
|
|
14
|
+
* from the recorded phase.
|
|
15
|
+
*/
|
|
16
|
+
import { acquireHandoverLock, isOwnerLive, newHandoverId, readActiveReleasePointer, readCandidateDiscovery, readHandoverFence, readHandoverReceipt, readRuntimeIdentity, removeCandidateDiscovery, removeHandoverFence, writeActiveReleasePointer, writeHandoverFence, writeHandoverReceipt } from "./runtimeRelease.js";
|
|
17
|
+
export const DEFAULT_CANDIDATE_READY_TIMEOUT_MS = 30_000;
|
|
18
|
+
export const DEFAULT_PROMOTION_TIMEOUT_MS = 45_000;
|
|
19
|
+
export const DEFAULT_POLL_INTERVAL_MS = 100;
|
|
20
|
+
/**
|
|
21
|
+
* Optional confirmation debounce after the candidate latches `dualOwner:
|
|
22
|
+
* true`. Defaults to 0: the candidate's own exit grace
|
|
23
|
+
* (`DEFAULT_DUAL_OWNER_GRACE_MS` in `handoverCandidate.ts`, 30s) is the single
|
|
24
|
+
* authoritative old-owner exit window, and the activator trusts the candidate's
|
|
25
|
+
* latched signal. A non-zero value only adds a short extra confirmation before
|
|
26
|
+
* reporting dual-owner; it must never be used to re-litigate the exit grace.
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_DUAL_OWNER_GRACE_MS = 0;
|
|
29
|
+
export async function activateRelease(ports, options) {
|
|
30
|
+
const { home, manifest } = options;
|
|
31
|
+
const releaseId = `${manifest.version}-${manifest.packageDigest}`;
|
|
32
|
+
const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
33
|
+
const candidateReadyTimeoutMs = options.candidateReadyTimeoutMs
|
|
34
|
+
?? DEFAULT_CANDIDATE_READY_TIMEOUT_MS;
|
|
35
|
+
const promotionTimeoutMs = options.promotionTimeoutMs ?? DEFAULT_PROMOTION_TIMEOUT_MS;
|
|
36
|
+
const dualOwnerGraceMs = options.dualOwnerGraceMs ?? DEFAULT_DUAL_OWNER_GRACE_MS;
|
|
37
|
+
const active = readActiveReleasePointer(home);
|
|
38
|
+
// An in-flight handover (durable fence) always takes precedence over the
|
|
39
|
+
// already-active shortcut: the pointer may name this release while a
|
|
40
|
+
// crashed activation left the old Controller live or the candidate
|
|
41
|
+
// unpromoted. Recovery must run first.
|
|
42
|
+
if (active !== null && active.releaseId === releaseId && readHandoverFence(home) === null) {
|
|
43
|
+
return { outcome: "already-active", releaseId };
|
|
44
|
+
}
|
|
45
|
+
// 1) Read-only storage compatibility preflight. A failure leaves the old
|
|
46
|
+
// Controller and pointer untouched.
|
|
47
|
+
try {
|
|
48
|
+
ports.runPreflight(options.releaseDir, home);
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
return {
|
|
52
|
+
outcome: "aborted",
|
|
53
|
+
phase: "preflight",
|
|
54
|
+
message: `Storage compatibility preflight failed: ${messageOf(error)}`,
|
|
55
|
+
action: "The current Controller and active release pointer are unchanged.",
|
|
56
|
+
recoverable: true
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const lock = acquireHandoverLock(home);
|
|
60
|
+
try {
|
|
61
|
+
return await activateLocked(ports, {
|
|
62
|
+
home,
|
|
63
|
+
releaseDir: options.releaseDir,
|
|
64
|
+
manifest,
|
|
65
|
+
releaseId,
|
|
66
|
+
active,
|
|
67
|
+
pollIntervalMs,
|
|
68
|
+
candidateReadyTimeoutMs,
|
|
69
|
+
promotionTimeoutMs,
|
|
70
|
+
dualOwnerGraceMs
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
lock.release();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
async function activateLocked(ports, locked) {
|
|
78
|
+
const { home, manifest, releaseDir } = locked;
|
|
79
|
+
const { releaseId, active } = locked;
|
|
80
|
+
// 2) Crash recovery: a durable fence records the exact phase a previous
|
|
81
|
+
// activator reached. Resume or roll back deterministically.
|
|
82
|
+
const recovery = await recoverInterruptedHandover(ports, home, locked);
|
|
83
|
+
if (recovery !== null)
|
|
84
|
+
return recovery;
|
|
85
|
+
// 3) Capture the old Controller identity. When no Controller is running,
|
|
86
|
+
// activation only needs the pointer switch plus a fresh start.
|
|
87
|
+
const old = await readOldController(ports, home);
|
|
88
|
+
if (old === null) {
|
|
89
|
+
return await activateWithoutOldController(ports, home, manifest, releaseDir, releaseId, active);
|
|
90
|
+
}
|
|
91
|
+
// 4) Begin the handover: the old Controller fences mutations and records
|
|
92
|
+
// the durable fence.
|
|
93
|
+
const handoverId = newHandoverId();
|
|
94
|
+
let beginResult;
|
|
95
|
+
try {
|
|
96
|
+
beginResult = await ports.call(home, "controller.begin-handover", {
|
|
97
|
+
handoverId,
|
|
98
|
+
fromReleaseId: active?.releaseId ?? null,
|
|
99
|
+
toReleaseId: releaseId
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
return {
|
|
104
|
+
outcome: "aborted",
|
|
105
|
+
phase: "begin-handover",
|
|
106
|
+
message: `Old Controller rejected the handover: ${messageOf(error)}`,
|
|
107
|
+
action: "The old Controller is still serving; inspect it and retry.",
|
|
108
|
+
recoverable: true
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
const fence = readHandoverFence(home);
|
|
112
|
+
if (fence === null || fence.handoverId !== handoverId) {
|
|
113
|
+
return {
|
|
114
|
+
outcome: "aborted",
|
|
115
|
+
phase: "begin-handover",
|
|
116
|
+
message: "Old Controller did not record the handover fence.",
|
|
117
|
+
action: "The old Controller is still serving; inspect it and retry.",
|
|
118
|
+
recoverable: true
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
void beginResult;
|
|
122
|
+
// 5) Start the candidate and wait for its identity read-back.
|
|
123
|
+
ports.spawnCandidate(home, releaseDir, handoverId);
|
|
124
|
+
const candidateReady = await waitForCandidateReady(ports, home, manifest, locked.candidateReadyTimeoutMs, locked.pollIntervalMs);
|
|
125
|
+
if (candidateReady === null) {
|
|
126
|
+
await rollbackHandover(ports, home, fence, "candidate did not become ready");
|
|
127
|
+
return {
|
|
128
|
+
outcome: "aborted",
|
|
129
|
+
phase: "candidate-ready",
|
|
130
|
+
message: "The new Controller candidate did not become ready in time.",
|
|
131
|
+
action: "The old Controller has resumed accepting mutations; retry the activation.",
|
|
132
|
+
recoverable: true
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
// 6) Switch the active release pointer. This is the atomic commit point:
|
|
136
|
+
// after it succeeds, the stable launcher and new Sessions resolve the new
|
|
137
|
+
// release.
|
|
138
|
+
const pointer = Object.freeze({
|
|
139
|
+
schemaVersion: 1,
|
|
140
|
+
releaseId,
|
|
141
|
+
version: manifest.version,
|
|
142
|
+
buildId: manifest.buildId,
|
|
143
|
+
packageDigest: manifest.packageDigest,
|
|
144
|
+
activatedAt: ports.now().toISOString()
|
|
145
|
+
});
|
|
146
|
+
writeActiveReleasePointer(home, pointer);
|
|
147
|
+
// 7) Commit the old Controller: it closes its socket and exits.
|
|
148
|
+
try {
|
|
149
|
+
await ports.call(home, "controller.commit-handover", { handoverId });
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
// The commit RPC may fail because the old Controller exited after
|
|
153
|
+
// responding. The promotion wait below resolves the true state.
|
|
154
|
+
if (!isConnectionError(error))
|
|
155
|
+
throw error;
|
|
156
|
+
}
|
|
157
|
+
// 8) Wait for the candidate to promote itself (old owner gone + pointer
|
|
158
|
+
// switched). A stuck old owner leaves the candidate read-only.
|
|
159
|
+
const promotion = await waitForPromotion(ports, home, fence, locked.promotionTimeoutMs, locked.pollIntervalMs, locked.dualOwnerGraceMs);
|
|
160
|
+
if (promotion === "promoted") {
|
|
161
|
+
return { outcome: "activated", releaseId, handoverId, oldPid: old.pid };
|
|
162
|
+
}
|
|
163
|
+
if (promotion === "dual-owner") {
|
|
164
|
+
return {
|
|
165
|
+
outcome: "dual-owner",
|
|
166
|
+
releaseId,
|
|
167
|
+
handoverId,
|
|
168
|
+
message: "The old Controller did not exit after the handover commit; "
|
|
169
|
+
+ "the candidate is read-only and no Controller is writing.",
|
|
170
|
+
action: "Stop the old Controller with a fenced `controller stop` "
|
|
171
|
+
+ `(PID ${old.pid}); the candidate will then promote itself.`
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
// Promotion timed out without a clear dual-owner signal. The receipt and
|
|
175
|
+
// fence record the exact state for manual recovery.
|
|
176
|
+
return {
|
|
177
|
+
outcome: "aborted",
|
|
178
|
+
phase: "promotion",
|
|
179
|
+
message: "The candidate did not promote within the timeout.",
|
|
180
|
+
action: "Inspect the handover fence and receipt; the old Controller may "
|
|
181
|
+
+ "still be running or the candidate may have exited.",
|
|
182
|
+
recoverable: false
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
async function activateWithoutOldController(ports, home, manifest, releaseDir, releaseId, active) {
|
|
186
|
+
const handoverId = newHandoverId();
|
|
187
|
+
const pointer = Object.freeze({
|
|
188
|
+
schemaVersion: 1,
|
|
189
|
+
releaseId,
|
|
190
|
+
version: manifest.version,
|
|
191
|
+
buildId: manifest.buildId,
|
|
192
|
+
packageDigest: manifest.packageDigest,
|
|
193
|
+
activatedAt: ports.now().toISOString()
|
|
194
|
+
});
|
|
195
|
+
writeActiveReleasePointer(home, pointer);
|
|
196
|
+
try {
|
|
197
|
+
await ports.startControllerFromRelease(home, releaseDir);
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
// Restore the previous pointer so the launcher keeps resolving the old
|
|
201
|
+
// release.
|
|
202
|
+
if (active !== null)
|
|
203
|
+
writeActiveReleasePointer(home, active);
|
|
204
|
+
return {
|
|
205
|
+
outcome: "aborted",
|
|
206
|
+
phase: "start-controller",
|
|
207
|
+
message: `The new Controller did not start: ${messageOf(error)}`,
|
|
208
|
+
action: "The active release pointer was restored; the old release is unchanged.",
|
|
209
|
+
recoverable: true
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
writeHandoverReceipt(home, Object.freeze({
|
|
213
|
+
schemaVersion: 1,
|
|
214
|
+
handoverId,
|
|
215
|
+
outcome: "completed",
|
|
216
|
+
old: null,
|
|
217
|
+
candidate: null,
|
|
218
|
+
previousReleaseId: active?.releaseId ?? null,
|
|
219
|
+
activatedReleaseId: releaseId,
|
|
220
|
+
startedAt: ports.now().toISOString(),
|
|
221
|
+
completedAt: ports.now().toISOString()
|
|
222
|
+
}));
|
|
223
|
+
return { outcome: "activated", releaseId, handoverId };
|
|
224
|
+
}
|
|
225
|
+
async function readOldController(ports, home) {
|
|
226
|
+
try {
|
|
227
|
+
const identity = await ports.call(home, "controller.identity", {});
|
|
228
|
+
if (typeof identity !== "object"
|
|
229
|
+
|| identity === null
|
|
230
|
+
|| typeof identity.pid !== "number") {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
const record = identity;
|
|
234
|
+
if (record.releaseDrifted === true) {
|
|
235
|
+
throw new Error("The old Controller's release has drifted from its installed manifest; "
|
|
236
|
+
+ "refusing to hand over. Reinstall or restore the active release before activating.");
|
|
237
|
+
}
|
|
238
|
+
const startIdentity = typeof record.processStartIdentity === "string"
|
|
239
|
+
? record.processStartIdentity
|
|
240
|
+
: undefined;
|
|
241
|
+
if (startIdentity === undefined)
|
|
242
|
+
return null;
|
|
243
|
+
return Object.freeze({
|
|
244
|
+
pid: record.pid,
|
|
245
|
+
processStartIdentity: startIdentity,
|
|
246
|
+
buildId: typeof record.buildId === "string" ? record.buildId : "unknown",
|
|
247
|
+
version: typeof record.version === "string" ? record.version : "unknown"
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
if (isControllerNotRunning(error))
|
|
252
|
+
return null;
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
async function waitForCandidateReady(ports, home, manifest, timeoutMs, pollIntervalMs) {
|
|
257
|
+
const deadline = Date.now() + timeoutMs;
|
|
258
|
+
for (;;) {
|
|
259
|
+
const candidate = readCandidateDiscovery(home);
|
|
260
|
+
if (candidate !== null) {
|
|
261
|
+
const identity = readRuntimeIdentity(home);
|
|
262
|
+
if (identity !== null
|
|
263
|
+
&& identity.mode === "candidate"
|
|
264
|
+
&& identity.buildId === manifest.buildId
|
|
265
|
+
&& identity.pid === candidate.pid) {
|
|
266
|
+
return Object.freeze({
|
|
267
|
+
pid: candidate.pid,
|
|
268
|
+
processStartIdentity: candidate.processStartIdentity,
|
|
269
|
+
buildId: identity.buildId,
|
|
270
|
+
version: identity.version
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (Date.now() >= deadline)
|
|
275
|
+
return null;
|
|
276
|
+
await ports.sleep(pollIntervalMs);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
async function waitForPromotion(ports, home, fence, timeoutMs, pollIntervalMs, dualOwnerGraceMs) {
|
|
280
|
+
const deadline = Date.now() + timeoutMs;
|
|
281
|
+
let dualOwnerSince = null;
|
|
282
|
+
for (;;) {
|
|
283
|
+
const receipt = readHandoverReceipt(home);
|
|
284
|
+
if (receipt !== null
|
|
285
|
+
&& receipt.handoverId === fence.handoverId
|
|
286
|
+
&& receipt.outcome === "completed") {
|
|
287
|
+
return "promoted";
|
|
288
|
+
}
|
|
289
|
+
const identity = readRuntimeIdentity(home);
|
|
290
|
+
if (identity !== null && identity.mode === "candidate" && identity.dualOwner) {
|
|
291
|
+
if (dualOwnerSince === null)
|
|
292
|
+
dualOwnerSince = Date.now();
|
|
293
|
+
// Give the old owner a short grace window to exit before declaring
|
|
294
|
+
// dual-owner.
|
|
295
|
+
if (Date.now() - dualOwnerSince > dualOwnerGraceMs)
|
|
296
|
+
return "dual-owner";
|
|
297
|
+
}
|
|
298
|
+
else {
|
|
299
|
+
dualOwnerSince = null;
|
|
300
|
+
}
|
|
301
|
+
if (Date.now() >= deadline)
|
|
302
|
+
return "timeout";
|
|
303
|
+
await ports.sleep(pollIntervalMs);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
async function rollbackHandover(ports, home, fence, reason) {
|
|
307
|
+
const candidate = readCandidateDiscovery(home);
|
|
308
|
+
if (candidate !== null) {
|
|
309
|
+
try {
|
|
310
|
+
ports.killOwnedProcess({
|
|
311
|
+
pid: candidate.pid,
|
|
312
|
+
processStartIdentity: candidate.processStartIdentity,
|
|
313
|
+
buildId: "candidate",
|
|
314
|
+
version: "candidate"
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
catch {
|
|
318
|
+
// A dead candidate is the desired end state; ignore kill failures.
|
|
319
|
+
}
|
|
320
|
+
removeCandidateDiscovery(home);
|
|
321
|
+
}
|
|
322
|
+
try {
|
|
323
|
+
await ports.call(home, "controller.rollback-handover", {
|
|
324
|
+
handoverId: fence.handoverId
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
// The old Controller may have exited; the fence phase records the truth.
|
|
329
|
+
}
|
|
330
|
+
writeHandoverFence(home, Object.freeze({
|
|
331
|
+
...fence,
|
|
332
|
+
phase: "rolled-back",
|
|
333
|
+
updatedAt: new Date().toISOString()
|
|
334
|
+
}));
|
|
335
|
+
writeHandoverReceipt(home, Object.freeze({
|
|
336
|
+
schemaVersion: 1,
|
|
337
|
+
handoverId: fence.handoverId,
|
|
338
|
+
outcome: "rolled-back",
|
|
339
|
+
old: fence.old,
|
|
340
|
+
candidate: fence.candidate,
|
|
341
|
+
previousReleaseId: fence.fromReleaseId,
|
|
342
|
+
activatedReleaseId: fence.toReleaseId,
|
|
343
|
+
startedAt: fence.createdAt,
|
|
344
|
+
completedAt: new Date().toISOString()
|
|
345
|
+
}));
|
|
346
|
+
void reason;
|
|
347
|
+
}
|
|
348
|
+
async function recoverInterruptedHandover(ports, home, locked) {
|
|
349
|
+
const fence = readHandoverFence(home);
|
|
350
|
+
if (fence === null)
|
|
351
|
+
return null;
|
|
352
|
+
if (fence.phase === "rolled-back") {
|
|
353
|
+
removeHandoverFence(home);
|
|
354
|
+
return null;
|
|
355
|
+
}
|
|
356
|
+
const candidateLive = readCandidateDiscovery(home) !== null
|
|
357
|
+
&& (() => {
|
|
358
|
+
const discovery = readCandidateDiscovery(home);
|
|
359
|
+
return discovery !== null && isOwnerLive({
|
|
360
|
+
pid: discovery.pid,
|
|
361
|
+
processStartIdentity: discovery.processStartIdentity,
|
|
362
|
+
buildId: "candidate",
|
|
363
|
+
version: "candidate"
|
|
364
|
+
});
|
|
365
|
+
})();
|
|
366
|
+
const oldLive = isOwnerLive(fence.old);
|
|
367
|
+
if (fence.phase === "committed") {
|
|
368
|
+
if (candidateLive) {
|
|
369
|
+
const promotion = await waitForPromotion(ports, home, fence, locked.promotionTimeoutMs, locked.pollIntervalMs, locked.dualOwnerGraceMs);
|
|
370
|
+
if (promotion === "promoted") {
|
|
371
|
+
return {
|
|
372
|
+
outcome: "activated",
|
|
373
|
+
releaseId: fence.toReleaseId,
|
|
374
|
+
handoverId: fence.handoverId,
|
|
375
|
+
oldPid: fence.old.pid
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
if (promotion === "dual-owner") {
|
|
379
|
+
return {
|
|
380
|
+
outcome: "dual-owner",
|
|
381
|
+
releaseId: fence.toReleaseId,
|
|
382
|
+
handoverId: fence.handoverId,
|
|
383
|
+
message: "The old Controller did not exit after the handover commit; "
|
|
384
|
+
+ "the candidate is read-only and no Controller is writing.",
|
|
385
|
+
action: "Stop the old Controller with a fenced `controller stop`; "
|
|
386
|
+
+ "the candidate will then promote itself."
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (!oldLive) {
|
|
391
|
+
// Both old and candidate are gone; the pointer may already be switched.
|
|
392
|
+
removeHandoverFence(home);
|
|
393
|
+
return null;
|
|
394
|
+
}
|
|
395
|
+
return {
|
|
396
|
+
outcome: "dual-owner",
|
|
397
|
+
releaseId: fence.toReleaseId,
|
|
398
|
+
handoverId: fence.handoverId,
|
|
399
|
+
message: "The handover was committed but the candidate is gone and the "
|
|
400
|
+
+ "old Controller is still live.",
|
|
401
|
+
action: "Stop the old Controller with a fenced `controller stop`, then "
|
|
402
|
+
+ "retry the activation."
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
// phase "fenced" or "candidate-ready"
|
|
406
|
+
if (candidateLive) {
|
|
407
|
+
// Resume the in-flight handover: wait for readiness, switch pointer,
|
|
408
|
+
// commit, promote. The simplest safe resume is to re-drive from the
|
|
409
|
+
// candidate-ready wait.
|
|
410
|
+
const candidateReady = await waitForCandidateReady(ports, home, locked.manifest, locked.candidateReadyTimeoutMs, locked.pollIntervalMs);
|
|
411
|
+
if (candidateReady === null) {
|
|
412
|
+
await rollbackHandover(ports, home, fence, "candidate did not become ready");
|
|
413
|
+
return null;
|
|
414
|
+
}
|
|
415
|
+
const pointer = Object.freeze({
|
|
416
|
+
schemaVersion: 1,
|
|
417
|
+
releaseId: locked.releaseId,
|
|
418
|
+
version: locked.manifest.version,
|
|
419
|
+
buildId: locked.manifest.buildId,
|
|
420
|
+
packageDigest: locked.manifest.packageDigest,
|
|
421
|
+
activatedAt: ports.now().toISOString()
|
|
422
|
+
});
|
|
423
|
+
writeActiveReleasePointer(home, pointer);
|
|
424
|
+
try {
|
|
425
|
+
await ports.call(home, "controller.commit-handover", {
|
|
426
|
+
handoverId: fence.handoverId
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
catch (error) {
|
|
430
|
+
if (!isConnectionError(error))
|
|
431
|
+
throw error;
|
|
432
|
+
}
|
|
433
|
+
const promotion = await waitForPromotion(ports, home, fence, locked.promotionTimeoutMs, locked.pollIntervalMs, locked.dualOwnerGraceMs);
|
|
434
|
+
if (promotion === "promoted") {
|
|
435
|
+
return {
|
|
436
|
+
outcome: "activated",
|
|
437
|
+
releaseId: locked.releaseId,
|
|
438
|
+
handoverId: fence.handoverId,
|
|
439
|
+
oldPid: fence.old.pid
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
return null;
|
|
443
|
+
}
|
|
444
|
+
// Candidate is dead: roll back the old Controller and start fresh.
|
|
445
|
+
if (oldLive) {
|
|
446
|
+
try {
|
|
447
|
+
await ports.call(home, "controller.rollback-handover", {
|
|
448
|
+
handoverId: fence.handoverId
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
// Old Controller may be unresponsive; the fence phase records the truth.
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
removeHandoverFence(home);
|
|
456
|
+
return null;
|
|
457
|
+
}
|
|
458
|
+
function isControllerNotRunning(error) {
|
|
459
|
+
return typeof error === "object"
|
|
460
|
+
&& error !== null
|
|
461
|
+
&& "code" in error
|
|
462
|
+
&& error.code === "CONTROLLER_NOT_RUNNING";
|
|
463
|
+
}
|
|
464
|
+
function isConnectionError(error) {
|
|
465
|
+
if (typeof error !== "object" || error === null || !("code" in error))
|
|
466
|
+
return false;
|
|
467
|
+
const code = error.code;
|
|
468
|
+
return code === "CONTROLLER_NOT_RUNNING"
|
|
469
|
+
|| code === "ECONNREFUSED"
|
|
470
|
+
|| code === "ECONNRESET"
|
|
471
|
+
|| code === "EPIPE";
|
|
472
|
+
}
|
|
473
|
+
function messageOf(error) {
|
|
474
|
+
return error instanceof Error ? error.message : String(error);
|
|
475
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { writeTextFileAtomically } from "../storage/durableFile.js";
|
|
4
|
+
/**
|
|
5
|
+
* Durable adapter-level idempotency for the real release-workflow adapter.
|
|
6
|
+
*
|
|
7
|
+
* The engine already prevents duplicate effects within a workflow run (it
|
|
8
|
+
* never re-submits a step it has marked terminal, and re-attempts after a
|
|
9
|
+
* crash are serialized by a cross-process lock). The residual window is the
|
|
10
|
+
* crash AFTER the external effect landed but BEFORE the engine persisted the
|
|
11
|
+
* step result: a resumed process re-invokes `executeStep` under the same
|
|
12
|
+
* idempotency key, and without this store the shell command runs a second
|
|
13
|
+
* time. This store makes `executeStep` idempotent across processes: a
|
|
14
|
+
* recorded successful effect is replayed instead of re-executed.
|
|
15
|
+
*
|
|
16
|
+
* Only SUCCEEDED effects are recorded. A failed effect produced no confirmed
|
|
17
|
+
* landing (and the engine retries failed steps), so caching it would strand
|
|
18
|
+
* recoverable failures; a timeout is re-queried authoritatively by the
|
|
19
|
+
* engine and re-executed only when the effect is proven absent.
|
|
20
|
+
*
|
|
21
|
+
* Layout
|
|
22
|
+
* ------
|
|
23
|
+
* Each key is its own record file under `release-idempotency/`, named by the
|
|
24
|
+
* percent-encoded key. There is deliberately no whole-map cache and no
|
|
25
|
+
* whole-file rewrite: two processes that record different keys touch
|
|
26
|
+
* different files, so neither can clobber the other, and a process that
|
|
27
|
+
* crashes mid-write leaves at most one stale temp file (the atomic
|
|
28
|
+
* temp+rename in `writeTextFileAtomically` never exposes a partial record).
|
|
29
|
+
* The temp filename carries the writer pid and a UUID, so two processes
|
|
30
|
+
* never share a temp pathname.
|
|
31
|
+
*
|
|
32
|
+
* Every record carries `schemaVersion: 1`. A record whose version is unknown,
|
|
33
|
+
* whose key does not match the file it was read from, or whose effect is not
|
|
34
|
+
* a succeeded effect fails closed (the read throws) rather than being
|
|
35
|
+
* silently skipped: at-most-once cannot be proven from a corrupt record.
|
|
36
|
+
*
|
|
37
|
+
* A persistence failure is propagated, never swallowed. The adapter converts
|
|
38
|
+
* an unpersisted success into an ambiguous/fail-closed outcome; reporting
|
|
39
|
+
* success for an effect whose dedup record did not land would let a crash
|
|
40
|
+
* re-run the effect.
|
|
41
|
+
*
|
|
42
|
+
* Unsupported layouts
|
|
43
|
+
* -------------------
|
|
44
|
+
* The per-key `schemaVersion: 1` layout is the only supported format. A
|
|
45
|
+
* legacy whole-map file (`release-idempotency.json`) from an unsupported
|
|
46
|
+
* older layout is never auto-migrated: it fails closed with a bounded
|
|
47
|
+
* diagnostic so the operator can inspect and clean it up deliberately.
|
|
48
|
+
*/
|
|
49
|
+
export const RELEASE_IDEMPOTENCY_SCHEMA_VERSION = 1;
|
|
50
|
+
const DIRECTORY_NAME = "release-idempotency";
|
|
51
|
+
const LEGACY_FILE_NAME = "release-idempotency.json";
|
|
52
|
+
export function createFileReleaseIdempotencyStore(home, now = () => new Date()) {
|
|
53
|
+
const directory = join(home, DIRECTORY_NAME);
|
|
54
|
+
const legacyPath = join(home, LEGACY_FILE_NAME);
|
|
55
|
+
function recordPath(key) {
|
|
56
|
+
// encodeURIComponent is injective and leaves only Linux-filename-safe
|
|
57
|
+
// characters unescaped, so distinct keys never collide on disk.
|
|
58
|
+
return join(directory, `${encodeURIComponent(key)}.json`);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Fail closed when an unsupported legacy whole-map file is present.
|
|
62
|
+
* The per-key layout is the only supported format; a whole-map file
|
|
63
|
+
* from an unsupported older layout must not be silently trusted.
|
|
64
|
+
*/
|
|
65
|
+
function rejectLegacyLayout() {
|
|
66
|
+
if (existsSync(legacyPath)) {
|
|
67
|
+
throw new Error(`unsupported legacy release idempotency layout: ${legacyPath} ` +
|
|
68
|
+
`(whole-map file). Remove it after inspecting its contents; ` +
|
|
69
|
+
`the per-key schemaVersion 1 layout is the only supported format.`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
async load(key) {
|
|
74
|
+
try {
|
|
75
|
+
rejectLegacyLayout();
|
|
76
|
+
const path = recordPath(key);
|
|
77
|
+
if (!existsSync(path))
|
|
78
|
+
return undefined;
|
|
79
|
+
const parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
80
|
+
return requireRecord(parsed, key).effect;
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
throw new Error(`release idempotency store cannot be read: ${error instanceof Error ? error.message : String(error)}`);
|
|
84
|
+
}
|
|
85
|
+
},
|
|
86
|
+
async recordSuccess(key, effect) {
|
|
87
|
+
if (effect.outcome !== "succeeded") {
|
|
88
|
+
throw new Error(`release idempotency store only records succeeded effects: ${effect.outcome}`);
|
|
89
|
+
}
|
|
90
|
+
try {
|
|
91
|
+
rejectLegacyLayout();
|
|
92
|
+
const record = Object.freeze({
|
|
93
|
+
schemaVersion: RELEASE_IDEMPOTENCY_SCHEMA_VERSION,
|
|
94
|
+
key,
|
|
95
|
+
effect,
|
|
96
|
+
recordedAt: now().toISOString()
|
|
97
|
+
});
|
|
98
|
+
writeTextFileAtomically(recordPath(key), JSON.stringify(record, null, 2));
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
// Propagate: the adapter must not turn an unpersisted success into a
|
|
102
|
+
// successful return.
|
|
103
|
+
throw new Error(`release idempotency store cannot be written: ${error instanceof Error ? error.message : String(error)}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/** An in-memory store for tests; two instances over the same map simulate processes. */
|
|
109
|
+
export function createInMemoryReleaseIdempotencyStore(backing) {
|
|
110
|
+
const records = backing ?? new Map();
|
|
111
|
+
return {
|
|
112
|
+
async load(key) {
|
|
113
|
+
return records.get(key)?.effect;
|
|
114
|
+
},
|
|
115
|
+
async recordSuccess(key, effect) {
|
|
116
|
+
if (effect.outcome !== "succeeded") {
|
|
117
|
+
throw new Error(`release idempotency store only records succeeded effects: ${effect.outcome}`);
|
|
118
|
+
}
|
|
119
|
+
records.set(key, {
|
|
120
|
+
schemaVersion: RELEASE_IDEMPOTENCY_SCHEMA_VERSION,
|
|
121
|
+
key,
|
|
122
|
+
effect,
|
|
123
|
+
recordedAt: new Date(0).toISOString()
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Strictly validate one per-key record. Fails closed on any drift: an unknown
|
|
130
|
+
* schema version, a key that does not name the file it was read from, or a
|
|
131
|
+
* non-succeeded effect.
|
|
132
|
+
*/
|
|
133
|
+
function requireRecord(value, expectedKey) {
|
|
134
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
135
|
+
throw new Error("idempotency record is not an object");
|
|
136
|
+
}
|
|
137
|
+
const record = value;
|
|
138
|
+
if (record.schemaVersion !== RELEASE_IDEMPOTENCY_SCHEMA_VERSION) {
|
|
139
|
+
throw new Error(`idempotency record schemaVersion is ${String(record.schemaVersion)}, ` +
|
|
140
|
+
`expected ${RELEASE_IDEMPOTENCY_SCHEMA_VERSION}`);
|
|
141
|
+
}
|
|
142
|
+
if (typeof record.key !== "string" || record.key !== expectedKey) {
|
|
143
|
+
throw new Error(`idempotency record key ${String(record.key)} does not match ${expectedKey}`);
|
|
144
|
+
}
|
|
145
|
+
const effect = requireSucceededEffect(record.effect);
|
|
146
|
+
if (typeof record.recordedAt !== "string" || record.recordedAt.length === 0) {
|
|
147
|
+
throw new Error("idempotency record recordedAt is invalid");
|
|
148
|
+
}
|
|
149
|
+
return Object.freeze({
|
|
150
|
+
schemaVersion: RELEASE_IDEMPOTENCY_SCHEMA_VERSION,
|
|
151
|
+
key: record.key,
|
|
152
|
+
effect,
|
|
153
|
+
recordedAt: record.recordedAt
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
function requireSucceededEffect(value) {
|
|
157
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
158
|
+
throw new Error("idempotency record effect is not an object");
|
|
159
|
+
}
|
|
160
|
+
const effect = value;
|
|
161
|
+
if (effect.outcome !== "succeeded") {
|
|
162
|
+
throw new Error(`idempotency record effect outcome is ${String(effect.outcome)}, expected succeeded`);
|
|
163
|
+
}
|
|
164
|
+
return value;
|
|
165
|
+
}
|