@ricsam/r5d-worker 0.0.107 → 0.0.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/main.cjs +431 -145
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/plan-lint.cjs +165 -0
- package/dist/cjs/plan-parser.cjs +127 -0
- package/dist/cjs/plan-paths.cjs +53 -0
- package/dist/cjs/recovery-output-window.cjs +77 -0
- package/dist/cjs/recovery-protocol.cjs +31 -0
- package/dist/cjs/recovery-store.cjs +229 -0
- package/dist/cjs/supervisor.cjs +11 -4
- package/dist/mjs/main.mjs +433 -144
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/plan-lint.mjs +129 -0
- package/dist/mjs/plan-parser.mjs +94 -0
- package/dist/mjs/plan-paths.mjs +25 -0
- package/dist/mjs/recovery-output-window.mjs +53 -0
- package/dist/mjs/recovery-protocol.mjs +6 -0
- package/dist/mjs/recovery-store.mjs +194 -0
- package/dist/mjs/supervisor.mjs +10 -4
- package/dist/types/main.d.ts +49 -3
- package/dist/types/plan-lint.d.ts +16 -0
- package/dist/types/plan-parser.d.ts +38 -0
- package/dist/types/plan-paths.d.ts +36 -0
- package/dist/types/recovery-output-window.d.ts +19 -0
- package/dist/types/recovery-protocol.d.ts +59 -0
- package/dist/types/recovery-store.d.ts +38 -0
- package/dist/types/supervisor.d.ts +2 -1
- package/package.json +3 -2
package/dist/cjs/main.cjs
CHANGED
|
@@ -56,7 +56,13 @@ __export(main_exports, {
|
|
|
56
56
|
writeWorkerTextFile: () => writeWorkerTextFile
|
|
57
57
|
});
|
|
58
58
|
module.exports = __toCommonJS(main_exports);
|
|
59
|
+
var import_plan_paths = require("./plan-paths.cjs");
|
|
60
|
+
var import_plan_parser = require("./plan-parser.cjs");
|
|
61
|
+
var import_plan_lint = require("./plan-lint.cjs");
|
|
59
62
|
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
63
|
+
var import_recovery_store = require("./recovery-store.cjs");
|
|
64
|
+
var import_recovery_output_window = require("./recovery-output-window.cjs");
|
|
65
|
+
var import_recovery_protocol = require("./recovery-protocol.cjs");
|
|
60
66
|
var import_node_path = __toESM(require("node:path"), 1);
|
|
61
67
|
var import_node_crypto = require("node:crypto");
|
|
62
68
|
var import_node_os = __toESM(require("node:os"), 1);
|
|
@@ -158,6 +164,63 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
|
|
|
158
164
|
const activePtys = /* @__PURE__ */ new Map();
|
|
159
165
|
const workspaceSyncPriorityPtyTargets = /* @__PURE__ */ new Map();
|
|
160
166
|
let currentWorkerSocket = null;
|
|
167
|
+
const workerRuntimeId = crypto.randomUUID();
|
|
168
|
+
const workerOutputWindow = new import_recovery_output_window.WorkerOutputWindow();
|
|
169
|
+
let recoveryStore;
|
|
170
|
+
let workerReconnectAttempt = 0;
|
|
171
|
+
let workerRecoveryId;
|
|
172
|
+
let workerExecutionLeaseDeadline = 0;
|
|
173
|
+
let workerExecutionLeaseTimer;
|
|
174
|
+
let workerLeaseExpired = false;
|
|
175
|
+
const infrastructureStoppedRunIds = /* @__PURE__ */ new Set();
|
|
176
|
+
function isWorkerCommunicationFailure(message) {
|
|
177
|
+
return /Could not resolve host|Couldn't connect|Failed to connect|Connection reset|Connection timed out|remote end hung up|Empty reply|requested URL returned error: (?:408|429|5\d\d)|fetch failed|socket connection|ECONNRESET|ECONNREFUSED|ETIMEDOUT|Unable to connect|network connection was lost|Worker communication recovery expired|operation was aborted due to timeout|operation timed out|Failed to (?:fetch|download).*\((?:408|429|5\d\d)\)/i.test(
|
|
178
|
+
message
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
async function retryWorkerServerRead(read) {
|
|
182
|
+
let deadline;
|
|
183
|
+
let retry = 0;
|
|
184
|
+
while (true) {
|
|
185
|
+
try {
|
|
186
|
+
return await read();
|
|
187
|
+
} catch (error) {
|
|
188
|
+
if (!recoveryStore || !isWorkerCommunicationFailure(error instanceof Error ? error.message : String(error))) throw error;
|
|
189
|
+
deadline ??= Date.now() + import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS;
|
|
190
|
+
if (workerLeaseExpired || Date.now() >= deadline) throw new WorkerServerUnavailableError("Worker communication recovery expired");
|
|
191
|
+
currentWorkerSocket?.close(1012, "Communication recovery");
|
|
192
|
+
await Bun.sleep((0, import_supervisor.workerReconnectDelayMs)(Math.random, retry++));
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
function renewWorkerExecutionLease(leaseMs) {
|
|
197
|
+
if (!Number.isFinite(leaseMs) || leaseMs <= 0) return;
|
|
198
|
+
workerLeaseExpired = false;
|
|
199
|
+
workerExecutionLeaseDeadline = Date.now() + Math.min(leaseMs, import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS);
|
|
200
|
+
if (workerExecutionLeaseTimer) clearTimeout(workerExecutionLeaseTimer);
|
|
201
|
+
workerExecutionLeaseTimer = setTimeout(
|
|
202
|
+
() => {
|
|
203
|
+
if (Date.now() < workerExecutionLeaseDeadline) return;
|
|
204
|
+
workerLeaseExpired = true;
|
|
205
|
+
for (const operation of recoveryStore?.operations() ?? []) {
|
|
206
|
+
if (operation.sessionId && operation.state !== "completed") recoveryStore?.cancelSession(operation.sessionId);
|
|
207
|
+
}
|
|
208
|
+
for (const [runId, active] of activeProcesses) {
|
|
209
|
+
if (!active.sessionId) continue;
|
|
210
|
+
infrastructureStoppedRunIds.add(runId);
|
|
211
|
+
recoveryStore?.unknownRun(runId);
|
|
212
|
+
cancelledProcessRuns.add(runId);
|
|
213
|
+
closeProcessStdin(active);
|
|
214
|
+
void (0, import_process_tree.terminateProcessTree)(active.process).catch(
|
|
215
|
+
(error) => process.stderr.write(`[r5d-worker] execution lease termination failed: ${error}
|
|
216
|
+
`)
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
},
|
|
220
|
+
Math.min(leaseMs, import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS)
|
|
221
|
+
);
|
|
222
|
+
workerExecutionLeaseTimer.unref();
|
|
223
|
+
}
|
|
161
224
|
let workerAdmissionGeneration = 0;
|
|
162
225
|
const workspaceMutationGate = new import_workspace_mutation_gate.WorkspaceMutationGate();
|
|
163
226
|
let workspaceSyncQueue = Promise.resolve();
|
|
@@ -565,7 +628,7 @@ function targetPlanDir(planRoot, target) {
|
|
|
565
628
|
function planEnv(activePlanId) {
|
|
566
629
|
const env = {};
|
|
567
630
|
if (activePlanId) {
|
|
568
|
-
validatePlanId(activePlanId);
|
|
631
|
+
(0, import_plan_paths.validatePlanId)(activePlanId);
|
|
569
632
|
env.R5D_ACTIVE_PLAN_ID = activePlanId;
|
|
570
633
|
}
|
|
571
634
|
return env;
|
|
@@ -616,40 +679,46 @@ function parseRemoteFileManifest(value, collectionKey, label) {
|
|
|
616
679
|
});
|
|
617
680
|
}
|
|
618
681
|
async function fetchRemoteFileManifest(input) {
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
682
|
+
return retryWorkerServerRead(async () => {
|
|
683
|
+
const response = await fetch(input.url, {
|
|
684
|
+
signal: AbortSignal.timeout(3e4),
|
|
685
|
+
headers: {
|
|
686
|
+
Authorization: `Bearer ${input.token}`
|
|
687
|
+
}
|
|
688
|
+
});
|
|
689
|
+
if (!response.ok) {
|
|
690
|
+
const detail = (await readResponseText(response)).trim();
|
|
691
|
+
throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
|
|
622
692
|
}
|
|
693
|
+
return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
|
|
623
694
|
});
|
|
624
|
-
if (!response.ok) {
|
|
625
|
-
const detail = (await readResponseText(response)).trim();
|
|
626
|
-
throw new Error(`Failed to fetch ${input.label} manifest (${response.status})${detail ? `: ${detail}` : ""}`);
|
|
627
|
-
}
|
|
628
|
-
return parseRemoteFileManifest(await response.json(), input.collectionKey, input.label);
|
|
629
695
|
}
|
|
630
696
|
async function downloadRemoteSyncedFile(input) {
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
697
|
+
return retryWorkerServerRead(async () => {
|
|
698
|
+
const targetPath = import_node_path.default.join(input.targetDir, input.file.filename);
|
|
699
|
+
assertInsideRoot(input.targetDir, targetPath, input.label);
|
|
700
|
+
const response = await fetch(input.url, {
|
|
701
|
+
signal: AbortSignal.timeout(3e4),
|
|
702
|
+
headers: {
|
|
703
|
+
Authorization: `Bearer ${input.token}`
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
if (!response.ok) {
|
|
707
|
+
const detail = (await readResponseText(response)).trim();
|
|
708
|
+
throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
|
|
636
709
|
}
|
|
710
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
711
|
+
const receivedHash = (0, import_node_crypto.createHash)("sha256").update(buffer).digest("hex");
|
|
712
|
+
if (receivedHash !== input.file.sha256) {
|
|
713
|
+
throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
|
|
714
|
+
}
|
|
715
|
+
if (buffer.length !== input.file.size) {
|
|
716
|
+
throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
|
|
717
|
+
}
|
|
718
|
+
const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
|
|
719
|
+
import_node_fs.default.writeFileSync(tempPath, buffer);
|
|
720
|
+
import_node_fs.default.renameSync(tempPath, targetPath);
|
|
637
721
|
});
|
|
638
|
-
if (!response.ok) {
|
|
639
|
-
const detail = (await readResponseText(response)).trim();
|
|
640
|
-
throw new Error(`Failed to download ${input.label} ${input.file.filename} (${response.status})${detail ? `: ${detail}` : ""}`);
|
|
641
|
-
}
|
|
642
|
-
const buffer = Buffer.from(await response.arrayBuffer());
|
|
643
|
-
const receivedHash = (0, import_node_crypto.createHash)("sha256").update(buffer).digest("hex");
|
|
644
|
-
if (receivedHash !== input.file.sha256) {
|
|
645
|
-
throw new Error(`Downloaded ${input.label} ${input.file.filename} failed sha256 verification`);
|
|
646
|
-
}
|
|
647
|
-
if (buffer.length !== input.file.size) {
|
|
648
|
-
throw new Error(`Downloaded ${input.label} ${input.file.filename} has unexpected size`);
|
|
649
|
-
}
|
|
650
|
-
const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
|
|
651
|
-
import_node_fs.default.writeFileSync(tempPath, buffer);
|
|
652
|
-
import_node_fs.default.renameSync(tempPath, targetPath);
|
|
653
722
|
}
|
|
654
723
|
async function syncRemoteFileSet(input) {
|
|
655
724
|
import_node_fs.default.mkdirSync(input.targetDir, { recursive: true });
|
|
@@ -776,6 +845,7 @@ async function verifyR5dctlAuth(baseUrl, token, workerLabel) {
|
|
|
776
845
|
let response;
|
|
777
846
|
try {
|
|
778
847
|
response = await fetch(statusUrl, {
|
|
848
|
+
signal: AbortSignal.timeout(3e4),
|
|
779
849
|
headers: {
|
|
780
850
|
Authorization: `Bearer ${token}`
|
|
781
851
|
}
|
|
@@ -834,11 +904,6 @@ function validateProjectId(projectId) {
|
|
|
834
904
|
function validateBranchName(branchName) {
|
|
835
905
|
(0, import_managed_paths.validateManagedBranchName)(branchName);
|
|
836
906
|
}
|
|
837
|
-
function validatePlanId(planId) {
|
|
838
|
-
if (!/^[a-z0-9][a-z0-9-]*$/.test(planId)) {
|
|
839
|
-
throw new Error(`Invalid plan id from server: ${planId}`);
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
907
|
const NON_RECURSIVE_GIT_CONFIG_ARGS = [
|
|
843
908
|
"-c",
|
|
844
909
|
"submodule.recurse=false",
|
|
@@ -1745,6 +1810,7 @@ const workerGitSecurityTestHarness = {
|
|
|
1745
1810
|
workspaceSyncFailureHydrationIsSafe,
|
|
1746
1811
|
assertWorkerChildAdmission,
|
|
1747
1812
|
executeWriteFileOperation,
|
|
1813
|
+
executeOperation,
|
|
1748
1814
|
executeEditFileOperation,
|
|
1749
1815
|
terminateCredentialBearingChildren,
|
|
1750
1816
|
terminateCredentialBearingChildrenWithRetention,
|
|
@@ -2183,6 +2249,55 @@ function editWorkerTextFile(branchPath, filePath, edits, builtInPaths) {
|
|
|
2183
2249
|
edits: edits.length
|
|
2184
2250
|
};
|
|
2185
2251
|
}
|
|
2252
|
+
async function executePlanOperation(input) {
|
|
2253
|
+
const { message, resolvedTarget } = input;
|
|
2254
|
+
const filePath = message.type === "lint_plans" ? (0, import_plan_paths.getWorkerPlansPath)(resolvedTarget.target) : (0, import_plan_paths.getWorkerPlanPath)(resolvedTarget.target, message.planId);
|
|
2255
|
+
const builtInPaths = await prepareBuiltInToolPathsForTarget({
|
|
2256
|
+
target: resolvedTarget.target,
|
|
2257
|
+
filePath,
|
|
2258
|
+
baseUrl: input.baseUrl,
|
|
2259
|
+
token: input.token,
|
|
2260
|
+
rootDir: input.rootDir,
|
|
2261
|
+
sessionId: message.sessionId,
|
|
2262
|
+
access: message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing ? "write" : "read"
|
|
2263
|
+
});
|
|
2264
|
+
const resolved = resolveWorkerFilePath(resolvedTarget.rootPath, filePath, builtInPaths);
|
|
2265
|
+
if (message.type === "lint_plans") {
|
|
2266
|
+
const result = await (0, import_plan_lint.lintPlans)(resolved.absolutePath, message.glob);
|
|
2267
|
+
return {
|
|
2268
|
+
type: "lint_plans",
|
|
2269
|
+
content: result.lintedPlanCount === 0 ? message.glob ? `No plan files matched glob "${message.glob}".` : "No plan files found." : (0, import_plan_lint.formatPlanLintErrors)(result.errors) || "No plan lint issues found."
|
|
2270
|
+
};
|
|
2271
|
+
}
|
|
2272
|
+
return withFileMutationQueue(mutationQueueKey(resolvedTarget.target, resolved), async () => {
|
|
2273
|
+
input.assertAdmission();
|
|
2274
|
+
if (message.type === "read_plan") {
|
|
2275
|
+
let created = false;
|
|
2276
|
+
if (message.createIfMissing) {
|
|
2277
|
+
try {
|
|
2278
|
+
import_node_fs.default.writeFileSync(resolved.absolutePath, (0, import_plan_parser.createPlanMarkdown)(message.planId), { encoding: "utf8", flag: "wx" });
|
|
2279
|
+
created = true;
|
|
2280
|
+
} catch (error) {
|
|
2281
|
+
if (error.code !== "EEXIST") throw error;
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
return {
|
|
2285
|
+
type: "read_plan",
|
|
2286
|
+
created,
|
|
2287
|
+
read: readWorkerTextFile(resolvedTarget.rootPath, filePath, void 0, void 0, builtInPaths)
|
|
2288
|
+
};
|
|
2289
|
+
}
|
|
2290
|
+
const content = import_node_fs.default.readFileSync(resolved.absolutePath, "utf8");
|
|
2291
|
+
writeWorkerTextFile(resolvedTarget.rootPath, filePath, (0, import_plan_parser.updateTaskCheckbox)(content, message.taskId, message.checked), builtInPaths);
|
|
2292
|
+
return {
|
|
2293
|
+
type: "update_task_status",
|
|
2294
|
+
planId: message.planId,
|
|
2295
|
+
taskId: message.taskId,
|
|
2296
|
+
checked: message.checked,
|
|
2297
|
+
message: `Task ${message.taskId} marked ${message.checked ? "complete" : "incomplete"}.`
|
|
2298
|
+
};
|
|
2299
|
+
});
|
|
2300
|
+
}
|
|
2186
2301
|
async function executeReadFileOperation(input) {
|
|
2187
2302
|
const builtInPaths = await prepareBuiltInToolPathsForTarget({
|
|
2188
2303
|
target: input.resolvedTarget.target,
|
|
@@ -2561,6 +2676,10 @@ async function executeViewFileBytesOperation(input) {
|
|
|
2561
2676
|
}
|
|
2562
2677
|
async function executeOperation(input) {
|
|
2563
2678
|
switch (input.message.type) {
|
|
2679
|
+
case "read_plan":
|
|
2680
|
+
case "lint_plans":
|
|
2681
|
+
case "update_task_status":
|
|
2682
|
+
return executePlanOperation({ ...input, message: input.message });
|
|
2564
2683
|
case "read":
|
|
2565
2684
|
return executeReadFileOperation({ ...input, message: input.message });
|
|
2566
2685
|
case "write":
|
|
@@ -2890,23 +3009,46 @@ function closeProcessStdin(active) {
|
|
|
2890
3009
|
}
|
|
2891
3010
|
active.stdin = void 0;
|
|
2892
3011
|
}
|
|
2893
|
-
function
|
|
3012
|
+
function sendReplayWorkerMessage(ws, message) {
|
|
2894
3013
|
const target = currentWorkerSocket ?? ws;
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
3014
|
+
if (target.readyState !== WebSocket.OPEN) return false;
|
|
3015
|
+
const send = (payload) => {
|
|
3016
|
+
try {
|
|
3017
|
+
target.send(JSON.stringify(payload));
|
|
3018
|
+
return true;
|
|
3019
|
+
} catch {
|
|
3020
|
+
return false;
|
|
3021
|
+
}
|
|
3022
|
+
};
|
|
3023
|
+
return message.type === "exec_output" ? workerOutputWindow.send(message, send, target.bufferedAmount) : send(message);
|
|
3024
|
+
}
|
|
3025
|
+
function sendWorkerMessage(ws, message) {
|
|
3026
|
+
if (recoveryStore && "requestId" in message && typeof message.requestId === "string" && recoveryStore.isUnknown(message.requestId))
|
|
3027
|
+
return;
|
|
3028
|
+
const transportError = "error" in message && typeof message.error === "string" ? message.error : message.type === "workspace_sync_result" && message.result.outcome === "failed" ? message.result.error : void 0;
|
|
3029
|
+
if (recoveryStore && transportError && isWorkerCommunicationFailure(transportError) && "requestId" in message && typeof message.requestId === "string") {
|
|
3030
|
+
recoveryStore.unknown(message.requestId);
|
|
3031
|
+
currentWorkerSocket?.close(1012, "Communication recovery");
|
|
3032
|
+
return;
|
|
2900
3033
|
}
|
|
3034
|
+
const recorded = recoveryStore?.record(message) ?? message;
|
|
3035
|
+
if ((message.type === "exec_exit" || message.type === "exec_error") && infrastructureStoppedRunIds.has(message.runId)) {
|
|
3036
|
+
recoveryStore?.unknownRun(message.runId);
|
|
3037
|
+
return;
|
|
3038
|
+
}
|
|
3039
|
+
if (message.type === "exec_output" && workerRecoveryId !== void 0) return;
|
|
3040
|
+
sendReplayWorkerMessage(ws, recorded);
|
|
3041
|
+
}
|
|
3042
|
+
function sendSerializedWorkerMessage(ws, serialized) {
|
|
3043
|
+
sendWorkerMessage(ws, JSON.parse(serialized));
|
|
2901
3044
|
}
|
|
2902
3045
|
function sendWorkerMessageFromCurrentSource(ws, message) {
|
|
2903
|
-
if (
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
} catch (error) {
|
|
2907
|
-
process.stderr.write(`[r5d-worker] failed to send ${message.type}: ${error instanceof Error ? error.message : String(error)}
|
|
2908
|
-
`);
|
|
3046
|
+
if ("requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_configured") {
|
|
3047
|
+
sendWorkerMessage(ws, message);
|
|
3048
|
+
return;
|
|
2909
3049
|
}
|
|
3050
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
3051
|
+
sendWorkerMessage(ws, message);
|
|
2910
3052
|
}
|
|
2911
3053
|
function buildActiveProcessReports() {
|
|
2912
3054
|
return Array.from(activeProcesses.entries()).filter(([, active]) => active.sessionId.length > 0).map(([runId, active]) => ({
|
|
@@ -3420,14 +3562,21 @@ function websocketUrl(baseUrl, label) {
|
|
|
3420
3562
|
url.searchParams.set("version", getWorkerVersion());
|
|
3421
3563
|
return url.toString();
|
|
3422
3564
|
}
|
|
3423
|
-
async function startWorker(options
|
|
3424
|
-
|
|
3565
|
+
async function startWorker(options, projectRuntime = {
|
|
3566
|
+
projectConfigById: /* @__PURE__ */ new Map(),
|
|
3567
|
+
readyProjectIds: /* @__PURE__ */ new Set(),
|
|
3568
|
+
reconciledProjectConfigFingerprints: /* @__PURE__ */ new Map(),
|
|
3569
|
+
gitIdentity: null
|
|
3570
|
+
}) {
|
|
3425
3571
|
const config = readConfig(options.configPath);
|
|
3426
3572
|
const { baseUrl, token } = resolveCredentials(options, config);
|
|
3427
3573
|
initializeCredentialStoreDirectoryOnce();
|
|
3428
3574
|
const label = requireValue(options.label, "Missing required --label <label>");
|
|
3429
3575
|
validateLabel(label);
|
|
3430
3576
|
const rootDir = import_node_path.default.resolve(options.rootDir ?? process.env.R5D_ROOT ?? defaultRootDir());
|
|
3577
|
+
recoveryStore ??= new import_recovery_store.WorkerRecoveryStore(
|
|
3578
|
+
import_node_path.default.join(rootDir, ".worker-recovery", (0, import_node_crypto.createHash)("sha256").update(`${baseUrl}\0${label}`).digest("hex"), "protocol.sqlite")
|
|
3579
|
+
);
|
|
3431
3580
|
const projectsRoot = import_node_path.default.join(rootDir, "projects");
|
|
3432
3581
|
const syncRoot = import_node_path.default.join(rootDir, "sync");
|
|
3433
3582
|
const workspaceShadowRoot = import_node_path.default.join(syncRoot, "workspace");
|
|
@@ -3448,7 +3597,7 @@ async function startWorker(options) {
|
|
|
3448
3597
|
`);
|
|
3449
3598
|
runGit(["--version"]);
|
|
3450
3599
|
await verifyR5dctlAuth(baseUrl, token, label);
|
|
3451
|
-
const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
|
|
3600
|
+
const initializedWorkspaceState = projectRuntime.initializedWorkspaceState ?? await workspaceSyncSingleFlight.runExclusive(() => {
|
|
3452
3601
|
import_node_fs.default.mkdirSync(projectsRoot, { recursive: true });
|
|
3453
3602
|
import_node_fs.default.mkdirSync(syncRoot, { recursive: true });
|
|
3454
3603
|
import_node_fs.default.mkdirSync(artifactRoot, { recursive: true });
|
|
@@ -3459,12 +3608,11 @@ async function startWorker(options) {
|
|
|
3459
3608
|
});
|
|
3460
3609
|
return { store, state: store.read() };
|
|
3461
3610
|
});
|
|
3611
|
+
projectRuntime.initializedWorkspaceState = initializedWorkspaceState;
|
|
3462
3612
|
const projectWorkspaceStateStore = initializedWorkspaceState.store;
|
|
3463
|
-
let projectWorkspaceState = initializedWorkspaceState.
|
|
3464
|
-
const projectConfigById
|
|
3613
|
+
let projectWorkspaceState = initializedWorkspaceState.store.read();
|
|
3614
|
+
const { projectConfigById, readyProjectIds, reconciledProjectConfigFingerprints } = projectRuntime;
|
|
3465
3615
|
const pendingCheckouts = /* @__PURE__ */ new Map();
|
|
3466
|
-
const readyProjectIds = /* @__PURE__ */ new Set();
|
|
3467
|
-
const reconciledProjectConfigFingerprints = /* @__PURE__ */ new Map();
|
|
3468
3616
|
const lastObservedProjectHeads = /* @__PURE__ */ new Map();
|
|
3469
3617
|
const advertisedProjectMirrorRefsTokens = /* @__PURE__ */ new Map();
|
|
3470
3618
|
const advertisedProjectMirrorMountTokens = /* @__PURE__ */ new Map();
|
|
@@ -3494,7 +3642,7 @@ async function startWorker(options) {
|
|
|
3494
3642
|
let workspaceRemoteUrl = null;
|
|
3495
3643
|
let workspaceCredentialHelper = null;
|
|
3496
3644
|
let workspaceCredentialUsername = null;
|
|
3497
|
-
let workspaceGitIdentity =
|
|
3645
|
+
let workspaceGitIdentity = projectRuntime.gitIdentity;
|
|
3498
3646
|
let workspaceConfigured = false;
|
|
3499
3647
|
let activeWorkspaceIncidentId = null;
|
|
3500
3648
|
let deferredWorkspaceConfiguration = null;
|
|
@@ -3502,11 +3650,12 @@ async function startWorker(options) {
|
|
|
3502
3650
|
let workspaceConfigurationReceiptGeneration = 0;
|
|
3503
3651
|
let workspaceAutomaticTimer;
|
|
3504
3652
|
let workspacePeriodicTimer;
|
|
3505
|
-
let pendingAutomaticTrigger;
|
|
3653
|
+
let pendingAutomaticTrigger = projectRuntime.pendingAutomaticTrigger;
|
|
3506
3654
|
const pendingCreatedBranchPublicationNotBefore = /* @__PURE__ */ new Map();
|
|
3507
3655
|
const workspaceProjectionLedger = new import_workspace_projection_ledger.WorkspaceProjectionLedger();
|
|
3508
3656
|
let automaticSyncInFlight = false;
|
|
3509
3657
|
let terminalReplayTimer;
|
|
3658
|
+
let outputReplayTimer;
|
|
3510
3659
|
let workspaceSyncRequestsInFlight = 0;
|
|
3511
3660
|
let lastWorkspaceSyncMirrorObservationMs = 0;
|
|
3512
3661
|
let lastInventoriedMissingCheckouts;
|
|
@@ -3855,7 +4004,9 @@ async function startWorker(options) {
|
|
|
3855
4004
|
const reportedMissingCheckouts = (inventory) => inventory.missingCheckouts.map(({ projectId, branchName, reason }) => ({ projectId, branchName, reason }));
|
|
3856
4005
|
const logMissingCheckouts = (inventory) => {
|
|
3857
4006
|
if (inventory.missingCheckouts.length === 0) return;
|
|
3858
|
-
const described = inventory.missingCheckouts.map(
|
|
4007
|
+
const described = inventory.missingCheckouts.map(
|
|
4008
|
+
({ checkoutPath, reason }) => `${checkoutPath} (${reason === "missing" ? "directory missing" : "not a linked worktree of the project"})`
|
|
4009
|
+
).join(", ");
|
|
3859
4010
|
process.stderr.write(
|
|
3860
4011
|
`[r5d-worker] skipping ${inventory.missingCheckouts.length} configured project checkout(s) this cycle: ${described}; delete the branch through the platform or reconnect this worker to recreate the checkout
|
|
3861
4012
|
`
|
|
@@ -4426,7 +4577,9 @@ async function startWorker(options) {
|
|
|
4426
4577
|
error: error instanceof Error ? error.message : String(error)
|
|
4427
4578
|
};
|
|
4428
4579
|
};
|
|
4429
|
-
const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks(
|
|
4580
|
+
const workspaceHydrationTransactionHooks = createWorkspaceHydrationLeaseHooks(
|
|
4581
|
+
(message) => sendWorkerMessageFromCurrentSource(ws, message)
|
|
4582
|
+
);
|
|
4430
4583
|
const performWorkspaceSync = async (input) => {
|
|
4431
4584
|
if (!workspaceRemoteUrl || !workspaceCredentialHelper || !workspaceCredentialUsername || !workspaceGitIdentity) {
|
|
4432
4585
|
throw new Error("Worker workspace configuration has not been received");
|
|
@@ -4483,7 +4636,11 @@ async function startWorker(options) {
|
|
|
4483
4636
|
const outerRemediation = input.trigger.type === "remediation" || input.trigger.type === "remediation_confirm";
|
|
4484
4637
|
const mounts = outerRemediation ? (0, import_workspace_git_sync.workspaceGitMountsForRemediation)(configuredMounts) : configuredMounts;
|
|
4485
4638
|
if (outerRemediation) workspaceProjectionLedger.reset();
|
|
4486
|
-
const observedProjectHeads = outerRemediation ? [] : await observeProjectHeadsBeforeOuterWorkspace({
|
|
4639
|
+
const observedProjectHeads = outerRemediation ? [] : await observeProjectHeadsBeforeOuterWorkspace({
|
|
4640
|
+
allowNonFastForward: false,
|
|
4641
|
+
failClosed: false,
|
|
4642
|
+
projects: checkoutInventory.projects
|
|
4643
|
+
});
|
|
4487
4644
|
const expectedMirrorHeads = /* @__PURE__ */ new Map();
|
|
4488
4645
|
for (const bundle of observedProjectHeads) {
|
|
4489
4646
|
for (const observation of bundle.observations) {
|
|
@@ -4590,7 +4747,7 @@ async function startWorker(options) {
|
|
|
4590
4747
|
});
|
|
4591
4748
|
const workspaceSyncAdmissionSnapshotsMatch = (left, right) => left.generation === right.generation && left.activeIncidentId === right.activeIncidentId && left.deferredIncidentId === right.deferredIncidentId;
|
|
4592
4749
|
const assertWorkspaceSyncStillAdmitted = (admitted, trigger) => {
|
|
4593
|
-
if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) ||
|
|
4750
|
+
if (!workspaceSyncAdmissionSnapshotsMatch(admitted, captureWorkspaceSyncAdmission()) || !deferredWorkspaceSyncTriggerIsAllowed(deferredWorkspaceConfiguration, activeWorkspaceIncidentId, trigger)) {
|
|
4594
4751
|
throw new WorkspaceSyncAdmissionChangedError("Workspace synchronization admission changed while asynchronous work was in flight");
|
|
4595
4752
|
}
|
|
4596
4753
|
};
|
|
@@ -4644,6 +4801,7 @@ async function startWorker(options) {
|
|
|
4644
4801
|
bouncePending: unsafeWorkspaceSyncBouncePending,
|
|
4645
4802
|
invalidateExecution: () => {
|
|
4646
4803
|
workspaceConfigured = false;
|
|
4804
|
+
projectRuntime.configuration = void 0;
|
|
4647
4805
|
advanceWorkerAdmissionGeneration();
|
|
4648
4806
|
},
|
|
4649
4807
|
scheduleRecoveryBounce: () => {
|
|
@@ -4672,7 +4830,11 @@ async function startWorker(options) {
|
|
|
4672
4830
|
};
|
|
4673
4831
|
}
|
|
4674
4832
|
};
|
|
4675
|
-
const REMEDIATION_TRIGGER_TYPES = /* @__PURE__ */ new Set([
|
|
4833
|
+
const REMEDIATION_TRIGGER_TYPES = /* @__PURE__ */ new Set([
|
|
4834
|
+
"remediation",
|
|
4835
|
+
"remediation_confirm",
|
|
4836
|
+
"remediation_reset"
|
|
4837
|
+
]);
|
|
4676
4838
|
const isOrdinaryWorkspaceSyncRequest = (input) => input.confirmedLargeDiff !== true && input.resetToCanonical !== true && !(input.requiredAncestorHeads && input.requiredAncestorHeads.length > 0) && !REMEDIATION_TRIGGER_TYPES.has(input.trigger.type);
|
|
4677
4839
|
const LOCAL_ACTIVITY_TRIGGER_TYPES = /* @__PURE__ */ new Set([
|
|
4678
4840
|
"write",
|
|
@@ -4713,7 +4875,14 @@ async function startWorker(options) {
|
|
|
4713
4875
|
trigger: leader.trigger,
|
|
4714
4876
|
assertStillAdmitted: () => assertWorkspaceSyncStillAdmitted(admission, leader.trigger)
|
|
4715
4877
|
});
|
|
4716
|
-
return {
|
|
4878
|
+
return {
|
|
4879
|
+
result,
|
|
4880
|
+
ranCycle: true,
|
|
4881
|
+
startedAtMs,
|
|
4882
|
+
finishedAtMs: Date.now(),
|
|
4883
|
+
prepareMs: lastWorkspaceSyncMirrorObservationMs,
|
|
4884
|
+
staleAttemptIds
|
|
4885
|
+
};
|
|
4717
4886
|
}
|
|
4718
4887
|
});
|
|
4719
4888
|
const runWorkspaceSync = async (input) => {
|
|
@@ -4743,7 +4912,7 @@ async function startWorker(options) {
|
|
|
4743
4912
|
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4744
4913
|
throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
|
|
4745
4914
|
}
|
|
4746
|
-
ws
|
|
4915
|
+
sendSerializedWorkerMessage(ws, JSON.stringify({ type: "workspace_sync_started", attemptId }));
|
|
4747
4916
|
}
|
|
4748
4917
|
let queueMs = 0;
|
|
4749
4918
|
let prepareMs = 0;
|
|
@@ -4795,6 +4964,7 @@ async function startWorker(options) {
|
|
|
4795
4964
|
};
|
|
4796
4965
|
const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
|
|
4797
4966
|
pendingAutomaticTrigger = trigger;
|
|
4967
|
+
projectRuntime.pendingAutomaticTrigger = trigger;
|
|
4798
4968
|
const pendingBranchDeferral = () => (0, import_workspace_automatic_sync_policy.pendingCreatedBranchAutomaticSyncDeferral)(
|
|
4799
4969
|
projectWorkspaceState.locallyPendingCreatedBranches,
|
|
4800
4970
|
activeWorkspaceMutationTargets(),
|
|
@@ -4817,6 +4987,7 @@ async function startWorker(options) {
|
|
|
4817
4987
|
return;
|
|
4818
4988
|
}
|
|
4819
4989
|
pendingAutomaticTrigger = void 0;
|
|
4990
|
+
projectRuntime.pendingAutomaticTrigger = void 0;
|
|
4820
4991
|
automaticSyncInFlight = true;
|
|
4821
4992
|
void runWorkspaceSync({ trigger: scheduledTrigger }).then((result) => {
|
|
4822
4993
|
if (result.outcome === "failed") {
|
|
@@ -4840,9 +5011,14 @@ async function startWorker(options) {
|
|
|
4840
5011
|
workspaceAutomaticTimer.unref();
|
|
4841
5012
|
};
|
|
4842
5013
|
const markWorkspaceDirty = (trigger, force = false, target) => {
|
|
5014
|
+
if (projectRuntime.markWorkspaceDirty && projectRuntime.markWorkspaceDirty !== markWorkspaceDirty) {
|
|
5015
|
+
projectRuntime.markWorkspaceDirty(trigger, force, target);
|
|
5016
|
+
return;
|
|
5017
|
+
}
|
|
4843
5018
|
if (target) recordVisibleWorkspaceMutation(target);
|
|
4844
5019
|
scheduleAutomaticWorkspaceSync(trigger, force ? 0 : WORKSPACE_GIT_QUIET_MS);
|
|
4845
5020
|
};
|
|
5021
|
+
projectRuntime.markWorkspaceDirty = markWorkspaceDirty;
|
|
4846
5022
|
const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
|
|
4847
5023
|
const resolveMessageTarget = (target) => {
|
|
4848
5024
|
if (!workspaceConfigured) throw new Error("Worker workspace configuration has not completed successfully");
|
|
@@ -5145,6 +5321,7 @@ async function startWorker(options) {
|
|
|
5145
5321
|
bootstrapCredentialGenerationFingerprintAtProcessStart = null;
|
|
5146
5322
|
visibleGitIdentity = message.gitIdentity;
|
|
5147
5323
|
workspaceGitIdentity = message.gitIdentity;
|
|
5324
|
+
projectRuntime.gitIdentity = message.gitIdentity;
|
|
5148
5325
|
workspaceRemoteUrl = message.workspaceRemoteUrl;
|
|
5149
5326
|
workspaceCredentialHelper = nextWorkspaceCredentialHelper;
|
|
5150
5327
|
workspaceCredentialUsername = workerCredentialUsername;
|
|
@@ -5359,6 +5536,10 @@ async function startWorker(options) {
|
|
|
5359
5536
|
Authorization: `Bearer ${token}`
|
|
5360
5537
|
}
|
|
5361
5538
|
});
|
|
5539
|
+
const connectionAttemptTimer = setTimeout(() => {
|
|
5540
|
+
if (ws.readyState === WebSocket.CONNECTING) ws.terminate();
|
|
5541
|
+
}, 3e4);
|
|
5542
|
+
connectionAttemptTimer.unref();
|
|
5362
5543
|
let lastServerControlMessageAtMs = performance.now();
|
|
5363
5544
|
let serverHeartbeatProbeStartedAtMs;
|
|
5364
5545
|
let lastServerHeartbeatSweepAtMs = performance.now();
|
|
@@ -5454,7 +5635,9 @@ async function startWorker(options) {
|
|
|
5454
5635
|
process.on("SIGTERM", handleGracefulShutdown);
|
|
5455
5636
|
process.on("SIGINT", handleGracefulShutdown);
|
|
5456
5637
|
ws.addEventListener("open", () => {
|
|
5638
|
+
clearTimeout(connectionAttemptTimer);
|
|
5457
5639
|
currentWorkerSocket = ws;
|
|
5640
|
+
workerRecoveryId = "connecting";
|
|
5458
5641
|
noteServerControlActivity();
|
|
5459
5642
|
lastServerHeartbeatSweepAtMs = performance.now();
|
|
5460
5643
|
serverHeartbeatWatchdog = setInterval(() => {
|
|
@@ -5498,6 +5681,9 @@ async function startWorker(options) {
|
|
|
5498
5681
|
});
|
|
5499
5682
|
const hello = {
|
|
5500
5683
|
type: "hello",
|
|
5684
|
+
resumableProtocol: import_recovery_protocol.WORKER_RESUMABLE_PROTOCOL,
|
|
5685
|
+
runtimeId: workerRuntimeId,
|
|
5686
|
+
ledgerId: recoveryStore.ledgerId,
|
|
5501
5687
|
startupReady: startupRecoveryComplete,
|
|
5502
5688
|
hostInfo: {
|
|
5503
5689
|
hostname: (0, import_node_os.hostname)(),
|
|
@@ -5522,15 +5708,17 @@ async function startWorker(options) {
|
|
|
5522
5708
|
root: rootDir
|
|
5523
5709
|
}
|
|
5524
5710
|
};
|
|
5525
|
-
ws
|
|
5526
|
-
|
|
5527
|
-
|
|
5528
|
-
|
|
5529
|
-
|
|
5530
|
-
terminalReplayTimer = setInterval(() => {
|
|
5531
|
-
for (const terminal of pendingProcessTerminals.values()) {
|
|
5532
|
-
sendWorkerMessage(ws, terminal);
|
|
5711
|
+
sendSerializedWorkerMessage(ws, JSON.stringify(hello));
|
|
5712
|
+
outputReplayTimer = setInterval(() => {
|
|
5713
|
+
if (workerRecoveryId !== void 0 || currentWorkerSocket !== ws) return;
|
|
5714
|
+
for (const output of recoveryStore.output()) {
|
|
5715
|
+
if (!sendReplayWorkerMessage(ws, output)) break;
|
|
5533
5716
|
}
|
|
5717
|
+
}, 250);
|
|
5718
|
+
outputReplayTimer.unref();
|
|
5719
|
+
terminalReplayTimer = setInterval(() => {
|
|
5720
|
+
if (workerRecoveryId !== void 0) return;
|
|
5721
|
+
for (const terminal of recoveryStore?.terminals() ?? []) sendReplayWorkerMessage(ws, terminal);
|
|
5534
5722
|
}, 5e3);
|
|
5535
5723
|
terminalReplayTimer.unref();
|
|
5536
5724
|
process.stdout.write("[r5d-worker] connected\n");
|
|
@@ -5538,18 +5726,103 @@ async function startWorker(options) {
|
|
|
5538
5726
|
ws.addEventListener("message", (event) => {
|
|
5539
5727
|
noteServerControlActivity();
|
|
5540
5728
|
void (async () => {
|
|
5729
|
+
if (currentWorkerSocket !== ws) return;
|
|
5541
5730
|
const message = JSON.parse(String(event.data));
|
|
5731
|
+
if (message.type === "recovery_request") {
|
|
5732
|
+
workerRecoveryId = message.recoveryId;
|
|
5733
|
+
sendActiveProcessReport(ws);
|
|
5734
|
+
sendWorkerMessage(ws, {
|
|
5735
|
+
type: "recovery_state",
|
|
5736
|
+
recoveryId: message.recoveryId,
|
|
5737
|
+
runtimeId: workerRuntimeId,
|
|
5738
|
+
ledgerId: recoveryStore.ledgerId,
|
|
5739
|
+
operations: recoveryStore.operations()
|
|
5740
|
+
});
|
|
5741
|
+
const cursors = new Map(recoveryStore.outputCursors().map((cursor) => [cursor.runId, cursor]));
|
|
5742
|
+
for (const cursor of message.outputOffsets) cursors.set(cursor.runId, cursor);
|
|
5743
|
+
workerOutputWindow.reset([...cursors.values()]);
|
|
5744
|
+
for (const response of recoveryStore.responses(message.requests)) sendReplayWorkerMessage(ws, response);
|
|
5745
|
+
for (const output of recoveryStore.output(message.outputOffsets)) {
|
|
5746
|
+
while (true) {
|
|
5747
|
+
if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId || ws.readyState !== WebSocket.OPEN || workerLeaseExpired)
|
|
5748
|
+
return;
|
|
5749
|
+
if (sendReplayWorkerMessage(ws, output)) break;
|
|
5750
|
+
await Bun.sleep(10);
|
|
5751
|
+
}
|
|
5752
|
+
}
|
|
5753
|
+
if (currentWorkerSocket !== ws || workerRecoveryId !== message.recoveryId) return;
|
|
5754
|
+
for (const terminal of recoveryStore.terminals()) sendReplayWorkerMessage(ws, terminal);
|
|
5755
|
+
sendWorkerMessage(ws, { type: "recovery_complete", recoveryId: message.recoveryId });
|
|
5756
|
+
return;
|
|
5757
|
+
}
|
|
5758
|
+
if (message.type === "recovery_ready") {
|
|
5759
|
+
if (workerRecoveryId !== message.recoveryId) return;
|
|
5760
|
+
workerRecoveryId = void 0;
|
|
5761
|
+
workerReconnectAttempt = 0;
|
|
5762
|
+
renewWorkerExecutionLease(message.leaseMs);
|
|
5763
|
+
return;
|
|
5764
|
+
}
|
|
5765
|
+
if (message.type === "worker_execution_lease") {
|
|
5766
|
+
renewWorkerExecutionLease(message.leaseMs);
|
|
5767
|
+
return;
|
|
5768
|
+
}
|
|
5769
|
+
if (message.type === "operation_result_ack") {
|
|
5770
|
+
recoveryStore.acknowledgeResult(message.requestId);
|
|
5771
|
+
return;
|
|
5772
|
+
}
|
|
5773
|
+
if (message.type === "exec_output_ack") {
|
|
5774
|
+
recoveryStore.acknowledgeOutput(message.runId, message.stream, message.nextOffset);
|
|
5775
|
+
workerOutputWindow.acknowledge(message.runId, message.stream, message.nextOffset);
|
|
5776
|
+
return;
|
|
5777
|
+
}
|
|
5778
|
+
if (message.type === "cancel_session") {
|
|
5779
|
+
recoveryStore.cancelSession(message.sessionId);
|
|
5780
|
+
await Promise.all(
|
|
5781
|
+
[...activeProcesses].filter(([, active]) => active.sessionId === message.sessionId).map(async ([runId, active]) => {
|
|
5782
|
+
cancelledProcessRuns.add(runId);
|
|
5783
|
+
closeProcessStdin(active);
|
|
5784
|
+
await (0, import_process_tree.terminateProcessTree)(active.process);
|
|
5785
|
+
})
|
|
5786
|
+
);
|
|
5787
|
+
sendWorkerMessage(ws, { type: "cancel_session_ack", cancellationId: message.cancellationId });
|
|
5788
|
+
return;
|
|
5789
|
+
}
|
|
5542
5790
|
if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
|
|
5543
5791
|
if (!startupRecoveryComplete && message.type !== "ping") return;
|
|
5792
|
+
if (currentWorkerSocket !== ws) return;
|
|
5793
|
+
const operationRequestId = "requestId" in message && typeof message.requestId === "string" && message.type !== "workspace_config" ? message.requestId : void 0;
|
|
5794
|
+
if (operationRequestId) {
|
|
5795
|
+
const admission = recoveryStore.admit({ ...message, requestId: operationRequestId });
|
|
5796
|
+
if (admission === "conflict") {
|
|
5797
|
+
ws.close(1008, "Operation identity conflict");
|
|
5798
|
+
return;
|
|
5799
|
+
}
|
|
5800
|
+
sendWorkerMessage(ws, { type: "operation_received", requestId: operationRequestId });
|
|
5801
|
+
if (admission !== "new") {
|
|
5802
|
+
const response = recoveryStore.response(operationRequestId);
|
|
5803
|
+
if (response && admission !== "unknown") sendReplayWorkerMessage(ws, response);
|
|
5804
|
+
return;
|
|
5805
|
+
}
|
|
5806
|
+
}
|
|
5544
5807
|
const messageAdmissionGeneration = workerAdmissionGeneration;
|
|
5545
|
-
const
|
|
5546
|
-
|
|
5547
|
-
|
|
5548
|
-
|
|
5549
|
-
|
|
5550
|
-
|
|
5551
|
-
|
|
5552
|
-
|
|
5808
|
+
const admissionCredentials = configuredCredentialGenerationFingerprint;
|
|
5809
|
+
const assertMessageAdmission = () => {
|
|
5810
|
+
if (operationRequestId) {
|
|
5811
|
+
if (workerLeaseExpired || recoveryStore.isRequestCancelled(operationRequestId) || admissionCredentials !== configuredCredentialGenerationFingerprint || shutdownAfterClose || !workspaceConfigured) {
|
|
5812
|
+
recoveryStore.unknown(operationRequestId);
|
|
5813
|
+
throw new StaleWorkerAdmissionError();
|
|
5814
|
+
}
|
|
5815
|
+
return;
|
|
5816
|
+
}
|
|
5817
|
+
assertWorkerChildAdmission({
|
|
5818
|
+
capturedGeneration: messageAdmissionGeneration,
|
|
5819
|
+
currentGeneration: workerAdmissionGeneration,
|
|
5820
|
+
sourceSocketIsCurrent: currentWorkerSocket === ws,
|
|
5821
|
+
sourceSocketOpen: ws.readyState === WebSocket.OPEN,
|
|
5822
|
+
workspaceConfigured,
|
|
5823
|
+
runtimeHealthy: !shutdownAfterClose
|
|
5824
|
+
});
|
|
5825
|
+
};
|
|
5553
5826
|
if (message.type === "connected") {
|
|
5554
5827
|
return;
|
|
5555
5828
|
}
|
|
@@ -5565,8 +5838,17 @@ async function startWorker(options) {
|
|
|
5565
5838
|
if (message.type === "workspace_config") {
|
|
5566
5839
|
const receiptGeneration = ++workspaceConfigurationReceiptGeneration;
|
|
5567
5840
|
const refreshesDeferredConfiguration = deferredWorkspaceConfiguration !== null && activeWorkspaceIncidentId === null && message.deferWorkspaceSyncForIncidentId === void 0;
|
|
5568
|
-
|
|
5569
|
-
const
|
|
5841
|
+
const digest = (0, import_recovery_store.workerOperationDigest)(message);
|
|
5842
|
+
const cached = projectRuntime.configuration;
|
|
5843
|
+
if (cached?.digest !== digest) advanceWorkerAdmissionGeneration();
|
|
5844
|
+
const canReuseConfiguration = cached?.digest === digest && !unsafeWorkspaceSyncBouncePending && credentialGenerationFingerprint({ ...message, workerBaseUrl: baseUrl, workerAuthHeader: bearerAuthHeader }) === configuredCredentialGenerationFingerprint;
|
|
5845
|
+
const configured = canReuseConfiguration ? cached.configured : await configureWorkerWorkspace(message, receiptGeneration);
|
|
5846
|
+
if (canReuseConfiguration && configured.result.outcome !== "failed") {
|
|
5847
|
+
workspaceConfigured = true;
|
|
5848
|
+
activeWorkspaceIncidentId = cached.incidentId;
|
|
5849
|
+
deferredWorkspaceConfiguration = cached.deferred;
|
|
5850
|
+
}
|
|
5851
|
+
projectRuntime.configuration = configured.result.outcome !== "failed" ? { digest, configured, incidentId: activeWorkspaceIncidentId, deferred: deferredWorkspaceConfiguration } : void 0;
|
|
5570
5852
|
unsafeWorkspaceSyncBouncePending = false;
|
|
5571
5853
|
if (configured.result.outcome !== "failed") unsafeWorkspaceSyncRecoveryAttempts = 0;
|
|
5572
5854
|
sendWorkerMessageFromCurrentSource(ws, {
|
|
@@ -5612,16 +5894,31 @@ async function startWorker(options) {
|
|
|
5612
5894
|
return;
|
|
5613
5895
|
}
|
|
5614
5896
|
if (message.type === "sync_workspace") {
|
|
5615
|
-
|
|
5616
|
-
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
5621
|
-
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
|
|
5897
|
+
let retry = 0;
|
|
5898
|
+
let recoveryDeadline;
|
|
5899
|
+
while (true) {
|
|
5900
|
+
const result = await runWorkspaceSync({
|
|
5901
|
+
requestId: message.requestId,
|
|
5902
|
+
attemptId: message.attemptId,
|
|
5903
|
+
trigger: message.trigger,
|
|
5904
|
+
confirmedLargeDiff: message.confirmedLargeDiff,
|
|
5905
|
+
confirmationReason: message.confirmationReason,
|
|
5906
|
+
resetToCanonical: message.resetToCanonical,
|
|
5907
|
+
requiredAncestorHeads: message.requiredAncestorHeads,
|
|
5908
|
+
sendResult: false
|
|
5909
|
+
});
|
|
5910
|
+
if (result.outcome !== "failed" || !isWorkerCommunicationFailure(result.error ?? "")) {
|
|
5911
|
+
sendWorkspaceSyncResult(message.requestId, result);
|
|
5912
|
+
return;
|
|
5913
|
+
}
|
|
5914
|
+
recoveryDeadline ??= Date.now() + import_recovery_protocol.WORKER_RECOVERY_TIMEOUT_MS;
|
|
5915
|
+
if (workerLeaseExpired || Date.now() >= recoveryDeadline) {
|
|
5916
|
+
recoveryStore.unknown(message.requestId);
|
|
5917
|
+
return;
|
|
5918
|
+
}
|
|
5919
|
+
currentWorkerSocket?.close(1012, "Communication recovery");
|
|
5920
|
+
await Bun.sleep((0, import_supervisor.workerReconnectDelayMs)(Math.random, retry++));
|
|
5921
|
+
}
|
|
5625
5922
|
}
|
|
5626
5923
|
if (message.type === "create_project_branch") {
|
|
5627
5924
|
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
@@ -5887,6 +6184,7 @@ async function startWorker(options) {
|
|
|
5887
6184
|
}
|
|
5888
6185
|
if (message.type === "exec_terminal_ack") {
|
|
5889
6186
|
pendingProcessTerminals.delete(message.runId);
|
|
6187
|
+
recoveryStore.acknowledgeTerminal(message.runId);
|
|
5890
6188
|
return;
|
|
5891
6189
|
}
|
|
5892
6190
|
if (message.type === "port_forward_connect") {
|
|
@@ -5938,7 +6236,7 @@ async function startWorker(options) {
|
|
|
5938
6236
|
return;
|
|
5939
6237
|
}
|
|
5940
6238
|
if (message.type === "ping") {
|
|
5941
|
-
ws
|
|
6239
|
+
sendSerializedWorkerMessage(ws, JSON.stringify({ type: "pong" }));
|
|
5942
6240
|
return;
|
|
5943
6241
|
}
|
|
5944
6242
|
if (message.type === "cancel") {
|
|
@@ -5958,7 +6256,8 @@ async function startWorker(options) {
|
|
|
5958
6256
|
cancelledProcessRuns.add(message.runId);
|
|
5959
6257
|
cancelMessage = "Cancellation queued before command start";
|
|
5960
6258
|
}
|
|
5961
|
-
|
|
6259
|
+
sendSerializedWorkerMessage(
|
|
6260
|
+
ws,
|
|
5962
6261
|
JSON.stringify({
|
|
5963
6262
|
type: "cancel_result",
|
|
5964
6263
|
requestId: message.requestId,
|
|
@@ -5970,7 +6269,8 @@ async function startWorker(options) {
|
|
|
5970
6269
|
return;
|
|
5971
6270
|
}
|
|
5972
6271
|
if (message.type === "exec_stdin") {
|
|
5973
|
-
const sendAck = (payload) =>
|
|
6272
|
+
const sendAck = (payload) => sendSerializedWorkerMessage(
|
|
6273
|
+
ws,
|
|
5974
6274
|
JSON.stringify({
|
|
5975
6275
|
type: "operation_result",
|
|
5976
6276
|
requestId: message.requestId,
|
|
@@ -6093,16 +6393,19 @@ async function startWorker(options) {
|
|
|
6093
6393
|
ptyId: message.ptyId,
|
|
6094
6394
|
error: `Shell input is deferred for workspace incident ${deferredWorkspaceConfiguration?.incidentId ?? activeWorkspaceIncidentId ?? "unknown"}`
|
|
6095
6395
|
});
|
|
6396
|
+
sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
|
|
6096
6397
|
return;
|
|
6097
6398
|
}
|
|
6098
6399
|
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
6099
6400
|
markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` }, false, activePty.target);
|
|
6100
6401
|
}
|
|
6101
6402
|
writePty(ws, message);
|
|
6403
|
+
sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
|
|
6102
6404
|
return;
|
|
6103
6405
|
}
|
|
6104
6406
|
if (message.type === "pty_resize") {
|
|
6105
6407
|
resizePty(message);
|
|
6408
|
+
sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
|
|
6106
6409
|
return;
|
|
6107
6410
|
}
|
|
6108
6411
|
if (message.type === "pty_close") {
|
|
@@ -6111,6 +6414,7 @@ async function startWorker(options) {
|
|
|
6111
6414
|
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
6112
6415
|
markWorkspaceDirty({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, true, activePty.target);
|
|
6113
6416
|
}
|
|
6417
|
+
sendWorkerMessage(ws, { type: "operation_completed", requestId: message.requestId });
|
|
6114
6418
|
return;
|
|
6115
6419
|
}
|
|
6116
6420
|
if (message.type === "exec") {
|
|
@@ -6192,7 +6496,11 @@ async function startWorker(options) {
|
|
|
6192
6496
|
} finally {
|
|
6193
6497
|
if (targetReserved) workspaceSyncPriorityProcessTargets.delete(message.runId);
|
|
6194
6498
|
}
|
|
6195
|
-
|
|
6499
|
+
if (infrastructureStoppedRunIds.has(message.runId)) {
|
|
6500
|
+
recoveryStore.unknown(message.requestId);
|
|
6501
|
+
return;
|
|
6502
|
+
}
|
|
6503
|
+
sendSerializedWorkerMessage(ws, JSON.stringify(result));
|
|
6196
6504
|
if (hasWorkspaceEffect && targetMayMutateVisibleWorkspace(message.target)) {
|
|
6197
6505
|
markWorkspaceDirty(
|
|
6198
6506
|
{ type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
|
|
@@ -6285,7 +6593,7 @@ async function startWorker(options) {
|
|
|
6285
6593
|
});
|
|
6286
6594
|
return;
|
|
6287
6595
|
}
|
|
6288
|
-
if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
|
|
6596
|
+
if (message.type === "read_plan" || message.type === "lint_plans" || message.type === "update_task_status" || message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes" || message.type === "code_list" || message.type === "code_read") {
|
|
6289
6597
|
if (workspaceOperationsAreFenced(deferredWorkspaceConfiguration, activeWorkspaceIncidentId)) {
|
|
6290
6598
|
sendWorkerMessage(ws, {
|
|
6291
6599
|
type: "operation_result",
|
|
@@ -6295,7 +6603,7 @@ async function startWorker(options) {
|
|
|
6295
6603
|
return;
|
|
6296
6604
|
}
|
|
6297
6605
|
const reservesVisibleWorkspace = targetMayMutateVisibleWorkspace(message.target);
|
|
6298
|
-
const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target);
|
|
6606
|
+
const mutatesVisibleWorkspace = (message.type === "write" || message.type === "edit" || message.type === "update_task_status" || message.type === "read_plan" && message.createIfMissing) && targetMayMutateVisibleWorkspace(message.target);
|
|
6299
6607
|
if (reservesVisibleWorkspace) {
|
|
6300
6608
|
await (0, import_workspace_command_sync_policy.reserveWorkspaceCommandAfterCurrentSync)(message.target, workspaceSyncSingleFlight, () => {
|
|
6301
6609
|
workspaceSyncPriorityOperationTargets.set(message.requestId, message.target);
|
|
@@ -6315,16 +6623,17 @@ async function startWorker(options) {
|
|
|
6315
6623
|
assertAdmission: assertMessageAdmission
|
|
6316
6624
|
});
|
|
6317
6625
|
});
|
|
6318
|
-
|
|
6626
|
+
sendSerializedWorkerMessage(
|
|
6627
|
+
ws,
|
|
6319
6628
|
JSON.stringify({
|
|
6320
6629
|
type: "operation_result",
|
|
6321
6630
|
requestId: message.requestId,
|
|
6322
6631
|
result
|
|
6323
6632
|
})
|
|
6324
6633
|
);
|
|
6325
|
-
if (mutatesVisibleWorkspace) {
|
|
6634
|
+
if (mutatesVisibleWorkspace && (result.type !== "read_plan" || result.created)) {
|
|
6326
6635
|
dirtyTrigger = {
|
|
6327
|
-
type: message.type,
|
|
6636
|
+
type: message.type === "edit" || message.type === "update_task_status" ? "edit" : "write",
|
|
6328
6637
|
sessionId: message.sessionId,
|
|
6329
6638
|
toolCallId: message.requestId,
|
|
6330
6639
|
...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
|
|
@@ -6332,7 +6641,8 @@ async function startWorker(options) {
|
|
|
6332
6641
|
markWorkspaceDirty(dirtyTrigger, false, message.target);
|
|
6333
6642
|
}
|
|
6334
6643
|
} catch (error) {
|
|
6335
|
-
|
|
6644
|
+
sendSerializedWorkerMessage(
|
|
6645
|
+
ws,
|
|
6336
6646
|
JSON.stringify({
|
|
6337
6647
|
type: "operation_result",
|
|
6338
6648
|
requestId: message.requestId,
|
|
@@ -6351,7 +6661,8 @@ async function startWorker(options) {
|
|
|
6351
6661
|
}
|
|
6352
6662
|
const unhandledMessage = message;
|
|
6353
6663
|
if (typeof unhandledMessage.requestId === "string") {
|
|
6354
|
-
|
|
6664
|
+
sendSerializedWorkerMessage(
|
|
6665
|
+
ws,
|
|
6355
6666
|
JSON.stringify({
|
|
6356
6667
|
type: "operation_result",
|
|
6357
6668
|
requestId: unhandledMessage.requestId,
|
|
@@ -6365,6 +6676,7 @@ async function startWorker(options) {
|
|
|
6365
6676
|
});
|
|
6366
6677
|
});
|
|
6367
6678
|
ws.addEventListener("close", (event) => {
|
|
6679
|
+
clearTimeout(connectionAttemptTimer);
|
|
6368
6680
|
const processEvents = process;
|
|
6369
6681
|
processEvents.off("SIGTERM", handleGracefulShutdown);
|
|
6370
6682
|
processEvents.off("SIGINT", handleGracefulShutdown);
|
|
@@ -6381,15 +6693,20 @@ async function startWorker(options) {
|
|
|
6381
6693
|
clearTimeout(deferredWorkspaceConfigurationRefreshTimer);
|
|
6382
6694
|
deferredWorkspaceConfigurationRefreshTimer = void 0;
|
|
6383
6695
|
}
|
|
6696
|
+
if (outputReplayTimer) {
|
|
6697
|
+
clearInterval(outputReplayTimer);
|
|
6698
|
+
outputReplayTimer = void 0;
|
|
6699
|
+
}
|
|
6384
6700
|
if (terminalReplayTimer) {
|
|
6385
6701
|
clearInterval(terminalReplayTimer);
|
|
6386
6702
|
terminalReplayTimer = void 0;
|
|
6387
6703
|
}
|
|
6388
|
-
if (
|
|
6389
|
-
|
|
6390
|
-
|
|
6704
|
+
if (projectRuntime.configuration) {
|
|
6705
|
+
projectRuntime.configuration.incidentId = activeWorkspaceIncidentId;
|
|
6706
|
+
projectRuntime.configuration.deferred = deferredWorkspaceConfiguration;
|
|
6391
6707
|
}
|
|
6392
|
-
|
|
6708
|
+
if (currentWorkerSocket === ws) currentWorkerSocket = null;
|
|
6709
|
+
workerRecoveryId = "reconnecting";
|
|
6393
6710
|
const reason = event.reason ? `: ${event.reason}` : "";
|
|
6394
6711
|
process.stderr.write(`[r5d-worker] disconnected (${event.code}${reason})
|
|
6395
6712
|
`);
|
|
@@ -6399,50 +6716,19 @@ async function startWorker(options) {
|
|
|
6399
6716
|
if (shutdownAfterClose) {
|
|
6400
6717
|
process.exit(0);
|
|
6401
6718
|
}
|
|
6402
|
-
|
|
6403
|
-
|
|
6404
|
-
|
|
6405
|
-
if (!startupRecoveryComplete) {
|
|
6406
|
-
process.exit(1);
|
|
6407
|
-
return;
|
|
6408
|
-
}
|
|
6409
|
-
const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
|
|
6410
|
-
process.stderr.write(`[r5d-worker] workspace recovery complete; reconnecting in ${reconnectDelayMs}ms
|
|
6719
|
+
const reconnect = () => {
|
|
6720
|
+
const delay = (0, import_supervisor.workerReconnectDelayMs)(Math.random, workerReconnectAttempt++);
|
|
6721
|
+
process.stderr.write(`[r5d-worker] reconnecting in ${delay}ms
|
|
6411
6722
|
`);
|
|
6412
|
-
|
|
6413
|
-
|
|
6414
|
-
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6415
|
-
`);
|
|
6416
|
-
process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6417
|
-
});
|
|
6418
|
-
}, reconnectDelayMs);
|
|
6419
|
-
});
|
|
6420
|
-
return;
|
|
6421
|
-
}
|
|
6422
|
-
if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
|
|
6423
|
-
const scheduleReconnect = (reconnect2) => {
|
|
6424
|
-
const reconnectDelayMs = (0, import_supervisor.workerReconnectDelayMs)();
|
|
6425
|
-
process.stderr.write(
|
|
6426
|
-
`[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${reconnectDelayMs}ms
|
|
6427
|
-
`
|
|
6428
|
-
);
|
|
6429
|
-
setTimeout(reconnect2, reconnectDelayMs);
|
|
6430
|
-
};
|
|
6431
|
-
const reconnect = () => {
|
|
6432
|
-
void startWorker(options).catch((error) => {
|
|
6723
|
+
setTimeout(() => {
|
|
6724
|
+
void startupRecoveryPromise.then(() => startWorker(options, projectRuntime)).catch((error) => {
|
|
6433
6725
|
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6434
6726
|
`);
|
|
6435
|
-
|
|
6436
|
-
scheduleReconnect(reconnect);
|
|
6437
|
-
return;
|
|
6438
|
-
}
|
|
6439
|
-
process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6727
|
+
reconnect();
|
|
6440
6728
|
});
|
|
6441
|
-
};
|
|
6442
|
-
|
|
6443
|
-
|
|
6444
|
-
}
|
|
6445
|
-
process.exit(import_supervisor.WORKER_RECONNECT_EXIT_CODE);
|
|
6729
|
+
}, delay);
|
|
6730
|
+
};
|
|
6731
|
+
reconnect();
|
|
6446
6732
|
});
|
|
6447
6733
|
ws.addEventListener("error", (event) => {
|
|
6448
6734
|
const message = "message" in event && typeof event.message === "string" && event.message.trim() ? `: ${event.message.trim()}` : "";
|