@ricsam/r5d-worker 0.0.57 → 0.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/main.cjs +717 -140
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/workspace-convergence.cjs +320 -0
- package/dist/cjs/workspace-incident-state.cjs +32 -3
- package/dist/cjs/workspace-manifest-admission.cjs +70 -0
- package/dist/cjs/workspace-sync.cjs +259 -69
- package/dist/mjs/main.mjs +736 -143
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/workspace-convergence.mjs +275 -0
- package/dist/mjs/workspace-incident-state.mjs +31 -3
- package/dist/mjs/workspace-manifest-admission.mjs +35 -0
- package/dist/mjs/workspace-sync.mjs +256 -68
- package/dist/types/workspace-convergence.d.ts +102 -0
- package/dist/types/workspace-incident-state.d.ts +15 -1
- package/dist/types/workspace-manifest-admission.d.ts +27 -0
- package/dist/types/workspace-sync.d.ts +31 -2
- package/package.json +1 -1
package/dist/cjs/main.cjs
CHANGED
|
@@ -65,6 +65,8 @@ var import_process_tree = require("./process-tree.cjs");
|
|
|
65
65
|
var import_port_forward_client = require("./port-forward-client.cjs");
|
|
66
66
|
var import_workspace_incident_state = require("./workspace-incident-state.cjs");
|
|
67
67
|
var import_workspace_command_sync_policy = require("./workspace-command-sync-policy.cjs");
|
|
68
|
+
var import_workspace_convergence = require("./workspace-convergence.cjs");
|
|
69
|
+
var import_workspace_manifest_admission = require("./workspace-manifest-admission.cjs");
|
|
68
70
|
var import_supervisor = require("./supervisor.cjs");
|
|
69
71
|
var import_managed_paths = require("./managed-paths.cjs");
|
|
70
72
|
var import_workspace_sync = require("./workspace-sync.cjs");
|
|
@@ -84,6 +86,7 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
|
|
|
84
86
|
const activePtys = /* @__PURE__ */ new Map();
|
|
85
87
|
let currentWorkerSocket = null;
|
|
86
88
|
const workspaceSyncSingleFlight = new import_workspace_sync.WorkspaceSyncSingleFlight();
|
|
89
|
+
const workspaceFilesystemWriters = (0, import_workspace_convergence.processWorkspaceFilesystemWriterGate)();
|
|
87
90
|
let githubCredential = null;
|
|
88
91
|
let visibleGitIdentity = null;
|
|
89
92
|
function defaultConfigPath() {
|
|
@@ -2298,6 +2301,7 @@ async function openPty(input) {
|
|
|
2298
2301
|
onExit: (event) => {
|
|
2299
2302
|
input.releaseWorkspaceMutation?.();
|
|
2300
2303
|
activePtys.delete(input.message.ptyId);
|
|
2304
|
+
input.onTerminal?.();
|
|
2301
2305
|
sendWorkerMessage(input.ws, {
|
|
2302
2306
|
type: "pty_exit",
|
|
2303
2307
|
ptyId: input.message.ptyId,
|
|
@@ -2308,6 +2312,7 @@ async function openPty(input) {
|
|
|
2308
2312
|
onError: (error) => {
|
|
2309
2313
|
input.releaseWorkspaceMutation?.();
|
|
2310
2314
|
activePtys.delete(input.message.ptyId);
|
|
2315
|
+
input.onTerminal?.();
|
|
2311
2316
|
sendWorkerMessage(input.ws, {
|
|
2312
2317
|
type: "pty_error",
|
|
2313
2318
|
requestId: input.message.requestId,
|
|
@@ -2399,14 +2404,25 @@ async function startWorker(options) {
|
|
|
2399
2404
|
const pendingManifestCheckouts = /* @__PURE__ */ new Map();
|
|
2400
2405
|
let workspaceRemoteUrl = null;
|
|
2401
2406
|
let activeWorkspaceIncidentId = null;
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2407
|
+
const workspaceIncidentOrdering = new import_workspace_incident_state.WorkspaceIncidentOrderingFence();
|
|
2408
|
+
const workspaceConvergence = (0, import_workspace_convergence.processWorkspaceConvergenceState)(import_node_path.default.join(syncRoot, "workspace-convergence.json"));
|
|
2409
|
+
const workspaceManifestRevision = new import_workspace_convergence.WorkspaceManifestRevisionFence();
|
|
2410
|
+
let workspaceReady = false;
|
|
2411
|
+
let workspaceCheckpointTimer;
|
|
2412
|
+
let workspaceSafetyScan;
|
|
2413
|
+
let workspaceManifestHydrationTimer;
|
|
2414
|
+
let workspaceManifestHydrationInFlight = false;
|
|
2415
|
+
let workspaceSafetyScanInFlight = false;
|
|
2416
|
+
let workspaceHeadConvergenceInFlight = false;
|
|
2417
|
+
let pendingWorkspaceHead;
|
|
2418
|
+
let pendingCheckpoint;
|
|
2419
|
+
const explicitWorkspaceCheckpoints = new import_workspace_convergence.ExplicitWorkspaceCheckpointQueue();
|
|
2405
2420
|
let terminalReplayTimer;
|
|
2406
2421
|
let workspaceSyncRequestsInFlight = 0;
|
|
2407
2422
|
let sessionArtifactSyncRequestsInFlight = 0;
|
|
2408
2423
|
let cliUpdateInProgress = false;
|
|
2409
2424
|
let reloadAfterClose = false;
|
|
2425
|
+
let shutdownAfterClose = false;
|
|
2410
2426
|
const workspaceSyncInput = (trigger, overrides = {}) => {
|
|
2411
2427
|
if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
|
|
2412
2428
|
const projects = (0, import_workspace_sync.workspaceProjectsForSync)([...manifestByProjectId.values()], trigger);
|
|
@@ -2432,21 +2448,175 @@ async function startWorker(options) {
|
|
|
2432
2448
|
);
|
|
2433
2449
|
}
|
|
2434
2450
|
};
|
|
2435
|
-
const
|
|
2451
|
+
const sendWorkspaceReady = () => {
|
|
2452
|
+
const manifestRevision = workspaceManifestRevision.readyRevision();
|
|
2453
|
+
if (!workspaceReady || !manifestRevision) return;
|
|
2436
2454
|
try {
|
|
2437
2455
|
ws.send(
|
|
2438
2456
|
JSON.stringify({
|
|
2439
|
-
type: "
|
|
2440
|
-
|
|
2457
|
+
type: "workspace_ready",
|
|
2458
|
+
manifestRevision,
|
|
2459
|
+
pendingCheckouts: [...pendingManifestCheckouts.values()].sort(
|
|
2460
|
+
(left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
|
|
2461
|
+
),
|
|
2462
|
+
...workspaceConvergence.snapshot()
|
|
2463
|
+
})
|
|
2464
|
+
);
|
|
2465
|
+
} catch (error) {
|
|
2466
|
+
process.stderr.write(
|
|
2467
|
+
`[r5d-worker] failed to send workspace readiness: ${error instanceof Error ? error.message : String(error)}
|
|
2468
|
+
`
|
|
2469
|
+
);
|
|
2470
|
+
}
|
|
2471
|
+
};
|
|
2472
|
+
const workerHasActiveWriter = () => activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
|
|
2473
|
+
const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
|
|
2474
|
+
const armWorkspaceCheckpoint = (trigger, options2 = { delayMs: import_workspace_convergence.WORKSPACE_CHECKPOINT_QUIET_MS }) => {
|
|
2475
|
+
workspaceConvergence.schedule();
|
|
2476
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2477
|
+
sendWorkspaceReady();
|
|
2478
|
+
if (options2.requestId && !explicitWorkspaceCheckpoints.schedule({ requestId: options2.requestId, trigger }, pendingCheckpoint?.requestId)) {
|
|
2479
|
+
return;
|
|
2480
|
+
}
|
|
2481
|
+
if (!options2.requestId && explicitWorkspaceCheckpoints.hasScheduled()) return;
|
|
2482
|
+
if (workspaceCheckpointTimer) clearTimeout(workspaceCheckpointTimer);
|
|
2483
|
+
workspaceCheckpointTimer = setTimeout(() => {
|
|
2484
|
+
workspaceCheckpointTimer = void 0;
|
|
2485
|
+
if (options2.requestId) explicitWorkspaceCheckpoints.consumeScheduled(options2.requestId);
|
|
2486
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2487
|
+
if (!workspaceReady || !workspaceRemoteUrl) {
|
|
2488
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2489
|
+
return;
|
|
2490
|
+
}
|
|
2491
|
+
if (activeWorkspaceIncidentId) {
|
|
2492
|
+
workspaceConvergence.block();
|
|
2493
|
+
sendWorkspaceReady();
|
|
2494
|
+
if ((0, import_workspace_convergence.workspaceCheckpointIncidentAction)(options2.requestId) === "defer_automatic") return;
|
|
2495
|
+
if (pendingCheckpoint) {
|
|
2496
|
+
explicitWorkspaceCheckpoints.enqueue({ requestId: options2.requestId, trigger }, pendingCheckpoint.requestId);
|
|
2497
|
+
return;
|
|
2498
|
+
}
|
|
2499
|
+
const snapshot2 = workspaceConvergence.snapshot();
|
|
2500
|
+
pendingCheckpoint = {
|
|
2501
|
+
requestId: options2.requestId,
|
|
2441
2502
|
trigger,
|
|
2442
|
-
|
|
2503
|
+
dirtyGeneration: snapshot2.dirtyGeneration,
|
|
2504
|
+
explicit: true,
|
|
2505
|
+
admissionHeld: false,
|
|
2506
|
+
workInFlight: false,
|
|
2507
|
+
requestedAt: Date.now(),
|
|
2508
|
+
releaseCheckpointAdmission: () => {
|
|
2509
|
+
}
|
|
2510
|
+
};
|
|
2511
|
+
try {
|
|
2512
|
+
ws.send(
|
|
2513
|
+
JSON.stringify({
|
|
2514
|
+
type: "workspace_checkpoint_request",
|
|
2515
|
+
requestId: options2.requestId,
|
|
2516
|
+
trigger,
|
|
2517
|
+
dirtyGeneration: snapshot2.dirtyGeneration,
|
|
2518
|
+
localHead: snapshot2.localHead,
|
|
2519
|
+
desiredCanonicalHead: snapshot2.desiredCanonicalHead
|
|
2520
|
+
})
|
|
2521
|
+
);
|
|
2522
|
+
} catch (error) {
|
|
2523
|
+
pendingCheckpoint = void 0;
|
|
2524
|
+
workspaceConvergence.defer();
|
|
2525
|
+
process.stderr.write(
|
|
2526
|
+
`[r5d-worker] failed to request blocked workspace checkpoint authority: ${error instanceof Error ? error.message : String(error)}
|
|
2527
|
+
`
|
|
2528
|
+
);
|
|
2529
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, requestId: options2.requestId });
|
|
2530
|
+
}
|
|
2531
|
+
return;
|
|
2532
|
+
}
|
|
2533
|
+
if (pendingCheckpoint || workerHasActiveWriter()) {
|
|
2534
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2535
|
+
return;
|
|
2536
|
+
}
|
|
2537
|
+
const releaseCheckpointAdmission = workspaceFilesystemWriters.tryAcquireCheckpoint();
|
|
2538
|
+
if (!releaseCheckpointAdmission) {
|
|
2539
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2540
|
+
return;
|
|
2541
|
+
}
|
|
2542
|
+
const snapshot = workspaceConvergence.snapshot();
|
|
2543
|
+
const requestId = options2.requestId ?? crypto.randomUUID();
|
|
2544
|
+
pendingCheckpoint = {
|
|
2545
|
+
requestId,
|
|
2546
|
+
trigger,
|
|
2547
|
+
dirtyGeneration: snapshot.dirtyGeneration,
|
|
2548
|
+
explicit: options2.requestId !== void 0,
|
|
2549
|
+
admissionHeld: true,
|
|
2550
|
+
workInFlight: false,
|
|
2551
|
+
requestedAt: Date.now(),
|
|
2552
|
+
releaseCheckpointAdmission
|
|
2553
|
+
};
|
|
2554
|
+
try {
|
|
2555
|
+
ws.send(
|
|
2556
|
+
JSON.stringify({
|
|
2557
|
+
type: "workspace_checkpoint_request",
|
|
2558
|
+
requestId,
|
|
2559
|
+
trigger,
|
|
2560
|
+
dirtyGeneration: snapshot.dirtyGeneration,
|
|
2561
|
+
localHead: snapshot.localHead,
|
|
2562
|
+
desiredCanonicalHead: snapshot.desiredCanonicalHead
|
|
2563
|
+
})
|
|
2564
|
+
);
|
|
2565
|
+
} catch (error) {
|
|
2566
|
+
releaseCheckpointAdmission();
|
|
2567
|
+
pendingCheckpoint = void 0;
|
|
2568
|
+
workspaceConvergence.defer();
|
|
2569
|
+
process.stderr.write(
|
|
2570
|
+
`[r5d-worker] failed to request workspace checkpoint: ${error instanceof Error ? error.message : String(error)}
|
|
2571
|
+
`
|
|
2572
|
+
);
|
|
2573
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2574
|
+
}
|
|
2575
|
+
}, Math.max(0, options2.delayMs));
|
|
2576
|
+
workspaceCheckpointTimer.unref();
|
|
2577
|
+
};
|
|
2578
|
+
const markWorkspaceDirty = (trigger, force = false) => {
|
|
2579
|
+
workspaceConvergence.markDirty();
|
|
2580
|
+
armWorkspaceCheckpoint(trigger, { delayMs: force ? 0 : import_workspace_convergence.WORKSPACE_CHECKPOINT_QUIET_MS });
|
|
2581
|
+
};
|
|
2582
|
+
const convergeAvailableWorkspaceHead = async () => {
|
|
2583
|
+
if (pendingWorkspaceHead === void 0 || !workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || workspaceHeadConvergenceInFlight) {
|
|
2584
|
+
return;
|
|
2585
|
+
}
|
|
2586
|
+
const advertisedHead = pendingWorkspaceHead;
|
|
2587
|
+
pendingWorkspaceHead = void 0;
|
|
2588
|
+
workspaceConvergence.observeCanonicalHead(advertisedHead);
|
|
2589
|
+
workspaceHeadConvergenceInFlight = true;
|
|
2590
|
+
let releaseVisibleHydration = null;
|
|
2591
|
+
try {
|
|
2592
|
+
const snapshot = workspaceConvergence.snapshot();
|
|
2593
|
+
const mayHydrateVisible = !workerHasActiveWriter() && !pendingCheckpoint && snapshot.dirtyGeneration === snapshot.publishedGeneration;
|
|
2594
|
+
releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
|
|
2595
|
+
const hydrateVisible = releaseVisibleHydration !== null;
|
|
2596
|
+
const converged = await workspaceSyncSingleFlight.runExclusive(
|
|
2597
|
+
() => (0, import_workspace_sync.convergeWorkspaceHead)(workspaceSyncInput({ type: "inbound_head", detail: advertisedHead ?? "unborn" }), {
|
|
2598
|
+
hydrateVisible
|
|
2443
2599
|
})
|
|
2444
2600
|
);
|
|
2601
|
+
workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
|
|
2602
|
+
if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
|
|
2603
|
+
if (hydrateVisible && converged.localChanges) {
|
|
2604
|
+
markWorkspaceDirty({ type: "periodic", detail: "inbound overlap retained for next checkpoint" });
|
|
2605
|
+
}
|
|
2606
|
+
sendWorkspaceReady();
|
|
2445
2607
|
} catch (error) {
|
|
2608
|
+
pendingWorkspaceHead = advertisedHead;
|
|
2446
2609
|
process.stderr.write(
|
|
2447
|
-
`[r5d-worker] failed to
|
|
2610
|
+
`[r5d-worker] failed to fetch available workspace head: ${error instanceof Error ? error.message : String(error)}
|
|
2448
2611
|
`
|
|
2449
2612
|
);
|
|
2613
|
+
} finally {
|
|
2614
|
+
releaseVisibleHydration?.();
|
|
2615
|
+
workspaceHeadConvergenceInFlight = false;
|
|
2616
|
+
heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2617
|
+
if (pendingWorkspaceHead !== void 0) {
|
|
2618
|
+
setTimeout(() => void convergeAvailableWorkspaceHead(), 1e3).unref();
|
|
2619
|
+
}
|
|
2450
2620
|
}
|
|
2451
2621
|
};
|
|
2452
2622
|
const manifestCheckoutKey = (projectId, branchName) => `${projectId}\0${branchName}`;
|
|
@@ -2498,19 +2668,73 @@ async function startWorker(options) {
|
|
|
2498
2668
|
}
|
|
2499
2669
|
return created;
|
|
2500
2670
|
};
|
|
2671
|
+
const schedulePendingManifestCheckoutHydration = (delayMs = 0) => {
|
|
2672
|
+
if (workspaceManifestHydrationTimer || workspaceManifestHydrationInFlight || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2673
|
+
return;
|
|
2674
|
+
}
|
|
2675
|
+
workspaceManifestHydrationTimer = setTimeout(() => {
|
|
2676
|
+
workspaceManifestHydrationTimer = void 0;
|
|
2677
|
+
if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2678
|
+
if (pendingManifestCheckouts.size > 0 && currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
|
|
2679
|
+
schedulePendingManifestCheckoutHydration(1e3);
|
|
2680
|
+
}
|
|
2681
|
+
return;
|
|
2682
|
+
}
|
|
2683
|
+
const releaseHydration = workspaceFilesystemWriters.tryAcquireCheckpoint();
|
|
2684
|
+
if (!releaseHydration) {
|
|
2685
|
+
schedulePendingManifestCheckoutHydration(1e3);
|
|
2686
|
+
return;
|
|
2687
|
+
}
|
|
2688
|
+
workspaceManifestHydrationInFlight = true;
|
|
2689
|
+
void workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2690
|
+
if (!workspaceReady || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2691
|
+
return;
|
|
2692
|
+
}
|
|
2693
|
+
const targets = [...pendingManifestCheckouts.values()];
|
|
2694
|
+
ensureVisibleWorkspaceCheckouts(targets, { strictCanonicalRevalidation: true });
|
|
2695
|
+
for (const target of targets) {
|
|
2696
|
+
const manifest = manifestByProjectId.get(target.projectId);
|
|
2697
|
+
if (!manifest || !manifest.branches.includes(target.branchName)) continue;
|
|
2698
|
+
const branchPath = import_node_path.default.join(projectRootFor(projectsRoot, target.projectId, manifestByProjectId), target.branchName);
|
|
2699
|
+
if (hasNormalVisibleGitDir(branchPath)) {
|
|
2700
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
}).then(() => sendWorkspaceReady()).catch((error) => {
|
|
2704
|
+
process.stderr.write(
|
|
2705
|
+
`[r5d-worker] pending manifest checkout hydration failed: ${error instanceof Error ? error.message : String(error)}
|
|
2706
|
+
`
|
|
2707
|
+
);
|
|
2708
|
+
}).finally(() => {
|
|
2709
|
+
releaseHydration();
|
|
2710
|
+
workspaceManifestHydrationInFlight = false;
|
|
2711
|
+
heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2712
|
+
if (pendingManifestCheckouts.size > 0) schedulePendingManifestCheckoutHydration(1e3);
|
|
2713
|
+
});
|
|
2714
|
+
}, Math.max(0, delayMs));
|
|
2715
|
+
workspaceManifestHydrationTimer.unref();
|
|
2716
|
+
};
|
|
2501
2717
|
const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}) => {
|
|
2718
|
+
const requestedAt = Date.now();
|
|
2719
|
+
let queueEnteredAt = requestedAt;
|
|
2720
|
+
let prepareStartedAt = requestedAt;
|
|
2721
|
+
let synchronizeStartedAt = requestedAt;
|
|
2722
|
+
let synchronizeFinishedAt = requestedAt;
|
|
2502
2723
|
workspaceSyncRequestsInFlight += 1;
|
|
2503
2724
|
try {
|
|
2504
2725
|
let pendingTargets = [];
|
|
2505
|
-
const result = await workspaceSyncSingleFlight.
|
|
2726
|
+
const result = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2727
|
+
queueEnteredAt = Date.now();
|
|
2506
2728
|
const shouldEnsureCheckouts = !overrides.skipVisibleMirror && !overrides.resetToCanonical;
|
|
2507
2729
|
const syncProjects = (0, import_workspace_sync.workspaceProjectsForSync)([...manifestByProjectId.values()], trigger);
|
|
2730
|
+
if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
|
|
2508
2731
|
const allowedCheckoutKeys = new Set(
|
|
2509
2732
|
syncProjects.flatMap((project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName)))
|
|
2510
2733
|
);
|
|
2511
2734
|
const scopedPendingTargets = [...pendingManifestCheckouts.values()].filter(
|
|
2512
2735
|
(target) => allowedCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
|
|
2513
2736
|
);
|
|
2737
|
+
prepareStartedAt = Date.now();
|
|
2514
2738
|
let checkoutTargets = [];
|
|
2515
2739
|
if (shouldEnsureCheckouts) {
|
|
2516
2740
|
checkoutTargets = workspaceSyncCheckoutTargets({
|
|
@@ -2530,21 +2754,39 @@ async function startWorker(options) {
|
|
|
2530
2754
|
if (allowedCheckoutKeys.has(key)) newVisibleCheckoutByKey.set(key, target);
|
|
2531
2755
|
}
|
|
2532
2756
|
const newVisibleCheckouts = [...newVisibleCheckoutByKey.values()];
|
|
2533
|
-
|
|
2757
|
+
const input = workspaceSyncInput(trigger, {
|
|
2534
2758
|
...overrides,
|
|
2535
2759
|
...newVisibleCheckouts.length > 0 ? { newVisibleCheckouts } : {}
|
|
2536
2760
|
});
|
|
2761
|
+
synchronizeStartedAt = Date.now();
|
|
2762
|
+
let synchronized;
|
|
2763
|
+
try {
|
|
2764
|
+
synchronized = await (0, import_workspace_sync.synchronizeWorkspace)(input);
|
|
2765
|
+
} finally {
|
|
2766
|
+
synchronizeFinishedAt = Date.now();
|
|
2767
|
+
}
|
|
2768
|
+
return synchronized;
|
|
2537
2769
|
});
|
|
2538
|
-
|
|
2770
|
+
const completedAt = Date.now();
|
|
2771
|
+
const resultWithTelemetry = {
|
|
2772
|
+
...result,
|
|
2773
|
+
telemetry: {
|
|
2774
|
+
totalMs: completedAt - requestedAt,
|
|
2775
|
+
queueMs: queueEnteredAt - requestedAt,
|
|
2776
|
+
prepareMs: synchronizeStartedAt - prepareStartedAt,
|
|
2777
|
+
synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
|
|
2778
|
+
}
|
|
2779
|
+
};
|
|
2780
|
+
if (resultWithTelemetry.outcome === "no_change" || resultWithTelemetry.outcome === "published" || resultWithTelemetry.outcome === "updated" || resultWithTelemetry.outcome === "conflict_reset") {
|
|
2539
2781
|
for (const target of pendingTargets) {
|
|
2540
2782
|
const key = manifestCheckoutKey(target.projectId, target.branchName);
|
|
2541
2783
|
if (pendingManifestCheckouts.get(key) === target) pendingManifestCheckouts.delete(key);
|
|
2542
2784
|
}
|
|
2543
2785
|
}
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
return result;
|
|
2786
|
+
sendWorkspaceSyncResult(authority, resultWithTelemetry);
|
|
2787
|
+
return resultWithTelemetry;
|
|
2547
2788
|
} catch (error) {
|
|
2789
|
+
const completedAt = Date.now();
|
|
2548
2790
|
const result = {
|
|
2549
2791
|
type: "workspace_sync",
|
|
2550
2792
|
attemptId: overrides.attemptId ?? crypto.randomUUID(),
|
|
@@ -2559,9 +2801,14 @@ async function startWorker(options) {
|
|
|
2559
2801
|
affectedPaths: [],
|
|
2560
2802
|
discardedPaths: [],
|
|
2561
2803
|
localChangesDiscarded: false,
|
|
2804
|
+
telemetry: {
|
|
2805
|
+
totalMs: completedAt - requestedAt,
|
|
2806
|
+
queueMs: queueEnteredAt - requestedAt,
|
|
2807
|
+
prepareMs: Math.max(0, synchronizeStartedAt - prepareStartedAt),
|
|
2808
|
+
synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
|
|
2809
|
+
},
|
|
2562
2810
|
error: error instanceof Error ? error.message : String(error)
|
|
2563
2811
|
};
|
|
2564
|
-
previousPeriodicFingerprint = null;
|
|
2565
2812
|
sendWorkspaceSyncResult(authority, result);
|
|
2566
2813
|
return result;
|
|
2567
2814
|
} finally {
|
|
@@ -2577,6 +2824,24 @@ async function startWorker(options) {
|
|
|
2577
2824
|
let heartbeatBusyGrace = null;
|
|
2578
2825
|
let heartbeatTimedOut = false;
|
|
2579
2826
|
let heartbeatWatchdog;
|
|
2827
|
+
const handleGracefulShutdown = () => {
|
|
2828
|
+
if (shutdownAfterClose) return;
|
|
2829
|
+
shutdownAfterClose = true;
|
|
2830
|
+
void (0, import_workspace_convergence.waitForWorkspaceCheckpointOnShutdown)({
|
|
2831
|
+
snapshot: () => workspaceConvergence.snapshot(),
|
|
2832
|
+
forceCheckpoint: () => armWorkspaceCheckpoint({ type: "manual", detail: "graceful worker shutdown" }, { delayMs: 0 })
|
|
2833
|
+
}).then((outcome) => {
|
|
2834
|
+
process.stdout.write(`[r5d-worker] graceful workspace checkpoint ${outcome}
|
|
2835
|
+
`);
|
|
2836
|
+
if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
|
|
2837
|
+
ws.close(1e3, "Worker shutting down");
|
|
2838
|
+
} else {
|
|
2839
|
+
process.exit(0);
|
|
2840
|
+
}
|
|
2841
|
+
});
|
|
2842
|
+
};
|
|
2843
|
+
process.on("SIGTERM", handleGracefulShutdown);
|
|
2844
|
+
process.on("SIGINT", handleGracefulShutdown);
|
|
2580
2845
|
const stopHeartbeatWatchdog = () => {
|
|
2581
2846
|
if (heartbeatWatchdog) {
|
|
2582
2847
|
clearInterval(heartbeatWatchdog);
|
|
@@ -2607,7 +2872,8 @@ async function startWorker(options) {
|
|
|
2607
2872
|
updateClis: true,
|
|
2608
2873
|
canonicalResolverCheckout: true,
|
|
2609
2874
|
browserPortForwarding: true,
|
|
2610
|
-
globalWorkspaceSyncLeaseV1: true
|
|
2875
|
+
globalWorkspaceSyncLeaseV1: true,
|
|
2876
|
+
eventualWorkspaceSyncV1: true
|
|
2611
2877
|
},
|
|
2612
2878
|
projectRoot: projectsRoot,
|
|
2613
2879
|
artifactRoot,
|
|
@@ -2634,79 +2900,123 @@ async function startWorker(options) {
|
|
|
2634
2900
|
return;
|
|
2635
2901
|
}
|
|
2636
2902
|
if (message.type === "workspace_manifest") {
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
);
|
|
2644
|
-
const migratedProjectIds = (0, import_managed_paths.migrateLegacyProjectRoots)(projectsRoot, message.projects);
|
|
2645
|
-
manifestByProjectId.clear();
|
|
2646
|
-
for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
|
|
2647
|
-
const currentCheckouts = allManifestCheckouts();
|
|
2648
|
-
const currentCheckoutKeys = new Set(
|
|
2649
|
-
currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2650
|
-
);
|
|
2651
|
-
for (const key of pendingManifestCheckouts.keys()) {
|
|
2652
|
-
if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
|
|
2653
|
-
}
|
|
2654
|
-
for (const checkout of currentCheckouts) {
|
|
2655
|
-
const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
|
|
2656
|
-
if (previousCheckoutKeys.has(key)) continue;
|
|
2657
|
-
const branchPath = import_node_path.default.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
|
|
2658
|
-
if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
|
|
2659
|
-
}
|
|
2660
|
-
previousPeriodicFingerprint = null;
|
|
2661
|
-
process.stdout.write(
|
|
2662
|
-
`[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
|
|
2663
|
-
`
|
|
2664
|
-
);
|
|
2903
|
+
const revisionToken = workspaceManifestRevision.begin(message.revision);
|
|
2904
|
+
workspaceReady = false;
|
|
2905
|
+
const manifestAdmission = (0, import_workspace_manifest_admission.inspectWorkspaceManifestAdmission)({
|
|
2906
|
+
projectsRoot,
|
|
2907
|
+
workspaceShadowRoot,
|
|
2908
|
+
projects: message.projects
|
|
2665
2909
|
});
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2910
|
+
const releaseManifestHydration = await (0, import_workspace_manifest_admission.acquireWorkspaceManifestVisibleLease)(
|
|
2911
|
+
workspaceFilesystemWriters,
|
|
2912
|
+
manifestAdmission
|
|
2913
|
+
);
|
|
2914
|
+
let completed = false;
|
|
2915
|
+
try {
|
|
2916
|
+
completed = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2917
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2918
|
+
const firstBootstrap = manifestAdmission.firstBootstrap;
|
|
2919
|
+
configureGitHubAuth(message.githubCredential);
|
|
2920
|
+
visibleGitIdentity = message.gitIdentity;
|
|
2921
|
+
workspaceRemoteUrl = message.workspaceRemoteUrl;
|
|
2922
|
+
const migratedProjectIds = releaseManifestHydration ? (0, import_managed_paths.migrateLegacyProjectRoots)(projectsRoot, message.projects) : [];
|
|
2923
|
+
manifestByProjectId.clear();
|
|
2924
|
+
for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
|
|
2925
|
+
const currentCheckouts = allManifestCheckouts();
|
|
2926
|
+
const currentCheckoutKeys = new Set(
|
|
2927
|
+
currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2928
|
+
);
|
|
2929
|
+
for (const key of pendingManifestCheckouts.keys()) {
|
|
2930
|
+
if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
|
|
2931
|
+
}
|
|
2932
|
+
for (const checkout of currentCheckouts) {
|
|
2933
|
+
const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
|
|
2934
|
+
const branchPath = import_node_path.default.join(
|
|
2935
|
+
projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId),
|
|
2936
|
+
checkout.branchName
|
|
2679
2937
|
);
|
|
2680
|
-
if (
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2938
|
+
if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
|
|
2939
|
+
}
|
|
2940
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2941
|
+
const created = releaseManifestHydration ? ensureVisibleWorkspaceCheckouts(currentCheckouts, { strictCanonicalRevalidation: true }) : [];
|
|
2942
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2943
|
+
const bootstrapped = await (0, import_workspace_sync.convergeWorkspaceHead)(
|
|
2944
|
+
workspaceSyncInput({ type: "connect" }, { newVisibleCheckouts: created }),
|
|
2945
|
+
{
|
|
2946
|
+
// An existing checkout becomes routable after a fetch. Its
|
|
2947
|
+
// visible tree converges asynchronously; first bootstrap alone
|
|
2948
|
+
// must materialize the canonical snapshot before readiness.
|
|
2949
|
+
// Re-check the incident immediately before convergence so a
|
|
2950
|
+
// concurrently delivered fence forces fetch-only behavior.
|
|
2951
|
+
hydrateVisible: firstBootstrap && !activeWorkspaceIncidentId
|
|
2687
2952
|
}
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2953
|
+
);
|
|
2954
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2955
|
+
workspaceConvergence.setLocalHead(bootstrapped.localHead);
|
|
2956
|
+
workspaceConvergence.observeCanonicalHead(bootstrapped.canonicalHead);
|
|
2957
|
+
for (const checkout of currentCheckouts) {
|
|
2958
|
+
const manifest = manifestByProjectId.get(checkout.projectId);
|
|
2959
|
+
if (!manifest) continue;
|
|
2960
|
+
const branchPath = import_node_path.default.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
|
|
2961
|
+
if (hasNormalVisibleGitDir(branchPath)) {
|
|
2962
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(checkout.projectId, checkout.branchName));
|
|
2692
2963
|
}
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2964
|
+
}
|
|
2965
|
+
process.stdout.write(
|
|
2966
|
+
`[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
|
|
2967
|
+
`
|
|
2968
|
+
);
|
|
2969
|
+
return true;
|
|
2970
|
+
});
|
|
2971
|
+
} finally {
|
|
2972
|
+
releaseManifestHydration?.();
|
|
2973
|
+
}
|
|
2974
|
+
if (!completed || !workspaceManifestRevision.complete(revisionToken)) return;
|
|
2975
|
+
workspaceReady = true;
|
|
2976
|
+
sendWorkspaceReady();
|
|
2977
|
+
schedulePendingManifestCheckoutHydration();
|
|
2978
|
+
if (!workspaceSafetyScan) {
|
|
2979
|
+
workspaceSafetyScan = setInterval(() => {
|
|
2980
|
+
if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || workspaceSafetyScanInFlight) {
|
|
2981
|
+
return;
|
|
2982
|
+
}
|
|
2983
|
+
workspaceSafetyScanInFlight = true;
|
|
2984
|
+
void (async () => {
|
|
2985
|
+
const before = workspaceConvergence.snapshot();
|
|
2986
|
+
const mayHydrateVisible = !workerHasActiveWriter() && before.dirtyGeneration === before.publishedGeneration;
|
|
2987
|
+
const releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
|
|
2988
|
+
const hydrateVisible = releaseVisibleHydration !== null;
|
|
2989
|
+
try {
|
|
2990
|
+
const converged = await workspaceSyncSingleFlight.runExclusive(
|
|
2991
|
+
() => (0, import_workspace_sync.convergeWorkspaceHead)(workspaceSyncInput({ type: "periodic", detail: "idle convergence scan" }), {
|
|
2992
|
+
hydrateVisible
|
|
2993
|
+
})
|
|
2994
|
+
);
|
|
2995
|
+
workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
|
|
2996
|
+
if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
|
|
2997
|
+
if (converged.localChanges && hydrateVisible) {
|
|
2998
|
+
markWorkspaceDirty({ type: "periodic", detail: "idle safety scan" });
|
|
2999
|
+
}
|
|
3000
|
+
sendWorkspaceReady();
|
|
3001
|
+
} finally {
|
|
3002
|
+
releaseVisibleHydration?.();
|
|
2696
3003
|
}
|
|
2697
|
-
previousPeriodicFingerprint = null;
|
|
2698
|
-
requestWorkspaceSync({ type: "periodic" }, fingerprint);
|
|
2699
3004
|
})().catch((error) => {
|
|
2700
3005
|
process.stderr.write(
|
|
2701
|
-
`[r5d-worker]
|
|
3006
|
+
`[r5d-worker] idle workspace safety scan failed: ${error instanceof Error ? error.message : String(error)}
|
|
2702
3007
|
`
|
|
2703
3008
|
);
|
|
2704
3009
|
}).finally(() => {
|
|
2705
|
-
|
|
3010
|
+
workspaceSafetyScanInFlight = false;
|
|
2706
3011
|
heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2707
3012
|
});
|
|
2708
|
-
},
|
|
2709
|
-
|
|
3013
|
+
}, import_workspace_convergence.WORKSPACE_IDLE_SAFETY_SCAN_MS);
|
|
3014
|
+
workspaceSafetyScan.unref();
|
|
3015
|
+
}
|
|
3016
|
+
pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
|
|
3017
|
+
void convergeAvailableWorkspaceHead();
|
|
3018
|
+
if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3019
|
+
armWorkspaceCheckpoint({ type: "periodic", detail: "resume dirty generation after reconnect" }, { delayMs: 0 });
|
|
2710
3020
|
}
|
|
2711
3021
|
return;
|
|
2712
3022
|
}
|
|
@@ -2730,6 +3040,185 @@ async function startWorker(options) {
|
|
|
2730
3040
|
);
|
|
2731
3041
|
return;
|
|
2732
3042
|
}
|
|
3043
|
+
if (message.type === "workspace_head_available") {
|
|
3044
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3045
|
+
pendingWorkspaceHead = message.canonicalHead;
|
|
3046
|
+
sendWorkspaceReady();
|
|
3047
|
+
void convergeAvailableWorkspaceHead();
|
|
3048
|
+
return;
|
|
3049
|
+
}
|
|
3050
|
+
if (message.type === "workspace_checkpoint_now") {
|
|
3051
|
+
armWorkspaceCheckpoint(message.trigger, { delayMs: 0, requestId: message.requestId });
|
|
3052
|
+
return;
|
|
3053
|
+
}
|
|
3054
|
+
if (message.type === "workspace_checkpoint_authority") {
|
|
3055
|
+
const checkpoint = pendingCheckpoint;
|
|
3056
|
+
if (!checkpoint || checkpoint.requestId !== message.requestId) {
|
|
3057
|
+
process.stderr.write(`[r5d-worker] ignoring checkpoint authority for unknown request ${message.requestId}
|
|
3058
|
+
`);
|
|
3059
|
+
return;
|
|
3060
|
+
}
|
|
3061
|
+
if (message.status !== "granted") {
|
|
3062
|
+
checkpoint.releaseCheckpointAdmission();
|
|
3063
|
+
pendingCheckpoint = void 0;
|
|
3064
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3065
|
+
const applyBlockedResponse = message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId);
|
|
3066
|
+
if (applyBlockedResponse) {
|
|
3067
|
+
workspaceConvergence.block();
|
|
3068
|
+
if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
|
|
3069
|
+
} else {
|
|
3070
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3071
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3072
|
+
if (message.status === "busy") {
|
|
3073
|
+
armWorkspaceCheckpoint(checkpoint.trigger, {
|
|
3074
|
+
delayMs: message.retryAfterMs ?? 1e3,
|
|
3075
|
+
requestId: checkpoint.requestId
|
|
3076
|
+
});
|
|
3077
|
+
} else if (!activeWorkspaceIncidentId) {
|
|
3078
|
+
const explicit = explicitWorkspaceCheckpoints.next();
|
|
3079
|
+
if (explicit) {
|
|
3080
|
+
armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
|
|
3081
|
+
} else if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3082
|
+
armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: 0 });
|
|
3083
|
+
}
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
sendWorkspaceReady();
|
|
3087
|
+
return;
|
|
3088
|
+
}
|
|
3089
|
+
checkpoint.attemptId = message.attemptId;
|
|
3090
|
+
checkpoint.authorityToken = message.authorityToken;
|
|
3091
|
+
checkpoint.workInFlight = true;
|
|
3092
|
+
pendingCheckpoint = checkpoint;
|
|
3093
|
+
if (!checkpoint.admissionHeld) {
|
|
3094
|
+
const releaseCheckpointAdmission = await workspaceFilesystemWriters.acquireCheckpoint();
|
|
3095
|
+
if (currentWorkerSocket !== ws || pendingCheckpoint?.requestId !== checkpoint.requestId || pendingCheckpoint.attemptId !== message.attemptId) {
|
|
3096
|
+
releaseCheckpointAdmission();
|
|
3097
|
+
checkpoint.workInFlight = false;
|
|
3098
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3099
|
+
return;
|
|
3100
|
+
}
|
|
3101
|
+
checkpoint.admissionHeld = true;
|
|
3102
|
+
checkpoint.releaseCheckpointAdmission = releaseCheckpointAdmission;
|
|
3103
|
+
}
|
|
3104
|
+
workspaceConvergence.beginPreparing();
|
|
3105
|
+
sendWorkspaceReady();
|
|
3106
|
+
let prepareStartedAt = Date.now();
|
|
3107
|
+
let synchronizeStartedAt = prepareStartedAt;
|
|
3108
|
+
let synchronizeStarted = false;
|
|
3109
|
+
let result;
|
|
3110
|
+
try {
|
|
3111
|
+
result = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
3112
|
+
prepareStartedAt = Date.now();
|
|
3113
|
+
const pendingTargets = [...pendingManifestCheckouts.values()];
|
|
3114
|
+
const createdCheckouts = ensureVisibleWorkspaceCheckouts(pendingTargets, { strictCanonicalRevalidation: true });
|
|
3115
|
+
synchronizeStartedAt = Date.now();
|
|
3116
|
+
synchronizeStarted = true;
|
|
3117
|
+
const prepared = await (0, import_workspace_sync.synchronizeWorkspace)(
|
|
3118
|
+
workspaceSyncInput(checkpoint.trigger, {
|
|
3119
|
+
attemptId: message.attemptId,
|
|
3120
|
+
quarantineRef: message.quarantineRef,
|
|
3121
|
+
expectedCanonicalHead: message.expectedHead,
|
|
3122
|
+
newVisibleCheckouts: [...createdCheckouts, ...pendingTargets]
|
|
3123
|
+
})
|
|
3124
|
+
);
|
|
3125
|
+
for (const target of pendingTargets) {
|
|
3126
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
|
|
3127
|
+
}
|
|
3128
|
+
return prepared.outcome === "updated" ? { ...prepared, outcome: "no_change" } : prepared;
|
|
3129
|
+
});
|
|
3130
|
+
} catch (error) {
|
|
3131
|
+
if (!synchronizeStarted) synchronizeStartedAt = Date.now();
|
|
3132
|
+
result = {
|
|
3133
|
+
type: "workspace_sync",
|
|
3134
|
+
attemptId: message.attemptId,
|
|
3135
|
+
workerLabel: label,
|
|
3136
|
+
trigger: checkpoint.trigger,
|
|
3137
|
+
outcome: "failed",
|
|
3138
|
+
startingHead: message.expectedHead,
|
|
3139
|
+
expectedHead: message.expectedHead,
|
|
3140
|
+
rebaseCount: 0,
|
|
3141
|
+
diffSizeBytes: 0,
|
|
3142
|
+
gitStatus: "",
|
|
3143
|
+
affectedProjects: [],
|
|
3144
|
+
affectedPaths: [],
|
|
3145
|
+
discardedPaths: [],
|
|
3146
|
+
localChangesDiscarded: false,
|
|
3147
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3148
|
+
};
|
|
3149
|
+
}
|
|
3150
|
+
const finishedAt = Date.now();
|
|
3151
|
+
result = {
|
|
3152
|
+
...result,
|
|
3153
|
+
dirtyGeneration: checkpoint.dirtyGeneration,
|
|
3154
|
+
telemetry: (0, import_workspace_convergence.workspaceCheckpointTelemetry)({
|
|
3155
|
+
requestedAt: checkpoint.requestedAt,
|
|
3156
|
+
prepareStartedAt,
|
|
3157
|
+
synchronizeStartedAt,
|
|
3158
|
+
finishedAt
|
|
3159
|
+
})
|
|
3160
|
+
};
|
|
3161
|
+
workspaceConvergence.beginSubmitting();
|
|
3162
|
+
sendWorkspaceReady();
|
|
3163
|
+
try {
|
|
3164
|
+
ws.send(
|
|
3165
|
+
JSON.stringify({
|
|
3166
|
+
type: "workspace_checkpoint_result",
|
|
3167
|
+
requestId: checkpoint.requestId,
|
|
3168
|
+
attemptId: message.attemptId,
|
|
3169
|
+
authorityToken: message.authorityToken,
|
|
3170
|
+
dirtyGeneration: checkpoint.dirtyGeneration,
|
|
3171
|
+
result
|
|
3172
|
+
})
|
|
3173
|
+
);
|
|
3174
|
+
} catch (error) {
|
|
3175
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3176
|
+
throw error;
|
|
3177
|
+
} finally {
|
|
3178
|
+
checkpoint.workInFlight = false;
|
|
3179
|
+
checkpoint.releaseCheckpointAdmission();
|
|
3180
|
+
if (currentWorkerSocket !== ws) workspaceConvergence.resetTransientCheckpointState();
|
|
3181
|
+
}
|
|
3182
|
+
return;
|
|
3183
|
+
}
|
|
3184
|
+
if (message.type === "workspace_checkpoint_completed") {
|
|
3185
|
+
const checkpoint = pendingCheckpoint;
|
|
3186
|
+
if (!checkpoint || checkpoint.requestId !== message.requestId || checkpoint.attemptId !== message.attemptId) {
|
|
3187
|
+
process.stderr.write(`[r5d-worker] ignoring completion for unknown checkpoint ${message.requestId}
|
|
3188
|
+
`);
|
|
3189
|
+
return;
|
|
3190
|
+
}
|
|
3191
|
+
pendingCheckpoint = void 0;
|
|
3192
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3193
|
+
if (message.status === "published" || message.status === "no_change") {
|
|
3194
|
+
workspaceConvergence.complete({
|
|
3195
|
+
generation: Math.min(message.publishedGeneration ?? checkpoint.dirtyGeneration, checkpoint.dirtyGeneration),
|
|
3196
|
+
canonicalHead: message.canonicalHead,
|
|
3197
|
+
published: true
|
|
3198
|
+
});
|
|
3199
|
+
} else if (message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId)) {
|
|
3200
|
+
workspaceConvergence.block();
|
|
3201
|
+
if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
|
|
3202
|
+
} else {
|
|
3203
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3204
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3205
|
+
}
|
|
3206
|
+
sendWorkspaceReady();
|
|
3207
|
+
const explicit = message.status === "retry" && checkpoint.explicit ? { requestId: checkpoint.requestId, trigger: checkpoint.trigger } : explicitWorkspaceCheckpoints.next();
|
|
3208
|
+
if (explicit) {
|
|
3209
|
+
armWorkspaceCheckpoint(explicit.trigger, {
|
|
3210
|
+
delayMs: message.retryAfterMs ?? 0,
|
|
3211
|
+
requestId: explicit.requestId
|
|
3212
|
+
});
|
|
3213
|
+
} else if (message.status === "retry" || message.status === "failed" || workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3214
|
+
armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: message.retryAfterMs ?? 1e3 });
|
|
3215
|
+
}
|
|
3216
|
+
if (message.canonicalHead !== workspaceConvergence.snapshot().localHead) {
|
|
3217
|
+
pendingWorkspaceHead = message.canonicalHead;
|
|
3218
|
+
void convergeAvailableWorkspaceHead();
|
|
3219
|
+
}
|
|
3220
|
+
return;
|
|
3221
|
+
}
|
|
2733
3222
|
if (message.type === "sync_session_artifacts") {
|
|
2734
3223
|
sessionArtifactSyncRequestsInFlight += 1;
|
|
2735
3224
|
try {
|
|
@@ -2757,17 +3246,27 @@ async function startWorker(options) {
|
|
|
2757
3246
|
return;
|
|
2758
3247
|
}
|
|
2759
3248
|
if (message.type === "workspace_incident_updated") {
|
|
3249
|
+
workspaceIncidentOrdering.observe(message);
|
|
2760
3250
|
const previousIncidentId = activeWorkspaceIncidentId;
|
|
2761
3251
|
activeWorkspaceIncidentId = (0, import_workspace_incident_state.applyWorkspaceIncidentUpdate)(activeWorkspaceIncidentId, message);
|
|
2762
|
-
previousPeriodicFingerprint = null;
|
|
2763
3252
|
const releasedHead = (0, import_workspace_incident_state.releasePendingWorkspaceHead)(
|
|
2764
3253
|
previousIncidentId,
|
|
2765
3254
|
activeWorkspaceIncidentId,
|
|
2766
3255
|
message.status === "resolved" || message.status === "confirmed" || message.status === "reset" ? message.incidentId : null
|
|
2767
3256
|
);
|
|
2768
|
-
|
|
2769
|
-
|
|
3257
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3258
|
+
if (releasedHead.fetchAuthoritativeHead) {
|
|
3259
|
+
pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
|
|
3260
|
+
void convergeAvailableWorkspaceHead();
|
|
2770
3261
|
}
|
|
3262
|
+
const explicit = !activeWorkspaceIncidentId ? explicitWorkspaceCheckpoints.next() : void 0;
|
|
3263
|
+
if (explicit) {
|
|
3264
|
+
armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
|
|
3265
|
+
} else if (!activeWorkspaceIncidentId && workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3266
|
+
armWorkspaceCheckpoint({ type: "periodic", detail: "resume after workspace incident" }, { delayMs: 0 });
|
|
3267
|
+
}
|
|
3268
|
+
if (!activeWorkspaceIncidentId) schedulePendingManifestCheckoutHydration();
|
|
3269
|
+
sendWorkspaceReady();
|
|
2771
3270
|
return;
|
|
2772
3271
|
}
|
|
2773
3272
|
if (message.type === "exec_terminal_ack") {
|
|
@@ -2865,8 +3364,10 @@ async function startWorker(options) {
|
|
|
2865
3364
|
});
|
|
2866
3365
|
return;
|
|
2867
3366
|
}
|
|
3367
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2868
3368
|
const releaseWorkspaceMutation = await (0, import_workspace_command_sync_policy.acquireWorkspaceCommandMutation)(message.target, workspaceSyncSingleFlight);
|
|
2869
|
-
let
|
|
3369
|
+
let mutationLeaseTransferred = false;
|
|
3370
|
+
let writerLeaseTransferred = false;
|
|
2870
3371
|
try {
|
|
2871
3372
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
2872
3373
|
process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
|
|
@@ -2876,9 +3377,19 @@ async function startWorker(options) {
|
|
|
2876
3377
|
message,
|
|
2877
3378
|
resolvedTarget,
|
|
2878
3379
|
planRoot,
|
|
2879
|
-
...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {}
|
|
3380
|
+
...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {},
|
|
3381
|
+
...targetMayMutateVisibleWorkspace(message.target) ? {
|
|
3382
|
+
onTerminal: () => {
|
|
3383
|
+
markWorkspaceDirty(
|
|
3384
|
+
{ type: "process_terminal", detail: `pty ${message.ptyId} completed` },
|
|
3385
|
+
true
|
|
3386
|
+
);
|
|
3387
|
+
releaseWorkspaceWriter?.();
|
|
3388
|
+
}
|
|
3389
|
+
} : {}
|
|
2880
3390
|
});
|
|
2881
|
-
|
|
3391
|
+
mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
|
|
3392
|
+
writerLeaseTransferred = releaseWorkspaceWriter !== void 0;
|
|
2882
3393
|
} catch (error) {
|
|
2883
3394
|
sendWorkerMessage(ws, {
|
|
2884
3395
|
type: "pty_error",
|
|
@@ -2887,11 +3398,16 @@ async function startWorker(options) {
|
|
|
2887
3398
|
error: error instanceof Error ? error.message : String(error)
|
|
2888
3399
|
});
|
|
2889
3400
|
} finally {
|
|
2890
|
-
if (!
|
|
3401
|
+
if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
|
|
3402
|
+
if (!writerLeaseTransferred) releaseWorkspaceWriter?.();
|
|
2891
3403
|
}
|
|
2892
3404
|
return;
|
|
2893
3405
|
}
|
|
2894
3406
|
if (message.type === "pty_input") {
|
|
3407
|
+
const activePty = activePtys.get(message.ptyId);
|
|
3408
|
+
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
3409
|
+
markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` });
|
|
3410
|
+
}
|
|
2895
3411
|
writePty(ws, message);
|
|
2896
3412
|
return;
|
|
2897
3413
|
}
|
|
@@ -2900,7 +3416,11 @@ async function startWorker(options) {
|
|
|
2900
3416
|
return;
|
|
2901
3417
|
}
|
|
2902
3418
|
if (message.type === "pty_close") {
|
|
3419
|
+
const activePty = activePtys.get(message.ptyId);
|
|
2903
3420
|
closePty(message);
|
|
3421
|
+
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
3422
|
+
armWorkspaceCheckpoint({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, { delayMs: 0 });
|
|
3423
|
+
}
|
|
2904
3424
|
return;
|
|
2905
3425
|
}
|
|
2906
3426
|
if (message.type === "exec") {
|
|
@@ -2914,26 +3434,37 @@ async function startWorker(options) {
|
|
|
2914
3434
|
});
|
|
2915
3435
|
return;
|
|
2916
3436
|
}
|
|
2917
|
-
|
|
3437
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2918
3438
|
try {
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
3439
|
+
let result;
|
|
3440
|
+
try {
|
|
3441
|
+
const runCommand = async () => {
|
|
3442
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3443
|
+
process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
|
|
2922
3444
|
`);
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
3445
|
+
return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
|
|
3446
|
+
};
|
|
3447
|
+
result = await (0, import_workspace_command_sync_policy.runWorkspaceCommand)(message.target, workspaceSyncSingleFlight, runCommand);
|
|
3448
|
+
} catch (error) {
|
|
3449
|
+
result = {
|
|
3450
|
+
type: "exec_result",
|
|
3451
|
+
requestId: message.requestId,
|
|
3452
|
+
stdout: "",
|
|
3453
|
+
stderr: "",
|
|
3454
|
+
exitCode: 1,
|
|
3455
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3456
|
+
};
|
|
3457
|
+
}
|
|
3458
|
+
ws.send(JSON.stringify(result));
|
|
3459
|
+
if (targetMayMutateVisibleWorkspace(message.target)) {
|
|
3460
|
+
markWorkspaceDirty(
|
|
3461
|
+
{ type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
|
|
3462
|
+
true
|
|
3463
|
+
);
|
|
3464
|
+
}
|
|
3465
|
+
} finally {
|
|
3466
|
+
releaseWorkspaceWriter?.();
|
|
2935
3467
|
}
|
|
2936
|
-
ws.send(JSON.stringify(result));
|
|
2937
3468
|
return;
|
|
2938
3469
|
}
|
|
2939
3470
|
if (message.type === "exec_start") {
|
|
@@ -2952,18 +3483,34 @@ async function startWorker(options) {
|
|
|
2952
3483
|
runId: message.runId
|
|
2953
3484
|
});
|
|
2954
3485
|
const runCommand = async () => {
|
|
2955
|
-
const
|
|
2956
|
-
|
|
3486
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
3487
|
+
try {
|
|
3488
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3489
|
+
process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
|
|
2957
3490
|
`);
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
3491
|
+
await executeStreamingCommand({
|
|
3492
|
+
ws,
|
|
3493
|
+
message,
|
|
3494
|
+
resolvedTarget,
|
|
3495
|
+
baseUrl,
|
|
3496
|
+
token,
|
|
3497
|
+
artifactRoot,
|
|
3498
|
+
planRoot
|
|
3499
|
+
});
|
|
3500
|
+
} finally {
|
|
3501
|
+
if (targetMayMutateVisibleWorkspace(message.target)) {
|
|
3502
|
+
markWorkspaceDirty(
|
|
3503
|
+
{
|
|
3504
|
+
type: "process_terminal",
|
|
3505
|
+
sessionId: message.sessionId,
|
|
3506
|
+
processRunId: message.runId,
|
|
3507
|
+
detail: "process completed"
|
|
3508
|
+
},
|
|
3509
|
+
true
|
|
3510
|
+
);
|
|
3511
|
+
}
|
|
3512
|
+
releaseWorkspaceWriter?.();
|
|
3513
|
+
}
|
|
2967
3514
|
};
|
|
2968
3515
|
const execution = (0, import_workspace_command_sync_policy.runWorkspaceCommand)(message.target, workspaceSyncSingleFlight, runCommand);
|
|
2969
3516
|
void execution.catch((error) => {
|
|
@@ -2977,33 +3524,46 @@ async function startWorker(options) {
|
|
|
2977
3524
|
return;
|
|
2978
3525
|
}
|
|
2979
3526
|
if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes") {
|
|
3527
|
+
const releaseWorkspaceWriter = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2980
3528
|
try {
|
|
2981
|
-
|
|
2982
|
-
const
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
3529
|
+
try {
|
|
3530
|
+
const result = await workspaceSyncSingleFlight.runMutation(async () => {
|
|
3531
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3532
|
+
return await executeOperation({
|
|
3533
|
+
message,
|
|
3534
|
+
resolvedTarget,
|
|
3535
|
+
baseUrl,
|
|
3536
|
+
token,
|
|
3537
|
+
artifactRoot,
|
|
3538
|
+
planRoot
|
|
3539
|
+
});
|
|
2990
3540
|
});
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
}
|
|
3006
|
-
)
|
|
3541
|
+
ws.send(
|
|
3542
|
+
JSON.stringify({
|
|
3543
|
+
type: "operation_result",
|
|
3544
|
+
requestId: message.requestId,
|
|
3545
|
+
result
|
|
3546
|
+
})
|
|
3547
|
+
);
|
|
3548
|
+
if ((message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target)) {
|
|
3549
|
+
markWorkspaceDirty({
|
|
3550
|
+
type: message.type,
|
|
3551
|
+
sessionId: message.sessionId,
|
|
3552
|
+
toolCallId: message.requestId,
|
|
3553
|
+
...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
|
|
3554
|
+
});
|
|
3555
|
+
}
|
|
3556
|
+
} catch (error) {
|
|
3557
|
+
ws.send(
|
|
3558
|
+
JSON.stringify({
|
|
3559
|
+
type: "operation_result",
|
|
3560
|
+
requestId: message.requestId,
|
|
3561
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3562
|
+
})
|
|
3563
|
+
);
|
|
3564
|
+
}
|
|
3565
|
+
} finally {
|
|
3566
|
+
releaseWorkspaceWriter?.();
|
|
3007
3567
|
}
|
|
3008
3568
|
return;
|
|
3009
3569
|
}
|
|
@@ -3025,10 +3585,24 @@ async function startWorker(options) {
|
|
|
3025
3585
|
});
|
|
3026
3586
|
});
|
|
3027
3587
|
ws.addEventListener("close", (event) => {
|
|
3588
|
+
process.off("SIGTERM", handleGracefulShutdown);
|
|
3589
|
+
process.off("SIGINT", handleGracefulShutdown);
|
|
3028
3590
|
stopHeartbeatWatchdog();
|
|
3029
|
-
if (
|
|
3030
|
-
|
|
3031
|
-
|
|
3591
|
+
if (workspaceCheckpointTimer) {
|
|
3592
|
+
clearTimeout(workspaceCheckpointTimer);
|
|
3593
|
+
workspaceCheckpointTimer = void 0;
|
|
3594
|
+
}
|
|
3595
|
+
if (!pendingCheckpoint?.workInFlight) {
|
|
3596
|
+
pendingCheckpoint?.releaseCheckpointAdmission();
|
|
3597
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3598
|
+
}
|
|
3599
|
+
if (workspaceSafetyScan) {
|
|
3600
|
+
clearInterval(workspaceSafetyScan);
|
|
3601
|
+
workspaceSafetyScan = void 0;
|
|
3602
|
+
}
|
|
3603
|
+
if (workspaceManifestHydrationTimer) {
|
|
3604
|
+
clearTimeout(workspaceManifestHydrationTimer);
|
|
3605
|
+
workspaceManifestHydrationTimer = void 0;
|
|
3032
3606
|
}
|
|
3033
3607
|
if (terminalReplayTimer) {
|
|
3034
3608
|
clearInterval(terminalReplayTimer);
|
|
@@ -3044,6 +3618,9 @@ async function startWorker(options) {
|
|
|
3044
3618
|
if (reloadAfterClose) {
|
|
3045
3619
|
process.exit(import_supervisor.WORKER_RELOAD_EXIT_CODE);
|
|
3046
3620
|
}
|
|
3621
|
+
if (shutdownAfterClose) {
|
|
3622
|
+
process.exit(0);
|
|
3623
|
+
}
|
|
3047
3624
|
if (activeProcesses.size > 0 || pendingProcessTerminals.size > 0) {
|
|
3048
3625
|
process.stderr.write(
|
|
3049
3626
|
`[r5d-worker] ${activeProcesses.size} process(es) active and ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${import_supervisor.WORKER_RECONNECT_DELAY_MS}ms
|