@ricsam/r5d-worker 0.0.58 → 0.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mjs/main.mjs CHANGED
@@ -13,13 +13,27 @@ import {
13
13
  } from "./heartbeat.mjs";
14
14
  import { terminateProcessTree } from "./process-tree.mjs";
15
15
  import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
16
- import { applyWorkspaceIncidentUpdate, releasePendingWorkspaceHead } from "./workspace-incident-state.mjs";
16
+ import {
17
+ applyWorkspaceIncidentUpdate,
18
+ releasePendingWorkspaceHead,
19
+ WorkspaceIncidentOrderingFence
20
+ } from "./workspace-incident-state.mjs";
17
21
  import { acquireWorkspaceCommandMutation, runWorkspaceCommand } from "./workspace-command-sync-policy.mjs";
18
22
  import {
19
- RecentWorkspaceSyncProofCache,
20
- readShadowWorkspaceCanonicalHead,
21
- workspaceSyncScopeKey
22
- } from "./workspace-sync-fast-path.mjs";
23
+ WORKSPACE_CHECKPOINT_QUIET_MS,
24
+ WORKSPACE_IDLE_SAFETY_SCAN_MS,
25
+ ExplicitWorkspaceCheckpointQueue,
26
+ WorkspaceManifestRevisionFence,
27
+ processWorkspaceConvergenceState,
28
+ processWorkspaceFilesystemWriterGate,
29
+ waitForWorkspaceCheckpointOnShutdown,
30
+ workspaceCheckpointIncidentAction,
31
+ workspaceCheckpointTelemetry
32
+ } from "./workspace-convergence.mjs";
33
+ import {
34
+ acquireWorkspaceManifestVisibleLease,
35
+ inspectWorkspaceManifestAdmission
36
+ } from "./workspace-manifest-admission.mjs";
23
37
  import {
24
38
  isRetryableWorkerServerStatus,
25
39
  superviseWorkerRuntime,
@@ -30,15 +44,11 @@ import {
30
44
  } from "./supervisor.mjs";
31
45
  import { managedProjectRoot, migrateLegacyProjectRoots, validateManagedBranchName } from "./managed-paths.mjs";
32
46
  import {
33
- WORKSPACE_PERIODIC_SCAN_INTERVAL_MS,
34
47
  WorkspaceSyncSingleFlight,
35
- calculateWorkspaceDiffFingerprint,
48
+ convergeWorkspaceHead,
36
49
  synchronizeWorkspace,
37
50
  workspaceProjectsForSync
38
51
  } from "./workspace-sync.mjs";
39
- function workspaceSyncFastPathTelemetry(decision) {
40
- return decision.hit ? { hit: true, reason: decision.reason, proofAgeMs: decision.proofAgeMs } : { hit: false, reason: decision.reason };
41
- }
42
52
  class WorkerServerUnavailableError extends Error {
43
53
  name = "WorkerServerUnavailableError";
44
54
  }
@@ -54,6 +64,7 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
54
64
  const activePtys = /* @__PURE__ */ new Map();
55
65
  let currentWorkerSocket = null;
56
66
  const workspaceSyncSingleFlight = new WorkspaceSyncSingleFlight();
67
+ const workspaceFilesystemWriters = processWorkspaceFilesystemWriterGate();
57
68
  let githubCredential = null;
58
69
  let visibleGitIdentity = null;
59
70
  function defaultConfigPath() {
@@ -2268,6 +2279,7 @@ async function openPty(input) {
2268
2279
  onExit: (event) => {
2269
2280
  input.releaseWorkspaceMutation?.();
2270
2281
  activePtys.delete(input.message.ptyId);
2282
+ input.onTerminal?.();
2271
2283
  sendWorkerMessage(input.ws, {
2272
2284
  type: "pty_exit",
2273
2285
  ptyId: input.message.ptyId,
@@ -2278,6 +2290,7 @@ async function openPty(input) {
2278
2290
  onError: (error) => {
2279
2291
  input.releaseWorkspaceMutation?.();
2280
2292
  activePtys.delete(input.message.ptyId);
2293
+ input.onTerminal?.();
2281
2294
  sendWorkerMessage(input.ws, {
2282
2295
  type: "pty_error",
2283
2296
  requestId: input.message.requestId,
@@ -2369,15 +2382,25 @@ async function startWorker(options) {
2369
2382
  const pendingManifestCheckouts = /* @__PURE__ */ new Map();
2370
2383
  let workspaceRemoteUrl = null;
2371
2384
  let activeWorkspaceIncidentId = null;
2372
- let previousPeriodicFingerprint = null;
2373
- let periodicWorkspaceScan;
2374
- let periodicWorkspaceScanInFlight = false;
2385
+ const workspaceIncidentOrdering = new WorkspaceIncidentOrderingFence();
2386
+ const workspaceConvergence = processWorkspaceConvergenceState(path.join(syncRoot, "workspace-convergence.json"));
2387
+ const workspaceManifestRevision = new WorkspaceManifestRevisionFence();
2388
+ let workspaceReady = false;
2389
+ let workspaceCheckpointTimer;
2390
+ let workspaceSafetyScan;
2391
+ let workspaceManifestHydrationTimer;
2392
+ let workspaceManifestHydrationInFlight = false;
2393
+ let workspaceSafetyScanInFlight = false;
2394
+ let workspaceHeadConvergenceInFlight = false;
2395
+ let pendingWorkspaceHead;
2396
+ let pendingCheckpoint;
2397
+ const explicitWorkspaceCheckpoints = new ExplicitWorkspaceCheckpointQueue();
2375
2398
  let terminalReplayTimer;
2376
2399
  let workspaceSyncRequestsInFlight = 0;
2377
2400
  let sessionArtifactSyncRequestsInFlight = 0;
2378
2401
  let cliUpdateInProgress = false;
2379
2402
  let reloadAfterClose = false;
2380
- const recentWorkspaceSyncProof = new RecentWorkspaceSyncProofCache();
2403
+ let shutdownAfterClose = false;
2381
2404
  const workspaceSyncInput = (trigger, overrides = {}) => {
2382
2405
  if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
2383
2406
  const projects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
@@ -2403,21 +2426,175 @@ async function startWorker(options) {
2403
2426
  );
2404
2427
  }
2405
2428
  };
2406
- const requestWorkspaceSync = (trigger, fingerprint) => {
2429
+ const sendWorkspaceReady = () => {
2430
+ const manifestRevision = workspaceManifestRevision.readyRevision();
2431
+ if (!workspaceReady || !manifestRevision) return;
2407
2432
  try {
2408
2433
  ws.send(
2409
2434
  JSON.stringify({
2410
- type: "workspace_sync_requested",
2411
- observationId: crypto.randomUUID(),
2435
+ type: "workspace_ready",
2436
+ manifestRevision,
2437
+ pendingCheckouts: [...pendingManifestCheckouts.values()].sort(
2438
+ (left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
2439
+ ),
2440
+ ...workspaceConvergence.snapshot()
2441
+ })
2442
+ );
2443
+ } catch (error) {
2444
+ process.stderr.write(
2445
+ `[r5d-worker] failed to send workspace readiness: ${error instanceof Error ? error.message : String(error)}
2446
+ `
2447
+ );
2448
+ }
2449
+ };
2450
+ const workerHasActiveWriter = () => activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2451
+ const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
2452
+ const armWorkspaceCheckpoint = (trigger, options2 = { delayMs: WORKSPACE_CHECKPOINT_QUIET_MS }) => {
2453
+ workspaceConvergence.schedule();
2454
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2455
+ sendWorkspaceReady();
2456
+ if (options2.requestId && !explicitWorkspaceCheckpoints.schedule({ requestId: options2.requestId, trigger }, pendingCheckpoint?.requestId)) {
2457
+ return;
2458
+ }
2459
+ if (!options2.requestId && explicitWorkspaceCheckpoints.hasScheduled()) return;
2460
+ if (workspaceCheckpointTimer) clearTimeout(workspaceCheckpointTimer);
2461
+ workspaceCheckpointTimer = setTimeout(() => {
2462
+ workspaceCheckpointTimer = void 0;
2463
+ if (options2.requestId) explicitWorkspaceCheckpoints.consumeScheduled(options2.requestId);
2464
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2465
+ if (!workspaceReady || !workspaceRemoteUrl) {
2466
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2467
+ return;
2468
+ }
2469
+ if (activeWorkspaceIncidentId) {
2470
+ workspaceConvergence.block();
2471
+ sendWorkspaceReady();
2472
+ if (workspaceCheckpointIncidentAction(options2.requestId) === "defer_automatic") return;
2473
+ if (pendingCheckpoint) {
2474
+ explicitWorkspaceCheckpoints.enqueue({ requestId: options2.requestId, trigger }, pendingCheckpoint.requestId);
2475
+ return;
2476
+ }
2477
+ const snapshot2 = workspaceConvergence.snapshot();
2478
+ pendingCheckpoint = {
2479
+ requestId: options2.requestId,
2412
2480
  trigger,
2413
- ...fingerprint ? { fingerprint } : {}
2481
+ dirtyGeneration: snapshot2.dirtyGeneration,
2482
+ explicit: true,
2483
+ admissionHeld: false,
2484
+ workInFlight: false,
2485
+ requestedAt: Date.now(),
2486
+ releaseCheckpointAdmission: () => {
2487
+ }
2488
+ };
2489
+ try {
2490
+ ws.send(
2491
+ JSON.stringify({
2492
+ type: "workspace_checkpoint_request",
2493
+ requestId: options2.requestId,
2494
+ trigger,
2495
+ dirtyGeneration: snapshot2.dirtyGeneration,
2496
+ localHead: snapshot2.localHead,
2497
+ desiredCanonicalHead: snapshot2.desiredCanonicalHead
2498
+ })
2499
+ );
2500
+ } catch (error) {
2501
+ pendingCheckpoint = void 0;
2502
+ workspaceConvergence.defer();
2503
+ process.stderr.write(
2504
+ `[r5d-worker] failed to request blocked workspace checkpoint authority: ${error instanceof Error ? error.message : String(error)}
2505
+ `
2506
+ );
2507
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, requestId: options2.requestId });
2508
+ }
2509
+ return;
2510
+ }
2511
+ if (pendingCheckpoint || workerHasActiveWriter()) {
2512
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2513
+ return;
2514
+ }
2515
+ const releaseCheckpointAdmission = workspaceFilesystemWriters.tryAcquireCheckpoint();
2516
+ if (!releaseCheckpointAdmission) {
2517
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2518
+ return;
2519
+ }
2520
+ const snapshot = workspaceConvergence.snapshot();
2521
+ const requestId = options2.requestId ?? crypto.randomUUID();
2522
+ pendingCheckpoint = {
2523
+ requestId,
2524
+ trigger,
2525
+ dirtyGeneration: snapshot.dirtyGeneration,
2526
+ explicit: options2.requestId !== void 0,
2527
+ admissionHeld: true,
2528
+ workInFlight: false,
2529
+ requestedAt: Date.now(),
2530
+ releaseCheckpointAdmission
2531
+ };
2532
+ try {
2533
+ ws.send(
2534
+ JSON.stringify({
2535
+ type: "workspace_checkpoint_request",
2536
+ requestId,
2537
+ trigger,
2538
+ dirtyGeneration: snapshot.dirtyGeneration,
2539
+ localHead: snapshot.localHead,
2540
+ desiredCanonicalHead: snapshot.desiredCanonicalHead
2541
+ })
2542
+ );
2543
+ } catch (error) {
2544
+ releaseCheckpointAdmission();
2545
+ pendingCheckpoint = void 0;
2546
+ workspaceConvergence.defer();
2547
+ process.stderr.write(
2548
+ `[r5d-worker] failed to request workspace checkpoint: ${error instanceof Error ? error.message : String(error)}
2549
+ `
2550
+ );
2551
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2552
+ }
2553
+ }, Math.max(0, options2.delayMs));
2554
+ workspaceCheckpointTimer.unref();
2555
+ };
2556
+ const markWorkspaceDirty = (trigger, force = false) => {
2557
+ workspaceConvergence.markDirty();
2558
+ armWorkspaceCheckpoint(trigger, { delayMs: force ? 0 : WORKSPACE_CHECKPOINT_QUIET_MS });
2559
+ };
2560
+ const convergeAvailableWorkspaceHead = async () => {
2561
+ if (pendingWorkspaceHead === void 0 || !workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || workspaceHeadConvergenceInFlight) {
2562
+ return;
2563
+ }
2564
+ const advertisedHead = pendingWorkspaceHead;
2565
+ pendingWorkspaceHead = void 0;
2566
+ workspaceConvergence.observeCanonicalHead(advertisedHead);
2567
+ workspaceHeadConvergenceInFlight = true;
2568
+ let releaseVisibleHydration = null;
2569
+ try {
2570
+ const snapshot = workspaceConvergence.snapshot();
2571
+ const mayHydrateVisible = !workerHasActiveWriter() && !pendingCheckpoint && snapshot.dirtyGeneration === snapshot.publishedGeneration;
2572
+ releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
2573
+ const hydrateVisible = releaseVisibleHydration !== null;
2574
+ const converged = await workspaceSyncSingleFlight.runExclusive(
2575
+ () => convergeWorkspaceHead(workspaceSyncInput({ type: "inbound_head", detail: advertisedHead ?? "unborn" }), {
2576
+ hydrateVisible
2414
2577
  })
2415
2578
  );
2579
+ workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
2580
+ if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
2581
+ if (hydrateVisible && converged.localChanges) {
2582
+ markWorkspaceDirty({ type: "periodic", detail: "inbound overlap retained for next checkpoint" });
2583
+ }
2584
+ sendWorkspaceReady();
2416
2585
  } catch (error) {
2586
+ pendingWorkspaceHead = advertisedHead;
2417
2587
  process.stderr.write(
2418
- `[r5d-worker] failed to send workspace sync observation on its current connection: ${error instanceof Error ? error.message : String(error)}
2588
+ `[r5d-worker] failed to fetch available workspace head: ${error instanceof Error ? error.message : String(error)}
2419
2589
  `
2420
2590
  );
2591
+ } finally {
2592
+ releaseVisibleHydration?.();
2593
+ workspaceHeadConvergenceInFlight = false;
2594
+ heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
2595
+ if (pendingWorkspaceHead !== void 0) {
2596
+ setTimeout(() => void convergeAvailableWorkspaceHead(), 1e3).unref();
2597
+ }
2421
2598
  }
2422
2599
  };
2423
2600
  const manifestCheckoutKey = (projectId, branchName) => `${projectId}\0${branchName}`;
@@ -2469,13 +2646,58 @@ async function startWorker(options) {
2469
2646
  }
2470
2647
  return created;
2471
2648
  };
2472
- const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}, authoritativeCanonicalHead, allowRecentNoChangeFastPath) => {
2649
+ const schedulePendingManifestCheckoutHydration = (delayMs = 0) => {
2650
+ if (workspaceManifestHydrationTimer || workspaceManifestHydrationInFlight || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2651
+ return;
2652
+ }
2653
+ workspaceManifestHydrationTimer = setTimeout(() => {
2654
+ workspaceManifestHydrationTimer = void 0;
2655
+ if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2656
+ if (pendingManifestCheckouts.size > 0 && currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
2657
+ schedulePendingManifestCheckoutHydration(1e3);
2658
+ }
2659
+ return;
2660
+ }
2661
+ const releaseHydration = workspaceFilesystemWriters.tryAcquireCheckpoint();
2662
+ if (!releaseHydration) {
2663
+ schedulePendingManifestCheckoutHydration(1e3);
2664
+ return;
2665
+ }
2666
+ workspaceManifestHydrationInFlight = true;
2667
+ void workspaceSyncSingleFlight.runExclusive(async () => {
2668
+ if (!workspaceReady || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2669
+ return;
2670
+ }
2671
+ const targets = [...pendingManifestCheckouts.values()];
2672
+ ensureVisibleWorkspaceCheckouts(targets, { strictCanonicalRevalidation: true });
2673
+ for (const target of targets) {
2674
+ const manifest = manifestByProjectId.get(target.projectId);
2675
+ if (!manifest || !manifest.branches.includes(target.branchName)) continue;
2676
+ const branchPath = path.join(projectRootFor(projectsRoot, target.projectId, manifestByProjectId), target.branchName);
2677
+ if (hasNormalVisibleGitDir(branchPath)) {
2678
+ pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
2679
+ }
2680
+ }
2681
+ }).then(() => sendWorkspaceReady()).catch((error) => {
2682
+ process.stderr.write(
2683
+ `[r5d-worker] pending manifest checkout hydration failed: ${error instanceof Error ? error.message : String(error)}
2684
+ `
2685
+ );
2686
+ }).finally(() => {
2687
+ releaseHydration();
2688
+ workspaceManifestHydrationInFlight = false;
2689
+ heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
2690
+ if (pendingManifestCheckouts.size > 0) schedulePendingManifestCheckoutHydration(1e3);
2691
+ });
2692
+ }, Math.max(0, delayMs));
2693
+ workspaceManifestHydrationTimer.unref();
2694
+ };
2695
+ const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}) => {
2473
2696
  const requestedAt = Date.now();
2474
2697
  let queueEnteredAt = requestedAt;
2475
2698
  let prepareStartedAt = requestedAt;
2476
2699
  let synchronizeStartedAt = requestedAt;
2477
2700
  let synchronizeFinishedAt = requestedAt;
2478
- let fastPathDecision = { hit: false, reason: "no_recent_proof" };
2479
2701
  workspaceSyncRequestsInFlight += 1;
2480
2702
  try {
2481
2703
  let pendingTargets = [];
@@ -2484,48 +2706,12 @@ async function startWorker(options) {
2484
2706
  const shouldEnsureCheckouts = !overrides.skipVisibleMirror && !overrides.resetToCanonical;
2485
2707
  const syncProjects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
2486
2708
  if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
2487
- const scopeKey = workspaceSyncScopeKey(workspaceRemoteUrl, syncProjects);
2488
2709
  const allowedCheckoutKeys = new Set(
2489
2710
  syncProjects.flatMap((project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName)))
2490
2711
  );
2491
2712
  const scopedPendingTargets = [...pendingManifestCheckouts.values()].filter(
2492
2713
  (target) => allowedCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
2493
2714
  );
2494
- const workerBusy = activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2495
- fastPathDecision = recentWorkspaceSyncProof.evaluate({
2496
- trigger,
2497
- allowRecentNoChangeFastPath,
2498
- canonicalHead: authoritativeCanonicalHead,
2499
- scopeKey,
2500
- nowMs: Date.now(),
2501
- skipVisibleMirror: overrides.skipVisibleMirror,
2502
- resetToCanonical: overrides.resetToCanonical,
2503
- incidentActive: activeWorkspaceIncidentId !== null,
2504
- workerBusy,
2505
- pendingCheckouts: scopedPendingTargets.length > 0 || Boolean(overrides.newVisibleCheckouts?.length)
2506
- });
2507
- if (fastPathDecision.hit) {
2508
- prepareStartedAt = queueEnteredAt;
2509
- synchronizeStartedAt = queueEnteredAt;
2510
- synchronizeFinishedAt = queueEnteredAt;
2511
- return {
2512
- type: "workspace_sync",
2513
- attemptId: overrides.attemptId ?? crypto.randomUUID(),
2514
- workerLabel: label,
2515
- trigger,
2516
- outcome: "no_change",
2517
- startingHead: fastPathDecision.canonicalHead,
2518
- expectedHead: fastPathDecision.canonicalHead,
2519
- publishedHead: fastPathDecision.canonicalHead,
2520
- rebaseCount: 0,
2521
- diffSizeBytes: 0,
2522
- gitStatus: "",
2523
- affectedProjects: [],
2524
- affectedPaths: [],
2525
- discardedPaths: [],
2526
- localChangesDiscarded: false
2527
- };
2528
- }
2529
2715
  prepareStartedAt = Date.now();
2530
2716
  let checkoutTargets = [];
2531
2717
  if (shouldEnsureCheckouts) {
@@ -2551,24 +2737,12 @@ async function startWorker(options) {
2551
2737
  ...newVisibleCheckouts.length > 0 ? { newVisibleCheckouts } : {}
2552
2738
  });
2553
2739
  synchronizeStartedAt = Date.now();
2554
- recentWorkspaceSyncProof.invalidate();
2555
- const observationGeneration = recentWorkspaceSyncProof.beginObservation();
2556
- const workerWasBusyDuringObservation = workerBusy;
2557
2740
  let synchronized;
2558
2741
  try {
2559
2742
  synchronized = await synchronizeWorkspace(input);
2560
2743
  } finally {
2561
2744
  synchronizeFinishedAt = Date.now();
2562
2745
  }
2563
- if ((synchronized.outcome === "no_change" || synchronized.outcome === "updated") && pendingTargets.length === 0) {
2564
- recentWorkspaceSyncProof.recordStable({
2565
- observationGeneration,
2566
- canonicalHead: synchronized.publishedHead ?? null,
2567
- scopeKey,
2568
- observedAtMs: synchronizeFinishedAt,
2569
- workerBusy: workerWasBusyDuringObservation || activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress || activeWorkspaceIncidentId !== null
2570
- });
2571
- }
2572
2746
  return synchronized;
2573
2747
  });
2574
2748
  const completedAt = Date.now();
@@ -2578,8 +2752,7 @@ async function startWorker(options) {
2578
2752
  totalMs: completedAt - requestedAt,
2579
2753
  queueMs: queueEnteredAt - requestedAt,
2580
2754
  prepareMs: synchronizeStartedAt - prepareStartedAt,
2581
- synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt,
2582
- fastPath: workspaceSyncFastPathTelemetry(fastPathDecision)
2755
+ synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
2583
2756
  }
2584
2757
  };
2585
2758
  if (resultWithTelemetry.outcome === "no_change" || resultWithTelemetry.outcome === "published" || resultWithTelemetry.outcome === "updated" || resultWithTelemetry.outcome === "conflict_reset") {
@@ -2588,7 +2761,6 @@ async function startWorker(options) {
2588
2761
  if (pendingManifestCheckouts.get(key) === target) pendingManifestCheckouts.delete(key);
2589
2762
  }
2590
2763
  }
2591
- previousPeriodicFingerprint = null;
2592
2764
  sendWorkspaceSyncResult(authority, resultWithTelemetry);
2593
2765
  return resultWithTelemetry;
2594
2766
  } catch (error) {
@@ -2611,12 +2783,10 @@ async function startWorker(options) {
2611
2783
  totalMs: completedAt - requestedAt,
2612
2784
  queueMs: queueEnteredAt - requestedAt,
2613
2785
  prepareMs: Math.max(0, synchronizeStartedAt - prepareStartedAt),
2614
- synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt),
2615
- fastPath: workspaceSyncFastPathTelemetry(fastPathDecision)
2786
+ synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
2616
2787
  },
2617
2788
  error: error instanceof Error ? error.message : String(error)
2618
2789
  };
2619
- previousPeriodicFingerprint = null;
2620
2790
  sendWorkspaceSyncResult(authority, result);
2621
2791
  return result;
2622
2792
  } finally {
@@ -2632,6 +2802,24 @@ async function startWorker(options) {
2632
2802
  let heartbeatBusyGrace = null;
2633
2803
  let heartbeatTimedOut = false;
2634
2804
  let heartbeatWatchdog;
2805
+ const handleGracefulShutdown = () => {
2806
+ if (shutdownAfterClose) return;
2807
+ shutdownAfterClose = true;
2808
+ void waitForWorkspaceCheckpointOnShutdown({
2809
+ snapshot: () => workspaceConvergence.snapshot(),
2810
+ forceCheckpoint: () => armWorkspaceCheckpoint({ type: "manual", detail: "graceful worker shutdown" }, { delayMs: 0 })
2811
+ }).then((outcome) => {
2812
+ process.stdout.write(`[r5d-worker] graceful workspace checkpoint ${outcome}
2813
+ `);
2814
+ if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
2815
+ ws.close(1e3, "Worker shutting down");
2816
+ } else {
2817
+ process.exit(0);
2818
+ }
2819
+ });
2820
+ };
2821
+ process.on("SIGTERM", handleGracefulShutdown);
2822
+ process.on("SIGINT", handleGracefulShutdown);
2635
2823
  const stopHeartbeatWatchdog = () => {
2636
2824
  if (heartbeatWatchdog) {
2637
2825
  clearInterval(heartbeatWatchdog);
@@ -2662,7 +2850,8 @@ async function startWorker(options) {
2662
2850
  updateClis: true,
2663
2851
  canonicalResolverCheckout: true,
2664
2852
  browserPortForwarding: true,
2665
- globalWorkspaceSyncLeaseV1: true
2853
+ globalWorkspaceSyncLeaseV1: true,
2854
+ eventualWorkspaceSyncV1: true
2666
2855
  },
2667
2856
  projectRoot: projectsRoot,
2668
2857
  artifactRoot,
@@ -2689,100 +2878,123 @@ async function startWorker(options) {
2689
2878
  return;
2690
2879
  }
2691
2880
  if (message.type === "workspace_manifest") {
2692
- recentWorkspaceSyncProof.invalidate();
2693
- await workspaceSyncSingleFlight.runMutation(() => {
2694
- configureGitHubAuth(message.githubCredential);
2695
- visibleGitIdentity = message.gitIdentity;
2696
- workspaceRemoteUrl = message.workspaceRemoteUrl;
2697
- const previousCheckoutKeys = new Set(
2698
- allManifestCheckouts().map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2699
- );
2700
- const migratedProjectIds = migrateLegacyProjectRoots(projectsRoot, message.projects);
2701
- manifestByProjectId.clear();
2702
- for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
2703
- const currentCheckouts = allManifestCheckouts();
2704
- const currentCheckoutKeys = new Set(
2705
- currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2706
- );
2707
- for (const key of pendingManifestCheckouts.keys()) {
2708
- if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
2709
- }
2710
- for (const checkout of currentCheckouts) {
2711
- const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
2712
- if (previousCheckoutKeys.has(key)) continue;
2713
- const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
2714
- if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
2715
- }
2716
- previousPeriodicFingerprint = null;
2717
- process.stdout.write(
2718
- `[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
2719
- `
2720
- );
2881
+ const revisionToken = workspaceManifestRevision.begin(message.revision);
2882
+ workspaceReady = false;
2883
+ const manifestAdmission = inspectWorkspaceManifestAdmission({
2884
+ projectsRoot,
2885
+ workspaceShadowRoot,
2886
+ projects: message.projects
2721
2887
  });
2722
- if (!periodicWorkspaceScan) {
2723
- const emptyFingerprint = createHash("sha256").update("").digest("hex");
2724
- periodicWorkspaceScan = setInterval(() => {
2725
- if (!workspaceRemoteUrl || activeWorkspaceIncidentId || cliUpdateInProgress || periodicWorkspaceScanInFlight) return;
2726
- periodicWorkspaceScanInFlight = true;
2727
- void (async () => {
2728
- const genericCheckoutKeys = new Set(
2729
- workspaceProjectsForSync([...manifestByProjectId.values()], { type: "periodic" }).flatMap(
2730
- (project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName))
2731
- )
2732
- );
2733
- const hasPendingGenericCheckout = [...pendingManifestCheckouts.values()].some(
2734
- (target) => genericCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
2888
+ const releaseManifestHydration = await acquireWorkspaceManifestVisibleLease(
2889
+ workspaceFilesystemWriters,
2890
+ manifestAdmission
2891
+ );
2892
+ let completed = false;
2893
+ try {
2894
+ completed = await workspaceSyncSingleFlight.runExclusive(async () => {
2895
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2896
+ const firstBootstrap = manifestAdmission.firstBootstrap;
2897
+ configureGitHubAuth(message.githubCredential);
2898
+ visibleGitIdentity = message.gitIdentity;
2899
+ workspaceRemoteUrl = message.workspaceRemoteUrl;
2900
+ const migratedProjectIds = releaseManifestHydration ? migrateLegacyProjectRoots(projectsRoot, message.projects) : [];
2901
+ manifestByProjectId.clear();
2902
+ for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
2903
+ const currentCheckouts = allManifestCheckouts();
2904
+ const currentCheckoutKeys = new Set(
2905
+ currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2906
+ );
2907
+ for (const key of pendingManifestCheckouts.keys()) {
2908
+ if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
2909
+ }
2910
+ for (const checkout of currentCheckouts) {
2911
+ const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
2912
+ const branchPath = path.join(
2913
+ projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId),
2914
+ checkout.branchName
2735
2915
  );
2736
- if (hasPendingGenericCheckout) {
2737
- recentWorkspaceSyncProof.invalidate();
2738
- previousPeriodicFingerprint = null;
2739
- requestWorkspaceSync({
2740
- type: "periodic",
2741
- detail: "workspace manifest additions"
2742
- });
2743
- return;
2916
+ if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
2917
+ }
2918
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2919
+ const created = releaseManifestHydration ? ensureVisibleWorkspaceCheckouts(currentCheckouts, { strictCanonicalRevalidation: true }) : [];
2920
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2921
+ const bootstrapped = await convergeWorkspaceHead(
2922
+ workspaceSyncInput({ type: "connect" }, { newVisibleCheckouts: created }),
2923
+ {
2924
+ // An existing checkout becomes routable after a fetch. Its
2925
+ // visible tree converges asynchronously; first bootstrap alone
2926
+ // must materialize the canonical snapshot before readiness.
2927
+ // Re-check the incident immediately before convergence so a
2928
+ // concurrently delivered fence forces fetch-only behavior.
2929
+ hydrateVisible: firstBootstrap && !activeWorkspaceIncidentId
2744
2930
  }
2745
- const observation = await workspaceSyncSingleFlight.runExclusive(() => {
2746
- const observationGeneration = recentWorkspaceSyncProof.beginObservation();
2747
- const input = workspaceSyncInput({ type: "periodic" });
2748
- const workerBusy = activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2749
- return {
2750
- observationGeneration,
2751
- scopeKey: workspaceSyncScopeKey(input.remoteUrl, input.projects),
2752
- fingerprint: calculateWorkspaceDiffFingerprint(input),
2753
- canonicalHead: readShadowWorkspaceCanonicalHead(workspaceShadowRoot),
2754
- workerBusy
2755
- };
2756
- });
2757
- if (observation.fingerprint === emptyFingerprint) {
2758
- recentWorkspaceSyncProof.recordStable({
2759
- observationGeneration: observation.observationGeneration,
2760
- canonicalHead: observation.canonicalHead,
2761
- scopeKey: observation.scopeKey,
2762
- observedAtMs: Date.now(),
2763
- workerBusy: observation.workerBusy || activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress
2764
- });
2765
- previousPeriodicFingerprint = null;
2766
- return;
2931
+ );
2932
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2933
+ workspaceConvergence.setLocalHead(bootstrapped.localHead);
2934
+ workspaceConvergence.observeCanonicalHead(bootstrapped.canonicalHead);
2935
+ for (const checkout of currentCheckouts) {
2936
+ const manifest = manifestByProjectId.get(checkout.projectId);
2937
+ if (!manifest) continue;
2938
+ const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
2939
+ if (hasNormalVisibleGitDir(branchPath)) {
2940
+ pendingManifestCheckouts.delete(manifestCheckoutKey(checkout.projectId, checkout.branchName));
2767
2941
  }
2768
- recentWorkspaceSyncProof.invalidate();
2769
- if (previousPeriodicFingerprint !== observation.fingerprint) {
2770
- previousPeriodicFingerprint = observation.fingerprint;
2771
- return;
2942
+ }
2943
+ process.stdout.write(
2944
+ `[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
2945
+ `
2946
+ );
2947
+ return true;
2948
+ });
2949
+ } finally {
2950
+ releaseManifestHydration?.();
2951
+ }
2952
+ if (!completed || !workspaceManifestRevision.complete(revisionToken)) return;
2953
+ workspaceReady = true;
2954
+ sendWorkspaceReady();
2955
+ schedulePendingManifestCheckoutHydration();
2956
+ if (!workspaceSafetyScan) {
2957
+ workspaceSafetyScan = setInterval(() => {
2958
+ if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || workspaceSafetyScanInFlight) {
2959
+ return;
2960
+ }
2961
+ workspaceSafetyScanInFlight = true;
2962
+ void (async () => {
2963
+ const before = workspaceConvergence.snapshot();
2964
+ const mayHydrateVisible = !workerHasActiveWriter() && before.dirtyGeneration === before.publishedGeneration;
2965
+ const releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
2966
+ const hydrateVisible = releaseVisibleHydration !== null;
2967
+ try {
2968
+ const converged = await workspaceSyncSingleFlight.runExclusive(
2969
+ () => convergeWorkspaceHead(workspaceSyncInput({ type: "periodic", detail: "idle convergence scan" }), {
2970
+ hydrateVisible
2971
+ })
2972
+ );
2973
+ workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
2974
+ if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
2975
+ if (converged.localChanges && hydrateVisible) {
2976
+ markWorkspaceDirty({ type: "periodic", detail: "idle safety scan" });
2977
+ }
2978
+ sendWorkspaceReady();
2979
+ } finally {
2980
+ releaseVisibleHydration?.();
2772
2981
  }
2773
- previousPeriodicFingerprint = null;
2774
- requestWorkspaceSync({ type: "periodic" }, observation.fingerprint);
2775
2982
  })().catch((error) => {
2776
2983
  process.stderr.write(
2777
- `[r5d-worker] periodic workspace synchronization failed: ${error instanceof Error ? error.message : String(error)}
2984
+ `[r5d-worker] idle workspace safety scan failed: ${error instanceof Error ? error.message : String(error)}
2778
2985
  `
2779
2986
  );
2780
2987
  }).finally(() => {
2781
- periodicWorkspaceScanInFlight = false;
2988
+ workspaceSafetyScanInFlight = false;
2782
2989
  heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
2783
2990
  });
2784
- }, WORKSPACE_PERIODIC_SCAN_INTERVAL_MS);
2785
- periodicWorkspaceScan.unref();
2991
+ }, WORKSPACE_IDLE_SAFETY_SCAN_MS);
2992
+ workspaceSafetyScan.unref();
2993
+ }
2994
+ pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
2995
+ void convergeAvailableWorkspaceHead();
2996
+ if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
2997
+ armWorkspaceCheckpoint({ type: "periodic", detail: "resume dirty generation after reconnect" }, { delayMs: 0 });
2786
2998
  }
2787
2999
  return;
2788
3000
  }
@@ -2802,12 +3014,189 @@ async function startWorker(options) {
2802
3014
  confirmationReason: message.confirmationReason,
2803
3015
  resetToCanonical: message.resetToCanonical,
2804
3016
  skipVisibleMirror: message.skipVisibleMirror
2805
- },
2806
- message.canonicalHead,
2807
- message.allowRecentNoChangeFastPath
3017
+ }
2808
3018
  );
2809
3019
  return;
2810
3020
  }
3021
+ if (message.type === "workspace_head_available") {
3022
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3023
+ pendingWorkspaceHead = message.canonicalHead;
3024
+ sendWorkspaceReady();
3025
+ void convergeAvailableWorkspaceHead();
3026
+ return;
3027
+ }
3028
+ if (message.type === "workspace_checkpoint_now") {
3029
+ armWorkspaceCheckpoint(message.trigger, { delayMs: 0, requestId: message.requestId });
3030
+ return;
3031
+ }
3032
+ if (message.type === "workspace_checkpoint_authority") {
3033
+ const checkpoint = pendingCheckpoint;
3034
+ if (!checkpoint || checkpoint.requestId !== message.requestId) {
3035
+ process.stderr.write(`[r5d-worker] ignoring checkpoint authority for unknown request ${message.requestId}
3036
+ `);
3037
+ return;
3038
+ }
3039
+ if (message.status !== "granted") {
3040
+ checkpoint.releaseCheckpointAdmission();
3041
+ pendingCheckpoint = void 0;
3042
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3043
+ const applyBlockedResponse = message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId);
3044
+ if (applyBlockedResponse) {
3045
+ workspaceConvergence.block();
3046
+ if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
3047
+ } else {
3048
+ workspaceConvergence.resetTransientCheckpointState();
3049
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3050
+ if (message.status === "busy") {
3051
+ armWorkspaceCheckpoint(checkpoint.trigger, {
3052
+ delayMs: message.retryAfterMs ?? 1e3,
3053
+ requestId: checkpoint.requestId
3054
+ });
3055
+ } else if (!activeWorkspaceIncidentId) {
3056
+ const explicit = explicitWorkspaceCheckpoints.next();
3057
+ if (explicit) {
3058
+ armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
3059
+ } else if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3060
+ armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: 0 });
3061
+ }
3062
+ }
3063
+ }
3064
+ sendWorkspaceReady();
3065
+ return;
3066
+ }
3067
+ checkpoint.attemptId = message.attemptId;
3068
+ checkpoint.authorityToken = message.authorityToken;
3069
+ checkpoint.workInFlight = true;
3070
+ pendingCheckpoint = checkpoint;
3071
+ if (!checkpoint.admissionHeld) {
3072
+ const releaseCheckpointAdmission = await workspaceFilesystemWriters.acquireCheckpoint();
3073
+ if (currentWorkerSocket !== ws || pendingCheckpoint?.requestId !== checkpoint.requestId || pendingCheckpoint.attemptId !== message.attemptId) {
3074
+ releaseCheckpointAdmission();
3075
+ checkpoint.workInFlight = false;
3076
+ workspaceConvergence.resetTransientCheckpointState();
3077
+ return;
3078
+ }
3079
+ checkpoint.admissionHeld = true;
3080
+ checkpoint.releaseCheckpointAdmission = releaseCheckpointAdmission;
3081
+ }
3082
+ workspaceConvergence.beginPreparing();
3083
+ sendWorkspaceReady();
3084
+ let prepareStartedAt = Date.now();
3085
+ let synchronizeStartedAt = prepareStartedAt;
3086
+ let synchronizeStarted = false;
3087
+ let result;
3088
+ try {
3089
+ result = await workspaceSyncSingleFlight.runExclusive(async () => {
3090
+ prepareStartedAt = Date.now();
3091
+ const pendingTargets = [...pendingManifestCheckouts.values()];
3092
+ const createdCheckouts = ensureVisibleWorkspaceCheckouts(pendingTargets, { strictCanonicalRevalidation: true });
3093
+ synchronizeStartedAt = Date.now();
3094
+ synchronizeStarted = true;
3095
+ const prepared = await synchronizeWorkspace(
3096
+ workspaceSyncInput(checkpoint.trigger, {
3097
+ attemptId: message.attemptId,
3098
+ quarantineRef: message.quarantineRef,
3099
+ expectedCanonicalHead: message.expectedHead,
3100
+ newVisibleCheckouts: [...createdCheckouts, ...pendingTargets]
3101
+ })
3102
+ );
3103
+ for (const target of pendingTargets) {
3104
+ pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
3105
+ }
3106
+ return prepared.outcome === "updated" ? { ...prepared, outcome: "no_change" } : prepared;
3107
+ });
3108
+ } catch (error) {
3109
+ if (!synchronizeStarted) synchronizeStartedAt = Date.now();
3110
+ result = {
3111
+ type: "workspace_sync",
3112
+ attemptId: message.attemptId,
3113
+ workerLabel: label,
3114
+ trigger: checkpoint.trigger,
3115
+ outcome: "failed",
3116
+ startingHead: message.expectedHead,
3117
+ expectedHead: message.expectedHead,
3118
+ rebaseCount: 0,
3119
+ diffSizeBytes: 0,
3120
+ gitStatus: "",
3121
+ affectedProjects: [],
3122
+ affectedPaths: [],
3123
+ discardedPaths: [],
3124
+ localChangesDiscarded: false,
3125
+ error: error instanceof Error ? error.message : String(error)
3126
+ };
3127
+ }
3128
+ const finishedAt = Date.now();
3129
+ result = {
3130
+ ...result,
3131
+ dirtyGeneration: checkpoint.dirtyGeneration,
3132
+ telemetry: workspaceCheckpointTelemetry({
3133
+ requestedAt: checkpoint.requestedAt,
3134
+ prepareStartedAt,
3135
+ synchronizeStartedAt,
3136
+ finishedAt
3137
+ })
3138
+ };
3139
+ workspaceConvergence.beginSubmitting();
3140
+ sendWorkspaceReady();
3141
+ try {
3142
+ ws.send(
3143
+ JSON.stringify({
3144
+ type: "workspace_checkpoint_result",
3145
+ requestId: checkpoint.requestId,
3146
+ attemptId: message.attemptId,
3147
+ authorityToken: message.authorityToken,
3148
+ dirtyGeneration: checkpoint.dirtyGeneration,
3149
+ result
3150
+ })
3151
+ );
3152
+ } catch (error) {
3153
+ workspaceConvergence.resetTransientCheckpointState();
3154
+ throw error;
3155
+ } finally {
3156
+ checkpoint.workInFlight = false;
3157
+ checkpoint.releaseCheckpointAdmission();
3158
+ if (currentWorkerSocket !== ws) workspaceConvergence.resetTransientCheckpointState();
3159
+ }
3160
+ return;
3161
+ }
3162
+ if (message.type === "workspace_checkpoint_completed") {
3163
+ const checkpoint = pendingCheckpoint;
3164
+ if (!checkpoint || checkpoint.requestId !== message.requestId || checkpoint.attemptId !== message.attemptId) {
3165
+ process.stderr.write(`[r5d-worker] ignoring completion for unknown checkpoint ${message.requestId}
3166
+ `);
3167
+ return;
3168
+ }
3169
+ pendingCheckpoint = void 0;
3170
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3171
+ if (message.status === "published" || message.status === "no_change") {
3172
+ workspaceConvergence.complete({
3173
+ generation: Math.min(message.publishedGeneration ?? checkpoint.dirtyGeneration, checkpoint.dirtyGeneration),
3174
+ canonicalHead: message.canonicalHead,
3175
+ published: true
3176
+ });
3177
+ } else if (message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId)) {
3178
+ workspaceConvergence.block();
3179
+ if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
3180
+ } else {
3181
+ workspaceConvergence.resetTransientCheckpointState();
3182
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3183
+ }
3184
+ sendWorkspaceReady();
3185
+ const explicit = message.status === "retry" && checkpoint.explicit ? { requestId: checkpoint.requestId, trigger: checkpoint.trigger } : explicitWorkspaceCheckpoints.next();
3186
+ if (explicit) {
3187
+ armWorkspaceCheckpoint(explicit.trigger, {
3188
+ delayMs: message.retryAfterMs ?? 0,
3189
+ requestId: explicit.requestId
3190
+ });
3191
+ } else if (message.status === "retry" || message.status === "failed" || workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3192
+ armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: message.retryAfterMs ?? 1e3 });
3193
+ }
3194
+ if (message.canonicalHead !== workspaceConvergence.snapshot().localHead) {
3195
+ pendingWorkspaceHead = message.canonicalHead;
3196
+ void convergeAvailableWorkspaceHead();
3197
+ }
3198
+ return;
3199
+ }
2811
3200
  if (message.type === "sync_session_artifacts") {
2812
3201
  sessionArtifactSyncRequestsInFlight += 1;
2813
3202
  try {
@@ -2835,18 +3224,27 @@ async function startWorker(options) {
2835
3224
  return;
2836
3225
  }
2837
3226
  if (message.type === "workspace_incident_updated") {
2838
- recentWorkspaceSyncProof.invalidate();
3227
+ workspaceIncidentOrdering.observe(message);
2839
3228
  const previousIncidentId = activeWorkspaceIncidentId;
2840
3229
  activeWorkspaceIncidentId = applyWorkspaceIncidentUpdate(activeWorkspaceIncidentId, message);
2841
- previousPeriodicFingerprint = null;
2842
3230
  const releasedHead = releasePendingWorkspaceHead(
2843
3231
  previousIncidentId,
2844
3232
  activeWorkspaceIncidentId,
2845
3233
  message.status === "resolved" || message.status === "confirmed" || message.status === "reset" ? message.incidentId : null
2846
3234
  );
2847
- if (releasedHead.syncHead) {
2848
- requestWorkspaceSync({ type: "inbound_head", detail: releasedHead.syncHead });
3235
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3236
+ if (releasedHead.fetchAuthoritativeHead) {
3237
+ pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
3238
+ void convergeAvailableWorkspaceHead();
3239
+ }
3240
+ const explicit = !activeWorkspaceIncidentId ? explicitWorkspaceCheckpoints.next() : void 0;
3241
+ if (explicit) {
3242
+ armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
3243
+ } else if (!activeWorkspaceIncidentId && workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3244
+ armWorkspaceCheckpoint({ type: "periodic", detail: "resume after workspace incident" }, { delayMs: 0 });
2849
3245
  }
3246
+ if (!activeWorkspaceIncidentId) schedulePendingManifestCheckoutHydration();
3247
+ sendWorkspaceReady();
2850
3248
  return;
2851
3249
  }
2852
3250
  if (message.type === "exec_terminal_ack") {
@@ -2882,7 +3280,6 @@ async function startWorker(options) {
2882
3280
  return;
2883
3281
  }
2884
3282
  cliUpdateInProgress = true;
2885
- recentWorkspaceSyncProof.invalidate();
2886
3283
  try {
2887
3284
  const result = await installCliUpdate(message);
2888
3285
  sendWorkerMessage(ws, {
@@ -2945,9 +3342,10 @@ async function startWorker(options) {
2945
3342
  });
2946
3343
  return;
2947
3344
  }
2948
- recentWorkspaceSyncProof.invalidate();
3345
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
2949
3346
  const releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
2950
- let leaseTransferred = false;
3347
+ let mutationLeaseTransferred = false;
3348
+ let writerLeaseTransferred = false;
2951
3349
  try {
2952
3350
  const resolvedTarget = resolveMessageTarget(message.target);
2953
3351
  process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
@@ -2957,9 +3355,19 @@ async function startWorker(options) {
2957
3355
  message,
2958
3356
  resolvedTarget,
2959
3357
  planRoot,
2960
- ...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {}
3358
+ ...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {},
3359
+ ...targetMayMutateVisibleWorkspace(message.target) ? {
3360
+ onTerminal: () => {
3361
+ markWorkspaceDirty(
3362
+ { type: "process_terminal", detail: `pty ${message.ptyId} completed` },
3363
+ true
3364
+ );
3365
+ releaseWorkspaceWriter?.();
3366
+ }
3367
+ } : {}
2961
3368
  });
2962
- leaseTransferred = releaseWorkspaceMutation !== void 0;
3369
+ mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
3370
+ writerLeaseTransferred = releaseWorkspaceWriter !== void 0;
2963
3371
  } catch (error) {
2964
3372
  sendWorkerMessage(ws, {
2965
3373
  type: "pty_error",
@@ -2968,12 +3376,16 @@ async function startWorker(options) {
2968
3376
  error: error instanceof Error ? error.message : String(error)
2969
3377
  });
2970
3378
  } finally {
2971
- if (!leaseTransferred) releaseWorkspaceMutation?.();
3379
+ if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
3380
+ if (!writerLeaseTransferred) releaseWorkspaceWriter?.();
2972
3381
  }
2973
3382
  return;
2974
3383
  }
2975
3384
  if (message.type === "pty_input") {
2976
- recentWorkspaceSyncProof.invalidate();
3385
+ const activePty = activePtys.get(message.ptyId);
3386
+ if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
3387
+ markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` });
3388
+ }
2977
3389
  writePty(ws, message);
2978
3390
  return;
2979
3391
  }
@@ -2982,8 +3394,11 @@ async function startWorker(options) {
2982
3394
  return;
2983
3395
  }
2984
3396
  if (message.type === "pty_close") {
2985
- recentWorkspaceSyncProof.invalidate();
3397
+ const activePty = activePtys.get(message.ptyId);
2986
3398
  closePty(message);
3399
+ if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
3400
+ armWorkspaceCheckpoint({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, { delayMs: 0 });
3401
+ }
2987
3402
  return;
2988
3403
  }
2989
3404
  if (message.type === "exec") {
@@ -2997,27 +3412,37 @@ async function startWorker(options) {
2997
3412
  });
2998
3413
  return;
2999
3414
  }
3000
- recentWorkspaceSyncProof.invalidate();
3001
- let result;
3415
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3002
3416
  try {
3003
- const runCommand = async () => {
3004
- const resolvedTarget = resolveMessageTarget(message.target);
3005
- process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
3417
+ let result;
3418
+ try {
3419
+ const runCommand = async () => {
3420
+ const resolvedTarget = resolveMessageTarget(message.target);
3421
+ process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
3006
3422
  `);
3007
- return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
3008
- };
3009
- result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
3010
- } catch (error) {
3011
- result = {
3012
- type: "exec_result",
3013
- requestId: message.requestId,
3014
- stdout: "",
3015
- stderr: "",
3016
- exitCode: 1,
3017
- error: error instanceof Error ? error.message : String(error)
3018
- };
3423
+ return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
3424
+ };
3425
+ result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
3426
+ } catch (error) {
3427
+ result = {
3428
+ type: "exec_result",
3429
+ requestId: message.requestId,
3430
+ stdout: "",
3431
+ stderr: "",
3432
+ exitCode: 1,
3433
+ error: error instanceof Error ? error.message : String(error)
3434
+ };
3435
+ }
3436
+ ws.send(JSON.stringify(result));
3437
+ if (targetMayMutateVisibleWorkspace(message.target)) {
3438
+ markWorkspaceDirty(
3439
+ { type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
3440
+ true
3441
+ );
3442
+ }
3443
+ } finally {
3444
+ releaseWorkspaceWriter?.();
3019
3445
  }
3020
- ws.send(JSON.stringify(result));
3021
3446
  return;
3022
3447
  }
3023
3448
  if (message.type === "exec_start") {
@@ -3030,25 +3455,40 @@ async function startWorker(options) {
3030
3455
  });
3031
3456
  return;
3032
3457
  }
3033
- recentWorkspaceSyncProof.invalidate();
3034
3458
  sendWorkerMessage(ws, {
3035
3459
  type: "exec_accepted",
3036
3460
  requestId: message.requestId,
3037
3461
  runId: message.runId
3038
3462
  });
3039
3463
  const runCommand = async () => {
3040
- const resolvedTarget = resolveMessageTarget(message.target);
3041
- process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
3464
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3465
+ try {
3466
+ const resolvedTarget = resolveMessageTarget(message.target);
3467
+ process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
3042
3468
  `);
3043
- await executeStreamingCommand({
3044
- ws,
3045
- message,
3046
- resolvedTarget,
3047
- baseUrl,
3048
- token,
3049
- artifactRoot,
3050
- planRoot
3051
- });
3469
+ await executeStreamingCommand({
3470
+ ws,
3471
+ message,
3472
+ resolvedTarget,
3473
+ baseUrl,
3474
+ token,
3475
+ artifactRoot,
3476
+ planRoot
3477
+ });
3478
+ } finally {
3479
+ if (targetMayMutateVisibleWorkspace(message.target)) {
3480
+ markWorkspaceDirty(
3481
+ {
3482
+ type: "process_terminal",
3483
+ sessionId: message.sessionId,
3484
+ processRunId: message.runId,
3485
+ detail: "process completed"
3486
+ },
3487
+ true
3488
+ );
3489
+ }
3490
+ releaseWorkspaceWriter?.();
3491
+ }
3052
3492
  };
3053
3493
  const execution = runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
3054
3494
  void execution.catch((error) => {
@@ -3062,34 +3502,46 @@ async function startWorker(options) {
3062
3502
  return;
3063
3503
  }
3064
3504
  if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes") {
3065
- if (message.type === "write" || message.type === "edit") recentWorkspaceSyncProof.invalidate();
3505
+ const releaseWorkspaceWriter = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3066
3506
  try {
3067
- const result = await workspaceSyncSingleFlight.runMutation(async () => {
3068
- const resolvedTarget = resolveMessageTarget(message.target);
3069
- return await executeOperation({
3070
- message,
3071
- resolvedTarget,
3072
- baseUrl,
3073
- token,
3074
- artifactRoot,
3075
- planRoot
3507
+ try {
3508
+ const result = await workspaceSyncSingleFlight.runMutation(async () => {
3509
+ const resolvedTarget = resolveMessageTarget(message.target);
3510
+ return await executeOperation({
3511
+ message,
3512
+ resolvedTarget,
3513
+ baseUrl,
3514
+ token,
3515
+ artifactRoot,
3516
+ planRoot
3517
+ });
3076
3518
  });
3077
- });
3078
- ws.send(
3079
- JSON.stringify({
3080
- type: "operation_result",
3081
- requestId: message.requestId,
3082
- result
3083
- })
3084
- );
3085
- } catch (error) {
3086
- ws.send(
3087
- JSON.stringify({
3088
- type: "operation_result",
3089
- requestId: message.requestId,
3090
- error: error instanceof Error ? error.message : String(error)
3091
- })
3092
- );
3519
+ ws.send(
3520
+ JSON.stringify({
3521
+ type: "operation_result",
3522
+ requestId: message.requestId,
3523
+ result
3524
+ })
3525
+ );
3526
+ if ((message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target)) {
3527
+ markWorkspaceDirty({
3528
+ type: message.type,
3529
+ sessionId: message.sessionId,
3530
+ toolCallId: message.requestId,
3531
+ ...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
3532
+ });
3533
+ }
3534
+ } catch (error) {
3535
+ ws.send(
3536
+ JSON.stringify({
3537
+ type: "operation_result",
3538
+ requestId: message.requestId,
3539
+ error: error instanceof Error ? error.message : String(error)
3540
+ })
3541
+ );
3542
+ }
3543
+ } finally {
3544
+ releaseWorkspaceWriter?.();
3093
3545
  }
3094
3546
  return;
3095
3547
  }
@@ -3111,11 +3563,24 @@ async function startWorker(options) {
3111
3563
  });
3112
3564
  });
3113
3565
  ws.addEventListener("close", (event) => {
3114
- recentWorkspaceSyncProof.invalidate();
3566
+ process.off("SIGTERM", handleGracefulShutdown);
3567
+ process.off("SIGINT", handleGracefulShutdown);
3115
3568
  stopHeartbeatWatchdog();
3116
- if (periodicWorkspaceScan) {
3117
- clearInterval(periodicWorkspaceScan);
3118
- periodicWorkspaceScan = void 0;
3569
+ if (workspaceCheckpointTimer) {
3570
+ clearTimeout(workspaceCheckpointTimer);
3571
+ workspaceCheckpointTimer = void 0;
3572
+ }
3573
+ if (!pendingCheckpoint?.workInFlight) {
3574
+ pendingCheckpoint?.releaseCheckpointAdmission();
3575
+ workspaceConvergence.resetTransientCheckpointState();
3576
+ }
3577
+ if (workspaceSafetyScan) {
3578
+ clearInterval(workspaceSafetyScan);
3579
+ workspaceSafetyScan = void 0;
3580
+ }
3581
+ if (workspaceManifestHydrationTimer) {
3582
+ clearTimeout(workspaceManifestHydrationTimer);
3583
+ workspaceManifestHydrationTimer = void 0;
3119
3584
  }
3120
3585
  if (terminalReplayTimer) {
3121
3586
  clearInterval(terminalReplayTimer);
@@ -3131,6 +3596,9 @@ async function startWorker(options) {
3131
3596
  if (reloadAfterClose) {
3132
3597
  process.exit(WORKER_RELOAD_EXIT_CODE);
3133
3598
  }
3599
+ if (shutdownAfterClose) {
3600
+ process.exit(0);
3601
+ }
3134
3602
  if (activeProcesses.size > 0 || pendingProcessTerminals.size > 0) {
3135
3603
  process.stderr.write(
3136
3604
  `[r5d-worker] ${activeProcesses.size} process(es) active and ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms