@ricsam/r5d-worker 0.0.57 → 0.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/mjs/main.mjs CHANGED
@@ -13,8 +13,27 @@ import {
13
13
  } from "./heartbeat.mjs";
14
14
  import { terminateProcessTree } from "./process-tree.mjs";
15
15
  import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
16
- import { applyWorkspaceIncidentUpdate, releasePendingWorkspaceHead } from "./workspace-incident-state.mjs";
16
+ import {
17
+ applyWorkspaceIncidentUpdate,
18
+ releasePendingWorkspaceHead,
19
+ WorkspaceIncidentOrderingFence
20
+ } from "./workspace-incident-state.mjs";
17
21
  import { acquireWorkspaceCommandMutation, runWorkspaceCommand } from "./workspace-command-sync-policy.mjs";
22
+ import {
23
+ WORKSPACE_CHECKPOINT_QUIET_MS,
24
+ WORKSPACE_IDLE_SAFETY_SCAN_MS,
25
+ ExplicitWorkspaceCheckpointQueue,
26
+ WorkspaceManifestRevisionFence,
27
+ processWorkspaceConvergenceState,
28
+ processWorkspaceFilesystemWriterGate,
29
+ waitForWorkspaceCheckpointOnShutdown,
30
+ workspaceCheckpointIncidentAction,
31
+ workspaceCheckpointTelemetry
32
+ } from "./workspace-convergence.mjs";
33
+ import {
34
+ acquireWorkspaceManifestVisibleLease,
35
+ inspectWorkspaceManifestAdmission
36
+ } from "./workspace-manifest-admission.mjs";
18
37
  import {
19
38
  isRetryableWorkerServerStatus,
20
39
  superviseWorkerRuntime,
@@ -25,9 +44,8 @@ import {
25
44
  } from "./supervisor.mjs";
26
45
  import { managedProjectRoot, migrateLegacyProjectRoots, validateManagedBranchName } from "./managed-paths.mjs";
27
46
  import {
28
- WORKSPACE_PERIODIC_SCAN_INTERVAL_MS,
29
47
  WorkspaceSyncSingleFlight,
30
- calculateWorkspaceDiffFingerprint,
48
+ convergeWorkspaceHead,
31
49
  synchronizeWorkspace,
32
50
  workspaceProjectsForSync
33
51
  } from "./workspace-sync.mjs";
@@ -46,6 +64,7 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
46
64
  const activePtys = /* @__PURE__ */ new Map();
47
65
  let currentWorkerSocket = null;
48
66
  const workspaceSyncSingleFlight = new WorkspaceSyncSingleFlight();
67
+ const workspaceFilesystemWriters = processWorkspaceFilesystemWriterGate();
49
68
  let githubCredential = null;
50
69
  let visibleGitIdentity = null;
51
70
  function defaultConfigPath() {
@@ -2260,6 +2279,7 @@ async function openPty(input) {
2260
2279
  onExit: (event) => {
2261
2280
  input.releaseWorkspaceMutation?.();
2262
2281
  activePtys.delete(input.message.ptyId);
2282
+ input.onTerminal?.();
2263
2283
  sendWorkerMessage(input.ws, {
2264
2284
  type: "pty_exit",
2265
2285
  ptyId: input.message.ptyId,
@@ -2270,6 +2290,7 @@ async function openPty(input) {
2270
2290
  onError: (error) => {
2271
2291
  input.releaseWorkspaceMutation?.();
2272
2292
  activePtys.delete(input.message.ptyId);
2293
+ input.onTerminal?.();
2273
2294
  sendWorkerMessage(input.ws, {
2274
2295
  type: "pty_error",
2275
2296
  requestId: input.message.requestId,
@@ -2361,14 +2382,25 @@ async function startWorker(options) {
2361
2382
  const pendingManifestCheckouts = /* @__PURE__ */ new Map();
2362
2383
  let workspaceRemoteUrl = null;
2363
2384
  let activeWorkspaceIncidentId = null;
2364
- let previousPeriodicFingerprint = null;
2365
- let periodicWorkspaceScan;
2366
- let periodicWorkspaceScanInFlight = false;
2385
+ const workspaceIncidentOrdering = new WorkspaceIncidentOrderingFence();
2386
+ const workspaceConvergence = processWorkspaceConvergenceState(path.join(syncRoot, "workspace-convergence.json"));
2387
+ const workspaceManifestRevision = new WorkspaceManifestRevisionFence();
2388
+ let workspaceReady = false;
2389
+ let workspaceCheckpointTimer;
2390
+ let workspaceSafetyScan;
2391
+ let workspaceManifestHydrationTimer;
2392
+ let workspaceManifestHydrationInFlight = false;
2393
+ let workspaceSafetyScanInFlight = false;
2394
+ let workspaceHeadConvergenceInFlight = false;
2395
+ let pendingWorkspaceHead;
2396
+ let pendingCheckpoint;
2397
+ const explicitWorkspaceCheckpoints = new ExplicitWorkspaceCheckpointQueue();
2367
2398
  let terminalReplayTimer;
2368
2399
  let workspaceSyncRequestsInFlight = 0;
2369
2400
  let sessionArtifactSyncRequestsInFlight = 0;
2370
2401
  let cliUpdateInProgress = false;
2371
2402
  let reloadAfterClose = false;
2403
+ let shutdownAfterClose = false;
2372
2404
  const workspaceSyncInput = (trigger, overrides = {}) => {
2373
2405
  if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
2374
2406
  const projects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
@@ -2394,21 +2426,175 @@ async function startWorker(options) {
2394
2426
  );
2395
2427
  }
2396
2428
  };
2397
- const requestWorkspaceSync = (trigger, fingerprint) => {
2429
+ const sendWorkspaceReady = () => {
2430
+ const manifestRevision = workspaceManifestRevision.readyRevision();
2431
+ if (!workspaceReady || !manifestRevision) return;
2398
2432
  try {
2399
2433
  ws.send(
2400
2434
  JSON.stringify({
2401
- type: "workspace_sync_requested",
2402
- observationId: crypto.randomUUID(),
2435
+ type: "workspace_ready",
2436
+ manifestRevision,
2437
+ pendingCheckouts: [...pendingManifestCheckouts.values()].sort(
2438
+ (left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
2439
+ ),
2440
+ ...workspaceConvergence.snapshot()
2441
+ })
2442
+ );
2443
+ } catch (error) {
2444
+ process.stderr.write(
2445
+ `[r5d-worker] failed to send workspace readiness: ${error instanceof Error ? error.message : String(error)}
2446
+ `
2447
+ );
2448
+ }
2449
+ };
2450
+ const workerHasActiveWriter = () => activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2451
+ const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
2452
+ const armWorkspaceCheckpoint = (trigger, options2 = { delayMs: WORKSPACE_CHECKPOINT_QUIET_MS }) => {
2453
+ workspaceConvergence.schedule();
2454
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2455
+ sendWorkspaceReady();
2456
+ if (options2.requestId && !explicitWorkspaceCheckpoints.schedule({ requestId: options2.requestId, trigger }, pendingCheckpoint?.requestId)) {
2457
+ return;
2458
+ }
2459
+ if (!options2.requestId && explicitWorkspaceCheckpoints.hasScheduled()) return;
2460
+ if (workspaceCheckpointTimer) clearTimeout(workspaceCheckpointTimer);
2461
+ workspaceCheckpointTimer = setTimeout(() => {
2462
+ workspaceCheckpointTimer = void 0;
2463
+ if (options2.requestId) explicitWorkspaceCheckpoints.consumeScheduled(options2.requestId);
2464
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2465
+ if (!workspaceReady || !workspaceRemoteUrl) {
2466
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2467
+ return;
2468
+ }
2469
+ if (activeWorkspaceIncidentId) {
2470
+ workspaceConvergence.block();
2471
+ sendWorkspaceReady();
2472
+ if (workspaceCheckpointIncidentAction(options2.requestId) === "defer_automatic") return;
2473
+ if (pendingCheckpoint) {
2474
+ explicitWorkspaceCheckpoints.enqueue({ requestId: options2.requestId, trigger }, pendingCheckpoint.requestId);
2475
+ return;
2476
+ }
2477
+ const snapshot2 = workspaceConvergence.snapshot();
2478
+ pendingCheckpoint = {
2479
+ requestId: options2.requestId,
2403
2480
  trigger,
2404
- ...fingerprint ? { fingerprint } : {}
2481
+ dirtyGeneration: snapshot2.dirtyGeneration,
2482
+ explicit: true,
2483
+ admissionHeld: false,
2484
+ workInFlight: false,
2485
+ requestedAt: Date.now(),
2486
+ releaseCheckpointAdmission: () => {
2487
+ }
2488
+ };
2489
+ try {
2490
+ ws.send(
2491
+ JSON.stringify({
2492
+ type: "workspace_checkpoint_request",
2493
+ requestId: options2.requestId,
2494
+ trigger,
2495
+ dirtyGeneration: snapshot2.dirtyGeneration,
2496
+ localHead: snapshot2.localHead,
2497
+ desiredCanonicalHead: snapshot2.desiredCanonicalHead
2498
+ })
2499
+ );
2500
+ } catch (error) {
2501
+ pendingCheckpoint = void 0;
2502
+ workspaceConvergence.defer();
2503
+ process.stderr.write(
2504
+ `[r5d-worker] failed to request blocked workspace checkpoint authority: ${error instanceof Error ? error.message : String(error)}
2505
+ `
2506
+ );
2507
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, requestId: options2.requestId });
2508
+ }
2509
+ return;
2510
+ }
2511
+ if (pendingCheckpoint || workerHasActiveWriter()) {
2512
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2513
+ return;
2514
+ }
2515
+ const releaseCheckpointAdmission = workspaceFilesystemWriters.tryAcquireCheckpoint();
2516
+ if (!releaseCheckpointAdmission) {
2517
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2518
+ return;
2519
+ }
2520
+ const snapshot = workspaceConvergence.snapshot();
2521
+ const requestId = options2.requestId ?? crypto.randomUUID();
2522
+ pendingCheckpoint = {
2523
+ requestId,
2524
+ trigger,
2525
+ dirtyGeneration: snapshot.dirtyGeneration,
2526
+ explicit: options2.requestId !== void 0,
2527
+ admissionHeld: true,
2528
+ workInFlight: false,
2529
+ requestedAt: Date.now(),
2530
+ releaseCheckpointAdmission
2531
+ };
2532
+ try {
2533
+ ws.send(
2534
+ JSON.stringify({
2535
+ type: "workspace_checkpoint_request",
2536
+ requestId,
2537
+ trigger,
2538
+ dirtyGeneration: snapshot.dirtyGeneration,
2539
+ localHead: snapshot.localHead,
2540
+ desiredCanonicalHead: snapshot.desiredCanonicalHead
2541
+ })
2542
+ );
2543
+ } catch (error) {
2544
+ releaseCheckpointAdmission();
2545
+ pendingCheckpoint = void 0;
2546
+ workspaceConvergence.defer();
2547
+ process.stderr.write(
2548
+ `[r5d-worker] failed to request workspace checkpoint: ${error instanceof Error ? error.message : String(error)}
2549
+ `
2550
+ );
2551
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2552
+ }
2553
+ }, Math.max(0, options2.delayMs));
2554
+ workspaceCheckpointTimer.unref();
2555
+ };
2556
+ const markWorkspaceDirty = (trigger, force = false) => {
2557
+ workspaceConvergence.markDirty();
2558
+ armWorkspaceCheckpoint(trigger, { delayMs: force ? 0 : WORKSPACE_CHECKPOINT_QUIET_MS });
2559
+ };
2560
+ const convergeAvailableWorkspaceHead = async () => {
2561
+ if (pendingWorkspaceHead === void 0 || !workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || workspaceHeadConvergenceInFlight) {
2562
+ return;
2563
+ }
2564
+ const advertisedHead = pendingWorkspaceHead;
2565
+ pendingWorkspaceHead = void 0;
2566
+ workspaceConvergence.observeCanonicalHead(advertisedHead);
2567
+ workspaceHeadConvergenceInFlight = true;
2568
+ let releaseVisibleHydration = null;
2569
+ try {
2570
+ const snapshot = workspaceConvergence.snapshot();
2571
+ const mayHydrateVisible = !workerHasActiveWriter() && !pendingCheckpoint && snapshot.dirtyGeneration === snapshot.publishedGeneration;
2572
+ releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
2573
+ const hydrateVisible = releaseVisibleHydration !== null;
2574
+ const converged = await workspaceSyncSingleFlight.runExclusive(
2575
+ () => convergeWorkspaceHead(workspaceSyncInput({ type: "inbound_head", detail: advertisedHead ?? "unborn" }), {
2576
+ hydrateVisible
2405
2577
  })
2406
2578
  );
2579
+ workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
2580
+ if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
2581
+ if (hydrateVisible && converged.localChanges) {
2582
+ markWorkspaceDirty({ type: "periodic", detail: "inbound overlap retained for next checkpoint" });
2583
+ }
2584
+ sendWorkspaceReady();
2407
2585
  } catch (error) {
2586
+ pendingWorkspaceHead = advertisedHead;
2408
2587
  process.stderr.write(
2409
- `[r5d-worker] failed to send workspace sync observation on its current connection: ${error instanceof Error ? error.message : String(error)}
2588
+ `[r5d-worker] failed to fetch available workspace head: ${error instanceof Error ? error.message : String(error)}
2410
2589
  `
2411
2590
  );
2591
+ } finally {
2592
+ releaseVisibleHydration?.();
2593
+ workspaceHeadConvergenceInFlight = false;
2594
+ heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
2595
+ if (pendingWorkspaceHead !== void 0) {
2596
+ setTimeout(() => void convergeAvailableWorkspaceHead(), 1e3).unref();
2597
+ }
2412
2598
  }
2413
2599
  };
2414
2600
  const manifestCheckoutKey = (projectId, branchName) => `${projectId}\0${branchName}`;
@@ -2460,19 +2646,73 @@ async function startWorker(options) {
2460
2646
  }
2461
2647
  return created;
2462
2648
  };
2649
+ const schedulePendingManifestCheckoutHydration = (delayMs = 0) => {
2650
+ if (workspaceManifestHydrationTimer || workspaceManifestHydrationInFlight || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2651
+ return;
2652
+ }
2653
+ workspaceManifestHydrationTimer = setTimeout(() => {
2654
+ workspaceManifestHydrationTimer = void 0;
2655
+ if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2656
+ if (pendingManifestCheckouts.size > 0 && currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
2657
+ schedulePendingManifestCheckoutHydration(1e3);
2658
+ }
2659
+ return;
2660
+ }
2661
+ const releaseHydration = workspaceFilesystemWriters.tryAcquireCheckpoint();
2662
+ if (!releaseHydration) {
2663
+ schedulePendingManifestCheckoutHydration(1e3);
2664
+ return;
2665
+ }
2666
+ workspaceManifestHydrationInFlight = true;
2667
+ void workspaceSyncSingleFlight.runExclusive(async () => {
2668
+ if (!workspaceReady || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2669
+ return;
2670
+ }
2671
+ const targets = [...pendingManifestCheckouts.values()];
2672
+ ensureVisibleWorkspaceCheckouts(targets, { strictCanonicalRevalidation: true });
2673
+ for (const target of targets) {
2674
+ const manifest = manifestByProjectId.get(target.projectId);
2675
+ if (!manifest || !manifest.branches.includes(target.branchName)) continue;
2676
+ const branchPath = path.join(projectRootFor(projectsRoot, target.projectId, manifestByProjectId), target.branchName);
2677
+ if (hasNormalVisibleGitDir(branchPath)) {
2678
+ pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
2679
+ }
2680
+ }
2681
+ }).then(() => sendWorkspaceReady()).catch((error) => {
2682
+ process.stderr.write(
2683
+ `[r5d-worker] pending manifest checkout hydration failed: ${error instanceof Error ? error.message : String(error)}
2684
+ `
2685
+ );
2686
+ }).finally(() => {
2687
+ releaseHydration();
2688
+ workspaceManifestHydrationInFlight = false;
2689
+ heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
2690
+ if (pendingManifestCheckouts.size > 0) schedulePendingManifestCheckoutHydration(1e3);
2691
+ });
2692
+ }, Math.max(0, delayMs));
2693
+ workspaceManifestHydrationTimer.unref();
2694
+ };
2463
2695
  const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}) => {
2696
+ const requestedAt = Date.now();
2697
+ let queueEnteredAt = requestedAt;
2698
+ let prepareStartedAt = requestedAt;
2699
+ let synchronizeStartedAt = requestedAt;
2700
+ let synchronizeFinishedAt = requestedAt;
2464
2701
  workspaceSyncRequestsInFlight += 1;
2465
2702
  try {
2466
2703
  let pendingTargets = [];
2467
- const result = await workspaceSyncSingleFlight.runPrepared(() => {
2704
+ const result = await workspaceSyncSingleFlight.runExclusive(async () => {
2705
+ queueEnteredAt = Date.now();
2468
2706
  const shouldEnsureCheckouts = !overrides.skipVisibleMirror && !overrides.resetToCanonical;
2469
2707
  const syncProjects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
2708
+ if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
2470
2709
  const allowedCheckoutKeys = new Set(
2471
2710
  syncProjects.flatMap((project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName)))
2472
2711
  );
2473
2712
  const scopedPendingTargets = [...pendingManifestCheckouts.values()].filter(
2474
2713
  (target) => allowedCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
2475
2714
  );
2715
+ prepareStartedAt = Date.now();
2476
2716
  let checkoutTargets = [];
2477
2717
  if (shouldEnsureCheckouts) {
2478
2718
  checkoutTargets = workspaceSyncCheckoutTargets({
@@ -2492,21 +2732,39 @@ async function startWorker(options) {
2492
2732
  if (allowedCheckoutKeys.has(key)) newVisibleCheckoutByKey.set(key, target);
2493
2733
  }
2494
2734
  const newVisibleCheckouts = [...newVisibleCheckoutByKey.values()];
2495
- return workspaceSyncInput(trigger, {
2735
+ const input = workspaceSyncInput(trigger, {
2496
2736
  ...overrides,
2497
2737
  ...newVisibleCheckouts.length > 0 ? { newVisibleCheckouts } : {}
2498
2738
  });
2739
+ synchronizeStartedAt = Date.now();
2740
+ let synchronized;
2741
+ try {
2742
+ synchronized = await synchronizeWorkspace(input);
2743
+ } finally {
2744
+ synchronizeFinishedAt = Date.now();
2745
+ }
2746
+ return synchronized;
2499
2747
  });
2500
- if (result.outcome === "no_change" || result.outcome === "published" || result.outcome === "updated" || result.outcome === "conflict_reset") {
2748
+ const completedAt = Date.now();
2749
+ const resultWithTelemetry = {
2750
+ ...result,
2751
+ telemetry: {
2752
+ totalMs: completedAt - requestedAt,
2753
+ queueMs: queueEnteredAt - requestedAt,
2754
+ prepareMs: synchronizeStartedAt - prepareStartedAt,
2755
+ synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
2756
+ }
2757
+ };
2758
+ if (resultWithTelemetry.outcome === "no_change" || resultWithTelemetry.outcome === "published" || resultWithTelemetry.outcome === "updated" || resultWithTelemetry.outcome === "conflict_reset") {
2501
2759
  for (const target of pendingTargets) {
2502
2760
  const key = manifestCheckoutKey(target.projectId, target.branchName);
2503
2761
  if (pendingManifestCheckouts.get(key) === target) pendingManifestCheckouts.delete(key);
2504
2762
  }
2505
2763
  }
2506
- previousPeriodicFingerprint = null;
2507
- sendWorkspaceSyncResult(authority, result);
2508
- return result;
2764
+ sendWorkspaceSyncResult(authority, resultWithTelemetry);
2765
+ return resultWithTelemetry;
2509
2766
  } catch (error) {
2767
+ const completedAt = Date.now();
2510
2768
  const result = {
2511
2769
  type: "workspace_sync",
2512
2770
  attemptId: overrides.attemptId ?? crypto.randomUUID(),
@@ -2521,9 +2779,14 @@ async function startWorker(options) {
2521
2779
  affectedPaths: [],
2522
2780
  discardedPaths: [],
2523
2781
  localChangesDiscarded: false,
2782
+ telemetry: {
2783
+ totalMs: completedAt - requestedAt,
2784
+ queueMs: queueEnteredAt - requestedAt,
2785
+ prepareMs: Math.max(0, synchronizeStartedAt - prepareStartedAt),
2786
+ synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
2787
+ },
2524
2788
  error: error instanceof Error ? error.message : String(error)
2525
2789
  };
2526
- previousPeriodicFingerprint = null;
2527
2790
  sendWorkspaceSyncResult(authority, result);
2528
2791
  return result;
2529
2792
  } finally {
@@ -2539,6 +2802,24 @@ async function startWorker(options) {
2539
2802
  let heartbeatBusyGrace = null;
2540
2803
  let heartbeatTimedOut = false;
2541
2804
  let heartbeatWatchdog;
2805
+ const handleGracefulShutdown = () => {
2806
+ if (shutdownAfterClose) return;
2807
+ shutdownAfterClose = true;
2808
+ void waitForWorkspaceCheckpointOnShutdown({
2809
+ snapshot: () => workspaceConvergence.snapshot(),
2810
+ forceCheckpoint: () => armWorkspaceCheckpoint({ type: "manual", detail: "graceful worker shutdown" }, { delayMs: 0 })
2811
+ }).then((outcome) => {
2812
+ process.stdout.write(`[r5d-worker] graceful workspace checkpoint ${outcome}
2813
+ `);
2814
+ if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
2815
+ ws.close(1e3, "Worker shutting down");
2816
+ } else {
2817
+ process.exit(0);
2818
+ }
2819
+ });
2820
+ };
2821
+ process.on("SIGTERM", handleGracefulShutdown);
2822
+ process.on("SIGINT", handleGracefulShutdown);
2542
2823
  const stopHeartbeatWatchdog = () => {
2543
2824
  if (heartbeatWatchdog) {
2544
2825
  clearInterval(heartbeatWatchdog);
@@ -2569,7 +2850,8 @@ async function startWorker(options) {
2569
2850
  updateClis: true,
2570
2851
  canonicalResolverCheckout: true,
2571
2852
  browserPortForwarding: true,
2572
- globalWorkspaceSyncLeaseV1: true
2853
+ globalWorkspaceSyncLeaseV1: true,
2854
+ eventualWorkspaceSyncV1: true
2573
2855
  },
2574
2856
  projectRoot: projectsRoot,
2575
2857
  artifactRoot,
@@ -2596,79 +2878,123 @@ async function startWorker(options) {
2596
2878
  return;
2597
2879
  }
2598
2880
  if (message.type === "workspace_manifest") {
2599
- await workspaceSyncSingleFlight.runMutation(() => {
2600
- configureGitHubAuth(message.githubCredential);
2601
- visibleGitIdentity = message.gitIdentity;
2602
- workspaceRemoteUrl = message.workspaceRemoteUrl;
2603
- const previousCheckoutKeys = new Set(
2604
- allManifestCheckouts().map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2605
- );
2606
- const migratedProjectIds = migrateLegacyProjectRoots(projectsRoot, message.projects);
2607
- manifestByProjectId.clear();
2608
- for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
2609
- const currentCheckouts = allManifestCheckouts();
2610
- const currentCheckoutKeys = new Set(
2611
- currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2612
- );
2613
- for (const key of pendingManifestCheckouts.keys()) {
2614
- if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
2615
- }
2616
- for (const checkout of currentCheckouts) {
2617
- const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
2618
- if (previousCheckoutKeys.has(key)) continue;
2619
- const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
2620
- if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
2621
- }
2622
- previousPeriodicFingerprint = null;
2623
- process.stdout.write(
2624
- `[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
2625
- `
2626
- );
2881
+ const revisionToken = workspaceManifestRevision.begin(message.revision);
2882
+ workspaceReady = false;
2883
+ const manifestAdmission = inspectWorkspaceManifestAdmission({
2884
+ projectsRoot,
2885
+ workspaceShadowRoot,
2886
+ projects: message.projects
2627
2887
  });
2628
- if (!periodicWorkspaceScan) {
2629
- const emptyFingerprint = createHash("sha256").update("").digest("hex");
2630
- periodicWorkspaceScan = setInterval(() => {
2631
- if (!workspaceRemoteUrl || activeWorkspaceIncidentId || cliUpdateInProgress || periodicWorkspaceScanInFlight) return;
2632
- periodicWorkspaceScanInFlight = true;
2633
- void (async () => {
2634
- const genericCheckoutKeys = new Set(
2635
- workspaceProjectsForSync([...manifestByProjectId.values()], { type: "periodic" }).flatMap(
2636
- (project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName))
2637
- )
2638
- );
2639
- const hasPendingGenericCheckout = [...pendingManifestCheckouts.values()].some(
2640
- (target) => genericCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
2888
+ const releaseManifestHydration = await acquireWorkspaceManifestVisibleLease(
2889
+ workspaceFilesystemWriters,
2890
+ manifestAdmission
2891
+ );
2892
+ let completed = false;
2893
+ try {
2894
+ completed = await workspaceSyncSingleFlight.runExclusive(async () => {
2895
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2896
+ const firstBootstrap = manifestAdmission.firstBootstrap;
2897
+ configureGitHubAuth(message.githubCredential);
2898
+ visibleGitIdentity = message.gitIdentity;
2899
+ workspaceRemoteUrl = message.workspaceRemoteUrl;
2900
+ const migratedProjectIds = releaseManifestHydration ? migrateLegacyProjectRoots(projectsRoot, message.projects) : [];
2901
+ manifestByProjectId.clear();
2902
+ for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
2903
+ const currentCheckouts = allManifestCheckouts();
2904
+ const currentCheckoutKeys = new Set(
2905
+ currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2906
+ );
2907
+ for (const key of pendingManifestCheckouts.keys()) {
2908
+ if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
2909
+ }
2910
+ for (const checkout of currentCheckouts) {
2911
+ const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
2912
+ const branchPath = path.join(
2913
+ projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId),
2914
+ checkout.branchName
2641
2915
  );
2642
- if (hasPendingGenericCheckout) {
2643
- previousPeriodicFingerprint = null;
2644
- requestWorkspaceSync({
2645
- type: "periodic",
2646
- detail: "workspace manifest additions"
2647
- });
2648
- return;
2916
+ if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
2917
+ }
2918
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2919
+ const created = releaseManifestHydration ? ensureVisibleWorkspaceCheckouts(currentCheckouts, { strictCanonicalRevalidation: true }) : [];
2920
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2921
+ const bootstrapped = await convergeWorkspaceHead(
2922
+ workspaceSyncInput({ type: "connect" }, { newVisibleCheckouts: created }),
2923
+ {
2924
+ // An existing checkout becomes routable after a fetch. Its
2925
+ // visible tree converges asynchronously; first bootstrap alone
2926
+ // must materialize the canonical snapshot before readiness.
2927
+ // Re-check the incident immediately before convergence so a
2928
+ // concurrently delivered fence forces fetch-only behavior.
2929
+ hydrateVisible: firstBootstrap && !activeWorkspaceIncidentId
2649
2930
  }
2650
- const fingerprint = await workspaceSyncSingleFlight.fingerprintPrepared(() => workspaceSyncInput({ type: "periodic" }));
2651
- if (fingerprint === emptyFingerprint) {
2652
- previousPeriodicFingerprint = null;
2653
- return;
2931
+ );
2932
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2933
+ workspaceConvergence.setLocalHead(bootstrapped.localHead);
2934
+ workspaceConvergence.observeCanonicalHead(bootstrapped.canonicalHead);
2935
+ for (const checkout of currentCheckouts) {
2936
+ const manifest = manifestByProjectId.get(checkout.projectId);
2937
+ if (!manifest) continue;
2938
+ const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
2939
+ if (hasNormalVisibleGitDir(branchPath)) {
2940
+ pendingManifestCheckouts.delete(manifestCheckoutKey(checkout.projectId, checkout.branchName));
2654
2941
  }
2655
- if (previousPeriodicFingerprint !== fingerprint) {
2656
- previousPeriodicFingerprint = fingerprint;
2657
- return;
2942
+ }
2943
+ process.stdout.write(
2944
+ `[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
2945
+ `
2946
+ );
2947
+ return true;
2948
+ });
2949
+ } finally {
2950
+ releaseManifestHydration?.();
2951
+ }
2952
+ if (!completed || !workspaceManifestRevision.complete(revisionToken)) return;
2953
+ workspaceReady = true;
2954
+ sendWorkspaceReady();
2955
+ schedulePendingManifestCheckoutHydration();
2956
+ if (!workspaceSafetyScan) {
2957
+ workspaceSafetyScan = setInterval(() => {
2958
+ if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || workspaceSafetyScanInFlight) {
2959
+ return;
2960
+ }
2961
+ workspaceSafetyScanInFlight = true;
2962
+ void (async () => {
2963
+ const before = workspaceConvergence.snapshot();
2964
+ const mayHydrateVisible = !workerHasActiveWriter() && before.dirtyGeneration === before.publishedGeneration;
2965
+ const releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
2966
+ const hydrateVisible = releaseVisibleHydration !== null;
2967
+ try {
2968
+ const converged = await workspaceSyncSingleFlight.runExclusive(
2969
+ () => convergeWorkspaceHead(workspaceSyncInput({ type: "periodic", detail: "idle convergence scan" }), {
2970
+ hydrateVisible
2971
+ })
2972
+ );
2973
+ workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
2974
+ if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
2975
+ if (converged.localChanges && hydrateVisible) {
2976
+ markWorkspaceDirty({ type: "periodic", detail: "idle safety scan" });
2977
+ }
2978
+ sendWorkspaceReady();
2979
+ } finally {
2980
+ releaseVisibleHydration?.();
2658
2981
  }
2659
- previousPeriodicFingerprint = null;
2660
- requestWorkspaceSync({ type: "periodic" }, fingerprint);
2661
2982
  })().catch((error) => {
2662
2983
  process.stderr.write(
2663
- `[r5d-worker] periodic workspace synchronization failed: ${error instanceof Error ? error.message : String(error)}
2984
+ `[r5d-worker] idle workspace safety scan failed: ${error instanceof Error ? error.message : String(error)}
2664
2985
  `
2665
2986
  );
2666
2987
  }).finally(() => {
2667
- periodicWorkspaceScanInFlight = false;
2988
+ workspaceSafetyScanInFlight = false;
2668
2989
  heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
2669
2990
  });
2670
- }, WORKSPACE_PERIODIC_SCAN_INTERVAL_MS);
2671
- periodicWorkspaceScan.unref();
2991
+ }, WORKSPACE_IDLE_SAFETY_SCAN_MS);
2992
+ workspaceSafetyScan.unref();
2993
+ }
2994
+ pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
2995
+ void convergeAvailableWorkspaceHead();
2996
+ if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
2997
+ armWorkspaceCheckpoint({ type: "periodic", detail: "resume dirty generation after reconnect" }, { delayMs: 0 });
2672
2998
  }
2673
2999
  return;
2674
3000
  }
@@ -2692,6 +3018,185 @@ async function startWorker(options) {
2692
3018
  );
2693
3019
  return;
2694
3020
  }
3021
+ if (message.type === "workspace_head_available") {
3022
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3023
+ pendingWorkspaceHead = message.canonicalHead;
3024
+ sendWorkspaceReady();
3025
+ void convergeAvailableWorkspaceHead();
3026
+ return;
3027
+ }
3028
+ if (message.type === "workspace_checkpoint_now") {
3029
+ armWorkspaceCheckpoint(message.trigger, { delayMs: 0, requestId: message.requestId });
3030
+ return;
3031
+ }
3032
+ if (message.type === "workspace_checkpoint_authority") {
3033
+ const checkpoint = pendingCheckpoint;
3034
+ if (!checkpoint || checkpoint.requestId !== message.requestId) {
3035
+ process.stderr.write(`[r5d-worker] ignoring checkpoint authority for unknown request ${message.requestId}
3036
+ `);
3037
+ return;
3038
+ }
3039
+ if (message.status !== "granted") {
3040
+ checkpoint.releaseCheckpointAdmission();
3041
+ pendingCheckpoint = void 0;
3042
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3043
+ const applyBlockedResponse = message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId);
3044
+ if (applyBlockedResponse) {
3045
+ workspaceConvergence.block();
3046
+ if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
3047
+ } else {
3048
+ workspaceConvergence.resetTransientCheckpointState();
3049
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3050
+ if (message.status === "busy") {
3051
+ armWorkspaceCheckpoint(checkpoint.trigger, {
3052
+ delayMs: message.retryAfterMs ?? 1e3,
3053
+ requestId: checkpoint.requestId
3054
+ });
3055
+ } else if (!activeWorkspaceIncidentId) {
3056
+ const explicit = explicitWorkspaceCheckpoints.next();
3057
+ if (explicit) {
3058
+ armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
3059
+ } else if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3060
+ armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: 0 });
3061
+ }
3062
+ }
3063
+ }
3064
+ sendWorkspaceReady();
3065
+ return;
3066
+ }
3067
+ checkpoint.attemptId = message.attemptId;
3068
+ checkpoint.authorityToken = message.authorityToken;
3069
+ checkpoint.workInFlight = true;
3070
+ pendingCheckpoint = checkpoint;
3071
+ if (!checkpoint.admissionHeld) {
3072
+ const releaseCheckpointAdmission = await workspaceFilesystemWriters.acquireCheckpoint();
3073
+ if (currentWorkerSocket !== ws || pendingCheckpoint?.requestId !== checkpoint.requestId || pendingCheckpoint.attemptId !== message.attemptId) {
3074
+ releaseCheckpointAdmission();
3075
+ checkpoint.workInFlight = false;
3076
+ workspaceConvergence.resetTransientCheckpointState();
3077
+ return;
3078
+ }
3079
+ checkpoint.admissionHeld = true;
3080
+ checkpoint.releaseCheckpointAdmission = releaseCheckpointAdmission;
3081
+ }
3082
+ workspaceConvergence.beginPreparing();
3083
+ sendWorkspaceReady();
3084
+ let prepareStartedAt = Date.now();
3085
+ let synchronizeStartedAt = prepareStartedAt;
3086
+ let synchronizeStarted = false;
3087
+ let result;
3088
+ try {
3089
+ result = await workspaceSyncSingleFlight.runExclusive(async () => {
3090
+ prepareStartedAt = Date.now();
3091
+ const pendingTargets = [...pendingManifestCheckouts.values()];
3092
+ const createdCheckouts = ensureVisibleWorkspaceCheckouts(pendingTargets, { strictCanonicalRevalidation: true });
3093
+ synchronizeStartedAt = Date.now();
3094
+ synchronizeStarted = true;
3095
+ const prepared = await synchronizeWorkspace(
3096
+ workspaceSyncInput(checkpoint.trigger, {
3097
+ attemptId: message.attemptId,
3098
+ quarantineRef: message.quarantineRef,
3099
+ expectedCanonicalHead: message.expectedHead,
3100
+ newVisibleCheckouts: [...createdCheckouts, ...pendingTargets]
3101
+ })
3102
+ );
3103
+ for (const target of pendingTargets) {
3104
+ pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
3105
+ }
3106
+ return prepared.outcome === "updated" ? { ...prepared, outcome: "no_change" } : prepared;
3107
+ });
3108
+ } catch (error) {
3109
+ if (!synchronizeStarted) synchronizeStartedAt = Date.now();
3110
+ result = {
3111
+ type: "workspace_sync",
3112
+ attemptId: message.attemptId,
3113
+ workerLabel: label,
3114
+ trigger: checkpoint.trigger,
3115
+ outcome: "failed",
3116
+ startingHead: message.expectedHead,
3117
+ expectedHead: message.expectedHead,
3118
+ rebaseCount: 0,
3119
+ diffSizeBytes: 0,
3120
+ gitStatus: "",
3121
+ affectedProjects: [],
3122
+ affectedPaths: [],
3123
+ discardedPaths: [],
3124
+ localChangesDiscarded: false,
3125
+ error: error instanceof Error ? error.message : String(error)
3126
+ };
3127
+ }
3128
+ const finishedAt = Date.now();
3129
+ result = {
3130
+ ...result,
3131
+ dirtyGeneration: checkpoint.dirtyGeneration,
3132
+ telemetry: workspaceCheckpointTelemetry({
3133
+ requestedAt: checkpoint.requestedAt,
3134
+ prepareStartedAt,
3135
+ synchronizeStartedAt,
3136
+ finishedAt
3137
+ })
3138
+ };
3139
+ workspaceConvergence.beginSubmitting();
3140
+ sendWorkspaceReady();
3141
+ try {
3142
+ ws.send(
3143
+ JSON.stringify({
3144
+ type: "workspace_checkpoint_result",
3145
+ requestId: checkpoint.requestId,
3146
+ attemptId: message.attemptId,
3147
+ authorityToken: message.authorityToken,
3148
+ dirtyGeneration: checkpoint.dirtyGeneration,
3149
+ result
3150
+ })
3151
+ );
3152
+ } catch (error) {
3153
+ workspaceConvergence.resetTransientCheckpointState();
3154
+ throw error;
3155
+ } finally {
3156
+ checkpoint.workInFlight = false;
3157
+ checkpoint.releaseCheckpointAdmission();
3158
+ if (currentWorkerSocket !== ws) workspaceConvergence.resetTransientCheckpointState();
3159
+ }
3160
+ return;
3161
+ }
3162
+ if (message.type === "workspace_checkpoint_completed") {
3163
+ const checkpoint = pendingCheckpoint;
3164
+ if (!checkpoint || checkpoint.requestId !== message.requestId || checkpoint.attemptId !== message.attemptId) {
3165
+ process.stderr.write(`[r5d-worker] ignoring completion for unknown checkpoint ${message.requestId}
3166
+ `);
3167
+ return;
3168
+ }
3169
+ pendingCheckpoint = void 0;
3170
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3171
+ if (message.status === "published" || message.status === "no_change") {
3172
+ workspaceConvergence.complete({
3173
+ generation: Math.min(message.publishedGeneration ?? checkpoint.dirtyGeneration, checkpoint.dirtyGeneration),
3174
+ canonicalHead: message.canonicalHead,
3175
+ published: true
3176
+ });
3177
+ } else if (message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId)) {
3178
+ workspaceConvergence.block();
3179
+ if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
3180
+ } else {
3181
+ workspaceConvergence.resetTransientCheckpointState();
3182
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3183
+ }
3184
+ sendWorkspaceReady();
3185
+ const explicit = message.status === "retry" && checkpoint.explicit ? { requestId: checkpoint.requestId, trigger: checkpoint.trigger } : explicitWorkspaceCheckpoints.next();
3186
+ if (explicit) {
3187
+ armWorkspaceCheckpoint(explicit.trigger, {
3188
+ delayMs: message.retryAfterMs ?? 0,
3189
+ requestId: explicit.requestId
3190
+ });
3191
+ } else if (message.status === "retry" || message.status === "failed" || workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3192
+ armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: message.retryAfterMs ?? 1e3 });
3193
+ }
3194
+ if (message.canonicalHead !== workspaceConvergence.snapshot().localHead) {
3195
+ pendingWorkspaceHead = message.canonicalHead;
3196
+ void convergeAvailableWorkspaceHead();
3197
+ }
3198
+ return;
3199
+ }
2695
3200
  if (message.type === "sync_session_artifacts") {
2696
3201
  sessionArtifactSyncRequestsInFlight += 1;
2697
3202
  try {
@@ -2719,17 +3224,27 @@ async function startWorker(options) {
2719
3224
  return;
2720
3225
  }
2721
3226
  if (message.type === "workspace_incident_updated") {
3227
+ workspaceIncidentOrdering.observe(message);
2722
3228
  const previousIncidentId = activeWorkspaceIncidentId;
2723
3229
  activeWorkspaceIncidentId = applyWorkspaceIncidentUpdate(activeWorkspaceIncidentId, message);
2724
- previousPeriodicFingerprint = null;
2725
3230
  const releasedHead = releasePendingWorkspaceHead(
2726
3231
  previousIncidentId,
2727
3232
  activeWorkspaceIncidentId,
2728
3233
  message.status === "resolved" || message.status === "confirmed" || message.status === "reset" ? message.incidentId : null
2729
3234
  );
2730
- if (releasedHead.syncHead) {
2731
- requestWorkspaceSync({ type: "inbound_head", detail: releasedHead.syncHead });
3235
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3236
+ if (releasedHead.fetchAuthoritativeHead) {
3237
+ pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
3238
+ void convergeAvailableWorkspaceHead();
3239
+ }
3240
+ const explicit = !activeWorkspaceIncidentId ? explicitWorkspaceCheckpoints.next() : void 0;
3241
+ if (explicit) {
3242
+ armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
3243
+ } else if (!activeWorkspaceIncidentId && workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3244
+ armWorkspaceCheckpoint({ type: "periodic", detail: "resume after workspace incident" }, { delayMs: 0 });
2732
3245
  }
3246
+ if (!activeWorkspaceIncidentId) schedulePendingManifestCheckoutHydration();
3247
+ sendWorkspaceReady();
2733
3248
  return;
2734
3249
  }
2735
3250
  if (message.type === "exec_terminal_ack") {
@@ -2827,8 +3342,10 @@ async function startWorker(options) {
2827
3342
  });
2828
3343
  return;
2829
3344
  }
3345
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
2830
3346
  const releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
2831
- let leaseTransferred = false;
3347
+ let mutationLeaseTransferred = false;
3348
+ let writerLeaseTransferred = false;
2832
3349
  try {
2833
3350
  const resolvedTarget = resolveMessageTarget(message.target);
2834
3351
  process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
@@ -2838,9 +3355,19 @@ async function startWorker(options) {
2838
3355
  message,
2839
3356
  resolvedTarget,
2840
3357
  planRoot,
2841
- ...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {}
3358
+ ...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {},
3359
+ ...targetMayMutateVisibleWorkspace(message.target) ? {
3360
+ onTerminal: () => {
3361
+ markWorkspaceDirty(
3362
+ { type: "process_terminal", detail: `pty ${message.ptyId} completed` },
3363
+ true
3364
+ );
3365
+ releaseWorkspaceWriter?.();
3366
+ }
3367
+ } : {}
2842
3368
  });
2843
- leaseTransferred = releaseWorkspaceMutation !== void 0;
3369
+ mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
3370
+ writerLeaseTransferred = releaseWorkspaceWriter !== void 0;
2844
3371
  } catch (error) {
2845
3372
  sendWorkerMessage(ws, {
2846
3373
  type: "pty_error",
@@ -2849,11 +3376,16 @@ async function startWorker(options) {
2849
3376
  error: error instanceof Error ? error.message : String(error)
2850
3377
  });
2851
3378
  } finally {
2852
- if (!leaseTransferred) releaseWorkspaceMutation?.();
3379
+ if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
3380
+ if (!writerLeaseTransferred) releaseWorkspaceWriter?.();
2853
3381
  }
2854
3382
  return;
2855
3383
  }
2856
3384
  if (message.type === "pty_input") {
3385
+ const activePty = activePtys.get(message.ptyId);
3386
+ if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
3387
+ markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` });
3388
+ }
2857
3389
  writePty(ws, message);
2858
3390
  return;
2859
3391
  }
@@ -2862,7 +3394,11 @@ async function startWorker(options) {
2862
3394
  return;
2863
3395
  }
2864
3396
  if (message.type === "pty_close") {
3397
+ const activePty = activePtys.get(message.ptyId);
2865
3398
  closePty(message);
3399
+ if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
3400
+ armWorkspaceCheckpoint({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, { delayMs: 0 });
3401
+ }
2866
3402
  return;
2867
3403
  }
2868
3404
  if (message.type === "exec") {
@@ -2876,26 +3412,37 @@ async function startWorker(options) {
2876
3412
  });
2877
3413
  return;
2878
3414
  }
2879
- let result;
3415
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
2880
3416
  try {
2881
- const runCommand = async () => {
2882
- const resolvedTarget = resolveMessageTarget(message.target);
2883
- process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
3417
+ let result;
3418
+ try {
3419
+ const runCommand = async () => {
3420
+ const resolvedTarget = resolveMessageTarget(message.target);
3421
+ process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
2884
3422
  `);
2885
- return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
2886
- };
2887
- result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
2888
- } catch (error) {
2889
- result = {
2890
- type: "exec_result",
2891
- requestId: message.requestId,
2892
- stdout: "",
2893
- stderr: "",
2894
- exitCode: 1,
2895
- error: error instanceof Error ? error.message : String(error)
2896
- };
3423
+ return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
3424
+ };
3425
+ result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
3426
+ } catch (error) {
3427
+ result = {
3428
+ type: "exec_result",
3429
+ requestId: message.requestId,
3430
+ stdout: "",
3431
+ stderr: "",
3432
+ exitCode: 1,
3433
+ error: error instanceof Error ? error.message : String(error)
3434
+ };
3435
+ }
3436
+ ws.send(JSON.stringify(result));
3437
+ if (targetMayMutateVisibleWorkspace(message.target)) {
3438
+ markWorkspaceDirty(
3439
+ { type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
3440
+ true
3441
+ );
3442
+ }
3443
+ } finally {
3444
+ releaseWorkspaceWriter?.();
2897
3445
  }
2898
- ws.send(JSON.stringify(result));
2899
3446
  return;
2900
3447
  }
2901
3448
  if (message.type === "exec_start") {
@@ -2914,18 +3461,34 @@ async function startWorker(options) {
2914
3461
  runId: message.runId
2915
3462
  });
2916
3463
  const runCommand = async () => {
2917
- const resolvedTarget = resolveMessageTarget(message.target);
2918
- process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
3464
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3465
+ try {
3466
+ const resolvedTarget = resolveMessageTarget(message.target);
3467
+ process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
2919
3468
  `);
2920
- await executeStreamingCommand({
2921
- ws,
2922
- message,
2923
- resolvedTarget,
2924
- baseUrl,
2925
- token,
2926
- artifactRoot,
2927
- planRoot
2928
- });
3469
+ await executeStreamingCommand({
3470
+ ws,
3471
+ message,
3472
+ resolvedTarget,
3473
+ baseUrl,
3474
+ token,
3475
+ artifactRoot,
3476
+ planRoot
3477
+ });
3478
+ } finally {
3479
+ if (targetMayMutateVisibleWorkspace(message.target)) {
3480
+ markWorkspaceDirty(
3481
+ {
3482
+ type: "process_terminal",
3483
+ sessionId: message.sessionId,
3484
+ processRunId: message.runId,
3485
+ detail: "process completed"
3486
+ },
3487
+ true
3488
+ );
3489
+ }
3490
+ releaseWorkspaceWriter?.();
3491
+ }
2929
3492
  };
2930
3493
  const execution = runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
2931
3494
  void execution.catch((error) => {
@@ -2939,33 +3502,46 @@ async function startWorker(options) {
2939
3502
  return;
2940
3503
  }
2941
3504
  if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes") {
3505
+ const releaseWorkspaceWriter = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
2942
3506
  try {
2943
- const result = await workspaceSyncSingleFlight.runMutation(async () => {
2944
- const resolvedTarget = resolveMessageTarget(message.target);
2945
- return await executeOperation({
2946
- message,
2947
- resolvedTarget,
2948
- baseUrl,
2949
- token,
2950
- artifactRoot,
2951
- planRoot
3507
+ try {
3508
+ const result = await workspaceSyncSingleFlight.runMutation(async () => {
3509
+ const resolvedTarget = resolveMessageTarget(message.target);
3510
+ return await executeOperation({
3511
+ message,
3512
+ resolvedTarget,
3513
+ baseUrl,
3514
+ token,
3515
+ artifactRoot,
3516
+ planRoot
3517
+ });
2952
3518
  });
2953
- });
2954
- ws.send(
2955
- JSON.stringify({
2956
- type: "operation_result",
2957
- requestId: message.requestId,
2958
- result
2959
- })
2960
- );
2961
- } catch (error) {
2962
- ws.send(
2963
- JSON.stringify({
2964
- type: "operation_result",
2965
- requestId: message.requestId,
2966
- error: error instanceof Error ? error.message : String(error)
2967
- })
2968
- );
3519
+ ws.send(
3520
+ JSON.stringify({
3521
+ type: "operation_result",
3522
+ requestId: message.requestId,
3523
+ result
3524
+ })
3525
+ );
3526
+ if ((message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target)) {
3527
+ markWorkspaceDirty({
3528
+ type: message.type,
3529
+ sessionId: message.sessionId,
3530
+ toolCallId: message.requestId,
3531
+ ...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
3532
+ });
3533
+ }
3534
+ } catch (error) {
3535
+ ws.send(
3536
+ JSON.stringify({
3537
+ type: "operation_result",
3538
+ requestId: message.requestId,
3539
+ error: error instanceof Error ? error.message : String(error)
3540
+ })
3541
+ );
3542
+ }
3543
+ } finally {
3544
+ releaseWorkspaceWriter?.();
2969
3545
  }
2970
3546
  return;
2971
3547
  }
@@ -2987,10 +3563,24 @@ async function startWorker(options) {
2987
3563
  });
2988
3564
  });
2989
3565
  ws.addEventListener("close", (event) => {
3566
+ process.off("SIGTERM", handleGracefulShutdown);
3567
+ process.off("SIGINT", handleGracefulShutdown);
2990
3568
  stopHeartbeatWatchdog();
2991
- if (periodicWorkspaceScan) {
2992
- clearInterval(periodicWorkspaceScan);
2993
- periodicWorkspaceScan = void 0;
3569
+ if (workspaceCheckpointTimer) {
3570
+ clearTimeout(workspaceCheckpointTimer);
3571
+ workspaceCheckpointTimer = void 0;
3572
+ }
3573
+ if (!pendingCheckpoint?.workInFlight) {
3574
+ pendingCheckpoint?.releaseCheckpointAdmission();
3575
+ workspaceConvergence.resetTransientCheckpointState();
3576
+ }
3577
+ if (workspaceSafetyScan) {
3578
+ clearInterval(workspaceSafetyScan);
3579
+ workspaceSafetyScan = void 0;
3580
+ }
3581
+ if (workspaceManifestHydrationTimer) {
3582
+ clearTimeout(workspaceManifestHydrationTimer);
3583
+ workspaceManifestHydrationTimer = void 0;
2994
3584
  }
2995
3585
  if (terminalReplayTimer) {
2996
3586
  clearInterval(terminalReplayTimer);
@@ -3006,6 +3596,9 @@ async function startWorker(options) {
3006
3596
  if (reloadAfterClose) {
3007
3597
  process.exit(WORKER_RELOAD_EXIT_CODE);
3008
3598
  }
3599
+ if (shutdownAfterClose) {
3600
+ process.exit(0);
3601
+ }
3009
3602
  if (activeProcesses.size > 0 || pendingProcessTerminals.size > 0) {
3010
3603
  process.stderr.write(
3011
3604
  `[r5d-worker] ${activeProcesses.size} process(es) active and ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms