@ricsam/r5d-worker 0.0.58 → 0.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cjs/main.cjs CHANGED
@@ -65,14 +65,12 @@ var import_process_tree = require("./process-tree.cjs");
65
65
  var import_port_forward_client = require("./port-forward-client.cjs");
66
66
  var import_workspace_incident_state = require("./workspace-incident-state.cjs");
67
67
  var import_workspace_command_sync_policy = require("./workspace-command-sync-policy.cjs");
68
- var import_workspace_sync_fast_path = require("./workspace-sync-fast-path.cjs");
68
+ var import_workspace_convergence = require("./workspace-convergence.cjs");
69
+ var import_workspace_manifest_admission = require("./workspace-manifest-admission.cjs");
69
70
  var import_supervisor = require("./supervisor.cjs");
70
71
  var import_managed_paths = require("./managed-paths.cjs");
71
72
  var import_workspace_sync = require("./workspace-sync.cjs");
72
73
  const import_meta = {};
73
- function workspaceSyncFastPathTelemetry(decision) {
74
- return decision.hit ? { hit: true, reason: decision.reason, proofAgeMs: decision.proofAgeMs } : { hit: false, reason: decision.reason };
75
- }
76
74
  class WorkerServerUnavailableError extends Error {
77
75
  name = "WorkerServerUnavailableError";
78
76
  }
@@ -88,6 +86,7 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
88
86
  const activePtys = /* @__PURE__ */ new Map();
89
87
  let currentWorkerSocket = null;
90
88
  const workspaceSyncSingleFlight = new import_workspace_sync.WorkspaceSyncSingleFlight();
89
+ const workspaceFilesystemWriters = (0, import_workspace_convergence.processWorkspaceFilesystemWriterGate)();
91
90
  let githubCredential = null;
92
91
  let visibleGitIdentity = null;
93
92
  function defaultConfigPath() {
@@ -2302,6 +2301,7 @@ async function openPty(input) {
2302
2301
  onExit: (event) => {
2303
2302
  input.releaseWorkspaceMutation?.();
2304
2303
  activePtys.delete(input.message.ptyId);
2304
+ input.onTerminal?.();
2305
2305
  sendWorkerMessage(input.ws, {
2306
2306
  type: "pty_exit",
2307
2307
  ptyId: input.message.ptyId,
@@ -2312,6 +2312,7 @@ async function openPty(input) {
2312
2312
  onError: (error) => {
2313
2313
  input.releaseWorkspaceMutation?.();
2314
2314
  activePtys.delete(input.message.ptyId);
2315
+ input.onTerminal?.();
2315
2316
  sendWorkerMessage(input.ws, {
2316
2317
  type: "pty_error",
2317
2318
  requestId: input.message.requestId,
@@ -2403,15 +2404,25 @@ async function startWorker(options) {
2403
2404
  const pendingManifestCheckouts = /* @__PURE__ */ new Map();
2404
2405
  let workspaceRemoteUrl = null;
2405
2406
  let activeWorkspaceIncidentId = null;
2406
- let previousPeriodicFingerprint = null;
2407
- let periodicWorkspaceScan;
2408
- let periodicWorkspaceScanInFlight = false;
2407
+ const workspaceIncidentOrdering = new import_workspace_incident_state.WorkspaceIncidentOrderingFence();
2408
+ const workspaceConvergence = (0, import_workspace_convergence.processWorkspaceConvergenceState)(import_node_path.default.join(syncRoot, "workspace-convergence.json"));
2409
+ const workspaceManifestRevision = new import_workspace_convergence.WorkspaceManifestRevisionFence();
2410
+ let workspaceReady = false;
2411
+ let workspaceCheckpointTimer;
2412
+ let workspaceSafetyScan;
2413
+ let workspaceManifestHydrationTimer;
2414
+ let workspaceManifestHydrationInFlight = false;
2415
+ let workspaceSafetyScanInFlight = false;
2416
+ let workspaceHeadConvergenceInFlight = false;
2417
+ let pendingWorkspaceHead;
2418
+ let pendingCheckpoint;
2419
+ const explicitWorkspaceCheckpoints = new import_workspace_convergence.ExplicitWorkspaceCheckpointQueue();
2409
2420
  let terminalReplayTimer;
2410
2421
  let workspaceSyncRequestsInFlight = 0;
2411
2422
  let sessionArtifactSyncRequestsInFlight = 0;
2412
2423
  let cliUpdateInProgress = false;
2413
2424
  let reloadAfterClose = false;
2414
- const recentWorkspaceSyncProof = new import_workspace_sync_fast_path.RecentWorkspaceSyncProofCache();
2425
+ let shutdownAfterClose = false;
2415
2426
  const workspaceSyncInput = (trigger, overrides = {}) => {
2416
2427
  if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
2417
2428
  const projects = (0, import_workspace_sync.workspaceProjectsForSync)([...manifestByProjectId.values()], trigger);
@@ -2437,21 +2448,175 @@ async function startWorker(options) {
2437
2448
  );
2438
2449
  }
2439
2450
  };
2440
- const requestWorkspaceSync = (trigger, fingerprint) => {
2451
+ const sendWorkspaceReady = () => {
2452
+ const manifestRevision = workspaceManifestRevision.readyRevision();
2453
+ if (!workspaceReady || !manifestRevision) return;
2441
2454
  try {
2442
2455
  ws.send(
2443
2456
  JSON.stringify({
2444
- type: "workspace_sync_requested",
2445
- observationId: crypto.randomUUID(),
2457
+ type: "workspace_ready",
2458
+ manifestRevision,
2459
+ pendingCheckouts: [...pendingManifestCheckouts.values()].sort(
2460
+ (left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
2461
+ ),
2462
+ ...workspaceConvergence.snapshot()
2463
+ })
2464
+ );
2465
+ } catch (error) {
2466
+ process.stderr.write(
2467
+ `[r5d-worker] failed to send workspace readiness: ${error instanceof Error ? error.message : String(error)}
2468
+ `
2469
+ );
2470
+ }
2471
+ };
2472
+ const workerHasActiveWriter = () => activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2473
+ const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
2474
+ const armWorkspaceCheckpoint = (trigger, options2 = { delayMs: import_workspace_convergence.WORKSPACE_CHECKPOINT_QUIET_MS }) => {
2475
+ workspaceConvergence.schedule();
2476
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2477
+ sendWorkspaceReady();
2478
+ if (options2.requestId && !explicitWorkspaceCheckpoints.schedule({ requestId: options2.requestId, trigger }, pendingCheckpoint?.requestId)) {
2479
+ return;
2480
+ }
2481
+ if (!options2.requestId && explicitWorkspaceCheckpoints.hasScheduled()) return;
2482
+ if (workspaceCheckpointTimer) clearTimeout(workspaceCheckpointTimer);
2483
+ workspaceCheckpointTimer = setTimeout(() => {
2484
+ workspaceCheckpointTimer = void 0;
2485
+ if (options2.requestId) explicitWorkspaceCheckpoints.consumeScheduled(options2.requestId);
2486
+ if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
2487
+ if (!workspaceReady || !workspaceRemoteUrl) {
2488
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2489
+ return;
2490
+ }
2491
+ if (activeWorkspaceIncidentId) {
2492
+ workspaceConvergence.block();
2493
+ sendWorkspaceReady();
2494
+ if ((0, import_workspace_convergence.workspaceCheckpointIncidentAction)(options2.requestId) === "defer_automatic") return;
2495
+ if (pendingCheckpoint) {
2496
+ explicitWorkspaceCheckpoints.enqueue({ requestId: options2.requestId, trigger }, pendingCheckpoint.requestId);
2497
+ return;
2498
+ }
2499
+ const snapshot2 = workspaceConvergence.snapshot();
2500
+ pendingCheckpoint = {
2501
+ requestId: options2.requestId,
2446
2502
  trigger,
2447
- ...fingerprint ? { fingerprint } : {}
2503
+ dirtyGeneration: snapshot2.dirtyGeneration,
2504
+ explicit: true,
2505
+ admissionHeld: false,
2506
+ workInFlight: false,
2507
+ requestedAt: Date.now(),
2508
+ releaseCheckpointAdmission: () => {
2509
+ }
2510
+ };
2511
+ try {
2512
+ ws.send(
2513
+ JSON.stringify({
2514
+ type: "workspace_checkpoint_request",
2515
+ requestId: options2.requestId,
2516
+ trigger,
2517
+ dirtyGeneration: snapshot2.dirtyGeneration,
2518
+ localHead: snapshot2.localHead,
2519
+ desiredCanonicalHead: snapshot2.desiredCanonicalHead
2520
+ })
2521
+ );
2522
+ } catch (error) {
2523
+ pendingCheckpoint = void 0;
2524
+ workspaceConvergence.defer();
2525
+ process.stderr.write(
2526
+ `[r5d-worker] failed to request blocked workspace checkpoint authority: ${error instanceof Error ? error.message : String(error)}
2527
+ `
2528
+ );
2529
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, requestId: options2.requestId });
2530
+ }
2531
+ return;
2532
+ }
2533
+ if (pendingCheckpoint || workerHasActiveWriter()) {
2534
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2535
+ return;
2536
+ }
2537
+ const releaseCheckpointAdmission = workspaceFilesystemWriters.tryAcquireCheckpoint();
2538
+ if (!releaseCheckpointAdmission) {
2539
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2540
+ return;
2541
+ }
2542
+ const snapshot = workspaceConvergence.snapshot();
2543
+ const requestId = options2.requestId ?? crypto.randomUUID();
2544
+ pendingCheckpoint = {
2545
+ requestId,
2546
+ trigger,
2547
+ dirtyGeneration: snapshot.dirtyGeneration,
2548
+ explicit: options2.requestId !== void 0,
2549
+ admissionHeld: true,
2550
+ workInFlight: false,
2551
+ requestedAt: Date.now(),
2552
+ releaseCheckpointAdmission
2553
+ };
2554
+ try {
2555
+ ws.send(
2556
+ JSON.stringify({
2557
+ type: "workspace_checkpoint_request",
2558
+ requestId,
2559
+ trigger,
2560
+ dirtyGeneration: snapshot.dirtyGeneration,
2561
+ localHead: snapshot.localHead,
2562
+ desiredCanonicalHead: snapshot.desiredCanonicalHead
2563
+ })
2564
+ );
2565
+ } catch (error) {
2566
+ releaseCheckpointAdmission();
2567
+ pendingCheckpoint = void 0;
2568
+ workspaceConvergence.defer();
2569
+ process.stderr.write(
2570
+ `[r5d-worker] failed to request workspace checkpoint: ${error instanceof Error ? error.message : String(error)}
2571
+ `
2572
+ );
2573
+ armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
2574
+ }
2575
+ }, Math.max(0, options2.delayMs));
2576
+ workspaceCheckpointTimer.unref();
2577
+ };
2578
+ const markWorkspaceDirty = (trigger, force = false) => {
2579
+ workspaceConvergence.markDirty();
2580
+ armWorkspaceCheckpoint(trigger, { delayMs: force ? 0 : import_workspace_convergence.WORKSPACE_CHECKPOINT_QUIET_MS });
2581
+ };
2582
+ const convergeAvailableWorkspaceHead = async () => {
2583
+ if (pendingWorkspaceHead === void 0 || !workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || workspaceHeadConvergenceInFlight) {
2584
+ return;
2585
+ }
2586
+ const advertisedHead = pendingWorkspaceHead;
2587
+ pendingWorkspaceHead = void 0;
2588
+ workspaceConvergence.observeCanonicalHead(advertisedHead);
2589
+ workspaceHeadConvergenceInFlight = true;
2590
+ let releaseVisibleHydration = null;
2591
+ try {
2592
+ const snapshot = workspaceConvergence.snapshot();
2593
+ const mayHydrateVisible = !workerHasActiveWriter() && !pendingCheckpoint && snapshot.dirtyGeneration === snapshot.publishedGeneration;
2594
+ releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
2595
+ const hydrateVisible = releaseVisibleHydration !== null;
2596
+ const converged = await workspaceSyncSingleFlight.runExclusive(
2597
+ () => (0, import_workspace_sync.convergeWorkspaceHead)(workspaceSyncInput({ type: "inbound_head", detail: advertisedHead ?? "unborn" }), {
2598
+ hydrateVisible
2448
2599
  })
2449
2600
  );
2601
+ workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
2602
+ if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
2603
+ if (hydrateVisible && converged.localChanges) {
2604
+ markWorkspaceDirty({ type: "periodic", detail: "inbound overlap retained for next checkpoint" });
2605
+ }
2606
+ sendWorkspaceReady();
2450
2607
  } catch (error) {
2608
+ pendingWorkspaceHead = advertisedHead;
2451
2609
  process.stderr.write(
2452
- `[r5d-worker] failed to send workspace sync observation on its current connection: ${error instanceof Error ? error.message : String(error)}
2610
+ `[r5d-worker] failed to fetch available workspace head: ${error instanceof Error ? error.message : String(error)}
2453
2611
  `
2454
2612
  );
2613
+ } finally {
2614
+ releaseVisibleHydration?.();
2615
+ workspaceHeadConvergenceInFlight = false;
2616
+ heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
2617
+ if (pendingWorkspaceHead !== void 0) {
2618
+ setTimeout(() => void convergeAvailableWorkspaceHead(), 1e3).unref();
2619
+ }
2455
2620
  }
2456
2621
  };
2457
2622
  const manifestCheckoutKey = (projectId, branchName) => `${projectId}\0${branchName}`;
@@ -2503,13 +2668,58 @@ async function startWorker(options) {
2503
2668
  }
2504
2669
  return created;
2505
2670
  };
2506
- const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}, authoritativeCanonicalHead, allowRecentNoChangeFastPath) => {
2671
+ const schedulePendingManifestCheckoutHydration = (delayMs = 0) => {
2672
+ if (workspaceManifestHydrationTimer || workspaceManifestHydrationInFlight || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2673
+ return;
2674
+ }
2675
+ workspaceManifestHydrationTimer = setTimeout(() => {
2676
+ workspaceManifestHydrationTimer = void 0;
2677
+ if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2678
+ if (pendingManifestCheckouts.size > 0 && currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
2679
+ schedulePendingManifestCheckoutHydration(1e3);
2680
+ }
2681
+ return;
2682
+ }
2683
+ const releaseHydration = workspaceFilesystemWriters.tryAcquireCheckpoint();
2684
+ if (!releaseHydration) {
2685
+ schedulePendingManifestCheckoutHydration(1e3);
2686
+ return;
2687
+ }
2688
+ workspaceManifestHydrationInFlight = true;
2689
+ void workspaceSyncSingleFlight.runExclusive(async () => {
2690
+ if (!workspaceReady || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
2691
+ return;
2692
+ }
2693
+ const targets = [...pendingManifestCheckouts.values()];
2694
+ ensureVisibleWorkspaceCheckouts(targets, { strictCanonicalRevalidation: true });
2695
+ for (const target of targets) {
2696
+ const manifest = manifestByProjectId.get(target.projectId);
2697
+ if (!manifest || !manifest.branches.includes(target.branchName)) continue;
2698
+ const branchPath = import_node_path.default.join(projectRootFor(projectsRoot, target.projectId, manifestByProjectId), target.branchName);
2699
+ if (hasNormalVisibleGitDir(branchPath)) {
2700
+ pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
2701
+ }
2702
+ }
2703
+ }).then(() => sendWorkspaceReady()).catch((error) => {
2704
+ process.stderr.write(
2705
+ `[r5d-worker] pending manifest checkout hydration failed: ${error instanceof Error ? error.message : String(error)}
2706
+ `
2707
+ );
2708
+ }).finally(() => {
2709
+ releaseHydration();
2710
+ workspaceManifestHydrationInFlight = false;
2711
+ heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
2712
+ if (pendingManifestCheckouts.size > 0) schedulePendingManifestCheckoutHydration(1e3);
2713
+ });
2714
+ }, Math.max(0, delayMs));
2715
+ workspaceManifestHydrationTimer.unref();
2716
+ };
2717
+ const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}) => {
2507
2718
  const requestedAt = Date.now();
2508
2719
  let queueEnteredAt = requestedAt;
2509
2720
  let prepareStartedAt = requestedAt;
2510
2721
  let synchronizeStartedAt = requestedAt;
2511
2722
  let synchronizeFinishedAt = requestedAt;
2512
- let fastPathDecision = { hit: false, reason: "no_recent_proof" };
2513
2723
  workspaceSyncRequestsInFlight += 1;
2514
2724
  try {
2515
2725
  let pendingTargets = [];
@@ -2518,48 +2728,12 @@ async function startWorker(options) {
2518
2728
  const shouldEnsureCheckouts = !overrides.skipVisibleMirror && !overrides.resetToCanonical;
2519
2729
  const syncProjects = (0, import_workspace_sync.workspaceProjectsForSync)([...manifestByProjectId.values()], trigger);
2520
2730
  if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
2521
- const scopeKey = (0, import_workspace_sync_fast_path.workspaceSyncScopeKey)(workspaceRemoteUrl, syncProjects);
2522
2731
  const allowedCheckoutKeys = new Set(
2523
2732
  syncProjects.flatMap((project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName)))
2524
2733
  );
2525
2734
  const scopedPendingTargets = [...pendingManifestCheckouts.values()].filter(
2526
2735
  (target) => allowedCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
2527
2736
  );
2528
- const workerBusy = activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2529
- fastPathDecision = recentWorkspaceSyncProof.evaluate({
2530
- trigger,
2531
- allowRecentNoChangeFastPath,
2532
- canonicalHead: authoritativeCanonicalHead,
2533
- scopeKey,
2534
- nowMs: Date.now(),
2535
- skipVisibleMirror: overrides.skipVisibleMirror,
2536
- resetToCanonical: overrides.resetToCanonical,
2537
- incidentActive: activeWorkspaceIncidentId !== null,
2538
- workerBusy,
2539
- pendingCheckouts: scopedPendingTargets.length > 0 || Boolean(overrides.newVisibleCheckouts?.length)
2540
- });
2541
- if (fastPathDecision.hit) {
2542
- prepareStartedAt = queueEnteredAt;
2543
- synchronizeStartedAt = queueEnteredAt;
2544
- synchronizeFinishedAt = queueEnteredAt;
2545
- return {
2546
- type: "workspace_sync",
2547
- attemptId: overrides.attemptId ?? crypto.randomUUID(),
2548
- workerLabel: label,
2549
- trigger,
2550
- outcome: "no_change",
2551
- startingHead: fastPathDecision.canonicalHead,
2552
- expectedHead: fastPathDecision.canonicalHead,
2553
- publishedHead: fastPathDecision.canonicalHead,
2554
- rebaseCount: 0,
2555
- diffSizeBytes: 0,
2556
- gitStatus: "",
2557
- affectedProjects: [],
2558
- affectedPaths: [],
2559
- discardedPaths: [],
2560
- localChangesDiscarded: false
2561
- };
2562
- }
2563
2737
  prepareStartedAt = Date.now();
2564
2738
  let checkoutTargets = [];
2565
2739
  if (shouldEnsureCheckouts) {
@@ -2585,24 +2759,12 @@ async function startWorker(options) {
2585
2759
  ...newVisibleCheckouts.length > 0 ? { newVisibleCheckouts } : {}
2586
2760
  });
2587
2761
  synchronizeStartedAt = Date.now();
2588
- recentWorkspaceSyncProof.invalidate();
2589
- const observationGeneration = recentWorkspaceSyncProof.beginObservation();
2590
- const workerWasBusyDuringObservation = workerBusy;
2591
2762
  let synchronized;
2592
2763
  try {
2593
2764
  synchronized = await (0, import_workspace_sync.synchronizeWorkspace)(input);
2594
2765
  } finally {
2595
2766
  synchronizeFinishedAt = Date.now();
2596
2767
  }
2597
- if ((synchronized.outcome === "no_change" || synchronized.outcome === "updated") && pendingTargets.length === 0) {
2598
- recentWorkspaceSyncProof.recordStable({
2599
- observationGeneration,
2600
- canonicalHead: synchronized.publishedHead ?? null,
2601
- scopeKey,
2602
- observedAtMs: synchronizeFinishedAt,
2603
- workerBusy: workerWasBusyDuringObservation || activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress || activeWorkspaceIncidentId !== null
2604
- });
2605
- }
2606
2768
  return synchronized;
2607
2769
  });
2608
2770
  const completedAt = Date.now();
@@ -2612,8 +2774,7 @@ async function startWorker(options) {
2612
2774
  totalMs: completedAt - requestedAt,
2613
2775
  queueMs: queueEnteredAt - requestedAt,
2614
2776
  prepareMs: synchronizeStartedAt - prepareStartedAt,
2615
- synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt,
2616
- fastPath: workspaceSyncFastPathTelemetry(fastPathDecision)
2777
+ synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
2617
2778
  }
2618
2779
  };
2619
2780
  if (resultWithTelemetry.outcome === "no_change" || resultWithTelemetry.outcome === "published" || resultWithTelemetry.outcome === "updated" || resultWithTelemetry.outcome === "conflict_reset") {
@@ -2622,7 +2783,6 @@ async function startWorker(options) {
2622
2783
  if (pendingManifestCheckouts.get(key) === target) pendingManifestCheckouts.delete(key);
2623
2784
  }
2624
2785
  }
2625
- previousPeriodicFingerprint = null;
2626
2786
  sendWorkspaceSyncResult(authority, resultWithTelemetry);
2627
2787
  return resultWithTelemetry;
2628
2788
  } catch (error) {
@@ -2645,12 +2805,10 @@ async function startWorker(options) {
2645
2805
  totalMs: completedAt - requestedAt,
2646
2806
  queueMs: queueEnteredAt - requestedAt,
2647
2807
  prepareMs: Math.max(0, synchronizeStartedAt - prepareStartedAt),
2648
- synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt),
2649
- fastPath: workspaceSyncFastPathTelemetry(fastPathDecision)
2808
+ synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
2650
2809
  },
2651
2810
  error: error instanceof Error ? error.message : String(error)
2652
2811
  };
2653
- previousPeriodicFingerprint = null;
2654
2812
  sendWorkspaceSyncResult(authority, result);
2655
2813
  return result;
2656
2814
  } finally {
@@ -2666,6 +2824,24 @@ async function startWorker(options) {
2666
2824
  let heartbeatBusyGrace = null;
2667
2825
  let heartbeatTimedOut = false;
2668
2826
  let heartbeatWatchdog;
2827
+ const handleGracefulShutdown = () => {
2828
+ if (shutdownAfterClose) return;
2829
+ shutdownAfterClose = true;
2830
+ void (0, import_workspace_convergence.waitForWorkspaceCheckpointOnShutdown)({
2831
+ snapshot: () => workspaceConvergence.snapshot(),
2832
+ forceCheckpoint: () => armWorkspaceCheckpoint({ type: "manual", detail: "graceful worker shutdown" }, { delayMs: 0 })
2833
+ }).then((outcome) => {
2834
+ process.stdout.write(`[r5d-worker] graceful workspace checkpoint ${outcome}
2835
+ `);
2836
+ if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
2837
+ ws.close(1e3, "Worker shutting down");
2838
+ } else {
2839
+ process.exit(0);
2840
+ }
2841
+ });
2842
+ };
2843
+ process.on("SIGTERM", handleGracefulShutdown);
2844
+ process.on("SIGINT", handleGracefulShutdown);
2669
2845
  const stopHeartbeatWatchdog = () => {
2670
2846
  if (heartbeatWatchdog) {
2671
2847
  clearInterval(heartbeatWatchdog);
@@ -2696,7 +2872,8 @@ async function startWorker(options) {
2696
2872
  updateClis: true,
2697
2873
  canonicalResolverCheckout: true,
2698
2874
  browserPortForwarding: true,
2699
- globalWorkspaceSyncLeaseV1: true
2875
+ globalWorkspaceSyncLeaseV1: true,
2876
+ eventualWorkspaceSyncV1: true
2700
2877
  },
2701
2878
  projectRoot: projectsRoot,
2702
2879
  artifactRoot,
@@ -2723,100 +2900,123 @@ async function startWorker(options) {
2723
2900
  return;
2724
2901
  }
2725
2902
  if (message.type === "workspace_manifest") {
2726
- recentWorkspaceSyncProof.invalidate();
2727
- await workspaceSyncSingleFlight.runMutation(() => {
2728
- configureGitHubAuth(message.githubCredential);
2729
- visibleGitIdentity = message.gitIdentity;
2730
- workspaceRemoteUrl = message.workspaceRemoteUrl;
2731
- const previousCheckoutKeys = new Set(
2732
- allManifestCheckouts().map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2733
- );
2734
- const migratedProjectIds = (0, import_managed_paths.migrateLegacyProjectRoots)(projectsRoot, message.projects);
2735
- manifestByProjectId.clear();
2736
- for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
2737
- const currentCheckouts = allManifestCheckouts();
2738
- const currentCheckoutKeys = new Set(
2739
- currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2740
- );
2741
- for (const key of pendingManifestCheckouts.keys()) {
2742
- if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
2743
- }
2744
- for (const checkout of currentCheckouts) {
2745
- const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
2746
- if (previousCheckoutKeys.has(key)) continue;
2747
- const branchPath = import_node_path.default.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
2748
- if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
2749
- }
2750
- previousPeriodicFingerprint = null;
2751
- process.stdout.write(
2752
- `[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
2753
- `
2754
- );
2903
+ const revisionToken = workspaceManifestRevision.begin(message.revision);
2904
+ workspaceReady = false;
2905
+ const manifestAdmission = (0, import_workspace_manifest_admission.inspectWorkspaceManifestAdmission)({
2906
+ projectsRoot,
2907
+ workspaceShadowRoot,
2908
+ projects: message.projects
2755
2909
  });
2756
- if (!periodicWorkspaceScan) {
2757
- const emptyFingerprint = (0, import_node_crypto.createHash)("sha256").update("").digest("hex");
2758
- periodicWorkspaceScan = setInterval(() => {
2759
- if (!workspaceRemoteUrl || activeWorkspaceIncidentId || cliUpdateInProgress || periodicWorkspaceScanInFlight) return;
2760
- periodicWorkspaceScanInFlight = true;
2761
- void (async () => {
2762
- const genericCheckoutKeys = new Set(
2763
- (0, import_workspace_sync.workspaceProjectsForSync)([...manifestByProjectId.values()], { type: "periodic" }).flatMap(
2764
- (project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName))
2765
- )
2766
- );
2767
- const hasPendingGenericCheckout = [...pendingManifestCheckouts.values()].some(
2768
- (target) => genericCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
2910
+ const releaseManifestHydration = await (0, import_workspace_manifest_admission.acquireWorkspaceManifestVisibleLease)(
2911
+ workspaceFilesystemWriters,
2912
+ manifestAdmission
2913
+ );
2914
+ let completed = false;
2915
+ try {
2916
+ completed = await workspaceSyncSingleFlight.runExclusive(async () => {
2917
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2918
+ const firstBootstrap = manifestAdmission.firstBootstrap;
2919
+ configureGitHubAuth(message.githubCredential);
2920
+ visibleGitIdentity = message.gitIdentity;
2921
+ workspaceRemoteUrl = message.workspaceRemoteUrl;
2922
+ const migratedProjectIds = releaseManifestHydration ? (0, import_managed_paths.migrateLegacyProjectRoots)(projectsRoot, message.projects) : [];
2923
+ manifestByProjectId.clear();
2924
+ for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
2925
+ const currentCheckouts = allManifestCheckouts();
2926
+ const currentCheckoutKeys = new Set(
2927
+ currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
2928
+ );
2929
+ for (const key of pendingManifestCheckouts.keys()) {
2930
+ if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
2931
+ }
2932
+ for (const checkout of currentCheckouts) {
2933
+ const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
2934
+ const branchPath = import_node_path.default.join(
2935
+ projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId),
2936
+ checkout.branchName
2769
2937
  );
2770
- if (hasPendingGenericCheckout) {
2771
- recentWorkspaceSyncProof.invalidate();
2772
- previousPeriodicFingerprint = null;
2773
- requestWorkspaceSync({
2774
- type: "periodic",
2775
- detail: "workspace manifest additions"
2776
- });
2777
- return;
2938
+ if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
2939
+ }
2940
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2941
+ const created = releaseManifestHydration ? ensureVisibleWorkspaceCheckouts(currentCheckouts, { strictCanonicalRevalidation: true }) : [];
2942
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2943
+ const bootstrapped = await (0, import_workspace_sync.convergeWorkspaceHead)(
2944
+ workspaceSyncInput({ type: "connect" }, { newVisibleCheckouts: created }),
2945
+ {
2946
+ // An existing checkout becomes routable after a fetch. Its
2947
+ // visible tree converges asynchronously; first bootstrap alone
2948
+ // must materialize the canonical snapshot before readiness.
2949
+ // Re-check the incident immediately before convergence so a
2950
+ // concurrently delivered fence forces fetch-only behavior.
2951
+ hydrateVisible: firstBootstrap && !activeWorkspaceIncidentId
2778
2952
  }
2779
- const observation = await workspaceSyncSingleFlight.runExclusive(() => {
2780
- const observationGeneration = recentWorkspaceSyncProof.beginObservation();
2781
- const input = workspaceSyncInput({ type: "periodic" });
2782
- const workerBusy = activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
2783
- return {
2784
- observationGeneration,
2785
- scopeKey: (0, import_workspace_sync_fast_path.workspaceSyncScopeKey)(input.remoteUrl, input.projects),
2786
- fingerprint: (0, import_workspace_sync.calculateWorkspaceDiffFingerprint)(input),
2787
- canonicalHead: (0, import_workspace_sync_fast_path.readShadowWorkspaceCanonicalHead)(workspaceShadowRoot),
2788
- workerBusy
2789
- };
2790
- });
2791
- if (observation.fingerprint === emptyFingerprint) {
2792
- recentWorkspaceSyncProof.recordStable({
2793
- observationGeneration: observation.observationGeneration,
2794
- canonicalHead: observation.canonicalHead,
2795
- scopeKey: observation.scopeKey,
2796
- observedAtMs: Date.now(),
2797
- workerBusy: observation.workerBusy || activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress
2798
- });
2799
- previousPeriodicFingerprint = null;
2800
- return;
2953
+ );
2954
+ if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
2955
+ workspaceConvergence.setLocalHead(bootstrapped.localHead);
2956
+ workspaceConvergence.observeCanonicalHead(bootstrapped.canonicalHead);
2957
+ for (const checkout of currentCheckouts) {
2958
+ const manifest = manifestByProjectId.get(checkout.projectId);
2959
+ if (!manifest) continue;
2960
+ const branchPath = import_node_path.default.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
2961
+ if (hasNormalVisibleGitDir(branchPath)) {
2962
+ pendingManifestCheckouts.delete(manifestCheckoutKey(checkout.projectId, checkout.branchName));
2801
2963
  }
2802
- recentWorkspaceSyncProof.invalidate();
2803
- if (previousPeriodicFingerprint !== observation.fingerprint) {
2804
- previousPeriodicFingerprint = observation.fingerprint;
2805
- return;
2964
+ }
2965
+ process.stdout.write(
2966
+ `[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
2967
+ `
2968
+ );
2969
+ return true;
2970
+ });
2971
+ } finally {
2972
+ releaseManifestHydration?.();
2973
+ }
2974
+ if (!completed || !workspaceManifestRevision.complete(revisionToken)) return;
2975
+ workspaceReady = true;
2976
+ sendWorkspaceReady();
2977
+ schedulePendingManifestCheckoutHydration();
2978
+ if (!workspaceSafetyScan) {
2979
+ workspaceSafetyScan = setInterval(() => {
2980
+ if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || workspaceSafetyScanInFlight) {
2981
+ return;
2982
+ }
2983
+ workspaceSafetyScanInFlight = true;
2984
+ void (async () => {
2985
+ const before = workspaceConvergence.snapshot();
2986
+ const mayHydrateVisible = !workerHasActiveWriter() && before.dirtyGeneration === before.publishedGeneration;
2987
+ const releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
2988
+ const hydrateVisible = releaseVisibleHydration !== null;
2989
+ try {
2990
+ const converged = await workspaceSyncSingleFlight.runExclusive(
2991
+ () => (0, import_workspace_sync.convergeWorkspaceHead)(workspaceSyncInput({ type: "periodic", detail: "idle convergence scan" }), {
2992
+ hydrateVisible
2993
+ })
2994
+ );
2995
+ workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
2996
+ if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
2997
+ if (converged.localChanges && hydrateVisible) {
2998
+ markWorkspaceDirty({ type: "periodic", detail: "idle safety scan" });
2999
+ }
3000
+ sendWorkspaceReady();
3001
+ } finally {
3002
+ releaseVisibleHydration?.();
2806
3003
  }
2807
- previousPeriodicFingerprint = null;
2808
- requestWorkspaceSync({ type: "periodic" }, observation.fingerprint);
2809
3004
  })().catch((error) => {
2810
3005
  process.stderr.write(
2811
- `[r5d-worker] periodic workspace synchronization failed: ${error instanceof Error ? error.message : String(error)}
3006
+ `[r5d-worker] idle workspace safety scan failed: ${error instanceof Error ? error.message : String(error)}
2812
3007
  `
2813
3008
  );
2814
3009
  }).finally(() => {
2815
- periodicWorkspaceScanInFlight = false;
3010
+ workspaceSafetyScanInFlight = false;
2816
3011
  heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
2817
3012
  });
2818
- }, import_workspace_sync.WORKSPACE_PERIODIC_SCAN_INTERVAL_MS);
2819
- periodicWorkspaceScan.unref();
3013
+ }, import_workspace_convergence.WORKSPACE_IDLE_SAFETY_SCAN_MS);
3014
+ workspaceSafetyScan.unref();
3015
+ }
3016
+ pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
3017
+ void convergeAvailableWorkspaceHead();
3018
+ if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3019
+ armWorkspaceCheckpoint({ type: "periodic", detail: "resume dirty generation after reconnect" }, { delayMs: 0 });
2820
3020
  }
2821
3021
  return;
2822
3022
  }
@@ -2836,12 +3036,189 @@ async function startWorker(options) {
2836
3036
  confirmationReason: message.confirmationReason,
2837
3037
  resetToCanonical: message.resetToCanonical,
2838
3038
  skipVisibleMirror: message.skipVisibleMirror
2839
- },
2840
- message.canonicalHead,
2841
- message.allowRecentNoChangeFastPath
3039
+ }
2842
3040
  );
2843
3041
  return;
2844
3042
  }
3043
+ if (message.type === "workspace_head_available") {
3044
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3045
+ pendingWorkspaceHead = message.canonicalHead;
3046
+ sendWorkspaceReady();
3047
+ void convergeAvailableWorkspaceHead();
3048
+ return;
3049
+ }
3050
+ if (message.type === "workspace_checkpoint_now") {
3051
+ armWorkspaceCheckpoint(message.trigger, { delayMs: 0, requestId: message.requestId });
3052
+ return;
3053
+ }
3054
+ if (message.type === "workspace_checkpoint_authority") {
3055
+ const checkpoint = pendingCheckpoint;
3056
+ if (!checkpoint || checkpoint.requestId !== message.requestId) {
3057
+ process.stderr.write(`[r5d-worker] ignoring checkpoint authority for unknown request ${message.requestId}
3058
+ `);
3059
+ return;
3060
+ }
3061
+ if (message.status !== "granted") {
3062
+ checkpoint.releaseCheckpointAdmission();
3063
+ pendingCheckpoint = void 0;
3064
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3065
+ const applyBlockedResponse = message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId);
3066
+ if (applyBlockedResponse) {
3067
+ workspaceConvergence.block();
3068
+ if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
3069
+ } else {
3070
+ workspaceConvergence.resetTransientCheckpointState();
3071
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3072
+ if (message.status === "busy") {
3073
+ armWorkspaceCheckpoint(checkpoint.trigger, {
3074
+ delayMs: message.retryAfterMs ?? 1e3,
3075
+ requestId: checkpoint.requestId
3076
+ });
3077
+ } else if (!activeWorkspaceIncidentId) {
3078
+ const explicit = explicitWorkspaceCheckpoints.next();
3079
+ if (explicit) {
3080
+ armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
3081
+ } else if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3082
+ armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: 0 });
3083
+ }
3084
+ }
3085
+ }
3086
+ sendWorkspaceReady();
3087
+ return;
3088
+ }
3089
+ checkpoint.attemptId = message.attemptId;
3090
+ checkpoint.authorityToken = message.authorityToken;
3091
+ checkpoint.workInFlight = true;
3092
+ pendingCheckpoint = checkpoint;
3093
+ if (!checkpoint.admissionHeld) {
3094
+ const releaseCheckpointAdmission = await workspaceFilesystemWriters.acquireCheckpoint();
3095
+ if (currentWorkerSocket !== ws || pendingCheckpoint?.requestId !== checkpoint.requestId || pendingCheckpoint.attemptId !== message.attemptId) {
3096
+ releaseCheckpointAdmission();
3097
+ checkpoint.workInFlight = false;
3098
+ workspaceConvergence.resetTransientCheckpointState();
3099
+ return;
3100
+ }
3101
+ checkpoint.admissionHeld = true;
3102
+ checkpoint.releaseCheckpointAdmission = releaseCheckpointAdmission;
3103
+ }
3104
+ workspaceConvergence.beginPreparing();
3105
+ sendWorkspaceReady();
3106
+ let prepareStartedAt = Date.now();
3107
+ let synchronizeStartedAt = prepareStartedAt;
3108
+ let synchronizeStarted = false;
3109
+ let result;
3110
+ try {
3111
+ result = await workspaceSyncSingleFlight.runExclusive(async () => {
3112
+ prepareStartedAt = Date.now();
3113
+ const pendingTargets = [...pendingManifestCheckouts.values()];
3114
+ const createdCheckouts = ensureVisibleWorkspaceCheckouts(pendingTargets, { strictCanonicalRevalidation: true });
3115
+ synchronizeStartedAt = Date.now();
3116
+ synchronizeStarted = true;
3117
+ const prepared = await (0, import_workspace_sync.synchronizeWorkspace)(
3118
+ workspaceSyncInput(checkpoint.trigger, {
3119
+ attemptId: message.attemptId,
3120
+ quarantineRef: message.quarantineRef,
3121
+ expectedCanonicalHead: message.expectedHead,
3122
+ newVisibleCheckouts: [...createdCheckouts, ...pendingTargets]
3123
+ })
3124
+ );
3125
+ for (const target of pendingTargets) {
3126
+ pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
3127
+ }
3128
+ return prepared.outcome === "updated" ? { ...prepared, outcome: "no_change" } : prepared;
3129
+ });
3130
+ } catch (error) {
3131
+ if (!synchronizeStarted) synchronizeStartedAt = Date.now();
3132
+ result = {
3133
+ type: "workspace_sync",
3134
+ attemptId: message.attemptId,
3135
+ workerLabel: label,
3136
+ trigger: checkpoint.trigger,
3137
+ outcome: "failed",
3138
+ startingHead: message.expectedHead,
3139
+ expectedHead: message.expectedHead,
3140
+ rebaseCount: 0,
3141
+ diffSizeBytes: 0,
3142
+ gitStatus: "",
3143
+ affectedProjects: [],
3144
+ affectedPaths: [],
3145
+ discardedPaths: [],
3146
+ localChangesDiscarded: false,
3147
+ error: error instanceof Error ? error.message : String(error)
3148
+ };
3149
+ }
3150
+ const finishedAt = Date.now();
3151
+ result = {
3152
+ ...result,
3153
+ dirtyGeneration: checkpoint.dirtyGeneration,
3154
+ telemetry: (0, import_workspace_convergence.workspaceCheckpointTelemetry)({
3155
+ requestedAt: checkpoint.requestedAt,
3156
+ prepareStartedAt,
3157
+ synchronizeStartedAt,
3158
+ finishedAt
3159
+ })
3160
+ };
3161
+ workspaceConvergence.beginSubmitting();
3162
+ sendWorkspaceReady();
3163
+ try {
3164
+ ws.send(
3165
+ JSON.stringify({
3166
+ type: "workspace_checkpoint_result",
3167
+ requestId: checkpoint.requestId,
3168
+ attemptId: message.attemptId,
3169
+ authorityToken: message.authorityToken,
3170
+ dirtyGeneration: checkpoint.dirtyGeneration,
3171
+ result
3172
+ })
3173
+ );
3174
+ } catch (error) {
3175
+ workspaceConvergence.resetTransientCheckpointState();
3176
+ throw error;
3177
+ } finally {
3178
+ checkpoint.workInFlight = false;
3179
+ checkpoint.releaseCheckpointAdmission();
3180
+ if (currentWorkerSocket !== ws) workspaceConvergence.resetTransientCheckpointState();
3181
+ }
3182
+ return;
3183
+ }
3184
+ if (message.type === "workspace_checkpoint_completed") {
3185
+ const checkpoint = pendingCheckpoint;
3186
+ if (!checkpoint || checkpoint.requestId !== message.requestId || checkpoint.attemptId !== message.attemptId) {
3187
+ process.stderr.write(`[r5d-worker] ignoring completion for unknown checkpoint ${message.requestId}
3188
+ `);
3189
+ return;
3190
+ }
3191
+ pendingCheckpoint = void 0;
3192
+ workspaceConvergence.observeCanonicalHead(message.canonicalHead);
3193
+ if (message.status === "published" || message.status === "no_change") {
3194
+ workspaceConvergence.complete({
3195
+ generation: Math.min(message.publishedGeneration ?? checkpoint.dirtyGeneration, checkpoint.dirtyGeneration),
3196
+ canonicalHead: message.canonicalHead,
3197
+ published: true
3198
+ });
3199
+ } else if (message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId)) {
3200
+ workspaceConvergence.block();
3201
+ if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
3202
+ } else {
3203
+ workspaceConvergence.resetTransientCheckpointState();
3204
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3205
+ }
3206
+ sendWorkspaceReady();
3207
+ const explicit = message.status === "retry" && checkpoint.explicit ? { requestId: checkpoint.requestId, trigger: checkpoint.trigger } : explicitWorkspaceCheckpoints.next();
3208
+ if (explicit) {
3209
+ armWorkspaceCheckpoint(explicit.trigger, {
3210
+ delayMs: message.retryAfterMs ?? 0,
3211
+ requestId: explicit.requestId
3212
+ });
3213
+ } else if (message.status === "retry" || message.status === "failed" || workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3214
+ armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: message.retryAfterMs ?? 1e3 });
3215
+ }
3216
+ if (message.canonicalHead !== workspaceConvergence.snapshot().localHead) {
3217
+ pendingWorkspaceHead = message.canonicalHead;
3218
+ void convergeAvailableWorkspaceHead();
3219
+ }
3220
+ return;
3221
+ }
2845
3222
  if (message.type === "sync_session_artifacts") {
2846
3223
  sessionArtifactSyncRequestsInFlight += 1;
2847
3224
  try {
@@ -2869,18 +3246,27 @@ async function startWorker(options) {
2869
3246
  return;
2870
3247
  }
2871
3248
  if (message.type === "workspace_incident_updated") {
2872
- recentWorkspaceSyncProof.invalidate();
3249
+ workspaceIncidentOrdering.observe(message);
2873
3250
  const previousIncidentId = activeWorkspaceIncidentId;
2874
3251
  activeWorkspaceIncidentId = (0, import_workspace_incident_state.applyWorkspaceIncidentUpdate)(activeWorkspaceIncidentId, message);
2875
- previousPeriodicFingerprint = null;
2876
3252
  const releasedHead = (0, import_workspace_incident_state.releasePendingWorkspaceHead)(
2877
3253
  previousIncidentId,
2878
3254
  activeWorkspaceIncidentId,
2879
3255
  message.status === "resolved" || message.status === "confirmed" || message.status === "reset" ? message.incidentId : null
2880
3256
  );
2881
- if (releasedHead.syncHead) {
2882
- requestWorkspaceSync({ type: "inbound_head", detail: releasedHead.syncHead });
3257
+ workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
3258
+ if (releasedHead.fetchAuthoritativeHead) {
3259
+ pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
3260
+ void convergeAvailableWorkspaceHead();
2883
3261
  }
3262
+ const explicit = !activeWorkspaceIncidentId ? explicitWorkspaceCheckpoints.next() : void 0;
3263
+ if (explicit) {
3264
+ armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
3265
+ } else if (!activeWorkspaceIncidentId && workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
3266
+ armWorkspaceCheckpoint({ type: "periodic", detail: "resume after workspace incident" }, { delayMs: 0 });
3267
+ }
3268
+ if (!activeWorkspaceIncidentId) schedulePendingManifestCheckoutHydration();
3269
+ sendWorkspaceReady();
2884
3270
  return;
2885
3271
  }
2886
3272
  if (message.type === "exec_terminal_ack") {
@@ -2916,7 +3302,6 @@ async function startWorker(options) {
2916
3302
  return;
2917
3303
  }
2918
3304
  cliUpdateInProgress = true;
2919
- recentWorkspaceSyncProof.invalidate();
2920
3305
  try {
2921
3306
  const result = await (0, import_cli_update.installCliUpdate)(message);
2922
3307
  sendWorkerMessage(ws, {
@@ -2979,9 +3364,10 @@ async function startWorker(options) {
2979
3364
  });
2980
3365
  return;
2981
3366
  }
2982
- recentWorkspaceSyncProof.invalidate();
3367
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
2983
3368
  const releaseWorkspaceMutation = await (0, import_workspace_command_sync_policy.acquireWorkspaceCommandMutation)(message.target, workspaceSyncSingleFlight);
2984
- let leaseTransferred = false;
3369
+ let mutationLeaseTransferred = false;
3370
+ let writerLeaseTransferred = false;
2985
3371
  try {
2986
3372
  const resolvedTarget = resolveMessageTarget(message.target);
2987
3373
  process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
@@ -2991,9 +3377,19 @@ async function startWorker(options) {
2991
3377
  message,
2992
3378
  resolvedTarget,
2993
3379
  planRoot,
2994
- ...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {}
3380
+ ...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {},
3381
+ ...targetMayMutateVisibleWorkspace(message.target) ? {
3382
+ onTerminal: () => {
3383
+ markWorkspaceDirty(
3384
+ { type: "process_terminal", detail: `pty ${message.ptyId} completed` },
3385
+ true
3386
+ );
3387
+ releaseWorkspaceWriter?.();
3388
+ }
3389
+ } : {}
2995
3390
  });
2996
- leaseTransferred = releaseWorkspaceMutation !== void 0;
3391
+ mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
3392
+ writerLeaseTransferred = releaseWorkspaceWriter !== void 0;
2997
3393
  } catch (error) {
2998
3394
  sendWorkerMessage(ws, {
2999
3395
  type: "pty_error",
@@ -3002,12 +3398,16 @@ async function startWorker(options) {
3002
3398
  error: error instanceof Error ? error.message : String(error)
3003
3399
  });
3004
3400
  } finally {
3005
- if (!leaseTransferred) releaseWorkspaceMutation?.();
3401
+ if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
3402
+ if (!writerLeaseTransferred) releaseWorkspaceWriter?.();
3006
3403
  }
3007
3404
  return;
3008
3405
  }
3009
3406
  if (message.type === "pty_input") {
3010
- recentWorkspaceSyncProof.invalidate();
3407
+ const activePty = activePtys.get(message.ptyId);
3408
+ if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
3409
+ markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` });
3410
+ }
3011
3411
  writePty(ws, message);
3012
3412
  return;
3013
3413
  }
@@ -3016,8 +3416,11 @@ async function startWorker(options) {
3016
3416
  return;
3017
3417
  }
3018
3418
  if (message.type === "pty_close") {
3019
- recentWorkspaceSyncProof.invalidate();
3419
+ const activePty = activePtys.get(message.ptyId);
3020
3420
  closePty(message);
3421
+ if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
3422
+ armWorkspaceCheckpoint({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, { delayMs: 0 });
3423
+ }
3021
3424
  return;
3022
3425
  }
3023
3426
  if (message.type === "exec") {
@@ -3031,27 +3434,37 @@ async function startWorker(options) {
3031
3434
  });
3032
3435
  return;
3033
3436
  }
3034
- recentWorkspaceSyncProof.invalidate();
3035
- let result;
3437
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3036
3438
  try {
3037
- const runCommand = async () => {
3038
- const resolvedTarget = resolveMessageTarget(message.target);
3039
- process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
3439
+ let result;
3440
+ try {
3441
+ const runCommand = async () => {
3442
+ const resolvedTarget = resolveMessageTarget(message.target);
3443
+ process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
3040
3444
  `);
3041
- return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
3042
- };
3043
- result = await (0, import_workspace_command_sync_policy.runWorkspaceCommand)(message.target, workspaceSyncSingleFlight, runCommand);
3044
- } catch (error) {
3045
- result = {
3046
- type: "exec_result",
3047
- requestId: message.requestId,
3048
- stdout: "",
3049
- stderr: "",
3050
- exitCode: 1,
3051
- error: error instanceof Error ? error.message : String(error)
3052
- };
3445
+ return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
3446
+ };
3447
+ result = await (0, import_workspace_command_sync_policy.runWorkspaceCommand)(message.target, workspaceSyncSingleFlight, runCommand);
3448
+ } catch (error) {
3449
+ result = {
3450
+ type: "exec_result",
3451
+ requestId: message.requestId,
3452
+ stdout: "",
3453
+ stderr: "",
3454
+ exitCode: 1,
3455
+ error: error instanceof Error ? error.message : String(error)
3456
+ };
3457
+ }
3458
+ ws.send(JSON.stringify(result));
3459
+ if (targetMayMutateVisibleWorkspace(message.target)) {
3460
+ markWorkspaceDirty(
3461
+ { type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
3462
+ true
3463
+ );
3464
+ }
3465
+ } finally {
3466
+ releaseWorkspaceWriter?.();
3053
3467
  }
3054
- ws.send(JSON.stringify(result));
3055
3468
  return;
3056
3469
  }
3057
3470
  if (message.type === "exec_start") {
@@ -3064,25 +3477,40 @@ async function startWorker(options) {
3064
3477
  });
3065
3478
  return;
3066
3479
  }
3067
- recentWorkspaceSyncProof.invalidate();
3068
3480
  sendWorkerMessage(ws, {
3069
3481
  type: "exec_accepted",
3070
3482
  requestId: message.requestId,
3071
3483
  runId: message.runId
3072
3484
  });
3073
3485
  const runCommand = async () => {
3074
- const resolvedTarget = resolveMessageTarget(message.target);
3075
- process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
3486
+ const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3487
+ try {
3488
+ const resolvedTarget = resolveMessageTarget(message.target);
3489
+ process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
3076
3490
  `);
3077
- await executeStreamingCommand({
3078
- ws,
3079
- message,
3080
- resolvedTarget,
3081
- baseUrl,
3082
- token,
3083
- artifactRoot,
3084
- planRoot
3085
- });
3491
+ await executeStreamingCommand({
3492
+ ws,
3493
+ message,
3494
+ resolvedTarget,
3495
+ baseUrl,
3496
+ token,
3497
+ artifactRoot,
3498
+ planRoot
3499
+ });
3500
+ } finally {
3501
+ if (targetMayMutateVisibleWorkspace(message.target)) {
3502
+ markWorkspaceDirty(
3503
+ {
3504
+ type: "process_terminal",
3505
+ sessionId: message.sessionId,
3506
+ processRunId: message.runId,
3507
+ detail: "process completed"
3508
+ },
3509
+ true
3510
+ );
3511
+ }
3512
+ releaseWorkspaceWriter?.();
3513
+ }
3086
3514
  };
3087
3515
  const execution = (0, import_workspace_command_sync_policy.runWorkspaceCommand)(message.target, workspaceSyncSingleFlight, runCommand);
3088
3516
  void execution.catch((error) => {
@@ -3096,34 +3524,46 @@ async function startWorker(options) {
3096
3524
  return;
3097
3525
  }
3098
3526
  if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes") {
3099
- if (message.type === "write" || message.type === "edit") recentWorkspaceSyncProof.invalidate();
3527
+ const releaseWorkspaceWriter = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
3100
3528
  try {
3101
- const result = await workspaceSyncSingleFlight.runMutation(async () => {
3102
- const resolvedTarget = resolveMessageTarget(message.target);
3103
- return await executeOperation({
3104
- message,
3105
- resolvedTarget,
3106
- baseUrl,
3107
- token,
3108
- artifactRoot,
3109
- planRoot
3529
+ try {
3530
+ const result = await workspaceSyncSingleFlight.runMutation(async () => {
3531
+ const resolvedTarget = resolveMessageTarget(message.target);
3532
+ return await executeOperation({
3533
+ message,
3534
+ resolvedTarget,
3535
+ baseUrl,
3536
+ token,
3537
+ artifactRoot,
3538
+ planRoot
3539
+ });
3110
3540
  });
3111
- });
3112
- ws.send(
3113
- JSON.stringify({
3114
- type: "operation_result",
3115
- requestId: message.requestId,
3116
- result
3117
- })
3118
- );
3119
- } catch (error) {
3120
- ws.send(
3121
- JSON.stringify({
3122
- type: "operation_result",
3123
- requestId: message.requestId,
3124
- error: error instanceof Error ? error.message : String(error)
3125
- })
3126
- );
3541
+ ws.send(
3542
+ JSON.stringify({
3543
+ type: "operation_result",
3544
+ requestId: message.requestId,
3545
+ result
3546
+ })
3547
+ );
3548
+ if ((message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target)) {
3549
+ markWorkspaceDirty({
3550
+ type: message.type,
3551
+ sessionId: message.sessionId,
3552
+ toolCallId: message.requestId,
3553
+ ...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
3554
+ });
3555
+ }
3556
+ } catch (error) {
3557
+ ws.send(
3558
+ JSON.stringify({
3559
+ type: "operation_result",
3560
+ requestId: message.requestId,
3561
+ error: error instanceof Error ? error.message : String(error)
3562
+ })
3563
+ );
3564
+ }
3565
+ } finally {
3566
+ releaseWorkspaceWriter?.();
3127
3567
  }
3128
3568
  return;
3129
3569
  }
@@ -3145,11 +3585,24 @@ async function startWorker(options) {
3145
3585
  });
3146
3586
  });
3147
3587
  ws.addEventListener("close", (event) => {
3148
- recentWorkspaceSyncProof.invalidate();
3588
+ process.off("SIGTERM", handleGracefulShutdown);
3589
+ process.off("SIGINT", handleGracefulShutdown);
3149
3590
  stopHeartbeatWatchdog();
3150
- if (periodicWorkspaceScan) {
3151
- clearInterval(periodicWorkspaceScan);
3152
- periodicWorkspaceScan = void 0;
3591
+ if (workspaceCheckpointTimer) {
3592
+ clearTimeout(workspaceCheckpointTimer);
3593
+ workspaceCheckpointTimer = void 0;
3594
+ }
3595
+ if (!pendingCheckpoint?.workInFlight) {
3596
+ pendingCheckpoint?.releaseCheckpointAdmission();
3597
+ workspaceConvergence.resetTransientCheckpointState();
3598
+ }
3599
+ if (workspaceSafetyScan) {
3600
+ clearInterval(workspaceSafetyScan);
3601
+ workspaceSafetyScan = void 0;
3602
+ }
3603
+ if (workspaceManifestHydrationTimer) {
3604
+ clearTimeout(workspaceManifestHydrationTimer);
3605
+ workspaceManifestHydrationTimer = void 0;
3153
3606
  }
3154
3607
  if (terminalReplayTimer) {
3155
3608
  clearInterval(terminalReplayTimer);
@@ -3165,6 +3618,9 @@ async function startWorker(options) {
3165
3618
  if (reloadAfterClose) {
3166
3619
  process.exit(import_supervisor.WORKER_RELOAD_EXIT_CODE);
3167
3620
  }
3621
+ if (shutdownAfterClose) {
3622
+ process.exit(0);
3623
+ }
3168
3624
  if (activeProcesses.size > 0 || pendingProcessTerminals.size > 0) {
3169
3625
  process.stderr.write(
3170
3626
  `[r5d-worker] ${activeProcesses.size} process(es) active and ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${import_supervisor.WORKER_RECONNECT_DELAY_MS}ms