@ricsam/r5d-worker 0.0.58 → 0.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/main.cjs +683 -227
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/workspace-convergence.cjs +320 -0
- package/dist/cjs/workspace-incident-state.cjs +32 -3
- package/dist/cjs/workspace-manifest-admission.cjs +70 -0
- package/dist/cjs/workspace-sync.cjs +135 -16
- package/dist/mjs/main.mjs +701 -233
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/workspace-convergence.mjs +275 -0
- package/dist/mjs/workspace-incident-state.mjs +31 -3
- package/dist/mjs/workspace-manifest-admission.mjs +35 -0
- package/dist/mjs/workspace-sync.mjs +132 -15
- package/dist/types/workspace-convergence.d.ts +102 -0
- package/dist/types/workspace-incident-state.d.ts +15 -1
- package/dist/types/workspace-manifest-admission.d.ts +27 -0
- package/dist/types/workspace-sync.d.ts +23 -6
- package/package.json +1 -1
- package/dist/cjs/workspace-sync-fast-path.cjs +0 -108
- package/dist/mjs/workspace-sync-fast-path.mjs +0 -81
- package/dist/types/workspace-sync-fast-path.d.ts +0 -69
package/dist/mjs/main.mjs
CHANGED
|
@@ -13,13 +13,27 @@ import {
|
|
|
13
13
|
} from "./heartbeat.mjs";
|
|
14
14
|
import { terminateProcessTree } from "./process-tree.mjs";
|
|
15
15
|
import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
applyWorkspaceIncidentUpdate,
|
|
18
|
+
releasePendingWorkspaceHead,
|
|
19
|
+
WorkspaceIncidentOrderingFence
|
|
20
|
+
} from "./workspace-incident-state.mjs";
|
|
17
21
|
import { acquireWorkspaceCommandMutation, runWorkspaceCommand } from "./workspace-command-sync-policy.mjs";
|
|
18
22
|
import {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
WORKSPACE_CHECKPOINT_QUIET_MS,
|
|
24
|
+
WORKSPACE_IDLE_SAFETY_SCAN_MS,
|
|
25
|
+
ExplicitWorkspaceCheckpointQueue,
|
|
26
|
+
WorkspaceManifestRevisionFence,
|
|
27
|
+
processWorkspaceConvergenceState,
|
|
28
|
+
processWorkspaceFilesystemWriterGate,
|
|
29
|
+
waitForWorkspaceCheckpointOnShutdown,
|
|
30
|
+
workspaceCheckpointIncidentAction,
|
|
31
|
+
workspaceCheckpointTelemetry
|
|
32
|
+
} from "./workspace-convergence.mjs";
|
|
33
|
+
import {
|
|
34
|
+
acquireWorkspaceManifestVisibleLease,
|
|
35
|
+
inspectWorkspaceManifestAdmission
|
|
36
|
+
} from "./workspace-manifest-admission.mjs";
|
|
23
37
|
import {
|
|
24
38
|
isRetryableWorkerServerStatus,
|
|
25
39
|
superviseWorkerRuntime,
|
|
@@ -30,15 +44,11 @@ import {
|
|
|
30
44
|
} from "./supervisor.mjs";
|
|
31
45
|
import { managedProjectRoot, migrateLegacyProjectRoots, validateManagedBranchName } from "./managed-paths.mjs";
|
|
32
46
|
import {
|
|
33
|
-
WORKSPACE_PERIODIC_SCAN_INTERVAL_MS,
|
|
34
47
|
WorkspaceSyncSingleFlight,
|
|
35
|
-
|
|
48
|
+
convergeWorkspaceHead,
|
|
36
49
|
synchronizeWorkspace,
|
|
37
50
|
workspaceProjectsForSync
|
|
38
51
|
} from "./workspace-sync.mjs";
|
|
39
|
-
function workspaceSyncFastPathTelemetry(decision) {
|
|
40
|
-
return decision.hit ? { hit: true, reason: decision.reason, proofAgeMs: decision.proofAgeMs } : { hit: false, reason: decision.reason };
|
|
41
|
-
}
|
|
42
52
|
class WorkerServerUnavailableError extends Error {
|
|
43
53
|
name = "WorkerServerUnavailableError";
|
|
44
54
|
}
|
|
@@ -54,6 +64,7 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
|
|
|
54
64
|
const activePtys = /* @__PURE__ */ new Map();
|
|
55
65
|
let currentWorkerSocket = null;
|
|
56
66
|
const workspaceSyncSingleFlight = new WorkspaceSyncSingleFlight();
|
|
67
|
+
const workspaceFilesystemWriters = processWorkspaceFilesystemWriterGate();
|
|
57
68
|
let githubCredential = null;
|
|
58
69
|
let visibleGitIdentity = null;
|
|
59
70
|
function defaultConfigPath() {
|
|
@@ -2268,6 +2279,7 @@ async function openPty(input) {
|
|
|
2268
2279
|
onExit: (event) => {
|
|
2269
2280
|
input.releaseWorkspaceMutation?.();
|
|
2270
2281
|
activePtys.delete(input.message.ptyId);
|
|
2282
|
+
input.onTerminal?.();
|
|
2271
2283
|
sendWorkerMessage(input.ws, {
|
|
2272
2284
|
type: "pty_exit",
|
|
2273
2285
|
ptyId: input.message.ptyId,
|
|
@@ -2278,6 +2290,7 @@ async function openPty(input) {
|
|
|
2278
2290
|
onError: (error) => {
|
|
2279
2291
|
input.releaseWorkspaceMutation?.();
|
|
2280
2292
|
activePtys.delete(input.message.ptyId);
|
|
2293
|
+
input.onTerminal?.();
|
|
2281
2294
|
sendWorkerMessage(input.ws, {
|
|
2282
2295
|
type: "pty_error",
|
|
2283
2296
|
requestId: input.message.requestId,
|
|
@@ -2369,15 +2382,25 @@ async function startWorker(options) {
|
|
|
2369
2382
|
const pendingManifestCheckouts = /* @__PURE__ */ new Map();
|
|
2370
2383
|
let workspaceRemoteUrl = null;
|
|
2371
2384
|
let activeWorkspaceIncidentId = null;
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2385
|
+
const workspaceIncidentOrdering = new WorkspaceIncidentOrderingFence();
|
|
2386
|
+
const workspaceConvergence = processWorkspaceConvergenceState(path.join(syncRoot, "workspace-convergence.json"));
|
|
2387
|
+
const workspaceManifestRevision = new WorkspaceManifestRevisionFence();
|
|
2388
|
+
let workspaceReady = false;
|
|
2389
|
+
let workspaceCheckpointTimer;
|
|
2390
|
+
let workspaceSafetyScan;
|
|
2391
|
+
let workspaceManifestHydrationTimer;
|
|
2392
|
+
let workspaceManifestHydrationInFlight = false;
|
|
2393
|
+
let workspaceSafetyScanInFlight = false;
|
|
2394
|
+
let workspaceHeadConvergenceInFlight = false;
|
|
2395
|
+
let pendingWorkspaceHead;
|
|
2396
|
+
let pendingCheckpoint;
|
|
2397
|
+
const explicitWorkspaceCheckpoints = new ExplicitWorkspaceCheckpointQueue();
|
|
2375
2398
|
let terminalReplayTimer;
|
|
2376
2399
|
let workspaceSyncRequestsInFlight = 0;
|
|
2377
2400
|
let sessionArtifactSyncRequestsInFlight = 0;
|
|
2378
2401
|
let cliUpdateInProgress = false;
|
|
2379
2402
|
let reloadAfterClose = false;
|
|
2380
|
-
|
|
2403
|
+
let shutdownAfterClose = false;
|
|
2381
2404
|
const workspaceSyncInput = (trigger, overrides = {}) => {
|
|
2382
2405
|
if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
|
|
2383
2406
|
const projects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
|
|
@@ -2403,21 +2426,175 @@ async function startWorker(options) {
|
|
|
2403
2426
|
);
|
|
2404
2427
|
}
|
|
2405
2428
|
};
|
|
2406
|
-
const
|
|
2429
|
+
const sendWorkspaceReady = () => {
|
|
2430
|
+
const manifestRevision = workspaceManifestRevision.readyRevision();
|
|
2431
|
+
if (!workspaceReady || !manifestRevision) return;
|
|
2407
2432
|
try {
|
|
2408
2433
|
ws.send(
|
|
2409
2434
|
JSON.stringify({
|
|
2410
|
-
type: "
|
|
2411
|
-
|
|
2435
|
+
type: "workspace_ready",
|
|
2436
|
+
manifestRevision,
|
|
2437
|
+
pendingCheckouts: [...pendingManifestCheckouts.values()].sort(
|
|
2438
|
+
(left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
|
|
2439
|
+
),
|
|
2440
|
+
...workspaceConvergence.snapshot()
|
|
2441
|
+
})
|
|
2442
|
+
);
|
|
2443
|
+
} catch (error) {
|
|
2444
|
+
process.stderr.write(
|
|
2445
|
+
`[r5d-worker] failed to send workspace readiness: ${error instanceof Error ? error.message : String(error)}
|
|
2446
|
+
`
|
|
2447
|
+
);
|
|
2448
|
+
}
|
|
2449
|
+
};
|
|
2450
|
+
const workerHasActiveWriter = () => activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
|
|
2451
|
+
const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
|
|
2452
|
+
const armWorkspaceCheckpoint = (trigger, options2 = { delayMs: WORKSPACE_CHECKPOINT_QUIET_MS }) => {
|
|
2453
|
+
workspaceConvergence.schedule();
|
|
2454
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2455
|
+
sendWorkspaceReady();
|
|
2456
|
+
if (options2.requestId && !explicitWorkspaceCheckpoints.schedule({ requestId: options2.requestId, trigger }, pendingCheckpoint?.requestId)) {
|
|
2457
|
+
return;
|
|
2458
|
+
}
|
|
2459
|
+
if (!options2.requestId && explicitWorkspaceCheckpoints.hasScheduled()) return;
|
|
2460
|
+
if (workspaceCheckpointTimer) clearTimeout(workspaceCheckpointTimer);
|
|
2461
|
+
workspaceCheckpointTimer = setTimeout(() => {
|
|
2462
|
+
workspaceCheckpointTimer = void 0;
|
|
2463
|
+
if (options2.requestId) explicitWorkspaceCheckpoints.consumeScheduled(options2.requestId);
|
|
2464
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2465
|
+
if (!workspaceReady || !workspaceRemoteUrl) {
|
|
2466
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2467
|
+
return;
|
|
2468
|
+
}
|
|
2469
|
+
if (activeWorkspaceIncidentId) {
|
|
2470
|
+
workspaceConvergence.block();
|
|
2471
|
+
sendWorkspaceReady();
|
|
2472
|
+
if (workspaceCheckpointIncidentAction(options2.requestId) === "defer_automatic") return;
|
|
2473
|
+
if (pendingCheckpoint) {
|
|
2474
|
+
explicitWorkspaceCheckpoints.enqueue({ requestId: options2.requestId, trigger }, pendingCheckpoint.requestId);
|
|
2475
|
+
return;
|
|
2476
|
+
}
|
|
2477
|
+
const snapshot2 = workspaceConvergence.snapshot();
|
|
2478
|
+
pendingCheckpoint = {
|
|
2479
|
+
requestId: options2.requestId,
|
|
2412
2480
|
trigger,
|
|
2413
|
-
|
|
2481
|
+
dirtyGeneration: snapshot2.dirtyGeneration,
|
|
2482
|
+
explicit: true,
|
|
2483
|
+
admissionHeld: false,
|
|
2484
|
+
workInFlight: false,
|
|
2485
|
+
requestedAt: Date.now(),
|
|
2486
|
+
releaseCheckpointAdmission: () => {
|
|
2487
|
+
}
|
|
2488
|
+
};
|
|
2489
|
+
try {
|
|
2490
|
+
ws.send(
|
|
2491
|
+
JSON.stringify({
|
|
2492
|
+
type: "workspace_checkpoint_request",
|
|
2493
|
+
requestId: options2.requestId,
|
|
2494
|
+
trigger,
|
|
2495
|
+
dirtyGeneration: snapshot2.dirtyGeneration,
|
|
2496
|
+
localHead: snapshot2.localHead,
|
|
2497
|
+
desiredCanonicalHead: snapshot2.desiredCanonicalHead
|
|
2498
|
+
})
|
|
2499
|
+
);
|
|
2500
|
+
} catch (error) {
|
|
2501
|
+
pendingCheckpoint = void 0;
|
|
2502
|
+
workspaceConvergence.defer();
|
|
2503
|
+
process.stderr.write(
|
|
2504
|
+
`[r5d-worker] failed to request blocked workspace checkpoint authority: ${error instanceof Error ? error.message : String(error)}
|
|
2505
|
+
`
|
|
2506
|
+
);
|
|
2507
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, requestId: options2.requestId });
|
|
2508
|
+
}
|
|
2509
|
+
return;
|
|
2510
|
+
}
|
|
2511
|
+
if (pendingCheckpoint || workerHasActiveWriter()) {
|
|
2512
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2513
|
+
return;
|
|
2514
|
+
}
|
|
2515
|
+
const releaseCheckpointAdmission = workspaceFilesystemWriters.tryAcquireCheckpoint();
|
|
2516
|
+
if (!releaseCheckpointAdmission) {
|
|
2517
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2518
|
+
return;
|
|
2519
|
+
}
|
|
2520
|
+
const snapshot = workspaceConvergence.snapshot();
|
|
2521
|
+
const requestId = options2.requestId ?? crypto.randomUUID();
|
|
2522
|
+
pendingCheckpoint = {
|
|
2523
|
+
requestId,
|
|
2524
|
+
trigger,
|
|
2525
|
+
dirtyGeneration: snapshot.dirtyGeneration,
|
|
2526
|
+
explicit: options2.requestId !== void 0,
|
|
2527
|
+
admissionHeld: true,
|
|
2528
|
+
workInFlight: false,
|
|
2529
|
+
requestedAt: Date.now(),
|
|
2530
|
+
releaseCheckpointAdmission
|
|
2531
|
+
};
|
|
2532
|
+
try {
|
|
2533
|
+
ws.send(
|
|
2534
|
+
JSON.stringify({
|
|
2535
|
+
type: "workspace_checkpoint_request",
|
|
2536
|
+
requestId,
|
|
2537
|
+
trigger,
|
|
2538
|
+
dirtyGeneration: snapshot.dirtyGeneration,
|
|
2539
|
+
localHead: snapshot.localHead,
|
|
2540
|
+
desiredCanonicalHead: snapshot.desiredCanonicalHead
|
|
2541
|
+
})
|
|
2542
|
+
);
|
|
2543
|
+
} catch (error) {
|
|
2544
|
+
releaseCheckpointAdmission();
|
|
2545
|
+
pendingCheckpoint = void 0;
|
|
2546
|
+
workspaceConvergence.defer();
|
|
2547
|
+
process.stderr.write(
|
|
2548
|
+
`[r5d-worker] failed to request workspace checkpoint: ${error instanceof Error ? error.message : String(error)}
|
|
2549
|
+
`
|
|
2550
|
+
);
|
|
2551
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2552
|
+
}
|
|
2553
|
+
}, Math.max(0, options2.delayMs));
|
|
2554
|
+
workspaceCheckpointTimer.unref();
|
|
2555
|
+
};
|
|
2556
|
+
const markWorkspaceDirty = (trigger, force = false) => {
|
|
2557
|
+
workspaceConvergence.markDirty();
|
|
2558
|
+
armWorkspaceCheckpoint(trigger, { delayMs: force ? 0 : WORKSPACE_CHECKPOINT_QUIET_MS });
|
|
2559
|
+
};
|
|
2560
|
+
const convergeAvailableWorkspaceHead = async () => {
|
|
2561
|
+
if (pendingWorkspaceHead === void 0 || !workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || workspaceHeadConvergenceInFlight) {
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
const advertisedHead = pendingWorkspaceHead;
|
|
2565
|
+
pendingWorkspaceHead = void 0;
|
|
2566
|
+
workspaceConvergence.observeCanonicalHead(advertisedHead);
|
|
2567
|
+
workspaceHeadConvergenceInFlight = true;
|
|
2568
|
+
let releaseVisibleHydration = null;
|
|
2569
|
+
try {
|
|
2570
|
+
const snapshot = workspaceConvergence.snapshot();
|
|
2571
|
+
const mayHydrateVisible = !workerHasActiveWriter() && !pendingCheckpoint && snapshot.dirtyGeneration === snapshot.publishedGeneration;
|
|
2572
|
+
releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
|
|
2573
|
+
const hydrateVisible = releaseVisibleHydration !== null;
|
|
2574
|
+
const converged = await workspaceSyncSingleFlight.runExclusive(
|
|
2575
|
+
() => convergeWorkspaceHead(workspaceSyncInput({ type: "inbound_head", detail: advertisedHead ?? "unborn" }), {
|
|
2576
|
+
hydrateVisible
|
|
2414
2577
|
})
|
|
2415
2578
|
);
|
|
2579
|
+
workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
|
|
2580
|
+
if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
|
|
2581
|
+
if (hydrateVisible && converged.localChanges) {
|
|
2582
|
+
markWorkspaceDirty({ type: "periodic", detail: "inbound overlap retained for next checkpoint" });
|
|
2583
|
+
}
|
|
2584
|
+
sendWorkspaceReady();
|
|
2416
2585
|
} catch (error) {
|
|
2586
|
+
pendingWorkspaceHead = advertisedHead;
|
|
2417
2587
|
process.stderr.write(
|
|
2418
|
-
`[r5d-worker] failed to
|
|
2588
|
+
`[r5d-worker] failed to fetch available workspace head: ${error instanceof Error ? error.message : String(error)}
|
|
2419
2589
|
`
|
|
2420
2590
|
);
|
|
2591
|
+
} finally {
|
|
2592
|
+
releaseVisibleHydration?.();
|
|
2593
|
+
workspaceHeadConvergenceInFlight = false;
|
|
2594
|
+
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2595
|
+
if (pendingWorkspaceHead !== void 0) {
|
|
2596
|
+
setTimeout(() => void convergeAvailableWorkspaceHead(), 1e3).unref();
|
|
2597
|
+
}
|
|
2421
2598
|
}
|
|
2422
2599
|
};
|
|
2423
2600
|
const manifestCheckoutKey = (projectId, branchName) => `${projectId}\0${branchName}`;
|
|
@@ -2469,13 +2646,58 @@ async function startWorker(options) {
|
|
|
2469
2646
|
}
|
|
2470
2647
|
return created;
|
|
2471
2648
|
};
|
|
2472
|
-
const
|
|
2649
|
+
const schedulePendingManifestCheckoutHydration = (delayMs = 0) => {
|
|
2650
|
+
if (workspaceManifestHydrationTimer || workspaceManifestHydrationInFlight || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2651
|
+
return;
|
|
2652
|
+
}
|
|
2653
|
+
workspaceManifestHydrationTimer = setTimeout(() => {
|
|
2654
|
+
workspaceManifestHydrationTimer = void 0;
|
|
2655
|
+
if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2656
|
+
if (pendingManifestCheckouts.size > 0 && currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
|
|
2657
|
+
schedulePendingManifestCheckoutHydration(1e3);
|
|
2658
|
+
}
|
|
2659
|
+
return;
|
|
2660
|
+
}
|
|
2661
|
+
const releaseHydration = workspaceFilesystemWriters.tryAcquireCheckpoint();
|
|
2662
|
+
if (!releaseHydration) {
|
|
2663
|
+
schedulePendingManifestCheckoutHydration(1e3);
|
|
2664
|
+
return;
|
|
2665
|
+
}
|
|
2666
|
+
workspaceManifestHydrationInFlight = true;
|
|
2667
|
+
void workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2668
|
+
if (!workspaceReady || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2669
|
+
return;
|
|
2670
|
+
}
|
|
2671
|
+
const targets = [...pendingManifestCheckouts.values()];
|
|
2672
|
+
ensureVisibleWorkspaceCheckouts(targets, { strictCanonicalRevalidation: true });
|
|
2673
|
+
for (const target of targets) {
|
|
2674
|
+
const manifest = manifestByProjectId.get(target.projectId);
|
|
2675
|
+
if (!manifest || !manifest.branches.includes(target.branchName)) continue;
|
|
2676
|
+
const branchPath = path.join(projectRootFor(projectsRoot, target.projectId, manifestByProjectId), target.branchName);
|
|
2677
|
+
if (hasNormalVisibleGitDir(branchPath)) {
|
|
2678
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
|
|
2679
|
+
}
|
|
2680
|
+
}
|
|
2681
|
+
}).then(() => sendWorkspaceReady()).catch((error) => {
|
|
2682
|
+
process.stderr.write(
|
|
2683
|
+
`[r5d-worker] pending manifest checkout hydration failed: ${error instanceof Error ? error.message : String(error)}
|
|
2684
|
+
`
|
|
2685
|
+
);
|
|
2686
|
+
}).finally(() => {
|
|
2687
|
+
releaseHydration();
|
|
2688
|
+
workspaceManifestHydrationInFlight = false;
|
|
2689
|
+
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2690
|
+
if (pendingManifestCheckouts.size > 0) schedulePendingManifestCheckoutHydration(1e3);
|
|
2691
|
+
});
|
|
2692
|
+
}, Math.max(0, delayMs));
|
|
2693
|
+
workspaceManifestHydrationTimer.unref();
|
|
2694
|
+
};
|
|
2695
|
+
const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}) => {
|
|
2473
2696
|
const requestedAt = Date.now();
|
|
2474
2697
|
let queueEnteredAt = requestedAt;
|
|
2475
2698
|
let prepareStartedAt = requestedAt;
|
|
2476
2699
|
let synchronizeStartedAt = requestedAt;
|
|
2477
2700
|
let synchronizeFinishedAt = requestedAt;
|
|
2478
|
-
let fastPathDecision = { hit: false, reason: "no_recent_proof" };
|
|
2479
2701
|
workspaceSyncRequestsInFlight += 1;
|
|
2480
2702
|
try {
|
|
2481
2703
|
let pendingTargets = [];
|
|
@@ -2484,48 +2706,12 @@ async function startWorker(options) {
|
|
|
2484
2706
|
const shouldEnsureCheckouts = !overrides.skipVisibleMirror && !overrides.resetToCanonical;
|
|
2485
2707
|
const syncProjects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
|
|
2486
2708
|
if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
|
|
2487
|
-
const scopeKey = workspaceSyncScopeKey(workspaceRemoteUrl, syncProjects);
|
|
2488
2709
|
const allowedCheckoutKeys = new Set(
|
|
2489
2710
|
syncProjects.flatMap((project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName)))
|
|
2490
2711
|
);
|
|
2491
2712
|
const scopedPendingTargets = [...pendingManifestCheckouts.values()].filter(
|
|
2492
2713
|
(target) => allowedCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
|
|
2493
2714
|
);
|
|
2494
|
-
const workerBusy = activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
|
|
2495
|
-
fastPathDecision = recentWorkspaceSyncProof.evaluate({
|
|
2496
|
-
trigger,
|
|
2497
|
-
allowRecentNoChangeFastPath,
|
|
2498
|
-
canonicalHead: authoritativeCanonicalHead,
|
|
2499
|
-
scopeKey,
|
|
2500
|
-
nowMs: Date.now(),
|
|
2501
|
-
skipVisibleMirror: overrides.skipVisibleMirror,
|
|
2502
|
-
resetToCanonical: overrides.resetToCanonical,
|
|
2503
|
-
incidentActive: activeWorkspaceIncidentId !== null,
|
|
2504
|
-
workerBusy,
|
|
2505
|
-
pendingCheckouts: scopedPendingTargets.length > 0 || Boolean(overrides.newVisibleCheckouts?.length)
|
|
2506
|
-
});
|
|
2507
|
-
if (fastPathDecision.hit) {
|
|
2508
|
-
prepareStartedAt = queueEnteredAt;
|
|
2509
|
-
synchronizeStartedAt = queueEnteredAt;
|
|
2510
|
-
synchronizeFinishedAt = queueEnteredAt;
|
|
2511
|
-
return {
|
|
2512
|
-
type: "workspace_sync",
|
|
2513
|
-
attemptId: overrides.attemptId ?? crypto.randomUUID(),
|
|
2514
|
-
workerLabel: label,
|
|
2515
|
-
trigger,
|
|
2516
|
-
outcome: "no_change",
|
|
2517
|
-
startingHead: fastPathDecision.canonicalHead,
|
|
2518
|
-
expectedHead: fastPathDecision.canonicalHead,
|
|
2519
|
-
publishedHead: fastPathDecision.canonicalHead,
|
|
2520
|
-
rebaseCount: 0,
|
|
2521
|
-
diffSizeBytes: 0,
|
|
2522
|
-
gitStatus: "",
|
|
2523
|
-
affectedProjects: [],
|
|
2524
|
-
affectedPaths: [],
|
|
2525
|
-
discardedPaths: [],
|
|
2526
|
-
localChangesDiscarded: false
|
|
2527
|
-
};
|
|
2528
|
-
}
|
|
2529
2715
|
prepareStartedAt = Date.now();
|
|
2530
2716
|
let checkoutTargets = [];
|
|
2531
2717
|
if (shouldEnsureCheckouts) {
|
|
@@ -2551,24 +2737,12 @@ async function startWorker(options) {
|
|
|
2551
2737
|
...newVisibleCheckouts.length > 0 ? { newVisibleCheckouts } : {}
|
|
2552
2738
|
});
|
|
2553
2739
|
synchronizeStartedAt = Date.now();
|
|
2554
|
-
recentWorkspaceSyncProof.invalidate();
|
|
2555
|
-
const observationGeneration = recentWorkspaceSyncProof.beginObservation();
|
|
2556
|
-
const workerWasBusyDuringObservation = workerBusy;
|
|
2557
2740
|
let synchronized;
|
|
2558
2741
|
try {
|
|
2559
2742
|
synchronized = await synchronizeWorkspace(input);
|
|
2560
2743
|
} finally {
|
|
2561
2744
|
synchronizeFinishedAt = Date.now();
|
|
2562
2745
|
}
|
|
2563
|
-
if ((synchronized.outcome === "no_change" || synchronized.outcome === "updated") && pendingTargets.length === 0) {
|
|
2564
|
-
recentWorkspaceSyncProof.recordStable({
|
|
2565
|
-
observationGeneration,
|
|
2566
|
-
canonicalHead: synchronized.publishedHead ?? null,
|
|
2567
|
-
scopeKey,
|
|
2568
|
-
observedAtMs: synchronizeFinishedAt,
|
|
2569
|
-
workerBusy: workerWasBusyDuringObservation || activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress || activeWorkspaceIncidentId !== null
|
|
2570
|
-
});
|
|
2571
|
-
}
|
|
2572
2746
|
return synchronized;
|
|
2573
2747
|
});
|
|
2574
2748
|
const completedAt = Date.now();
|
|
@@ -2578,8 +2752,7 @@ async function startWorker(options) {
|
|
|
2578
2752
|
totalMs: completedAt - requestedAt,
|
|
2579
2753
|
queueMs: queueEnteredAt - requestedAt,
|
|
2580
2754
|
prepareMs: synchronizeStartedAt - prepareStartedAt,
|
|
2581
|
-
synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
|
|
2582
|
-
fastPath: workspaceSyncFastPathTelemetry(fastPathDecision)
|
|
2755
|
+
synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
|
|
2583
2756
|
}
|
|
2584
2757
|
};
|
|
2585
2758
|
if (resultWithTelemetry.outcome === "no_change" || resultWithTelemetry.outcome === "published" || resultWithTelemetry.outcome === "updated" || resultWithTelemetry.outcome === "conflict_reset") {
|
|
@@ -2588,7 +2761,6 @@ async function startWorker(options) {
|
|
|
2588
2761
|
if (pendingManifestCheckouts.get(key) === target) pendingManifestCheckouts.delete(key);
|
|
2589
2762
|
}
|
|
2590
2763
|
}
|
|
2591
|
-
previousPeriodicFingerprint = null;
|
|
2592
2764
|
sendWorkspaceSyncResult(authority, resultWithTelemetry);
|
|
2593
2765
|
return resultWithTelemetry;
|
|
2594
2766
|
} catch (error) {
|
|
@@ -2611,12 +2783,10 @@ async function startWorker(options) {
|
|
|
2611
2783
|
totalMs: completedAt - requestedAt,
|
|
2612
2784
|
queueMs: queueEnteredAt - requestedAt,
|
|
2613
2785
|
prepareMs: Math.max(0, synchronizeStartedAt - prepareStartedAt),
|
|
2614
|
-
synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
|
|
2615
|
-
fastPath: workspaceSyncFastPathTelemetry(fastPathDecision)
|
|
2786
|
+
synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
|
|
2616
2787
|
},
|
|
2617
2788
|
error: error instanceof Error ? error.message : String(error)
|
|
2618
2789
|
};
|
|
2619
|
-
previousPeriodicFingerprint = null;
|
|
2620
2790
|
sendWorkspaceSyncResult(authority, result);
|
|
2621
2791
|
return result;
|
|
2622
2792
|
} finally {
|
|
@@ -2632,6 +2802,24 @@ async function startWorker(options) {
|
|
|
2632
2802
|
let heartbeatBusyGrace = null;
|
|
2633
2803
|
let heartbeatTimedOut = false;
|
|
2634
2804
|
let heartbeatWatchdog;
|
|
2805
|
+
const handleGracefulShutdown = () => {
|
|
2806
|
+
if (shutdownAfterClose) return;
|
|
2807
|
+
shutdownAfterClose = true;
|
|
2808
|
+
void waitForWorkspaceCheckpointOnShutdown({
|
|
2809
|
+
snapshot: () => workspaceConvergence.snapshot(),
|
|
2810
|
+
forceCheckpoint: () => armWorkspaceCheckpoint({ type: "manual", detail: "graceful worker shutdown" }, { delayMs: 0 })
|
|
2811
|
+
}).then((outcome) => {
|
|
2812
|
+
process.stdout.write(`[r5d-worker] graceful workspace checkpoint ${outcome}
|
|
2813
|
+
`);
|
|
2814
|
+
if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
|
|
2815
|
+
ws.close(1e3, "Worker shutting down");
|
|
2816
|
+
} else {
|
|
2817
|
+
process.exit(0);
|
|
2818
|
+
}
|
|
2819
|
+
});
|
|
2820
|
+
};
|
|
2821
|
+
process.on("SIGTERM", handleGracefulShutdown);
|
|
2822
|
+
process.on("SIGINT", handleGracefulShutdown);
|
|
2635
2823
|
const stopHeartbeatWatchdog = () => {
|
|
2636
2824
|
if (heartbeatWatchdog) {
|
|
2637
2825
|
clearInterval(heartbeatWatchdog);
|
|
@@ -2662,7 +2850,8 @@ async function startWorker(options) {
|
|
|
2662
2850
|
updateClis: true,
|
|
2663
2851
|
canonicalResolverCheckout: true,
|
|
2664
2852
|
browserPortForwarding: true,
|
|
2665
|
-
globalWorkspaceSyncLeaseV1: true
|
|
2853
|
+
globalWorkspaceSyncLeaseV1: true,
|
|
2854
|
+
eventualWorkspaceSyncV1: true
|
|
2666
2855
|
},
|
|
2667
2856
|
projectRoot: projectsRoot,
|
|
2668
2857
|
artifactRoot,
|
|
@@ -2689,100 +2878,123 @@ async function startWorker(options) {
|
|
|
2689
2878
|
return;
|
|
2690
2879
|
}
|
|
2691
2880
|
if (message.type === "workspace_manifest") {
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
allManifestCheckouts().map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2699
|
-
);
|
|
2700
|
-
const migratedProjectIds = migrateLegacyProjectRoots(projectsRoot, message.projects);
|
|
2701
|
-
manifestByProjectId.clear();
|
|
2702
|
-
for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
|
|
2703
|
-
const currentCheckouts = allManifestCheckouts();
|
|
2704
|
-
const currentCheckoutKeys = new Set(
|
|
2705
|
-
currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2706
|
-
);
|
|
2707
|
-
for (const key of pendingManifestCheckouts.keys()) {
|
|
2708
|
-
if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
|
|
2709
|
-
}
|
|
2710
|
-
for (const checkout of currentCheckouts) {
|
|
2711
|
-
const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
|
|
2712
|
-
if (previousCheckoutKeys.has(key)) continue;
|
|
2713
|
-
const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
|
|
2714
|
-
if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
|
|
2715
|
-
}
|
|
2716
|
-
previousPeriodicFingerprint = null;
|
|
2717
|
-
process.stdout.write(
|
|
2718
|
-
`[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
|
|
2719
|
-
`
|
|
2720
|
-
);
|
|
2881
|
+
const revisionToken = workspaceManifestRevision.begin(message.revision);
|
|
2882
|
+
workspaceReady = false;
|
|
2883
|
+
const manifestAdmission = inspectWorkspaceManifestAdmission({
|
|
2884
|
+
projectsRoot,
|
|
2885
|
+
workspaceShadowRoot,
|
|
2886
|
+
projects: message.projects
|
|
2721
2887
|
});
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2888
|
+
const releaseManifestHydration = await acquireWorkspaceManifestVisibleLease(
|
|
2889
|
+
workspaceFilesystemWriters,
|
|
2890
|
+
manifestAdmission
|
|
2891
|
+
);
|
|
2892
|
+
let completed = false;
|
|
2893
|
+
try {
|
|
2894
|
+
completed = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2895
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2896
|
+
const firstBootstrap = manifestAdmission.firstBootstrap;
|
|
2897
|
+
configureGitHubAuth(message.githubCredential);
|
|
2898
|
+
visibleGitIdentity = message.gitIdentity;
|
|
2899
|
+
workspaceRemoteUrl = message.workspaceRemoteUrl;
|
|
2900
|
+
const migratedProjectIds = releaseManifestHydration ? migrateLegacyProjectRoots(projectsRoot, message.projects) : [];
|
|
2901
|
+
manifestByProjectId.clear();
|
|
2902
|
+
for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
|
|
2903
|
+
const currentCheckouts = allManifestCheckouts();
|
|
2904
|
+
const currentCheckoutKeys = new Set(
|
|
2905
|
+
currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2906
|
+
);
|
|
2907
|
+
for (const key of pendingManifestCheckouts.keys()) {
|
|
2908
|
+
if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
|
|
2909
|
+
}
|
|
2910
|
+
for (const checkout of currentCheckouts) {
|
|
2911
|
+
const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
|
|
2912
|
+
const branchPath = path.join(
|
|
2913
|
+
projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId),
|
|
2914
|
+
checkout.branchName
|
|
2735
2915
|
);
|
|
2736
|
-
if (
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2916
|
+
if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
|
|
2917
|
+
}
|
|
2918
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2919
|
+
const created = releaseManifestHydration ? ensureVisibleWorkspaceCheckouts(currentCheckouts, { strictCanonicalRevalidation: true }) : [];
|
|
2920
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2921
|
+
const bootstrapped = await convergeWorkspaceHead(
|
|
2922
|
+
workspaceSyncInput({ type: "connect" }, { newVisibleCheckouts: created }),
|
|
2923
|
+
{
|
|
2924
|
+
// An existing checkout becomes routable after a fetch. Its
|
|
2925
|
+
// visible tree converges asynchronously; first bootstrap alone
|
|
2926
|
+
// must materialize the canonical snapshot before readiness.
|
|
2927
|
+
// Re-check the incident immediately before convergence so a
|
|
2928
|
+
// concurrently delivered fence forces fetch-only behavior.
|
|
2929
|
+
hydrateVisible: firstBootstrap && !activeWorkspaceIncidentId
|
|
2744
2930
|
}
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
};
|
|
2756
|
-
});
|
|
2757
|
-
if (observation.fingerprint === emptyFingerprint) {
|
|
2758
|
-
recentWorkspaceSyncProof.recordStable({
|
|
2759
|
-
observationGeneration: observation.observationGeneration,
|
|
2760
|
-
canonicalHead: observation.canonicalHead,
|
|
2761
|
-
scopeKey: observation.scopeKey,
|
|
2762
|
-
observedAtMs: Date.now(),
|
|
2763
|
-
workerBusy: observation.workerBusy || activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress
|
|
2764
|
-
});
|
|
2765
|
-
previousPeriodicFingerprint = null;
|
|
2766
|
-
return;
|
|
2931
|
+
);
|
|
2932
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2933
|
+
workspaceConvergence.setLocalHead(bootstrapped.localHead);
|
|
2934
|
+
workspaceConvergence.observeCanonicalHead(bootstrapped.canonicalHead);
|
|
2935
|
+
for (const checkout of currentCheckouts) {
|
|
2936
|
+
const manifest = manifestByProjectId.get(checkout.projectId);
|
|
2937
|
+
if (!manifest) continue;
|
|
2938
|
+
const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
|
|
2939
|
+
if (hasNormalVisibleGitDir(branchPath)) {
|
|
2940
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(checkout.projectId, checkout.branchName));
|
|
2767
2941
|
}
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2942
|
+
}
|
|
2943
|
+
process.stdout.write(
|
|
2944
|
+
`[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
|
|
2945
|
+
`
|
|
2946
|
+
);
|
|
2947
|
+
return true;
|
|
2948
|
+
});
|
|
2949
|
+
} finally {
|
|
2950
|
+
releaseManifestHydration?.();
|
|
2951
|
+
}
|
|
2952
|
+
if (!completed || !workspaceManifestRevision.complete(revisionToken)) return;
|
|
2953
|
+
workspaceReady = true;
|
|
2954
|
+
sendWorkspaceReady();
|
|
2955
|
+
schedulePendingManifestCheckoutHydration();
|
|
2956
|
+
if (!workspaceSafetyScan) {
|
|
2957
|
+
workspaceSafetyScan = setInterval(() => {
|
|
2958
|
+
if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || workspaceSafetyScanInFlight) {
|
|
2959
|
+
return;
|
|
2960
|
+
}
|
|
2961
|
+
workspaceSafetyScanInFlight = true;
|
|
2962
|
+
void (async () => {
|
|
2963
|
+
const before = workspaceConvergence.snapshot();
|
|
2964
|
+
const mayHydrateVisible = !workerHasActiveWriter() && before.dirtyGeneration === before.publishedGeneration;
|
|
2965
|
+
const releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
|
|
2966
|
+
const hydrateVisible = releaseVisibleHydration !== null;
|
|
2967
|
+
try {
|
|
2968
|
+
const converged = await workspaceSyncSingleFlight.runExclusive(
|
|
2969
|
+
() => convergeWorkspaceHead(workspaceSyncInput({ type: "periodic", detail: "idle convergence scan" }), {
|
|
2970
|
+
hydrateVisible
|
|
2971
|
+
})
|
|
2972
|
+
);
|
|
2973
|
+
workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
|
|
2974
|
+
if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
|
|
2975
|
+
if (converged.localChanges && hydrateVisible) {
|
|
2976
|
+
markWorkspaceDirty({ type: "periodic", detail: "idle safety scan" });
|
|
2977
|
+
}
|
|
2978
|
+
sendWorkspaceReady();
|
|
2979
|
+
} finally {
|
|
2980
|
+
releaseVisibleHydration?.();
|
|
2772
2981
|
}
|
|
2773
|
-
previousPeriodicFingerprint = null;
|
|
2774
|
-
requestWorkspaceSync({ type: "periodic" }, observation.fingerprint);
|
|
2775
2982
|
})().catch((error) => {
|
|
2776
2983
|
process.stderr.write(
|
|
2777
|
-
`[r5d-worker]
|
|
2984
|
+
`[r5d-worker] idle workspace safety scan failed: ${error instanceof Error ? error.message : String(error)}
|
|
2778
2985
|
`
|
|
2779
2986
|
);
|
|
2780
2987
|
}).finally(() => {
|
|
2781
|
-
|
|
2988
|
+
workspaceSafetyScanInFlight = false;
|
|
2782
2989
|
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2783
2990
|
});
|
|
2784
|
-
},
|
|
2785
|
-
|
|
2991
|
+
}, WORKSPACE_IDLE_SAFETY_SCAN_MS);
|
|
2992
|
+
workspaceSafetyScan.unref();
|
|
2993
|
+
}
|
|
2994
|
+
pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
|
|
2995
|
+
void convergeAvailableWorkspaceHead();
|
|
2996
|
+
if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
2997
|
+
armWorkspaceCheckpoint({ type: "periodic", detail: "resume dirty generation after reconnect" }, { delayMs: 0 });
|
|
2786
2998
|
}
|
|
2787
2999
|
return;
|
|
2788
3000
|
}
|
|
@@ -2802,12 +3014,189 @@ async function startWorker(options) {
|
|
|
2802
3014
|
confirmationReason: message.confirmationReason,
|
|
2803
3015
|
resetToCanonical: message.resetToCanonical,
|
|
2804
3016
|
skipVisibleMirror: message.skipVisibleMirror
|
|
2805
|
-
}
|
|
2806
|
-
message.canonicalHead,
|
|
2807
|
-
message.allowRecentNoChangeFastPath
|
|
3017
|
+
}
|
|
2808
3018
|
);
|
|
2809
3019
|
return;
|
|
2810
3020
|
}
|
|
3021
|
+
if (message.type === "workspace_head_available") {
|
|
3022
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3023
|
+
pendingWorkspaceHead = message.canonicalHead;
|
|
3024
|
+
sendWorkspaceReady();
|
|
3025
|
+
void convergeAvailableWorkspaceHead();
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
if (message.type === "workspace_checkpoint_now") {
|
|
3029
|
+
armWorkspaceCheckpoint(message.trigger, { delayMs: 0, requestId: message.requestId });
|
|
3030
|
+
return;
|
|
3031
|
+
}
|
|
3032
|
+
if (message.type === "workspace_checkpoint_authority") {
|
|
3033
|
+
const checkpoint = pendingCheckpoint;
|
|
3034
|
+
if (!checkpoint || checkpoint.requestId !== message.requestId) {
|
|
3035
|
+
process.stderr.write(`[r5d-worker] ignoring checkpoint authority for unknown request ${message.requestId}
|
|
3036
|
+
`);
|
|
3037
|
+
return;
|
|
3038
|
+
}
|
|
3039
|
+
if (message.status !== "granted") {
|
|
3040
|
+
checkpoint.releaseCheckpointAdmission();
|
|
3041
|
+
pendingCheckpoint = void 0;
|
|
3042
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3043
|
+
const applyBlockedResponse = message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId);
|
|
3044
|
+
if (applyBlockedResponse) {
|
|
3045
|
+
workspaceConvergence.block();
|
|
3046
|
+
if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
|
|
3047
|
+
} else {
|
|
3048
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3049
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3050
|
+
if (message.status === "busy") {
|
|
3051
|
+
armWorkspaceCheckpoint(checkpoint.trigger, {
|
|
3052
|
+
delayMs: message.retryAfterMs ?? 1e3,
|
|
3053
|
+
requestId: checkpoint.requestId
|
|
3054
|
+
});
|
|
3055
|
+
} else if (!activeWorkspaceIncidentId) {
|
|
3056
|
+
const explicit = explicitWorkspaceCheckpoints.next();
|
|
3057
|
+
if (explicit) {
|
|
3058
|
+
armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
|
|
3059
|
+
} else if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3060
|
+
armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: 0 });
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
}
|
|
3064
|
+
sendWorkspaceReady();
|
|
3065
|
+
return;
|
|
3066
|
+
}
|
|
3067
|
+
checkpoint.attemptId = message.attemptId;
|
|
3068
|
+
checkpoint.authorityToken = message.authorityToken;
|
|
3069
|
+
checkpoint.workInFlight = true;
|
|
3070
|
+
pendingCheckpoint = checkpoint;
|
|
3071
|
+
if (!checkpoint.admissionHeld) {
|
|
3072
|
+
const releaseCheckpointAdmission = await workspaceFilesystemWriters.acquireCheckpoint();
|
|
3073
|
+
if (currentWorkerSocket !== ws || pendingCheckpoint?.requestId !== checkpoint.requestId || pendingCheckpoint.attemptId !== message.attemptId) {
|
|
3074
|
+
releaseCheckpointAdmission();
|
|
3075
|
+
checkpoint.workInFlight = false;
|
|
3076
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3077
|
+
return;
|
|
3078
|
+
}
|
|
3079
|
+
checkpoint.admissionHeld = true;
|
|
3080
|
+
checkpoint.releaseCheckpointAdmission = releaseCheckpointAdmission;
|
|
3081
|
+
}
|
|
3082
|
+
workspaceConvergence.beginPreparing();
|
|
3083
|
+
sendWorkspaceReady();
|
|
3084
|
+
let prepareStartedAt = Date.now();
|
|
3085
|
+
let synchronizeStartedAt = prepareStartedAt;
|
|
3086
|
+
let synchronizeStarted = false;
|
|
3087
|
+
let result;
|
|
3088
|
+
try {
|
|
3089
|
+
result = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
3090
|
+
prepareStartedAt = Date.now();
|
|
3091
|
+
const pendingTargets = [...pendingManifestCheckouts.values()];
|
|
3092
|
+
const createdCheckouts = ensureVisibleWorkspaceCheckouts(pendingTargets, { strictCanonicalRevalidation: true });
|
|
3093
|
+
synchronizeStartedAt = Date.now();
|
|
3094
|
+
synchronizeStarted = true;
|
|
3095
|
+
const prepared = await synchronizeWorkspace(
|
|
3096
|
+
workspaceSyncInput(checkpoint.trigger, {
|
|
3097
|
+
attemptId: message.attemptId,
|
|
3098
|
+
quarantineRef: message.quarantineRef,
|
|
3099
|
+
expectedCanonicalHead: message.expectedHead,
|
|
3100
|
+
newVisibleCheckouts: [...createdCheckouts, ...pendingTargets]
|
|
3101
|
+
})
|
|
3102
|
+
);
|
|
3103
|
+
for (const target of pendingTargets) {
|
|
3104
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
|
|
3105
|
+
}
|
|
3106
|
+
return prepared.outcome === "updated" ? { ...prepared, outcome: "no_change" } : prepared;
|
|
3107
|
+
});
|
|
3108
|
+
} catch (error) {
|
|
3109
|
+
if (!synchronizeStarted) synchronizeStartedAt = Date.now();
|
|
3110
|
+
result = {
|
|
3111
|
+
type: "workspace_sync",
|
|
3112
|
+
attemptId: message.attemptId,
|
|
3113
|
+
workerLabel: label,
|
|
3114
|
+
trigger: checkpoint.trigger,
|
|
3115
|
+
outcome: "failed",
|
|
3116
|
+
startingHead: message.expectedHead,
|
|
3117
|
+
expectedHead: message.expectedHead,
|
|
3118
|
+
rebaseCount: 0,
|
|
3119
|
+
diffSizeBytes: 0,
|
|
3120
|
+
gitStatus: "",
|
|
3121
|
+
affectedProjects: [],
|
|
3122
|
+
affectedPaths: [],
|
|
3123
|
+
discardedPaths: [],
|
|
3124
|
+
localChangesDiscarded: false,
|
|
3125
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3126
|
+
};
|
|
3127
|
+
}
|
|
3128
|
+
const finishedAt = Date.now();
|
|
3129
|
+
result = {
|
|
3130
|
+
...result,
|
|
3131
|
+
dirtyGeneration: checkpoint.dirtyGeneration,
|
|
3132
|
+
telemetry: workspaceCheckpointTelemetry({
|
|
3133
|
+
requestedAt: checkpoint.requestedAt,
|
|
3134
|
+
prepareStartedAt,
|
|
3135
|
+
synchronizeStartedAt,
|
|
3136
|
+
finishedAt
|
|
3137
|
+
})
|
|
3138
|
+
};
|
|
3139
|
+
workspaceConvergence.beginSubmitting();
|
|
3140
|
+
sendWorkspaceReady();
|
|
3141
|
+
try {
|
|
3142
|
+
ws.send(
|
|
3143
|
+
JSON.stringify({
|
|
3144
|
+
type: "workspace_checkpoint_result",
|
|
3145
|
+
requestId: checkpoint.requestId,
|
|
3146
|
+
attemptId: message.attemptId,
|
|
3147
|
+
authorityToken: message.authorityToken,
|
|
3148
|
+
dirtyGeneration: checkpoint.dirtyGeneration,
|
|
3149
|
+
result
|
|
3150
|
+
})
|
|
3151
|
+
);
|
|
3152
|
+
} catch (error) {
|
|
3153
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3154
|
+
throw error;
|
|
3155
|
+
} finally {
|
|
3156
|
+
checkpoint.workInFlight = false;
|
|
3157
|
+
checkpoint.releaseCheckpointAdmission();
|
|
3158
|
+
if (currentWorkerSocket !== ws) workspaceConvergence.resetTransientCheckpointState();
|
|
3159
|
+
}
|
|
3160
|
+
return;
|
|
3161
|
+
}
|
|
3162
|
+
if (message.type === "workspace_checkpoint_completed") {
|
|
3163
|
+
const checkpoint = pendingCheckpoint;
|
|
3164
|
+
if (!checkpoint || checkpoint.requestId !== message.requestId || checkpoint.attemptId !== message.attemptId) {
|
|
3165
|
+
process.stderr.write(`[r5d-worker] ignoring completion for unknown checkpoint ${message.requestId}
|
|
3166
|
+
`);
|
|
3167
|
+
return;
|
|
3168
|
+
}
|
|
3169
|
+
pendingCheckpoint = void 0;
|
|
3170
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3171
|
+
if (message.status === "published" || message.status === "no_change") {
|
|
3172
|
+
workspaceConvergence.complete({
|
|
3173
|
+
generation: Math.min(message.publishedGeneration ?? checkpoint.dirtyGeneration, checkpoint.dirtyGeneration),
|
|
3174
|
+
canonicalHead: message.canonicalHead,
|
|
3175
|
+
published: true
|
|
3176
|
+
});
|
|
3177
|
+
} else if (message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId)) {
|
|
3178
|
+
workspaceConvergence.block();
|
|
3179
|
+
if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
|
|
3180
|
+
} else {
|
|
3181
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3182
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3183
|
+
}
|
|
3184
|
+
sendWorkspaceReady();
|
|
3185
|
+
const explicit = message.status === "retry" && checkpoint.explicit ? { requestId: checkpoint.requestId, trigger: checkpoint.trigger } : explicitWorkspaceCheckpoints.next();
|
|
3186
|
+
if (explicit) {
|
|
3187
|
+
armWorkspaceCheckpoint(explicit.trigger, {
|
|
3188
|
+
delayMs: message.retryAfterMs ?? 0,
|
|
3189
|
+
requestId: explicit.requestId
|
|
3190
|
+
});
|
|
3191
|
+
} else if (message.status === "retry" || message.status === "failed" || workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3192
|
+
armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: message.retryAfterMs ?? 1e3 });
|
|
3193
|
+
}
|
|
3194
|
+
if (message.canonicalHead !== workspaceConvergence.snapshot().localHead) {
|
|
3195
|
+
pendingWorkspaceHead = message.canonicalHead;
|
|
3196
|
+
void convergeAvailableWorkspaceHead();
|
|
3197
|
+
}
|
|
3198
|
+
return;
|
|
3199
|
+
}
|
|
2811
3200
|
if (message.type === "sync_session_artifacts") {
|
|
2812
3201
|
sessionArtifactSyncRequestsInFlight += 1;
|
|
2813
3202
|
try {
|
|
@@ -2835,18 +3224,27 @@ async function startWorker(options) {
|
|
|
2835
3224
|
return;
|
|
2836
3225
|
}
|
|
2837
3226
|
if (message.type === "workspace_incident_updated") {
|
|
2838
|
-
|
|
3227
|
+
workspaceIncidentOrdering.observe(message);
|
|
2839
3228
|
const previousIncidentId = activeWorkspaceIncidentId;
|
|
2840
3229
|
activeWorkspaceIncidentId = applyWorkspaceIncidentUpdate(activeWorkspaceIncidentId, message);
|
|
2841
|
-
previousPeriodicFingerprint = null;
|
|
2842
3230
|
const releasedHead = releasePendingWorkspaceHead(
|
|
2843
3231
|
previousIncidentId,
|
|
2844
3232
|
activeWorkspaceIncidentId,
|
|
2845
3233
|
message.status === "resolved" || message.status === "confirmed" || message.status === "reset" ? message.incidentId : null
|
|
2846
3234
|
);
|
|
2847
|
-
|
|
2848
|
-
|
|
3235
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3236
|
+
if (releasedHead.fetchAuthoritativeHead) {
|
|
3237
|
+
pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
|
|
3238
|
+
void convergeAvailableWorkspaceHead();
|
|
3239
|
+
}
|
|
3240
|
+
const explicit = !activeWorkspaceIncidentId ? explicitWorkspaceCheckpoints.next() : void 0;
|
|
3241
|
+
if (explicit) {
|
|
3242
|
+
armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
|
|
3243
|
+
} else if (!activeWorkspaceIncidentId && workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3244
|
+
armWorkspaceCheckpoint({ type: "periodic", detail: "resume after workspace incident" }, { delayMs: 0 });
|
|
2849
3245
|
}
|
|
3246
|
+
if (!activeWorkspaceIncidentId) schedulePendingManifestCheckoutHydration();
|
|
3247
|
+
sendWorkspaceReady();
|
|
2850
3248
|
return;
|
|
2851
3249
|
}
|
|
2852
3250
|
if (message.type === "exec_terminal_ack") {
|
|
@@ -2882,7 +3280,6 @@ async function startWorker(options) {
|
|
|
2882
3280
|
return;
|
|
2883
3281
|
}
|
|
2884
3282
|
cliUpdateInProgress = true;
|
|
2885
|
-
recentWorkspaceSyncProof.invalidate();
|
|
2886
3283
|
try {
|
|
2887
3284
|
const result = await installCliUpdate(message);
|
|
2888
3285
|
sendWorkerMessage(ws, {
|
|
@@ -2945,9 +3342,10 @@ async function startWorker(options) {
|
|
|
2945
3342
|
});
|
|
2946
3343
|
return;
|
|
2947
3344
|
}
|
|
2948
|
-
|
|
3345
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2949
3346
|
const releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
|
|
2950
|
-
let
|
|
3347
|
+
let mutationLeaseTransferred = false;
|
|
3348
|
+
let writerLeaseTransferred = false;
|
|
2951
3349
|
try {
|
|
2952
3350
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
2953
3351
|
process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
|
|
@@ -2957,9 +3355,19 @@ async function startWorker(options) {
|
|
|
2957
3355
|
message,
|
|
2958
3356
|
resolvedTarget,
|
|
2959
3357
|
planRoot,
|
|
2960
|
-
...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {}
|
|
3358
|
+
...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {},
|
|
3359
|
+
...targetMayMutateVisibleWorkspace(message.target) ? {
|
|
3360
|
+
onTerminal: () => {
|
|
3361
|
+
markWorkspaceDirty(
|
|
3362
|
+
{ type: "process_terminal", detail: `pty ${message.ptyId} completed` },
|
|
3363
|
+
true
|
|
3364
|
+
);
|
|
3365
|
+
releaseWorkspaceWriter?.();
|
|
3366
|
+
}
|
|
3367
|
+
} : {}
|
|
2961
3368
|
});
|
|
2962
|
-
|
|
3369
|
+
mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
|
|
3370
|
+
writerLeaseTransferred = releaseWorkspaceWriter !== void 0;
|
|
2963
3371
|
} catch (error) {
|
|
2964
3372
|
sendWorkerMessage(ws, {
|
|
2965
3373
|
type: "pty_error",
|
|
@@ -2968,12 +3376,16 @@ async function startWorker(options) {
|
|
|
2968
3376
|
error: error instanceof Error ? error.message : String(error)
|
|
2969
3377
|
});
|
|
2970
3378
|
} finally {
|
|
2971
|
-
if (!
|
|
3379
|
+
if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
|
|
3380
|
+
if (!writerLeaseTransferred) releaseWorkspaceWriter?.();
|
|
2972
3381
|
}
|
|
2973
3382
|
return;
|
|
2974
3383
|
}
|
|
2975
3384
|
if (message.type === "pty_input") {
|
|
2976
|
-
|
|
3385
|
+
const activePty = activePtys.get(message.ptyId);
|
|
3386
|
+
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
3387
|
+
markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` });
|
|
3388
|
+
}
|
|
2977
3389
|
writePty(ws, message);
|
|
2978
3390
|
return;
|
|
2979
3391
|
}
|
|
@@ -2982,8 +3394,11 @@ async function startWorker(options) {
|
|
|
2982
3394
|
return;
|
|
2983
3395
|
}
|
|
2984
3396
|
if (message.type === "pty_close") {
|
|
2985
|
-
|
|
3397
|
+
const activePty = activePtys.get(message.ptyId);
|
|
2986
3398
|
closePty(message);
|
|
3399
|
+
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
3400
|
+
armWorkspaceCheckpoint({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, { delayMs: 0 });
|
|
3401
|
+
}
|
|
2987
3402
|
return;
|
|
2988
3403
|
}
|
|
2989
3404
|
if (message.type === "exec") {
|
|
@@ -2997,27 +3412,37 @@ async function startWorker(options) {
|
|
|
2997
3412
|
});
|
|
2998
3413
|
return;
|
|
2999
3414
|
}
|
|
3000
|
-
|
|
3001
|
-
let result;
|
|
3415
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
3002
3416
|
try {
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3417
|
+
let result;
|
|
3418
|
+
try {
|
|
3419
|
+
const runCommand = async () => {
|
|
3420
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3421
|
+
process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
|
|
3006
3422
|
`);
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3423
|
+
return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
|
|
3424
|
+
};
|
|
3425
|
+
result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
|
|
3426
|
+
} catch (error) {
|
|
3427
|
+
result = {
|
|
3428
|
+
type: "exec_result",
|
|
3429
|
+
requestId: message.requestId,
|
|
3430
|
+
stdout: "",
|
|
3431
|
+
stderr: "",
|
|
3432
|
+
exitCode: 1,
|
|
3433
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3434
|
+
};
|
|
3435
|
+
}
|
|
3436
|
+
ws.send(JSON.stringify(result));
|
|
3437
|
+
if (targetMayMutateVisibleWorkspace(message.target)) {
|
|
3438
|
+
markWorkspaceDirty(
|
|
3439
|
+
{ type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
|
|
3440
|
+
true
|
|
3441
|
+
);
|
|
3442
|
+
}
|
|
3443
|
+
} finally {
|
|
3444
|
+
releaseWorkspaceWriter?.();
|
|
3019
3445
|
}
|
|
3020
|
-
ws.send(JSON.stringify(result));
|
|
3021
3446
|
return;
|
|
3022
3447
|
}
|
|
3023
3448
|
if (message.type === "exec_start") {
|
|
@@ -3030,25 +3455,40 @@ async function startWorker(options) {
|
|
|
3030
3455
|
});
|
|
3031
3456
|
return;
|
|
3032
3457
|
}
|
|
3033
|
-
recentWorkspaceSyncProof.invalidate();
|
|
3034
3458
|
sendWorkerMessage(ws, {
|
|
3035
3459
|
type: "exec_accepted",
|
|
3036
3460
|
requestId: message.requestId,
|
|
3037
3461
|
runId: message.runId
|
|
3038
3462
|
});
|
|
3039
3463
|
const runCommand = async () => {
|
|
3040
|
-
const
|
|
3041
|
-
|
|
3464
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
3465
|
+
try {
|
|
3466
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3467
|
+
process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
|
|
3042
3468
|
`);
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3469
|
+
await executeStreamingCommand({
|
|
3470
|
+
ws,
|
|
3471
|
+
message,
|
|
3472
|
+
resolvedTarget,
|
|
3473
|
+
baseUrl,
|
|
3474
|
+
token,
|
|
3475
|
+
artifactRoot,
|
|
3476
|
+
planRoot
|
|
3477
|
+
});
|
|
3478
|
+
} finally {
|
|
3479
|
+
if (targetMayMutateVisibleWorkspace(message.target)) {
|
|
3480
|
+
markWorkspaceDirty(
|
|
3481
|
+
{
|
|
3482
|
+
type: "process_terminal",
|
|
3483
|
+
sessionId: message.sessionId,
|
|
3484
|
+
processRunId: message.runId,
|
|
3485
|
+
detail: "process completed"
|
|
3486
|
+
},
|
|
3487
|
+
true
|
|
3488
|
+
);
|
|
3489
|
+
}
|
|
3490
|
+
releaseWorkspaceWriter?.();
|
|
3491
|
+
}
|
|
3052
3492
|
};
|
|
3053
3493
|
const execution = runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
|
|
3054
3494
|
void execution.catch((error) => {
|
|
@@ -3062,34 +3502,46 @@ async function startWorker(options) {
|
|
|
3062
3502
|
return;
|
|
3063
3503
|
}
|
|
3064
3504
|
if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes") {
|
|
3065
|
-
|
|
3505
|
+
const releaseWorkspaceWriter = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
3066
3506
|
try {
|
|
3067
|
-
|
|
3068
|
-
const
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3507
|
+
try {
|
|
3508
|
+
const result = await workspaceSyncSingleFlight.runMutation(async () => {
|
|
3509
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3510
|
+
return await executeOperation({
|
|
3511
|
+
message,
|
|
3512
|
+
resolvedTarget,
|
|
3513
|
+
baseUrl,
|
|
3514
|
+
token,
|
|
3515
|
+
artifactRoot,
|
|
3516
|
+
planRoot
|
|
3517
|
+
});
|
|
3076
3518
|
});
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
}
|
|
3092
|
-
)
|
|
3519
|
+
ws.send(
|
|
3520
|
+
JSON.stringify({
|
|
3521
|
+
type: "operation_result",
|
|
3522
|
+
requestId: message.requestId,
|
|
3523
|
+
result
|
|
3524
|
+
})
|
|
3525
|
+
);
|
|
3526
|
+
if ((message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target)) {
|
|
3527
|
+
markWorkspaceDirty({
|
|
3528
|
+
type: message.type,
|
|
3529
|
+
sessionId: message.sessionId,
|
|
3530
|
+
toolCallId: message.requestId,
|
|
3531
|
+
...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
|
|
3532
|
+
});
|
|
3533
|
+
}
|
|
3534
|
+
} catch (error) {
|
|
3535
|
+
ws.send(
|
|
3536
|
+
JSON.stringify({
|
|
3537
|
+
type: "operation_result",
|
|
3538
|
+
requestId: message.requestId,
|
|
3539
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3540
|
+
})
|
|
3541
|
+
);
|
|
3542
|
+
}
|
|
3543
|
+
} finally {
|
|
3544
|
+
releaseWorkspaceWriter?.();
|
|
3093
3545
|
}
|
|
3094
3546
|
return;
|
|
3095
3547
|
}
|
|
@@ -3111,11 +3563,24 @@ async function startWorker(options) {
|
|
|
3111
3563
|
});
|
|
3112
3564
|
});
|
|
3113
3565
|
ws.addEventListener("close", (event) => {
|
|
3114
|
-
|
|
3566
|
+
process.off("SIGTERM", handleGracefulShutdown);
|
|
3567
|
+
process.off("SIGINT", handleGracefulShutdown);
|
|
3115
3568
|
stopHeartbeatWatchdog();
|
|
3116
|
-
if (
|
|
3117
|
-
|
|
3118
|
-
|
|
3569
|
+
if (workspaceCheckpointTimer) {
|
|
3570
|
+
clearTimeout(workspaceCheckpointTimer);
|
|
3571
|
+
workspaceCheckpointTimer = void 0;
|
|
3572
|
+
}
|
|
3573
|
+
if (!pendingCheckpoint?.workInFlight) {
|
|
3574
|
+
pendingCheckpoint?.releaseCheckpointAdmission();
|
|
3575
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3576
|
+
}
|
|
3577
|
+
if (workspaceSafetyScan) {
|
|
3578
|
+
clearInterval(workspaceSafetyScan);
|
|
3579
|
+
workspaceSafetyScan = void 0;
|
|
3580
|
+
}
|
|
3581
|
+
if (workspaceManifestHydrationTimer) {
|
|
3582
|
+
clearTimeout(workspaceManifestHydrationTimer);
|
|
3583
|
+
workspaceManifestHydrationTimer = void 0;
|
|
3119
3584
|
}
|
|
3120
3585
|
if (terminalReplayTimer) {
|
|
3121
3586
|
clearInterval(terminalReplayTimer);
|
|
@@ -3131,6 +3596,9 @@ async function startWorker(options) {
|
|
|
3131
3596
|
if (reloadAfterClose) {
|
|
3132
3597
|
process.exit(WORKER_RELOAD_EXIT_CODE);
|
|
3133
3598
|
}
|
|
3599
|
+
if (shutdownAfterClose) {
|
|
3600
|
+
process.exit(0);
|
|
3601
|
+
}
|
|
3134
3602
|
if (activeProcesses.size > 0 || pendingProcessTerminals.size > 0) {
|
|
3135
3603
|
process.stderr.write(
|
|
3136
3604
|
`[r5d-worker] ${activeProcesses.size} process(es) active and ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
|