@ricsam/r5d-worker 0.0.57 → 0.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/main.cjs +717 -140
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/workspace-convergence.cjs +320 -0
- package/dist/cjs/workspace-incident-state.cjs +32 -3
- package/dist/cjs/workspace-manifest-admission.cjs +70 -0
- package/dist/cjs/workspace-sync.cjs +259 -69
- package/dist/mjs/main.mjs +736 -143
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/workspace-convergence.mjs +275 -0
- package/dist/mjs/workspace-incident-state.mjs +31 -3
- package/dist/mjs/workspace-manifest-admission.mjs +35 -0
- package/dist/mjs/workspace-sync.mjs +256 -68
- package/dist/types/workspace-convergence.d.ts +102 -0
- package/dist/types/workspace-incident-state.d.ts +15 -1
- package/dist/types/workspace-manifest-admission.d.ts +27 -0
- package/dist/types/workspace-sync.d.ts +31 -2
- package/package.json +1 -1
package/dist/mjs/main.mjs
CHANGED
|
@@ -13,8 +13,27 @@ import {
|
|
|
13
13
|
} from "./heartbeat.mjs";
|
|
14
14
|
import { terminateProcessTree } from "./process-tree.mjs";
|
|
15
15
|
import { openWorkerPortForwardRelay } from "./port-forward-client.mjs";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
applyWorkspaceIncidentUpdate,
|
|
18
|
+
releasePendingWorkspaceHead,
|
|
19
|
+
WorkspaceIncidentOrderingFence
|
|
20
|
+
} from "./workspace-incident-state.mjs";
|
|
17
21
|
import { acquireWorkspaceCommandMutation, runWorkspaceCommand } from "./workspace-command-sync-policy.mjs";
|
|
22
|
+
import {
|
|
23
|
+
WORKSPACE_CHECKPOINT_QUIET_MS,
|
|
24
|
+
WORKSPACE_IDLE_SAFETY_SCAN_MS,
|
|
25
|
+
ExplicitWorkspaceCheckpointQueue,
|
|
26
|
+
WorkspaceManifestRevisionFence,
|
|
27
|
+
processWorkspaceConvergenceState,
|
|
28
|
+
processWorkspaceFilesystemWriterGate,
|
|
29
|
+
waitForWorkspaceCheckpointOnShutdown,
|
|
30
|
+
workspaceCheckpointIncidentAction,
|
|
31
|
+
workspaceCheckpointTelemetry
|
|
32
|
+
} from "./workspace-convergence.mjs";
|
|
33
|
+
import {
|
|
34
|
+
acquireWorkspaceManifestVisibleLease,
|
|
35
|
+
inspectWorkspaceManifestAdmission
|
|
36
|
+
} from "./workspace-manifest-admission.mjs";
|
|
18
37
|
import {
|
|
19
38
|
isRetryableWorkerServerStatus,
|
|
20
39
|
superviseWorkerRuntime,
|
|
@@ -25,9 +44,8 @@ import {
|
|
|
25
44
|
} from "./supervisor.mjs";
|
|
26
45
|
import { managedProjectRoot, migrateLegacyProjectRoots, validateManagedBranchName } from "./managed-paths.mjs";
|
|
27
46
|
import {
|
|
28
|
-
WORKSPACE_PERIODIC_SCAN_INTERVAL_MS,
|
|
29
47
|
WorkspaceSyncSingleFlight,
|
|
30
|
-
|
|
48
|
+
convergeWorkspaceHead,
|
|
31
49
|
synchronizeWorkspace,
|
|
32
50
|
workspaceProjectsForSync
|
|
33
51
|
} from "./workspace-sync.mjs";
|
|
@@ -46,6 +64,7 @@ const cancelledProcessRuns = /* @__PURE__ */ new Set();
|
|
|
46
64
|
const activePtys = /* @__PURE__ */ new Map();
|
|
47
65
|
let currentWorkerSocket = null;
|
|
48
66
|
const workspaceSyncSingleFlight = new WorkspaceSyncSingleFlight();
|
|
67
|
+
const workspaceFilesystemWriters = processWorkspaceFilesystemWriterGate();
|
|
49
68
|
let githubCredential = null;
|
|
50
69
|
let visibleGitIdentity = null;
|
|
51
70
|
function defaultConfigPath() {
|
|
@@ -2260,6 +2279,7 @@ async function openPty(input) {
|
|
|
2260
2279
|
onExit: (event) => {
|
|
2261
2280
|
input.releaseWorkspaceMutation?.();
|
|
2262
2281
|
activePtys.delete(input.message.ptyId);
|
|
2282
|
+
input.onTerminal?.();
|
|
2263
2283
|
sendWorkerMessage(input.ws, {
|
|
2264
2284
|
type: "pty_exit",
|
|
2265
2285
|
ptyId: input.message.ptyId,
|
|
@@ -2270,6 +2290,7 @@ async function openPty(input) {
|
|
|
2270
2290
|
onError: (error) => {
|
|
2271
2291
|
input.releaseWorkspaceMutation?.();
|
|
2272
2292
|
activePtys.delete(input.message.ptyId);
|
|
2293
|
+
input.onTerminal?.();
|
|
2273
2294
|
sendWorkerMessage(input.ws, {
|
|
2274
2295
|
type: "pty_error",
|
|
2275
2296
|
requestId: input.message.requestId,
|
|
@@ -2361,14 +2382,25 @@ async function startWorker(options) {
|
|
|
2361
2382
|
const pendingManifestCheckouts = /* @__PURE__ */ new Map();
|
|
2362
2383
|
let workspaceRemoteUrl = null;
|
|
2363
2384
|
let activeWorkspaceIncidentId = null;
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2385
|
+
const workspaceIncidentOrdering = new WorkspaceIncidentOrderingFence();
|
|
2386
|
+
const workspaceConvergence = processWorkspaceConvergenceState(path.join(syncRoot, "workspace-convergence.json"));
|
|
2387
|
+
const workspaceManifestRevision = new WorkspaceManifestRevisionFence();
|
|
2388
|
+
let workspaceReady = false;
|
|
2389
|
+
let workspaceCheckpointTimer;
|
|
2390
|
+
let workspaceSafetyScan;
|
|
2391
|
+
let workspaceManifestHydrationTimer;
|
|
2392
|
+
let workspaceManifestHydrationInFlight = false;
|
|
2393
|
+
let workspaceSafetyScanInFlight = false;
|
|
2394
|
+
let workspaceHeadConvergenceInFlight = false;
|
|
2395
|
+
let pendingWorkspaceHead;
|
|
2396
|
+
let pendingCheckpoint;
|
|
2397
|
+
const explicitWorkspaceCheckpoints = new ExplicitWorkspaceCheckpointQueue();
|
|
2367
2398
|
let terminalReplayTimer;
|
|
2368
2399
|
let workspaceSyncRequestsInFlight = 0;
|
|
2369
2400
|
let sessionArtifactSyncRequestsInFlight = 0;
|
|
2370
2401
|
let cliUpdateInProgress = false;
|
|
2371
2402
|
let reloadAfterClose = false;
|
|
2403
|
+
let shutdownAfterClose = false;
|
|
2372
2404
|
const workspaceSyncInput = (trigger, overrides = {}) => {
|
|
2373
2405
|
if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
|
|
2374
2406
|
const projects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
|
|
@@ -2394,21 +2426,175 @@ async function startWorker(options) {
|
|
|
2394
2426
|
);
|
|
2395
2427
|
}
|
|
2396
2428
|
};
|
|
2397
|
-
const
|
|
2429
|
+
const sendWorkspaceReady = () => {
|
|
2430
|
+
const manifestRevision = workspaceManifestRevision.readyRevision();
|
|
2431
|
+
if (!workspaceReady || !manifestRevision) return;
|
|
2398
2432
|
try {
|
|
2399
2433
|
ws.send(
|
|
2400
2434
|
JSON.stringify({
|
|
2401
|
-
type: "
|
|
2402
|
-
|
|
2435
|
+
type: "workspace_ready",
|
|
2436
|
+
manifestRevision,
|
|
2437
|
+
pendingCheckouts: [...pendingManifestCheckouts.values()].sort(
|
|
2438
|
+
(left, right) => left.projectId.localeCompare(right.projectId) || left.branchName.localeCompare(right.branchName)
|
|
2439
|
+
),
|
|
2440
|
+
...workspaceConvergence.snapshot()
|
|
2441
|
+
})
|
|
2442
|
+
);
|
|
2443
|
+
} catch (error) {
|
|
2444
|
+
process.stderr.write(
|
|
2445
|
+
`[r5d-worker] failed to send workspace readiness: ${error instanceof Error ? error.message : String(error)}
|
|
2446
|
+
`
|
|
2447
|
+
);
|
|
2448
|
+
}
|
|
2449
|
+
};
|
|
2450
|
+
const workerHasActiveWriter = () => activeProcesses.size > 0 || activePtys.size > 0 || cliUpdateInProgress;
|
|
2451
|
+
const targetMayMutateVisibleWorkspace = (target) => target.type === "project" || target.rootProfile === "visible_projects";
|
|
2452
|
+
const armWorkspaceCheckpoint = (trigger, options2 = { delayMs: WORKSPACE_CHECKPOINT_QUIET_MS }) => {
|
|
2453
|
+
workspaceConvergence.schedule();
|
|
2454
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2455
|
+
sendWorkspaceReady();
|
|
2456
|
+
if (options2.requestId && !explicitWorkspaceCheckpoints.schedule({ requestId: options2.requestId, trigger }, pendingCheckpoint?.requestId)) {
|
|
2457
|
+
return;
|
|
2458
|
+
}
|
|
2459
|
+
if (!options2.requestId && explicitWorkspaceCheckpoints.hasScheduled()) return;
|
|
2460
|
+
if (workspaceCheckpointTimer) clearTimeout(workspaceCheckpointTimer);
|
|
2461
|
+
workspaceCheckpointTimer = setTimeout(() => {
|
|
2462
|
+
workspaceCheckpointTimer = void 0;
|
|
2463
|
+
if (options2.requestId) explicitWorkspaceCheckpoints.consumeScheduled(options2.requestId);
|
|
2464
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2465
|
+
if (!workspaceReady || !workspaceRemoteUrl) {
|
|
2466
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2467
|
+
return;
|
|
2468
|
+
}
|
|
2469
|
+
if (activeWorkspaceIncidentId) {
|
|
2470
|
+
workspaceConvergence.block();
|
|
2471
|
+
sendWorkspaceReady();
|
|
2472
|
+
if (workspaceCheckpointIncidentAction(options2.requestId) === "defer_automatic") return;
|
|
2473
|
+
if (pendingCheckpoint) {
|
|
2474
|
+
explicitWorkspaceCheckpoints.enqueue({ requestId: options2.requestId, trigger }, pendingCheckpoint.requestId);
|
|
2475
|
+
return;
|
|
2476
|
+
}
|
|
2477
|
+
const snapshot2 = workspaceConvergence.snapshot();
|
|
2478
|
+
pendingCheckpoint = {
|
|
2479
|
+
requestId: options2.requestId,
|
|
2403
2480
|
trigger,
|
|
2404
|
-
|
|
2481
|
+
dirtyGeneration: snapshot2.dirtyGeneration,
|
|
2482
|
+
explicit: true,
|
|
2483
|
+
admissionHeld: false,
|
|
2484
|
+
workInFlight: false,
|
|
2485
|
+
requestedAt: Date.now(),
|
|
2486
|
+
releaseCheckpointAdmission: () => {
|
|
2487
|
+
}
|
|
2488
|
+
};
|
|
2489
|
+
try {
|
|
2490
|
+
ws.send(
|
|
2491
|
+
JSON.stringify({
|
|
2492
|
+
type: "workspace_checkpoint_request",
|
|
2493
|
+
requestId: options2.requestId,
|
|
2494
|
+
trigger,
|
|
2495
|
+
dirtyGeneration: snapshot2.dirtyGeneration,
|
|
2496
|
+
localHead: snapshot2.localHead,
|
|
2497
|
+
desiredCanonicalHead: snapshot2.desiredCanonicalHead
|
|
2498
|
+
})
|
|
2499
|
+
);
|
|
2500
|
+
} catch (error) {
|
|
2501
|
+
pendingCheckpoint = void 0;
|
|
2502
|
+
workspaceConvergence.defer();
|
|
2503
|
+
process.stderr.write(
|
|
2504
|
+
`[r5d-worker] failed to request blocked workspace checkpoint authority: ${error instanceof Error ? error.message : String(error)}
|
|
2505
|
+
`
|
|
2506
|
+
);
|
|
2507
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, requestId: options2.requestId });
|
|
2508
|
+
}
|
|
2509
|
+
return;
|
|
2510
|
+
}
|
|
2511
|
+
if (pendingCheckpoint || workerHasActiveWriter()) {
|
|
2512
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2513
|
+
return;
|
|
2514
|
+
}
|
|
2515
|
+
const releaseCheckpointAdmission = workspaceFilesystemWriters.tryAcquireCheckpoint();
|
|
2516
|
+
if (!releaseCheckpointAdmission) {
|
|
2517
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2518
|
+
return;
|
|
2519
|
+
}
|
|
2520
|
+
const snapshot = workspaceConvergence.snapshot();
|
|
2521
|
+
const requestId = options2.requestId ?? crypto.randomUUID();
|
|
2522
|
+
pendingCheckpoint = {
|
|
2523
|
+
requestId,
|
|
2524
|
+
trigger,
|
|
2525
|
+
dirtyGeneration: snapshot.dirtyGeneration,
|
|
2526
|
+
explicit: options2.requestId !== void 0,
|
|
2527
|
+
admissionHeld: true,
|
|
2528
|
+
workInFlight: false,
|
|
2529
|
+
requestedAt: Date.now(),
|
|
2530
|
+
releaseCheckpointAdmission
|
|
2531
|
+
};
|
|
2532
|
+
try {
|
|
2533
|
+
ws.send(
|
|
2534
|
+
JSON.stringify({
|
|
2535
|
+
type: "workspace_checkpoint_request",
|
|
2536
|
+
requestId,
|
|
2537
|
+
trigger,
|
|
2538
|
+
dirtyGeneration: snapshot.dirtyGeneration,
|
|
2539
|
+
localHead: snapshot.localHead,
|
|
2540
|
+
desiredCanonicalHead: snapshot.desiredCanonicalHead
|
|
2541
|
+
})
|
|
2542
|
+
);
|
|
2543
|
+
} catch (error) {
|
|
2544
|
+
releaseCheckpointAdmission();
|
|
2545
|
+
pendingCheckpoint = void 0;
|
|
2546
|
+
workspaceConvergence.defer();
|
|
2547
|
+
process.stderr.write(
|
|
2548
|
+
`[r5d-worker] failed to request workspace checkpoint: ${error instanceof Error ? error.message : String(error)}
|
|
2549
|
+
`
|
|
2550
|
+
);
|
|
2551
|
+
armWorkspaceCheckpoint(trigger, { delayMs: 1e3, ...options2.requestId ? { requestId: options2.requestId } : {} });
|
|
2552
|
+
}
|
|
2553
|
+
}, Math.max(0, options2.delayMs));
|
|
2554
|
+
workspaceCheckpointTimer.unref();
|
|
2555
|
+
};
|
|
2556
|
+
const markWorkspaceDirty = (trigger, force = false) => {
|
|
2557
|
+
workspaceConvergence.markDirty();
|
|
2558
|
+
armWorkspaceCheckpoint(trigger, { delayMs: force ? 0 : WORKSPACE_CHECKPOINT_QUIET_MS });
|
|
2559
|
+
};
|
|
2560
|
+
const convergeAvailableWorkspaceHead = async () => {
|
|
2561
|
+
if (pendingWorkspaceHead === void 0 || !workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || workspaceHeadConvergenceInFlight) {
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
const advertisedHead = pendingWorkspaceHead;
|
|
2565
|
+
pendingWorkspaceHead = void 0;
|
|
2566
|
+
workspaceConvergence.observeCanonicalHead(advertisedHead);
|
|
2567
|
+
workspaceHeadConvergenceInFlight = true;
|
|
2568
|
+
let releaseVisibleHydration = null;
|
|
2569
|
+
try {
|
|
2570
|
+
const snapshot = workspaceConvergence.snapshot();
|
|
2571
|
+
const mayHydrateVisible = !workerHasActiveWriter() && !pendingCheckpoint && snapshot.dirtyGeneration === snapshot.publishedGeneration;
|
|
2572
|
+
releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
|
|
2573
|
+
const hydrateVisible = releaseVisibleHydration !== null;
|
|
2574
|
+
const converged = await workspaceSyncSingleFlight.runExclusive(
|
|
2575
|
+
() => convergeWorkspaceHead(workspaceSyncInput({ type: "inbound_head", detail: advertisedHead ?? "unborn" }), {
|
|
2576
|
+
hydrateVisible
|
|
2405
2577
|
})
|
|
2406
2578
|
);
|
|
2579
|
+
workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
|
|
2580
|
+
if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
|
|
2581
|
+
if (hydrateVisible && converged.localChanges) {
|
|
2582
|
+
markWorkspaceDirty({ type: "periodic", detail: "inbound overlap retained for next checkpoint" });
|
|
2583
|
+
}
|
|
2584
|
+
sendWorkspaceReady();
|
|
2407
2585
|
} catch (error) {
|
|
2586
|
+
pendingWorkspaceHead = advertisedHead;
|
|
2408
2587
|
process.stderr.write(
|
|
2409
|
-
`[r5d-worker] failed to
|
|
2588
|
+
`[r5d-worker] failed to fetch available workspace head: ${error instanceof Error ? error.message : String(error)}
|
|
2410
2589
|
`
|
|
2411
2590
|
);
|
|
2591
|
+
} finally {
|
|
2592
|
+
releaseVisibleHydration?.();
|
|
2593
|
+
workspaceHeadConvergenceInFlight = false;
|
|
2594
|
+
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2595
|
+
if (pendingWorkspaceHead !== void 0) {
|
|
2596
|
+
setTimeout(() => void convergeAvailableWorkspaceHead(), 1e3).unref();
|
|
2597
|
+
}
|
|
2412
2598
|
}
|
|
2413
2599
|
};
|
|
2414
2600
|
const manifestCheckoutKey = (projectId, branchName) => `${projectId}\0${branchName}`;
|
|
@@ -2460,19 +2646,73 @@ async function startWorker(options) {
|
|
|
2460
2646
|
}
|
|
2461
2647
|
return created;
|
|
2462
2648
|
};
|
|
2649
|
+
const schedulePendingManifestCheckoutHydration = (delayMs = 0) => {
|
|
2650
|
+
if (workspaceManifestHydrationTimer || workspaceManifestHydrationInFlight || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2651
|
+
return;
|
|
2652
|
+
}
|
|
2653
|
+
workspaceManifestHydrationTimer = setTimeout(() => {
|
|
2654
|
+
workspaceManifestHydrationTimer = void 0;
|
|
2655
|
+
if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || pendingManifestCheckouts.size === 0 || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2656
|
+
if (pendingManifestCheckouts.size > 0 && currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
|
|
2657
|
+
schedulePendingManifestCheckoutHydration(1e3);
|
|
2658
|
+
}
|
|
2659
|
+
return;
|
|
2660
|
+
}
|
|
2661
|
+
const releaseHydration = workspaceFilesystemWriters.tryAcquireCheckpoint();
|
|
2662
|
+
if (!releaseHydration) {
|
|
2663
|
+
schedulePendingManifestCheckoutHydration(1e3);
|
|
2664
|
+
return;
|
|
2665
|
+
}
|
|
2666
|
+
workspaceManifestHydrationInFlight = true;
|
|
2667
|
+
void workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2668
|
+
if (!workspaceReady || activeWorkspaceIncidentId || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
2669
|
+
return;
|
|
2670
|
+
}
|
|
2671
|
+
const targets = [...pendingManifestCheckouts.values()];
|
|
2672
|
+
ensureVisibleWorkspaceCheckouts(targets, { strictCanonicalRevalidation: true });
|
|
2673
|
+
for (const target of targets) {
|
|
2674
|
+
const manifest = manifestByProjectId.get(target.projectId);
|
|
2675
|
+
if (!manifest || !manifest.branches.includes(target.branchName)) continue;
|
|
2676
|
+
const branchPath = path.join(projectRootFor(projectsRoot, target.projectId, manifestByProjectId), target.branchName);
|
|
2677
|
+
if (hasNormalVisibleGitDir(branchPath)) {
|
|
2678
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
|
|
2679
|
+
}
|
|
2680
|
+
}
|
|
2681
|
+
}).then(() => sendWorkspaceReady()).catch((error) => {
|
|
2682
|
+
process.stderr.write(
|
|
2683
|
+
`[r5d-worker] pending manifest checkout hydration failed: ${error instanceof Error ? error.message : String(error)}
|
|
2684
|
+
`
|
|
2685
|
+
);
|
|
2686
|
+
}).finally(() => {
|
|
2687
|
+
releaseHydration();
|
|
2688
|
+
workspaceManifestHydrationInFlight = false;
|
|
2689
|
+
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2690
|
+
if (pendingManifestCheckouts.size > 0) schedulePendingManifestCheckoutHydration(1e3);
|
|
2691
|
+
});
|
|
2692
|
+
}, Math.max(0, delayMs));
|
|
2693
|
+
workspaceManifestHydrationTimer.unref();
|
|
2694
|
+
};
|
|
2463
2695
|
const runRequestedWorkspaceSync = async (authority, trigger, overrides = {}) => {
|
|
2696
|
+
const requestedAt = Date.now();
|
|
2697
|
+
let queueEnteredAt = requestedAt;
|
|
2698
|
+
let prepareStartedAt = requestedAt;
|
|
2699
|
+
let synchronizeStartedAt = requestedAt;
|
|
2700
|
+
let synchronizeFinishedAt = requestedAt;
|
|
2464
2701
|
workspaceSyncRequestsInFlight += 1;
|
|
2465
2702
|
try {
|
|
2466
2703
|
let pendingTargets = [];
|
|
2467
|
-
const result = await workspaceSyncSingleFlight.
|
|
2704
|
+
const result = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2705
|
+
queueEnteredAt = Date.now();
|
|
2468
2706
|
const shouldEnsureCheckouts = !overrides.skipVisibleMirror && !overrides.resetToCanonical;
|
|
2469
2707
|
const syncProjects = workspaceProjectsForSync([...manifestByProjectId.values()], trigger);
|
|
2708
|
+
if (!workspaceRemoteUrl) throw new Error("Worker workspace manifest has not been received");
|
|
2470
2709
|
const allowedCheckoutKeys = new Set(
|
|
2471
2710
|
syncProjects.flatMap((project) => project.branches.map((branchName) => manifestCheckoutKey(project.projectId, branchName)))
|
|
2472
2711
|
);
|
|
2473
2712
|
const scopedPendingTargets = [...pendingManifestCheckouts.values()].filter(
|
|
2474
2713
|
(target) => allowedCheckoutKeys.has(manifestCheckoutKey(target.projectId, target.branchName))
|
|
2475
2714
|
);
|
|
2715
|
+
prepareStartedAt = Date.now();
|
|
2476
2716
|
let checkoutTargets = [];
|
|
2477
2717
|
if (shouldEnsureCheckouts) {
|
|
2478
2718
|
checkoutTargets = workspaceSyncCheckoutTargets({
|
|
@@ -2492,21 +2732,39 @@ async function startWorker(options) {
|
|
|
2492
2732
|
if (allowedCheckoutKeys.has(key)) newVisibleCheckoutByKey.set(key, target);
|
|
2493
2733
|
}
|
|
2494
2734
|
const newVisibleCheckouts = [...newVisibleCheckoutByKey.values()];
|
|
2495
|
-
|
|
2735
|
+
const input = workspaceSyncInput(trigger, {
|
|
2496
2736
|
...overrides,
|
|
2497
2737
|
...newVisibleCheckouts.length > 0 ? { newVisibleCheckouts } : {}
|
|
2498
2738
|
});
|
|
2739
|
+
synchronizeStartedAt = Date.now();
|
|
2740
|
+
let synchronized;
|
|
2741
|
+
try {
|
|
2742
|
+
synchronized = await synchronizeWorkspace(input);
|
|
2743
|
+
} finally {
|
|
2744
|
+
synchronizeFinishedAt = Date.now();
|
|
2745
|
+
}
|
|
2746
|
+
return synchronized;
|
|
2499
2747
|
});
|
|
2500
|
-
|
|
2748
|
+
const completedAt = Date.now();
|
|
2749
|
+
const resultWithTelemetry = {
|
|
2750
|
+
...result,
|
|
2751
|
+
telemetry: {
|
|
2752
|
+
totalMs: completedAt - requestedAt,
|
|
2753
|
+
queueMs: queueEnteredAt - requestedAt,
|
|
2754
|
+
prepareMs: synchronizeStartedAt - prepareStartedAt,
|
|
2755
|
+
synchronizeMs: synchronizeFinishedAt - synchronizeStartedAt
|
|
2756
|
+
}
|
|
2757
|
+
};
|
|
2758
|
+
if (resultWithTelemetry.outcome === "no_change" || resultWithTelemetry.outcome === "published" || resultWithTelemetry.outcome === "updated" || resultWithTelemetry.outcome === "conflict_reset") {
|
|
2501
2759
|
for (const target of pendingTargets) {
|
|
2502
2760
|
const key = manifestCheckoutKey(target.projectId, target.branchName);
|
|
2503
2761
|
if (pendingManifestCheckouts.get(key) === target) pendingManifestCheckouts.delete(key);
|
|
2504
2762
|
}
|
|
2505
2763
|
}
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
return result;
|
|
2764
|
+
sendWorkspaceSyncResult(authority, resultWithTelemetry);
|
|
2765
|
+
return resultWithTelemetry;
|
|
2509
2766
|
} catch (error) {
|
|
2767
|
+
const completedAt = Date.now();
|
|
2510
2768
|
const result = {
|
|
2511
2769
|
type: "workspace_sync",
|
|
2512
2770
|
attemptId: overrides.attemptId ?? crypto.randomUUID(),
|
|
@@ -2521,9 +2779,14 @@ async function startWorker(options) {
|
|
|
2521
2779
|
affectedPaths: [],
|
|
2522
2780
|
discardedPaths: [],
|
|
2523
2781
|
localChangesDiscarded: false,
|
|
2782
|
+
telemetry: {
|
|
2783
|
+
totalMs: completedAt - requestedAt,
|
|
2784
|
+
queueMs: queueEnteredAt - requestedAt,
|
|
2785
|
+
prepareMs: Math.max(0, synchronizeStartedAt - prepareStartedAt),
|
|
2786
|
+
synchronizeMs: Math.max(0, synchronizeFinishedAt - synchronizeStartedAt)
|
|
2787
|
+
},
|
|
2524
2788
|
error: error instanceof Error ? error.message : String(error)
|
|
2525
2789
|
};
|
|
2526
|
-
previousPeriodicFingerprint = null;
|
|
2527
2790
|
sendWorkspaceSyncResult(authority, result);
|
|
2528
2791
|
return result;
|
|
2529
2792
|
} finally {
|
|
@@ -2539,6 +2802,24 @@ async function startWorker(options) {
|
|
|
2539
2802
|
let heartbeatBusyGrace = null;
|
|
2540
2803
|
let heartbeatTimedOut = false;
|
|
2541
2804
|
let heartbeatWatchdog;
|
|
2805
|
+
const handleGracefulShutdown = () => {
|
|
2806
|
+
if (shutdownAfterClose) return;
|
|
2807
|
+
shutdownAfterClose = true;
|
|
2808
|
+
void waitForWorkspaceCheckpointOnShutdown({
|
|
2809
|
+
snapshot: () => workspaceConvergence.snapshot(),
|
|
2810
|
+
forceCheckpoint: () => armWorkspaceCheckpoint({ type: "manual", detail: "graceful worker shutdown" }, { delayMs: 0 })
|
|
2811
|
+
}).then((outcome) => {
|
|
2812
|
+
process.stdout.write(`[r5d-worker] graceful workspace checkpoint ${outcome}
|
|
2813
|
+
`);
|
|
2814
|
+
if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
|
|
2815
|
+
ws.close(1e3, "Worker shutting down");
|
|
2816
|
+
} else {
|
|
2817
|
+
process.exit(0);
|
|
2818
|
+
}
|
|
2819
|
+
});
|
|
2820
|
+
};
|
|
2821
|
+
process.on("SIGTERM", handleGracefulShutdown);
|
|
2822
|
+
process.on("SIGINT", handleGracefulShutdown);
|
|
2542
2823
|
const stopHeartbeatWatchdog = () => {
|
|
2543
2824
|
if (heartbeatWatchdog) {
|
|
2544
2825
|
clearInterval(heartbeatWatchdog);
|
|
@@ -2569,7 +2850,8 @@ async function startWorker(options) {
|
|
|
2569
2850
|
updateClis: true,
|
|
2570
2851
|
canonicalResolverCheckout: true,
|
|
2571
2852
|
browserPortForwarding: true,
|
|
2572
|
-
globalWorkspaceSyncLeaseV1: true
|
|
2853
|
+
globalWorkspaceSyncLeaseV1: true,
|
|
2854
|
+
eventualWorkspaceSyncV1: true
|
|
2573
2855
|
},
|
|
2574
2856
|
projectRoot: projectsRoot,
|
|
2575
2857
|
artifactRoot,
|
|
@@ -2596,79 +2878,123 @@ async function startWorker(options) {
|
|
|
2596
2878
|
return;
|
|
2597
2879
|
}
|
|
2598
2880
|
if (message.type === "workspace_manifest") {
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
);
|
|
2606
|
-
const migratedProjectIds = migrateLegacyProjectRoots(projectsRoot, message.projects);
|
|
2607
|
-
manifestByProjectId.clear();
|
|
2608
|
-
for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
|
|
2609
|
-
const currentCheckouts = allManifestCheckouts();
|
|
2610
|
-
const currentCheckoutKeys = new Set(
|
|
2611
|
-
currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2612
|
-
);
|
|
2613
|
-
for (const key of pendingManifestCheckouts.keys()) {
|
|
2614
|
-
if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
|
|
2615
|
-
}
|
|
2616
|
-
for (const checkout of currentCheckouts) {
|
|
2617
|
-
const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
|
|
2618
|
-
if (previousCheckoutKeys.has(key)) continue;
|
|
2619
|
-
const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
|
|
2620
|
-
if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
|
|
2621
|
-
}
|
|
2622
|
-
previousPeriodicFingerprint = null;
|
|
2623
|
-
process.stdout.write(
|
|
2624
|
-
`[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
|
|
2625
|
-
`
|
|
2626
|
-
);
|
|
2881
|
+
const revisionToken = workspaceManifestRevision.begin(message.revision);
|
|
2882
|
+
workspaceReady = false;
|
|
2883
|
+
const manifestAdmission = inspectWorkspaceManifestAdmission({
|
|
2884
|
+
projectsRoot,
|
|
2885
|
+
workspaceShadowRoot,
|
|
2886
|
+
projects: message.projects
|
|
2627
2887
|
});
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2888
|
+
const releaseManifestHydration = await acquireWorkspaceManifestVisibleLease(
|
|
2889
|
+
workspaceFilesystemWriters,
|
|
2890
|
+
manifestAdmission
|
|
2891
|
+
);
|
|
2892
|
+
let completed = false;
|
|
2893
|
+
try {
|
|
2894
|
+
completed = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
2895
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2896
|
+
const firstBootstrap = manifestAdmission.firstBootstrap;
|
|
2897
|
+
configureGitHubAuth(message.githubCredential);
|
|
2898
|
+
visibleGitIdentity = message.gitIdentity;
|
|
2899
|
+
workspaceRemoteUrl = message.workspaceRemoteUrl;
|
|
2900
|
+
const migratedProjectIds = releaseManifestHydration ? migrateLegacyProjectRoots(projectsRoot, message.projects) : [];
|
|
2901
|
+
manifestByProjectId.clear();
|
|
2902
|
+
for (const project of message.projects) manifestByProjectId.set(project.projectId, project);
|
|
2903
|
+
const currentCheckouts = allManifestCheckouts();
|
|
2904
|
+
const currentCheckoutKeys = new Set(
|
|
2905
|
+
currentCheckouts.map(({ projectId, branchName }) => manifestCheckoutKey(projectId, branchName))
|
|
2906
|
+
);
|
|
2907
|
+
for (const key of pendingManifestCheckouts.keys()) {
|
|
2908
|
+
if (!currentCheckoutKeys.has(key)) pendingManifestCheckouts.delete(key);
|
|
2909
|
+
}
|
|
2910
|
+
for (const checkout of currentCheckouts) {
|
|
2911
|
+
const key = manifestCheckoutKey(checkout.projectId, checkout.branchName);
|
|
2912
|
+
const branchPath = path.join(
|
|
2913
|
+
projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId),
|
|
2914
|
+
checkout.branchName
|
|
2641
2915
|
);
|
|
2642
|
-
if (
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2916
|
+
if (!hasNormalVisibleGitDir(branchPath)) pendingManifestCheckouts.set(key, checkout);
|
|
2917
|
+
}
|
|
2918
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2919
|
+
const created = releaseManifestHydration ? ensureVisibleWorkspaceCheckouts(currentCheckouts, { strictCanonicalRevalidation: true }) : [];
|
|
2920
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2921
|
+
const bootstrapped = await convergeWorkspaceHead(
|
|
2922
|
+
workspaceSyncInput({ type: "connect" }, { newVisibleCheckouts: created }),
|
|
2923
|
+
{
|
|
2924
|
+
// An existing checkout becomes routable after a fetch. Its
|
|
2925
|
+
// visible tree converges asynchronously; first bootstrap alone
|
|
2926
|
+
// must materialize the canonical snapshot before readiness.
|
|
2927
|
+
// Re-check the incident immediately before convergence so a
|
|
2928
|
+
// concurrently delivered fence forces fetch-only behavior.
|
|
2929
|
+
hydrateVisible: firstBootstrap && !activeWorkspaceIncidentId
|
|
2649
2930
|
}
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2931
|
+
);
|
|
2932
|
+
if (!workspaceManifestRevision.isCurrent(revisionToken)) return false;
|
|
2933
|
+
workspaceConvergence.setLocalHead(bootstrapped.localHead);
|
|
2934
|
+
workspaceConvergence.observeCanonicalHead(bootstrapped.canonicalHead);
|
|
2935
|
+
for (const checkout of currentCheckouts) {
|
|
2936
|
+
const manifest = manifestByProjectId.get(checkout.projectId);
|
|
2937
|
+
if (!manifest) continue;
|
|
2938
|
+
const branchPath = path.join(projectRootFor(projectsRoot, checkout.projectId, manifestByProjectId), checkout.branchName);
|
|
2939
|
+
if (hasNormalVisibleGitDir(branchPath)) {
|
|
2940
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(checkout.projectId, checkout.branchName));
|
|
2654
2941
|
}
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2942
|
+
}
|
|
2943
|
+
process.stdout.write(
|
|
2944
|
+
`[r5d-worker] workspace manifest: ${message.projects.length} projects${migratedProjectIds.length > 0 ? `; migrated ${migratedProjectIds.length} project checkout root(s)` : ""}
|
|
2945
|
+
`
|
|
2946
|
+
);
|
|
2947
|
+
return true;
|
|
2948
|
+
});
|
|
2949
|
+
} finally {
|
|
2950
|
+
releaseManifestHydration?.();
|
|
2951
|
+
}
|
|
2952
|
+
if (!completed || !workspaceManifestRevision.complete(revisionToken)) return;
|
|
2953
|
+
workspaceReady = true;
|
|
2954
|
+
sendWorkspaceReady();
|
|
2955
|
+
schedulePendingManifestCheckoutHydration();
|
|
2956
|
+
if (!workspaceSafetyScan) {
|
|
2957
|
+
workspaceSafetyScan = setInterval(() => {
|
|
2958
|
+
if (!workspaceReady || !workspaceRemoteUrl || activeWorkspaceIncidentId || pendingCheckpoint || workspaceSafetyScanInFlight) {
|
|
2959
|
+
return;
|
|
2960
|
+
}
|
|
2961
|
+
workspaceSafetyScanInFlight = true;
|
|
2962
|
+
void (async () => {
|
|
2963
|
+
const before = workspaceConvergence.snapshot();
|
|
2964
|
+
const mayHydrateVisible = !workerHasActiveWriter() && before.dirtyGeneration === before.publishedGeneration;
|
|
2965
|
+
const releaseVisibleHydration = mayHydrateVisible ? workspaceFilesystemWriters.tryAcquireCheckpoint() : null;
|
|
2966
|
+
const hydrateVisible = releaseVisibleHydration !== null;
|
|
2967
|
+
try {
|
|
2968
|
+
const converged = await workspaceSyncSingleFlight.runExclusive(
|
|
2969
|
+
() => convergeWorkspaceHead(workspaceSyncInput({ type: "periodic", detail: "idle convergence scan" }), {
|
|
2970
|
+
hydrateVisible
|
|
2971
|
+
})
|
|
2972
|
+
);
|
|
2973
|
+
workspaceConvergence.observeCanonicalHead(converged.canonicalHead);
|
|
2974
|
+
if (converged.hydrated) workspaceConvergence.setLocalHead(converged.localHead);
|
|
2975
|
+
if (converged.localChanges && hydrateVisible) {
|
|
2976
|
+
markWorkspaceDirty({ type: "periodic", detail: "idle safety scan" });
|
|
2977
|
+
}
|
|
2978
|
+
sendWorkspaceReady();
|
|
2979
|
+
} finally {
|
|
2980
|
+
releaseVisibleHydration?.();
|
|
2658
2981
|
}
|
|
2659
|
-
previousPeriodicFingerprint = null;
|
|
2660
|
-
requestWorkspaceSync({ type: "periodic" }, fingerprint);
|
|
2661
2982
|
})().catch((error) => {
|
|
2662
2983
|
process.stderr.write(
|
|
2663
|
-
`[r5d-worker]
|
|
2984
|
+
`[r5d-worker] idle workspace safety scan failed: ${error instanceof Error ? error.message : String(error)}
|
|
2664
2985
|
`
|
|
2665
2986
|
);
|
|
2666
2987
|
}).finally(() => {
|
|
2667
|
-
|
|
2988
|
+
workspaceSafetyScanInFlight = false;
|
|
2668
2989
|
heartbeatBusyGrace = grantWorkerHeartbeatBusyGrace(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
2669
2990
|
});
|
|
2670
|
-
},
|
|
2671
|
-
|
|
2991
|
+
}, WORKSPACE_IDLE_SAFETY_SCAN_MS);
|
|
2992
|
+
workspaceSafetyScan.unref();
|
|
2993
|
+
}
|
|
2994
|
+
pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
|
|
2995
|
+
void convergeAvailableWorkspaceHead();
|
|
2996
|
+
if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
2997
|
+
armWorkspaceCheckpoint({ type: "periodic", detail: "resume dirty generation after reconnect" }, { delayMs: 0 });
|
|
2672
2998
|
}
|
|
2673
2999
|
return;
|
|
2674
3000
|
}
|
|
@@ -2692,6 +3018,185 @@ async function startWorker(options) {
|
|
|
2692
3018
|
);
|
|
2693
3019
|
return;
|
|
2694
3020
|
}
|
|
3021
|
+
if (message.type === "workspace_head_available") {
|
|
3022
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3023
|
+
pendingWorkspaceHead = message.canonicalHead;
|
|
3024
|
+
sendWorkspaceReady();
|
|
3025
|
+
void convergeAvailableWorkspaceHead();
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
if (message.type === "workspace_checkpoint_now") {
|
|
3029
|
+
armWorkspaceCheckpoint(message.trigger, { delayMs: 0, requestId: message.requestId });
|
|
3030
|
+
return;
|
|
3031
|
+
}
|
|
3032
|
+
if (message.type === "workspace_checkpoint_authority") {
|
|
3033
|
+
const checkpoint = pendingCheckpoint;
|
|
3034
|
+
if (!checkpoint || checkpoint.requestId !== message.requestId) {
|
|
3035
|
+
process.stderr.write(`[r5d-worker] ignoring checkpoint authority for unknown request ${message.requestId}
|
|
3036
|
+
`);
|
|
3037
|
+
return;
|
|
3038
|
+
}
|
|
3039
|
+
if (message.status !== "granted") {
|
|
3040
|
+
checkpoint.releaseCheckpointAdmission();
|
|
3041
|
+
pendingCheckpoint = void 0;
|
|
3042
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3043
|
+
const applyBlockedResponse = message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId);
|
|
3044
|
+
if (applyBlockedResponse) {
|
|
3045
|
+
workspaceConvergence.block();
|
|
3046
|
+
if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
|
|
3047
|
+
} else {
|
|
3048
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3049
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3050
|
+
if (message.status === "busy") {
|
|
3051
|
+
armWorkspaceCheckpoint(checkpoint.trigger, {
|
|
3052
|
+
delayMs: message.retryAfterMs ?? 1e3,
|
|
3053
|
+
requestId: checkpoint.requestId
|
|
3054
|
+
});
|
|
3055
|
+
} else if (!activeWorkspaceIncidentId) {
|
|
3056
|
+
const explicit = explicitWorkspaceCheckpoints.next();
|
|
3057
|
+
if (explicit) {
|
|
3058
|
+
armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
|
|
3059
|
+
} else if (workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3060
|
+
armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: 0 });
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
}
|
|
3064
|
+
sendWorkspaceReady();
|
|
3065
|
+
return;
|
|
3066
|
+
}
|
|
3067
|
+
checkpoint.attemptId = message.attemptId;
|
|
3068
|
+
checkpoint.authorityToken = message.authorityToken;
|
|
3069
|
+
checkpoint.workInFlight = true;
|
|
3070
|
+
pendingCheckpoint = checkpoint;
|
|
3071
|
+
if (!checkpoint.admissionHeld) {
|
|
3072
|
+
const releaseCheckpointAdmission = await workspaceFilesystemWriters.acquireCheckpoint();
|
|
3073
|
+
if (currentWorkerSocket !== ws || pendingCheckpoint?.requestId !== checkpoint.requestId || pendingCheckpoint.attemptId !== message.attemptId) {
|
|
3074
|
+
releaseCheckpointAdmission();
|
|
3075
|
+
checkpoint.workInFlight = false;
|
|
3076
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3077
|
+
return;
|
|
3078
|
+
}
|
|
3079
|
+
checkpoint.admissionHeld = true;
|
|
3080
|
+
checkpoint.releaseCheckpointAdmission = releaseCheckpointAdmission;
|
|
3081
|
+
}
|
|
3082
|
+
workspaceConvergence.beginPreparing();
|
|
3083
|
+
sendWorkspaceReady();
|
|
3084
|
+
let prepareStartedAt = Date.now();
|
|
3085
|
+
let synchronizeStartedAt = prepareStartedAt;
|
|
3086
|
+
let synchronizeStarted = false;
|
|
3087
|
+
let result;
|
|
3088
|
+
try {
|
|
3089
|
+
result = await workspaceSyncSingleFlight.runExclusive(async () => {
|
|
3090
|
+
prepareStartedAt = Date.now();
|
|
3091
|
+
const pendingTargets = [...pendingManifestCheckouts.values()];
|
|
3092
|
+
const createdCheckouts = ensureVisibleWorkspaceCheckouts(pendingTargets, { strictCanonicalRevalidation: true });
|
|
3093
|
+
synchronizeStartedAt = Date.now();
|
|
3094
|
+
synchronizeStarted = true;
|
|
3095
|
+
const prepared = await synchronizeWorkspace(
|
|
3096
|
+
workspaceSyncInput(checkpoint.trigger, {
|
|
3097
|
+
attemptId: message.attemptId,
|
|
3098
|
+
quarantineRef: message.quarantineRef,
|
|
3099
|
+
expectedCanonicalHead: message.expectedHead,
|
|
3100
|
+
newVisibleCheckouts: [...createdCheckouts, ...pendingTargets]
|
|
3101
|
+
})
|
|
3102
|
+
);
|
|
3103
|
+
for (const target of pendingTargets) {
|
|
3104
|
+
pendingManifestCheckouts.delete(manifestCheckoutKey(target.projectId, target.branchName));
|
|
3105
|
+
}
|
|
3106
|
+
return prepared.outcome === "updated" ? { ...prepared, outcome: "no_change" } : prepared;
|
|
3107
|
+
});
|
|
3108
|
+
} catch (error) {
|
|
3109
|
+
if (!synchronizeStarted) synchronizeStartedAt = Date.now();
|
|
3110
|
+
result = {
|
|
3111
|
+
type: "workspace_sync",
|
|
3112
|
+
attemptId: message.attemptId,
|
|
3113
|
+
workerLabel: label,
|
|
3114
|
+
trigger: checkpoint.trigger,
|
|
3115
|
+
outcome: "failed",
|
|
3116
|
+
startingHead: message.expectedHead,
|
|
3117
|
+
expectedHead: message.expectedHead,
|
|
3118
|
+
rebaseCount: 0,
|
|
3119
|
+
diffSizeBytes: 0,
|
|
3120
|
+
gitStatus: "",
|
|
3121
|
+
affectedProjects: [],
|
|
3122
|
+
affectedPaths: [],
|
|
3123
|
+
discardedPaths: [],
|
|
3124
|
+
localChangesDiscarded: false,
|
|
3125
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3126
|
+
};
|
|
3127
|
+
}
|
|
3128
|
+
const finishedAt = Date.now();
|
|
3129
|
+
result = {
|
|
3130
|
+
...result,
|
|
3131
|
+
dirtyGeneration: checkpoint.dirtyGeneration,
|
|
3132
|
+
telemetry: workspaceCheckpointTelemetry({
|
|
3133
|
+
requestedAt: checkpoint.requestedAt,
|
|
3134
|
+
prepareStartedAt,
|
|
3135
|
+
synchronizeStartedAt,
|
|
3136
|
+
finishedAt
|
|
3137
|
+
})
|
|
3138
|
+
};
|
|
3139
|
+
workspaceConvergence.beginSubmitting();
|
|
3140
|
+
sendWorkspaceReady();
|
|
3141
|
+
try {
|
|
3142
|
+
ws.send(
|
|
3143
|
+
JSON.stringify({
|
|
3144
|
+
type: "workspace_checkpoint_result",
|
|
3145
|
+
requestId: checkpoint.requestId,
|
|
3146
|
+
attemptId: message.attemptId,
|
|
3147
|
+
authorityToken: message.authorityToken,
|
|
3148
|
+
dirtyGeneration: checkpoint.dirtyGeneration,
|
|
3149
|
+
result
|
|
3150
|
+
})
|
|
3151
|
+
);
|
|
3152
|
+
} catch (error) {
|
|
3153
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3154
|
+
throw error;
|
|
3155
|
+
} finally {
|
|
3156
|
+
checkpoint.workInFlight = false;
|
|
3157
|
+
checkpoint.releaseCheckpointAdmission();
|
|
3158
|
+
if (currentWorkerSocket !== ws) workspaceConvergence.resetTransientCheckpointState();
|
|
3159
|
+
}
|
|
3160
|
+
return;
|
|
3161
|
+
}
|
|
3162
|
+
if (message.type === "workspace_checkpoint_completed") {
|
|
3163
|
+
const checkpoint = pendingCheckpoint;
|
|
3164
|
+
if (!checkpoint || checkpoint.requestId !== message.requestId || checkpoint.attemptId !== message.attemptId) {
|
|
3165
|
+
process.stderr.write(`[r5d-worker] ignoring completion for unknown checkpoint ${message.requestId}
|
|
3166
|
+
`);
|
|
3167
|
+
return;
|
|
3168
|
+
}
|
|
3169
|
+
pendingCheckpoint = void 0;
|
|
3170
|
+
workspaceConvergence.observeCanonicalHead(message.canonicalHead);
|
|
3171
|
+
if (message.status === "published" || message.status === "no_change") {
|
|
3172
|
+
workspaceConvergence.complete({
|
|
3173
|
+
generation: Math.min(message.publishedGeneration ?? checkpoint.dirtyGeneration, checkpoint.dirtyGeneration),
|
|
3174
|
+
canonicalHead: message.canonicalHead,
|
|
3175
|
+
published: true
|
|
3176
|
+
});
|
|
3177
|
+
} else if (message.status === "blocked" && workspaceIncidentOrdering.permitsBlockedResponse(message.incidentId)) {
|
|
3178
|
+
workspaceConvergence.block();
|
|
3179
|
+
if (message.incidentId) activeWorkspaceIncidentId = message.incidentId;
|
|
3180
|
+
} else {
|
|
3181
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3182
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3183
|
+
}
|
|
3184
|
+
sendWorkspaceReady();
|
|
3185
|
+
const explicit = message.status === "retry" && checkpoint.explicit ? { requestId: checkpoint.requestId, trigger: checkpoint.trigger } : explicitWorkspaceCheckpoints.next();
|
|
3186
|
+
if (explicit) {
|
|
3187
|
+
armWorkspaceCheckpoint(explicit.trigger, {
|
|
3188
|
+
delayMs: message.retryAfterMs ?? 0,
|
|
3189
|
+
requestId: explicit.requestId
|
|
3190
|
+
});
|
|
3191
|
+
} else if (message.status === "retry" || message.status === "failed" || workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3192
|
+
armWorkspaceCheckpoint(checkpoint.trigger, { delayMs: message.retryAfterMs ?? 1e3 });
|
|
3193
|
+
}
|
|
3194
|
+
if (message.canonicalHead !== workspaceConvergence.snapshot().localHead) {
|
|
3195
|
+
pendingWorkspaceHead = message.canonicalHead;
|
|
3196
|
+
void convergeAvailableWorkspaceHead();
|
|
3197
|
+
}
|
|
3198
|
+
return;
|
|
3199
|
+
}
|
|
2695
3200
|
if (message.type === "sync_session_artifacts") {
|
|
2696
3201
|
sessionArtifactSyncRequestsInFlight += 1;
|
|
2697
3202
|
try {
|
|
@@ -2719,17 +3224,27 @@ async function startWorker(options) {
|
|
|
2719
3224
|
return;
|
|
2720
3225
|
}
|
|
2721
3226
|
if (message.type === "workspace_incident_updated") {
|
|
3227
|
+
workspaceIncidentOrdering.observe(message);
|
|
2722
3228
|
const previousIncidentId = activeWorkspaceIncidentId;
|
|
2723
3229
|
activeWorkspaceIncidentId = applyWorkspaceIncidentUpdate(activeWorkspaceIncidentId, message);
|
|
2724
|
-
previousPeriodicFingerprint = null;
|
|
2725
3230
|
const releasedHead = releasePendingWorkspaceHead(
|
|
2726
3231
|
previousIncidentId,
|
|
2727
3232
|
activeWorkspaceIncidentId,
|
|
2728
3233
|
message.status === "resolved" || message.status === "confirmed" || message.status === "reset" ? message.incidentId : null
|
|
2729
3234
|
);
|
|
2730
|
-
|
|
2731
|
-
|
|
3235
|
+
workspaceConvergence.observeIncidentState(Boolean(activeWorkspaceIncidentId));
|
|
3236
|
+
if (releasedHead.fetchAuthoritativeHead) {
|
|
3237
|
+
pendingWorkspaceHead = workspaceConvergence.snapshot().desiredCanonicalHead;
|
|
3238
|
+
void convergeAvailableWorkspaceHead();
|
|
3239
|
+
}
|
|
3240
|
+
const explicit = !activeWorkspaceIncidentId ? explicitWorkspaceCheckpoints.next() : void 0;
|
|
3241
|
+
if (explicit) {
|
|
3242
|
+
armWorkspaceCheckpoint(explicit.trigger, { delayMs: 0, requestId: explicit.requestId });
|
|
3243
|
+
} else if (!activeWorkspaceIncidentId && workspaceConvergence.snapshot().dirtyGeneration > workspaceConvergence.snapshot().publishedGeneration) {
|
|
3244
|
+
armWorkspaceCheckpoint({ type: "periodic", detail: "resume after workspace incident" }, { delayMs: 0 });
|
|
2732
3245
|
}
|
|
3246
|
+
if (!activeWorkspaceIncidentId) schedulePendingManifestCheckoutHydration();
|
|
3247
|
+
sendWorkspaceReady();
|
|
2733
3248
|
return;
|
|
2734
3249
|
}
|
|
2735
3250
|
if (message.type === "exec_terminal_ack") {
|
|
@@ -2827,8 +3342,10 @@ async function startWorker(options) {
|
|
|
2827
3342
|
});
|
|
2828
3343
|
return;
|
|
2829
3344
|
}
|
|
3345
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2830
3346
|
const releaseWorkspaceMutation = await acquireWorkspaceCommandMutation(message.target, workspaceSyncSingleFlight);
|
|
2831
|
-
let
|
|
3347
|
+
let mutationLeaseTransferred = false;
|
|
3348
|
+
let writerLeaseTransferred = false;
|
|
2832
3349
|
try {
|
|
2833
3350
|
const resolvedTarget = resolveMessageTarget(message.target);
|
|
2834
3351
|
process.stdout.write(`[r5d-worker] pty ${message.ptyId}: ${describeWorkerSessionTarget(message.target)}
|
|
@@ -2838,9 +3355,19 @@ async function startWorker(options) {
|
|
|
2838
3355
|
message,
|
|
2839
3356
|
resolvedTarget,
|
|
2840
3357
|
planRoot,
|
|
2841
|
-
...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {}
|
|
3358
|
+
...releaseWorkspaceMutation ? { releaseWorkspaceMutation } : {},
|
|
3359
|
+
...targetMayMutateVisibleWorkspace(message.target) ? {
|
|
3360
|
+
onTerminal: () => {
|
|
3361
|
+
markWorkspaceDirty(
|
|
3362
|
+
{ type: "process_terminal", detail: `pty ${message.ptyId} completed` },
|
|
3363
|
+
true
|
|
3364
|
+
);
|
|
3365
|
+
releaseWorkspaceWriter?.();
|
|
3366
|
+
}
|
|
3367
|
+
} : {}
|
|
2842
3368
|
});
|
|
2843
|
-
|
|
3369
|
+
mutationLeaseTransferred = releaseWorkspaceMutation !== void 0;
|
|
3370
|
+
writerLeaseTransferred = releaseWorkspaceWriter !== void 0;
|
|
2844
3371
|
} catch (error) {
|
|
2845
3372
|
sendWorkerMessage(ws, {
|
|
2846
3373
|
type: "pty_error",
|
|
@@ -2849,11 +3376,16 @@ async function startWorker(options) {
|
|
|
2849
3376
|
error: error instanceof Error ? error.message : String(error)
|
|
2850
3377
|
});
|
|
2851
3378
|
} finally {
|
|
2852
|
-
if (!
|
|
3379
|
+
if (!mutationLeaseTransferred) releaseWorkspaceMutation?.();
|
|
3380
|
+
if (!writerLeaseTransferred) releaseWorkspaceWriter?.();
|
|
2853
3381
|
}
|
|
2854
3382
|
return;
|
|
2855
3383
|
}
|
|
2856
3384
|
if (message.type === "pty_input") {
|
|
3385
|
+
const activePty = activePtys.get(message.ptyId);
|
|
3386
|
+
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
3387
|
+
markWorkspaceDirty({ type: "shell_inline", detail: `pty ${message.ptyId}` });
|
|
3388
|
+
}
|
|
2857
3389
|
writePty(ws, message);
|
|
2858
3390
|
return;
|
|
2859
3391
|
}
|
|
@@ -2862,7 +3394,11 @@ async function startWorker(options) {
|
|
|
2862
3394
|
return;
|
|
2863
3395
|
}
|
|
2864
3396
|
if (message.type === "pty_close") {
|
|
3397
|
+
const activePty = activePtys.get(message.ptyId);
|
|
2865
3398
|
closePty(message);
|
|
3399
|
+
if (activePty && targetMayMutateVisibleWorkspace(activePty.target)) {
|
|
3400
|
+
armWorkspaceCheckpoint({ type: "process_terminal", detail: `pty ${message.ptyId} closed` }, { delayMs: 0 });
|
|
3401
|
+
}
|
|
2866
3402
|
return;
|
|
2867
3403
|
}
|
|
2868
3404
|
if (message.type === "exec") {
|
|
@@ -2876,26 +3412,37 @@ async function startWorker(options) {
|
|
|
2876
3412
|
});
|
|
2877
3413
|
return;
|
|
2878
3414
|
}
|
|
2879
|
-
|
|
3415
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2880
3416
|
try {
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
3417
|
+
let result;
|
|
3418
|
+
try {
|
|
3419
|
+
const runCommand = async () => {
|
|
3420
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3421
|
+
process.stdout.write(`[r5d-worker] exec ${message.runId}: ${message.argv.join(" ")}
|
|
2884
3422
|
`);
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
3423
|
+
return await executeCommand({ message, resolvedTarget, baseUrl, token, artifactRoot, planRoot });
|
|
3424
|
+
};
|
|
3425
|
+
result = await runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
|
|
3426
|
+
} catch (error) {
|
|
3427
|
+
result = {
|
|
3428
|
+
type: "exec_result",
|
|
3429
|
+
requestId: message.requestId,
|
|
3430
|
+
stdout: "",
|
|
3431
|
+
stderr: "",
|
|
3432
|
+
exitCode: 1,
|
|
3433
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3434
|
+
};
|
|
3435
|
+
}
|
|
3436
|
+
ws.send(JSON.stringify(result));
|
|
3437
|
+
if (targetMayMutateVisibleWorkspace(message.target)) {
|
|
3438
|
+
markWorkspaceDirty(
|
|
3439
|
+
{ type: "shell_inline", sessionId: message.sessionId, processRunId: message.runId, detail: "foreground command completed" },
|
|
3440
|
+
true
|
|
3441
|
+
);
|
|
3442
|
+
}
|
|
3443
|
+
} finally {
|
|
3444
|
+
releaseWorkspaceWriter?.();
|
|
2897
3445
|
}
|
|
2898
|
-
ws.send(JSON.stringify(result));
|
|
2899
3446
|
return;
|
|
2900
3447
|
}
|
|
2901
3448
|
if (message.type === "exec_start") {
|
|
@@ -2914,18 +3461,34 @@ async function startWorker(options) {
|
|
|
2914
3461
|
runId: message.runId
|
|
2915
3462
|
});
|
|
2916
3463
|
const runCommand = async () => {
|
|
2917
|
-
const
|
|
2918
|
-
|
|
3464
|
+
const releaseWorkspaceWriter = targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
3465
|
+
try {
|
|
3466
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3467
|
+
process.stdout.write(`[r5d-worker] exec_start ${message.runId}: ${message.argv.join(" ")}
|
|
2919
3468
|
`);
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
3469
|
+
await executeStreamingCommand({
|
|
3470
|
+
ws,
|
|
3471
|
+
message,
|
|
3472
|
+
resolvedTarget,
|
|
3473
|
+
baseUrl,
|
|
3474
|
+
token,
|
|
3475
|
+
artifactRoot,
|
|
3476
|
+
planRoot
|
|
3477
|
+
});
|
|
3478
|
+
} finally {
|
|
3479
|
+
if (targetMayMutateVisibleWorkspace(message.target)) {
|
|
3480
|
+
markWorkspaceDirty(
|
|
3481
|
+
{
|
|
3482
|
+
type: "process_terminal",
|
|
3483
|
+
sessionId: message.sessionId,
|
|
3484
|
+
processRunId: message.runId,
|
|
3485
|
+
detail: "process completed"
|
|
3486
|
+
},
|
|
3487
|
+
true
|
|
3488
|
+
);
|
|
3489
|
+
}
|
|
3490
|
+
releaseWorkspaceWriter?.();
|
|
3491
|
+
}
|
|
2929
3492
|
};
|
|
2930
3493
|
const execution = runWorkspaceCommand(message.target, workspaceSyncSingleFlight, runCommand);
|
|
2931
3494
|
void execution.catch((error) => {
|
|
@@ -2939,33 +3502,46 @@ async function startWorker(options) {
|
|
|
2939
3502
|
return;
|
|
2940
3503
|
}
|
|
2941
3504
|
if (message.type === "read" || message.type === "write" || message.type === "edit" || message.type === "grep" || message.type === "find" || message.type === "ls" || message.type === "view_file_bytes") {
|
|
3505
|
+
const releaseWorkspaceWriter = (message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target) ? await workspaceFilesystemWriters.acquireWriter() : void 0;
|
|
2942
3506
|
try {
|
|
2943
|
-
|
|
2944
|
-
const
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
3507
|
+
try {
|
|
3508
|
+
const result = await workspaceSyncSingleFlight.runMutation(async () => {
|
|
3509
|
+
const resolvedTarget = resolveMessageTarget(message.target);
|
|
3510
|
+
return await executeOperation({
|
|
3511
|
+
message,
|
|
3512
|
+
resolvedTarget,
|
|
3513
|
+
baseUrl,
|
|
3514
|
+
token,
|
|
3515
|
+
artifactRoot,
|
|
3516
|
+
planRoot
|
|
3517
|
+
});
|
|
2952
3518
|
});
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
}
|
|
2968
|
-
)
|
|
3519
|
+
ws.send(
|
|
3520
|
+
JSON.stringify({
|
|
3521
|
+
type: "operation_result",
|
|
3522
|
+
requestId: message.requestId,
|
|
3523
|
+
result
|
|
3524
|
+
})
|
|
3525
|
+
);
|
|
3526
|
+
if ((message.type === "write" || message.type === "edit") && targetMayMutateVisibleWorkspace(message.target)) {
|
|
3527
|
+
markWorkspaceDirty({
|
|
3528
|
+
type: message.type,
|
|
3529
|
+
sessionId: message.sessionId,
|
|
3530
|
+
toolCallId: message.requestId,
|
|
3531
|
+
...message.target.type === "project" ? { projectId: message.target.projectId, branchName: message.target.branchName } : {}
|
|
3532
|
+
});
|
|
3533
|
+
}
|
|
3534
|
+
} catch (error) {
|
|
3535
|
+
ws.send(
|
|
3536
|
+
JSON.stringify({
|
|
3537
|
+
type: "operation_result",
|
|
3538
|
+
requestId: message.requestId,
|
|
3539
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3540
|
+
})
|
|
3541
|
+
);
|
|
3542
|
+
}
|
|
3543
|
+
} finally {
|
|
3544
|
+
releaseWorkspaceWriter?.();
|
|
2969
3545
|
}
|
|
2970
3546
|
return;
|
|
2971
3547
|
}
|
|
@@ -2987,10 +3563,24 @@ async function startWorker(options) {
|
|
|
2987
3563
|
});
|
|
2988
3564
|
});
|
|
2989
3565
|
ws.addEventListener("close", (event) => {
|
|
3566
|
+
process.off("SIGTERM", handleGracefulShutdown);
|
|
3567
|
+
process.off("SIGINT", handleGracefulShutdown);
|
|
2990
3568
|
stopHeartbeatWatchdog();
|
|
2991
|
-
if (
|
|
2992
|
-
|
|
2993
|
-
|
|
3569
|
+
if (workspaceCheckpointTimer) {
|
|
3570
|
+
clearTimeout(workspaceCheckpointTimer);
|
|
3571
|
+
workspaceCheckpointTimer = void 0;
|
|
3572
|
+
}
|
|
3573
|
+
if (!pendingCheckpoint?.workInFlight) {
|
|
3574
|
+
pendingCheckpoint?.releaseCheckpointAdmission();
|
|
3575
|
+
workspaceConvergence.resetTransientCheckpointState();
|
|
3576
|
+
}
|
|
3577
|
+
if (workspaceSafetyScan) {
|
|
3578
|
+
clearInterval(workspaceSafetyScan);
|
|
3579
|
+
workspaceSafetyScan = void 0;
|
|
3580
|
+
}
|
|
3581
|
+
if (workspaceManifestHydrationTimer) {
|
|
3582
|
+
clearTimeout(workspaceManifestHydrationTimer);
|
|
3583
|
+
workspaceManifestHydrationTimer = void 0;
|
|
2994
3584
|
}
|
|
2995
3585
|
if (terminalReplayTimer) {
|
|
2996
3586
|
clearInterval(terminalReplayTimer);
|
|
@@ -3006,6 +3596,9 @@ async function startWorker(options) {
|
|
|
3006
3596
|
if (reloadAfterClose) {
|
|
3007
3597
|
process.exit(WORKER_RELOAD_EXIT_CODE);
|
|
3008
3598
|
}
|
|
3599
|
+
if (shutdownAfterClose) {
|
|
3600
|
+
process.exit(0);
|
|
3601
|
+
}
|
|
3009
3602
|
if (activeProcesses.size > 0 || pendingProcessTerminals.size > 0) {
|
|
3010
3603
|
process.stderr.write(
|
|
3011
3604
|
`[r5d-worker] ${activeProcesses.size} process(es) active and ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${WORKER_RECONNECT_DELAY_MS}ms
|