@ricsam/r5d-worker 0.0.89 → 0.0.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/cjs/main.cjs +155 -52
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/project-snapshot-recovery-runner.cjs +171 -0
- package/dist/cjs/project-worktrees.cjs +183 -23
- package/dist/mjs/main.mjs +159 -57
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/project-snapshot-recovery-runner.mjs +135 -0
- package/dist/mjs/project-worktrees.mjs +183 -23
- package/dist/types/project-snapshot-recovery-runner.d.ts +10 -0
- package/dist/types/project-worktrees.d.ts +13 -1
- package/package.json +1 -1
- package/dist/cjs/heartbeat.cjs +0 -49
- package/dist/mjs/heartbeat.mjs +0 -21
- package/dist/types/heartbeat.d.ts +0 -10
package/README.md
CHANGED
|
@@ -41,6 +41,10 @@ Worker labels are mandatory and unique per user. Choose labels that describe hos
|
|
|
41
41
|
|
|
42
42
|
The supervisor also reconnects automatically after server rollouts and transient network failures. A brief bounded delay prevents a tight retry loop while the server is unavailable. Pressing Ctrl+C still stops the worker, and agent-managed processes retain their existing in-process recovery path while disconnected.
|
|
43
43
|
|
|
44
|
+
The runtime opens its control connection before recovering interrupted project snapshots. During recovery and initial project configuration it remains connected but non-dispatchable, reports its current phase plus branch/byte progress to the server, and prints the same progress locally. Recovery runs in a credential-sanitized helper process so the control loop can continue answering heartbeats. A durable `restoring` transaction state permits same-filesystem snapshots to be restored with one atomic directory move; cross-filesystem layouts retain the copy-and-fsync fallback.
|
|
45
|
+
|
|
46
|
+
The server registry is the sole heartbeat timeout authority. The worker answers server pings but does not run an independent connection watchdog, because synchronous bounded workspace operations can temporarily occupy its event loop. The server keeps those operations leased for their bounded lifetime and closes genuinely stale connections after the lease ends.
|
|
47
|
+
|
|
44
48
|
Credential-generation restart fencing protects against stale or accidentally detached descendants. It is not a hostile same-UID sandbox: code running as the worker's OS user can read or tamper with that user's config and auth files, install a persistent user service, or return after reboot. Run untrusted agents in a disposable VM that is reset at the boundary, or use a separately privileged credential broker/service account. When rotating the worker's own local `r5dctl` credential, stop the worker and reboot/reset the host before writing the replacement credential so an old same-UID descendant cannot read the new config.
|
|
45
49
|
|
|
46
50
|
One-click updates use the current user's global npm prefix and never invoke `sudo`. If that prefix is not writable, User Settings provides the exact command to run manually.
|
package/dist/cjs/main.cjs
CHANGED
|
@@ -63,7 +63,6 @@ var import_node_child_process = require("node:child_process");
|
|
|
63
63
|
var import_bun_sqlite = require("bun:sqlite");
|
|
64
64
|
var import_cli_update = require("./cli-update.cjs");
|
|
65
65
|
var import_git_process_environment = require("./git-process-environment.cjs");
|
|
66
|
-
var import_heartbeat = require("./heartbeat.cjs");
|
|
67
66
|
var import_process_tree = require("./process-tree.cjs");
|
|
68
67
|
var import_pty_output_coalescer = require("./pty-output-coalescer.cjs");
|
|
69
68
|
var import_port_forward_client = require("./port-forward-client.cjs");
|
|
@@ -81,6 +80,7 @@ var import_repository_transition_policy = require("./repository-transition-polic
|
|
|
81
80
|
var import_supervisor = require("./supervisor.cjs");
|
|
82
81
|
var import_managed_paths = require("./managed-paths.cjs");
|
|
83
82
|
var import_project_worktrees = require("./project-worktrees.cjs");
|
|
83
|
+
var import_project_snapshot_recovery_runner = require("./project-snapshot-recovery-runner.cjs");
|
|
84
84
|
var import_project_workspace_state = require("./project-workspace-state.cjs");
|
|
85
85
|
var import_workspace_git_sync = require("./workspace-git-sync.cjs");
|
|
86
86
|
const import_meta = {};
|
|
@@ -3408,28 +3408,6 @@ async function startWorker(options) {
|
|
|
3408
3408
|
runGit(["--version"]);
|
|
3409
3409
|
await verifyR5dctlAuth(baseUrl, token, label);
|
|
3410
3410
|
const initializedWorkspaceState = await workspaceSyncSingleFlight.runExclusive(() => {
|
|
3411
|
-
if (!startupProjectSnapshotRecoveryCompleted) {
|
|
3412
|
-
const snapshotRecovery = (0, import_project_worktrees.recoverStaleProjectWorktreeSnapshots)({
|
|
3413
|
-
projectsRoot,
|
|
3414
|
-
temporaryRoot: projectWorktreeSnapshotsRoot
|
|
3415
|
-
});
|
|
3416
|
-
if (snapshotRecovery.restored.length > 0) {
|
|
3417
|
-
process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
|
|
3418
|
-
`);
|
|
3419
|
-
}
|
|
3420
|
-
if (snapshotRecovery.removed.length > 0) {
|
|
3421
|
-
process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
|
|
3422
|
-
`);
|
|
3423
|
-
}
|
|
3424
|
-
for (const failure of snapshotRecovery.failed) {
|
|
3425
|
-
process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
|
|
3426
|
-
`);
|
|
3427
|
-
}
|
|
3428
|
-
if (snapshotRecovery.failed.length > 0) {
|
|
3429
|
-
throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
|
|
3430
|
-
}
|
|
3431
|
-
startupProjectSnapshotRecoveryCompleted = true;
|
|
3432
|
-
}
|
|
3433
3411
|
import_node_fs.default.mkdirSync(projectsRoot, { recursive: true });
|
|
3434
3412
|
import_node_fs.default.mkdirSync(syncRoot, { recursive: true });
|
|
3435
3413
|
import_node_fs.default.mkdirSync(artifactRoot, { recursive: true });
|
|
@@ -3468,6 +3446,8 @@ async function startWorker(options) {
|
|
|
3468
3446
|
let cliUpdateInProgress = false;
|
|
3469
3447
|
let reloadAfterClose = false;
|
|
3470
3448
|
let shutdownAfterClose = false;
|
|
3449
|
+
let startupRecoveryComplete = startupProjectSnapshotRecoveryCompleted;
|
|
3450
|
+
let startupRecoveryPromise = Promise.resolve();
|
|
3471
3451
|
const automaticWorkspaceSyncIsCurrentlyAdmitted = () => workspaceAutomaticSyncIsAdmitted({
|
|
3472
3452
|
workspaceConfigured,
|
|
3473
3453
|
activeIncidentId: activeWorkspaceIncidentId,
|
|
@@ -3884,8 +3864,14 @@ async function startWorker(options) {
|
|
|
3884
3864
|
})
|
|
3885
3865
|
)
|
|
3886
3866
|
].sort();
|
|
3887
|
-
const ensureConfiguredProjects = async () => {
|
|
3888
|
-
|
|
3867
|
+
const ensureConfiguredProjects = async (onProgress) => {
|
|
3868
|
+
const projects = [...projectConfigById.values()];
|
|
3869
|
+
for (const [projectIndex, project] of projects.entries()) {
|
|
3870
|
+
onProgress?.({
|
|
3871
|
+
detail: `Configuring ${project.projectPath}`,
|
|
3872
|
+
completedItems: projectIndex,
|
|
3873
|
+
totalItems: projects.length
|
|
3874
|
+
});
|
|
3889
3875
|
validateProjectId(project.projectId);
|
|
3890
3876
|
(0, import_repository_transition_policy.assertRepositoryTransitionState)(project);
|
|
3891
3877
|
for (const branch of project.branches) validateBranchName(branch.branchName);
|
|
@@ -3933,7 +3919,14 @@ async function startWorker(options) {
|
|
|
3933
3919
|
mirrorUrl: connection.mirrorUrl,
|
|
3934
3920
|
mirrorCredentialUsername: connection.mirrorCredentialUsername,
|
|
3935
3921
|
credentialHelper: connection.credentialHelper,
|
|
3936
|
-
gitIdentity: workspaceGitIdentity
|
|
3922
|
+
gitIdentity: workspaceGitIdentity,
|
|
3923
|
+
onSnapshotProgress: (progress) => onProgress?.({
|
|
3924
|
+
detail: progress.phase === "scanning" ? `Measuring ${project.projectPath} workspace snapshot` : progress.phase === "syncing" ? `Making ${project.projectPath} workspace snapshot durable` : `${progress.operation === "restore" ? "Restoring" : "Snapshotting"} ${project.projectPath}${progress.branchName ? `/${progress.branchName}` : ""}`,
|
|
3925
|
+
completedBytes: progress.completedBytes,
|
|
3926
|
+
completedItems: progress.completedBranches,
|
|
3927
|
+
totalItems: progress.totalBranches,
|
|
3928
|
+
...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
|
|
3929
|
+
})
|
|
3937
3930
|
});
|
|
3938
3931
|
readyProjectIds.add(project.projectId);
|
|
3939
3932
|
reconciledProjectConfigFingerprints.set(project.projectId, configFingerprint);
|
|
@@ -3968,6 +3961,11 @@ async function startWorker(options) {
|
|
|
3968
3961
|
`
|
|
3969
3962
|
);
|
|
3970
3963
|
}
|
|
3964
|
+
onProgress?.({
|
|
3965
|
+
detail: `Configured ${project.projectPath}`,
|
|
3966
|
+
completedItems: projectIndex + 1,
|
|
3967
|
+
totalItems: projects.length
|
|
3968
|
+
});
|
|
3971
3969
|
}
|
|
3972
3970
|
};
|
|
3973
3971
|
const collectAheadOfOriginBranches = async () => {
|
|
@@ -4360,6 +4358,11 @@ async function startWorker(options) {
|
|
|
4360
4358
|
if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result2);
|
|
4361
4359
|
return result2;
|
|
4362
4360
|
}
|
|
4361
|
+
sendLifecycleProgress({
|
|
4362
|
+
phase: "syncing",
|
|
4363
|
+
operationId: attemptId,
|
|
4364
|
+
detail: input.trigger.detail ?? `Synchronizing workspace (${input.trigger.type})`
|
|
4365
|
+
});
|
|
4363
4366
|
if (!input.requestId && input.sendResult !== false) {
|
|
4364
4367
|
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
4365
4368
|
throw new Error("Cannot start autonomous workspace synchronization without an open worker control socket");
|
|
@@ -4462,6 +4465,9 @@ async function startWorker(options) {
|
|
|
4462
4465
|
}
|
|
4463
4466
|
};
|
|
4464
4467
|
if (input.sendResult !== false) sendWorkspaceSyncResult(input.requestId, result);
|
|
4468
|
+
sendLifecycleProgress(
|
|
4469
|
+
result.outcome === "failed" ? { phase: "failed", operationId: attemptId, detail: result.error ?? "Workspace synchronization failed" } : { phase: "ready", operationId: attemptId, detail: "Workspace synchronized" }
|
|
4470
|
+
);
|
|
4465
4471
|
return result;
|
|
4466
4472
|
};
|
|
4467
4473
|
const scheduleAutomaticWorkspaceSync = (trigger, delayMs = WORKSPACE_GIT_QUIET_MS) => {
|
|
@@ -4501,7 +4507,6 @@ async function startWorker(options) {
|
|
|
4501
4507
|
);
|
|
4502
4508
|
}).finally(() => {
|
|
4503
4509
|
automaticSyncInFlight = false;
|
|
4504
|
-
heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
4505
4510
|
if (pendingAutomaticTrigger && automaticWorkspaceSyncIsCurrentlyAdmitted()) {
|
|
4506
4511
|
scheduleAutomaticWorkspaceSync(pendingAutomaticTrigger, WORKSPACE_GIT_QUIET_MS);
|
|
4507
4512
|
}
|
|
@@ -4537,6 +4542,14 @@ async function startWorker(options) {
|
|
|
4537
4542
|
});
|
|
4538
4543
|
};
|
|
4539
4544
|
const configureWorkerWorkspace = async (message, receiptGeneration) => {
|
|
4545
|
+
const configurationOperationId = crypto.randomUUID();
|
|
4546
|
+
sendLifecycleProgress({
|
|
4547
|
+
phase: "configuring",
|
|
4548
|
+
operationId: configurationOperationId,
|
|
4549
|
+
detail: `Preparing ${message.projects.length} project(s)`,
|
|
4550
|
+
completedItems: 0,
|
|
4551
|
+
totalItems: message.projects.length
|
|
4552
|
+
});
|
|
4540
4553
|
const incidentDeferral = workspaceConfigurationIncidentDeferral({
|
|
4541
4554
|
requestedIncidentId: message.deferWorkspaceSyncForIncidentId,
|
|
4542
4555
|
activeIncidentId: activeWorkspaceIncidentId
|
|
@@ -4901,7 +4914,13 @@ async function startWorker(options) {
|
|
|
4901
4914
|
// checkout initialization from erasing synchronized dirt.
|
|
4902
4915
|
{ ignoreBusy: true }
|
|
4903
4916
|
);
|
|
4904
|
-
await ensureConfiguredProjects()
|
|
4917
|
+
await ensureConfiguredProjects((progress) => {
|
|
4918
|
+
sendLifecycleProgress({ phase: "configuring", operationId: configurationOperationId, ...progress });
|
|
4919
|
+
process.stdout.write(
|
|
4920
|
+
`[r5d-worker] ${progress.detail ?? "Configuring workspace"}${progress.completedBytes === void 0 ? "" : `: ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes`}${progress.completedItems === void 0 ? "" : `, ${progress.completedItems}/${progress.totalItems ?? "?"} item(s)`}
|
|
4921
|
+
`
|
|
4922
|
+
);
|
|
4923
|
+
});
|
|
4905
4924
|
const configurationSyncAdmissionGeneration = workerAdmissionGeneration;
|
|
4906
4925
|
const assertConfigurationSyncStillAdmitted = () => {
|
|
4907
4926
|
if (receiptGeneration !== workspaceConfigurationReceiptGeneration || configurationSyncAdmissionGeneration !== workerAdmissionGeneration || !workspaceConfigurationIncidentSnapshotIsCurrent(incidentDeferral.incidentId, activeWorkspaceIncidentId) || currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) {
|
|
@@ -5000,10 +5019,60 @@ async function startWorker(options) {
|
|
|
5000
5019
|
Authorization: `Bearer ${token}`
|
|
5001
5020
|
}
|
|
5002
5021
|
});
|
|
5003
|
-
|
|
5004
|
-
|
|
5005
|
-
|
|
5006
|
-
|
|
5022
|
+
const sendLifecycleProgress = (progress) => {
|
|
5023
|
+
if (currentWorkerSocket !== ws || ws.readyState !== WebSocket.OPEN) return;
|
|
5024
|
+
sendWorkerMessage(ws, { type: "worker_progress", progress });
|
|
5025
|
+
};
|
|
5026
|
+
const describeSnapshotProgress = (progress) => {
|
|
5027
|
+
const project = import_node_path.default.relative(projectsRoot, progress.projectRoot) || import_node_path.default.basename(progress.projectRoot);
|
|
5028
|
+
const branch = progress.branchName ? `/${progress.branchName}` : "";
|
|
5029
|
+
if (progress.phase === "scanning") return `Scanning ${project} recovery snapshot`;
|
|
5030
|
+
if (progress.phase === "syncing") return `Making restored ${project} files durable`;
|
|
5031
|
+
if (progress.phase === "cleaning") return `Cleaning recovered ${project} snapshot`;
|
|
5032
|
+
return `Restoring ${project}${branch}`;
|
|
5033
|
+
};
|
|
5034
|
+
const performStartupRecovery = async (operationId) => {
|
|
5035
|
+
if (startupProjectSnapshotRecoveryCompleted) return;
|
|
5036
|
+
sendLifecycleProgress({ phase: "recovering", operationId, detail: "Checking interrupted workspace transactions" });
|
|
5037
|
+
const snapshotRecovery = await workspaceSyncSingleFlight.runExclusive(
|
|
5038
|
+
async () => await (0, import_project_snapshot_recovery_runner.recoverProjectSnapshotsInChild)({
|
|
5039
|
+
projectsRoot,
|
|
5040
|
+
snapshotRoot: projectWorktreeSnapshotsRoot,
|
|
5041
|
+
onProgress: (progress) => {
|
|
5042
|
+
const lifecycleProgress = {
|
|
5043
|
+
phase: "recovering",
|
|
5044
|
+
operationId,
|
|
5045
|
+
detail: describeSnapshotProgress(progress),
|
|
5046
|
+
completedBytes: progress.completedBytes,
|
|
5047
|
+
completedItems: progress.completedBranches,
|
|
5048
|
+
totalItems: progress.totalBranches,
|
|
5049
|
+
...progress.totalBytes === void 0 ? {} : { totalBytes: progress.totalBytes }
|
|
5050
|
+
};
|
|
5051
|
+
sendLifecycleProgress(lifecycleProgress);
|
|
5052
|
+
process.stdout.write(
|
|
5053
|
+
`[r5d-worker] ${lifecycleProgress.detail}: ${progress.completedBranches}/${progress.totalBranches} branch(es), ${progress.completedBytes}/${progress.totalBytes ?? "?"} bytes
|
|
5054
|
+
`
|
|
5055
|
+
);
|
|
5056
|
+
}
|
|
5057
|
+
})
|
|
5058
|
+
);
|
|
5059
|
+
if (snapshotRecovery.restored.length > 0) {
|
|
5060
|
+
process.stdout.write(`[r5d-worker] restored ${snapshotRecovery.restored.length} interrupted project snapshot(s)
|
|
5061
|
+
`);
|
|
5062
|
+
}
|
|
5063
|
+
if (snapshotRecovery.removed.length > 0) {
|
|
5064
|
+
process.stdout.write(`[r5d-worker] removed ${snapshotRecovery.removed.length} completed/incomplete project snapshot(s)
|
|
5065
|
+
`);
|
|
5066
|
+
}
|
|
5067
|
+
for (const failure of snapshotRecovery.failed) {
|
|
5068
|
+
process.stderr.write(`[r5d-worker] failed to recover stale project snapshot ${failure.path}: ${failure.error}
|
|
5069
|
+
`);
|
|
5070
|
+
}
|
|
5071
|
+
if (snapshotRecovery.failed.length > 0) {
|
|
5072
|
+
throw new Error("Project snapshot recovery is incomplete; refusing to mutate project checkouts");
|
|
5073
|
+
}
|
|
5074
|
+
startupProjectSnapshotRecoveryCompleted = true;
|
|
5075
|
+
};
|
|
5007
5076
|
const handleGracefulShutdown = () => {
|
|
5008
5077
|
if (shutdownAfterClose) return;
|
|
5009
5078
|
shutdownAfterClose = true;
|
|
@@ -5031,25 +5100,29 @@ async function startWorker(options) {
|
|
|
5031
5100
|
};
|
|
5032
5101
|
process.on("SIGTERM", handleGracefulShutdown);
|
|
5033
5102
|
process.on("SIGINT", handleGracefulShutdown);
|
|
5034
|
-
const stopHeartbeatWatchdog = () => {
|
|
5035
|
-
if (heartbeatWatchdog) {
|
|
5036
|
-
clearInterval(heartbeatWatchdog);
|
|
5037
|
-
heartbeatWatchdog = void 0;
|
|
5038
|
-
}
|
|
5039
|
-
};
|
|
5040
5103
|
ws.addEventListener("open", () => {
|
|
5041
5104
|
currentWorkerSocket = ws;
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5105
|
+
const startupOperationId = crypto.randomUUID();
|
|
5106
|
+
startupRecoveryPromise = Promise.resolve().then(async () => await performStartupRecovery(startupOperationId)).then(() => {
|
|
5107
|
+
startupRecoveryComplete = true;
|
|
5108
|
+
if (currentWorkerSocket === ws && ws.readyState === WebSocket.OPEN) {
|
|
5109
|
+
sendWorkerMessage(ws, { type: "startup_ready" });
|
|
5110
|
+
sendLifecycleProgress({
|
|
5111
|
+
phase: "configuring",
|
|
5112
|
+
operationId: crypto.randomUUID(),
|
|
5113
|
+
detail: "Waiting for workspace configuration"
|
|
5114
|
+
});
|
|
5045
5115
|
}
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5116
|
+
}).catch((error) => {
|
|
5117
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5118
|
+
sendLifecycleProgress({ phase: "failed", operationId: startupOperationId, detail: message });
|
|
5119
|
+
process.stderr.write(`[r5d-worker] startup recovery failed: ${message}
|
|
5120
|
+
`);
|
|
5121
|
+
ws.close(1011, "Project snapshot recovery failed");
|
|
5122
|
+
});
|
|
5051
5123
|
const hello = {
|
|
5052
5124
|
type: "hello",
|
|
5125
|
+
startupReady: startupRecoveryComplete,
|
|
5053
5126
|
hostInfo: {
|
|
5054
5127
|
hostname: (0, import_node_os.hostname)(),
|
|
5055
5128
|
platform: process.platform,
|
|
@@ -5087,6 +5160,8 @@ async function startWorker(options) {
|
|
|
5087
5160
|
ws.addEventListener("message", (event) => {
|
|
5088
5161
|
void (async () => {
|
|
5089
5162
|
const message = JSON.parse(String(event.data));
|
|
5163
|
+
if (!startupRecoveryComplete && message.type !== "ping") await startupRecoveryPromise;
|
|
5164
|
+
if (!startupRecoveryComplete && message.type !== "ping") return;
|
|
5090
5165
|
const messageAdmissionGeneration = workerAdmissionGeneration;
|
|
5091
5166
|
const assertMessageAdmission = () => assertWorkerChildAdmission({
|
|
5092
5167
|
capturedGeneration: messageAdmissionGeneration,
|
|
@@ -5094,7 +5169,7 @@ async function startWorker(options) {
|
|
|
5094
5169
|
sourceSocketIsCurrent: currentWorkerSocket === ws,
|
|
5095
5170
|
sourceSocketOpen: ws.readyState === WebSocket.OPEN,
|
|
5096
5171
|
workspaceConfigured,
|
|
5097
|
-
runtimeHealthy: !shutdownAfterClose
|
|
5172
|
+
runtimeHealthy: !shutdownAfterClose
|
|
5098
5173
|
});
|
|
5099
5174
|
if (message.type === "connected") {
|
|
5100
5175
|
return;
|
|
@@ -5114,6 +5189,19 @@ async function startWorker(options) {
|
|
|
5114
5189
|
aheadOfOriginBranches: configured.aheadOfOriginBranches,
|
|
5115
5190
|
...configured.deferredWorkspaceSyncForIncidentId ? { deferredWorkspaceSyncForIncidentId: configured.deferredWorkspaceSyncForIncidentId } : {}
|
|
5116
5191
|
});
|
|
5192
|
+
sendLifecycleProgress(
|
|
5193
|
+
configured.result.outcome === "failed" ? {
|
|
5194
|
+
phase: "failed",
|
|
5195
|
+
operationId: configured.result.attemptId,
|
|
5196
|
+
detail: configured.result.error ?? "Workspace configuration failed"
|
|
5197
|
+
} : {
|
|
5198
|
+
phase: "ready",
|
|
5199
|
+
operationId: configured.result.attemptId,
|
|
5200
|
+
detail: `${message.projects.length} project(s) configured`,
|
|
5201
|
+
completedItems: message.projects.length,
|
|
5202
|
+
totalItems: message.projects.length
|
|
5203
|
+
}
|
|
5204
|
+
);
|
|
5117
5205
|
process.stdout.write(
|
|
5118
5206
|
`[r5d-worker] workspace configured: ${message.projects.length} project(s), ${configured.pending.length} pending checkout(s)
|
|
5119
5207
|
`
|
|
@@ -5453,8 +5541,6 @@ async function startWorker(options) {
|
|
|
5453
5541
|
return;
|
|
5454
5542
|
}
|
|
5455
5543
|
if (message.type === "ping") {
|
|
5456
|
-
lastServerHeartbeatAt = Date.now();
|
|
5457
|
-
heartbeatBusyGrace = null;
|
|
5458
5544
|
ws.send(JSON.stringify({ type: "pong" }));
|
|
5459
5545
|
return;
|
|
5460
5546
|
}
|
|
@@ -5868,15 +5954,12 @@ async function startWorker(options) {
|
|
|
5868
5954
|
})().catch((error) => {
|
|
5869
5955
|
process.stderr.write(`[r5d-worker] message handling failed: ${error instanceof Error ? error.message : String(error)}
|
|
5870
5956
|
`);
|
|
5871
|
-
}).finally(() => {
|
|
5872
|
-
heartbeatBusyGrace = (0, import_heartbeat.grantWorkerHeartbeatBusyGrace)(lastServerHeartbeatAt, heartbeatBusyGrace);
|
|
5873
5957
|
});
|
|
5874
5958
|
});
|
|
5875
5959
|
ws.addEventListener("close", (event) => {
|
|
5876
5960
|
const processEvents = process;
|
|
5877
5961
|
processEvents.off("SIGTERM", handleGracefulShutdown);
|
|
5878
5962
|
processEvents.off("SIGINT", handleGracefulShutdown);
|
|
5879
|
-
stopHeartbeatWatchdog();
|
|
5880
5963
|
if (workspaceAutomaticTimer) {
|
|
5881
5964
|
clearTimeout(workspaceAutomaticTimer);
|
|
5882
5965
|
workspaceAutomaticTimer = void 0;
|
|
@@ -5907,6 +5990,21 @@ async function startWorker(options) {
|
|
|
5907
5990
|
if (shutdownAfterClose) {
|
|
5908
5991
|
process.exit(0);
|
|
5909
5992
|
}
|
|
5993
|
+
if (!startupRecoveryComplete) {
|
|
5994
|
+
process.stderr.write("[r5d-worker] control connection closed during workspace recovery; finishing recovery before reconnecting\n");
|
|
5995
|
+
void startupRecoveryPromise.then(() => {
|
|
5996
|
+
if (!startupRecoveryComplete) {
|
|
5997
|
+
process.exit(1);
|
|
5998
|
+
return;
|
|
5999
|
+
}
|
|
6000
|
+
void startWorker(options).catch((error) => {
|
|
6001
|
+
process.stderr.write(`[r5d-worker] reconnect failed: ${error instanceof Error ? error.message : String(error)}
|
|
6002
|
+
`);
|
|
6003
|
+
process.exit(error instanceof WorkerServerUnavailableError ? import_supervisor.WORKER_RECONNECT_EXIT_CODE : 1);
|
|
6004
|
+
});
|
|
6005
|
+
});
|
|
6006
|
+
return;
|
|
6007
|
+
}
|
|
5910
6008
|
if (activeProcesses.size > 0 || credentialBearingProcessGroups.size > 0 || activePtys.size > 0 || pendingProcessTerminals.size > 0) {
|
|
5911
6009
|
process.stderr.write(
|
|
5912
6010
|
`[r5d-worker] ${activeProcesses.size} process(es), ${credentialBearingProcessGroups.size} owned process group(s), and ${activePtys.size} PTY(s) active with ${pendingProcessTerminals.size} terminal report(s) pending; reconnecting in ${import_supervisor.WORKER_RECONNECT_DELAY_MS}ms
|
|
@@ -5937,6 +6035,11 @@ async function startWorker(options) {
|
|
|
5937
6035
|
});
|
|
5938
6036
|
}
|
|
5939
6037
|
async function main() {
|
|
6038
|
+
if (process.argv[2] === import_project_snapshot_recovery_runner.PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND) {
|
|
6039
|
+
const exitCode2 = (0, import_project_snapshot_recovery_runner.runProjectSnapshotRecoveryHelper)(process.argv.slice(3));
|
|
6040
|
+
if (exitCode2 !== 0) process.exit(exitCode2);
|
|
6041
|
+
return;
|
|
6042
|
+
}
|
|
5940
6043
|
const parsed = parseArgs(process.argv.slice(2));
|
|
5941
6044
|
if (parsed.command === "help") {
|
|
5942
6045
|
printHelp();
|
package/dist/cjs/package.json
CHANGED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
var project_snapshot_recovery_runner_exports = {};
|
|
30
|
+
__export(project_snapshot_recovery_runner_exports, {
|
|
31
|
+
PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND: () => PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
|
|
32
|
+
recoverProjectSnapshotsInChild: () => recoverProjectSnapshotsInChild,
|
|
33
|
+
runProjectSnapshotRecoveryHelper: () => runProjectSnapshotRecoveryHelper
|
|
34
|
+
});
|
|
35
|
+
module.exports = __toCommonJS(project_snapshot_recovery_runner_exports);
|
|
36
|
+
var import_node_path = __toESM(require("node:path"), 1);
|
|
37
|
+
var import_node_child_process = require("node:child_process");
|
|
38
|
+
var import_project_worktrees = require("./project-worktrees.cjs");
|
|
39
|
+
const PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND = "__recover-project-snapshots";
|
|
40
|
+
function requireAbsolutePath(value, label) {
|
|
41
|
+
if (!value) throw new Error(`Missing ${label}`);
|
|
42
|
+
const resolved = import_node_path.default.resolve(value);
|
|
43
|
+
if (resolved !== value) throw new Error(`${label} must be an absolute path`);
|
|
44
|
+
return resolved;
|
|
45
|
+
}
|
|
46
|
+
function parseHelperArguments(args) {
|
|
47
|
+
let projectsRoot;
|
|
48
|
+
let snapshotRoot;
|
|
49
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
50
|
+
const argument = args[index];
|
|
51
|
+
if (argument === "--projects-root") {
|
|
52
|
+
projectsRoot = args[index + 1];
|
|
53
|
+
index += 1;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (argument === "--snapshot-root") {
|
|
57
|
+
snapshotRoot = args[index + 1];
|
|
58
|
+
index += 1;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
throw new Error(`Unknown project snapshot recovery helper argument: ${argument ?? "(missing)"}`);
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
projectsRoot: requireAbsolutePath(projectsRoot, "projects root"),
|
|
65
|
+
snapshotRoot: requireAbsolutePath(snapshotRoot, "snapshot root")
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
function writeHelperMessage(message) {
|
|
69
|
+
process.stdout.write(`${JSON.stringify(message)}
|
|
70
|
+
`);
|
|
71
|
+
}
|
|
72
|
+
function runProjectSnapshotRecoveryHelper(args) {
|
|
73
|
+
const { projectsRoot, snapshotRoot } = parseHelperArguments(args);
|
|
74
|
+
const result = (0, import_project_worktrees.recoverStaleProjectWorktreeSnapshots)({
|
|
75
|
+
projectsRoot,
|
|
76
|
+
temporaryRoot: snapshotRoot,
|
|
77
|
+
onProgress: (progress) => writeHelperMessage({ type: "progress", progress })
|
|
78
|
+
});
|
|
79
|
+
writeHelperMessage({ type: "result", result });
|
|
80
|
+
return result.failed.length === 0 ? 0 : 1;
|
|
81
|
+
}
|
|
82
|
+
function isRecoveryProgress(value) {
|
|
83
|
+
if (!value || typeof value !== "object") return false;
|
|
84
|
+
const progress = value;
|
|
85
|
+
return progress.operation === "restore" && (progress.phase === "scanning" || progress.phase === "copying" || progress.phase === "moving" || progress.phase === "syncing" || progress.phase === "cleaning") && typeof progress.projectRoot === "string" && Number.isSafeInteger(progress.completedBranches) && Number.isSafeInteger(progress.totalBranches) && typeof progress.completedBytes === "number" && Number.isFinite(progress.completedBytes) && (progress.totalBytes === void 0 || typeof progress.totalBytes === "number" && Number.isFinite(progress.totalBytes));
|
|
86
|
+
}
|
|
87
|
+
function isRecoveryResult(value) {
|
|
88
|
+
if (!value || typeof value !== "object") return false;
|
|
89
|
+
const result = value;
|
|
90
|
+
return Array.isArray(result.restored) && Array.isArray(result.removed) && Array.isArray(result.failed);
|
|
91
|
+
}
|
|
92
|
+
function recoveryChildEnvironment() {
|
|
93
|
+
const environment = {};
|
|
94
|
+
for (const name of ["PATH", "TMPDIR", "TMP", "TEMP", "SystemRoot", "WINDIR"]) {
|
|
95
|
+
const value = process.env[name];
|
|
96
|
+
if (value !== void 0) environment[name] = value;
|
|
97
|
+
}
|
|
98
|
+
return environment;
|
|
99
|
+
}
|
|
100
|
+
async function recoverProjectSnapshotsInChild(input) {
|
|
101
|
+
const entrypoint = process.argv[1];
|
|
102
|
+
if (!entrypoint) throw new Error("Cannot locate the r5d-worker runtime entrypoint for snapshot recovery");
|
|
103
|
+
const child = (0, import_node_child_process.spawn)(
|
|
104
|
+
process.execPath,
|
|
105
|
+
[
|
|
106
|
+
entrypoint,
|
|
107
|
+
PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
|
|
108
|
+
"--projects-root",
|
|
109
|
+
import_node_path.default.resolve(input.projectsRoot),
|
|
110
|
+
"--snapshot-root",
|
|
111
|
+
import_node_path.default.resolve(input.snapshotRoot)
|
|
112
|
+
],
|
|
113
|
+
{
|
|
114
|
+
env: recoveryChildEnvironment(),
|
|
115
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
116
|
+
}
|
|
117
|
+
);
|
|
118
|
+
const terminateOnParentExit = () => {
|
|
119
|
+
if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM");
|
|
120
|
+
};
|
|
121
|
+
const parentProcessEvents = process;
|
|
122
|
+
parentProcessEvents.once("exit", terminateOnParentExit);
|
|
123
|
+
let stdout = "";
|
|
124
|
+
let stderr = "";
|
|
125
|
+
let result;
|
|
126
|
+
child.stdout.setEncoding("utf8");
|
|
127
|
+
child.stdout.on("data", (chunk) => {
|
|
128
|
+
stdout += chunk;
|
|
129
|
+
while (true) {
|
|
130
|
+
const newline = stdout.indexOf("\n");
|
|
131
|
+
if (newline < 0) break;
|
|
132
|
+
const line = stdout.slice(0, newline);
|
|
133
|
+
stdout = stdout.slice(newline + 1);
|
|
134
|
+
if (!line.trim()) continue;
|
|
135
|
+
let message;
|
|
136
|
+
try {
|
|
137
|
+
message = JSON.parse(line);
|
|
138
|
+
} catch {
|
|
139
|
+
stderr += `Invalid recovery helper output: ${line}
|
|
140
|
+
`;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (!message || typeof message !== "object") continue;
|
|
144
|
+
const record = message;
|
|
145
|
+
if (record.type === "progress" && isRecoveryProgress(record.progress)) input.onProgress?.(record.progress);
|
|
146
|
+
if (record.type === "result" && isRecoveryResult(record.result)) result = record.result;
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
child.stderr.setEncoding("utf8");
|
|
150
|
+
child.stderr.on("data", (chunk) => {
|
|
151
|
+
stderr = `${stderr}${chunk}`.slice(-32768);
|
|
152
|
+
});
|
|
153
|
+
const exitCode = await new Promise((resolve, reject) => {
|
|
154
|
+
child.once("error", reject);
|
|
155
|
+
child.once("close", (code, signal) => {
|
|
156
|
+
if (signal) reject(new Error(`Project snapshot recovery helper exited on signal ${signal}`));
|
|
157
|
+
else resolve(code ?? 1);
|
|
158
|
+
});
|
|
159
|
+
}).finally(() => parentProcessEvents.off("exit", terminateOnParentExit));
|
|
160
|
+
if (exitCode !== 0 || !result) {
|
|
161
|
+
const detail = stderr.trim();
|
|
162
|
+
throw new Error(detail || `Project snapshot recovery helper exited with status ${exitCode}`);
|
|
163
|
+
}
|
|
164
|
+
return result;
|
|
165
|
+
}
|
|
166
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
167
|
+
0 && (module.exports = {
|
|
168
|
+
PROJECT_SNAPSHOT_RECOVERY_HELPER_COMMAND,
|
|
169
|
+
recoverProjectSnapshotsInChild,
|
|
170
|
+
runProjectSnapshotRecoveryHelper
|
|
171
|
+
});
|