machine-bridge-mcp 2.0.0 → 3.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -0
- package/CONTRIBUTING.md +20 -25
- package/README.md +21 -14
- package/SECURITY.md +147 -123
- package/browser-extension/manifest.json +3 -2
- package/browser-extension/service-worker.js +31 -12
- package/docs/ARCHITECTURE.md +28 -13
- package/docs/AUDIT.md +101 -2
- package/docs/CLIENTS.md +2 -2
- package/docs/ENGINEERING.md +9 -3
- package/docs/GETTING_STARTED.md +10 -9
- package/docs/LOCAL_AUTHORIZATION.md +115 -72
- package/docs/LOGGING.md +14 -3
- package/docs/MANAGED_JOBS.md +24 -29
- package/docs/MULTI_ACCOUNT.md +98 -37
- package/docs/OPERATIONS.md +34 -23
- package/docs/OVERVIEW.md +7 -7
- package/docs/PROJECT_STANDARDS.md +9 -5
- package/docs/RELEASING.md +171 -64
- package/docs/TESTING.md +21 -9
- package/docs/THREAT_MODEL.md +138 -67
- package/docs/TOOL_REFERENCE.md +2 -2
- package/docs/UPGRADING.md +102 -18
- package/native/macos/MachineBridgeTrustBroker.swift +233 -0
- package/package.json +37 -6
- package/scripts/candidate-runtime-store.mjs +38 -0
- package/scripts/check-plan.mjs +19 -0
- package/scripts/check-runner.mjs +4 -1
- package/scripts/commit-message-check.mjs +5 -2
- package/scripts/coverage-check.mjs +30 -2
- package/scripts/github-push.mjs +11 -0
- package/scripts/github-release.mjs +38 -7
- package/scripts/install-published-prerelease.mjs +167 -0
- package/scripts/local-release-acceptance.mjs +23 -12
- package/scripts/npm-publication-policy.mjs +16 -0
- package/scripts/prerelease-activation.mjs +79 -0
- package/scripts/privacy-check.mjs +7 -5
- package/scripts/promotion-digest.mjs +121 -0
- package/scripts/publish-npm.mjs +51 -0
- package/scripts/published-release.mjs +87 -0
- package/scripts/release-acceptance.mjs +30 -1
- package/scripts/release-candidate-manifest.mjs +40 -0
- package/scripts/release-channel.mjs +106 -0
- package/scripts/release-impact-check.mjs +12 -22
- package/scripts/release-soak.mjs +237 -0
- package/scripts/start-release-candidate.mjs +113 -7
- package/src/local/account-access.mjs +6 -0
- package/src/local/account-admin.mjs +34 -16
- package/src/local/agent-context.mjs +45 -42
- package/src/local/app-automation.mjs +9 -4
- package/src/local/authority-context.mjs +106 -0
- package/src/local/browser-bridge-http.mjs +4 -1
- package/src/local/browser-bridge.mjs +14 -9
- package/src/local/browser-broker-routes.mjs +4 -0
- package/src/local/browser-broker-server.mjs +5 -4
- package/src/local/browser-extension-identity.mjs +50 -0
- package/src/local/browser-extension-protocol.mjs +9 -4
- package/src/local/browser-operation-service.mjs +7 -0
- package/src/local/browser-pairing-http.mjs +36 -0
- package/src/local/browser-pairing-store.mjs +56 -47
- package/src/local/cli-account-admin.mjs +41 -7
- package/src/local/cli-activate.mjs +90 -0
- package/src/local/cli-approval.mjs +14 -58
- package/src/local/cli-local-admin.mjs +5 -15
- package/src/local/cli-options.mjs +11 -7
- package/src/local/cli-service.mjs +35 -5
- package/src/local/cli.mjs +119 -58
- package/src/local/daemon-process.mjs +6 -0
- package/src/local/delegated-process-sandbox.mjs +147 -0
- package/src/local/device-identity.mjs +157 -48
- package/src/local/device-root-provider.mjs +87 -0
- package/src/local/errors.mjs +25 -3
- package/src/local/git-service.mjs +16 -15
- package/src/local/job-runner.mjs +35 -27
- package/src/local/log.mjs +18 -44
- package/src/local/macos-trust-broker.mjs +320 -0
- package/src/local/managed-job-lock.mjs +18 -0
- package/src/local/managed-job-projection.mjs +9 -1
- package/src/local/managed-job-runner.mjs +13 -1
- package/src/local/managed-job-storage.mjs +2 -1
- package/src/local/managed-job-terminal.mjs +138 -0
- package/src/local/managed-jobs.mjs +88 -32
- package/src/local/operation-authorization.mjs +75 -239
- package/src/local/operation-risk.mjs +28 -0
- package/src/local/operation-state-lock.mjs +7 -86
- package/src/local/owner-state-lock.mjs +104 -0
- package/src/local/policy.mjs +19 -0
- package/src/local/process-execution.mjs +21 -9
- package/src/local/process-sessions.mjs +27 -17
- package/src/local/process-tracker.mjs +30 -6
- package/src/local/process-tree-ownership.mjs +59 -0
- package/src/local/process-tree.mjs +22 -18
- package/src/local/relay-connection.mjs +47 -35
- package/src/local/relay-diagnostics.mjs +65 -0
- package/src/local/remote-configuration.mjs +48 -0
- package/src/local/runtime-activation.mjs +141 -0
- package/src/local/runtime-diagnostics.mjs +21 -0
- package/src/local/runtime-relay.mjs +9 -8
- package/src/local/runtime-reporting.mjs +8 -3
- package/src/local/runtime-resource-service.mjs +77 -0
- package/src/local/runtime-tool-handlers.mjs +8 -8
- package/src/local/runtime.mjs +85 -95
- package/src/local/secure-file.mjs +5 -1
- package/src/local/security-audit-log.mjs +204 -0
- package/src/local/service-convergence.mjs +17 -1
- package/src/local/service-ownership.mjs +17 -0
- package/src/local/service-restart-handoff.mjs +50 -0
- package/src/local/service-restart-scheduler.mjs +49 -0
- package/src/local/service-status.mjs +47 -0
- package/src/local/service.mjs +142 -43
- package/src/local/state-inventory.mjs +21 -3
- package/src/local/state.mjs +155 -17
- package/src/local/stdio.mjs +5 -0
- package/src/local/tool-executor.mjs +39 -6
- package/src/local/tool-result-boundary.mjs +51 -0
- package/src/local/tools.mjs +1 -0
- package/src/local/trusted-executable.mjs +53 -0
- package/src/local/trusted-git-executable.mjs +34 -0
- package/src/local/trusted-github-cli.mjs +24 -0
- package/src/local/windows-service.mjs +20 -0
- package/src/local/worker-deployment.mjs +3 -4
- package/src/local/worker-secret-file.mjs +2 -2
- package/src/local/workspace-file-service.mjs +34 -31
- package/src/shared/admin-auth.d.mts +2 -1
- package/src/shared/admin-auth.mjs +3 -3
- package/src/shared/device-session-auth.d.mts +20 -0
- package/src/shared/device-session-auth.mjs +51 -0
- package/src/shared/log-redaction.d.mts +9 -0
- package/src/shared/log-redaction.mjs +51 -0
- package/src/shared/server-metadata.json +1 -1
- package/src/shared/tool-catalog.json +2 -2
- package/src/worker/account-admin.ts +23 -22
- package/src/worker/authority.ts +23 -4
- package/src/worker/daemon-auth.ts +45 -67
- package/src/worker/daemon-sockets.ts +12 -2
- package/src/worker/device-session-verifier.ts +129 -0
- package/src/worker/dpop.ts +151 -0
- package/src/worker/http.ts +47 -16
- package/src/worker/index.ts +53 -16
- package/src/worker/nonce-store.ts +1 -5
- package/src/worker/oauth-client-admin.ts +45 -0
- package/src/worker/oauth-controller.ts +55 -6
- package/src/worker/oauth-refresh-families.ts +21 -7
- package/src/worker/oauth-state.ts +6 -0
- package/src/worker/oauth-tokens.ts +28 -8
- package/src/worker/observability.ts +4 -2
- package/src/worker/websocket-protocol.ts +8 -2
package/src/local/policy.mjs
CHANGED
|
@@ -111,6 +111,25 @@ export function normalizePolicy(policy = {}) {
|
|
|
111
111
|
return Object.freeze(normalized);
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
/** @param {PolicyInput} left @param {PolicyInput} right @param {{profile?: string, origin?: string}} [options] */
|
|
115
|
+
export function intersectPolicies(left, right, options = {}) {
|
|
116
|
+
const a = normalizePolicy(left);
|
|
117
|
+
const b = normalizePolicy(right);
|
|
118
|
+
const rank = { off: 0, direct: 1, shell: 2 };
|
|
119
|
+
const execMode = rank[a.execMode] <= rank[b.execMode] ? a.execMode : b.execMode;
|
|
120
|
+
return Object.freeze({
|
|
121
|
+
profile: typeof options.profile === "string" && options.profile ? options.profile : "custom",
|
|
122
|
+
origin: typeof options.origin === "string" && options.origin ? options.origin : "custom",
|
|
123
|
+
revision: Math.max(a.revision, b.revision),
|
|
124
|
+
allowWrite: a.allowWrite && b.allowWrite,
|
|
125
|
+
allowExec: execMode !== "off",
|
|
126
|
+
execMode,
|
|
127
|
+
unrestrictedPaths: a.unrestrictedPaths && b.unrestrictedPaths,
|
|
128
|
+
minimalEnv: a.minimalEnv || b.minimalEnv,
|
|
129
|
+
exposeAbsolutePaths: a.exposeAbsolutePaths && b.exposeAbsolutePaths,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
114
133
|
/** @param {PolicyCapabilities} left @param {PolicyCapabilities} right */
|
|
115
134
|
export function policyCapabilitiesEqual(left, right) {
|
|
116
135
|
return left.allowWrite === right.allowWrite
|
|
@@ -6,6 +6,7 @@ import { executionEnv, workspaceShellCommand } from "./shell.mjs";
|
|
|
6
6
|
import { MAX_COMMAND_BYTES, validateArgv } from "./process-contract.mjs";
|
|
7
7
|
import { terminateProcessTreeWithEscalation } from "./process-tree.mjs";
|
|
8
8
|
import { BridgeError } from "./errors.mjs";
|
|
9
|
+
import { delegatedProcessCommand } from "./delegated-process-sandbox.mjs";
|
|
9
10
|
import { clampInteger } from "./numbers.mjs";
|
|
10
11
|
import { ProcessOutputStream } from "./process-output-stream.mjs";
|
|
11
12
|
import { processFailureMessage, publicProcessToolResult } from "./process-result-projection.mjs";
|
|
@@ -19,8 +20,7 @@ import {
|
|
|
19
20
|
PUBLIC_PROCESS_INLINE_OUTPUT_BYTES,
|
|
20
21
|
} from "./execution-limits.mjs";
|
|
21
22
|
|
|
22
|
-
const PROCESS_OUTPUT_CAPTURE = Symbol("process-output-capture");
|
|
23
|
-
const CONTINUATION_READ_BYTES = 64 * 1024;
|
|
23
|
+
const PROCESS_OUTPUT_CAPTURE = Symbol("process-output-capture"); const CONTINUATION_READ_BYTES = 64 * 1024;
|
|
24
24
|
|
|
25
25
|
function spawnDirectProcess(command, args, options) {
|
|
26
26
|
// Keep the production child_process API call structurally separate from the
|
|
@@ -35,10 +35,11 @@ function spawnDirectProcess(command, args, options) {
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
export class ProcessExecutionService {
|
|
38
|
-
constructor({ workspace, policy, policyGate, runtimeDir, processTracker, resolveExistingPath, resolveLocalCommand, displayPath, throwIfCancelled, retainCompletedOutput = null, spawnProcess = spawnDirectProcess, terminateProcess = terminateProcessTreeWithEscalation }) {
|
|
38
|
+
constructor({ workspace, policy, policyGate, policyForContext = null, runtimeDir, processTracker, resolveExistingPath, resolveLocalCommand, displayPath, throwIfCancelled, retainCompletedOutput = null, spawnProcess = spawnDirectProcess, terminateProcess = terminateProcessTreeWithEscalation }) {
|
|
39
39
|
this.workspace = workspace;
|
|
40
40
|
this.policy = policy;
|
|
41
41
|
this.policyGate = policyGate;
|
|
42
|
+
this.policyForContext = typeof policyForContext === "function" ? policyForContext : () => this.policy;
|
|
42
43
|
this.runtimeDir = runtimeDir;
|
|
43
44
|
this.processTracker = processTracker;
|
|
44
45
|
this.resolveExistingPath = resolveExistingPath;
|
|
@@ -53,7 +54,7 @@ export class ProcessExecutionService {
|
|
|
53
54
|
async runDirect(args, context = {}) {
|
|
54
55
|
this.policyGate.assert("run_process");
|
|
55
56
|
const argv = validateArgv(args.argv);
|
|
56
|
-
const cwd = await this.resolveExistingPath(args.cwd || ".");
|
|
57
|
+
const cwd = await this.resolveExistingPath(args.cwd || ".", context);
|
|
57
58
|
if (!(await stat(cwd)).isDirectory()) throw new BridgeError("invalid_request", "cwd is not a directory");
|
|
58
59
|
const result = await this.runPublic(
|
|
59
60
|
argv[0], argv.slice(1),
|
|
@@ -67,14 +68,14 @@ export class ProcessExecutionService {
|
|
|
67
68
|
this.policyGate.assert("run_local_command");
|
|
68
69
|
const command = await this.resolveLocalCommand(args, context);
|
|
69
70
|
const argv = validateArgv(command.argv);
|
|
70
|
-
const cwd = await this.resolveExistingPath(command.cwd);
|
|
71
|
+
const cwd = await this.resolveExistingPath(command.cwd, context);
|
|
71
72
|
if (!(await stat(cwd)).isDirectory()) throw new BridgeError("invalid_request", "registered command cwd is not a directory");
|
|
72
73
|
const requested = args.timeout_seconds === undefined
|
|
73
74
|
? command.timeoutSeconds
|
|
74
75
|
: clampInteger(args.timeout_seconds, command.timeoutSeconds, MIN_PROCESS_TIMEOUT_SECONDS, MAX_PROCESS_TIMEOUT_SECONDS);
|
|
75
76
|
const timeoutSeconds = Math.min(requested, command.timeoutSeconds);
|
|
76
77
|
const result = await this.runPublic(argv[0], argv.slice(1), timeoutSeconds * 1000, context, cwd);
|
|
77
|
-
return publicProcessToolResult({ name: command.name, cwd: this.displayPath(cwd), timeout_seconds: timeoutSeconds, ...result });
|
|
78
|
+
return publicProcessToolResult({ name: command.name, cwd: this.displayPath(cwd, context), timeout_seconds: timeoutSeconds, ...result });
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
async probeShell(context = {}) {
|
|
@@ -82,6 +83,10 @@ export class ProcessExecutionService {
|
|
|
82
83
|
return this.run(shell.cmd, shell.args, 5000, true, 64 * 1024, context);
|
|
83
84
|
}
|
|
84
85
|
|
|
86
|
+
async runFixedInternal(cmd, args, timeoutMs, allowFailure = false, maxOutputBytes = DEFAULT_PROCESS_OUTPUT_BYTES, context = {}, cwd = this.workspace) {
|
|
87
|
+
const argv = validateArgv([cmd, ...args]);
|
|
88
|
+
return this.run(argv[0], argv.slice(1), timeoutMs, allowFailure, maxOutputBytes, context, cwd, null, { internalFixed: true });
|
|
89
|
+
}
|
|
85
90
|
async runShell(command, timeoutSeconds, context = {}) {
|
|
86
91
|
this.policyGate.assert("exec_command");
|
|
87
92
|
if (!command || typeof command !== "string") throw new BridgeError("invalid_request", "command is required");
|
|
@@ -114,7 +119,7 @@ export class ProcessExecutionService {
|
|
|
114
119
|
command: basename(cmd), cwd,
|
|
115
120
|
stdout: capture.stdout, stderr: capture.stderr,
|
|
116
121
|
exitCode: result.code, startedAt, closedAt: Date.now(),
|
|
117
|
-
});
|
|
122
|
+
}, context);
|
|
118
123
|
if (session) {
|
|
119
124
|
continuation = {
|
|
120
125
|
output_session_id: session.session_id,
|
|
@@ -155,9 +160,16 @@ export class ProcessExecutionService {
|
|
|
155
160
|
const retainedStderr = options.retainOutput ? new ProcessOutputStream(MAX_PROCESS_SESSION_OUTPUT_BYTES) : null;
|
|
156
161
|
let child;
|
|
157
162
|
try {
|
|
158
|
-
|
|
163
|
+
const internalFixed = options.internalFixed === true;
|
|
164
|
+
const launch = internalFixed
|
|
165
|
+
? { command: cmd, args }
|
|
166
|
+
: delegatedProcessCommand({ command: cmd, args, workspace: this.workspace, runtimeDir: this.runtimeDir, context });
|
|
167
|
+
child = this.spawnProcess(launch.command, launch.args, {
|
|
159
168
|
cwd,
|
|
160
|
-
env: executionEnv(this.workspace, {
|
|
169
|
+
env: executionEnv(this.workspace, {
|
|
170
|
+
fullEnv: internalFixed ? false : this.policyForContext(context).minimalEnv === false,
|
|
171
|
+
runtimeDir: this.runtimeDir,
|
|
172
|
+
}),
|
|
161
173
|
detached: process.platform !== "win32",
|
|
162
174
|
windowsHide: true,
|
|
163
175
|
shell: false,
|
|
@@ -2,6 +2,8 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { basename } from "node:path";
|
|
4
4
|
import { executionEnv } from "./shell.mjs";
|
|
5
|
+
import { assertOwnedByContext, principalBinding } from "./authority-context.mjs";
|
|
6
|
+
import { delegatedProcessCommand } from "./delegated-process-sandbox.mjs";
|
|
5
7
|
import { validateArgv } from "./process-contract.mjs";
|
|
6
8
|
import { terminateProcessTree, terminateProcessTreeWithEscalation } from "./process-tree.mjs";
|
|
7
9
|
import { createToolAuthorizer } from "./policy.mjs";
|
|
@@ -13,10 +15,11 @@ import {
|
|
|
13
15
|
} from "./execution-limits.mjs";
|
|
14
16
|
|
|
15
17
|
export class ProcessSessionManager {
|
|
16
|
-
constructor({ workspace, policy, authorizeTool = null, runtimeDir, processTracker, resolveCwd, displayPath, throwIfCancelled }) {
|
|
18
|
+
constructor({ workspace, policy, authorizeTool = null, policyForContext = null, runtimeDir, processTracker, resolveCwd, displayPath, throwIfCancelled }) {
|
|
17
19
|
this.workspace = workspace;
|
|
18
20
|
this.policy = policy;
|
|
19
21
|
this.authorizeTool = createToolAuthorizer(this.policy, authorizeTool);
|
|
22
|
+
this.policyForContext = typeof policyForContext === "function" ? policyForContext : () => this.policy;
|
|
20
23
|
this.runtimeDir = runtimeDir;
|
|
21
24
|
this.processTracker = processTracker;
|
|
22
25
|
this.resolveCwd = resolveCwd;
|
|
@@ -34,6 +37,7 @@ export class ProcessSessionManager {
|
|
|
34
37
|
}
|
|
35
38
|
|
|
36
39
|
clear() {
|
|
40
|
+
for (const session of this.sessions.values()) notifySessionWaiters(session);
|
|
37
41
|
this.sessions.clear();
|
|
38
42
|
}
|
|
39
43
|
|
|
@@ -41,7 +45,7 @@ export class ProcessSessionManager {
|
|
|
41
45
|
for (const session of this.sessions.values()) notifySessionWaiters(session);
|
|
42
46
|
}
|
|
43
47
|
|
|
44
|
-
retainCompletedOutput({ command, cwd, stdout, stderr, exitCode, startedAt, closedAt = Date.now() }) {
|
|
48
|
+
retainCompletedOutput({ command, cwd, stdout, stderr, exitCode, startedAt, closedAt = Date.now() }, context = {}) {
|
|
45
49
|
this.prune();
|
|
46
50
|
this.evictExitedForCapacity();
|
|
47
51
|
if (this.sessions.size >= MAX_PROCESS_SESSIONS) return null;
|
|
@@ -61,23 +65,25 @@ export class ProcessSessionManager {
|
|
|
61
65
|
stdinClosed: true,
|
|
62
66
|
waiters: new Set(),
|
|
63
67
|
terminationTimer: null,
|
|
68
|
+
...principalBinding(context),
|
|
64
69
|
};
|
|
65
70
|
this.sessions.set(session.id, session);
|
|
66
|
-
return this.summary(session);
|
|
71
|
+
return this.summary(session, context);
|
|
67
72
|
}
|
|
68
73
|
|
|
69
74
|
async start(args, context = {}) {
|
|
70
75
|
this.authorizeTool("start_process");
|
|
71
76
|
const argv = validateArgv(args.argv);
|
|
72
|
-
const cwd = await this.resolveCwd(args.cwd || ".");
|
|
77
|
+
const cwd = await this.resolveCwd(args.cwd || ".", context);
|
|
73
78
|
this.prune();
|
|
74
79
|
this.evictExitedForCapacity();
|
|
75
80
|
if (this.sessions.size >= MAX_PROCESS_SESSIONS) throw new Error(`process session limit reached (${MAX_PROCESS_SESSIONS})`);
|
|
76
81
|
this.throwIfCancelled(context);
|
|
77
82
|
|
|
78
|
-
const
|
|
83
|
+
const launch = delegatedProcessCommand({ command: argv[0], args: argv.slice(1), workspace: this.workspace, runtimeDir: this.runtimeDir, context });
|
|
84
|
+
const child = spawn(launch.command, launch.args, {
|
|
79
85
|
cwd,
|
|
80
|
-
env: executionEnv(this.workspace, { fullEnv: this.
|
|
86
|
+
env: executionEnv(this.workspace, { fullEnv: this.policyForContext(context).minimalEnv === false, runtimeDir: this.runtimeDir }),
|
|
81
87
|
detached: process.platform !== "win32",
|
|
82
88
|
windowsHide: true,
|
|
83
89
|
});
|
|
@@ -96,6 +102,7 @@ export class ProcessSessionManager {
|
|
|
96
102
|
stdinClosed: false,
|
|
97
103
|
waiters: new Set(),
|
|
98
104
|
terminationTimer: null,
|
|
105
|
+
...principalBinding(context),
|
|
99
106
|
};
|
|
100
107
|
this.sessions.set(session.id, session);
|
|
101
108
|
this.trackChild(child, context.callId);
|
|
@@ -133,12 +140,12 @@ export class ProcessSessionManager {
|
|
|
133
140
|
throw error;
|
|
134
141
|
}
|
|
135
142
|
this.throwIfCancelled(context);
|
|
136
|
-
return this.summary(session);
|
|
143
|
+
return this.summary(session, context);
|
|
137
144
|
}
|
|
138
145
|
|
|
139
146
|
async read(args, context = {}) {
|
|
140
147
|
this.authorizeTool("read_process");
|
|
141
|
-
const session = this.get(args.session_id);
|
|
148
|
+
const session = this.get(args.session_id, context);
|
|
142
149
|
const stdoutOffset = clampInteger(args.stdout_offset, 0, 0, Number.MAX_SAFE_INTEGER);
|
|
143
150
|
const stderrOffset = clampInteger(args.stderr_offset, 0, 0, Number.MAX_SAFE_INTEGER);
|
|
144
151
|
const maxBytes = clampInteger(args.max_bytes, 64 * 1024, 1, 256 * 1024);
|
|
@@ -158,7 +165,7 @@ export class ProcessSessionManager {
|
|
|
158
165
|
this.throwIfCancelled(context);
|
|
159
166
|
session.lastActivity = Date.now();
|
|
160
167
|
return {
|
|
161
|
-
...this.summary(session),
|
|
168
|
+
...this.summary(session, context),
|
|
162
169
|
stdout: session.stdout.read(stdoutOffset, maxBytes),
|
|
163
170
|
stderr: session.stderr.read(stderrOffset, maxBytes),
|
|
164
171
|
};
|
|
@@ -166,7 +173,7 @@ export class ProcessSessionManager {
|
|
|
166
173
|
|
|
167
174
|
async write(args, context = {}) {
|
|
168
175
|
this.authorizeTool("write_process");
|
|
169
|
-
const session = this.get(args.session_id);
|
|
176
|
+
const session = this.get(args.session_id, context);
|
|
170
177
|
if (session.closedAt !== null) throw new Error("process session has already exited");
|
|
171
178
|
const data = String(args.data ?? "");
|
|
172
179
|
if (Buffer.byteLength(data) > MAX_PROCESS_SESSION_STDIN_BYTES) throw new Error(`stdin data exceeds maximum size (${MAX_PROCESS_SESSION_STDIN_BYTES} bytes)`);
|
|
@@ -182,42 +189,45 @@ export class ProcessSessionManager {
|
|
|
182
189
|
session.stdinClosed = true;
|
|
183
190
|
}
|
|
184
191
|
session.lastActivity = Date.now();
|
|
185
|
-
return { ...this.summary(session), bytes_written: Buffer.byteLength(data) };
|
|
192
|
+
return { ...this.summary(session, context), bytes_written: Buffer.byteLength(data) };
|
|
186
193
|
}
|
|
187
194
|
|
|
188
195
|
async kill(args, context = {}) {
|
|
189
196
|
this.authorizeTool("kill_process");
|
|
190
|
-
const session = this.get(args.session_id);
|
|
197
|
+
const session = this.get(args.session_id, context);
|
|
191
198
|
this.throwIfCancelled(context);
|
|
192
199
|
const wasRunning = session.closedAt === null;
|
|
193
200
|
const force = args.force === true;
|
|
194
201
|
if (wasRunning && force) terminateProcessTree(session.child, "SIGKILL");
|
|
195
202
|
else if (wasRunning && !session.terminationTimer) {
|
|
196
|
-
session.terminationTimer = terminateProcessTreeWithEscalation(session.child
|
|
203
|
+
session.terminationTimer = terminateProcessTreeWithEscalation(session.child, {
|
|
204
|
+
onTerminationSettled: () => { session.terminationTimer = null; },
|
|
205
|
+
});
|
|
197
206
|
}
|
|
198
207
|
session.lastActivity = Date.now();
|
|
199
208
|
return {
|
|
200
|
-
...this.summary(session),
|
|
209
|
+
...this.summary(session, context),
|
|
201
210
|
termination_requested: wasRunning,
|
|
202
211
|
force,
|
|
203
212
|
force_after_ms: wasRunning && !force ? 2000 : null,
|
|
204
213
|
};
|
|
205
214
|
}
|
|
206
215
|
|
|
207
|
-
get(sessionId) {
|
|
216
|
+
get(sessionId, context = {}) {
|
|
208
217
|
this.prune();
|
|
209
218
|
const id = String(sessionId || "");
|
|
210
219
|
if (!/^proc_[A-Za-z0-9_-]{20,}$/.test(id)) throw new Error("invalid process session id");
|
|
211
220
|
const session = this.sessions.get(id);
|
|
212
221
|
if (!session) throw new Error("process session not found or expired");
|
|
222
|
+
assertOwnedByContext(session, context, "process session");
|
|
213
223
|
return session;
|
|
214
224
|
}
|
|
215
225
|
|
|
216
|
-
summary(session) {
|
|
226
|
+
summary(session, context = {}) {
|
|
217
227
|
return {
|
|
218
228
|
session_id: session.id,
|
|
219
229
|
command: session.argv0,
|
|
220
|
-
cwd: this.displayPath(session.cwd),
|
|
230
|
+
cwd: this.displayPath(session.cwd, context),
|
|
221
231
|
running: session.closedAt === null,
|
|
222
232
|
exit_code: session.exitCode,
|
|
223
233
|
signal: session.signal,
|
|
@@ -6,8 +6,12 @@ export class ProcessTracker {
|
|
|
6
6
|
this.terminateWithEscalation = typeof options.terminateWithEscalation === "function"
|
|
7
7
|
? options.terminateWithEscalation
|
|
8
8
|
: terminateProcessTreeWithEscalation;
|
|
9
|
+
this.clearScheduledTermination = typeof options.clearScheduledTermination === "function"
|
|
10
|
+
? options.clearScheduledTermination
|
|
11
|
+
: clearTimeout;
|
|
9
12
|
this.active = new Set();
|
|
10
13
|
this.byCall = new Map();
|
|
14
|
+
this.terminationTimers = new Map();
|
|
11
15
|
}
|
|
12
16
|
|
|
13
17
|
track(child, callId = "") {
|
|
@@ -35,21 +39,41 @@ export class ProcessTracker {
|
|
|
35
39
|
|
|
36
40
|
terminateCall(callId, { force = false } = {}) {
|
|
37
41
|
const children = [...(this.byCall.get(String(callId || "")) || [])];
|
|
38
|
-
for (const child of children)
|
|
39
|
-
if (force) this.terminate(child, "SIGKILL");
|
|
40
|
-
else this.terminateWithEscalation(child);
|
|
41
|
-
}
|
|
42
|
+
for (const child of children) this.requestTermination(child, force);
|
|
42
43
|
return children.length;
|
|
43
44
|
}
|
|
44
45
|
|
|
45
46
|
terminateAll(signal = "SIGTERM", escalate = false) {
|
|
46
47
|
for (const child of [...this.active]) {
|
|
47
|
-
if (escalate && signal !== "SIGKILL") this.
|
|
48
|
-
else
|
|
48
|
+
if (escalate && signal !== "SIGKILL") this.requestTermination(child, false);
|
|
49
|
+
else {
|
|
50
|
+
this.clearTermination(child);
|
|
51
|
+
this.terminate(child, signal);
|
|
52
|
+
}
|
|
49
53
|
}
|
|
50
54
|
}
|
|
51
55
|
|
|
52
56
|
snapshot() {
|
|
53
57
|
return { active_processes: this.active.size, calls_with_processes: this.byCall.size };
|
|
54
58
|
}
|
|
59
|
+
|
|
60
|
+
requestTermination(child, force) {
|
|
61
|
+
if (force) {
|
|
62
|
+
this.clearTermination(child);
|
|
63
|
+
this.terminate(child, "SIGKILL");
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (this.terminationTimers.has(child)) return;
|
|
67
|
+
const timer = this.terminateWithEscalation(child, {
|
|
68
|
+
onTerminationSettled: () => { this.terminationTimers.delete(child); },
|
|
69
|
+
});
|
|
70
|
+
if (timer) this.terminationTimers.set(child, timer);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
clearTermination(child) {
|
|
74
|
+
const timer = this.terminationTimers.get(child);
|
|
75
|
+
if (!timer) return;
|
|
76
|
+
this.clearScheduledTermination(timer);
|
|
77
|
+
this.terminationTimers.delete(child);
|
|
78
|
+
}
|
|
55
79
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
|
|
3
|
+
const PROCESS_SNAPSHOT_TIMEOUT_MS = 3000;
|
|
4
|
+
const PROCESS_SNAPSHOT_BYTES = 512 * 1024;
|
|
5
|
+
const START_TIME_TOLERANCE_MS = 1500;
|
|
6
|
+
|
|
7
|
+
export function captureProcessTreeOwnership(child, options = {}) {
|
|
8
|
+
const pid = positivePid(child?.pid);
|
|
9
|
+
const platform = String(options.platform || process.platform);
|
|
10
|
+
if (!pid) return { platform, pid: 0, members: [] };
|
|
11
|
+
if (platform === "win32") return { platform, pid, members: [{ pid, startedAt: null }] };
|
|
12
|
+
return { platform, pid, members: processGroupMembers(pid, options) };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function processTreeOwnershipStillCurrent(snapshot, child, options = {}) {
|
|
16
|
+
if (!snapshot?.pid) return false;
|
|
17
|
+
if (snapshot.platform === "win32") return !childHasExited(child);
|
|
18
|
+
if (!Array.isArray(snapshot.members) || snapshot.members.length === 0) return !childHasExited(child);
|
|
19
|
+
const current = processGroupMembers(snapshot.pid, options);
|
|
20
|
+
return snapshot.members.some((expected) => current.some((observed) => sameProcessIdentity(expected, observed)));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function processGroupMembers(groupId, options = {}) {
|
|
24
|
+
const list = typeof options.listProcessGroups === "function" ? options.listProcessGroups : listProcessGroups;
|
|
25
|
+
return list(options).filter((entry) => entry.pgid === groupId).map(({ pid, startedAt }) => ({ pid, startedAt }));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function listProcessGroups(options = {}) {
|
|
29
|
+
const run = typeof options.spawnSyncProcess === "function" ? options.spawnSyncProcess : spawnSync;
|
|
30
|
+
const result = run("ps", ["-axo", "pid=,pgid=,lstart="], {
|
|
31
|
+
encoding: "utf8",
|
|
32
|
+
timeout: PROCESS_SNAPSHOT_TIMEOUT_MS,
|
|
33
|
+
maxBuffer: PROCESS_SNAPSHOT_BYTES,
|
|
34
|
+
windowsHide: true,
|
|
35
|
+
env: { ...process.env, LC_ALL: "C", LANG: "C" },
|
|
36
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
37
|
+
});
|
|
38
|
+
if (result?.error || result?.status !== 0) return [];
|
|
39
|
+
return String(result.stdout || "").split(/\r?\n/).map(parseProcessRow).filter(Boolean);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function parseProcessRow(line) {
|
|
43
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(String(line || ""));
|
|
44
|
+
if (!match) return null;
|
|
45
|
+
const pid = positivePid(match[1]);
|
|
46
|
+
const pgid = positivePid(match[2]);
|
|
47
|
+
const startedAt = Date.parse(match[3]);
|
|
48
|
+
return pid && pgid && Number.isFinite(startedAt) ? { pid, pgid, startedAt } : null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function sameProcessIdentity(left, right) {
|
|
52
|
+
return left.pid === right.pid && Number.isFinite(left.startedAt) && Number.isFinite(right.startedAt)
|
|
53
|
+
&& Math.abs(left.startedAt - right.startedAt) <= START_TIME_TOLERANCE_MS;
|
|
54
|
+
}
|
|
55
|
+
function positivePid(value) { const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : 0; }
|
|
56
|
+
function childHasExited(child) {
|
|
57
|
+
return child?.exitCode !== null && child?.exitCode !== undefined
|
|
58
|
+
|| child?.signalCode !== null && child?.signalCode !== undefined;
|
|
59
|
+
}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import {
|
|
3
|
+
captureProcessTreeOwnership,
|
|
4
|
+
processTreeOwnershipStillCurrent,
|
|
5
|
+
} from "./process-tree-ownership.mjs";
|
|
6
|
+
export { captureProcessTreeOwnership, processTreeOwnershipStillCurrent } from "./process-tree-ownership.mjs";
|
|
2
7
|
|
|
3
8
|
export const DEFAULT_PROCESS_TERMINATION_GRACE_MS = 2000;
|
|
4
9
|
|
|
@@ -10,13 +15,9 @@ export function terminateProcessTree(child, signal = "SIGTERM", options = {}) {
|
|
|
10
15
|
const spawnProcess = typeof options.spawnProcess === "function" ? options.spawnProcess : spawn;
|
|
11
16
|
const force = signal === "SIGKILL";
|
|
12
17
|
const killer = spawnProcess("taskkill.exe", ["/PID", String(child.pid), "/T", ...(force ? ["/F"] : [])], {
|
|
13
|
-
stdio: "ignore",
|
|
14
|
-
windowsHide: true,
|
|
15
|
-
shell: false,
|
|
18
|
+
stdio: "ignore", windowsHide: true, shell: false,
|
|
16
19
|
});
|
|
17
|
-
const fallback = () => {
|
|
18
|
-
try { child.kill(signal); } catch {}
|
|
19
|
-
};
|
|
20
|
+
const fallback = () => { try { child.kill(signal); } catch {} };
|
|
20
21
|
killer?.once?.("error", fallback);
|
|
21
22
|
killer?.once?.("exit", (code) => { if (code !== 0) fallback(); });
|
|
22
23
|
killer?.unref?.();
|
|
@@ -25,17 +26,11 @@ export function terminateProcessTree(child, signal = "SIGTERM", options = {}) {
|
|
|
25
26
|
// Fall through to ChildProcess.kill for environments without taskkill.
|
|
26
27
|
}
|
|
27
28
|
}
|
|
28
|
-
|
|
29
29
|
const killProcess = typeof options.killProcess === "function" ? options.killProcess : process.kill.bind(process);
|
|
30
|
-
try {
|
|
31
|
-
|
|
32
|
-
return
|
|
33
|
-
|
|
34
|
-
try {
|
|
35
|
-
return child.kill(signal) !== false;
|
|
36
|
-
} catch {
|
|
37
|
-
return false;
|
|
38
|
-
}
|
|
30
|
+
try { killProcess(-child.pid, signal); return true; }
|
|
31
|
+
catch {
|
|
32
|
+
try { return child.kill(signal) !== false; }
|
|
33
|
+
catch { return false; }
|
|
39
34
|
}
|
|
40
35
|
}
|
|
41
36
|
|
|
@@ -45,9 +40,18 @@ export function terminateProcessTreeWithEscalation(child, options = {}) {
|
|
|
45
40
|
: DEFAULT_PROCESS_TERMINATION_GRACE_MS;
|
|
46
41
|
const schedule = typeof options.setTimeout === "function" ? options.setTimeout : setTimeout;
|
|
47
42
|
const terminate = typeof options.terminate === "function" ? options.terminate : terminateProcessTree;
|
|
43
|
+
const ownership = typeof options.captureOwnership === "function"
|
|
44
|
+
? options.captureOwnership(child)
|
|
45
|
+
: captureProcessTreeOwnership(child, options);
|
|
48
46
|
terminate(child, "SIGTERM", options);
|
|
49
47
|
return schedule(() => {
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
try {
|
|
49
|
+
const owned = typeof options.isTerminationTargetOwned === "function"
|
|
50
|
+
? options.isTerminationTargetOwned(ownership, child)
|
|
51
|
+
: processTreeOwnershipStillCurrent(ownership, child, options);
|
|
52
|
+
if (!owned) return;
|
|
53
|
+
terminate(child, "SIGKILL", options);
|
|
54
|
+
options.onEscalated?.();
|
|
55
|
+
} finally { options.onTerminationSettled?.(); }
|
|
52
56
|
}, graceMs);
|
|
53
57
|
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import WebSocket from "ws";
|
|
2
2
|
import { classifyOperationalError } from "./log.mjs";
|
|
3
3
|
import { proxyAgentForWebSocket } from "./network-proxy.mjs";
|
|
4
|
+
import {
|
|
5
|
+
APPLICATION_PROXY_ROUTE_SCOPE, relayOutageFields, relayRecoveryFields, relayStatusSnapshot,
|
|
6
|
+
} from "./relay-diagnostics.mjs";
|
|
4
7
|
|
|
5
8
|
const DEFAULT_HEARTBEAT_INTERVAL_MS = 25_000;
|
|
6
9
|
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75_000;
|
|
@@ -39,6 +42,7 @@ export class RelayConnection {
|
|
|
39
42
|
this.reconnectDelay = typeof options.reconnectDelay === "function" ? options.reconnectDelay : reconnectDelay;
|
|
40
43
|
this.proxyAgentForUrl = typeof options.proxyAgentForUrl === "function" ? options.proxyAgentForUrl : proxyAgentForWebSocket;
|
|
41
44
|
this.networkRoute = "unresolved";
|
|
45
|
+
this.networkRouteScope = APPLICATION_PROXY_ROUTE_SCOPE;
|
|
42
46
|
this.maxPayload = boundedPositiveInteger(options.maxPayload, 8 * 1024 * 1024);
|
|
43
47
|
this.heartbeatIntervalMs = boundedPositiveInteger(options.heartbeatIntervalMs, DEFAULT_HEARTBEAT_INTERVAL_MS);
|
|
44
48
|
this.heartbeatTimeoutMs = boundedPositiveInteger(options.heartbeatTimeoutMs, DEFAULT_HEARTBEAT_TIMEOUT_MS);
|
|
@@ -72,8 +76,15 @@ export class RelayConnection {
|
|
|
72
76
|
this.outageNoticeEmitted = false;
|
|
73
77
|
this.outageWarningCount = 0;
|
|
74
78
|
this.lastOutageWarnAt = 0;
|
|
79
|
+
this.outageCount = 0;
|
|
75
80
|
this.lastCloseCategory = "connection_interrupted";
|
|
81
|
+
this.lastCloseCode = 0;
|
|
76
82
|
this.lastTransportErrorClass = "";
|
|
83
|
+
this.lastDisconnectedAt = 0;
|
|
84
|
+
this.lastReadyAt = 0;
|
|
85
|
+
this.lastReadyDurationMs = 0;
|
|
86
|
+
this.lastReconnectDelayMs = 0;
|
|
87
|
+
this.nextReconnectAt = 0;
|
|
77
88
|
this.pendingCloseCategory = "";
|
|
78
89
|
this.connectedOnce = null;
|
|
79
90
|
this.connectedOnceResolve = null;
|
|
@@ -83,16 +94,7 @@ export class RelayConnection {
|
|
|
83
94
|
}
|
|
84
95
|
|
|
85
96
|
status() {
|
|
86
|
-
return
|
|
87
|
-
authenticated: this.authenticated,
|
|
88
|
-
ready: this.ready,
|
|
89
|
-
readiness_probe_delivered: this.readinessProbeDelivered,
|
|
90
|
-
closed: this.closed,
|
|
91
|
-
network_route: this.networkRoute,
|
|
92
|
-
reconnect_attempt: this.reconnectAttempt,
|
|
93
|
-
outage_active: this.outageStartedAt > 0,
|
|
94
|
-
session_generation: this.sessionGeneration,
|
|
95
|
-
};
|
|
97
|
+
return relayStatusSnapshot(this, this.now());
|
|
96
98
|
}
|
|
97
99
|
|
|
98
100
|
currentSessionId() {
|
|
@@ -218,17 +220,18 @@ export class RelayConnection {
|
|
|
218
220
|
this.ready = true;
|
|
219
221
|
this.clearTimer("readinessTimer", "clearTimeout");
|
|
220
222
|
this.reconnectAttempt = 0;
|
|
223
|
+
this.lastReadyAt = this.now();
|
|
224
|
+
this.nextReconnectAt = 0;
|
|
225
|
+
this.lastReconnectDelayMs = 0;
|
|
221
226
|
|
|
222
227
|
if (!this.hasConnected) {
|
|
223
228
|
this.logger.info?.("remote relay connected and end-to-end result delivery verified");
|
|
224
229
|
} else if (this.outageStartedAt > 0) {
|
|
225
230
|
const outageMs = Math.max(0, this.now() - this.outageStartedAt);
|
|
226
231
|
if (this.outageNoticeEmitted) {
|
|
227
|
-
|
|
228
|
-
this.logger.
|
|
229
|
-
|
|
230
|
-
attempts: this.outageAttempts,
|
|
231
|
-
});
|
|
232
|
+
const recoveryFields = relayRecoveryFields(this, outageMs);
|
|
233
|
+
this.logger.info?.(`remote relay connection restored after ${formatDuration(outageMs)} (${formatAttempts(this.outageAttempts)})`, recoveryFields);
|
|
234
|
+
this.logger.debug?.("remote relay outage recovery details", recoveryFields);
|
|
232
235
|
} else {
|
|
233
236
|
this.logger.debug?.("remote relay connection recovered after a brief interruption", {
|
|
234
237
|
outage_ms: outageMs,
|
|
@@ -281,7 +284,7 @@ export class RelayConnection {
|
|
|
281
284
|
const proxy = this.proxyAgentForUrl(wsUrl);
|
|
282
285
|
const headers = this.connectionHeaders();
|
|
283
286
|
if (!headers || typeof headers !== "object" || Array.isArray(headers)) throw new Error("relay connection headers are invalid");
|
|
284
|
-
this.networkRoute = proxy?.agent ? "proxy" : "
|
|
287
|
+
this.networkRoute = proxy?.agent ? "application-http-proxy" : "system-network-stack";
|
|
285
288
|
socket = new this.WebSocketClass(wsUrl, {
|
|
286
289
|
headers,
|
|
287
290
|
maxPayload: this.maxPayload,
|
|
@@ -290,11 +293,13 @@ export class RelayConnection {
|
|
|
290
293
|
this.logger.debug?.("remote relay network route selected", { route: this.networkRoute });
|
|
291
294
|
} catch (error) {
|
|
292
295
|
if (error?.code === "relay_proxy_configuration") {
|
|
293
|
-
this.networkRoute = "invalid-proxy-configuration";
|
|
296
|
+
this.networkRoute = "invalid-application-proxy-configuration";
|
|
294
297
|
this.failPermanently("relay_proxy_configuration");
|
|
295
298
|
return;
|
|
296
299
|
}
|
|
297
300
|
this.lastTransportErrorClass = classifyRelayTransportError(error);
|
|
301
|
+
this.lastCloseCode = 0;
|
|
302
|
+
this.lastDisconnectedAt = this.now();
|
|
298
303
|
this.logger.debug?.("remote relay connection could not be created", { error_class: this.lastTransportErrorClass });
|
|
299
304
|
this.scheduleReconnect("connection_interrupted");
|
|
300
305
|
return;
|
|
@@ -362,9 +367,14 @@ export class RelayConnection {
|
|
|
362
367
|
const reasonText = sanitizeCloseReason(reason);
|
|
363
368
|
const category = this.pendingCloseCategory || relayCloseCategory(code, reasonText);
|
|
364
369
|
this.pendingCloseCategory = "";
|
|
365
|
-
|
|
370
|
+
if (category !== "relay_transport_error") this.lastTransportErrorClass = "";
|
|
371
|
+
const disconnectedAt = this.now();
|
|
372
|
+
const connectedForMs = wasAuthenticated && this.connectedAt > 0 ? Math.max(0, disconnectedAt - this.connectedAt) : 0;
|
|
373
|
+
this.lastDisconnectedAt = disconnectedAt;
|
|
374
|
+
this.lastReadyDurationMs = wasReady && this.lastReadyAt > 0 ? Math.max(0, disconnectedAt - this.lastReadyAt) : 0;
|
|
375
|
+
this.lastCloseCode = Number(code) || 0;
|
|
366
376
|
this.logger.debug?.("remote relay transport closed", {
|
|
367
|
-
close_code:
|
|
377
|
+
close_code: this.lastCloseCode,
|
|
368
378
|
close_reason: reasonText || "<none>",
|
|
369
379
|
category,
|
|
370
380
|
ready: wasReady,
|
|
@@ -457,10 +467,20 @@ export class RelayConnection {
|
|
|
457
467
|
if (this.closed || this.reconnectTimer) return;
|
|
458
468
|
this.recordOutage(category);
|
|
459
469
|
const delay = this.reconnectDelay(this.reconnectAttempt++);
|
|
470
|
+
this.lastReconnectDelayMs = delay;
|
|
471
|
+
this.nextReconnectAt = this.now() + delay;
|
|
460
472
|
this.scheduleOutageWarning();
|
|
461
|
-
this.logger.debug?.("scheduling daemon reconnect", {
|
|
473
|
+
this.logger.debug?.("scheduling daemon reconnect", {
|
|
474
|
+
delay_ms: delay,
|
|
475
|
+
next_reconnect_at: new Date(this.nextReconnectAt).toISOString(),
|
|
476
|
+
attempt: this.outageAttempts,
|
|
477
|
+
close_category: this.lastCloseCategory,
|
|
478
|
+
network_route: this.networkRoute,
|
|
479
|
+
network_route_scope: this.networkRouteScope,
|
|
480
|
+
});
|
|
462
481
|
this.reconnectTimer = this.scheduler.setTimeout(() => {
|
|
463
482
|
this.reconnectTimer = null;
|
|
483
|
+
this.nextReconnectAt = 0;
|
|
464
484
|
this.connect();
|
|
465
485
|
}, delay);
|
|
466
486
|
this.reconnectTimer?.unref?.();
|
|
@@ -501,6 +521,7 @@ export class RelayConnection {
|
|
|
501
521
|
const now = this.now();
|
|
502
522
|
if (this.outageStartedAt === 0) {
|
|
503
523
|
this.outageStartedAt = now;
|
|
524
|
+
this.outageCount += 1;
|
|
504
525
|
this.outageAttempts = 0;
|
|
505
526
|
this.outageNoticeEmitted = false;
|
|
506
527
|
this.lastOutageWarnAt = 0;
|
|
@@ -538,13 +559,9 @@ export class RelayConnection {
|
|
|
538
559
|
const action = outageMs >= 5 * 60_000
|
|
539
560
|
? " If this persists, check internet access and the deployed Worker."
|
|
540
561
|
: "";
|
|
541
|
-
|
|
542
|
-
this.logger.
|
|
543
|
-
|
|
544
|
-
attempts: this.outageAttempts,
|
|
545
|
-
cause,
|
|
546
|
-
...(this.lastTransportErrorClass ? { error_class: this.lastTransportErrorClass } : {}),
|
|
547
|
-
});
|
|
562
|
+
const outageFields = relayOutageFields(this, this.now(), cause);
|
|
563
|
+
this.logger.warn?.(`remote relay unavailable for ${formatDuration(outageMs)}; reconnecting automatically (${formatAttempts(this.outageAttempts)}; ${cause}).${action}`, outageFields);
|
|
564
|
+
this.logger.debug?.("remote relay outage details", outageFields);
|
|
548
565
|
}
|
|
549
566
|
|
|
550
567
|
resetOutage() {
|
|
@@ -554,8 +571,7 @@ export class RelayConnection {
|
|
|
554
571
|
this.outageNoticeEmitted = false;
|
|
555
572
|
this.outageWarningCount = 0;
|
|
556
573
|
this.lastOutageWarnAt = 0;
|
|
557
|
-
this.
|
|
558
|
-
this.lastTransportErrorClass = "";
|
|
574
|
+
this.nextReconnectAt = 0;
|
|
559
575
|
this.pendingCloseCategory = "";
|
|
560
576
|
}
|
|
561
577
|
|
|
@@ -666,8 +682,8 @@ export function isSupersededClose(code, reason) {
|
|
|
666
682
|
|
|
667
683
|
export function reconnectDelay(attempt, random = Math.random) {
|
|
668
684
|
const safeAttempt = Math.max(0, Number.isFinite(Number(attempt)) ? Number(attempt) : 0);
|
|
669
|
-
const base = Math.min(
|
|
670
|
-
return base + Math.floor(random() *
|
|
685
|
+
const base = Math.min(1000 * (2 ** Math.min(safeAttempt, 4)), 15_000);
|
|
686
|
+
return base + Math.floor(random() * 500);
|
|
671
687
|
}
|
|
672
688
|
|
|
673
689
|
function normalizeWorkerUrl(value) {
|
|
@@ -744,7 +760,3 @@ function formatDuration(milliseconds) {
|
|
|
744
760
|
}
|
|
745
761
|
return parts.join(" ") || "1 second";
|
|
746
762
|
}
|
|
747
|
-
|
|
748
|
-
function roundSeconds(milliseconds) {
|
|
749
|
-
return Math.max(1, Math.round(milliseconds / 1000));
|
|
750
|
-
}
|