@gajae-code/coding-agent 0.18.11 → 0.18.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +293 -0
- package/dist/types/async/job-manager.d.ts +7 -3
- package/dist/types/async/result-formatting.d.ts +2 -0
- package/dist/types/autoresearch/git.d.ts +1 -1
- package/dist/types/capability/fs.d.ts +2 -2
- package/dist/types/cli/accounts-cli.d.ts +2 -2
- package/dist/types/cli/doctor/install-repairs.d.ts +4 -1
- package/dist/types/cli/install-activation.d.ts +10 -4
- package/dist/types/cli/public-command-errors.d.ts +1 -0
- package/dist/types/cli/update-cli.d.ts +14 -1
- package/dist/types/cli-ordinary.d.ts +2 -2
- package/dist/types/commands/accounts.d.ts +2 -1
- package/dist/types/commands/migrate.d.ts +1 -1
- package/dist/types/commands/sdk.d.ts +5 -0
- package/dist/types/commit/changelog/index.d.ts +1 -0
- package/dist/types/config/autorouting-tier-map.d.ts +30 -0
- package/dist/types/config/config-hot-reload-trust.d.ts +12 -0
- package/dist/types/config/config-hot-reload.d.ts +39 -0
- package/dist/types/config/file-lock.d.ts +8 -1
- package/dist/types/config/model-discovery-manager.d.ts +2 -0
- package/dist/types/config/model-profile-activation.d.ts +62 -6
- package/dist/types/config/model-registry.d.ts +56 -5
- package/dist/types/config/model-resolver.d.ts +2 -0
- package/dist/types/config/models-config-schema.d.ts +10 -0
- package/dist/types/config/settings-schema.d.ts +46 -3
- package/dist/types/config/settings.d.ts +33 -1
- package/dist/types/dap/config.d.ts +8 -4
- package/dist/types/debug/system-info.d.ts +6 -1
- package/dist/types/discovery/mcp-compat.d.ts +3 -1
- package/dist/types/edit/modes/patch.d.ts +5 -0
- package/dist/types/edit/modes/replace.d.ts +3 -0
- package/dist/types/exec/bash-executor.d.ts +12 -0
- package/dist/types/gjc-runtime/managed-owner-admission.d.ts +3 -0
- package/dist/types/gjc-runtime/session-state-lock.d.ts +2 -0
- package/dist/types/gjc-runtime/state-runtime.d.ts +0 -2
- package/dist/types/migrate/adapters/codex.d.ts +2 -1
- package/dist/types/migrate/adapters/index.d.ts +10 -1
- package/dist/types/migrate/executor.d.ts +3 -1
- package/dist/types/migrate/types.d.ts +15 -3
- package/dist/types/modes/components/settings-selector.d.ts +6 -0
- package/dist/types/modes/controllers/selector-controller.d.ts +2 -0
- package/dist/types/modes/theme/theme.d.ts +1 -0
- package/dist/types/runtime-mcp/content-limits.d.ts +4 -0
- package/dist/types/runtime-mcp/manager.d.ts +7 -0
- package/dist/types/runtime-mcp/plugin-mcp-name-filter.d.ts +22 -0
- package/dist/types/runtime-mcp/tool-bridge.d.ts +4 -0
- package/dist/types/sdk/broker/broker-exit.d.ts +2 -2
- package/dist/types/sdk/broker/broker.d.ts +7 -1
- package/dist/types/sdk/broker/daemon-entry.d.ts +11 -1
- package/dist/types/sdk/broker/doctor-restart.d.ts +1 -1
- package/dist/types/sdk/broker/ensure.d.ts +136 -7
- package/dist/types/sdk/broker/hop.d.ts +36 -0
- package/dist/types/sdk/broker/lifecycle-ledger.d.ts +18 -10
- package/dist/types/sdk/broker/lock-host-identity.d.ts +5 -0
- package/dist/types/sdk/broker/session-index.d.ts +4 -1
- package/dist/types/sdk/bus/chat-command-policy.d.ts +2 -1
- package/dist/types/sdk/bus/index.d.ts +23 -0
- package/dist/types/sdk/bus/reconciliation-store.d.ts +10 -0
- package/dist/types/sdk/bus/telegram-daemon-contract.d.ts +3 -1
- package/dist/types/sdk/cli/session-cli.d.ts +0 -1
- package/dist/types/sdk/host/session-runtime.d.ts +32 -0
- package/dist/types/sdk/prompt-deadline-flush.d.ts +42 -4
- package/dist/types/sdk/prompt-deadline-manager.d.ts +2 -0
- package/dist/types/sdk/protocol/adapter-validation.d.ts +10 -4
- package/dist/types/sdk/reconciliation-extensions.d.ts +1 -1
- package/dist/types/sdk/session-list.d.ts +1 -1
- package/dist/types/session/agent-session.d.ts +58 -2
- package/dist/types/session/internal/managed-session-scope.d.ts +2 -2
- package/dist/types/session/internal/managed-session-storage.d.ts +1 -1
- package/dist/types/session/internal/native-publish-outcome.d.ts +1 -1
- package/dist/types/session/messages.d.ts +5 -1
- package/dist/types/session/session-manager.d.ts +13 -3
- package/dist/types/session/startup-auth-config.d.ts +24 -2
- package/dist/types/session/streaming-output.d.ts +11 -0
- package/dist/types/session/terminal-abort.d.ts +3 -1
- package/dist/types/session-import/service.d.ts +1 -1
- package/dist/types/skill-state/workflow-mutation-guard.d.ts +7 -3
- package/dist/types/slash-commands/helpers/provider-auth-status.d.ts +8 -0
- package/dist/types/task/executor.d.ts +19 -2
- package/dist/types/tools/ast-edit-write-path.d.ts +10 -0
- package/dist/types/tools/atomic-file-write.d.ts +9 -7
- package/dist/types/tools/bash-allowed-prefixes.d.ts +11 -0
- package/dist/types/tools/descriptors.d.ts +2 -0
- package/dist/types/tools/fetch.d.ts +1 -1
- package/dist/types/tools/index.d.ts +2 -0
- package/dist/types/tools/python.d.ts +1 -1
- package/dist/types/tools/report-issue.d.ts +41 -0
- package/dist/types/tools/subagent.d.ts +23 -0
- package/dist/types/web/scrapers/types.d.ts +8 -1
- package/dist/types/web/search/providers/insane.d.ts +7 -1
- package/dist/types/web/search/providers/searxng.d.ts +5 -0
- package/package.json +8 -8
- package/scripts/build-binary.ts +2 -1
- package/scripts/capture-subagent-compact-panel-showcase.ts +105 -0
- package/scripts/generate-docs-index.ts +8 -1
- package/scripts/generate-sdk-operation-inventory.ts +22 -4
- package/src/async/job-manager.ts +356 -56
- package/src/async/result-formatting.ts +40 -0
- package/src/autoresearch/git.ts +9 -2
- package/src/autoresearch/prompts/prompt-setup.md +1 -1
- package/src/capability/fs.ts +27 -19
- package/src/capability/index.ts +2 -2
- package/src/cli/accounts-cli.ts +176 -8
- package/src/cli/auth-broker-cli.ts +21 -11
- package/src/cli/doctor/install-repairs.ts +35 -5
- package/src/cli/fast-help.ts +1 -1
- package/src/cli/install-activation.ts +47 -4
- package/src/cli/migrate-cli.ts +7 -5
- package/src/cli/public-command-errors.ts +4 -0
- package/src/cli/root-flags.ts +2 -2
- package/src/cli/stats-cli.ts +9 -5
- package/src/cli/update-cli.ts +224 -7
- package/src/cli-ordinary.ts +7 -2
- package/src/commands/accounts.ts +10 -5
- package/src/commands/migrate.ts +3 -3
- package/src/commands/sdk.ts +199 -42
- package/src/commands/ultragoal.ts +16 -8
- package/src/commit/changelog/index.ts +39 -1
- package/src/commit/message.ts +10 -3
- package/src/config/autorouting-tier-map.ts +30 -0
- package/src/config/config-hot-reload-trust.ts +148 -0
- package/src/config/config-hot-reload.ts +734 -0
- package/src/config/file-lock.ts +216 -56
- package/src/config/model-bindings-applier.ts +85 -47
- package/src/config/model-discovery-manager.ts +10 -0
- package/src/config/model-profile-activation.ts +716 -113
- package/src/config/model-registry.ts +1334 -158
- package/src/config/model-resolver.ts +3 -1
- package/src/config/models-config-schema.ts +6 -0
- package/src/config/settings-schema.ts +60 -3
- package/src/config/settings.ts +458 -24
- package/src/coordinator-mcp/policy.ts +26 -1
- package/src/crash/sanitize.ts +4 -0
- package/src/customization/import.ts +3 -1
- package/src/dap/config.ts +31 -11
- package/src/dap/defaults.json +1 -1
- package/src/debug/crash-diagnostics.ts +2 -1
- package/src/debug/report-bundle.ts +19 -0
- package/src/debug/system-info.ts +8 -3
- package/src/defaults/gjc/extensions/grok-cli-vendor/src/shared/base-url.ts +21 -7
- package/src/discovery/agents-md.ts +1 -1
- package/src/discovery/helpers.ts +1 -2
- package/src/discovery/mcp-compat.ts +120 -17
- package/src/edit/apply-patch/index.ts +12 -1
- package/src/edit/index.ts +13 -17
- package/src/edit/modes/patch.ts +187 -8
- package/src/edit/modes/replace.ts +136 -0
- package/src/eval/js/context-manager.ts +29 -3
- package/src/eval/js/worker-entry.ts +51 -18
- package/src/eval/py/executor.ts +14 -4
- package/src/eval/py/runner.py +144 -5
- package/src/exec/bash-executor.ts +81 -4
- package/src/exec/bash-shell-guardian.ts +4 -1
- package/src/exec/bash-shell-supervisor.ts +8 -4
- package/src/exec/isolated-shell.ts +5 -1
- package/src/export/html/index.ts +46 -6
- package/src/extensibility/custom-tools/wrapper.ts +3 -1
- package/src/extensibility/extensions/loader.ts +8 -1
- package/src/extensibility/plugins/marketplace/fetcher.ts +2 -2
- package/src/gjc-runtime/gc-runtime.ts +33 -6
- package/src/gjc-runtime/launch-tmux.ts +7 -2
- package/src/gjc-runtime/launch-worktree.test.ts +148 -0
- package/src/gjc-runtime/launch-worktree.ts +101 -14
- package/src/gjc-runtime/managed-owner-admission.ts +46 -2
- package/src/gjc-runtime/session-resolution.ts +38 -6
- package/src/gjc-runtime/session-state-lock.ts +59 -14
- package/src/gjc-runtime/state-runtime.ts +23 -19
- package/src/hooks/native-skill-hook.ts +14 -11
- package/src/internal-urls/docs-index.generated.ts +14 -13
- package/src/internal-urls/local-protocol.ts +142 -8
- package/src/lsp/index.ts +29 -0
- package/src/lsp/workspace-path.ts +23 -0
- package/src/migrate/action-planner.ts +57 -0
- package/src/migrate/adapters/claude-code.ts +23 -4
- package/src/migrate/adapters/codex.ts +22 -4
- package/src/migrate/adapters/index.ts +22 -4
- package/src/migrate/adapters/opencode.ts +6 -1
- package/src/migrate/executor.ts +84 -7
- package/src/migrate/report.ts +19 -2
- package/src/migrate/types.ts +14 -4
- package/src/modes/DESIGN.md +51 -0
- package/src/modes/acp/permission-mode.ts +18 -1
- package/src/modes/components/assistant-message.ts +55 -1
- package/src/modes/components/settings-selector.ts +103 -37
- package/src/modes/controllers/command-controller.ts +38 -2
- package/src/modes/controllers/plan-mode-controller.ts +17 -7
- package/src/modes/controllers/selector-controller.ts +62 -6
- package/src/modes/interactive-mode.ts +68 -0
- package/src/modes/theme/theme.ts +15 -0
- package/src/prompts/tools/debug.md +1 -1
- package/src/prompts/tools/read.md +1 -1
- package/src/prompts/tools/report-issue.md +13 -0
- package/src/prompts/tools/task.md +1 -1
- package/src/runtime-credential-selector.ts +2 -2
- package/src/runtime-mcp/content-limits.ts +7 -0
- package/src/runtime-mcp/manager-pool.test.ts +5 -4
- package/src/runtime-mcp/manager.ts +32 -0
- package/src/runtime-mcp/oauth-flow.ts +32 -6
- package/src/runtime-mcp/plugin-mcp-name-filter.ts +62 -0
- package/src/runtime-mcp/render.ts +16 -1
- package/src/runtime-mcp/tool-bridge.ts +126 -17
- package/src/sdk/broker/broker-exit.ts +8 -5
- package/src/sdk/broker/broker.ts +342 -82
- package/src/sdk/broker/daemon-entry.ts +338 -116
- package/src/sdk/broker/doctor-restart.ts +2 -1
- package/src/sdk/broker/ensure.ts +1143 -265
- package/src/sdk/broker/hop.ts +274 -0
- package/src/sdk/broker/lifecycle-ledger.ts +35 -11
- package/src/sdk/broker/lock-host-identity.ts +29 -0
- package/src/sdk/broker/session-index.ts +62 -12
- package/src/sdk/bus/chat-command-policy.ts +16 -3
- package/src/sdk/bus/index.ts +52 -21
- package/src/sdk/bus/reconciliation-store.ts +201 -8
- package/src/sdk/bus/telegram-daemon-contract.ts +3 -1
- package/src/sdk/cli/session-cli.ts +21 -6
- package/src/sdk/host/session-runtime.test.ts +118 -23
- package/src/sdk/host/session-runtime.ts +352 -47
- package/src/sdk/lifecycle/service.ts +1 -0
- package/src/sdk/prompt-deadline-flush.ts +78 -5
- package/src/sdk/prompt-deadline-manager.ts +68 -14
- package/src/sdk/protocol/adapter-validation.ts +53 -11
- package/src/sdk/protocol/operation-inventory.generated.json +101 -2
- package/src/sdk/reconciliation-extensions.ts +1 -0
- package/src/sdk/session-list.ts +1 -0
- package/src/sdk/session.ts +114 -61
- package/src/session/agent-session.ts +1043 -81
- package/src/session/internal/managed-session-scope.ts +126 -39
- package/src/session/internal/managed-session-storage.ts +14 -11
- package/src/session/internal/native-publish-outcome.ts +5 -1
- package/src/session/messages.ts +47 -6
- package/src/session/session-manager.ts +236 -14
- package/src/session/startup-auth-config.ts +116 -8
- package/src/session/streaming-output.ts +262 -4
- package/src/session/terminal-abort.ts +14 -2
- package/src/session-import/service.ts +1 -0
- package/src/skill-state/workflow-mutation-guard.ts +491 -32
- package/src/slash-commands/builtin-registry.ts +15 -3
- package/src/slash-commands/helpers/credential-switch.ts +10 -5
- package/src/slash-commands/helpers/provider-auth-status.ts +456 -0
- package/src/task/executor.ts +103 -12
- package/src/task/index.ts +389 -175
- package/src/tools/ast-edit-write-path.ts +64 -0
- package/src/tools/ast-edit.ts +18 -1
- package/src/tools/atomic-file-write.ts +46 -15
- package/src/tools/auto-generated-guard.ts +3 -0
- package/src/tools/bash-allowed-prefixes.ts +153 -5
- package/src/tools/bash-interactive.ts +14 -2
- package/src/tools/bash.ts +14 -0
- package/src/tools/debug.ts +7 -6
- package/src/tools/descriptors.test.ts +44 -0
- package/src/tools/descriptors.ts +34 -2
- package/src/tools/fetch.ts +3 -2
- package/src/tools/index.ts +2 -0
- package/src/tools/output-meta.ts +23 -2
- package/src/tools/python.ts +1 -1
- package/src/tools/read.ts +17 -35
- package/src/tools/report-issue.ts +184 -0
- package/src/tools/subagent-render.ts +114 -12
- package/src/tools/subagent.ts +108 -1
- package/src/tools/todo-write.ts +41 -3
- package/src/tools/tool-catalog.generated.ts +31 -3
- package/src/tools/write.ts +35 -3
- package/src/utils/commit-message-generator.ts +9 -2
- package/src/utils/file-mentions.ts +29 -1
- package/src/utils/title-generator.ts +7 -2
- package/src/web/scrapers/index.ts +3 -1
- package/src/web/scrapers/types.ts +8 -0
- package/src/web/scrapers/youtube.ts +3 -2
- package/src/web/search/providers/insane.ts +91 -28
- package/src/web/search/providers/searxng.ts +62 -10
package/src/sdk/broker/ensure.ts
CHANGED
|
@@ -2,10 +2,16 @@ import { type ChildProcess, spawn } from "node:child_process";
|
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import type { FileHandle } from "node:fs/promises";
|
|
4
4
|
import * as fs from "node:fs/promises";
|
|
5
|
+
import * as os from "node:os";
|
|
5
6
|
import path from "node:path";
|
|
7
|
+
import { nativeProcessBindings } from "@gajae-code/utils/native-process";
|
|
6
8
|
import packageJson from "../../../package.json" with { type: "json" };
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
+
import {
|
|
10
|
+
acquireFileLock,
|
|
11
|
+
FILE_LOCK_ORPHAN_TRANSITION_ADOPTION_AGE_MS,
|
|
12
|
+
type FileLockOptions,
|
|
13
|
+
withFileLock,
|
|
14
|
+
} from "../../config/file-lock";
|
|
9
15
|
import { SdkClient, SdkClientError } from "../client/client";
|
|
10
16
|
import { type BrokerStartupExitRecord, clearBrokerStartupExitRecord, readBrokerStartupExitRecord } from "./broker-exit";
|
|
11
17
|
import {
|
|
@@ -14,7 +20,13 @@ import {
|
|
|
14
20
|
readBrokerDiscovery,
|
|
15
21
|
readBrokerRestartIntent,
|
|
16
22
|
} from "./discovery";
|
|
17
|
-
import {
|
|
23
|
+
import { BROKER_HANDOFF_ABORT, BROKER_HANDOFF_ACKNOWLEDGEMENT, BROKER_HANDOFF_COMMIT } from "./hop";
|
|
24
|
+
import { brokerLockHostIdentity } from "./lock-host-identity";
|
|
25
|
+
import {
|
|
26
|
+
isProcessIncarnation,
|
|
27
|
+
observeProcessIncarnation,
|
|
28
|
+
type ProcessIncarnationObservation,
|
|
29
|
+
} from "./process-incarnation";
|
|
18
30
|
import {
|
|
19
31
|
isSdkInternalRuntimeImagePresent,
|
|
20
32
|
resolveSdkInternalSpawnCommand,
|
|
@@ -28,6 +40,567 @@ import {
|
|
|
28
40
|
readBrokerStartupFailureMarker,
|
|
29
41
|
} from "./startup-failure";
|
|
30
42
|
|
|
43
|
+
/** Result of launching a broker through a hop/trampoline or as an exact fixture child. */
|
|
44
|
+
export interface BrokerSpawnResult {
|
|
45
|
+
/** The launcher process, or the exact broker process for fixture leases. */
|
|
46
|
+
process: ChildProcess;
|
|
47
|
+
/** The real broker's pid (from the launcher reply or exact child). */
|
|
48
|
+
realBrokerPid: number | undefined;
|
|
49
|
+
/** Kernel identity returned by the POSIX trampoline when available. */
|
|
50
|
+
realBrokerIncarnation?: string;
|
|
51
|
+
handoff?: BrokerLaunchHandoff;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface BrokerLaunchHandoff {
|
|
55
|
+
release(): Promise<void>;
|
|
56
|
+
terminate(): Promise<void>;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Typed error returned when a Windows broker hop fails.
|
|
61
|
+
* The hop is mandatory on Windows for broker detachment; failure is unrecoverable.
|
|
62
|
+
*/
|
|
63
|
+
export class BrokerHopError extends Error {
|
|
64
|
+
readonly code = "broker_hop_failed";
|
|
65
|
+
readonly hopExitCode: number | null;
|
|
66
|
+
readonly hopStdout: string;
|
|
67
|
+
readonly hopStderr: string;
|
|
68
|
+
readonly reason: string;
|
|
69
|
+
|
|
70
|
+
constructor(fields: { exitCode: number | null; stdout: string; stderr: string; reason: string }) {
|
|
71
|
+
const stderr = fields.stderr.trim();
|
|
72
|
+
super(`Windows broker hop failed: ${fields.reason}${stderr ? `; ${stderr}` : ""}`);
|
|
73
|
+
this.name = "BrokerHopError";
|
|
74
|
+
this.hopExitCode = fields.exitCode;
|
|
75
|
+
this.hopStdout = fields.stdout;
|
|
76
|
+
this.hopStderr = fields.stderr;
|
|
77
|
+
this.reason = fields.reason;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function resolveBrokerLaunchMode(
|
|
82
|
+
platform: NodeJS.Platform,
|
|
83
|
+
initiator: "discovery" | "fixture-lease",
|
|
84
|
+
): "windows-hop" | "posix-trampoline" | "darwin-trampoline" | "direct" {
|
|
85
|
+
if (initiator === "fixture-lease") return "direct";
|
|
86
|
+
if (platform === "win32") return "windows-hop";
|
|
87
|
+
return platform === "darwin" ? "darwin-trampoline" : "posix-trampoline";
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Resolve the hop invocation command for the current runtime (source or compiled).
|
|
92
|
+
* Returns the executable path and arguments to spawn the hop process.
|
|
93
|
+
*/
|
|
94
|
+
function resolveHopInvocation(hopMessage: string): { file: string; args: string[] } {
|
|
95
|
+
const brokerCmd = resolveSdkInternalSpawnCommand("broker-internal");
|
|
96
|
+
if (brokerCmd.kind === "bun-source") {
|
|
97
|
+
// Source mode: args are ["--no-env-file", "--config=...", cli.ts, "sdk", "broker-internal"]
|
|
98
|
+
// Replace the last two with ["internal", "broker-hop", hopMessage]
|
|
99
|
+
const args = [...brokerCmd.args.slice(0, -2), "internal", "broker-hop", hopMessage];
|
|
100
|
+
return { file: brokerCmd.file, args };
|
|
101
|
+
} else {
|
|
102
|
+
// Compiled mode: file is gjc executable
|
|
103
|
+
return { file: brokerCmd.file, args: ["internal", "broker-hop", hopMessage] };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Spawn a broker as an exact child so its process handle can be retained for cleanup. */
|
|
108
|
+
function spawnBrokerDirect(
|
|
109
|
+
brokerFile: string,
|
|
110
|
+
brokerArgs: readonly string[],
|
|
111
|
+
options: {
|
|
112
|
+
stdioFd?: number;
|
|
113
|
+
env: NodeJS.ProcessEnv;
|
|
114
|
+
cwd?: string;
|
|
115
|
+
detached: boolean;
|
|
116
|
+
},
|
|
117
|
+
): BrokerSpawnResult {
|
|
118
|
+
const child = spawn(brokerFile, Array.from(brokerArgs), {
|
|
119
|
+
detached: options.detached,
|
|
120
|
+
stdio: ["ignore", "ignore", options.stdioFd ?? "ignore"],
|
|
121
|
+
env: options.env,
|
|
122
|
+
...(options.cwd ? { cwd: options.cwd } : {}),
|
|
123
|
+
});
|
|
124
|
+
return { process: child, realBrokerPid: child.pid };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** What the parent sends the Windows hop. The broker environment is never serialized here. */
|
|
128
|
+
export interface BrokerHopMessage {
|
|
129
|
+
command: { file: string; args: string[] };
|
|
130
|
+
cwd?: string;
|
|
131
|
+
/** Path (never an inherited fd number) the hop opens for the broker's stderr. */
|
|
132
|
+
stderrLogPath?: string;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface BrokerLaunchResult {
|
|
136
|
+
process: ChildProcess;
|
|
137
|
+
realBrokerPid: number | undefined;
|
|
138
|
+
realBrokerIncarnation?: string;
|
|
139
|
+
error: Error | undefined;
|
|
140
|
+
handoff?: BrokerLaunchHandoff;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const MAX_LAUNCHER_OUTPUT_CHARS = 4_096;
|
|
144
|
+
const DEFAULT_LAUNCHER_TIMEOUT_MS = 30_000;
|
|
145
|
+
const HOP_HANDOFF_CLEANUP_TIMEOUT_MS = 1_000;
|
|
146
|
+
|
|
147
|
+
function appendBoundedOutput(current: string, chunk: Buffer | string): string {
|
|
148
|
+
const next = current + chunk.toString();
|
|
149
|
+
return next.length <= MAX_LAUNCHER_OUTPUT_CHARS ? next : next.slice(-MAX_LAUNCHER_OUTPUT_CHARS);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function endBrokerLauncherInput(child: ChildProcess, chunk?: string): void {
|
|
153
|
+
const input = child.stdin;
|
|
154
|
+
if (!input || input.destroyed || input.writableEnded) return;
|
|
155
|
+
try {
|
|
156
|
+
input.end(chunk);
|
|
157
|
+
} catch {
|
|
158
|
+
// The launcher treats an ended acknowledgement pipe as a failed handoff.
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function writeBrokerLauncherInput(child: ChildProcess, chunk: string): Promise<void> {
|
|
163
|
+
const input = child.stdin;
|
|
164
|
+
if (!input || input.destroyed || input.writableEnded)
|
|
165
|
+
return Promise.reject(new Error("Broker trampoline control pipe is closed."));
|
|
166
|
+
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
|
167
|
+
try {
|
|
168
|
+
input.write(chunk, error => (error ? reject(error) : resolve()));
|
|
169
|
+
} catch (error) {
|
|
170
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
171
|
+
}
|
|
172
|
+
return promise;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function awaitLauncherClose(child: ChildProcess): Promise<{ code: number | null; spawnError?: Error }> {
|
|
176
|
+
const { promise, resolve } = Promise.withResolvers<{ code: number | null; spawnError?: Error }>();
|
|
177
|
+
let spawnError: Error | undefined;
|
|
178
|
+
child.once("error", error => {
|
|
179
|
+
spawnError = error;
|
|
180
|
+
resolve({ code: null, spawnError });
|
|
181
|
+
});
|
|
182
|
+
child.once("close", code => resolve({ code, ...(spawnError ? { spawnError } : {}) }));
|
|
183
|
+
return promise;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function terminateLauncher(child: ChildProcess): Promise<boolean> {
|
|
187
|
+
if (child.exitCode !== null || child.signalCode !== null) return true;
|
|
188
|
+
const exited = Promise.withResolvers<void>();
|
|
189
|
+
child.once("exit", exited.resolve);
|
|
190
|
+
child.once("close", exited.resolve);
|
|
191
|
+
child.on("error", () => {});
|
|
192
|
+
try {
|
|
193
|
+
child.kill("SIGKILL");
|
|
194
|
+
} catch {
|
|
195
|
+
// The exit events below still distinguish a launcher that already exited.
|
|
196
|
+
}
|
|
197
|
+
await Promise.race([exited.promise, Bun.sleep(1_000)]);
|
|
198
|
+
return child.exitCode !== null || child.signalCode !== null;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async function awaitLauncherCloseBeforeDeadline(
|
|
202
|
+
child: ChildProcess,
|
|
203
|
+
timeoutMs: number,
|
|
204
|
+
onTimeout?: () => void,
|
|
205
|
+
): Promise<
|
|
206
|
+
{ kind: "closed"; outcome: { code: number | null; spawnError?: Error } } | { kind: "timeout"; terminated: boolean }
|
|
207
|
+
> {
|
|
208
|
+
const closed = awaitLauncherClose(child);
|
|
209
|
+
const result = await Promise.race([
|
|
210
|
+
closed.then(outcome => ({ kind: "closed" as const, outcome })),
|
|
211
|
+
Bun.sleep(timeoutMs).then(() => ({ kind: "timeout" as const })),
|
|
212
|
+
]);
|
|
213
|
+
if (result.kind === "closed") return result;
|
|
214
|
+
if (onTimeout) {
|
|
215
|
+
onTimeout();
|
|
216
|
+
const cleanupFinished = await Promise.race([
|
|
217
|
+
closed.then(() => true),
|
|
218
|
+
Bun.sleep(HOP_HANDOFF_CLEANUP_TIMEOUT_MS).then(() => false),
|
|
219
|
+
]);
|
|
220
|
+
if (cleanupFinished) return { kind: "timeout", terminated: true };
|
|
221
|
+
}
|
|
222
|
+
return { kind: "timeout", terminated: await terminateLauncher(child) };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Test hook: proves a stuck launcher is terminated at its deadline. */
|
|
226
|
+
export async function awaitBrokerLauncherForTest(
|
|
227
|
+
child: ChildProcess,
|
|
228
|
+
timeoutMs: number,
|
|
229
|
+
onTimeout?: () => void,
|
|
230
|
+
): Promise<{ kind: "closed" } | { kind: "timeout"; terminated: boolean }> {
|
|
231
|
+
const result = await awaitLauncherCloseBeforeDeadline(child, timeoutMs, onTimeout);
|
|
232
|
+
return result.kind === "closed" ? { kind: "closed" } : result;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Windows only: launch the broker through the internal hop and await its reply.
|
|
237
|
+
*
|
|
238
|
+
* `taskkill /T /F` walks ParentProcessId, so a broker spawned directly by the client
|
|
239
|
+
* dies with it even when detached. The hop spawns the broker and exits after
|
|
240
|
+
* the parent acknowledges the complete reply, leaving the broker without a
|
|
241
|
+
* live parent in the client's tree. The broker inherits the environment set on
|
|
242
|
+
* the hop, never passed on the command line; stderr is passed as a path because
|
|
243
|
+
* a parent fd number does not exist inside the hop.
|
|
244
|
+
*/
|
|
245
|
+
export async function launchBrokerViaHop(
|
|
246
|
+
message: BrokerHopMessage,
|
|
247
|
+
options: { env: NodeJS.ProcessEnv; cwd?: string; timeoutMs?: number },
|
|
248
|
+
): Promise<BrokerLaunchResult> {
|
|
249
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_LAUNCHER_TIMEOUT_MS;
|
|
250
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("Windows broker hop startup deadline elapsed.");
|
|
251
|
+
const hopCmd = resolveHopInvocation(JSON.stringify(message));
|
|
252
|
+
const hop = spawn(hopCmd.file, hopCmd.args, {
|
|
253
|
+
detached: false,
|
|
254
|
+
windowsHide: true,
|
|
255
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
256
|
+
env: options.env,
|
|
257
|
+
...(options.cwd ? { cwd: options.cwd } : {}),
|
|
258
|
+
});
|
|
259
|
+
let stdout = "";
|
|
260
|
+
let stderr = "";
|
|
261
|
+
let acknowledgementSent = false;
|
|
262
|
+
hop.stdout?.on("data", chunk => {
|
|
263
|
+
stdout = appendBoundedOutput(stdout, chunk);
|
|
264
|
+
if (acknowledgementSent) return;
|
|
265
|
+
// A successful hop write callback does not prove this parent received the
|
|
266
|
+
// record. Keep the hop's pinned broker until this complete identity parses.
|
|
267
|
+
const newlineIndex = stdout.indexOf("\n");
|
|
268
|
+
if (newlineIndex < 0) return;
|
|
269
|
+
acknowledgementSent = true;
|
|
270
|
+
const replyText = stdout.slice(0, newlineIndex + 1);
|
|
271
|
+
const trailingOutput = stdout.slice(newlineIndex + 1);
|
|
272
|
+
const parsed = parseBrokerHopReply(0, replyText);
|
|
273
|
+
if (parsed.error === undefined && trailingOutput.length === 0)
|
|
274
|
+
endBrokerLauncherInput(hop, BROKER_HANDOFF_ACKNOWLEDGEMENT);
|
|
275
|
+
else endBrokerLauncherInput(hop);
|
|
276
|
+
});
|
|
277
|
+
hop.stderr?.on("data", chunk => {
|
|
278
|
+
stderr = appendBoundedOutput(stderr, chunk);
|
|
279
|
+
});
|
|
280
|
+
hop.stdin?.on("error", () => {});
|
|
281
|
+
const wait = await awaitLauncherCloseBeforeDeadline(hop, timeoutMs, () => {
|
|
282
|
+
hop.stdout?.destroy();
|
|
283
|
+
endBrokerLauncherInput(hop);
|
|
284
|
+
});
|
|
285
|
+
if (wait.kind === "timeout") {
|
|
286
|
+
return await brokerHopTimeoutResult(hop, stdout, stderr, timeoutMs, wait.terminated);
|
|
287
|
+
}
|
|
288
|
+
return { process: hop, ...parseBrokerHopReply(wait.outcome.code, stdout, wait.outcome.spawnError, stderr) };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function brokerHopTimeoutResult(
|
|
292
|
+
hop: ChildProcess,
|
|
293
|
+
stdout: string,
|
|
294
|
+
stderr: string,
|
|
295
|
+
timeoutMs: number,
|
|
296
|
+
launcherTerminated: boolean,
|
|
297
|
+
): Promise<BrokerLaunchResult> {
|
|
298
|
+
// A complete reply can arrive before the hop's close event. Preserve its
|
|
299
|
+
// incarnation and reap that exact broker instead of leaving it detached after
|
|
300
|
+
// reporting a timeout to the caller.
|
|
301
|
+
const handoff = parseBrokerHopReply(0, stdout, undefined, stderr);
|
|
302
|
+
const brokerDetail = await reapTimedOutBrokerHandoff(handoff.realBrokerPid, handoff.realBrokerIncarnation);
|
|
303
|
+
const launcherDetail = launcherTerminated
|
|
304
|
+
? "hop exited after handoff cleanup"
|
|
305
|
+
: "hop did not exit after termination";
|
|
306
|
+
return {
|
|
307
|
+
process: hop,
|
|
308
|
+
realBrokerPid: handoff.realBrokerPid,
|
|
309
|
+
...(handoff.realBrokerIncarnation !== undefined ? { realBrokerIncarnation: handoff.realBrokerIncarnation } : {}),
|
|
310
|
+
error: new BrokerHopError({
|
|
311
|
+
exitCode: hop.exitCode,
|
|
312
|
+
stdout,
|
|
313
|
+
stderr,
|
|
314
|
+
reason: `hop exceeded its ${timeoutMs}ms startup deadline; ${launcherDetail}${brokerDetail}`,
|
|
315
|
+
}),
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
async function reapTimedOutBrokerHandoff(
|
|
320
|
+
realBrokerPid: number | undefined,
|
|
321
|
+
realBrokerIncarnation: string | undefined,
|
|
322
|
+
): Promise<string> {
|
|
323
|
+
if (realBrokerPid === undefined || realBrokerIncarnation === undefined) return "";
|
|
324
|
+
try {
|
|
325
|
+
await reapDetachedBrokerPid(realBrokerPid, realBrokerIncarnation, {
|
|
326
|
+
gracefulMs: HOP_HANDOFF_CLEANUP_TIMEOUT_MS,
|
|
327
|
+
killVerifyMs: HOP_HANDOFF_CLEANUP_TIMEOUT_MS,
|
|
328
|
+
});
|
|
329
|
+
return "; captured broker handoff was reaped";
|
|
330
|
+
} catch (error) {
|
|
331
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
332
|
+
return `; exact broker cleanup failed: ${detail}`;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** Parses the hop's single-line pid/incarnation reply into broker identity or a typed error. */
|
|
337
|
+
export function parseBrokerHopReply(
|
|
338
|
+
code: number | null,
|
|
339
|
+
stdout: string,
|
|
340
|
+
spawnError?: Error,
|
|
341
|
+
stderr = "",
|
|
342
|
+
): { realBrokerPid: number | undefined; realBrokerIncarnation?: string; error: Error | undefined } {
|
|
343
|
+
const fail = (reason: string) => ({
|
|
344
|
+
realBrokerPid: undefined,
|
|
345
|
+
error: new BrokerHopError({ exitCode: code, stdout, stderr, reason }),
|
|
346
|
+
});
|
|
347
|
+
if (spawnError) return fail(`hop could not be spawned: ${spawnError.message}`);
|
|
348
|
+
if (code !== 0) return fail(`hop process exited with non-zero code ${code}`);
|
|
349
|
+
const line = stdout.trim();
|
|
350
|
+
if (!line) return fail("hop exited with code 0 but no response on stdout");
|
|
351
|
+
let reply: unknown;
|
|
352
|
+
try {
|
|
353
|
+
reply = JSON.parse(line);
|
|
354
|
+
} catch (error) {
|
|
355
|
+
return fail(`failed to parse hop JSON response: ${error instanceof Error ? error.message : String(error)}`);
|
|
356
|
+
}
|
|
357
|
+
const pid = (reply as { pid?: unknown } | null)?.pid;
|
|
358
|
+
if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0)
|
|
359
|
+
return fail(`hop response missing or invalid pid: ${String(pid)}`);
|
|
360
|
+
const incarnation = (reply as { incarnation?: unknown } | null)?.incarnation;
|
|
361
|
+
if (!isProcessIncarnation(incarnation)) return fail("hop response missing or invalid process incarnation");
|
|
362
|
+
return { realBrokerPid: pid, realBrokerIncarnation: incarnation, error: undefined };
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** POSIX only: the trampoline holds its exact broker child until the publication decision. */
|
|
366
|
+
export async function launchBrokerViaPosixTrampoline(
|
|
367
|
+
agentDir: string,
|
|
368
|
+
options: {
|
|
369
|
+
env: NodeJS.ProcessEnv;
|
|
370
|
+
cwd?: string;
|
|
371
|
+
stderrFd?: number;
|
|
372
|
+
timeoutMs?: number;
|
|
373
|
+
retainChildUntilPublication?: boolean;
|
|
374
|
+
},
|
|
375
|
+
): Promise<BrokerLaunchResult> {
|
|
376
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_LAUNCHER_TIMEOUT_MS;
|
|
377
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
378
|
+
throw new Error("POSIX broker trampoline startup deadline elapsed.");
|
|
379
|
+
const command = resolveSdkInternalSpawnCommand("broker-trampoline-internal");
|
|
380
|
+
const trampolineEnvironment = { ...options.env };
|
|
381
|
+
if (options.retainChildUntilPublication) trampolineEnvironment.GJC_BROKER_TRAMPOLINE_RETAIN_UNTIL_PUBLICATION = "1";
|
|
382
|
+
else delete trampolineEnvironment.GJC_BROKER_TRAMPOLINE_RETAIN_UNTIL_PUBLICATION;
|
|
383
|
+
const child = spawn(command.file, [...command.args, "--agent-dir", agentDir], {
|
|
384
|
+
detached: true,
|
|
385
|
+
stdio: ["pipe", "pipe", options.stderrFd ?? "ignore"],
|
|
386
|
+
env: trampolineEnvironment,
|
|
387
|
+
...(options.cwd ? { cwd: options.cwd } : {}),
|
|
388
|
+
});
|
|
389
|
+
if (options.retainChildUntilPublication) return await launchBrokerViaRetainedPosixTrampoline(child, timeoutMs);
|
|
390
|
+
let stdout = "";
|
|
391
|
+
let acknowledgementSent = false;
|
|
392
|
+
child.stdout?.on("data", chunk => {
|
|
393
|
+
stdout = appendBoundedOutput(stdout, chunk);
|
|
394
|
+
if (acknowledgementSent) return;
|
|
395
|
+
const newlineIndex = stdout.indexOf("\n");
|
|
396
|
+
if (newlineIndex < 0) return;
|
|
397
|
+
acknowledgementSent = true;
|
|
398
|
+
const replyText = stdout.slice(0, newlineIndex + 1);
|
|
399
|
+
const trailingOutput = stdout.slice(newlineIndex + 1);
|
|
400
|
+
const parsed = parseBrokerTrampolineReply(0, replyText);
|
|
401
|
+
if (parsed.error === undefined && trailingOutput.length === 0)
|
|
402
|
+
endBrokerLauncherInput(child, BROKER_HANDOFF_ACKNOWLEDGEMENT);
|
|
403
|
+
else endBrokerLauncherInput(child);
|
|
404
|
+
});
|
|
405
|
+
child.stdin?.on("error", () => {});
|
|
406
|
+
const wait = await awaitLauncherCloseBeforeDeadline(child, timeoutMs, () => {
|
|
407
|
+
child.stdout?.destroy();
|
|
408
|
+
endBrokerLauncherInput(child);
|
|
409
|
+
});
|
|
410
|
+
if (wait.kind === "timeout") {
|
|
411
|
+
return await brokerTrampolineTimeoutResult(child, stdout, timeoutMs, wait.terminated);
|
|
412
|
+
}
|
|
413
|
+
const parsed = parseBrokerTrampolineReply(wait.outcome.code, stdout, wait.outcome.spawnError);
|
|
414
|
+
return { process: child, ...parsed };
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const RETAINED_TRAMPOLINE_CLEANUP_TIMEOUT_MS = 5_000;
|
|
418
|
+
|
|
419
|
+
function createBrokerLaunchHandoff(
|
|
420
|
+
trampoline: ChildProcess,
|
|
421
|
+
brokerPid: number,
|
|
422
|
+
brokerIncarnation: string,
|
|
423
|
+
): BrokerLaunchHandoff {
|
|
424
|
+
const closed = awaitLauncherClose(trampoline);
|
|
425
|
+
let disposition: "release" | "terminate" | undefined;
|
|
426
|
+
let completion: Promise<void> | undefined;
|
|
427
|
+
const finish = (requested: "release" | "terminate"): Promise<void> => {
|
|
428
|
+
if (completion) {
|
|
429
|
+
if (disposition !== requested)
|
|
430
|
+
return Promise.reject(new Error(`Broker trampoline already received ${disposition}.`));
|
|
431
|
+
return completion;
|
|
432
|
+
}
|
|
433
|
+
disposition = requested;
|
|
434
|
+
completion = (async () => {
|
|
435
|
+
const message = requested === "release" ? BROKER_HANDOFF_COMMIT : BROKER_HANDOFF_ABORT;
|
|
436
|
+
let inputError: unknown;
|
|
437
|
+
if (!trampoline.stdin?.destroyed && !trampoline.stdin?.writableEnded) {
|
|
438
|
+
try {
|
|
439
|
+
await writeBrokerLauncherInput(trampoline, message);
|
|
440
|
+
} catch (error) {
|
|
441
|
+
inputError = error;
|
|
442
|
+
} finally {
|
|
443
|
+
endBrokerLauncherInput(trampoline);
|
|
444
|
+
}
|
|
445
|
+
} else if (requested === "release") {
|
|
446
|
+
inputError = new Error("POSIX broker trampoline control pipe closed before publication commit.");
|
|
447
|
+
}
|
|
448
|
+
const outcome = await Promise.race([
|
|
449
|
+
closed,
|
|
450
|
+
Bun.sleep(RETAINED_TRAMPOLINE_CLEANUP_TIMEOUT_MS).then(() => undefined),
|
|
451
|
+
]);
|
|
452
|
+
if (!outcome) throw new Error("POSIX broker trampoline did not complete its publication decision.");
|
|
453
|
+
if (outcome.spawnError) throw outcome.spawnError;
|
|
454
|
+
if (requested === "release") {
|
|
455
|
+
if (inputError)
|
|
456
|
+
throw new Error("Could not commit the POSIX broker handoff after publication.", { cause: inputError });
|
|
457
|
+
if (outcome.code !== 0)
|
|
458
|
+
throw new Error(`POSIX broker trampoline exited with code ${outcome.code} before release completed.`);
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
const observation = observeProcessIncarnation(brokerPid);
|
|
462
|
+
if (
|
|
463
|
+
observation.status !== "absent" &&
|
|
464
|
+
!(observation.status === "present" && observation.incarnation !== brokerIncarnation)
|
|
465
|
+
)
|
|
466
|
+
throw new Error(`POSIX broker trampoline did not verify cleanup of broker ${brokerPid}.`);
|
|
467
|
+
})();
|
|
468
|
+
return completion;
|
|
469
|
+
};
|
|
470
|
+
return { release: () => finish("release"), terminate: () => finish("terminate") };
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
async function launchBrokerViaRetainedPosixTrampoline(
|
|
474
|
+
trampoline: ChildProcess,
|
|
475
|
+
timeoutMs: number,
|
|
476
|
+
): Promise<BrokerLaunchResult> {
|
|
477
|
+
let stdout = "";
|
|
478
|
+
let parsed: ReturnType<typeof parseBrokerTrampolineReply> | undefined;
|
|
479
|
+
let failure: Error | undefined;
|
|
480
|
+
let settled = false;
|
|
481
|
+
const reply = Promise.withResolvers<void>();
|
|
482
|
+
const finish = (error?: Error): void => {
|
|
483
|
+
if (settled) return;
|
|
484
|
+
settled = true;
|
|
485
|
+
failure = error;
|
|
486
|
+
reply.resolve();
|
|
487
|
+
};
|
|
488
|
+
const closed = awaitLauncherClose(trampoline);
|
|
489
|
+
void closed.then(outcome => {
|
|
490
|
+
if (settled) return;
|
|
491
|
+
const result = parseBrokerTrampolineReply(outcome.code, stdout, outcome.spawnError);
|
|
492
|
+
parsed = result;
|
|
493
|
+
finish(result.error ?? new Error("POSIX broker trampoline exited before publication was confirmed."));
|
|
494
|
+
});
|
|
495
|
+
trampoline.stdout?.on("data", chunk => {
|
|
496
|
+
stdout = appendBoundedOutput(stdout, chunk);
|
|
497
|
+
if (settled) return;
|
|
498
|
+
const newlineIndex = stdout.indexOf("\n");
|
|
499
|
+
if (newlineIndex < 0) return;
|
|
500
|
+
const replyText = stdout.slice(0, newlineIndex + 1);
|
|
501
|
+
const trailingOutput = stdout.slice(newlineIndex + 1);
|
|
502
|
+
const result = parseBrokerTrampolineReply(0, replyText);
|
|
503
|
+
if (result.error || trailingOutput.length > 0) {
|
|
504
|
+
parsed = result;
|
|
505
|
+
endBrokerLauncherInput(trampoline);
|
|
506
|
+
finish(result.error ?? new Error("POSIX broker trampoline wrote unexpected output."));
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
parsed = result;
|
|
510
|
+
void writeBrokerLauncherInput(trampoline, BROKER_HANDOFF_ACKNOWLEDGEMENT).then(
|
|
511
|
+
() => finish(),
|
|
512
|
+
error => {
|
|
513
|
+
endBrokerLauncherInput(trampoline);
|
|
514
|
+
finish(error instanceof Error ? error : new Error(String(error)));
|
|
515
|
+
},
|
|
516
|
+
);
|
|
517
|
+
});
|
|
518
|
+
trampoline.stdin?.on("error", () => {});
|
|
519
|
+
const wait = await Promise.race([
|
|
520
|
+
reply.promise.then(() => ({ kind: "reply" as const })),
|
|
521
|
+
Bun.sleep(timeoutMs).then(() => ({ kind: "timeout" as const })),
|
|
522
|
+
]);
|
|
523
|
+
const identity = {
|
|
524
|
+
realBrokerPid: parsed?.realBrokerPid,
|
|
525
|
+
...(parsed?.realBrokerIncarnation === undefined ? {} : { realBrokerIncarnation: parsed.realBrokerIncarnation }),
|
|
526
|
+
};
|
|
527
|
+
if (wait.kind === "timeout") {
|
|
528
|
+
endBrokerLauncherInput(trampoline);
|
|
529
|
+
return {
|
|
530
|
+
process: trampoline,
|
|
531
|
+
...identity,
|
|
532
|
+
...(identity.realBrokerPid !== undefined && identity.realBrokerIncarnation !== undefined
|
|
533
|
+
? { handoff: createBrokerLaunchHandoff(trampoline, identity.realBrokerPid, identity.realBrokerIncarnation) }
|
|
534
|
+
: {}),
|
|
535
|
+
error: new Error(`POSIX broker trampoline did not report a verified identity within ${timeoutMs}ms.`),
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
if (failure) {
|
|
539
|
+
endBrokerLauncherInput(trampoline);
|
|
540
|
+
return {
|
|
541
|
+
process: trampoline,
|
|
542
|
+
...identity,
|
|
543
|
+
...(identity.realBrokerPid !== undefined && identity.realBrokerIncarnation !== undefined
|
|
544
|
+
? { handoff: createBrokerLaunchHandoff(trampoline, identity.realBrokerPid, identity.realBrokerIncarnation) }
|
|
545
|
+
: {}),
|
|
546
|
+
error: failure,
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
if (identity.realBrokerPid === undefined || identity.realBrokerIncarnation === undefined)
|
|
550
|
+
return {
|
|
551
|
+
process: trampoline,
|
|
552
|
+
...identity,
|
|
553
|
+
error: new Error("POSIX broker trampoline returned no verified process identity."),
|
|
554
|
+
};
|
|
555
|
+
return {
|
|
556
|
+
process: trampoline,
|
|
557
|
+
...identity,
|
|
558
|
+
error: undefined,
|
|
559
|
+
handoff: createBrokerLaunchHandoff(trampoline, identity.realBrokerPid, identity.realBrokerIncarnation),
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async function brokerTrampolineTimeoutResult(
|
|
564
|
+
trampoline: ChildProcess,
|
|
565
|
+
stdout: string,
|
|
566
|
+
timeoutMs: number,
|
|
567
|
+
trampolineTerminated: boolean,
|
|
568
|
+
): Promise<BrokerLaunchResult> {
|
|
569
|
+
const handoff = parseBrokerTrampolineReply(0, stdout);
|
|
570
|
+
const brokerDetail = await reapTimedOutBrokerHandoff(handoff.realBrokerPid, handoff.realBrokerIncarnation);
|
|
571
|
+
const trampolineDetail = trampolineTerminated
|
|
572
|
+
? "trampoline was terminated"
|
|
573
|
+
: "trampoline did not exit after termination";
|
|
574
|
+
const handoffFailure = handoff.realBrokerPid === undefined && handoff.error ? `; ${handoff.error.message}` : "";
|
|
575
|
+
return {
|
|
576
|
+
process: trampoline,
|
|
577
|
+
realBrokerPid: handoff.realBrokerPid,
|
|
578
|
+
...(handoff.realBrokerIncarnation !== undefined ? { realBrokerIncarnation: handoff.realBrokerIncarnation } : {}),
|
|
579
|
+
error: new Error(
|
|
580
|
+
`POSIX broker trampoline exceeded its ${timeoutMs}ms startup deadline; ${trampolineDetail}${brokerDetail}${handoffFailure}`,
|
|
581
|
+
),
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
export function parseBrokerTrampolineReply(
|
|
586
|
+
code: number | null,
|
|
587
|
+
stdout: string,
|
|
588
|
+
spawnError?: Error,
|
|
589
|
+
): { realBrokerPid: number | undefined; realBrokerIncarnation?: string; error: Error | undefined } {
|
|
590
|
+
const fail = (reason: string) => ({
|
|
591
|
+
realBrokerPid: undefined,
|
|
592
|
+
error: new Error(`POSIX broker trampoline failed: ${reason}`),
|
|
593
|
+
});
|
|
594
|
+
if (spawnError) return fail(`could not be spawned: ${spawnError.message}`);
|
|
595
|
+
if (code !== 0) return fail(`exited with non-zero code ${code}: ${stdout.trim()}`);
|
|
596
|
+
const match = /^(\d+)\t([^\t\r\n]+)$/.exec(stdout.trim());
|
|
597
|
+
const pid = match ? Number(match[1]) : Number.NaN;
|
|
598
|
+
const incarnation = match?.[2];
|
|
599
|
+
if (!Number.isSafeInteger(pid) || pid <= 0 || !isProcessIncarnation(incarnation))
|
|
600
|
+
return fail("reply did not contain a valid pid and process incarnation");
|
|
601
|
+
return { realBrokerPid: pid, realBrokerIncarnation: incarnation, error: undefined };
|
|
602
|
+
}
|
|
603
|
+
|
|
31
604
|
function resolveExpectedBrokerGeneration(): string {
|
|
32
605
|
const v = (packageJson as { version?: unknown }).version;
|
|
33
606
|
return typeof v === "string" && v.length > 0 ? v : "unknown";
|
|
@@ -39,15 +612,36 @@ function brokerStartupFailureReason(marker: BrokerStartupFailureMarker | undefin
|
|
|
39
612
|
return marker.reason;
|
|
40
613
|
}
|
|
41
614
|
|
|
615
|
+
function brokerSpawnFailureError(spawnError: Error): Error {
|
|
616
|
+
return spawnError instanceof BrokerHopError
|
|
617
|
+
? spawnError
|
|
618
|
+
: new Error(`Failed to spawn detached SDK broker: ${spawnError.message}`);
|
|
619
|
+
}
|
|
620
|
+
|
|
42
621
|
function brokerStartupExitReason(record: BrokerStartupExitRecord | undefined): string | undefined {
|
|
43
622
|
if (!record) return undefined;
|
|
44
623
|
if (record.reason === "startup-lock-blocked")
|
|
45
624
|
return `SDK broker startup blocked by retained removal transition ${record.blockingLockPath}.`;
|
|
625
|
+
if (record.reason === "startup-lock-unavailable")
|
|
626
|
+
return `SDK broker startup could not acquire required lock ${record.blockingLockPath}.`;
|
|
46
627
|
if (record.reason === "startup-deadline")
|
|
47
628
|
return `SDK broker startup exceeded its ${record.timeoutMs}ms fence deadline.`;
|
|
48
629
|
return `SDK broker startup interrupted by ${record.signal} before readiness.`;
|
|
49
630
|
}
|
|
50
631
|
|
|
632
|
+
function brokerStartupExitStatus(
|
|
633
|
+
childExitCode: number | null,
|
|
634
|
+
childSignal: NodeJS.Signals | null,
|
|
635
|
+
startupExitRecord: Pick<BrokerStartupExitRecord, "exitCode" | "signal"> | undefined,
|
|
636
|
+
startupFailureMarker: Pick<BrokerStartupFailureMarker, "exitCode" | "signal"> | undefined,
|
|
637
|
+
): { exitCode: number | null; signal: string | null } {
|
|
638
|
+
const brokerExitEvidence = startupExitRecord ?? startupFailureMarker;
|
|
639
|
+
return {
|
|
640
|
+
exitCode: brokerExitEvidence ? brokerExitEvidence.exitCode : childExitCode,
|
|
641
|
+
signal: brokerExitEvidence ? brokerExitEvidence.signal : childSignal,
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
|
|
51
645
|
export function isBrokerGenerationCompatible(discovery: BrokerDiscovery | null): boolean {
|
|
52
646
|
if (!discovery) return false;
|
|
53
647
|
return discovery.packageGeneration === resolveExpectedBrokerGeneration();
|
|
@@ -83,13 +677,17 @@ export interface EnsureBrokerSettings {
|
|
|
83
677
|
}
|
|
84
678
|
|
|
85
679
|
const BROKER_PUBLICATION_TIMEOUT_MS = 15_000;
|
|
86
|
-
const STARTUP_LOCK_WAIT_MS = 9_000;
|
|
87
680
|
const STALE_BROKER_RETIREMENT_TIMEOUT_MS = 5_000;
|
|
681
|
+
const SPAWN_LOCK_RETRY_DELAY_MS = 50;
|
|
682
|
+
/** Keep the startup fence alive long enough to adopt an ownerless removal transition. */
|
|
683
|
+
const STARTUP_LOCK_WAIT_MS = Math.max(
|
|
684
|
+
9_000,
|
|
685
|
+
FILE_LOCK_ORPHAN_TRANSITION_ADOPTION_AGE_MS + SPAWN_LOCK_RETRY_DELAY_MS * 2,
|
|
686
|
+
);
|
|
88
687
|
/** Parent budget covers stale retirement, a full child-fence wait, and one publication attempt. */
|
|
89
688
|
const DISCOVERY_TIMEOUT_MS = STALE_BROKER_RETIREMENT_TIMEOUT_MS + STARTUP_LOCK_WAIT_MS + BROKER_PUBLICATION_TIMEOUT_MS;
|
|
90
689
|
/** A process that loses the spawn lock waits this long for the winner's discovery before giving up. */
|
|
91
690
|
const SPAWN_LOCK_WAIT_MS = STALE_BROKER_RETIREMENT_TIMEOUT_MS + DISCOVERY_TIMEOUT_MS + 5_000;
|
|
92
|
-
const SPAWN_LOCK_RETRY_DELAY_MS = 50;
|
|
93
691
|
const SPAWN_LOCK_TARGET_NAME = "broker.spawn";
|
|
94
692
|
const STARTUP_LOCK_TARGET_NAME = "broker.startup";
|
|
95
693
|
const BROKER_SESSION_ENV_NAMES = new Set([
|
|
@@ -121,7 +719,9 @@ const BROKER_SESSION_ENV_PREFIXES = [
|
|
|
121
719
|
"GJC_MANAGED_OWNER_",
|
|
122
720
|
] as const;
|
|
123
721
|
|
|
124
|
-
export interface
|
|
722
|
+
export interface BrokerStartupLockOptions {
|
|
723
|
+
/** Stop waiting for the startup lock at this absolute wall-clock deadline. */
|
|
724
|
+
deadlineAt?: number;
|
|
125
725
|
onAcquired?: () => void;
|
|
126
726
|
onContended?: () => void;
|
|
127
727
|
}
|
|
@@ -189,33 +789,6 @@ export function setEnsureBrokerTimingForTest(timing: EnsureBrokerTiming | undefi
|
|
|
189
789
|
|
|
190
790
|
type SpawnLockOptions = Pick<FileLockOptions, "retries" | "retryDelayMs" | "signal">;
|
|
191
791
|
|
|
192
|
-
interface BrokerLockHostIdentity {
|
|
193
|
-
ownerHostId: string;
|
|
194
|
-
previousOwnerHostIds: readonly string[];
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
const brokerLockHostIdentityPromises = new Map<string, Promise<BrokerLockHostIdentity>>();
|
|
198
|
-
|
|
199
|
-
async function brokerLockHostIdentity(agentDir: string): Promise<BrokerLockHostIdentity> {
|
|
200
|
-
const coordinationRoot = path.resolve(agentDir, "sdk");
|
|
201
|
-
const pending =
|
|
202
|
-
brokerLockHostIdentityPromises.get(coordinationRoot) ??
|
|
203
|
-
Promise.all([loadInstallationHostId({ configRootDir: coordinationRoot }), loadLegacyInstallationHostId()]).then(
|
|
204
|
-
([ownerHostId, legacyHostId]) => ({
|
|
205
|
-
ownerHostId,
|
|
206
|
-
previousOwnerHostIds: legacyHostId === ownerHostId ? [] : [legacyHostId],
|
|
207
|
-
}),
|
|
208
|
-
);
|
|
209
|
-
brokerLockHostIdentityPromises.set(coordinationRoot, pending);
|
|
210
|
-
try {
|
|
211
|
-
return await pending;
|
|
212
|
-
} catch (error) {
|
|
213
|
-
if (brokerLockHostIdentityPromises.get(coordinationRoot) === pending)
|
|
214
|
-
brokerLockHostIdentityPromises.delete(coordinationRoot);
|
|
215
|
-
throw error;
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
792
|
async function readBrokerDiscoveryBeforeDeadline(
|
|
220
793
|
agentDir: string,
|
|
221
794
|
heartbeatTtlMs: number | undefined,
|
|
@@ -268,17 +841,30 @@ async function acquireSpawnLock(agentDir: string, options: SpawnLockOptions = {}
|
|
|
268
841
|
export async function withBrokerStartupLock<T>(
|
|
269
842
|
agentDir: string,
|
|
270
843
|
operation: (deadline: number) => Promise<T>,
|
|
271
|
-
|
|
844
|
+
options: BrokerStartupLockOptions = {},
|
|
272
845
|
): Promise<T> {
|
|
846
|
+
if (options.deadlineAt !== undefined && !Number.isFinite(options.deadlineAt))
|
|
847
|
+
throw new TypeError("Broker startup-lock deadline must be finite.");
|
|
273
848
|
const hostIdentity = await brokerLockHostIdentity(agentDir);
|
|
849
|
+
const now = ensureBrokerTiming.now();
|
|
850
|
+
const lockWaitMs =
|
|
851
|
+
options.deadlineAt === undefined
|
|
852
|
+
? STARTUP_LOCK_WAIT_MS
|
|
853
|
+
: Math.max(0, Math.min(STARTUP_LOCK_WAIT_MS, options.deadlineAt - now));
|
|
274
854
|
return withFileLock(
|
|
275
855
|
path.join(agentDir, "sdk", STARTUP_LOCK_TARGET_NAME),
|
|
276
|
-
() =>
|
|
856
|
+
() => {
|
|
857
|
+
const operationDeadline = ensureBrokerTiming.now() + BROKER_DISCOVERY_BUDGET.fenceOperationMs;
|
|
858
|
+
return operation(
|
|
859
|
+
options.deadlineAt === undefined ? operationDeadline : Math.min(operationDeadline, options.deadlineAt),
|
|
860
|
+
);
|
|
861
|
+
},
|
|
277
862
|
{
|
|
278
|
-
retries: Math.ceil(
|
|
863
|
+
retries: Math.ceil(lockWaitMs / SPAWN_LOCK_RETRY_DELAY_MS),
|
|
279
864
|
retryDelayMs: SPAWN_LOCK_RETRY_DELAY_MS,
|
|
280
|
-
|
|
281
|
-
|
|
865
|
+
orphanTransitionAgeMs: FILE_LOCK_ORPHAN_TRANSITION_ADOPTION_AGE_MS,
|
|
866
|
+
onAcquired: options.onAcquired,
|
|
867
|
+
onContended: options.onContended,
|
|
282
868
|
...hostIdentity,
|
|
283
869
|
},
|
|
284
870
|
);
|
|
@@ -371,8 +957,7 @@ export interface StartedFixtureBroker {
|
|
|
371
957
|
interface BrokerOwner {
|
|
372
958
|
stop(): Promise<void>;
|
|
373
959
|
canReuse(discovery: BrokerDiscovery | null): boolean;
|
|
374
|
-
markReady(discovery: BrokerDiscovery): boolean
|
|
375
|
-
setTarget(pid: number, incarnation: string): void;
|
|
960
|
+
markReady(discovery: BrokerDiscovery): Promise<boolean>;
|
|
376
961
|
}
|
|
377
962
|
type EnsureInitiator = "discovery" | "fixture-lease";
|
|
378
963
|
type EnsureOutcome =
|
|
@@ -392,6 +977,7 @@ interface ReapTiming {
|
|
|
392
977
|
gracefulMs: number;
|
|
393
978
|
killVerifyMs: number;
|
|
394
979
|
}
|
|
980
|
+
type DetachedBrokerSignal = (pid: number, brokerIncarnation: string, signal: NodeJS.Signals) => boolean;
|
|
395
981
|
const DEFAULT_REAP_TIMING: ReapTiming = {
|
|
396
982
|
gracefulMs: REAP_GRACEFUL_MS,
|
|
397
983
|
killVerifyMs: REAP_SIGKILL_CAP_MS,
|
|
@@ -408,12 +994,25 @@ const DEFAULT_REAP_TIMING: ReapTiming = {
|
|
|
408
994
|
* teardown (e.g. a transient signal-delivery failure); that is diagnostic only
|
|
409
995
|
* and never counts as exit, so the escalation cannot be skipped mid-shutdown.
|
|
410
996
|
*/
|
|
411
|
-
async function reapSpawnedBroker(
|
|
997
|
+
async function reapSpawnedBroker(
|
|
998
|
+
child: ChildProcess,
|
|
999
|
+
realBrokerPid?: number,
|
|
1000
|
+
brokerIncarnation?: string,
|
|
1001
|
+
timing: ReapTiming = DEFAULT_REAP_TIMING,
|
|
1002
|
+
): Promise<void> {
|
|
1003
|
+
// `child` is the actual broker only for direct fixture launches; hop and
|
|
1004
|
+
// trampoline launches keep the broker identity separate from the launcher.
|
|
1005
|
+
const pidToReap = realBrokerPid ?? child.pid;
|
|
412
1006
|
// A spawn failure (e.g. ENOENT) never created a kernel process: pid is
|
|
413
1007
|
// undefined and there is nothing to signal or await. The `error` event is the
|
|
414
1008
|
// only signal and is diagnostic here — termination trivially holds, so do not
|
|
415
1009
|
// run out the TERM/KILL windows or report a stuck child that never existed.
|
|
416
|
-
if (
|
|
1010
|
+
if (pidToReap === undefined) return;
|
|
1011
|
+
if (pidToReap !== child.pid) {
|
|
1012
|
+
// The retained ChildProcess is a hop/trampoline, not the broker itself.
|
|
1013
|
+
await reapDetachedBrokerPid(pidToReap, brokerIncarnation, timing);
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
417
1016
|
// Reaping owns repeated teardown diagnostics too. Keep exactly one error
|
|
418
1017
|
// listener for the retained child so a later signal-delivery error cannot
|
|
419
1018
|
// become an unhandled EventEmitter error after the spawn listener is consumed.
|
|
@@ -454,81 +1053,196 @@ async function reapSpawnedBroker(child: ChildProcess, timing: ReapTiming = DEFAU
|
|
|
454
1053
|
if (hasExited()) return;
|
|
455
1054
|
// SIGKILL is uninterruptible; a child still alive past this bounded wait is a
|
|
456
1055
|
// kernel-level stuck state. Surface it rather than silently orphaning the spawn.
|
|
457
|
-
throw new Error(`Detached SDK broker (pid ${
|
|
1056
|
+
throw new Error(`Detached SDK broker (pid ${pidToReap}) did not exit after SIGKILL during reap.`);
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
/**
|
|
1060
|
+
* Reap a broker launched by a short-lived hop or trampoline. The launcher has
|
|
1061
|
+
* already exited, so its ChildProcess carries no signal to await: the real broker
|
|
1062
|
+
* is targeted by the reported pid, and every poll is fenced by incarnation.
|
|
1063
|
+
* Signals use a pinned OS process handle; unsupported platforms fail closed.
|
|
1064
|
+
*/
|
|
1065
|
+
interface PinnedBrokerProcess {
|
|
1066
|
+
readonly incarnation: string;
|
|
1067
|
+
signalRoot(signal: number): boolean;
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
function signalPinnedBrokerProcess(
|
|
1071
|
+
reference: PinnedBrokerProcess | null,
|
|
1072
|
+
expectedIncarnation: string,
|
|
1073
|
+
signal: NodeJS.Signals,
|
|
1074
|
+
): boolean {
|
|
1075
|
+
if (!reference || reference.incarnation !== expectedIncarnation) return false;
|
|
1076
|
+
const signalNumber = os.constants.signals[signal];
|
|
1077
|
+
return signalNumber !== undefined && reference.signalRoot(signalNumber);
|
|
458
1078
|
}
|
|
459
1079
|
|
|
460
|
-
|
|
1080
|
+
function signalDetachedBrokerProcess(
|
|
461
1081
|
pid: number,
|
|
462
|
-
|
|
1082
|
+
brokerIncarnation: string,
|
|
1083
|
+
signal: NodeJS.Signals,
|
|
1084
|
+
platform: NodeJS.Platform = process.platform,
|
|
1085
|
+
): boolean {
|
|
1086
|
+
// Darwin has no atomic identity-bound signal primitive. A separate
|
|
1087
|
+
// incarnation check cannot close the PID-reuse window before process.kill.
|
|
1088
|
+
if (platform === "darwin") return false;
|
|
1089
|
+
const reference = nativeProcessBindings().Process.fromPid(pid);
|
|
1090
|
+
return signalPinnedBrokerProcess(reference, brokerIncarnation, signal);
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
async function reapDetachedBrokerPid(
|
|
1094
|
+
pid: number,
|
|
1095
|
+
brokerIncarnation?: string,
|
|
463
1096
|
timing: ReapTiming = DEFAULT_REAP_TIMING,
|
|
1097
|
+
observe: (pid: number) => ProcessIncarnationObservation = observeProcessIncarnation,
|
|
1098
|
+
signalProcess: DetachedBrokerSignal = signalDetachedBrokerProcess,
|
|
464
1099
|
): Promise<void> {
|
|
465
|
-
const
|
|
466
|
-
const
|
|
467
|
-
|
|
1100
|
+
const inspect = (): "owned" | "gone" | "unknown" => {
|
|
1101
|
+
const observation = observe(pid);
|
|
1102
|
+
if (observation.status === "absent") return "gone";
|
|
1103
|
+
if (observation.status !== "present" || !brokerIncarnation) return "unknown";
|
|
1104
|
+
return observation.incarnation === brokerIncarnation ? "owned" : "gone";
|
|
468
1105
|
};
|
|
469
|
-
const
|
|
470
|
-
|
|
471
|
-
|
|
1106
|
+
const awaitGone = async (windowMs: number): Promise<boolean> => {
|
|
1107
|
+
const deadline = Date.now() + windowMs;
|
|
1108
|
+
for (;;) {
|
|
1109
|
+
const state = inspect();
|
|
1110
|
+
if (state === "gone") return true;
|
|
1111
|
+
if (Date.now() >= deadline) return false;
|
|
1112
|
+
await sleep(Math.min(20, Math.max(1, deadline - Date.now())));
|
|
1113
|
+
}
|
|
1114
|
+
};
|
|
1115
|
+
const signal = (sig: NodeJS.Signals): boolean => {
|
|
1116
|
+
const state = inspect();
|
|
1117
|
+
if (state === "gone") return false;
|
|
1118
|
+
if (state === "unknown") {
|
|
1119
|
+
throw new Error(`Detached SDK broker (pid ${pid}) cannot be signaled without a verified process incarnation.`);
|
|
1120
|
+
}
|
|
1121
|
+
if (!brokerIncarnation) {
|
|
1122
|
+
throw new Error(`Detached SDK broker (pid ${pid}) cannot be signaled without a verified process incarnation.`);
|
|
1123
|
+
}
|
|
1124
|
+
let delivered = false;
|
|
472
1125
|
try {
|
|
473
|
-
|
|
1126
|
+
delivered = signalProcess(pid, brokerIncarnation, sig);
|
|
474
1127
|
} catch {
|
|
475
|
-
//
|
|
1128
|
+
// Re-inspect below; only a confirmed gone/replaced process settles cleanup.
|
|
476
1129
|
}
|
|
1130
|
+
if (!delivered && inspect() !== "gone")
|
|
1131
|
+
throw new Error(`Detached SDK broker (pid ${pid}) could not be signaled under its pinned incarnation.`);
|
|
1132
|
+
return true;
|
|
477
1133
|
};
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
signal("
|
|
484
|
-
|
|
485
|
-
if (!
|
|
1134
|
+
const beforeTermination = inspect();
|
|
1135
|
+
if (beforeTermination === "gone") return;
|
|
1136
|
+
if (beforeTermination === "unknown") {
|
|
1137
|
+
throw new Error(`Detached SDK broker (pid ${pid}) cannot be reaped without a verified process incarnation.`);
|
|
1138
|
+
}
|
|
1139
|
+
if (!signal("SIGTERM")) return;
|
|
1140
|
+
if (await awaitGone(timing.gracefulMs)) return;
|
|
1141
|
+
if (!signal("SIGKILL")) return;
|
|
1142
|
+
if (await awaitGone(timing.killVerifyMs)) return;
|
|
1143
|
+
throw new Error(`Detached SDK broker (pid ${pid}) did not exit after SIGKILL during reap.`);
|
|
486
1144
|
}
|
|
487
1145
|
|
|
488
|
-
|
|
489
|
-
|
|
1146
|
+
/** Reap a failed launch through its pending trampoline lease or retained broker handle. */
|
|
1147
|
+
export async function reapFailedBrokerLaunch(
|
|
1148
|
+
launch: Pick<BrokerLaunchResult, "process" | "realBrokerPid" | "realBrokerIncarnation" | "handoff">,
|
|
1149
|
+
): Promise<void> {
|
|
1150
|
+
if (launch.handoff) {
|
|
1151
|
+
await launch.handoff.terminate();
|
|
1152
|
+
return;
|
|
1153
|
+
}
|
|
1154
|
+
if (launch.realBrokerPid === undefined) return;
|
|
1155
|
+
if (launch.realBrokerPid === launch.process.pid) {
|
|
1156
|
+
await reapSpawnedBroker(launch.process, launch.realBrokerPid, launch.realBrokerIncarnation);
|
|
1157
|
+
return;
|
|
1158
|
+
}
|
|
1159
|
+
await reapDetachedBrokerPid(launch.realBrokerPid, launch.realBrokerIncarnation);
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
/** Reap the direct broker child held by the startup trampoline before publication. */
|
|
1163
|
+
export async function reapBrokerTrampolineChild(
|
|
1164
|
+
child: ChildProcess,
|
|
1165
|
+
brokerPid: number | undefined,
|
|
1166
|
+
brokerIncarnation?: string,
|
|
1167
|
+
): Promise<void> {
|
|
1168
|
+
await reapSpawnedBroker(child, brokerPid, brokerIncarnation);
|
|
490
1169
|
}
|
|
491
1170
|
|
|
492
1171
|
function registerBrokerOwner(
|
|
493
1172
|
agentDir: string,
|
|
494
1173
|
child: ChildProcess,
|
|
1174
|
+
realBrokerPid: number | undefined,
|
|
1175
|
+
brokerIncarnation: string | undefined,
|
|
495
1176
|
timing: ReapTiming = DEFAULT_REAP_TIMING,
|
|
1177
|
+
launchHandoff?: BrokerLaunchHandoff,
|
|
496
1178
|
): BrokerOwner {
|
|
497
|
-
|
|
498
|
-
|
|
1179
|
+
// Windows retains the hop process handle; POSIX retains the trampoline until publication
|
|
1180
|
+
// when requested. `realBrokerPid` and incarnation always identify the broker itself.
|
|
1181
|
+
const expectedBrokerPid = realBrokerPid ?? child.pid;
|
|
1182
|
+
const expectedBrokerIncarnation = brokerIncarnation;
|
|
1183
|
+
let handoff = launchHandoff;
|
|
1184
|
+
let publishedDiscovery: BrokerDiscovery | undefined;
|
|
499
1185
|
let state: "starting" | "ready" | "cleanup-unverified" = "starting";
|
|
500
|
-
const matches = (discovery: BrokerDiscovery | null): boolean =>
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
discovery.
|
|
1186
|
+
const matches = (discovery: BrokerDiscovery | null): boolean => {
|
|
1187
|
+
if (!discovery) return false;
|
|
1188
|
+
// Match discovery to the exact broker identity captured by the launch protocol.
|
|
1189
|
+
return Boolean(
|
|
1190
|
+
expectedBrokerPid !== undefined &&
|
|
1191
|
+
expectedBrokerIncarnation &&
|
|
1192
|
+
discovery.pid === expectedBrokerPid &&
|
|
1193
|
+
discovery.incarnation === expectedBrokerIncarnation,
|
|
507
1194
|
);
|
|
508
|
-
|
|
509
|
-
|
|
1195
|
+
};
|
|
1196
|
+
let stopTask: Promise<void> | undefined;
|
|
1197
|
+
const stop = (): Promise<void> => {
|
|
1198
|
+
if (stopTask) return stopTask;
|
|
1199
|
+
const task = (async () => {
|
|
510
1200
|
try {
|
|
511
|
-
if (
|
|
512
|
-
await
|
|
513
|
-
|
|
1201
|
+
if (handoff) {
|
|
1202
|
+
await handoff.terminate();
|
|
1203
|
+
handoff = undefined;
|
|
1204
|
+
} else if (publishedDiscovery) {
|
|
1205
|
+
try {
|
|
1206
|
+
await shutdownPublishedBrokerViaRpc(
|
|
1207
|
+
agentDir,
|
|
1208
|
+
publishedDiscovery,
|
|
1209
|
+
Date.now() +
|
|
1210
|
+
Math.max(BROKER_DISCOVERY_BUDGET.staleRetirementMs, timing.gracefulMs + timing.killVerifyMs),
|
|
1211
|
+
);
|
|
1212
|
+
} catch (error) {
|
|
1213
|
+
if (process.platform === "darwin" && expectedBrokerPid !== child.pid) throw error;
|
|
1214
|
+
await reapSpawnedBroker(child, expectedBrokerPid, expectedBrokerIncarnation, timing);
|
|
1215
|
+
}
|
|
1216
|
+
} else {
|
|
1217
|
+
await reapSpawnedBroker(child, expectedBrokerPid, expectedBrokerIncarnation, timing);
|
|
1218
|
+
}
|
|
514
1219
|
} catch (error) {
|
|
515
1220
|
state = "cleanup-unverified";
|
|
516
1221
|
throw error;
|
|
517
1222
|
}
|
|
518
1223
|
if (owners.get(agentDir) === owner) owners.delete(agentDir);
|
|
519
|
-
}
|
|
1224
|
+
})();
|
|
1225
|
+
stopTask = task;
|
|
1226
|
+
void task.catch(() => {
|
|
1227
|
+
if (stopTask === task) stopTask = undefined;
|
|
1228
|
+
});
|
|
1229
|
+
return task;
|
|
1230
|
+
};
|
|
1231
|
+
const owner: BrokerOwner = {
|
|
1232
|
+
stop,
|
|
520
1233
|
canReuse(discovery): boolean {
|
|
521
1234
|
return state === "ready" && matches(discovery);
|
|
522
1235
|
},
|
|
523
|
-
markReady(discovery): boolean {
|
|
1236
|
+
async markReady(discovery): Promise<boolean> {
|
|
524
1237
|
if (!matches(discovery)) return false;
|
|
1238
|
+
if (handoff) {
|
|
1239
|
+
await handoff.release();
|
|
1240
|
+
handoff = undefined;
|
|
1241
|
+
}
|
|
1242
|
+
publishedDiscovery = discovery;
|
|
525
1243
|
state = "ready";
|
|
526
1244
|
return true;
|
|
527
1245
|
},
|
|
528
|
-
setTarget(pid, incarnation): void {
|
|
529
|
-
targetPid = pid;
|
|
530
|
-
targetIncarnation = incarnation;
|
|
531
|
-
},
|
|
532
1246
|
};
|
|
533
1247
|
owners.set(agentDir, owner);
|
|
534
1248
|
return owner;
|
|
@@ -585,39 +1299,107 @@ export async function closeBrokerClientBeforeDeadline(
|
|
|
585
1299
|
await Promise.race([close, ensureBrokerTiming.sleep(remainingMs)]);
|
|
586
1300
|
}
|
|
587
1301
|
|
|
1302
|
+
function brokerIdentityIsGone(discovery: BrokerDiscovery): boolean {
|
|
1303
|
+
const observation = observeProcessIncarnation(discovery.pid);
|
|
1304
|
+
return (
|
|
1305
|
+
observation.status === "absent" ||
|
|
1306
|
+
(observation.status === "present" && observation.incarnation !== discovery.incarnation)
|
|
1307
|
+
);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
async function waitForBrokerIdentityGone(discovery: BrokerDiscovery, deadline: number): Promise<void> {
|
|
1311
|
+
for (;;) {
|
|
1312
|
+
if (brokerIdentityIsGone(discovery)) return;
|
|
1313
|
+
const remainingMs = deadline - ensureBrokerTiming.now();
|
|
1314
|
+
if (remainingMs <= 0) throw new Error(`SDK broker ${discovery.pid} did not exit after authenticated shutdown.`);
|
|
1315
|
+
await ensureBrokerTiming.sleep(Math.min(STALE_BROKER_POLL_MS, remainingMs));
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
async function waitForBrokerDiscoveryOwnerGone(
|
|
1320
|
+
agentDir: string,
|
|
1321
|
+
discovery: BrokerDiscovery,
|
|
1322
|
+
deadline: number,
|
|
1323
|
+
): Promise<void> {
|
|
1324
|
+
for (;;) {
|
|
1325
|
+
// Heartbeat expiry is not proof that this owner removed its publication.
|
|
1326
|
+
const current = await readBrokerDiscoveryBeforeDeadline(agentDir, Number.POSITIVE_INFINITY, deadline);
|
|
1327
|
+
if (!current || !sameBrokerOwner(current, discovery)) return;
|
|
1328
|
+
const remainingMs = deadline - ensureBrokerTiming.now();
|
|
1329
|
+
if (remainingMs <= 0) throw new Error(`SDK broker ${discovery.pid} retained discovery during retirement.`);
|
|
1330
|
+
await ensureBrokerTiming.sleep(Math.min(STALE_BROKER_POLL_MS, remainingMs));
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
async function shutdownPublishedBrokerViaRpc(
|
|
1335
|
+
agentDir: string,
|
|
1336
|
+
discovery: BrokerDiscovery,
|
|
1337
|
+
deadline: number,
|
|
1338
|
+
): Promise<void> {
|
|
1339
|
+
if (brokerIdentityIsGone(discovery)) return;
|
|
1340
|
+
const current = await readBrokerDiscovery(agentDir);
|
|
1341
|
+
if (!current || !sameBrokerOwner(current, discovery)) {
|
|
1342
|
+
if (discovery.pid === process.pid) await waitForBrokerDiscoveryOwnerGone(agentDir, discovery, deadline);
|
|
1343
|
+
else await waitForBrokerIdentityGone(discovery, deadline);
|
|
1344
|
+
return;
|
|
1345
|
+
}
|
|
1346
|
+
const remainingMs = deadline - ensureBrokerTiming.now();
|
|
1347
|
+
if (remainingMs <= 0) throw new Error("SDK broker shutdown deadline elapsed before RPC connection.");
|
|
1348
|
+
const client = await SdkClient.connect(current.url, current.token, {
|
|
1349
|
+
timeoutMs: Math.max(1, Math.min(2_000, remainingMs)),
|
|
1350
|
+
});
|
|
1351
|
+
try {
|
|
1352
|
+
await client.global("broker.shutdown", {}, { timeoutMs: Math.max(1, deadline - ensureBrokerTiming.now()) });
|
|
1353
|
+
} finally {
|
|
1354
|
+
await closeBrokerClientBeforeDeadline(client, deadline).catch(() => undefined);
|
|
1355
|
+
}
|
|
1356
|
+
if (discovery.pid === process.pid) await waitForBrokerDiscoveryOwnerGone(agentDir, discovery, deadline);
|
|
1357
|
+
else await waitForBrokerIdentityGone(discovery, deadline);
|
|
1358
|
+
}
|
|
1359
|
+
|
|
588
1360
|
async function retireUnusableBroker(
|
|
589
1361
|
stale: BrokerDiscovery,
|
|
590
1362
|
settings: EnsureBrokerSettings,
|
|
591
1363
|
outerDeadline = Number.POSITIVE_INFINITY,
|
|
592
1364
|
): Promise<void> {
|
|
593
1365
|
const deadline = Math.min(outerDeadline, ensureBrokerTiming.now() + BROKER_DISCOVERY_BUDGET.staleRetirementMs);
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
try {
|
|
600
|
-
await client.global("broker.shutdown", {});
|
|
601
|
-
shutdownSucceeded = true;
|
|
602
|
-
} finally {
|
|
603
|
-
await closeBrokerClientBeforeDeadline(client, deadline).catch(() => undefined);
|
|
1366
|
+
const current = await readBrokerDiscoveryBeforeDeadline(settings.agentDir, settings.heartbeatTtlMs, deadline);
|
|
1367
|
+
if (!current || !sameBrokerOwner(current, stale)) {
|
|
1368
|
+
if (stale.pid === process.pid) {
|
|
1369
|
+
await waitForBrokerDiscoveryOwnerGone(settings.agentDir, stale, deadline);
|
|
1370
|
+
return;
|
|
604
1371
|
}
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
if (brokerProcessIncarnation(stale.pid) === stale.incarnation) {
|
|
609
|
-
try {
|
|
610
|
-
process.kill(stale.pid, "SIGTERM");
|
|
611
|
-
} catch {}
|
|
612
|
-
}
|
|
613
|
-
} catch {}
|
|
1372
|
+
await reapDetachedBrokerPid(stale.pid, stale.incarnation);
|
|
1373
|
+
await waitForBrokerIdentityGone(stale, deadline);
|
|
1374
|
+
return;
|
|
614
1375
|
}
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
1376
|
+
try {
|
|
1377
|
+
await shutdownPublishedBrokerViaRpc(settings.agentDir, current, deadline);
|
|
1378
|
+
return;
|
|
1379
|
+
} catch (rpcError) {
|
|
1380
|
+
if (stale.pid !== process.pid) {
|
|
1381
|
+
try {
|
|
1382
|
+
await reapDetachedBrokerPid(stale.pid, stale.incarnation);
|
|
1383
|
+
} catch (cleanupError) {
|
|
1384
|
+
throw new AggregateError(
|
|
1385
|
+
[rpcError, cleanupError],
|
|
1386
|
+
`SDK broker ${stale.pid} shutdown and identity-bound cleanup both failed.`,
|
|
1387
|
+
);
|
|
1388
|
+
}
|
|
1389
|
+
} else {
|
|
1390
|
+
try {
|
|
1391
|
+
await waitForBrokerDiscoveryOwnerGone(settings.agentDir, stale, deadline);
|
|
1392
|
+
} catch (retirementError) {
|
|
1393
|
+
throw new AggregateError(
|
|
1394
|
+
[rpcError, retirementError],
|
|
1395
|
+
`SDK broker ${stale.pid} shutdown RPC and discovery retirement both failed.`,
|
|
1396
|
+
);
|
|
1397
|
+
}
|
|
1398
|
+
return;
|
|
1399
|
+
}
|
|
1400
|
+
// If pid === process.pid, fall through to wait on identity
|
|
620
1401
|
}
|
|
1402
|
+
await waitForBrokerIdentityGone(stale, deadline);
|
|
621
1403
|
}
|
|
622
1404
|
|
|
623
1405
|
/** Reconcile an observed broker generation before entering a startup critical section. */
|
|
@@ -675,7 +1457,11 @@ function createFixtureLease(owner: BrokerOwner, child: ChildProcess): ExactFixtu
|
|
|
675
1457
|
return createFixtureLeaseFromChild(child, () => owner.stop());
|
|
676
1458
|
}
|
|
677
1459
|
|
|
678
|
-
async function ensureBrokerOnce(
|
|
1460
|
+
async function ensureBrokerOnce(
|
|
1461
|
+
settings: EnsureBrokerSettings,
|
|
1462
|
+
initiator: EnsureInitiator,
|
|
1463
|
+
retriedStartupLockUnavailable = false,
|
|
1464
|
+
): Promise<EnsureOutcome> {
|
|
679
1465
|
const initialDiscoveryDeadline =
|
|
680
1466
|
ensureBrokerTiming.now() + (initiator === "fixture-lease" ? FIXTURE_DISCOVERY_TIMEOUT_MS : DISCOVERY_TIMEOUT_MS);
|
|
681
1467
|
const priorOwner = owners.get(settings.agentDir);
|
|
@@ -716,6 +1502,7 @@ async function ensureBrokerOnce(settings: EnsureBrokerSettings, initiator: Ensur
|
|
|
716
1502
|
// Fixture leases always spawn their own.
|
|
717
1503
|
let spawnLog: BrokerSpawnLog | undefined;
|
|
718
1504
|
const releaseSpawnLock = initiator === "fixture-lease" ? async () => {} : await acquireSpawnLock(settings.agentDir);
|
|
1505
|
+
let spawnLockReleased = false;
|
|
719
1506
|
try {
|
|
720
1507
|
const deadline =
|
|
721
1508
|
ensureBrokerTiming.now() +
|
|
@@ -731,176 +1518,200 @@ async function ensureBrokerOnce(settings: EnsureBrokerSettings, initiator: Ensur
|
|
|
731
1518
|
// that finished between our first read and the lock acquisition.
|
|
732
1519
|
const discoveredUnderLock = await reconcileBrokerGenerationForStartup(settings, deadline);
|
|
733
1520
|
if (discoveredUnderLock) return { kind: "external-discovery", discovery: discoveredUnderLock };
|
|
734
|
-
const command = resolveSdkInternalSpawnCommand(
|
|
735
|
-
process.platform === "win32" || initiator === "fixture-lease"
|
|
736
|
-
? "broker-internal"
|
|
737
|
-
: "broker-trampoline-internal",
|
|
738
|
-
);
|
|
1521
|
+
const command = resolveSdkInternalSpawnCommand("broker-internal");
|
|
739
1522
|
spawnLog = await openBrokerSpawnLog(settings.agentDir);
|
|
740
1523
|
// A stale marker must never be misattributed to this spawn; clear it first.
|
|
741
1524
|
await clearBrokerStartupExitRecord(settings.agentDir);
|
|
742
1525
|
await clearBrokerStartupFailureMarker(settings.agentDir);
|
|
743
1526
|
const childSpawnedAt = Date.now();
|
|
744
|
-
const
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
env: brokerSpawnEnvironment(command, settings.env),
|
|
748
|
-
...(command.kind === "bun-source" ? { cwd: command.cwd } : {}),
|
|
749
|
-
});
|
|
750
|
-
const isTrampoline = process.platform !== "win32" && initiator !== "fixture-lease";
|
|
1527
|
+
const brokerArgs = [...command.args, "--agent-dir", settings.agentDir];
|
|
1528
|
+
const env = brokerSpawnEnvironment(command, settings.env);
|
|
1529
|
+
let spawnResult: BrokerSpawnResult;
|
|
751
1530
|
let spawnError: Error | undefined;
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1531
|
+
const launchMode = resolveBrokerLaunchMode(process.platform, initiator);
|
|
1532
|
+
if (launchMode === "windows-hop") {
|
|
1533
|
+
const launched = await launchBrokerViaHop(
|
|
1534
|
+
{
|
|
1535
|
+
command: { file: command.file, args: brokerArgs },
|
|
1536
|
+
...(command.kind === "bun-source" ? { cwd: command.cwd } : {}),
|
|
1537
|
+
...(spawnLog ? { stderrLogPath: spawnLog.path } : {}),
|
|
1538
|
+
},
|
|
1539
|
+
{
|
|
1540
|
+
env,
|
|
1541
|
+
cwd: command.kind === "bun-source" ? command.cwd : undefined,
|
|
1542
|
+
timeoutMs: Math.max(0, deadline - ensureBrokerTiming.now()),
|
|
1543
|
+
},
|
|
1544
|
+
);
|
|
1545
|
+
spawnResult = {
|
|
1546
|
+
process: launched.process,
|
|
1547
|
+
realBrokerPid: launched.realBrokerPid,
|
|
1548
|
+
realBrokerIncarnation: launched.realBrokerIncarnation,
|
|
1549
|
+
};
|
|
1550
|
+
spawnError = launched.error;
|
|
1551
|
+
} else if (launchMode === "posix-trampoline" || launchMode === "darwin-trampoline") {
|
|
1552
|
+
const launched = await launchBrokerViaPosixTrampoline(settings.agentDir, {
|
|
1553
|
+
env,
|
|
1554
|
+
...(command.kind === "bun-source" ? { cwd: command.cwd } : {}),
|
|
1555
|
+
...(spawnLog ? { stderrFd: spawnLog.handle.fd } : {}),
|
|
1556
|
+
timeoutMs: Math.max(0, deadline - ensureBrokerTiming.now()),
|
|
1557
|
+
retainChildUntilPublication: launchMode === "darwin-trampoline",
|
|
1558
|
+
});
|
|
1559
|
+
spawnResult = {
|
|
1560
|
+
process: launched.process,
|
|
1561
|
+
realBrokerPid: launched.realBrokerPid,
|
|
1562
|
+
realBrokerIncarnation: launched.realBrokerIncarnation,
|
|
1563
|
+
handoff: launched.handoff,
|
|
1564
|
+
};
|
|
1565
|
+
spawnError = launched.error;
|
|
1566
|
+
} else {
|
|
1567
|
+
spawnResult = spawnBrokerDirect(command.file, brokerArgs, {
|
|
1568
|
+
stdioFd: spawnLog?.handle.fd,
|
|
1569
|
+
env,
|
|
1570
|
+
cwd: command.kind === "bun-source" ? command.cwd : undefined,
|
|
1571
|
+
detached: process.platform !== "win32",
|
|
1572
|
+
});
|
|
1573
|
+
spawnResult.process.once("error", error => {
|
|
1574
|
+
spawnError = error;
|
|
786
1575
|
});
|
|
787
1576
|
}
|
|
788
|
-
|
|
1577
|
+
|
|
1578
|
+
const child = spawnResult.process;
|
|
1579
|
+
const realBrokerPid = spawnResult.realBrokerPid;
|
|
1580
|
+
let childIncarnation =
|
|
1581
|
+
spawnResult.realBrokerIncarnation ??
|
|
1582
|
+
(launchMode === "direct" && realBrokerPid !== undefined ? brokerProcessIncarnation(realBrokerPid) : undefined);
|
|
1583
|
+
let owner = registerBrokerOwner(
|
|
1584
|
+
settings.agentDir,
|
|
1585
|
+
child,
|
|
1586
|
+
realBrokerPid,
|
|
1587
|
+
childIncarnation,
|
|
1588
|
+
DEFAULT_REAP_TIMING,
|
|
1589
|
+
spawnResult.handoff,
|
|
1590
|
+
);
|
|
1591
|
+
if (!spawnResult.handoff) child.unref();
|
|
789
1592
|
// The child holds its own duplicate of the descriptor. Failure to close the
|
|
790
1593
|
// parent's diagnostic handle must not discard exact ownership of a live child.
|
|
791
1594
|
await spawnLog?.handle.close().catch(() => undefined);
|
|
792
1595
|
let discoveryError: unknown;
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
spawnError ||
|
|
796
|
-
(isTrampoline && childExited && (!trampolineReported || childExitCode !== 0 || childSignalCode !== null)) ||
|
|
797
|
-
(!isTrampoline && process.platform === "win32" && (child.exitCode !== null || child.signalCode !== null))
|
|
798
|
-
)
|
|
799
|
-
break;
|
|
800
|
-
if (
|
|
801
|
-
isTrampoline &&
|
|
802
|
-
trampolineReported &&
|
|
803
|
-
spawnedBrokerPid !== undefined &&
|
|
804
|
-
spawnedBrokerIncarnation !== undefined
|
|
805
|
-
) {
|
|
806
|
-
const observed = observeProcessIncarnation(spawnedBrokerPid);
|
|
807
|
-
if (observed.status === "present" && observed.incarnation !== spawnedBrokerIncarnation) {
|
|
808
|
-
brokerDeathObserved = true;
|
|
809
|
-
break;
|
|
810
|
-
}
|
|
811
|
-
if (observed.status === "absent") {
|
|
812
|
-
brokerDeathObserved = true;
|
|
813
|
-
break;
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
// The trampoline's stdout line identifies the real broker. Do not read
|
|
817
|
-
// discovery until that line has been parsed, or a marker from the
|
|
818
|
-
// trampoline could be trusted as the broker's own startup evidence.
|
|
819
|
-
if (isTrampoline && !trampolineReported) {
|
|
820
|
-
await ensureBrokerTiming.sleep(50);
|
|
821
|
-
continue;
|
|
822
|
-
}
|
|
1596
|
+
let brokerExitedBeforeDiscovery = false;
|
|
1597
|
+
const observeDiscovery = async (): Promise<EnsureOutcome | undefined> => {
|
|
823
1598
|
try {
|
|
824
1599
|
const discovered = await readBrokerDiscoveryBeforeDeadline(
|
|
825
1600
|
settings.agentDir,
|
|
826
1601
|
settings.heartbeatTtlMs,
|
|
827
1602
|
deadline,
|
|
828
1603
|
);
|
|
829
|
-
if (discovered)
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1604
|
+
if (!discovered || !(await isBrokerReusable(discovered))) return undefined;
|
|
1605
|
+
if (
|
|
1606
|
+
childIncarnation === undefined &&
|
|
1607
|
+
realBrokerPid !== undefined &&
|
|
1608
|
+
realBrokerPid === child.pid &&
|
|
1609
|
+
discovered.pid === realBrokerPid
|
|
1610
|
+
) {
|
|
1611
|
+
const verifiedIncarnation = brokerProcessIncarnation(realBrokerPid);
|
|
1612
|
+
if (verifiedIncarnation === discovered.incarnation) {
|
|
1613
|
+
childIncarnation = verifiedIncarnation;
|
|
1614
|
+
owner = registerBrokerOwner(
|
|
1615
|
+
settings.agentDir,
|
|
1616
|
+
child,
|
|
1617
|
+
realBrokerPid,
|
|
1618
|
+
childIncarnation,
|
|
1619
|
+
DEFAULT_REAP_TIMING,
|
|
1620
|
+
spawnResult.handoff,
|
|
1621
|
+
);
|
|
847
1622
|
}
|
|
848
|
-
if (owner.markReady(discovered)) {
|
|
849
|
-
return initiator === "fixture-lease"
|
|
850
|
-
? { kind: "local-started-fixture", discovery: discovered, owner, child }
|
|
851
|
-
: { kind: "local-started-discovery", discovery: discovered };
|
|
852
|
-
}
|
|
853
|
-
await owner.stop();
|
|
854
|
-
return { kind: "external-discovery", discovery: discovered };
|
|
855
1623
|
}
|
|
1624
|
+
if (await owner.markReady(discovered)) {
|
|
1625
|
+
return initiator === "fixture-lease"
|
|
1626
|
+
? { kind: "local-started-fixture", discovery: discovered, owner, child }
|
|
1627
|
+
: { kind: "local-started-discovery", discovery: discovered };
|
|
1628
|
+
}
|
|
1629
|
+
await owner.stop();
|
|
1630
|
+
return { kind: "external-discovery", discovery: discovered };
|
|
856
1631
|
} catch (error) {
|
|
857
1632
|
discoveryError = error;
|
|
1633
|
+
return undefined;
|
|
858
1634
|
}
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
1635
|
+
};
|
|
1636
|
+
while (ensureBrokerTiming.now() < deadline) {
|
|
1637
|
+
const failedSpawn =
|
|
1638
|
+
spawnError || child.signalCode !== null || (child.exitCode !== null && child.exitCode !== 0);
|
|
1639
|
+
if (failedSpawn) {
|
|
1640
|
+
const discovered = await observeDiscovery();
|
|
1641
|
+
if (discovered) return discovered;
|
|
1642
|
+
brokerExitedBeforeDiscovery = true;
|
|
1643
|
+
break;
|
|
1644
|
+
}
|
|
1645
|
+
let brokerObservation: ProcessIncarnationObservation | undefined;
|
|
1646
|
+
if (realBrokerPid !== undefined) {
|
|
1647
|
+
brokerObservation = observeProcessIncarnation(realBrokerPid);
|
|
1648
|
+
if (brokerObservation.status === "absent") {
|
|
1649
|
+
const discovered = await observeDiscovery();
|
|
1650
|
+
if (discovered) return discovered;
|
|
1651
|
+
brokerExitedBeforeDiscovery = true;
|
|
1652
|
+
break;
|
|
1653
|
+
}
|
|
1654
|
+
if (
|
|
1655
|
+
brokerObservation.status === "present" &&
|
|
1656
|
+
childIncarnation !== undefined &&
|
|
1657
|
+
brokerObservation.incarnation !== childIncarnation
|
|
1658
|
+
) {
|
|
1659
|
+
const discovered = await observeDiscovery();
|
|
1660
|
+
if (discovered) return discovered;
|
|
1661
|
+
brokerExitedBeforeDiscovery = true;
|
|
1662
|
+
break;
|
|
1663
|
+
}
|
|
1664
|
+
if (
|
|
1665
|
+
brokerObservation.status === "present" &&
|
|
1666
|
+
childIncarnation === undefined &&
|
|
1667
|
+
realBrokerPid === child.pid
|
|
1668
|
+
) {
|
|
1669
|
+
childIncarnation = brokerObservation.incarnation;
|
|
1670
|
+
owner = registerBrokerOwner(settings.agentDir, child, realBrokerPid, childIncarnation);
|
|
1671
|
+
}
|
|
1672
|
+
const startupExitRecord = await readBrokerStartupExitRecord(settings.agentDir);
|
|
1673
|
+
if (startupExitRecord?.pid === realBrokerPid && startupExitRecord.writtenAt >= childSpawnedAt) {
|
|
1674
|
+
const discovered = await observeDiscovery();
|
|
1675
|
+
if (discovered) return discovered;
|
|
1676
|
+
brokerExitedBeforeDiscovery = true;
|
|
1677
|
+
break;
|
|
1678
|
+
}
|
|
1679
|
+
if (brokerObservation.status === "present" && childIncarnation === brokerObservation.incarnation) {
|
|
1680
|
+
const marker = await readBrokerStartupFailureMarker(settings.agentDir);
|
|
1681
|
+
if (
|
|
1682
|
+
marker?.pid === realBrokerPid &&
|
|
1683
|
+
marker?.incarnation === childIncarnation &&
|
|
1684
|
+
marker.writtenAt >= childSpawnedAt
|
|
1685
|
+
) {
|
|
1686
|
+
const discovered = await observeDiscovery();
|
|
1687
|
+
if (discovered) return discovered;
|
|
1688
|
+
brokerExitedBeforeDiscovery = true;
|
|
1689
|
+
break;
|
|
883
1690
|
}
|
|
884
|
-
await ensureBrokerTiming.sleep(50);
|
|
885
1691
|
}
|
|
886
|
-
} catch {
|
|
887
|
-
// fall through to cleanup + failure
|
|
888
1692
|
}
|
|
1693
|
+
const discovered = await observeDiscovery();
|
|
1694
|
+
if (discovered) return discovered;
|
|
1695
|
+
await ensureBrokerTiming.sleep(50);
|
|
889
1696
|
}
|
|
1697
|
+
const exitedBeforeDiscovery = brokerExitedBeforeDiscovery;
|
|
890
1698
|
const marker = await readBrokerStartupFailureMarker(settings.agentDir);
|
|
891
1699
|
const startupExitRecord = await readBrokerStartupExitRecord(settings.agentDir);
|
|
1700
|
+
// Hops and committed trampolines may exit before the parent finishes polling.
|
|
1701
|
+
// Failure records are attributed to the real broker pid, not the launcher.
|
|
892
1702
|
const trustedMarker =
|
|
893
1703
|
marker &&
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
marker.pid ===
|
|
897
|
-
marker.incarnation ===
|
|
1704
|
+
childIncarnation !== undefined &&
|
|
1705
|
+
realBrokerPid !== undefined &&
|
|
1706
|
+
marker.pid === realBrokerPid &&
|
|
1707
|
+
marker.incarnation === childIncarnation &&
|
|
1708
|
+
marker.writtenAt >= childSpawnedAt
|
|
898
1709
|
? marker
|
|
899
1710
|
: undefined;
|
|
900
1711
|
const trustedStartupExitRecord =
|
|
901
1712
|
startupExitRecord &&
|
|
902
|
-
|
|
903
|
-
startupExitRecord.pid ===
|
|
1713
|
+
realBrokerPid !== undefined &&
|
|
1714
|
+
startupExitRecord.pid === realBrokerPid &&
|
|
904
1715
|
startupExitRecord.writtenAt >= childSpawnedAt
|
|
905
1716
|
? startupExitRecord
|
|
906
1717
|
: undefined;
|
|
@@ -909,7 +1720,7 @@ async function ensureBrokerOnce(settings: EnsureBrokerSettings, initiator: Ensur
|
|
|
909
1720
|
exitedBeforeDiscovery &&
|
|
910
1721
|
(trustedMarker?.reason.startsWith(`Failed to acquire lock for ${startupLockPath} after `) === true ||
|
|
911
1722
|
brokerStartupFailureCleanupTargetsLock(trustedMarker, `${startupLockPath}.lock`));
|
|
912
|
-
if (!startupFenceContention && exitedBeforeDiscovery
|
|
1723
|
+
if (!startupFenceContention && exitedBeforeDiscovery) {
|
|
913
1724
|
try {
|
|
914
1725
|
startupFenceContention = (await fs.stat(`${startupLockPath}.lock`)).isDirectory();
|
|
915
1726
|
} catch {
|
|
@@ -951,23 +1762,10 @@ async function ensureBrokerOnce(settings: EnsureBrokerSettings, initiator: Ensur
|
|
|
951
1762
|
// rejects that marker instead of misattributing a foreign failure to this
|
|
952
1763
|
// spawn's caller.
|
|
953
1764
|
const failure = spawnError
|
|
954
|
-
?
|
|
1765
|
+
? brokerSpawnFailureError(spawnError)
|
|
955
1766
|
: exitedBeforeDiscovery
|
|
956
1767
|
? new BrokerStartupError({
|
|
957
|
-
exitCode
|
|
958
|
-
? (trustedStartupExitRecord?.exitCode ??
|
|
959
|
-
trustedMarker?.exitCode ??
|
|
960
|
-
childExitCode ??
|
|
961
|
-
child.exitCode ??
|
|
962
|
-
null)
|
|
963
|
-
: (childExitCode ?? child.exitCode ?? trustedStartupExitRecord?.exitCode ?? null),
|
|
964
|
-
signal: isTrampoline
|
|
965
|
-
? (trustedStartupExitRecord?.signal ??
|
|
966
|
-
trustedMarker?.signal ??
|
|
967
|
-
childSignalCode ??
|
|
968
|
-
child.signalCode ??
|
|
969
|
-
null)
|
|
970
|
-
: (childSignalCode ?? child.signalCode ?? trustedStartupExitRecord?.signal ?? null),
|
|
1768
|
+
...brokerStartupExitStatus(child.exitCode, child.signalCode, trustedStartupExitRecord, trustedMarker),
|
|
971
1769
|
reason:
|
|
972
1770
|
brokerStartupExitReason(trustedStartupExitRecord) ?? brokerStartupFailureReason(trustedMarker),
|
|
973
1771
|
stderrExcerpt: spawnLogTail.length > 0 ? spawnLogTail : undefined,
|
|
@@ -983,10 +1781,20 @@ async function ensureBrokerOnce(settings: EnsureBrokerSettings, initiator: Ensur
|
|
|
983
1781
|
"SDK broker discovery and spawned broker cleanup both failed.",
|
|
984
1782
|
);
|
|
985
1783
|
}
|
|
1784
|
+
if (
|
|
1785
|
+
!retriedStartupLockUnavailable &&
|
|
1786
|
+
initiator === "discovery" &&
|
|
1787
|
+
trustedStartupExitRecord?.reason === "startup-lock-unavailable"
|
|
1788
|
+
) {
|
|
1789
|
+
await releaseSpawnLock();
|
|
1790
|
+
spawnLockReleased = true;
|
|
1791
|
+
await ensureBrokerTiming.sleep(SPAWN_LOCK_RETRY_DELAY_MS);
|
|
1792
|
+
return ensureBrokerOnce(settings, initiator, true);
|
|
1793
|
+
}
|
|
986
1794
|
throw failure;
|
|
987
1795
|
} finally {
|
|
988
1796
|
if (spawnLog) await removeBrokerSpawnLog(spawnLog.path);
|
|
989
|
-
await releaseSpawnLock();
|
|
1797
|
+
if (!spawnLockReleased) await releaseSpawnLock();
|
|
990
1798
|
}
|
|
991
1799
|
}
|
|
992
1800
|
|
|
@@ -1052,12 +1860,16 @@ export function startFixtureBrokerWithLeaseForTest(settings: EnsureBrokerSetting
|
|
|
1052
1860
|
* Test-only launch surface for topology fixtures. It accepts an already-resolved
|
|
1053
1861
|
* command and retains the exact spawned child; no production selection path
|
|
1054
1862
|
* reaches this function.
|
|
1863
|
+
*
|
|
1864
|
+
* Note: This function does NOT use the Windows broker hop. Fixtures are test-only
|
|
1865
|
+
* and do not require Windows process-tree detachment; the fixture needs to retain
|
|
1866
|
+
* exact ownership of the actual broker process to control its stdio[3] control pipe.
|
|
1055
1867
|
*/
|
|
1056
1868
|
export function startFixtureBrokerCommandWithLeaseForTest(command: FixtureBrokerCommand): StartedFixtureBrokerCommand {
|
|
1057
1869
|
if (!command.file || !Array.isArray(command.args)) throw new Error("Invalid fixture broker command.");
|
|
1058
|
-
const child = spawn(command.file,
|
|
1870
|
+
const child = spawn(command.file, command.args, {
|
|
1059
1871
|
cwd: command.cwd,
|
|
1060
|
-
detached:
|
|
1872
|
+
detached: process.platform !== "win32",
|
|
1061
1873
|
stdio: ["ignore", "ignore", "ignore", "pipe"],
|
|
1062
1874
|
env: command.env,
|
|
1063
1875
|
});
|
|
@@ -1107,8 +1919,40 @@ export function brokerOwnerIdentityMatchesForTest(
|
|
|
1107
1919
|
return sameBrokerOwner(left, right);
|
|
1108
1920
|
}
|
|
1109
1921
|
/** Test hook: drives the detached-broker reap on a controllable child surface. */
|
|
1110
|
-
export function reapSpawnedBrokerForTest(
|
|
1111
|
-
|
|
1922
|
+
export function reapSpawnedBrokerForTest(
|
|
1923
|
+
child: ChildProcess,
|
|
1924
|
+
realBrokerPid?: number,
|
|
1925
|
+
brokerIncarnation?: string,
|
|
1926
|
+
timing: ReapTiming = DEFAULT_REAP_TIMING,
|
|
1927
|
+
): Promise<void> {
|
|
1928
|
+
return reapSpawnedBroker(child, realBrokerPid, brokerIncarnation, timing);
|
|
1929
|
+
}
|
|
1930
|
+
/** Test hook: verifies detached PID reaping fences every signal and poll by incarnation. */
|
|
1931
|
+
export function reapDetachedBrokerPidForTest(
|
|
1932
|
+
pid: number,
|
|
1933
|
+
brokerIncarnation: string | undefined,
|
|
1934
|
+
timing: ReapTiming = DEFAULT_REAP_TIMING,
|
|
1935
|
+
observe: (pid: number) => ProcessIncarnationObservation = observeProcessIncarnation,
|
|
1936
|
+
signalProcess: DetachedBrokerSignal = signalDetachedBrokerProcess,
|
|
1937
|
+
): Promise<void> {
|
|
1938
|
+
return reapDetachedBrokerPid(pid, brokerIncarnation, timing, observe, signalProcess);
|
|
1939
|
+
}
|
|
1940
|
+
/** Test hook: verifies a pinned process handle matches before sending a signal. */
|
|
1941
|
+
export function signalPinnedBrokerProcessForTest(
|
|
1942
|
+
reference: PinnedBrokerProcess | null,
|
|
1943
|
+
expectedIncarnation: string,
|
|
1944
|
+
signal: NodeJS.Signals,
|
|
1945
|
+
): boolean {
|
|
1946
|
+
return signalPinnedBrokerProcess(reference, expectedIncarnation, signal);
|
|
1947
|
+
}
|
|
1948
|
+
/** Test hook: verifies platforms without identity-bound signaling fail closed. */
|
|
1949
|
+
export function signalDetachedBrokerProcessForTest(
|
|
1950
|
+
pid: number,
|
|
1951
|
+
brokerIncarnation: string,
|
|
1952
|
+
signal: NodeJS.Signals,
|
|
1953
|
+
platform: NodeJS.Platform,
|
|
1954
|
+
): boolean {
|
|
1955
|
+
return signalDetachedBrokerProcess(pid, brokerIncarnation, signal, platform);
|
|
1112
1956
|
}
|
|
1113
1957
|
/** Test hook: resolves the complete broker environment without spawning. */
|
|
1114
1958
|
export function brokerSpawnEnvironmentForTest(
|
|
@@ -1121,12 +1965,46 @@ export function brokerSpawnEnvironmentForTest(
|
|
|
1121
1965
|
export function registerBrokerOwnerForTest(
|
|
1122
1966
|
agentDir: string,
|
|
1123
1967
|
child: ChildProcess,
|
|
1968
|
+
realBrokerPid?: number,
|
|
1969
|
+
brokerIncarnation?: string,
|
|
1124
1970
|
timing: ReapTiming = DEFAULT_REAP_TIMING,
|
|
1125
1971
|
): BrokerOwner {
|
|
1126
|
-
return registerBrokerOwner(agentDir, child, timing);
|
|
1972
|
+
return registerBrokerOwner(agentDir, child, realBrokerPid, brokerIncarnation, timing);
|
|
1127
1973
|
}
|
|
1128
1974
|
|
|
1129
1975
|
/** Test hook: exercises the same trusted-marker reason reconstruction used by ensureBroker. */
|
|
1130
1976
|
export function brokerStartupFailureReasonForTest(marker: BrokerStartupFailureMarker | undefined): string {
|
|
1131
1977
|
return brokerStartupFailureReason(marker);
|
|
1132
1978
|
}
|
|
1979
|
+
/** Test hook: verifies typed hop errors survive ensureBroker's spawn-failure mapping. */
|
|
1980
|
+
export function brokerSpawnFailureErrorForTest(spawnError: Error): Error {
|
|
1981
|
+
return brokerSpawnFailureError(spawnError);
|
|
1982
|
+
}
|
|
1983
|
+
/** Test hook: preserves and reaps a verified broker reply captured at the hop deadline. */
|
|
1984
|
+
export function brokerHopTimeoutResultForTest(
|
|
1985
|
+
hop: ChildProcess,
|
|
1986
|
+
stdout: string,
|
|
1987
|
+
stderr: string,
|
|
1988
|
+
timeoutMs: number,
|
|
1989
|
+
launcherTerminated: boolean,
|
|
1990
|
+
): Promise<BrokerLaunchResult> {
|
|
1991
|
+
return brokerHopTimeoutResult(hop, stdout, stderr, timeoutMs, launcherTerminated);
|
|
1992
|
+
}
|
|
1993
|
+
/** Test hook: preserves and reaps a verified POSIX trampoline reply at its deadline. */
|
|
1994
|
+
export function brokerTrampolineTimeoutResultForTest(
|
|
1995
|
+
trampoline: ChildProcess,
|
|
1996
|
+
stdout: string,
|
|
1997
|
+
timeoutMs: number,
|
|
1998
|
+
trampolineTerminated: boolean,
|
|
1999
|
+
): Promise<BrokerLaunchResult> {
|
|
2000
|
+
return brokerTrampolineTimeoutResult(trampoline, stdout, timeoutMs, trampolineTerminated);
|
|
2001
|
+
}
|
|
2002
|
+
/** Test hook: verifies broker exit evidence takes precedence over the launcher's status. */
|
|
2003
|
+
export function brokerStartupExitStatusForTest(
|
|
2004
|
+
childExitCode: number | null,
|
|
2005
|
+
childSignal: NodeJS.Signals | null,
|
|
2006
|
+
startupExitRecord: Pick<BrokerStartupExitRecord, "exitCode" | "signal"> | undefined,
|
|
2007
|
+
startupFailureMarker: Pick<BrokerStartupFailureMarker, "exitCode" | "signal"> | undefined,
|
|
2008
|
+
): { exitCode: number | null; signal: string | null } {
|
|
2009
|
+
return brokerStartupExitStatus(childExitCode, childSignal, startupExitRecord, startupFailureMarker);
|
|
2010
|
+
}
|