@agenthippo-ai/agenthippo 0.1.1 → 1.110.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/agentpack/index.js +2 -1
- package/out/agentpack/paths.js +23 -0
- package/out/analytics/dashboard/fleetDisplayUtils.js +3 -0
- package/out/analytics/dashboard/provider.js +13 -2
- package/out/analytics/dashboard/webview/panel.js +293 -82
- package/out/analytics/identity.js +0 -0
- package/out/analytics/service.js +23 -25
- package/out/analytics/sessionEvents.js +109 -22
- package/out/auth/authBaseUrl.js +59 -0
- package/out/auth/bootstrapApply.js +9 -10
- package/out/auth/brokerClient.js +49 -15
- package/out/auth/onboardingPrompt.js +112 -0
- package/out/auth/provider.js +9 -22
- package/out/auth/sessionStore.js +3 -40
- package/out/auth/signInFlow.js +2 -1
- package/out/auth-broker/config.js +54 -4
- package/out/auth-broker/server.js +152 -20
- package/out/auth-broker/store.js +25 -24
- package/out/authority/warrant.js +206 -0
- package/out/branding.js +27 -1
- package/out/cli/commands/ask.js +3 -0
- package/out/cli/commands/chat.js +3 -0
- package/out/cli/commands/conformance.js +249 -0
- package/out/cli/commands/data-plane.js +67 -0
- package/out/cli/commands/deploy.js +50 -10
- package/out/cli/commands/gateway.js +58 -0
- package/out/cli/commands/idp.js +85 -0
- package/out/cli/commands/pack.js +182 -3
- package/out/cli/commands/sandbox.js +332 -0
- package/out/cli/commands/serve.js +1165 -82
- package/out/cli/commands/store.js +12 -4
- package/out/cli/credential-context.js +42 -80
- package/out/cli/data-plane/broker.js +78 -0
- package/out/cli/data-plane/connector-host.js +121 -0
- package/out/cli/data-plane/egress.js +189 -0
- package/out/cli/data-plane/gateway.js +218 -0
- package/out/cli/data-plane/index.js +152 -0
- package/out/cli/data-plane/registry.js +89 -0
- package/out/cli/data-plane/remote.js +106 -0
- package/out/cli/data-plane/sidecar.js +161 -0
- package/out/cli/deploy/blueprints/registry.js +84 -0
- package/out/cli/deploy/layers/identity.js +172 -0
- package/out/cli/deploy-init.js +1182 -347
- package/out/cli/deps.js +106 -2
- package/out/cli/idp-init.js +338 -0
- package/out/cli/index.js +192 -23
- package/out/cli/pack-install-ref.js +15 -0
- package/out/cli/pack-loader.js +45 -3
- package/out/cli/pack-runtime.js +61 -0
- package/out/cli/runtime.js +44 -3
- package/out/cli/serve-hardening.js +139 -3
- package/out/cli/state-sync.js +16 -15
- package/out/cli/utils.js +12 -1
- package/out/engines/base.js +32 -3
- package/out/engines/claude.js +47 -8
- package/out/engines/codex.js +125 -22
- package/out/engines/custom.js +54 -12
- package/out/evidence/sessionShip.js +610 -0
- package/out/extension.js +13 -2
- package/out/gateway/bridge.js +40 -2
- package/out/gateway/client.js +34 -1
- package/out/gateway/installSource.js +68 -0
- package/out/gateway/lifecycle.js +185 -51
- package/out/gateway/outboundPaths.js +4 -1
- package/out/gateway/outboundSend.js +6 -6
- package/out/gateway/secretsPreflight.js +90 -0
- package/out/gateway/service.js +7 -2
- package/out/headless/index.js +40 -9
- package/out/live-e2e/mcp-demo/server.mjs +32 -0
- package/out/live-probe/connector.mjs +8 -0
- package/out/live-probe/env-report.mjs +8 -0
- package/out/mcp.js +115 -67
- package/out/pack/caps.js +271 -0
- package/out/pack/governor.js +309 -0
- package/out/pack/lease-schema.js +270 -0
- package/out/pack/lease.js +640 -0
- package/out/pack/reconcile.js +217 -0
- package/out/pack/signature.js +839 -0
- package/out/remote/handleRemoteChat.js +44 -3
- package/out/remote/remoteTrust.js +277 -0
- package/out/remote/serveClient.js +52 -5
- package/out/runner/host.js +131 -0
- package/out/runner/protocol.js +108 -0
- package/out/runner/remoteEngine.js +130 -0
- package/out/runtime/interfaces/edit-tracker.js +2 -0
- package/out/runtime/vscode/vscode-edit-tracker.js +2 -0
- package/out/sandbox/containerDriver.js +570 -0
- package/out/sandbox/processDriver.js +315 -0
- package/out/sandbox/sandboxRunner.js +191 -0
- package/out/sandbox/support.js +301 -0
- package/out/sandbox/types.js +55 -0
- package/out/scripts/agentide-litellm.sh +67 -5
- package/out/scripts/agentide_litellm_hooks_v1.py +182 -21
- package/out/scripts/analytics/agentide-litellm-otel.config.yaml +94 -118
- package/out/scripts/analytics/setup-analytics.ps1 +37 -13
- package/out/scripts/analytics/setup-analytics.sh +93 -13
- package/out/scripts/analytics/start-analytics.sh +86 -14
- package/out/scripts/gateway/setup-gateway.sh +72 -32
- package/out/shared/models.js +12 -111
- package/out/shared/providers.js +2 -2
- package/out/spotlight/install.js +7 -1
- package/out/store/index.js +41 -10
- package/out/sync.js +3 -1
- package/out/test/analytics.test.js +198 -34
- package/out/test/auth-baseurl.test.js +132 -0
- package/out/test/auth-bootstrapApply.test.js +27 -6
- package/out/test/auth-broker-hardening.test.js +227 -0
- package/out/test/auth-broker.test.js +112 -9
- package/out/test/blueprint-registry.test.js +105 -0
- package/out/test/branding.test.js +13 -0
- package/out/test/caps-governor.test.js +455 -0
- package/out/test/codex-new-file-survival.test.js +119 -0
- package/out/test/configuration-scopes.test.js +131 -0
- package/out/test/credential-context.test.js +23 -28
- package/out/test/custom-env-layering.test.js +151 -0
- package/out/test/data-plane-engine-wiring.test.js +250 -0
- package/out/test/data-plane.test.js +311 -0
- package/out/test/deploy-init.test.js +295 -4
- package/out/test/egress-broker.test.js +172 -0
- package/out/test/fleetDisplayUtils.test.js +4 -0
- package/out/test/gateway-client.test.js +28 -0
- package/out/test/gateway-install-source.test.js +111 -0
- package/out/test/gateway-lifecycle.test.js +1 -1
- package/out/test/gateway-outbound.test.js +7 -0
- package/out/test/gateway-secrets-preflight.test.js +127 -0
- package/out/test/gateway-session-routing.test.js +18 -0
- package/out/test/governor-hardening.test.js +217 -0
- package/out/test/headless-sync.test.js +22 -0
- package/out/test/idp-init.test.js +151 -0
- package/out/test/lease-signature.test.js +357 -0
- package/out/test/lease-trust-root.test.js +273 -0
- package/out/test/lease-validation.test.js +169 -0
- package/out/test/lease.test.js +114 -0
- package/out/test/mcp-obo.test.js +52 -27
- package/out/test/mcp.test.js +69 -3
- package/out/test/model-routing.test.js +133 -0
- package/out/test/pack-admission.test.js +216 -0
- package/out/test/pack-digest-parity.test.js +140 -0
- package/out/test/pack-reconcile.test.js +183 -0
- package/out/test/pack-signature-artifact.test.js +103 -0
- package/out/test/pack-signature.test.js +176 -0
- package/out/test/pack-trust-root.test.js +202 -0
- package/out/test/remote-trust.test.js +242 -0
- package/out/test/runner-protocol.test.js +249 -0
- package/out/test/sandbox-driver.test.js +553 -0
- package/out/test/sandbox-hardening.test.js +395 -0
- package/out/test/sandbox-runner.test.js +343 -0
- package/out/test/serve-gateway-turn.test.js +267 -0
- package/out/test/serve-hardening.test.js +125 -6
- package/out/test/serveClient.test.js +38 -5
- package/out/test/sessionShip.test.js +560 -0
- package/out/test/spotlightPublish.test.js +28 -0
- package/out/test/state-sync-peer.test.js +18 -5
- package/out/test/store-install-safe.test.js +69 -0
- package/out/test/tls-bypass.test.js +90 -0
- package/out/test/warrant.test.js +138 -0
- package/out/utils.js +73 -20
- package/package.json +1 -1
|
@@ -37,18 +37,38 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
37
37
|
})();
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.serveCommand = serveCommand;
|
|
40
|
+
exports.resolveGatewaySessionKey = resolveGatewaySessionKey;
|
|
41
|
+
exports.runGatewayTurn = runGatewayTurn;
|
|
40
42
|
/**
|
|
41
43
|
* Serve Command - HTTP Server
|
|
42
44
|
*/
|
|
45
|
+
const crypto = __importStar(require("crypto"));
|
|
43
46
|
const fs = __importStar(require("fs"));
|
|
47
|
+
const os = __importStar(require("os"));
|
|
44
48
|
const http = __importStar(require("http"));
|
|
45
49
|
const path = __importStar(require("path"));
|
|
50
|
+
const node_crypto_1 = require("node:crypto");
|
|
51
|
+
const sandboxRunner_1 = require("../../sandbox/sandboxRunner");
|
|
52
|
+
const deps_1 = require("../deps");
|
|
53
|
+
const signature_1 = require("../../pack/signature");
|
|
54
|
+
const reconcile_1 = require("../../pack/reconcile");
|
|
55
|
+
const index_1 = require("../../store/index");
|
|
56
|
+
const lease_1 = require("../../pack/lease");
|
|
57
|
+
const caps_1 = require("../../pack/caps");
|
|
58
|
+
const governor_1 = require("../../pack/governor");
|
|
59
|
+
const containerDriver_1 = require("../../sandbox/containerDriver");
|
|
60
|
+
const processDriver_1 = require("../../sandbox/processDriver");
|
|
61
|
+
const support_1 = require("../../sandbox/support");
|
|
46
62
|
const headless_1 = require("../../headless");
|
|
47
63
|
const pack_loader_1 = require("../pack-loader");
|
|
48
64
|
const utils_1 = require("../utils");
|
|
49
|
-
const
|
|
65
|
+
const pack_runtime_1 = require("../pack-runtime");
|
|
66
|
+
const sessionShip_1 = require("../../evidence/sessionShip");
|
|
50
67
|
const serve_hardening_1 = require("../serve-hardening");
|
|
51
68
|
const credential_context_1 = require("../credential-context");
|
|
69
|
+
const data_plane_1 = require("../data-plane");
|
|
70
|
+
const runtime_1 = require("../runtime");
|
|
71
|
+
const outboundSend_1 = require("../../gateway/outboundSend");
|
|
52
72
|
const sessionEvents_1 = require("../../analytics/sessionEvents");
|
|
53
73
|
const base_1 = require("../../engines/base");
|
|
54
74
|
const state_sync_1 = require("../state-sync");
|
|
@@ -58,6 +78,11 @@ const customRegistry_1 = require("../../engines/customRegistry");
|
|
|
58
78
|
const BANNER_LINE = '-'.repeat(62);
|
|
59
79
|
async function serveCommand(options) {
|
|
60
80
|
const apiConfig = (0, pack_loader_1.getApiConfig)();
|
|
81
|
+
if (apiConfig.useLiteLLM && !apiConfig.litellmApiKey) {
|
|
82
|
+
// Loud, because the failure mode it replaces was silent: traffic reached the gateway
|
|
83
|
+
// authenticated with whatever sat in the provider-key slot, with analytics disabled.
|
|
84
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} AGENTHIPPO_LITELLM_BASE_URL is set without AGENTHIPPO_LITELLM_API_KEY — gateway calls will authenticate as 'sk-dummy' (a strict gateway will 401). Provider keys are never sent to the gateway.`);
|
|
85
|
+
}
|
|
61
86
|
// Load agent pack (required)
|
|
62
87
|
let pack;
|
|
63
88
|
const syncLogger = options.verbose ? new headless_1.ConsoleLogger('[Sync]') : new headless_1.NoOpLogger();
|
|
@@ -81,25 +106,247 @@ async function serveCommand(options) {
|
|
|
81
106
|
const defaultModel = (0, headless_1.resolveModelAlias)(rawModel);
|
|
82
107
|
const rawCheapModel = pack.manifest.spec.cheapModel?.trim();
|
|
83
108
|
const claudeCheapModel = rawCheapModel ? (0, headless_1.resolveModelAlias)(rawCheapModel) : undefined;
|
|
84
|
-
|
|
85
|
-
|
|
109
|
+
// Engine-aware: a codex deploy needs an OpenAI (or LiteLLM) key; a claude deploy needs an
|
|
110
|
+
// Anthropic (or LiteLLM) key. Fail loudly at boot rather than 401 at the first model call.
|
|
111
|
+
if (!(0, pack_loader_1.hasCredentialForEngine)(defaultEngine, apiConfig)) {
|
|
112
|
+
const need = defaultEngine === 'claude' ? 'ANTHROPIC_API_KEY' : 'OPENAI_API_KEY';
|
|
113
|
+
console.error(`Error: No usable API key for engine "${defaultEngine}". Set ${need}, or configure the LiteLLM proxy (AGENTHIPPO_LITELLM_API_KEY + AGENTHIPPO_LITELLM_BASE_URL).`);
|
|
86
114
|
for (const line of (0, utils_1.getLiteLLMQuickstartHintLines)()) {
|
|
87
115
|
console.error(line);
|
|
88
116
|
}
|
|
89
117
|
process.exit(1);
|
|
90
118
|
}
|
|
91
|
-
|
|
119
|
+
const logger = options.verbose ? new headless_1.ConsoleLogger('[Server]') : new headless_1.NoOpLogger();
|
|
120
|
+
// ---------------------------------------------------------------------------------------
|
|
121
|
+
// Pack admission — BEFORE anything from the pack reaches the workspace.
|
|
122
|
+
//
|
|
123
|
+
// `syncPackToWorkspace` used to run here, hundreds of lines before the signature was checked,
|
|
124
|
+
// so a pack that was about to be refused had already materialised its skills, AGENTS.md and
|
|
125
|
+
// MCP server definitions into the workspace and into the derived .claude/.codex directories.
|
|
126
|
+
// The refusal then stopped the turn while the content it was refusing sat on disk, reachable
|
|
127
|
+
// by anything else pointed at that workspace. Verification moves ahead of the write.
|
|
128
|
+
// ---------------------------------------------------------------------------------------
|
|
129
|
+
let activePackDir = pack.packPath;
|
|
130
|
+
let activeVersion = pack.manifest.metadata.version ?? '0.0.0';
|
|
131
|
+
let packDigestValue;
|
|
132
|
+
/**
|
|
133
|
+
* Trusted pack-signing keys. NOT frozen at boot.
|
|
134
|
+
*
|
|
135
|
+
* A trust root that is CONFIGURED BUT UNREACHABLE (a 503 during a rolling restart, a DNS blip)
|
|
136
|
+
* is now distinguishable from one that was never configured, and it is retried on the lease
|
|
137
|
+
* poll — one transient fetch at the wrong second must fail the turn, not brick the process.
|
|
138
|
+
*/
|
|
139
|
+
let packTrust = await (0, signature_1.loadPackTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
|
|
140
|
+
const reportTrustRoot = (t, label = 'pack') => {
|
|
141
|
+
for (const w of t.warnings) {
|
|
142
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${w}`);
|
|
143
|
+
}
|
|
144
|
+
if (t.status === 'unavailable') {
|
|
145
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${label} trust root is configured but unusable: ${t.reason}`);
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
reportTrustRoot(packTrust);
|
|
149
|
+
/**
|
|
150
|
+
* Verified per turn, with a cheap change probe — see `createPackVerifier`. The RESULT is
|
|
151
|
+
* cached only as long as the pack tree looks untouched and never longer than the full-reverify
|
|
152
|
+
* interval; the REQUIREMENT is re-read from the lease on every poll, so an operator can turn
|
|
153
|
+
* enforcement on for a fleet already running.
|
|
154
|
+
*/
|
|
155
|
+
const packVerifier = (0, signature_1.createPackVerifier)();
|
|
156
|
+
let packSignature = { ok: false, kind: 'no-pack', reason: 'not verified' };
|
|
157
|
+
const verifyPackAt = (packDir, version) => {
|
|
158
|
+
packSignature = packVerifier.refresh({ packDir, pack: packId, packVersion: version, keys: packTrust.keys });
|
|
159
|
+
packDigestValue = packSignature.ok ? packSignature.claims.digest : undefined;
|
|
160
|
+
logger.info(packSignature.ok
|
|
161
|
+
? `[Pack] signature verified for ${packId}`
|
|
162
|
+
: `[Pack] unverified — ${packSignature.reason}`);
|
|
163
|
+
return packSignature;
|
|
164
|
+
};
|
|
165
|
+
/** The refusal an operator sees, with the remedy for THIS failure rather than a generic one. */
|
|
166
|
+
const packRefusalText = (result) => {
|
|
167
|
+
if (result.ok) {
|
|
168
|
+
return '';
|
|
169
|
+
}
|
|
170
|
+
// "Configured but unreachable" outranks whatever the verifier concluded: with no usable
|
|
171
|
+
// keys every signature fails as `no-trust-root`, and telling an operator to configure a
|
|
172
|
+
// trust root they already configured is the least useful thing we could say.
|
|
173
|
+
const kind = packTrust.status === 'unavailable' && result.kind === 'no-trust-root' ? 'trust-root-unavailable' : result.kind;
|
|
174
|
+
return (0, signature_1.describePackRefusal)({
|
|
175
|
+
kind,
|
|
176
|
+
detail: kind === 'trust-root-unavailable' ? (packTrust.reason ?? result.reason) : result.reason,
|
|
177
|
+
pack: packId,
|
|
178
|
+
packVersion: activeVersion,
|
|
179
|
+
packDir: activePackDir,
|
|
180
|
+
trustRootHint: packTrust.url ? `Trust root: ${packTrust.url} (${packTrust.reason ?? 'unavailable'})` : packTrust.reason,
|
|
181
|
+
});
|
|
182
|
+
};
|
|
183
|
+
verifyPackAt(activePackDir, activeVersion);
|
|
184
|
+
/**
|
|
185
|
+
* Signing REQUIRED by default.
|
|
186
|
+
*
|
|
187
|
+
* This is a deliberate break with the previous shipped behaviour, which logged
|
|
188
|
+
* `[Pack] unsigned or unverifiable — …` and then ran the pack anyway. Warn-and-run is not a
|
|
189
|
+
* gate; it is a log line next to an ungated code path, and it meant the default deployment
|
|
190
|
+
* shape executed whatever was in the pack directory.
|
|
191
|
+
*
|
|
192
|
+
* Tighten-only survives intact: the escape hatch below relaxes only this INSTANCE's default,
|
|
193
|
+
* and `resolveRequireSignedPack` checks the lease's demand first, so a control plane that sets
|
|
194
|
+
* `requireSignedPack` cannot be opted out of locally.
|
|
195
|
+
*/
|
|
196
|
+
const allowUnsignedPack = Boolean(options.allowUnsignedPack) || (0, signature_1.allowUnsignedPackFromEnv)();
|
|
197
|
+
if (!packSignature.ok) {
|
|
198
|
+
if (!allowUnsignedPack) {
|
|
199
|
+
console.error(`${utils_1.COLORS.red}Error: ${packRefusalText(packSignature)}${utils_1.COLORS.reset}`);
|
|
200
|
+
process.exit(1);
|
|
201
|
+
}
|
|
202
|
+
// Unmissable, every boot, at warn level regardless of --verbose.
|
|
203
|
+
console.warn(BANNER_LINE);
|
|
204
|
+
console.warn(`${utils_1.COLORS.yellow}!! RUNNING AN UNVERIFIED PACK — signature checking is DISABLED on this instance.${utils_1.COLORS.reset}`);
|
|
205
|
+
console.warn(`${utils_1.COLORS.yellow}!! ${packSignature.reason}${utils_1.COLORS.reset}`);
|
|
206
|
+
console.warn(`${utils_1.COLORS.yellow}!! Whatever is in ${activePackDir} will be executed, verified by nobody.${utils_1.COLORS.reset}`);
|
|
207
|
+
console.warn(`${utils_1.COLORS.yellow}!! This is a LOCAL DEVELOPMENT setting (AGENTHIPPO_ALLOW_UNSIGNED_PACK). Do not ship it.${utils_1.COLORS.reset}`);
|
|
208
|
+
console.warn(BANNER_LINE);
|
|
209
|
+
}
|
|
210
|
+
// Sync pack content to workspace — only now that the pack has been admitted.
|
|
92
211
|
const bundledPromptsDir = (0, utils_1.findBundledBasePromptsDir)();
|
|
93
212
|
await (0, headless_1.syncPackToWorkspace)(options.workspace, pack.packPath, packId, defaultEngine, syncLogger, bundledPromptsDir);
|
|
94
213
|
console.log(`OK: Loaded agent: ${pack.manifest.metadata.name} v${pack.manifest.metadata.version}`);
|
|
214
|
+
// Structural attribution (pack-standard WS2) — greppable for CISO/SRE boot audits.
|
|
215
|
+
console.log(`agenthippo pack=${pack.manifest.metadata.name}@${pack.manifest.metadata.version} engine=${defaultEngine}`);
|
|
95
216
|
// Resolve persistent session state (serve-v3 §A1: replaces the tmpdir default).
|
|
96
217
|
const { dir: storagePath, ephemeral } = (0, serve_hardening_1.resolveStateDir)(options.stateDir);
|
|
97
218
|
const bindHost = (0, serve_hardening_1.resolveBindHost)(options.host);
|
|
98
|
-
//
|
|
99
|
-
// so
|
|
100
|
-
|
|
101
|
-
|
|
219
|
+
// Who serve believes a caller to be decides whose session state and whose data credentials they
|
|
220
|
+
// reach, so an unpinned identity header is not a detail to leave in a docstring.
|
|
221
|
+
for (const line of (0, serve_hardening_1.identityHeaderAdvice)(process.env, bindHost)) {
|
|
222
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${line}`);
|
|
223
|
+
}
|
|
224
|
+
// Passed to createPackRunner below so the runner's own SessionEventWriter uses the same
|
|
225
|
+
// root — a fail-closed warrant denial lands in the same per-session JSONL as the tool events
|
|
226
|
+
// it precedes. The denial happens before runner.run() is ever called, so it needs its own
|
|
227
|
+
// writer here too. See resolveAnalyticsSessionsDir's docstring for the precedence.
|
|
228
|
+
const analyticsSessionsDir = (0, serve_hardening_1.resolveAnalyticsSessionsDir)(storagePath, ephemeral);
|
|
229
|
+
const sessionEventWriter = new sessionEvents_1.SessionEventWriter(analyticsSessionsDir);
|
|
102
230
|
const logWarrantDenied = (sessionKey) => (reason) => sessionEventWriter.append(sessionKey, (0, base_1.generateRequestId)(), 'warrant_denied', { reason });
|
|
231
|
+
// Data-plane gateway (secure-obo-design §3): when the pack declares spec.dataPlane,
|
|
232
|
+
// serve runs the connector broker-side and engines get a per-turn nonce instead of the
|
|
233
|
+
// credential. The raw data credential never crosses into the agent's trust domain.
|
|
234
|
+
const dataPlaneCfg = (0, data_plane_1.resolveDataPlaneConfig)(pack.manifest.spec, pack.packPath);
|
|
235
|
+
// EDGE-registration mode (P1 fix, obo-inprogress §2): the trusted edge (IdP Caddy)
|
|
236
|
+
// registers each turn with the sidecar and injects the nonce as a request header; serve
|
|
237
|
+
// merely adopts it. serve holds NO registration credential at all — nothing in this
|
|
238
|
+
// container's /proc can mint a nonce for an arbitrary userId. Takes precedence over the
|
|
239
|
+
// token-based sidecar mode below.
|
|
240
|
+
const edgeCfg = (0, data_plane_1.resolveEdgeDataPlane)();
|
|
241
|
+
const edgeDataPlane = edgeCfg ? new data_plane_1.EdgeDataPlane(edgeCfg) : null;
|
|
242
|
+
// Data-plane SIDECAR mode (secure-obo-design §7.6): when the connector runs in its OWN
|
|
243
|
+
// container, AGENTHIPPO_DATA_PLANE_CONTROL_URL/TOKEN + AGENTHIPPO_DATA_MCP_URL are set and
|
|
244
|
+
// serve holds NO backend base credential — it registers turns with the sidecar and hands the
|
|
245
|
+
// engine only the nonce. Takes precedence over an in-process connector.
|
|
246
|
+
const remoteCfg = edgeDataPlane ? null : (0, data_plane_1.resolveRemoteDataPlane)();
|
|
247
|
+
const remoteDataPlane = remoteCfg ? new data_plane_1.RemoteDataPlane(remoteCfg) : null;
|
|
248
|
+
// One shared registry: a single per-turn nonce grants BOTH the data-plane and egress
|
|
249
|
+
// capabilities (secure-obo-design §7.6 step 2). Created when either LOCAL broker will run.
|
|
250
|
+
const turnRegistry = ((dataPlaneCfg && !remoteDataPlane && !edgeDataPlane) || options.gatewayBridge) ? new data_plane_1.TurnRegistry() : null;
|
|
251
|
+
const auditEvent = (sessionKey, event, fields) => sessionEventWriter.append(sessionKey, (0, base_1.generateRequestId)(), event, fields);
|
|
252
|
+
let dataPlane = null;
|
|
253
|
+
if (edgeDataPlane) {
|
|
254
|
+
process.env[data_plane_1.DATA_MCP_URL_ENV] = edgeDataPlane.url;
|
|
255
|
+
// Belt-and-braces: edge mode must never see a register token, but if one leaked into the
|
|
256
|
+
// deploy env anyway, strip it before any engine child inherits it.
|
|
257
|
+
delete process.env.AGENTHIPPO_DATA_PLANE_TOKEN;
|
|
258
|
+
delete process.env.AGENTHIPPO_DATA_PLANE_CONTROL_URL;
|
|
259
|
+
console.log(`OK: Data plane EDGE-registered — gateway ${edgeDataPlane.url}; turn nonces are minted at the edge, serve holds no registration credential`);
|
|
260
|
+
}
|
|
261
|
+
else if (remoteDataPlane) {
|
|
262
|
+
process.env[data_plane_1.DATA_MCP_URL_ENV] = remoteDataPlane.url;
|
|
263
|
+
// The control-API token is captured in the RemoteDataPlane (serve memory). Strip it from
|
|
264
|
+
// the process env so no engine child inherits it — otherwise the agent could register a
|
|
265
|
+
// turn as ANY user and read their rows (forge-identity escalation). The engine keeps the
|
|
266
|
+
// gateway URL + its own per-turn nonce, which is all it needs (secure-obo-design §7.6).
|
|
267
|
+
delete process.env.AGENTHIPPO_DATA_PLANE_TOKEN;
|
|
268
|
+
console.log(`OK: Data plane SIDECAR — gateway ${remoteDataPlane.url}; base credentials stay in the connector container, not here`);
|
|
269
|
+
}
|
|
270
|
+
else if (dataPlaneCfg) {
|
|
271
|
+
try {
|
|
272
|
+
dataPlane = await data_plane_1.DataPlane.start(dataPlaneCfg, {
|
|
273
|
+
logger: options.verbose ? new headless_1.ConsoleLogger('[DataPlane]') : new headless_1.NoOpLogger(),
|
|
274
|
+
audit: auditEvent,
|
|
275
|
+
registry: turnRegistry ?? undefined,
|
|
276
|
+
});
|
|
277
|
+
// Boot-stable process env: engines resolve ${env:AGENTHIPPO_DATA_MCP_URL} from it,
|
|
278
|
+
// so codex's config.toml stays byte-identical across turns. The NONCE is never set
|
|
279
|
+
// here — it rides the per-spawn child env only.
|
|
280
|
+
process.env[data_plane_1.DATA_MCP_URL_ENV] = dataPlane.url;
|
|
281
|
+
console.log(`OK: Data gateway (provider=${dataPlane.broker.provider.name}) on loopback — connector: ${dataPlaneCfg.connectorPath}`);
|
|
282
|
+
}
|
|
283
|
+
catch (err) {
|
|
284
|
+
console.error(`${utils_1.COLORS.red}Error: failed to start the data-plane gateway:${utils_1.COLORS.reset} ${err instanceof Error ? err.message : err}`);
|
|
285
|
+
process.exit(1);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
// Egress broker (secure-obo-design §7.6): capture the gateway token broker-side BEFORE any
|
|
289
|
+
// engine spawns, so it can be stripped from the process env the children inherit. The broker
|
|
290
|
+
// starts alongside the gateway bridge below; the token is held only in this closure.
|
|
291
|
+
// Captured UNCONDITIONALLY, and the env strip is immediate (serve-v4 §10 hole 2): the strip
|
|
292
|
+
// used to live inside the broker branch, so a serve without --gateway-bridge kept the token
|
|
293
|
+
// in process.env and every engine child (they all spread it) inherited a credential nothing
|
|
294
|
+
// in this deployment would ever legitimately use.
|
|
295
|
+
const brokerGatewayToken = (0, runtime_1.readOpenClawGatewayToken)();
|
|
296
|
+
delete process.env.OPENCLAW_GATEWAY_TOKEN;
|
|
297
|
+
let egressBroker = null;
|
|
298
|
+
// Mint one nonce per turn and assemble the per-spawn child env. Returns null outside any
|
|
299
|
+
// broker (no capability to grant). The URLs ride process.env (boot-stable) for config
|
|
300
|
+
// invariance; only the nonce is per-spawn.
|
|
301
|
+
const registerServeTurn = async (reg) => {
|
|
302
|
+
// Edge mode: the trusted edge already registered this turn and injected the nonce as a
|
|
303
|
+
// request header — adopt it. No edge nonce (misrouted request, or a channel turn that
|
|
304
|
+
// never traversed the edge) ⇒ no data capability, fail closed at the gateway.
|
|
305
|
+
if (edgeDataPlane) {
|
|
306
|
+
const edgeNonce = reg.edgeNonce?.trim();
|
|
307
|
+
if (!edgeNonce) {
|
|
308
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} edge data plane active but no ${data_plane_1.TURN_NONCE_HEADER} header on this turn — data tools will fail closed (check the edge route's forward_auth)`);
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
const r = edgeDataPlane.adoptTurn(edgeNonce);
|
|
312
|
+
const env = { ...r.env };
|
|
313
|
+
if (egressBroker) {
|
|
314
|
+
env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
|
|
315
|
+
}
|
|
316
|
+
return { nonce: r.nonce, env, release: r.release };
|
|
317
|
+
}
|
|
318
|
+
// Sidecar: register with the connector container; serve never sees a base credential.
|
|
319
|
+
if (remoteDataPlane) {
|
|
320
|
+
const r = await remoteDataPlane.registerTurn(reg);
|
|
321
|
+
const env = { ...r.env };
|
|
322
|
+
if (egressBroker) {
|
|
323
|
+
env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
|
|
324
|
+
}
|
|
325
|
+
return { nonce: r.nonce, env, release: r.release };
|
|
326
|
+
}
|
|
327
|
+
if (!turnRegistry) {
|
|
328
|
+
return null;
|
|
329
|
+
}
|
|
330
|
+
const { nonce, release } = turnRegistry.register(reg);
|
|
331
|
+
const env = { [data_plane_1.TURN_NONCE_ENV]: nonce };
|
|
332
|
+
if (dataPlane) {
|
|
333
|
+
env[data_plane_1.DATA_MCP_URL_ENV] = dataPlane.url;
|
|
334
|
+
}
|
|
335
|
+
if (egressBroker) {
|
|
336
|
+
env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
|
|
337
|
+
}
|
|
338
|
+
return { nonce, env, release };
|
|
339
|
+
};
|
|
340
|
+
// Edge-registered turn nonce, injected by the trusted edge per chat request (edge mode only —
|
|
341
|
+
// otherwise ignored). Trusted like the pinned identity header: only the in-network edge can
|
|
342
|
+
// reach serve, and a forged value fails closed at the gateway (unknown nonce ⇒ 401).
|
|
343
|
+
const edgeTurnNonce = (headers) => {
|
|
344
|
+
if (!edgeDataPlane) {
|
|
345
|
+
return undefined;
|
|
346
|
+
}
|
|
347
|
+
const v = headers[data_plane_1.TURN_NONCE_HEADER];
|
|
348
|
+
return Array.isArray(v) ? v[0] : v;
|
|
349
|
+
};
|
|
103
350
|
const turnLimiter = new serve_hardening_1.TurnLimiter(options.maxTurnsPerSession ?? 0);
|
|
104
351
|
// Per-identity request-rate cap (req/min). Keyed on the trusted user id (falling back to
|
|
105
352
|
// client IP) so rotating session_id can't bypass it. AGENTHIPPO_SERVE_RATE_LIMIT=0 disables.
|
|
@@ -150,18 +397,22 @@ async function serveCommand(options) {
|
|
|
150
397
|
await (0, state_sync_1.ensureSessionLocal)(s3State, storagePath, sessionKey, stateLog);
|
|
151
398
|
}
|
|
152
399
|
};
|
|
153
|
-
const logger = options.verbose ? new headless_1.ConsoleLogger('[Server]') : new headless_1.NoOpLogger();
|
|
154
400
|
// Find SDK CLI paths
|
|
155
401
|
const claudeCodePath = (0, utils_1.findClaudeCodePath)();
|
|
156
402
|
const codexPath = (0, utils_1.findCodexPath)();
|
|
157
|
-
// Validate required CLI is available
|
|
158
|
-
|
|
403
|
+
// Validate required CLI is available — but ONLY when this process will run the engine.
|
|
404
|
+
// A container sandbox runs the engine inside its own image, so requiring the SDK on the host
|
|
405
|
+
// is wrong and refuses to boot on exactly the minimal host a tier-2 deployment wants
|
|
406
|
+
// (observed live: a Linux host with no npm/codex could not start `--sandbox container`).
|
|
407
|
+
// The process driver shares this host, so it still needs the CLI present.
|
|
408
|
+
const engineRunsInSandbox = options.sandbox === 'container';
|
|
409
|
+
if (!engineRunsInSandbox && defaultEngine === 'claude' && !claudeCodePath) {
|
|
159
410
|
console.error(`${utils_1.COLORS.red}Error: Claude Code CLI not found.${utils_1.COLORS.reset}`);
|
|
160
411
|
console.error(' Install: npm install -g @anthropic-ai/claude-code');
|
|
161
412
|
console.error(' Or set CLAUDE_CODE_PATH environment variable');
|
|
162
413
|
process.exit(1);
|
|
163
414
|
}
|
|
164
|
-
if (defaultEngine === 'codex' && !codexPath) {
|
|
415
|
+
if (!engineRunsInSandbox && defaultEngine === 'codex' && !codexPath) {
|
|
165
416
|
console.error(`${utils_1.COLORS.red}Error: Codex CLI not found.${utils_1.COLORS.reset}`);
|
|
166
417
|
console.error(' Install: npm install -g @openai/codex');
|
|
167
418
|
console.error(' Or set CODEX_PATH environment variable');
|
|
@@ -172,32 +423,589 @@ async function serveCommand(options) {
|
|
|
172
423
|
const permissions = {
|
|
173
424
|
fileAccess: (0, pack_loader_1.getFileAccess)(pack),
|
|
174
425
|
};
|
|
175
|
-
|
|
426
|
+
// R-3 policy provenance (PLAN/compliance.md §3.1): remember WHICH policy is in force and
|
|
427
|
+
// announce changes. `policy_applied` marks the boundary; every turn between two boundaries
|
|
428
|
+
// carries the version via the runner's provenance stamp below, so "was the cap enforced?"
|
|
429
|
+
// stays answerable after anyone edits a cap.
|
|
430
|
+
let lastPolicyVersion;
|
|
431
|
+
let provenanceAnnounced = false;
|
|
432
|
+
const notePolicy = (lease) => {
|
|
433
|
+
const v = (0, lease_1.policyVersion)(lease);
|
|
434
|
+
// Announce once at boot even with no lease (unmanaged deployments still have a digest,
|
|
435
|
+
// an engine build and a tier), then again on every policy change. This event is the
|
|
436
|
+
// PROVENANCE DIMENSION: turns don't repeat these facts — the evidence index joins a
|
|
437
|
+
// turn to the latest announcement for its instance at or before the turn's timestamp.
|
|
438
|
+
if (provenanceAnnounced && v === lastPolicyVersion) {
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
provenanceAnnounced = true;
|
|
442
|
+
lastPolicyVersion = v;
|
|
443
|
+
void sessionEventWriter.append('serve', (0, base_1.generateRequestId)(), 'policy_applied', {
|
|
444
|
+
instance_id: instanceId,
|
|
445
|
+
policy_version: v ?? null,
|
|
446
|
+
pack_digest: packDigestValue ?? null,
|
|
447
|
+
engine_version: (0, deps_1.installedEngineVersion)(defaultEngine) ?? null,
|
|
448
|
+
tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : null,
|
|
449
|
+
enabled: lease?.enabled ?? null,
|
|
450
|
+
pinned_version: lease?.version ?? null,
|
|
451
|
+
caps: lease?.caps ?? null,
|
|
452
|
+
deny_count: lease?.deny?.length ?? 0,
|
|
453
|
+
});
|
|
454
|
+
};
|
|
455
|
+
// Same factory a sandbox uses (cli/pack-runtime.ts) — built after activeVersion is known.
|
|
456
|
+
let inProcessRunner;
|
|
457
|
+
/**
|
|
458
|
+
* Where sandboxes read the pack from.
|
|
459
|
+
*
|
|
460
|
+
* With --pull-packs this is a SYMLINK (`<state>/packs/current`), never the resolved directory:
|
|
461
|
+
* the sandbox drivers capture this path once at construction, so a stable name whose target is
|
|
462
|
+
* swapped is what lets a later version reach new sandboxes without rebuilding the runner. A
|
|
463
|
+
* sandbox already starting keeps the directory it opened, which is exactly the drain semantics
|
|
464
|
+
* version convergence already has.
|
|
465
|
+
*/
|
|
466
|
+
const packsRoot = path.join(storagePath, 'packs');
|
|
467
|
+
const storeDownloader = async (slug, version) => {
|
|
468
|
+
const files = await (0, index_1.downloadFromStore)({ type: 'pack', slug, version });
|
|
469
|
+
return files.map((f) => ({ path: f.path, content: f.content }));
|
|
470
|
+
};
|
|
471
|
+
/** Converge to the pinned version if we can; never take the fleet down trying. */
|
|
472
|
+
const reconcileToPin = async (pinned) => {
|
|
473
|
+
if (!options.pullPacks || !pinned || pinned === activeVersion) {
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
const res = await (0, reconcile_1.reconcilePinnedPack)({
|
|
477
|
+
pack: packId,
|
|
478
|
+
currentVersion: activeVersion,
|
|
479
|
+
pinnedVersion: pinned,
|
|
480
|
+
packsRoot,
|
|
481
|
+
download: storeDownloader,
|
|
482
|
+
publicKeysByKid: packTrust.keys,
|
|
483
|
+
logger: { info: (m) => logger.info(m), warn: (m) => logger.info(m) },
|
|
484
|
+
});
|
|
485
|
+
if (res.ok) {
|
|
486
|
+
activeVersion = res.version;
|
|
487
|
+
activePackDir = res.packDir;
|
|
488
|
+
verifyPackAt(activePackDir, activeVersion);
|
|
489
|
+
logger.info(`[Pack] now serving ${packId}@${activeVersion} from ${res.packDir}`);
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
// Deliberately not fatal: the lease fails OPEN on transport, and a bad tag or a store
|
|
493
|
+
// outage must not become a fleet-wide stop. The instance keeps reporting the version it is
|
|
494
|
+
// really running, so the console shows "pinned to X, running Y" rather than an outage.
|
|
495
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} cannot converge to ${packId}@${pinned}: ${res.reason}. Still serving @${activeVersion}.`);
|
|
496
|
+
};
|
|
497
|
+
const warrantKey = (0, support_1.loadWarrantKeypair)(logger);
|
|
498
|
+
// Lease: kill switch + version pin. No URL configured ⇒ unmanaged deployment, always admits.
|
|
499
|
+
/** Set once the sandbox runner exists; the lease-poll identity reads it lazily. */
|
|
500
|
+
let sandboxRunnerRef;
|
|
501
|
+
// Stable across restarts, unique per instance: an operator watching the console must see one
|
|
502
|
+
// row per instance that survives a redeploy, not a new row each boot (and not one row per NAT
|
|
503
|
+
// address for a whole fleet). Persisted beside the instance's own state.
|
|
504
|
+
const instanceId = (() => {
|
|
505
|
+
try {
|
|
506
|
+
const file = path.join((0, serve_hardening_1.resolveStateDir)(options.stateDir).dir, 'instance-id');
|
|
507
|
+
if (fs.existsSync(file)) {
|
|
508
|
+
return fs.readFileSync(file, 'utf-8').trim();
|
|
509
|
+
}
|
|
510
|
+
const id = `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`;
|
|
511
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
512
|
+
fs.writeFileSync(file, id);
|
|
513
|
+
return id;
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
// Never let bookkeeping stop a boot; a per-process id still beats collapsing by address.
|
|
517
|
+
return `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`;
|
|
518
|
+
}
|
|
519
|
+
})();
|
|
520
|
+
/** Lazily read by the lease-poll identity, which is built before the pack path is resolved. */
|
|
521
|
+
const activeVersionRef = { get: () => pack?.manifest.metadata.version };
|
|
522
|
+
// ONE constant for the poll interval and everything derived from it. The cache used to default
|
|
523
|
+
// to 60s while every poll defaulted to 30s and the banner printed 30 — so the real kill latency
|
|
524
|
+
// was up to 3x what an operator was told.
|
|
525
|
+
const leasePollSeconds = (0, lease_1.resolveLeasePollSeconds)(options.leasePollSeconds);
|
|
526
|
+
/**
|
|
527
|
+
* Trusted LEASE-signing keys — a separate root from the pack one.
|
|
528
|
+
*
|
|
529
|
+
* These used to be the same keyring (`signingKeys: () => packTrust.keys`), which made signed
|
|
530
|
+
* leases unreachable end to end: the control plane publishes the lease public key under purpose
|
|
531
|
+
* `lease-signing` and refuses to register one kid under two purposes, so a correctly-configured
|
|
532
|
+
* plane and a correctly-configured serve could never name the same key. With `requireSignedLease`
|
|
533
|
+
* sticky and no plane-side undo, that disagreement latches an instance into refusing turns.
|
|
534
|
+
*
|
|
535
|
+
* A DERIVED root that comes back `unavailable` is the normal state of every plane that publishes
|
|
536
|
+
* no lease-signing key, so it is only reported at boot when an operator named one outright.
|
|
537
|
+
* Otherwise `leaseSigningKeys` says everything there is to say, and only once a signed lease has
|
|
538
|
+
* actually arrived.
|
|
539
|
+
*/
|
|
540
|
+
let leaseTrust = await (0, signature_1.loadLeaseTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
|
|
541
|
+
if (leaseTrust.explicitlyConfigured) {
|
|
542
|
+
reportTrustRoot(leaseTrust, 'lease');
|
|
543
|
+
}
|
|
544
|
+
let leaseTrustFallbackWarned = false;
|
|
545
|
+
/**
|
|
546
|
+
* BACK-COMPAT, deliberately not a cutover.
|
|
547
|
+
*
|
|
548
|
+
* A deployment that verifies signed leases against the pack keyring today keeps working: with no
|
|
549
|
+
* lease root named outright, the pack keys stay in play and the (possibly empty) derived lease
|
|
550
|
+
* keys are added to them. The warning fires ONCE, and only when a signed lease is actually being
|
|
551
|
+
* verified, so a deployment that never uses signed leases never sees it.
|
|
552
|
+
*
|
|
553
|
+
* Not a hard switch because `requireSignedLease` latches per instance and the plane has no undo:
|
|
554
|
+
* an instance that stopped recognising its lease key would refuse turns and could not be talked
|
|
555
|
+
* out of it from the console.
|
|
556
|
+
*/
|
|
557
|
+
const leaseSigningKeys = () => {
|
|
558
|
+
const selected = (0, signature_1.selectLeaseSigningKeys)(leaseTrust, packTrust);
|
|
559
|
+
if (selected.packFallback && !leaseTrustFallbackWarned) {
|
|
560
|
+
leaseTrustFallbackWarned = true;
|
|
561
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} verifying signed leases against the PACK trust root — ` +
|
|
562
|
+
'no lease trust root is configured on this instance. The control plane publishes lease keys under ' +
|
|
563
|
+
'purpose `lease-signing` and refuses to register one kid under two purposes, so this stops working ' +
|
|
564
|
+
'the moment the plane rotates to a dedicated lease key — and requireSignedLease is sticky with no ' +
|
|
565
|
+
'plane-side undo, so that failure cannot be fixed from the console. Set ' +
|
|
566
|
+
`${signature_1.LEASE_JWKS_URL_ENV}=https://<plane>/.well-known/jwks.json?purpose=lease-signing ` +
|
|
567
|
+
`(or ${signature_1.LEASE_SIGNING_KEYS_ENV} with inline {"<kid>":"<public PEM>"} for an air-gapped install).` +
|
|
568
|
+
(leaseTrust.reason ? ` The derived lease trust root is unusable: ${leaseTrust.reason}` : ''));
|
|
569
|
+
}
|
|
570
|
+
return selected.keys;
|
|
571
|
+
};
|
|
572
|
+
const leaseSource = (0, lease_1.createLeaseSource)({
|
|
573
|
+
url: process.env.AGENTHIPPO_LEASE_URL,
|
|
574
|
+
// The poll IS the registration — see LeaseSourceOptions.identity.
|
|
575
|
+
identity: () => ({
|
|
576
|
+
instance: instanceId,
|
|
577
|
+
pack: packId,
|
|
578
|
+
// The version actually being served: after a pull this is the pinned one, and the
|
|
579
|
+
// console's whole convergence view depends on it being the truth rather than boot state.
|
|
580
|
+
version: activeVersionRef.get(),
|
|
581
|
+
tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
|
|
582
|
+
sandboxes: sandboxRunnerRef?.sandboxes().length,
|
|
583
|
+
// Which engine, and which BUILD of it. Pack version alone cannot explain a fleet-wide
|
|
584
|
+
// behaviour change that came from an engine upgrade, and the CLI is not engine-version
|
|
585
|
+
// agnostic, so the console needs both to answer "what is actually running?".
|
|
586
|
+
engine: defaultEngine,
|
|
587
|
+
engineversion: (0, deps_1.installedEngineVersion)(defaultEngine),
|
|
588
|
+
}),
|
|
589
|
+
// Derived from the poll interval by ONE function, so the two can no longer drift. Worst-case
|
|
590
|
+
// kill latency is poll + cache, and that is the number the boot banner prints.
|
|
591
|
+
cacheSeconds: (0, lease_1.leaseCacheSecondsFor)(leasePollSeconds),
|
|
592
|
+
// Signed-lease verification uses the LEASE trust root, falling back to the pack one while no
|
|
593
|
+
// lease root is configured. Read through a closure so a keyring that arrives on a later poll
|
|
594
|
+
// is picked up without rebuilding this.
|
|
595
|
+
signingKeys: () => leaseSigningKeys(),
|
|
596
|
+
requireSignedLease: options.requireSignedLease,
|
|
597
|
+
instanceId,
|
|
598
|
+
...(options.leaseMaxStalenessSeconds !== undefined ? { maxStalenessSeconds: options.leaseMaxStalenessSeconds } : {}),
|
|
599
|
+
logger: {
|
|
600
|
+
// Lease warnings are security-relevant (unsigned lease, staleness ceiling, insecure URL)
|
|
601
|
+
// and must not be hidden behind --verbose.
|
|
602
|
+
info: (m) => logger.info(m),
|
|
603
|
+
warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
|
|
604
|
+
},
|
|
605
|
+
});
|
|
606
|
+
/**
|
|
607
|
+
* Retry the trust root while it is missing, and re-verify once it arrives.
|
|
608
|
+
*
|
|
609
|
+
* Only while MISSING: a loaded trust root is not re-fetched, so key rotation still needs the
|
|
610
|
+
* documented restart and a compromised plane cannot swap the trust root under a running
|
|
611
|
+
* instance mid-flight. This exists solely so a transient failure at boot is transient — the
|
|
612
|
+
* `unavailable` status is exactly what makes "transient" distinguishable from "not configured".
|
|
613
|
+
*/
|
|
614
|
+
const retrySigningKeys = async () => {
|
|
615
|
+
if (packTrust.status !== 'configured') {
|
|
616
|
+
const next = await (0, signature_1.loadPackTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
|
|
617
|
+
if (next.status === 'configured') {
|
|
618
|
+
packTrust = next;
|
|
619
|
+
reportTrustRoot(next);
|
|
620
|
+
logger.info(`[Pack] signing keys now available (${Object.keys(next.keys).length}); re-verifying`);
|
|
621
|
+
verifyPackAt(activePackDir, activeVersion);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
// The lease root gets the SAME rule, not a better one: retried only while missing, never
|
|
625
|
+
// re-fetched once configured. Improving one and not the other would leave two roots with two
|
|
626
|
+
// rotation stories, which is how the rotation runbook silently stops being true for half of it.
|
|
627
|
+
if (leaseTrust.status !== 'configured') {
|
|
628
|
+
const next = await (0, signature_1.loadLeaseTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
|
|
629
|
+
if (next.status === 'configured') {
|
|
630
|
+
leaseTrust = next;
|
|
631
|
+
if (next.explicitlyConfigured) {
|
|
632
|
+
reportTrustRoot(next, 'lease');
|
|
633
|
+
}
|
|
634
|
+
logger.info(`[Lease] lease-signing keys now available (${Object.keys(next.keys).length})`);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
};
|
|
638
|
+
// Record the boot policy and watch for changes on the lease-poll cadence. The lease source
|
|
639
|
+
// caches, so this adds no extra fetches — it re-reads whatever the poll already trusts.
|
|
640
|
+
void leaseSource.get().then(notePolicy).catch(() => undefined);
|
|
641
|
+
{
|
|
642
|
+
const everyMs = leasePollSeconds * 1000;
|
|
643
|
+
const policyTimer = setInterval(() => {
|
|
644
|
+
void leaseSource.get().then(notePolicy).catch(() => undefined);
|
|
645
|
+
void retrySigningKeys().catch(() => undefined);
|
|
646
|
+
}, everyMs);
|
|
647
|
+
policyTimer.unref?.();
|
|
648
|
+
}
|
|
649
|
+
if (options.pullPacks) {
|
|
650
|
+
fs.mkdirSync(packsRoot, { recursive: true });
|
|
651
|
+
// Materialise whatever we boot with, so `current` is valid before any pin is considered and
|
|
652
|
+
// the drivers can capture one stable path.
|
|
653
|
+
const bootDir = path.join(packsRoot, activeVersion);
|
|
654
|
+
if (!fs.existsSync(bootDir)) {
|
|
655
|
+
fs.cpSync(pack.packPath, bootDir, { recursive: true });
|
|
656
|
+
}
|
|
657
|
+
(0, reconcile_1.promoteCurrent)(packsRoot, bootDir);
|
|
658
|
+
activePackDir = path.join(packsRoot, reconcile_1.CURRENT_LINK);
|
|
659
|
+
await reconcileToPin((await leaseSource.get())?.version);
|
|
660
|
+
// The pack is now read through `current`, which is a different path than the one verified
|
|
661
|
+
// above. Re-verify so `packSignature` describes the tree that will actually run.
|
|
662
|
+
verifyPackAt(activePackDir, activeVersion);
|
|
663
|
+
}
|
|
664
|
+
activeVersionRef.get = () => activeVersion;
|
|
665
|
+
inProcessRunner = (0, pack_runtime_1.createPackRunner)({
|
|
176
666
|
storagePath,
|
|
667
|
+
analyticsSessionsDir,
|
|
177
668
|
packId,
|
|
178
|
-
packVersion:
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
669
|
+
packVersion: activeVersion,
|
|
670
|
+
provenance: () => ({
|
|
671
|
+
policyVersion: lastPolicyVersion,
|
|
672
|
+
packDigest: packDigestValue,
|
|
673
|
+
engineVersion: (0, deps_1.installedEngineVersion)(defaultEngine),
|
|
674
|
+
tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
|
|
675
|
+
instanceId,
|
|
676
|
+
}),
|
|
677
|
+
workspaceRoot: options.workspace,
|
|
678
|
+
credentials: {
|
|
679
|
+
apiKey: apiConfig.apiKey,
|
|
680
|
+
baseUrl: apiConfig.baseUrl,
|
|
681
|
+
useLiteLLM: apiConfig.useLiteLLM,
|
|
682
|
+
litellmApiKey: apiConfig.litellmApiKey,
|
|
683
|
+
openaiApiKey: apiConfig.openaiApiKey,
|
|
684
|
+
anthropicApiKey: apiConfig.anthropicApiKey,
|
|
685
|
+
},
|
|
183
686
|
logger,
|
|
184
|
-
freshState: false,
|
|
185
|
-
claudeCodeExecutablePath: claudeCodePath,
|
|
186
|
-
codexExecutablePath: codexPath,
|
|
187
687
|
permissionHandler,
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
workspaceRoot: options.workspace,
|
|
191
|
-
agentId: packId,
|
|
192
|
-
baseEnv: process.env,
|
|
193
|
-
}),
|
|
194
|
-
...(claudeCheapModel ? { engineSettings: { claudeCheapModel } } : {}),
|
|
688
|
+
claudeCheapModel,
|
|
689
|
+
omitGatewayToken: true,
|
|
195
690
|
});
|
|
691
|
+
// Converge on the same cadence the fleet already polls, so pinning a version in the console
|
|
692
|
+
// reaches instances without a redeploy. Errors are swallowed by reconcileToPin itself.
|
|
693
|
+
if (options.pullPacks) {
|
|
694
|
+
const every = leasePollSeconds * 1000;
|
|
695
|
+
const timer = setInterval(() => {
|
|
696
|
+
void leaseSource.get().then((l) => reconcileToPin(l?.version)).catch(() => undefined);
|
|
697
|
+
}, every);
|
|
698
|
+
timer.unref?.();
|
|
699
|
+
}
|
|
700
|
+
// Container sandboxes got NO filesystem mounts at all — the driver honours `fs`, but serve never
|
|
701
|
+
// passed it, so a sandbox's `/state/workspace` was container-private storage. Everything the
|
|
702
|
+
// agent wrote was invisible on the host and lost when the sandbox died. The process driver needs
|
|
703
|
+
// no mounts: tier 0 shares this host's filesystem already.
|
|
704
|
+
//
|
|
705
|
+
// The workspace is PER SESSION unless --shared-workspace says otherwise. One host directory
|
|
706
|
+
// mounted writable into every session's sandbox is a shared disk between users of the same
|
|
707
|
+
// agent: whatever session A writes (a query export, a downloaded file), session B reads on its
|
|
708
|
+
// next turn. That silently contradicts "one disposable sandbox per session" for everything that
|
|
709
|
+
// matters, since the container boundary does not extend to a directory both containers mount.
|
|
710
|
+
const sharedWorkspace = Boolean(options.sharedWorkspace);
|
|
711
|
+
const sandboxMounts = options.sandbox === 'container'
|
|
712
|
+
? {
|
|
713
|
+
// serve's resolved pack, not the image's baked copy — the two cannot then disagree
|
|
714
|
+
// about which pack version is running.
|
|
715
|
+
readonly: [`${activePackDir}:${containerDriver_1.SANDBOX_PACK_DIR}`],
|
|
716
|
+
writable: [`${options.workspace}:${containerDriver_1.SANDBOX_WORKSPACE_DIR}`],
|
|
717
|
+
}
|
|
718
|
+
: undefined;
|
|
719
|
+
if (sandboxMounts && sharedWorkspace) {
|
|
720
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} --shared-workspace: every session's sandbox mounts ` +
|
|
721
|
+
`${options.workspace} read-write, so sessions can read and overwrite each other's files. ` +
|
|
722
|
+
`Only safe when all sessions belong to one trust domain.`);
|
|
723
|
+
}
|
|
724
|
+
// serve wrote that workspace as root; the sandbox runs non-root and must be able to write it too.
|
|
725
|
+
// Boot-time and once, before the first turn can arrive.
|
|
726
|
+
let sandboxImageUser;
|
|
727
|
+
if (sandboxMounts && options.sandboxImage) {
|
|
728
|
+
await (0, support_1.alignSandboxWorkspaceOwnership)({
|
|
729
|
+
exec: support_1.dockerExec,
|
|
730
|
+
image: options.sandboxImage,
|
|
731
|
+
writable: sandboxMounts.writable,
|
|
732
|
+
// A failure here is what an operator must see even without --verbose; success is noise.
|
|
733
|
+
logger: { info: (m) => logger.info(m), warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`) },
|
|
734
|
+
});
|
|
735
|
+
sandboxImageUser = await (0, support_1.resolveSandboxImageUser)(support_1.dockerExec, options.sandboxImage);
|
|
736
|
+
}
|
|
737
|
+
// A session's engine state must outlive its sandboxes or the conversation restarts every time a
|
|
738
|
+
// sandbox is reaped (serve-v4-sandbox §8.4). Per session, under serve's own state dir, so the
|
|
739
|
+
// existing per-session sync/prune machinery already covers it.
|
|
740
|
+
const sandboxStateDirFor = (sessionKey) => (0, support_1.ensureSandboxStateDir)(path.join(storagePath, 'sandbox-state', (0, state_sync_1.safeSessionId)(sessionKey)), sandboxImageUser);
|
|
741
|
+
/**
|
|
742
|
+
* Mounts for ONE session's sandbox: the pack read-only, and a workspace only this session sees.
|
|
743
|
+
*
|
|
744
|
+
* The per-session workspace lives beside that session's engine state, so the same reap/prune
|
|
745
|
+
* machinery already covers it and the agent's output stays visible on the host — the property
|
|
746
|
+
* the shared mount was added for — without also being visible to every other session.
|
|
747
|
+
*/
|
|
748
|
+
const sandboxFsFor = async (sessionKey) => {
|
|
749
|
+
if (!sandboxMounts) {
|
|
750
|
+
return { readonly: [], writable: [] };
|
|
751
|
+
}
|
|
752
|
+
if (sharedWorkspace) {
|
|
753
|
+
return sandboxMounts;
|
|
754
|
+
}
|
|
755
|
+
const dir = (0, support_1.ensureSandboxStateDir)(path.join(storagePath, 'sandbox-workspace', (0, state_sync_1.safeSessionId)(sessionKey)), sandboxImageUser);
|
|
756
|
+
return { readonly: sandboxMounts.readonly, writable: [`${dir}:${containerDriver_1.SANDBOX_WORKSPACE_DIR}`] };
|
|
757
|
+
};
|
|
758
|
+
/**
|
|
759
|
+
* "May this pack run, at this version, for this session?" — signature requirement, version pin,
|
|
760
|
+
* image digest and scoped deny rules, in ONE place.
|
|
761
|
+
*
|
|
762
|
+
* Both gates call this: sandbox creation (where it decides whether a sandbox may be born) and,
|
|
763
|
+
* on deployments with no sandbox, the per-turn governor gate. Two copies of these four checks
|
|
764
|
+
* is exactly the drift this codebase keeps paying for, and the no-sandbox path having no copy
|
|
765
|
+
* at all is why `requireSignedPack` and a version pin could be switched on in the console and
|
|
766
|
+
* enforce nothing on the default deployment shape.
|
|
767
|
+
*/
|
|
768
|
+
const assertPackAdmitted = (currentLease, sessionKey, userId) => {
|
|
769
|
+
const required = (0, signature_1.resolveRequireSignedPack)({
|
|
770
|
+
// The lease's demand is read FIRST and cannot be waived by any local setting — that is
|
|
771
|
+
// what keeps the requirement tighten-only now that the local default is "required".
|
|
772
|
+
lease: currentLease?.requireSignedPack,
|
|
773
|
+
localRequire: options.requireSignedPack,
|
|
774
|
+
localAllowUnsigned: allowUnsignedPack,
|
|
775
|
+
});
|
|
776
|
+
if (required) {
|
|
777
|
+
// RE-VERIFIED, not read off a boot-time boolean. In the default no-sandbox shape the
|
|
778
|
+
// engine runs in THIS process against a pack scaffolded with `fileAccess: full`, so the
|
|
779
|
+
// agent can edit its own pack directory between turns; a cached "verified once at boot"
|
|
780
|
+
// would pass every one of those later turns. `createPackVerifier` keeps this cheap — a
|
|
781
|
+
// stat-only probe in the common case, a full digest when the tree looks touched or the
|
|
782
|
+
// re-verify interval has elapsed.
|
|
783
|
+
const result = packVerifier.verify({
|
|
784
|
+
packDir: activePackDir, pack: packId, packVersion: activeVersion, keys: packTrust.keys,
|
|
785
|
+
});
|
|
786
|
+
if (result !== packSignature) {
|
|
787
|
+
packSignature = result;
|
|
788
|
+
packDigestValue = result.ok ? result.claims.digest : undefined;
|
|
789
|
+
}
|
|
790
|
+
if (!result.ok) {
|
|
791
|
+
throw new lease_1.LeaseDeniedError(packRefusalText(result));
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
// sessionKey makes admission SCOPED: a deny rule naming this session or its user refuses
|
|
795
|
+
// at the same gate that enforces the fleet-wide switch.
|
|
796
|
+
(0, lease_1.assertAdmitted)(currentLease, {
|
|
797
|
+
pack: packId,
|
|
798
|
+
version: activeVersion,
|
|
799
|
+
image: options.sandboxImage,
|
|
800
|
+
sessionKey,
|
|
801
|
+
userHash: serve_hardening_1.userNamespaceHash,
|
|
802
|
+
userId,
|
|
803
|
+
});
|
|
804
|
+
};
|
|
805
|
+
// Assigned just below; the identity closure above reads it lazily so a poll can report live
|
|
806
|
+
// sandbox count without forcing the runner to be constructed before the lease source.
|
|
807
|
+
// eslint-disable-next-line prefer-const
|
|
808
|
+
// Sandbox mode: the same HeadlessRunner shape, body relocated into a per-session sandbox.
|
|
809
|
+
// serve's turn loop below is untouched — see sandbox/sandboxRunner.ts.
|
|
810
|
+
const sandboxRunner = options.sandbox
|
|
811
|
+
? (0, sandboxRunner_1.createSandboxRunner)({
|
|
812
|
+
driver: options.sandbox === 'container'
|
|
813
|
+
// A non-internal route network is a real weakening of the tier-2 claim, so its warning
|
|
814
|
+
// must reach an operator who did not pass --verbose.
|
|
815
|
+
? new containerDriver_1.ContainerSandboxDriver({
|
|
816
|
+
exec: support_1.dockerExec,
|
|
817
|
+
logger: {
|
|
818
|
+
info: (m) => logger.info(m),
|
|
819
|
+
warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
|
|
820
|
+
},
|
|
821
|
+
})
|
|
822
|
+
: new processDriver_1.ProcessSandboxDriver({
|
|
823
|
+
command: process.execPath,
|
|
824
|
+
// --workspace is REQUIRED, not optional: a tier-0 sandbox shares this host's
|
|
825
|
+
// filesystem, and serve sends this same path as `workspaceRoot` on every run
|
|
826
|
+
// frame. Omitting it left the sandbox on the container-image default
|
|
827
|
+
// (/state/workspace), so a serve-launched process sandbox died at boot with
|
|
828
|
+
// ENOENT: mkdir '/state/workspace'.
|
|
829
|
+
// pack.packPath, not options.agent: serve has ALREADY resolved the pack (a
|
|
830
|
+
// name resolves against ~/.agent-hippo/agents and the workspace), and the
|
|
831
|
+
// sandbox must run that exact directory rather than re-resolve a name from a
|
|
832
|
+
// different cwd and possibly land on a different pack.
|
|
833
|
+
args: [
|
|
834
|
+
// Only forward the script path when the process actually HAS one. A
|
|
835
|
+
// single-file build reports `process.argv[1]` as a path inside its own
|
|
836
|
+
// virtual filesystem (`/$bunfs/root/index.js` for `bun build --compile`,
|
|
837
|
+
// `/snapshot/...` for pkg / node SEA). Passing that made the child parse
|
|
838
|
+
// it as the subcommand and die with `unknown command '/$bunfs/...'`, so
|
|
839
|
+
// every tier-0 sandbox failed with a blank "sandbox channel closed".
|
|
840
|
+
// An existence check is NOT enough to detect this: those runtimes mount
|
|
841
|
+
// the virtual path, so `fs.existsSync` cheerfully returns true for it.
|
|
842
|
+
...(0, support_1.sandboxScriptArg)(),
|
|
843
|
+
'sandbox', '--agent', activePackDir, '--stdio',
|
|
844
|
+
'--state-dir', storagePath, '--workspace', options.workspace,
|
|
845
|
+
],
|
|
846
|
+
// PID/user-namespace wrap where the host allows it (residual P4: on a
|
|
847
|
+
// single-uid host, a concurrent tenant reads a turn's nonce from
|
|
848
|
+
// /proc/<pid>/environ — and the claude CLI's argv from /proc/<pid>/cmdline,
|
|
849
|
+
// which is world-readable). Default 'auto': hardened where possible, honest
|
|
850
|
+
// warning where not; 'require' makes multi-tenant hosts fail closed.
|
|
851
|
+
namespaceIsolation: (0, processDriver_1.namespaceIsolationFromEnv)(),
|
|
852
|
+
logger: {
|
|
853
|
+
info: (m) => logger.info(m),
|
|
854
|
+
warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
|
|
855
|
+
},
|
|
856
|
+
}),
|
|
857
|
+
image: options.sandboxImage ?? '',
|
|
858
|
+
tier: options.sandbox === 'container' ? 2 : 0,
|
|
859
|
+
...(sandboxMounts ? { fs: sandboxMounts, fsFor: sandboxFsFor, stateDirFor: sandboxStateDirFor } : {}),
|
|
860
|
+
// The only routes a sandbox may reach. Everything else is unroutable, not merely denied.
|
|
861
|
+
routes: [
|
|
862
|
+
...(process.env.AGENTHIPPO_DATA_MCP_URL ? [{ kind: 'data', target: process.env.AGENTHIPPO_DATA_MCP_URL }] : []),
|
|
863
|
+
...(apiConfig.baseUrl ? [{ kind: 'inference', target: apiConfig.baseUrl }] : []),
|
|
864
|
+
],
|
|
865
|
+
limits: { ttlSeconds: options.sandboxTtlSeconds ?? 30 * 60 },
|
|
866
|
+
provenanceEnv: () => (0, runtime_1.buildProvenanceEnv)({
|
|
867
|
+
policyVersion: lastPolicyVersion,
|
|
868
|
+
packDigest: packDigestValue,
|
|
869
|
+
engineVersion: (0, deps_1.installedEngineVersion)(defaultEngine),
|
|
870
|
+
tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
|
|
871
|
+
instanceId,
|
|
872
|
+
}),
|
|
873
|
+
// Kill switch + rollback, checked before a sandbox can exist. Revoking the lease stops
|
|
874
|
+
// new work fleet-wide immediately; TTL bounds the sandboxes already running.
|
|
875
|
+
admit: async (sessionKey) => {
|
|
876
|
+
const currentLease = await leaseSource.get();
|
|
877
|
+
assertPackAdmitted(currentLease, sessionKey);
|
|
878
|
+
},
|
|
879
|
+
inferenceBaseUrl: apiConfig.baseUrl,
|
|
880
|
+
// Per-sandbox inference warrant — signed, audience-bound, and naming the sandbox it was
|
|
881
|
+
// minted for. The provider key never enters a sandbox; the proxy verifies with a public
|
|
882
|
+
// key and can never mint, so a sandbox cannot re-issue itself more budget.
|
|
883
|
+
//
|
|
884
|
+
// `user` is the verified end user from the turn's credential scope, not the pack: the
|
|
885
|
+
// gateway strips caller-supplied x-litellm-metadata-user-id and re-stamps it from this
|
|
886
|
+
// claim, so the warrant is the only attribution channel that survives it — stamping
|
|
887
|
+
// packId here made every downstream user_id (LiteLLM, spend rows, evidence provenance)
|
|
888
|
+
// the pack. Every mint runs inside runWithCredential (create-time on first-turn
|
|
889
|
+
// acquire, then per-turn re-mints in RemoteEngine.stream), so the ambient credential is
|
|
890
|
+
// the turn's own. An unidentified turn gets a session-scoped marker rather than
|
|
891
|
+
// silently borrowing the pack's name — sessionKey is already user-namespaced, so the
|
|
892
|
+
// marker cannot collide across identified users.
|
|
893
|
+
mintModelToken: (sandboxId, sessionKey) => (0, support_1.mintSandboxInferenceWarrant)(warrantKey, {
|
|
894
|
+
user: (0, credential_context_1.currentCredential)()?.userId?.trim() || `anon:${sessionKey}`,
|
|
895
|
+
pack: packId,
|
|
896
|
+
packVersion: activeVersion,
|
|
897
|
+
sandboxId,
|
|
898
|
+
tier: options.sandbox === 'container' ? 2 : 0,
|
|
899
|
+
// A sandbox outlives one turn, so its model warrant gets the full cap rather than a
|
|
900
|
+
// turn-length TTL; the spend cap, not the clock, is what bounds it.
|
|
901
|
+
ttlSeconds: 300,
|
|
902
|
+
}),
|
|
903
|
+
logger,
|
|
904
|
+
runtime: { logger, editTracker: undefined, configProvider: undefined, permissionHandler },
|
|
905
|
+
})
|
|
906
|
+
: undefined;
|
|
907
|
+
sandboxRunnerRef = sandboxRunner ?? undefined;
|
|
908
|
+
// Caps + kill switch. Enforcement is non-cooperative — it revokes a route or destroys the
|
|
909
|
+
// sandbox — so it exists only where there ARE sandboxes. Without --sandbox we say so rather than
|
|
910
|
+
// pretending a cap is in force.
|
|
911
|
+
// 0 / NaN mean "unlimited" for every one of these — `--max-turns-per-session` has documented
|
|
912
|
+
// that meaning since before caps existed, and passing its default 0 through as a real cap
|
|
913
|
+
// refused the very first turn (`turn cap reached (1/0)`, observed live).
|
|
914
|
+
const positive = (n) => typeof n === 'number' && Number.isFinite(n) && n > 0 ? n : undefined;
|
|
915
|
+
const localCaps = {
|
|
916
|
+
...(0, caps_1.capsFromEnv)(),
|
|
917
|
+
...(positive(options.maxTurnsPerSession) !== undefined ? { maxTurnsPerSession: options.maxTurnsPerSession } : {}),
|
|
918
|
+
...(positive(options.maxSessionSeconds) !== undefined ? { maxSessionSeconds: options.maxSessionSeconds } : {}),
|
|
919
|
+
...(positive(options.maxConcurrentSandboxes) !== undefined ? { maxConcurrentSandboxes: options.maxConcurrentSandboxes } : {}),
|
|
920
|
+
...(options.capOnBreach ? { onBreach: options.capOnBreach } : {}),
|
|
921
|
+
};
|
|
922
|
+
const capsRequested = localCaps.maxTurnsPerSession !== undefined
|
|
923
|
+
|| localCaps.maxSessionSeconds !== undefined
|
|
924
|
+
|| localCaps.maxConcurrentSandboxes !== undefined;
|
|
925
|
+
// A deployment must be governable whether or not it runs sandboxes. Gating the governor on
|
|
926
|
+
// `sandboxRunner` meant a plain `serve` never polled its lease at all: invisible to the control
|
|
927
|
+
// plane AND immune to the kill switch, which is the opposite of what an operator configuring
|
|
928
|
+
// a lease URL is asking for. Without sandboxes the enforcement that needs one degrades (no route
|
|
929
|
+
// revocation, no concurrency bound) but the substance survives — the turn is still refused.
|
|
930
|
+
const nullSandboxes = {
|
|
931
|
+
sandboxes: () => [],
|
|
932
|
+
destroySession: async () => undefined,
|
|
933
|
+
revokeRoute: async () => undefined,
|
|
934
|
+
// There is nothing here that can stop a turn already executing in this process, so a
|
|
935
|
+
// `kill` deny rule degrades to `drain`. The governor now says so loudly instead of
|
|
936
|
+
// sweeping an empty list and reporting the rule as applied.
|
|
937
|
+
supportsInFlightTermination: false,
|
|
938
|
+
};
|
|
939
|
+
const governed = Boolean(sandboxRunner || process.env.AGENTHIPPO_LEASE_URL || capsRequested);
|
|
940
|
+
const governor = governed
|
|
941
|
+
? (0, governor_1.createGovernor)({
|
|
942
|
+
// The lease's caps bind; local flags may tighten them and never loosen them.
|
|
943
|
+
caps: (0, caps_1.mergeCaps)((await leaseSource.get())?.caps, localCaps),
|
|
944
|
+
sandboxes: sandboxRunner ?? nullSandboxes,
|
|
945
|
+
getLease: () => leaseSource.get(),
|
|
946
|
+
pollSeconds: leasePollSeconds,
|
|
947
|
+
// Scope for deny rules: the pack this instance runs, and the same hash that namespaces
|
|
948
|
+
// session keys — so the console can name a user by email without serve holding a roster.
|
|
949
|
+
pack: packId,
|
|
950
|
+
userHash: serve_hardening_1.userNamespaceHash,
|
|
951
|
+
// Only when there is no sandbox: with one, the same checks gate a sandbox's BIRTH and
|
|
952
|
+
// running sandboxes are meant to drain on TTL. Without one, this is the only gate that
|
|
953
|
+
// exists, so an unsigned pack or an unpinned version would otherwise keep serving.
|
|
954
|
+
...(sandboxRunner ? {} : {
|
|
955
|
+
admitPack: (sessionKey, lease) => {
|
|
956
|
+
try {
|
|
957
|
+
// No userId here on purpose: the governor has ALREADY evaluated the deny rules for
|
|
958
|
+
// this turn (with its followed-user map, which this call site cannot see), so the
|
|
959
|
+
// only thing left for this gate is the signature/pin/digest half.
|
|
960
|
+
assertPackAdmitted(lease, sessionKey);
|
|
961
|
+
return { allowed: true };
|
|
962
|
+
}
|
|
963
|
+
catch (err) {
|
|
964
|
+
return { allowed: false, reason: err instanceof Error ? err.message : String(err) };
|
|
965
|
+
}
|
|
966
|
+
},
|
|
967
|
+
}),
|
|
968
|
+
logger,
|
|
969
|
+
audit: (event, fields) => auditEvent(String(fields.sessionKey ?? 'instance'), event, fields),
|
|
970
|
+
})
|
|
971
|
+
: undefined;
|
|
972
|
+
if (capsRequested && !sandboxRunner) {
|
|
973
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} caps are configured but --sandbox is not set. Turn and duration caps still refuse the turn; the concurrency cap and route revocation need sandboxes, so those are IGNORED.`);
|
|
974
|
+
}
|
|
975
|
+
const baseRunner = sandboxRunner ?? inProcessRunner;
|
|
976
|
+
const runner = governor ? (0, governor_1.withGovernor)(baseRunner, governor) : baseRunner;
|
|
977
|
+
if (governor) {
|
|
978
|
+
governor.start();
|
|
979
|
+
const p = governor.policy();
|
|
980
|
+
// The banner prints the number an operator will be held to: worst-case kill latency is
|
|
981
|
+
// poll + cache, not the poll alone. It used to print "lease poll 30s" while the cache
|
|
982
|
+
// defaulted to 60, so a kill could take three times as long as advertised.
|
|
983
|
+
const killLatency = (0, lease_1.effectiveKillLatencySeconds)(leasePollSeconds);
|
|
984
|
+
logger.info(`[Serve] control plane: caps turns=${p.maxTurnsPerSession ?? '∞'} session=${p.maxSessionSeconds ?? '∞'}s sandboxes=${p.maxConcurrentSandboxes ?? '∞'} onBreach=${p.onBreach ?? 'revoke-inference'}; lease poll ${leasePollSeconds}s (worst-case kill latency ${killLatency}s; an admin refresh is immediate)`);
|
|
985
|
+
if (!sandboxRunner) {
|
|
986
|
+
logger.info('[Serve] no sandbox driver: a deny rule with mode=kill takes effect as DRAIN — the in-flight turn finishes and only the next one is refused.');
|
|
987
|
+
}
|
|
988
|
+
const leaseStatus = leaseSource.status();
|
|
989
|
+
if (process.env.AGENTHIPPO_LEASE_URL && !leaseStatus.requireSignedLease) {
|
|
990
|
+
logger.info('[Serve] lease signature is NOT required — set --require-signed-lease (or lease.requireSignedLease) once the control plane signs leases.');
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
if (sandboxRunner) {
|
|
994
|
+
logger.info(`[Serve] sandbox mode: ${options.sandbox} (tier ${options.sandbox === 'container' ? 2 : 0}) — engines run in per-session sandboxes`);
|
|
995
|
+
// A sandbox's ONLY model credential is the per-sandbox warrant, which is meaningful only to an
|
|
996
|
+
// inference proxy. With no proxy configured there is nothing to send it to, so the engine
|
|
997
|
+
// dials the provider bare and the first turn dies with an opaque 401/reconnect loop. Say so
|
|
998
|
+
// at boot — same reason the engine credential check above is a boot check, not a turn check.
|
|
999
|
+
if (!apiConfig.baseUrl) {
|
|
1000
|
+
console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} --sandbox without an inference proxy: a sandbox holds no provider key, so model calls will fail 401. Set AGENTHIPPO_LITELLM_BASE_URL.`);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
196
1003
|
/** Set when `--gateway-bridge` is enabled; cleared on shutdown. */
|
|
197
1004
|
let gatewayBridge = null;
|
|
198
1005
|
/** True once the pack is synced and (if bridged) the gateway is connected. Drives GET /ready. */
|
|
199
1006
|
let ready = false;
|
|
200
1007
|
// Create HTTP server
|
|
1008
|
+
let evidenceShipHandle;
|
|
201
1009
|
const server = http.createServer(async (req, res) => {
|
|
202
1010
|
const cors = (0, serve_hardening_1.corsHeaders)();
|
|
203
1011
|
for (const [k, v] of Object.entries(cors)) {
|
|
@@ -211,8 +1019,63 @@ async function serveCommand(options) {
|
|
|
211
1019
|
try {
|
|
212
1020
|
// Liveness — always open.
|
|
213
1021
|
if (req.method === 'GET' && req.url === '/health') {
|
|
1022
|
+
const health = { status: 'ok' };
|
|
1023
|
+
// Reported unconditionally: with only `last_ok_at`, "shipping is off" and "shipping
|
|
1024
|
+
// is on but has never once succeeded" look identical to an operator and to alerting.
|
|
1025
|
+
health.evidence_ship_enabled = Boolean(evidenceShipHandle);
|
|
1026
|
+
const lastOk = evidenceShipHandle?.lastOkAt();
|
|
1027
|
+
if (lastOk !== undefined) {
|
|
1028
|
+
health.evidence_ship_last_ok_at = new Date(lastOk).toISOString();
|
|
1029
|
+
}
|
|
1030
|
+
// R-13 gap detection: evidence write failures and permanently-skipped files are
|
|
1031
|
+
// COUNTED now rather than swallowed, but a counter nobody can read is the same as no
|
|
1032
|
+
// counter. `disabledReason` matters most — "shipping is off because
|
|
1033
|
+
// AGENTHIPPO_EVIDENCE_SHIP=0" and "shipping is on and failing" produce the same
|
|
1034
|
+
// `enabled: false` shaped alert otherwise.
|
|
1035
|
+
const ship = (0, sessionShip_1.getEvidenceShipStatus)();
|
|
1036
|
+
health.evidence_ship_disabled_reason = ship.disabledReason ?? null;
|
|
1037
|
+
health.evidence_ship_consecutive_failures = ship.consecutiveFailures;
|
|
1038
|
+
health.evidence_records_staged = ship.recordsStaged;
|
|
1039
|
+
health.evidence_records_shipped = ship.recordsShipped;
|
|
1040
|
+
// Non-zero means the record is known to be INCOMPLETE, not merely late.
|
|
1041
|
+
health.evidence_records_skipped_too_old = ship.recordsSkippedTooOld;
|
|
1042
|
+
health.evidence_files_skipped_too_old = ship.filesSkippedTooOld;
|
|
1043
|
+
health.evidence_content_capture = ship.contentCapture;
|
|
1044
|
+
if (ship.lastHeartbeatAt !== undefined) {
|
|
1045
|
+
health.evidence_ship_last_heartbeat_at = new Date(ship.lastHeartbeatAt).toISOString();
|
|
1046
|
+
}
|
|
1047
|
+
const writes = (0, sessionEvents_1.getSessionEventWriterStats)();
|
|
1048
|
+
health.evidence_writes_ok = writes.written;
|
|
1049
|
+
health.evidence_writes_failed = writes.failed;
|
|
1050
|
+
if (writes.lastError) {
|
|
1051
|
+
health.evidence_write_last_error = {
|
|
1052
|
+
at: new Date(writes.lastError.at).toISOString(),
|
|
1053
|
+
event_type: writes.lastError.eventType,
|
|
1054
|
+
message: writes.lastError.message,
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
1057
|
+
// Governance surface: an operator asking "is this instance actually governed?" gets
|
|
1058
|
+
// the answer here rather than from boot logs that have long since scrolled away.
|
|
1059
|
+
const leaseState = leaseSource.status();
|
|
1060
|
+
health.lease_configured = Boolean(process.env.AGENTHIPPO_LEASE_URL);
|
|
1061
|
+
health.lease_signed = leaseState.signed;
|
|
1062
|
+
health.lease_signature_required = leaseState.requireSignedLease;
|
|
1063
|
+
health.lease_staleness_seconds = leaseState.stalenessSeconds ?? null;
|
|
1064
|
+
health.lease_fail_closed = leaseState.failClosed;
|
|
1065
|
+
health.pack_signature_ok = packSignature.ok;
|
|
1066
|
+
health.pack_signature_required = (0, signature_1.resolveRequireSignedPack)({
|
|
1067
|
+
localRequire: options.requireSignedPack, localAllowUnsigned: allowUnsignedPack,
|
|
1068
|
+
});
|
|
1069
|
+
health.pack_trust_root = packTrust.status;
|
|
1070
|
+
// Reported separately from the pack root: "signed leases are required and the lease
|
|
1071
|
+
// keyring is unavailable" is an outage in the making, and it is invisible in
|
|
1072
|
+
// `pack_trust_root`. `lease_trust_root_source` distinguishes an operator-named root
|
|
1073
|
+
// from the pack-keyring fallback, which is the state that stops working on rotation.
|
|
1074
|
+
health.lease_trust_root = leaseTrust.status;
|
|
1075
|
+
health.lease_trust_root_source = leaseTrust.explicitlyConfigured ? 'lease' : 'pack-fallback';
|
|
1076
|
+
health.kill_in_flight_supported = Boolean(sandboxRunner);
|
|
214
1077
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
215
|
-
res.end(JSON.stringify(
|
|
1078
|
+
res.end(JSON.stringify(health));
|
|
216
1079
|
return;
|
|
217
1080
|
}
|
|
218
1081
|
// Readiness — pack synced (+ gateway connected when bridged).
|
|
@@ -246,7 +1109,15 @@ async function serveCommand(options) {
|
|
|
246
1109
|
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
|
247
1110
|
return;
|
|
248
1111
|
}
|
|
249
|
-
|
|
1112
|
+
// User-namespaced session key (secure-obo-design §3.4): kills cross-user session
|
|
1113
|
+
// attachment via a guessed session_id. Fails closed on pinned-identity deployments.
|
|
1114
|
+
const { sessionKey, rejected } = (0, serve_hardening_1.resolveServeSessionKey)(body.session_id, userId);
|
|
1115
|
+
if (rejected) {
|
|
1116
|
+
sessionEventWriter.append('unverified', (0, base_1.generateRequestId)(), 'warrant_denied', { reason: rejected });
|
|
1117
|
+
res.writeHead(403, { 'Content-Type': 'application/json' });
|
|
1118
|
+
res.end(JSON.stringify({ error: 'session_id requires an authenticated user' }));
|
|
1119
|
+
return;
|
|
1120
|
+
}
|
|
250
1121
|
// Acquire before turn-limiter so a capacity reject does not burn a turn slot.
|
|
251
1122
|
const release = await turnGate.acquire();
|
|
252
1123
|
if (!release) {
|
|
@@ -258,6 +1129,10 @@ async function serveCommand(options) {
|
|
|
258
1129
|
res.end(JSON.stringify({ error: 'At capacity' }));
|
|
259
1130
|
return;
|
|
260
1131
|
}
|
|
1132
|
+
// Turn-scoped data capability: mint a nonce bound to the verified user; the
|
|
1133
|
+
// engine gets it via per-spawn env only. Deleted in the finally — a stale
|
|
1134
|
+
// engine/tool fails closed at the gateway after that (secure-obo-design §3.1).
|
|
1135
|
+
let turnReg = null;
|
|
261
1136
|
try {
|
|
262
1137
|
if (!turnLimiter.tryConsume(sessionKey)) {
|
|
263
1138
|
res.writeHead(429, { 'Content-Type': 'application/json' });
|
|
@@ -265,12 +1140,16 @@ async function serveCommand(options) {
|
|
|
265
1140
|
return;
|
|
266
1141
|
}
|
|
267
1142
|
await prepareSession(sessionKey);
|
|
268
|
-
|
|
1143
|
+
if (userId?.trim()) {
|
|
1144
|
+
turnReg = await registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken, edgeNonce: edgeTurnNonce(req.headers) });
|
|
1145
|
+
}
|
|
1146
|
+
const result = await (0, credential_context_1.runWithCredential)({ userId, dataToken }, () => handleChat(runner, body, sessionKey, defaultEngine, defaultModel, options.workspace, permissions, userId, turnReg?.env), { onDenied: logWarrantDenied(sessionKey) });
|
|
269
1147
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
270
1148
|
res.end(JSON.stringify(result));
|
|
271
1149
|
statePusher?.schedule(sessionKey); // per-session delta push, off the response path
|
|
272
1150
|
}
|
|
273
1151
|
finally {
|
|
1152
|
+
await turnReg?.release();
|
|
274
1153
|
release();
|
|
275
1154
|
}
|
|
276
1155
|
return;
|
|
@@ -288,7 +1167,14 @@ async function serveCommand(options) {
|
|
|
288
1167
|
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
|
289
1168
|
return;
|
|
290
1169
|
}
|
|
291
|
-
|
|
1170
|
+
// Same user-namespaced session key + fail-closed rule as /v1/chat above.
|
|
1171
|
+
const { sessionKey, rejected } = (0, serve_hardening_1.resolveServeSessionKey)(body.session_id, userId);
|
|
1172
|
+
if (rejected) {
|
|
1173
|
+
sessionEventWriter.append('unverified', (0, base_1.generateRequestId)(), 'warrant_denied', { reason: rejected });
|
|
1174
|
+
res.writeHead(403, { 'Content-Type': 'application/json' });
|
|
1175
|
+
res.end(JSON.stringify({ error: 'session_id requires an authenticated user' }));
|
|
1176
|
+
return;
|
|
1177
|
+
}
|
|
292
1178
|
// Acquire BEFORE writeHead(200, SSE) so a reject still gets a clean 429.
|
|
293
1179
|
const release = await turnGate.acquire();
|
|
294
1180
|
if (!release) {
|
|
@@ -300,6 +1186,8 @@ async function serveCommand(options) {
|
|
|
300
1186
|
res.end(JSON.stringify({ error: 'At capacity' }));
|
|
301
1187
|
return;
|
|
302
1188
|
}
|
|
1189
|
+
// Turn-scoped data capability — same rules as /v1/chat above.
|
|
1190
|
+
let turnReg = null;
|
|
303
1191
|
try {
|
|
304
1192
|
if (!turnLimiter.tryConsume(sessionKey)) {
|
|
305
1193
|
res.writeHead(429, { 'Content-Type': 'application/json' });
|
|
@@ -307,15 +1195,25 @@ async function serveCommand(options) {
|
|
|
307
1195
|
return;
|
|
308
1196
|
}
|
|
309
1197
|
await prepareSession(sessionKey);
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
1198
|
+
if (userId?.trim()) {
|
|
1199
|
+
turnReg = await registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken, edgeNonce: edgeTurnNonce(req.headers) });
|
|
1200
|
+
}
|
|
1201
|
+
// Fail-closed credential check MUST run before SSE writeHead(200).
|
|
1202
|
+
// Otherwise a warrant denial lands after headers are committed, the outer
|
|
1203
|
+
// catch can only append raw JSON (not an SSE error event), and the IDE
|
|
1204
|
+
// client's SSE parser falls through to emitter.done() → silent success.
|
|
1205
|
+
await (0, credential_context_1.runWithCredential)({ userId, dataToken }, async () => {
|
|
1206
|
+
res.writeHead(200, {
|
|
1207
|
+
'Content-Type': 'text/event-stream',
|
|
1208
|
+
'Cache-Control': 'no-cache',
|
|
1209
|
+
'Connection': 'keep-alive',
|
|
1210
|
+
});
|
|
1211
|
+
await handleStreamingChat(runner, body, sessionKey, defaultEngine, defaultModel, options.workspace, permissions, res, userId, turnReg?.env);
|
|
1212
|
+
}, { onDenied: logWarrantDenied(sessionKey) });
|
|
316
1213
|
statePusher?.schedule(sessionKey); // per-session delta push, off the response path
|
|
317
1214
|
}
|
|
318
1215
|
finally {
|
|
1216
|
+
await turnReg?.release();
|
|
319
1217
|
release();
|
|
320
1218
|
}
|
|
321
1219
|
return;
|
|
@@ -357,7 +1255,7 @@ async function serveCommand(options) {
|
|
|
357
1255
|
}
|
|
358
1256
|
console.log(`Workspace: ${options.workspace}`);
|
|
359
1257
|
if (pack) {
|
|
360
|
-
console.log(`Pack: ${pack.manifest.metadata.name}`);
|
|
1258
|
+
console.log(`Pack: ${pack.manifest.metadata.name}@${pack.manifest.metadata.version}`);
|
|
361
1259
|
console.log(`Gateway: ${gatewayAgentId}`);
|
|
362
1260
|
}
|
|
363
1261
|
console.log(BANNER_LINE);
|
|
@@ -369,28 +1267,67 @@ async function serveCommand(options) {
|
|
|
369
1267
|
console.log('');
|
|
370
1268
|
console.log('Press Ctrl+C to stop.');
|
|
371
1269
|
console.log('');
|
|
1270
|
+
const shipConfig = (0, sessionShip_1.resolveSessionShipConfig)(instanceId, analyticsSessionsDir, storagePath);
|
|
1271
|
+
if (shipConfig) {
|
|
1272
|
+
evidenceShipHandle = (0, sessionShip_1.startSessionShipper)({
|
|
1273
|
+
...shipConfig,
|
|
1274
|
+
logger: { info: (m) => logger.info(m), warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`) },
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
372
1277
|
// Bind channels before bridge connect so config.patch reload does not race registration.
|
|
373
1278
|
const bindSpec = options.bind?.trim();
|
|
374
1279
|
if (bindSpec) {
|
|
375
1280
|
try {
|
|
376
|
-
|
|
1281
|
+
// The captured token — the env var was stripped at boot (§10 hole 2).
|
|
1282
|
+
await bindToGatewayChannels(gatewayAgentId, bindSpec, brokerGatewayToken);
|
|
377
1283
|
}
|
|
378
1284
|
catch (err) {
|
|
379
1285
|
console.error(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} Failed to bind channels: ${err instanceof Error ? err.message : String(err)}`);
|
|
380
1286
|
}
|
|
381
1287
|
}
|
|
382
1288
|
if (options.gatewayBridge) {
|
|
1289
|
+
// Egress broker (secure-obo-design §7.6): start before the bridge so the URL is in
|
|
1290
|
+
// process.env when the first channel turn spawns an engine. The broker holds the
|
|
1291
|
+
// gateway token; the agent presents only the turn nonce.
|
|
1292
|
+
if (turnRegistry) {
|
|
1293
|
+
try {
|
|
1294
|
+
egressBroker = await (0, data_plane_1.startEgressBroker)({
|
|
1295
|
+
registry: turnRegistry,
|
|
1296
|
+
workspaceRoot: options.workspace,
|
|
1297
|
+
agentId: pack.manifest.metadata.name,
|
|
1298
|
+
send: async (handle, message, media) => {
|
|
1299
|
+
const r = await (0, outboundSend_1.sendGatewayOutboundMessage)(handle, { message, media, token: brokerGatewayToken });
|
|
1300
|
+
return { ok: r.ok, via: r.via, stdout: r.stdout, stderr: r.stderr };
|
|
1301
|
+
},
|
|
1302
|
+
logger: options.verbose ? new headless_1.ConsoleLogger('[Egress]') : new headless_1.NoOpLogger(),
|
|
1303
|
+
audit: auditEvent,
|
|
1304
|
+
});
|
|
1305
|
+
process.env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
|
|
1306
|
+
// The token itself was stripped from process.env at boot, broker or not.
|
|
1307
|
+
console.log('OK: Egress broker on loopback — gateway token stays broker-side; agent sends via turn nonce');
|
|
1308
|
+
}
|
|
1309
|
+
catch (err) {
|
|
1310
|
+
// Fail CLOSED: without the broker, AGENTHIPPO_EGRESS_URL stays unset and the
|
|
1311
|
+
// runtime would bake the gateway token back into engine children (§7.6 undone
|
|
1312
|
+
// by a port-bind hiccup). Refuse to serve rather than degrade silently.
|
|
1313
|
+
console.error(`${utils_1.COLORS.red}Error:${utils_1.COLORS.reset} Egress broker failed to start: ${err instanceof Error ? err.message : String(err)}`);
|
|
1314
|
+
process.exit(1);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
383
1317
|
try {
|
|
384
1318
|
gatewayBridge = await startServeGatewayBridge({
|
|
385
1319
|
gatewayAgentId,
|
|
386
1320
|
packName: pack.manifest.metadata.name,
|
|
387
1321
|
workspace: options.workspace,
|
|
388
|
-
|
|
1322
|
+
registerTurn: registerServeTurn,
|
|
1323
|
+
brokerGatewayToken,
|
|
389
1324
|
engine: defaultEngine,
|
|
390
1325
|
defaultModel,
|
|
391
1326
|
runner,
|
|
392
1327
|
permissions,
|
|
393
1328
|
turnGate,
|
|
1329
|
+
rateLimiter,
|
|
1330
|
+
turnLimiter,
|
|
394
1331
|
warnGateReject,
|
|
395
1332
|
prepareSession,
|
|
396
1333
|
onTurnComplete: (sessionId) => statePusher?.schedule(sessionId),
|
|
@@ -432,6 +1369,33 @@ async function serveCommand(options) {
|
|
|
432
1369
|
}
|
|
433
1370
|
gatewayBridge = null;
|
|
434
1371
|
}
|
|
1372
|
+
if (dataPlane) {
|
|
1373
|
+
try {
|
|
1374
|
+
await dataPlane.close();
|
|
1375
|
+
}
|
|
1376
|
+
catch {
|
|
1377
|
+
// best-effort
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
if (egressBroker) {
|
|
1381
|
+
try {
|
|
1382
|
+
await egressBroker.close();
|
|
1383
|
+
}
|
|
1384
|
+
catch {
|
|
1385
|
+
// best-effort
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
// After the planes are closed, so the tail includes whatever they just wrote. SIGTERM is
|
|
1389
|
+
// how every redeploy ends, and without this each one drops up to one interval of
|
|
1390
|
+
// session evidence — permanently, when the state dir (and its cursor) is ephemeral.
|
|
1391
|
+
if (evidenceShipHandle) {
|
|
1392
|
+
try {
|
|
1393
|
+
await evidenceShipHandle.flush();
|
|
1394
|
+
}
|
|
1395
|
+
catch {
|
|
1396
|
+
// best-effort — never block shutdown on the evidence server being reachable
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
435
1399
|
server.close(() => {
|
|
436
1400
|
runner.dispose();
|
|
437
1401
|
process.exit(0);
|
|
@@ -455,15 +1419,18 @@ async function parseBody(req) {
|
|
|
455
1419
|
throw new Error('Invalid JSON');
|
|
456
1420
|
}
|
|
457
1421
|
}
|
|
458
|
-
async function handleChat(runner, body, defaultEngine, defaultModel, workspace, permissions, userId) {
|
|
1422
|
+
async function handleChat(runner, body, sessionKey, defaultEngine, defaultModel, workspace, permissions, userId, turnEnv) {
|
|
459
1423
|
const result = await runner.run({
|
|
460
1424
|
engineId: resolveRequestedEngineId(body.engine, defaultEngine),
|
|
461
1425
|
modelId: body.model || defaultModel,
|
|
462
1426
|
message: body.message,
|
|
463
1427
|
workspaceRoot: workspace,
|
|
464
|
-
|
|
1428
|
+
// The user-namespaced key (never raw body.session_id) so engine conversation
|
|
1429
|
+
// state and per-session artifacts are per-user — see resolveServeSessionKey.
|
|
1430
|
+
sessionId: sessionKey,
|
|
465
1431
|
analyticsUserId: userId,
|
|
466
1432
|
permissions,
|
|
1433
|
+
turnEnv,
|
|
467
1434
|
});
|
|
468
1435
|
return {
|
|
469
1436
|
success: result.success,
|
|
@@ -473,6 +1440,85 @@ async function handleChat(runner, body, defaultEngine, defaultModel, workspace,
|
|
|
473
1440
|
error: result.error,
|
|
474
1441
|
};
|
|
475
1442
|
}
|
|
1443
|
+
/**
|
|
1444
|
+
* Session key for a gateway turn. Prefers the caller's own session identifiers; when the
|
|
1445
|
+
* sender has a resolved identity but no session id, keys off that identity (stable, no
|
|
1446
|
+
* collision risk — it's unique to that sender). Only when NEITHER a session id nor an
|
|
1447
|
+
* identity is available does this fall back to a per-message unique key, rather than a
|
|
1448
|
+
* shared 'gateway-anon' bucket — a shared bucket would let two unrelated, genuinely
|
|
1449
|
+
* unidentified senders share one TurnLimiter budget and, when AGENTHIPPO_STATE_S3_URI is
|
|
1450
|
+
* set, one S3 session-state prefix (ensureSessionLocal/pushSessionState), letting one
|
|
1451
|
+
* anonymous sender's state be restored into another's turn.
|
|
1452
|
+
*
|
|
1453
|
+
* `durable` is false only for that one-off fallback key. A key that is by construction
|
|
1454
|
+
* never reused can never hit its own TurnLimiter cap and can never have prior S3 state to
|
|
1455
|
+
* restore or a future turn to restore it for — callers should skip per-session tracking
|
|
1456
|
+
* (TurnLimiter, prepareSession/ensureSessionLocal, onTurnComplete/statePusher.schedule) for
|
|
1457
|
+
* it entirely, rather than accumulate a permanent, never-useful entry in those stores.
|
|
1458
|
+
*
|
|
1459
|
+
* `sessionKey` is namespaced by the resolved sender identity when an explicit session id is
|
|
1460
|
+
* supplied (secure-obo-design §3.4 — a caller-supplied id must not attach to another
|
|
1461
|
+
* sender's engine session). `outboundKey` preserves the raw gateway key: the bridge records
|
|
1462
|
+
* reply handles under it (bridge.ts upsertGatewayOutboundHandle), so the outbound-routing
|
|
1463
|
+
* file must keep resolving from the un-namespaced key or reply-to-sender breaks.
|
|
1464
|
+
*/
|
|
1465
|
+
function resolveGatewaySessionKey(opts, userId) {
|
|
1466
|
+
const explicit = opts.sessionId?.trim() || opts.gatewaySessionKey?.trim();
|
|
1467
|
+
if (explicit) {
|
|
1468
|
+
const sessionKey = userId ? `u${(0, serve_hardening_1.userNamespaceHash)(userId)}--${explicit}` : explicit;
|
|
1469
|
+
return { sessionKey, outboundKey: explicit, durable: true };
|
|
1470
|
+
}
|
|
1471
|
+
if (userId) {
|
|
1472
|
+
// Already keyed on the sender's own identity — no cross-user attachment surface.
|
|
1473
|
+
return { sessionKey: `gateway-${userId}`, outboundKey: `gateway-${userId}`, durable: true };
|
|
1474
|
+
}
|
|
1475
|
+
// randomUUID (122 bits), not generateRequestId (~32 bits) — this is the sole thing
|
|
1476
|
+
// standing between two unrelated anonymous senders and a real collision, so it needs to
|
|
1477
|
+
// be architecturally collision-free, not just statistically unlikely.
|
|
1478
|
+
const anon = `gateway-anon-${(0, node_crypto_1.randomUUID)()}`;
|
|
1479
|
+
return { sessionKey: anon, outboundKey: anon, durable: false };
|
|
1480
|
+
}
|
|
1481
|
+
/**
|
|
1482
|
+
* Gateway-channel turn admission: rate limit → turn gate (capacity) → per-session turn cap,
|
|
1483
|
+
* then run. Extracted from the gateway bridge's messageHandler so this is directly testable
|
|
1484
|
+
* without a live gateway connection — this exact sequence (mirroring the HTTP endpoints'
|
|
1485
|
+
* admission order) previously didn't exist for gateway-originated turns at all: channel
|
|
1486
|
+
* messages skipped both the identity-keyed rate limiter and the per-session turn cap.
|
|
1487
|
+
* Gateway channels can't receive an HTTP 429, so each rejection emits a short channel
|
|
1488
|
+
* message instead (best-effort — a channel-notice failure must not mask the rejection).
|
|
1489
|
+
*/
|
|
1490
|
+
async function runGatewayTurn(params) {
|
|
1491
|
+
const notify = async (text) => {
|
|
1492
|
+
try {
|
|
1493
|
+
await params.emitter?.text(text);
|
|
1494
|
+
await params.emitter?.done();
|
|
1495
|
+
}
|
|
1496
|
+
catch {
|
|
1497
|
+
// best-effort channel notice
|
|
1498
|
+
}
|
|
1499
|
+
};
|
|
1500
|
+
if (!params.rateLimiter.tryConsume(params.rlKey)) {
|
|
1501
|
+
await notify(serve_hardening_1.RATE_LIMIT_BUSY_MESSAGE);
|
|
1502
|
+
return { success: false, error: 'rate_limited' };
|
|
1503
|
+
}
|
|
1504
|
+
// Acquire before the turn-limiter so a capacity reject does not burn a turn slot.
|
|
1505
|
+
const release = await params.turnGate.acquire();
|
|
1506
|
+
if (!release) {
|
|
1507
|
+
params.warnGateReject(`TurnGate: rejecting gateway turn (inflight=${params.turnGate.inflight} queued=${params.turnGate.queued} max=${params.turnGate.max})`);
|
|
1508
|
+
await notify(serve_hardening_1.TURN_GATE_BUSY_MESSAGE);
|
|
1509
|
+
return { success: false, error: 'busy' };
|
|
1510
|
+
}
|
|
1511
|
+
try {
|
|
1512
|
+
if (params.trackSession !== false && !params.turnLimiter.tryConsume(params.sessionKey)) {
|
|
1513
|
+
await notify(serve_hardening_1.TURN_LIMIT_BUSY_MESSAGE);
|
|
1514
|
+
return { success: false, error: 'turn_limit' };
|
|
1515
|
+
}
|
|
1516
|
+
return await params.run();
|
|
1517
|
+
}
|
|
1518
|
+
finally {
|
|
1519
|
+
release();
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
476
1522
|
/**
|
|
477
1523
|
* Register this serve process as an OpenClaw external agent (WebSocket `agent.run` → headless runner).
|
|
478
1524
|
* Enables `openclaw agent --agent <packId> --message "…"` against the same gateway.
|
|
@@ -481,7 +1527,9 @@ async function startServeGatewayBridge(params) {
|
|
|
481
1527
|
const { GatewayBridge: Bridge } = await import('../../gateway/bridge.js');
|
|
482
1528
|
const DEFAULT_GATEWAY_PORT = 18789;
|
|
483
1529
|
const gatewayUrl = process.env.OPENCLAW_GATEWAY_URL ?? `ws://127.0.0.1:${DEFAULT_GATEWAY_PORT}`;
|
|
484
|
-
|
|
1530
|
+
// The captured token — OPENCLAW_GATEWAY_TOKEN is stripped from process.env at boot (§7.6,
|
|
1531
|
+
// §10 hole 2), so the bridge must never depend on the env var.
|
|
1532
|
+
const token = params.brokerGatewayToken;
|
|
485
1533
|
const bridge = new Bridge({ gatewayUrl, token });
|
|
486
1534
|
await bridge.connect();
|
|
487
1535
|
await bridge.registerAgent({
|
|
@@ -498,39 +1546,65 @@ async function startServeGatewayBridge(params) {
|
|
|
498
1546
|
: (requestedEngine || params.engine);
|
|
499
1547
|
// #4: the channel sender becomes the per-user user_id (analytics + per-user data scoping).
|
|
500
1548
|
const userId = (0, serve_hardening_1.peerToUserId)(opts.peerInfo);
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
1549
|
+
const { sessionKey, outboundKey, durable } = resolveGatewaySessionKey(opts, userId);
|
|
1550
|
+
return runGatewayTurn({
|
|
1551
|
+
rateLimiter: params.rateLimiter,
|
|
1552
|
+
turnLimiter: params.turnLimiter,
|
|
1553
|
+
turnGate: params.turnGate,
|
|
1554
|
+
warnGateReject: params.warnGateReject,
|
|
1555
|
+
// Same identity-keyed cap the HTTP endpoints enforce — a channel sender is already
|
|
1556
|
+
// trusted enough here to gate the OBO credential scope below, so it's trusted enough
|
|
1557
|
+
// to rate-limit on directly. Unidentified senders share one 'gateway' bucket rather
|
|
1558
|
+
// than being exempt from throttling entirely.
|
|
1559
|
+
rlKey: userId ?? 'gateway',
|
|
1560
|
+
sessionKey,
|
|
1561
|
+
trackSession: durable,
|
|
1562
|
+
emitter: opts.emitter,
|
|
1563
|
+
run: async () => {
|
|
1564
|
+
// A one-off anonymous sessionKey can never have prior state to restore, and
|
|
1565
|
+
// nothing will ever look it up again to push state for — skip both S3 round-trips.
|
|
1566
|
+
if (durable) {
|
|
1567
|
+
await params.prepareSession(sessionKey);
|
|
1568
|
+
}
|
|
1569
|
+
// Channel-originated turns carry no inbound platform token: with the
|
|
1570
|
+
// passthrough provider they fail closed at the gateway (an unlinked chat
|
|
1571
|
+
// peer gets NO data credential — secure-obo-design §7.5 bind-before-
|
|
1572
|
+
// credential); the identity provider maps the peer id server-side instead.
|
|
1573
|
+
// egressStoreKey is the RAW gateway key (outboundKey) the bridge records reply
|
|
1574
|
+
// handles under — it lets the egress broker resolve this session's bound
|
|
1575
|
+
// destinations server-side (§7.6). userId gates BOTH capabilities: an
|
|
1576
|
+
// unidentified peer gets neither a nonce nor an egress route.
|
|
1577
|
+
const turnReg = userId?.trim()
|
|
1578
|
+
? await params.registerTurn({ userId: userId.trim(), sessionKey, egressStoreKey: outboundKey })
|
|
1579
|
+
: null;
|
|
1580
|
+
try {
|
|
1581
|
+
const result = await (0, credential_context_1.runWithCredential)({ userId }, () => params.runner.run({
|
|
1582
|
+
engineId: effectiveEngine,
|
|
1583
|
+
modelId: params.defaultModel,
|
|
1584
|
+
message,
|
|
1585
|
+
workspaceRoot: params.workspace,
|
|
1586
|
+
// Use the resolved sessionKey (not the raw opts fields) so the engine's own
|
|
1587
|
+
// session state and the S3 restore/push above key off the same, user-namespaced
|
|
1588
|
+
// identity. The outbound-routing file (buildSessionEnvPatch) keeps the raw
|
|
1589
|
+
// gateway key — the bridge records reply handles under it, so namespacing it
|
|
1590
|
+
// here would break reply-to-sender. See resolveGatewaySessionKey.
|
|
1591
|
+
sessionId: sessionKey,
|
|
1592
|
+
gatewaySessionKey: outboundKey,
|
|
1593
|
+
analyticsUserId: userId,
|
|
1594
|
+
emitter: opts.emitter,
|
|
1595
|
+
permissions: params.permissions,
|
|
1596
|
+
turnEnv: turnReg?.env,
|
|
1597
|
+
}), { onDenied: params.logWarrantDenied(sessionKey) });
|
|
1598
|
+
if (durable) {
|
|
1599
|
+
params.onTurnComplete?.(sessionKey);
|
|
1600
|
+
}
|
|
1601
|
+
return { success: result.success, error: result.error };
|
|
1602
|
+
}
|
|
1603
|
+
finally {
|
|
1604
|
+
await turnReg?.release();
|
|
1605
|
+
}
|
|
1606
|
+
},
|
|
1607
|
+
});
|
|
534
1608
|
},
|
|
535
1609
|
});
|
|
536
1610
|
console.log(`${utils_1.COLORS.green}✓${utils_1.COLORS.reset} Gateway bridge: registered "${params.gatewayAgentId}" at ${gatewayUrl}`);
|
|
@@ -542,11 +1616,10 @@ async function startServeGatewayBridge(params) {
|
|
|
542
1616
|
* Connect to the gateway and set channel bindings for this agent.
|
|
543
1617
|
* Called when --bind is specified (e.g., --bind "telegram,whatsapp" or --bind "*").
|
|
544
1618
|
*/
|
|
545
|
-
async function bindToGatewayChannels(agentId, bindSpec) {
|
|
1619
|
+
async function bindToGatewayChannels(agentId, bindSpec, token) {
|
|
546
1620
|
const { createAgentIDEGatewayClient } = await import('../../gateway/client.js');
|
|
547
1621
|
const DEFAULT_GATEWAY_PORT = 18789;
|
|
548
1622
|
const gatewayUrl = process.env.OPENCLAW_GATEWAY_URL ?? `ws://127.0.0.1:${DEFAULT_GATEWAY_PORT}`;
|
|
549
|
-
const token = process.env.OPENCLAW_GATEWAY_TOKEN;
|
|
550
1623
|
const { client, clientInfo } = createAgentIDEGatewayClient(gatewayUrl, token);
|
|
551
1624
|
try {
|
|
552
1625
|
await client.connect(clientInfo);
|
|
@@ -594,7 +1667,7 @@ async function bindToGatewayChannels(agentId, bindSpec) {
|
|
|
594
1667
|
client.close();
|
|
595
1668
|
}
|
|
596
1669
|
}
|
|
597
|
-
async function handleStreamingChat(runner, body, defaultEngine, defaultModel, workspace, permissions, res, userId) {
|
|
1670
|
+
async function handleStreamingChat(runner, body, sessionKey, defaultEngine, defaultModel, workspace, permissions, res, userId, turnEnv) {
|
|
598
1671
|
let streamEnded = false;
|
|
599
1672
|
const sendEvent = (type, data) => {
|
|
600
1673
|
if (streamEnded || res.writableEnded) {
|
|
@@ -643,16 +1716,26 @@ async function handleStreamingChat(runner, body, defaultEngine, defaultModel, wo
|
|
|
643
1716
|
},
|
|
644
1717
|
};
|
|
645
1718
|
try {
|
|
646
|
-
await runner.run({
|
|
1719
|
+
const result = await runner.run({
|
|
647
1720
|
engineId: resolveRequestedEngineId(body.engine, defaultEngine),
|
|
648
1721
|
modelId: body.model || defaultModel,
|
|
649
1722
|
message: body.message,
|
|
650
1723
|
workspaceRoot: workspace,
|
|
651
|
-
|
|
1724
|
+
// The user-namespaced key (never raw body.session_id) — see resolveServeSessionKey.
|
|
1725
|
+
sessionId: sessionKey,
|
|
652
1726
|
analyticsUserId: userId,
|
|
653
1727
|
emitter,
|
|
654
1728
|
permissions,
|
|
1729
|
+
turnEnv,
|
|
655
1730
|
});
|
|
1731
|
+
// A policy refusal is a RESULT, not an exception: `withGovernor` returns
|
|
1732
|
+
// `{success:false, error}` without ever touching the emitter. Ignoring it sent the client
|
|
1733
|
+
// HTTP 200, zero events, then `done` — a kill switch, a cap breach or a deny rule looked
|
|
1734
|
+
// exactly like the agent choosing to say nothing, and the operator's reason reached nobody.
|
|
1735
|
+
// /v1/chat already returns the reason in its body; the two endpoints must not disagree.
|
|
1736
|
+
if (!result.success && !streamEnded) {
|
|
1737
|
+
sendEvent('error', { message: result.error ?? 'refused by policy' });
|
|
1738
|
+
}
|
|
656
1739
|
if (!streamEnded) {
|
|
657
1740
|
sendEvent('done', {});
|
|
658
1741
|
endStream();
|