@agenthippo-ai/agenthippo 0.2.3-dataplane → 1.110.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/out/agentpack/index.js +3 -1
  2. package/out/agentpack/paths.js +25 -2
  3. package/out/analytics/dashboard/localFleetAdapter.js +82 -2
  4. package/out/analytics/dashboard/provider.js +13 -2
  5. package/out/analytics/dashboard/webview/panel.js +3 -2
  6. package/out/analytics/identity.js +0 -0
  7. package/out/analytics/service.js +29 -25
  8. package/out/analytics/sessionEvents.js +100 -24
  9. package/out/auth/authBaseUrl.js +59 -0
  10. package/out/auth/bootstrapApply.js +9 -10
  11. package/out/auth/brokerClient.js +48 -14
  12. package/out/auth/provider.js +8 -21
  13. package/out/auth/sessionStore.js +3 -40
  14. package/out/auth/signInFlow.js +2 -1
  15. package/out/auth-broker/config.js +50 -5
  16. package/out/auth-broker/server.js +126 -26
  17. package/out/auth-broker/store.js +25 -24
  18. package/out/authority/warrant.js +206 -0
  19. package/out/branding.js +27 -1
  20. package/out/cli/commands/conformance.js +249 -0
  21. package/out/cli/commands/data-plane.js +151 -0
  22. package/out/cli/commands/deploy.js +1 -0
  23. package/out/cli/commands/gateway.js +4 -0
  24. package/out/cli/commands/pack.js +182 -3
  25. package/out/cli/commands/sandbox.js +351 -0
  26. package/out/cli/commands/serve.js +1145 -130
  27. package/out/cli/commands/store.js +12 -4
  28. package/out/cli/credential-context.js +17 -0
  29. package/out/cli/data-plane/gateway.js +28 -5
  30. package/out/cli/data-plane/index.js +18 -2
  31. package/out/cli/data-plane/registry.js +9 -0
  32. package/out/cli/data-plane/remote.js +106 -0
  33. package/out/cli/data-plane/sidecar.js +225 -0
  34. package/out/cli/deploy/layers/identity.js +88 -4
  35. package/out/cli/deploy-init.js +89 -24
  36. package/out/cli/deps.js +106 -2
  37. package/out/cli/idp-init.js +35 -0
  38. package/out/cli/index.js +171 -17
  39. package/out/cli/pack-install-ref.js +15 -0
  40. package/out/cli/pack-loader.js +8 -1
  41. package/out/cli/pack-runtime.js +61 -0
  42. package/out/cli/runtime.js +35 -1
  43. package/out/cli/serve-hardening.js +260 -14
  44. package/out/cli/utils.js +12 -1
  45. package/out/engines/base.js +32 -3
  46. package/out/engines/claude.js +33 -15
  47. package/out/engines/codex.js +111 -27
  48. package/out/engines/custom.js +15 -1
  49. package/out/evidence/sessionShip.js +643 -0
  50. package/out/extension.js +11 -2
  51. package/out/gateway/bridge.js +40 -2
  52. package/out/gateway/client.js +34 -1
  53. package/out/gateway/installSource.js +68 -0
  54. package/out/gateway/lifecycle.js +185 -51
  55. package/out/gateway/outboundPaths.js +4 -1
  56. package/out/gateway/secretsPreflight.js +90 -0
  57. package/out/gateway/service.js +7 -2
  58. package/out/headless/index.js +25 -2
  59. package/out/live-e2e/mcp-demo/server.mjs +32 -0
  60. package/out/mcp.js +30 -0
  61. package/out/pack/caps.js +271 -0
  62. package/out/pack/governor.js +326 -0
  63. package/out/pack/lease-schema.js +273 -0
  64. package/out/pack/lease.js +682 -0
  65. package/out/pack/reconcile.js +217 -0
  66. package/out/pack/signature.js +839 -0
  67. package/out/remote/handleRemoteChat.js +44 -3
  68. package/out/remote/remoteTrust.js +277 -0
  69. package/out/remote/serveClient.js +52 -5
  70. package/out/runner/host.js +141 -0
  71. package/out/runner/protocol.js +128 -0
  72. package/out/runner/remoteEngine.js +166 -0
  73. package/out/runtime/interfaces/edit-tracker.js +2 -0
  74. package/out/runtime/vscode/vscode-edit-tracker.js +2 -0
  75. package/out/sandbox/containerDriver.js +572 -0
  76. package/out/sandbox/processDriver.js +318 -0
  77. package/out/sandbox/sandboxRunner.js +303 -0
  78. package/out/sandbox/support.js +301 -0
  79. package/out/sandbox/types.js +55 -0
  80. package/out/scripts/agentide-litellm.sh +67 -5
  81. package/out/scripts/agentide_litellm_hooks_v1.py +266 -21
  82. package/out/scripts/analytics/agentide-litellm-otel.config.yaml +98 -132
  83. package/out/scripts/analytics/setup-analytics.ps1 +45 -14
  84. package/out/scripts/analytics/setup-analytics.sh +106 -15
  85. package/out/scripts/analytics/start-analytics.sh +86 -14
  86. package/out/scripts/gateway/setup-gateway.sh +72 -32
  87. package/out/shared/models.js +11 -110
  88. package/out/spotlight/install.js +7 -1
  89. package/out/store/index.js +41 -10
  90. package/out/sync.js +3 -1
  91. package/out/test/analytics.test.js +162 -23
  92. package/out/test/auth-baseurl.test.js +132 -0
  93. package/out/test/auth-bootstrapApply.test.js +27 -6
  94. package/out/test/auth-broker-hardening.test.js +227 -0
  95. package/out/test/auth-broker.test.js +59 -9
  96. package/out/test/branding.test.js +13 -0
  97. package/out/test/caps-governor.test.js +460 -0
  98. package/out/test/codex-new-file-survival.test.js +119 -0
  99. package/out/test/configuration-scopes.test.js +131 -0
  100. package/out/test/data-plane-engine-wiring.test.js +32 -0
  101. package/out/test/data-plane.test.js +380 -0
  102. package/out/test/deploy-init.test.js +47 -1
  103. package/out/test/gateway-client.test.js +28 -0
  104. package/out/test/gateway-install-source.test.js +111 -0
  105. package/out/test/gateway-lifecycle.test.js +1 -1
  106. package/out/test/gateway-outbound.test.js +7 -0
  107. package/out/test/gateway-secrets-preflight.test.js +127 -0
  108. package/out/test/gateway-session-routing.test.js +18 -0
  109. package/out/test/governor-hardening.test.js +222 -0
  110. package/out/test/headless-sync.test.js +22 -0
  111. package/out/test/lease-signature.test.js +357 -0
  112. package/out/test/lease-trust-root.test.js +273 -0
  113. package/out/test/lease-validation.test.js +224 -0
  114. package/out/test/lease.test.js +114 -0
  115. package/out/test/localFleetAdapter.test.js +91 -0
  116. package/out/test/mcp.test.js +27 -1
  117. package/out/test/pack-admission.test.js +216 -0
  118. package/out/test/pack-digest-parity.test.js +140 -0
  119. package/out/test/pack-reconcile.test.js +183 -0
  120. package/out/test/pack-signature-artifact.test.js +103 -0
  121. package/out/test/pack-signature.test.js +176 -0
  122. package/out/test/pack-trust-root.test.js +202 -0
  123. package/out/test/remote-trust.test.js +242 -0
  124. package/out/test/runner-protocol.test.js +249 -0
  125. package/out/test/sandbox-driver.test.js +553 -0
  126. package/out/test/sandbox-hardening.test.js +406 -0
  127. package/out/test/sandbox-runner.test.js +343 -0
  128. package/out/test/sandbox-scope.test.js +449 -0
  129. package/out/test/serve-cron-acceptance.test.js +370 -0
  130. package/out/test/serve-gateway-turn.test.js +122 -0
  131. package/out/test/serve-hardening.test.js +227 -15
  132. package/out/test/serveClient.test.js +6 -5
  133. package/out/test/sessionShip.test.js +618 -0
  134. package/out/test/spotlightPublish.test.js +28 -0
  135. package/out/test/store-install-safe.test.js +69 -0
  136. package/out/test/tls-bypass.test.js +90 -0
  137. package/out/test/warrant.test.js +138 -0
  138. package/out/utils.js +46 -9
  139. package/package.json +1 -1
@@ -39,21 +39,38 @@ Object.defineProperty(exports, "__esModule", { value: true });
39
39
  exports.serveCommand = serveCommand;
40
40
  exports.resolveGatewaySessionKey = resolveGatewaySessionKey;
41
41
  exports.runGatewayTurn = runGatewayTurn;
42
+ exports.runBridgeTurn = runBridgeTurn;
42
43
  /**
43
44
  * Serve Command - HTTP Server
44
45
  */
46
+ const crypto = __importStar(require("crypto"));
45
47
  const fs = __importStar(require("fs"));
48
+ const os = __importStar(require("os"));
46
49
  const http = __importStar(require("http"));
47
50
  const path = __importStar(require("path"));
48
51
  const node_crypto_1 = require("node:crypto");
52
+ const sandboxRunner_1 = require("../../sandbox/sandboxRunner");
53
+ const deps_1 = require("../deps");
54
+ const signature_1 = require("../../pack/signature");
55
+ const reconcile_1 = require("../../pack/reconcile");
56
+ const index_1 = require("../../store/index");
57
+ const lease_1 = require("../../pack/lease");
58
+ const caps_1 = require("../../pack/caps");
59
+ const governor_1 = require("../../pack/governor");
60
+ const containerDriver_1 = require("../../sandbox/containerDriver");
61
+ const processDriver_1 = require("../../sandbox/processDriver");
62
+ const support_1 = require("../../sandbox/support");
49
63
  const headless_1 = require("../../headless");
50
64
  const pack_loader_1 = require("../pack-loader");
51
65
  const utils_1 = require("../utils");
52
- const runtime_1 = require("../runtime");
66
+ const pack_runtime_1 = require("../pack-runtime");
67
+ const sessionShip_1 = require("../../evidence/sessionShip");
53
68
  const serve_hardening_1 = require("../serve-hardening");
54
69
  const credential_context_1 = require("../credential-context");
70
+ const lease_2 = require("../../pack/lease");
71
+ const identity_1 = require("../../analytics/identity");
55
72
  const data_plane_1 = require("../data-plane");
56
- const runtime_2 = require("../runtime");
73
+ const runtime_1 = require("../runtime");
57
74
  const outboundSend_1 = require("../../gateway/outboundSend");
58
75
  const sessionEvents_1 = require("../../analytics/sessionEvents");
59
76
  const base_1 = require("../../engines/base");
@@ -64,6 +81,11 @@ const customRegistry_1 = require("../../engines/customRegistry");
64
81
  const BANNER_LINE = '-'.repeat(62);
65
82
  async function serveCommand(options) {
66
83
  const apiConfig = (0, pack_loader_1.getApiConfig)();
84
+ if (apiConfig.useLiteLLM && !apiConfig.litellmApiKey) {
85
+ // Loud, because the failure mode it replaces was silent: traffic reached the gateway
86
+ // authenticated with whatever sat in the provider-key slot, with analytics disabled.
87
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} AGENTHIPPO_LITELLM_BASE_URL is set without AGENTHIPPO_LITELLM_API_KEY — gateway calls will authenticate as 'sk-dummy' (a strict gateway will 401). Provider keys are never sent to the gateway.`);
88
+ }
67
89
  // Load agent pack (required)
68
90
  let pack;
69
91
  const syncLogger = options.verbose ? new headless_1.ConsoleLogger('[Sync]') : new headless_1.NoOpLogger();
@@ -97,7 +119,98 @@ async function serveCommand(options) {
97
119
  }
98
120
  process.exit(1);
99
121
  }
100
- // Sync pack content to workspace
122
+ const logger = options.verbose ? new headless_1.ConsoleLogger('[Server]') : new headless_1.NoOpLogger();
123
+ // ---------------------------------------------------------------------------------------
124
+ // Pack admission — BEFORE anything from the pack reaches the workspace.
125
+ //
126
+ // `syncPackToWorkspace` used to run here, hundreds of lines before the signature was checked,
127
+ // so a pack that was about to be refused had already materialised its skills, AGENTS.md and
128
+ // MCP server definitions into the workspace and into the derived .claude/.codex directories.
129
+ // The refusal then stopped the turn while the content it was refusing sat on disk, reachable
130
+ // by anything else pointed at that workspace. Verification moves ahead of the write.
131
+ // ---------------------------------------------------------------------------------------
132
+ let activePackDir = pack.packPath;
133
+ let activeVersion = pack.manifest.metadata.version ?? '0.0.0';
134
+ let packDigestValue;
135
+ /**
136
+ * Trusted pack-signing keys. NOT frozen at boot.
137
+ *
138
+ * A trust root that is CONFIGURED BUT UNREACHABLE (a 503 during a rolling restart, a DNS blip)
139
+ * is now distinguishable from one that was never configured, and it is retried on the lease
140
+ * poll — one transient fetch at the wrong second must fail the turn, not brick the process.
141
+ */
142
+ let packTrust = await (0, signature_1.loadPackTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
143
+ const reportTrustRoot = (t, label = 'pack') => {
144
+ for (const w of t.warnings) {
145
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${w}`);
146
+ }
147
+ if (t.status === 'unavailable') {
148
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${label} trust root is configured but unusable: ${t.reason}`);
149
+ }
150
+ };
151
+ reportTrustRoot(packTrust);
152
+ /**
153
+ * Verified per turn, with a cheap change probe — see `createPackVerifier`. The RESULT is
154
+ * cached only as long as the pack tree looks untouched and never longer than the full-reverify
155
+ * interval; the REQUIREMENT is re-read from the lease on every poll, so an operator can turn
156
+ * enforcement on for a fleet already running.
157
+ */
158
+ const packVerifier = (0, signature_1.createPackVerifier)();
159
+ let packSignature = { ok: false, kind: 'no-pack', reason: 'not verified' };
160
+ const verifyPackAt = (packDir, version) => {
161
+ packSignature = packVerifier.refresh({ packDir, pack: packId, packVersion: version, keys: packTrust.keys });
162
+ packDigestValue = packSignature.ok ? packSignature.claims.digest : undefined;
163
+ logger.info(packSignature.ok
164
+ ? `[Pack] signature verified for ${packId}`
165
+ : `[Pack] unverified — ${packSignature.reason}`);
166
+ return packSignature;
167
+ };
168
+ /** The refusal an operator sees, with the remedy for THIS failure rather than a generic one. */
169
+ const packRefusalText = (result) => {
170
+ if (result.ok) {
171
+ return '';
172
+ }
173
+ // "Configured but unreachable" outranks whatever the verifier concluded: with no usable
174
+ // keys every signature fails as `no-trust-root`, and telling an operator to configure a
175
+ // trust root they already configured is the least useful thing we could say.
176
+ const kind = packTrust.status === 'unavailable' && result.kind === 'no-trust-root' ? 'trust-root-unavailable' : result.kind;
177
+ return (0, signature_1.describePackRefusal)({
178
+ kind,
179
+ detail: kind === 'trust-root-unavailable' ? (packTrust.reason ?? result.reason) : result.reason,
180
+ pack: packId,
181
+ packVersion: activeVersion,
182
+ packDir: activePackDir,
183
+ trustRootHint: packTrust.url ? `Trust root: ${packTrust.url} (${packTrust.reason ?? 'unavailable'})` : packTrust.reason,
184
+ });
185
+ };
186
+ verifyPackAt(activePackDir, activeVersion);
187
+ /**
188
+ * Signing REQUIRED by default.
189
+ *
190
+ * This is a deliberate break with the previous shipped behaviour, which logged
191
+ * `[Pack] unsigned or unverifiable — …` and then ran the pack anyway. Warn-and-run is not a
192
+ * gate; it is a log line next to an ungated code path, and it meant the default deployment
193
+ * shape executed whatever was in the pack directory.
194
+ *
195
+ * Tighten-only survives intact: the escape hatch below relaxes only this INSTANCE's default,
196
+ * and `resolveRequireSignedPack` checks the lease's demand first, so a control plane that sets
197
+ * `requireSignedPack` cannot be opted out of locally.
198
+ */
199
+ const allowUnsignedPack = Boolean(options.allowUnsignedPack) || (0, signature_1.allowUnsignedPackFromEnv)();
200
+ if (!packSignature.ok) {
201
+ if (!allowUnsignedPack) {
202
+ console.error(`${utils_1.COLORS.red}Error: ${packRefusalText(packSignature)}${utils_1.COLORS.reset}`);
203
+ process.exit(1);
204
+ }
205
+ // Unmissable, every boot, at warn level regardless of --verbose.
206
+ console.warn(BANNER_LINE);
207
+ console.warn(`${utils_1.COLORS.yellow}!! RUNNING AN UNVERIFIED PACK — signature checking is DISABLED on this instance.${utils_1.COLORS.reset}`);
208
+ console.warn(`${utils_1.COLORS.yellow}!! ${packSignature.reason}${utils_1.COLORS.reset}`);
209
+ console.warn(`${utils_1.COLORS.yellow}!! Whatever is in ${activePackDir} will be executed, verified by nobody.${utils_1.COLORS.reset}`);
210
+ console.warn(`${utils_1.COLORS.yellow}!! This is a LOCAL DEVELOPMENT setting (AGENTHIPPO_ALLOW_UNSIGNED_PACK). Do not ship it.${utils_1.COLORS.reset}`);
211
+ console.warn(BANNER_LINE);
212
+ }
213
+ // Sync pack content to workspace — only now that the pack has been admitted.
101
214
  const bundledPromptsDir = (0, utils_1.findBundledBasePromptsDir)();
102
215
  await (0, headless_1.syncPackToWorkspace)(options.workspace, pack.packPath, packId, defaultEngine, syncLogger, bundledPromptsDir);
103
216
  console.log(`OK: Loaded agent: ${pack.manifest.metadata.name} v${pack.manifest.metadata.version}`);
@@ -106,7 +219,31 @@ async function serveCommand(options) {
106
219
  // Resolve persistent session state (serve-v3 §A1: replaces the tmpdir default).
107
220
  const { dir: storagePath, ephemeral } = (0, serve_hardening_1.resolveStateDir)(options.stateDir);
108
221
  const bindHost = (0, serve_hardening_1.resolveBindHost)(options.host);
109
- // Passed to createHeadlessRunner below so the runner's own SessionEventWriter uses the same
222
+ // Who serve believes a caller to be decides whose session state and whose data credentials they
223
+ // reach, so an unpinned identity header is not a detail to leave in a docstring.
224
+ for (const line of (0, serve_hardening_1.identityHeaderAdvice)(process.env, bindHost)) {
225
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${line}`);
226
+ }
227
+ // A pinned header means multiple users; a tier-0 or sandbox-less serve means one filesystem for
228
+ // all of them. That pairing had no boot-time signal at all, while the strictly SAFER container
229
+ // case (--shared-workspace) has warned since it shipped.
230
+ const tenantIsolation = (0, serve_hardening_1.tenantIsolationAdvice)({
231
+ sandbox: options.sandbox,
232
+ sharedWorkspace: Boolean(options.sharedWorkspace),
233
+ workspace: options.workspace,
234
+ stateDir: storagePath,
235
+ });
236
+ if (tenantIsolation.level === 'refuse') {
237
+ for (const line of tenantIsolation.lines) {
238
+ console.error(`${utils_1.COLORS.red}Error:${utils_1.COLORS.reset} ${line}`);
239
+ }
240
+ console.error(`Refusing to start because ${serve_hardening_1.REQUIRE_TENANT_ISOLATION_ENV} is set. Unset it to downgrade this to a warning.`);
241
+ process.exit(1);
242
+ }
243
+ for (const line of tenantIsolation.lines) {
244
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${line}`);
245
+ }
246
+ // Passed to createPackRunner below so the runner's own SessionEventWriter uses the same
110
247
  // root — a fail-closed warrant denial lands in the same per-session JSONL as the tool events
111
248
  // it precedes. The denial happens before runner.run() is ever called, so it needs its own
112
249
  // writer here too. See resolveAnalyticsSessionsDir's docstring for the precedence.
@@ -117,12 +254,61 @@ async function serveCommand(options) {
117
254
  // serve runs the connector broker-side and engines get a per-turn nonce instead of the
118
255
  // credential. The raw data credential never crosses into the agent's trust domain.
119
256
  const dataPlaneCfg = (0, data_plane_1.resolveDataPlaneConfig)(pack.manifest.spec, pack.packPath);
257
+ // EDGE-registration mode (P1 fix, obo-inprogress §2): the trusted edge (IdP Caddy)
258
+ // registers each turn with the sidecar and injects the nonce as a request header; serve
259
+ // merely adopts it. serve holds NO registration credential at all — nothing in this
260
+ // container's /proc can mint a nonce for an arbitrary userId. Takes precedence over the
261
+ // token-based sidecar mode below.
262
+ const edgeCfg = (0, data_plane_1.resolveEdgeDataPlane)();
263
+ const edgeDataPlane = edgeCfg ? new data_plane_1.EdgeDataPlane(edgeCfg) : null;
264
+ // Data-plane SIDECAR mode (secure-obo-design §7.6): when the connector runs in its OWN
265
+ // container, AGENTHIPPO_DATA_PLANE_CONTROL_URL/TOKEN + AGENTHIPPO_DATA_MCP_URL are set and
266
+ // serve holds NO backend base credential — it registers turns with the sidecar and hands the
267
+ // engine only the nonce. Takes precedence over an in-process connector.
268
+ const remoteCfg = edgeDataPlane ? null : (0, data_plane_1.resolveRemoteDataPlane)();
269
+ const remoteDataPlane = remoteCfg ? new data_plane_1.RemoteDataPlane(remoteCfg) : null;
120
270
  // One shared registry: a single per-turn nonce grants BOTH the data-plane and egress
121
- // capabilities (secure-obo-design §7.6 step 2). Created when either broker will run.
122
- const turnRegistry = (dataPlaneCfg || options.gatewayBridge) ? new data_plane_1.TurnRegistry() : null;
123
- const auditEvent = (sessionKey, event, fields) => sessionEventWriter.append(sessionKey, (0, base_1.generateRequestId)(), event, fields);
271
+ // capabilities (secure-obo-design §7.6 step 2). Created when either LOCAL broker will run.
272
+ const turnRegistry = ((dataPlaneCfg && !remoteDataPlane && !edgeDataPlane) || options.gatewayBridge) ? new data_plane_1.TurnRegistry() : null;
273
+ /**
274
+ * Every data-plane audit row carries BOTH principals (PLAN/agent-identity.md §1 failure 3,
275
+ * §10).
276
+ *
277
+ * The user dimension was already on these rows. The agent was not, so evidence could answer
278
+ * "which user touched which rows" but not "via which agent" — the auditor's second question,
279
+ * and the one the whole design exists to make answerable. It was only ever *implied*, by which
280
+ * deployment happened to write the file, and that implication breaks entirely the moment two
281
+ * serves share a connector (§4.4).
282
+ *
283
+ * Read at emit time, not captured once: `activeVersion` and `packDigestValue` both change on a
284
+ * pack hot-swap, and a row must name the pack that actually served it.
285
+ *
286
+ * The digest is carried in full rather than truncated — it is not a secret, and a prefix cannot
287
+ * be checked against a signed artifact, which is the entire point of recording it.
288
+ */
289
+ const auditEvent = (sessionKey, event, fields) => sessionEventWriter.append(sessionKey, (0, base_1.generateRequestId)(), event, {
290
+ ...fields,
291
+ agent: { pack: packId, version: activeVersion, digest: packDigestValue ?? null },
292
+ });
124
293
  let dataPlane = null;
125
- if (dataPlaneCfg) {
294
+ if (edgeDataPlane) {
295
+ process.env[data_plane_1.DATA_MCP_URL_ENV] = edgeDataPlane.url;
296
+ // Belt-and-braces: edge mode must never see a register token, but if one leaked into the
297
+ // deploy env anyway, strip it before any engine child inherits it.
298
+ delete process.env.AGENTHIPPO_DATA_PLANE_TOKEN;
299
+ delete process.env.AGENTHIPPO_DATA_PLANE_CONTROL_URL;
300
+ console.log(`OK: Data plane EDGE-registered — gateway ${edgeDataPlane.url}; turn nonces are minted at the edge, serve holds no registration credential`);
301
+ }
302
+ else if (remoteDataPlane) {
303
+ process.env[data_plane_1.DATA_MCP_URL_ENV] = remoteDataPlane.url;
304
+ // The control-API token is captured in the RemoteDataPlane (serve memory). Strip it from
305
+ // the process env so no engine child inherits it — otherwise the agent could register a
306
+ // turn as ANY user and read their rows (forge-identity escalation). The engine keeps the
307
+ // gateway URL + its own per-turn nonce, which is all it needs (secure-obo-design §7.6).
308
+ delete process.env.AGENTHIPPO_DATA_PLANE_TOKEN;
309
+ console.log(`OK: Data plane SIDECAR — gateway ${remoteDataPlane.url}; base credentials stay in the connector container, not here`);
310
+ }
311
+ else if (dataPlaneCfg) {
126
312
  try {
127
313
  dataPlane = await data_plane_1.DataPlane.start(dataPlaneCfg, {
128
314
  logger: options.verbose ? new headless_1.ConsoleLogger('[DataPlane]') : new headless_1.NoOpLogger(),
@@ -143,12 +329,48 @@ async function serveCommand(options) {
143
329
  // Egress broker (secure-obo-design §7.6): capture the gateway token broker-side BEFORE any
144
330
  // engine spawns, so it can be stripped from the process env the children inherit. The broker
145
331
  // starts alongside the gateway bridge below; the token is held only in this closure.
146
- const brokerGatewayToken = options.gatewayBridge ? (0, runtime_2.readOpenClawGatewayToken)() : undefined;
332
+ // Captured UNCONDITIONALLY, and the env strip is immediate (serve-v4 §10 hole 2): the strip
333
+ // used to live inside the broker branch, so a serve without --gateway-bridge kept the token
334
+ // in process.env and every engine child (they all spread it) inherited a credential nothing
335
+ // in this deployment would ever legitimately use.
336
+ const brokerGatewayToken = (0, runtime_1.readOpenClawGatewayToken)();
337
+ delete process.env.OPENCLAW_GATEWAY_TOKEN;
147
338
  let egressBroker = null;
148
339
  // Mint one nonce per turn and assemble the per-spawn child env. Returns null outside any
149
340
  // broker (no capability to grant). The URLs ride process.env (boot-stable) for config
150
341
  // invariance; only the nonce is per-spawn.
151
- const registerServeTurn = (reg) => {
342
+ const registerServeTurn = async (reg) => {
343
+ // Edge mode: the trusted edge already registered this turn and injected the nonce as a
344
+ // request header — adopt it. No edge nonce (misrouted request, or a channel turn that
345
+ // never traversed the edge) ⇒ no data capability, fail closed at the gateway.
346
+ if (edgeDataPlane) {
347
+ const edgeNonce = reg.edgeNonce?.trim();
348
+ if (!edgeNonce) {
349
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} edge data plane active but no ${data_plane_1.TURN_NONCE_HEADER} header on this turn — data tools will fail closed (check the edge route's forward_auth)`);
350
+ return null;
351
+ }
352
+ const r = edgeDataPlane.adoptTurn(edgeNonce);
353
+ const env = { ...r.env };
354
+ if (egressBroker) {
355
+ env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
356
+ }
357
+ return { nonce: r.nonce, env, release: r.release };
358
+ }
359
+ // Sidecar: register with the connector container; serve never sees a base credential.
360
+ if (remoteDataPlane) {
361
+ // Name the agent on every registration (PR-8). Asserted, not proven — the sidecar
362
+ // records it and flags a mismatch against the pack it loaded, which is how "one
363
+ // connector, one agent" (§4.4) becomes observable rather than merely assumed.
364
+ const r = await remoteDataPlane.registerTurn({
365
+ ...reg,
366
+ agent: { pack: packId, version: activeVersion, digest: packDigestValue },
367
+ });
368
+ const env = { ...r.env };
369
+ if (egressBroker) {
370
+ env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
371
+ }
372
+ return { nonce: r.nonce, env, release: r.release };
373
+ }
152
374
  if (!turnRegistry) {
153
375
  return null;
154
376
  }
@@ -162,6 +384,16 @@ async function serveCommand(options) {
162
384
  }
163
385
  return { nonce, env, release };
164
386
  };
387
+ // Edge-registered turn nonce, injected by the trusted edge per chat request (edge mode only —
388
+ // otherwise ignored). Trusted like the pinned identity header: only the in-network edge can
389
+ // reach serve, and a forged value fails closed at the gateway (unknown nonce ⇒ 401).
390
+ const edgeTurnNonce = (headers) => {
391
+ if (!edgeDataPlane) {
392
+ return undefined;
393
+ }
394
+ const v = headers[data_plane_1.TURN_NONCE_HEADER];
395
+ return Array.isArray(v) ? v[0] : v;
396
+ };
165
397
  const turnLimiter = new serve_hardening_1.TurnLimiter(options.maxTurnsPerSession ?? 0);
166
398
  // Per-identity request-rate cap (req/min). Keyed on the trusted user id (falling back to
167
399
  // client IP) so rotating session_id can't bypass it. AGENTHIPPO_SERVE_RATE_LIMIT=0 disables.
@@ -212,18 +444,22 @@ async function serveCommand(options) {
212
444
  await (0, state_sync_1.ensureSessionLocal)(s3State, storagePath, sessionKey, stateLog);
213
445
  }
214
446
  };
215
- const logger = options.verbose ? new headless_1.ConsoleLogger('[Server]') : new headless_1.NoOpLogger();
216
447
  // Find SDK CLI paths
217
448
  const claudeCodePath = (0, utils_1.findClaudeCodePath)();
218
449
  const codexPath = (0, utils_1.findCodexPath)();
219
- // Validate required CLI is available
220
- if (defaultEngine === 'claude' && !claudeCodePath) {
450
+ // Validate required CLI is available — but ONLY when this process will run the engine.
451
+ // A container sandbox runs the engine inside its own image, so requiring the SDK on the host
452
+ // is wrong and refuses to boot on exactly the minimal host a tier-2 deployment wants
453
+ // (observed live: a Linux host with no npm/codex could not start `--sandbox container`).
454
+ // The process driver shares this host, so it still needs the CLI present.
455
+ const engineRunsInSandbox = options.sandbox === 'container';
456
+ if (!engineRunsInSandbox && defaultEngine === 'claude' && !claudeCodePath) {
221
457
  console.error(`${utils_1.COLORS.red}Error: Claude Code CLI not found.${utils_1.COLORS.reset}`);
222
458
  console.error(' Install: npm install -g @anthropic-ai/claude-code');
223
459
  console.error(' Or set CLAUDE_CODE_PATH environment variable');
224
460
  process.exit(1);
225
461
  }
226
- if (defaultEngine === 'codex' && !codexPath) {
462
+ if (!engineRunsInSandbox && defaultEngine === 'codex' && !codexPath) {
227
463
  console.error(`${utils_1.COLORS.red}Error: Codex CLI not found.${utils_1.COLORS.reset}`);
228
464
  console.error(' Install: npm install -g @openai/codex');
229
465
  console.error(' Or set CODEX_PATH environment variable');
@@ -234,38 +470,646 @@ async function serveCommand(options) {
234
470
  const permissions = {
235
471
  fileAccess: (0, pack_loader_1.getFileAccess)(pack),
236
472
  };
237
- const runner = (0, headless_1.createHeadlessRunner)({
473
+ // R-3 policy provenance (PLAN/compliance.md §3.1): remember WHICH policy is in force and
474
+ // announce changes. `policy_applied` marks the boundary; every turn between two boundaries
475
+ // carries the version via the runner's provenance stamp below, so "was the cap enforced?"
476
+ // stays answerable after anyone edits a cap.
477
+ let lastPolicyVersion;
478
+ let provenanceAnnounced = false;
479
+ const notePolicy = (lease) => {
480
+ const v = (0, lease_1.policyVersion)(lease);
481
+ // Announce once at boot even with no lease (unmanaged deployments still have a digest,
482
+ // an engine build and a tier), then again on every policy change. This event is the
483
+ // PROVENANCE DIMENSION: turns don't repeat these facts — the evidence index joins a
484
+ // turn to the latest announcement for its instance at or before the turn's timestamp.
485
+ if (provenanceAnnounced && v === lastPolicyVersion) {
486
+ return;
487
+ }
488
+ provenanceAnnounced = true;
489
+ lastPolicyVersion = v;
490
+ void sessionEventWriter.append('serve', (0, base_1.generateRequestId)(), 'policy_applied', {
491
+ instance_id: instanceId,
492
+ policy_version: v ?? null,
493
+ pack_digest: packDigestValue ?? null,
494
+ engine_version: (0, deps_1.installedEngineVersion)(defaultEngine) ?? null,
495
+ tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : null,
496
+ enabled: lease?.enabled ?? null,
497
+ pinned_version: lease?.version ?? null,
498
+ caps: lease?.caps ?? null,
499
+ deny_count: lease?.deny?.length ?? 0,
500
+ });
501
+ };
502
+ // Same factory a sandbox uses (cli/pack-runtime.ts) — built after activeVersion is known.
503
+ let inProcessRunner;
504
+ /**
505
+ * Where sandboxes read the pack from.
506
+ *
507
+ * With --pull-packs this is a SYMLINK (`<state>/packs/current`), never the resolved directory:
508
+ * the sandbox drivers capture this path once at construction, so a stable name whose target is
509
+ * swapped is what lets a later version reach new sandboxes without rebuilding the runner. A
510
+ * sandbox already starting keeps the directory it opened, which is exactly the drain semantics
511
+ * version convergence already has.
512
+ */
513
+ const packsRoot = path.join(storagePath, 'packs');
514
+ const storeDownloader = async (slug, version) => {
515
+ const files = await (0, index_1.downloadFromStore)({ type: 'pack', slug, version });
516
+ return files.map((f) => ({ path: f.path, content: f.content }));
517
+ };
518
+ /** Converge to the pinned version if we can; never take the fleet down trying. */
519
+ const reconcileToPin = async (pinned) => {
520
+ if (!options.pullPacks || !pinned || pinned === activeVersion) {
521
+ return;
522
+ }
523
+ const res = await (0, reconcile_1.reconcilePinnedPack)({
524
+ pack: packId,
525
+ currentVersion: activeVersion,
526
+ pinnedVersion: pinned,
527
+ packsRoot,
528
+ download: storeDownloader,
529
+ publicKeysByKid: packTrust.keys,
530
+ logger: { info: (m) => logger.info(m), warn: (m) => logger.info(m) },
531
+ });
532
+ if (res.ok) {
533
+ activeVersion = res.version;
534
+ activePackDir = res.packDir;
535
+ verifyPackAt(activePackDir, activeVersion);
536
+ logger.info(`[Pack] now serving ${packId}@${activeVersion} from ${res.packDir}`);
537
+ return;
538
+ }
539
+ // Deliberately not fatal: the lease fails OPEN on transport, and a bad tag or a store
540
+ // outage must not become a fleet-wide stop. The instance keeps reporting the version it is
541
+ // really running, so the console shows "pinned to X, running Y" rather than an outage.
542
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} cannot converge to ${packId}@${pinned}: ${res.reason}. Still serving @${activeVersion}.`);
543
+ };
544
+ const warrantKey = (0, support_1.loadWarrantKeypair)(logger);
545
+ // Lease: kill switch + version pin. No URL configured ⇒ unmanaged deployment, always admits.
546
+ /** Set once the sandbox runner exists; the lease-poll identity reads it lazily. */
547
+ let sandboxRunnerRef;
548
+ // Stable across restarts, unique per instance: an operator watching the console must see one
549
+ // row per instance that survives a redeploy, not a new row each boot (and not one row per NAT
550
+ // address for a whole fleet). Persisted beside the instance's own state.
551
+ const instanceId = (() => {
552
+ try {
553
+ const file = path.join((0, serve_hardening_1.resolveStateDir)(options.stateDir).dir, 'instance-id');
554
+ if (fs.existsSync(file)) {
555
+ return fs.readFileSync(file, 'utf-8').trim();
556
+ }
557
+ const id = `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`;
558
+ fs.mkdirSync(path.dirname(file), { recursive: true });
559
+ fs.writeFileSync(file, id);
560
+ return id;
561
+ }
562
+ catch {
563
+ // Never let bookkeeping stop a boot; a per-process id still beats collapsing by address.
564
+ return `${os.hostname()}-${crypto.randomBytes(4).toString('hex')}`;
565
+ }
566
+ })();
567
+ /** Lazily read by the lease-poll identity, which is built before the pack path is resolved. */
568
+ const activeVersionRef = { get: () => pack?.manifest.metadata.version };
569
+ // ONE constant for the poll interval and everything derived from it. The cache used to default
570
+ // to 60s while every poll defaulted to 30s and the banner printed 30 — so the real kill latency
571
+ // was up to 3x what an operator was told.
572
+ const leasePollSeconds = (0, lease_1.resolveLeasePollSeconds)(options.leasePollSeconds);
573
+ /**
574
+ * Trusted LEASE-signing keys — a separate root from the pack one.
575
+ *
576
+ * These used to be the same keyring (`signingKeys: () => packTrust.keys`), which made signed
577
+ * leases unreachable end to end: the control plane publishes the lease public key under purpose
578
+ * `lease-signing` and refuses to register one kid under two purposes, so a correctly-configured
579
+ * plane and a correctly-configured serve could never name the same key. With `requireSignedLease`
580
+ * sticky and no plane-side undo, that disagreement latches an instance into refusing turns.
581
+ *
582
+ * A DERIVED root that comes back `unavailable` is the normal state of every plane that publishes
583
+ * no lease-signing key, so it is only reported at boot when an operator named one outright.
584
+ * Otherwise `leaseSigningKeys` says everything there is to say, and only once a signed lease has
585
+ * actually arrived.
586
+ */
587
+ let leaseTrust = await (0, signature_1.loadLeaseTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
588
+ if (leaseTrust.explicitlyConfigured) {
589
+ reportTrustRoot(leaseTrust, 'lease');
590
+ }
591
+ let leaseTrustFallbackWarned = false;
592
+ /**
593
+ * BACK-COMPAT, deliberately not a cutover.
594
+ *
595
+ * A deployment that verifies signed leases against the pack keyring today keeps working: with no
596
+ * lease root named outright, the pack keys stay in play and the (possibly empty) derived lease
597
+ * keys are added to them. The warning fires ONCE, and only when a signed lease is actually being
598
+ * verified, so a deployment that never uses signed leases never sees it.
599
+ *
600
+ * Not a hard switch because `requireSignedLease` latches per instance and the plane has no undo:
601
+ * an instance that stopped recognising its lease key would refuse turns and could not be talked
602
+ * out of it from the console.
603
+ */
604
+ const leaseSigningKeys = () => {
605
+ const selected = (0, signature_1.selectLeaseSigningKeys)(leaseTrust, packTrust);
606
+ if (selected.packFallback && !leaseTrustFallbackWarned) {
607
+ leaseTrustFallbackWarned = true;
608
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} verifying signed leases against the PACK trust root — ` +
609
+ 'no lease trust root is configured on this instance. The control plane publishes lease keys under ' +
610
+ 'purpose `lease-signing` and refuses to register one kid under two purposes, so this stops working ' +
611
+ 'the moment the plane rotates to a dedicated lease key — and requireSignedLease is sticky with no ' +
612
+ 'plane-side undo, so that failure cannot be fixed from the console. Set ' +
613
+ `${signature_1.LEASE_JWKS_URL_ENV}=https://<plane>/.well-known/jwks.json?purpose=lease-signing ` +
614
+ `(or ${signature_1.LEASE_SIGNING_KEYS_ENV} with inline {"<kid>":"<public PEM>"} for an air-gapped install).` +
615
+ (leaseTrust.reason ? ` The derived lease trust root is unusable: ${leaseTrust.reason}` : ''));
616
+ }
617
+ return selected.keys;
618
+ };
619
+ const leaseSource = (0, lease_1.createLeaseSource)({
620
+ url: process.env.AGENTHIPPO_LEASE_URL,
621
+ // The poll IS the registration — see LeaseSourceOptions.identity.
622
+ identity: () => ({
623
+ instance: instanceId,
624
+ pack: packId,
625
+ // The version actually being served: after a pull this is the pinned one, and the
626
+ // console's whole convergence view depends on it being the truth rather than boot state.
627
+ version: activeVersionRef.get(),
628
+ tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
629
+ sandboxes: sandboxRunnerRef?.sandboxes().length,
630
+ // Which engine, and which BUILD of it. Pack version alone cannot explain a fleet-wide
631
+ // behaviour change that came from an engine upgrade, and the CLI is not engine-version
632
+ // agnostic, so the console needs both to answer "what is actually running?".
633
+ engine: defaultEngine,
634
+ engineversion: (0, deps_1.installedEngineVersion)(defaultEngine),
635
+ }),
636
+ // Derived from the poll interval by ONE function, so the two can no longer drift. Worst-case
637
+ // kill latency is poll + cache, and that is the number the boot banner prints.
638
+ cacheSeconds: (0, lease_1.leaseCacheSecondsFor)(leasePollSeconds),
639
+ // Signed-lease verification uses the LEASE trust root, falling back to the pack one while no
640
+ // lease root is configured. Read through a closure so a keyring that arrives on a later poll
641
+ // is picked up without rebuilding this.
642
+ signingKeys: () => leaseSigningKeys(),
643
+ requireSignedLease: options.requireSignedLease,
644
+ instanceId,
645
+ ...(options.leaseMaxStalenessSeconds !== undefined ? { maxStalenessSeconds: options.leaseMaxStalenessSeconds } : {}),
646
+ logger: {
647
+ // Lease warnings are security-relevant (unsigned lease, staleness ceiling, insecure URL)
648
+ // and must not be hidden behind --verbose.
649
+ info: (m) => logger.info(m),
650
+ warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
651
+ },
652
+ });
653
+ /**
654
+ * Retry the trust root while it is missing, and re-verify once it arrives.
655
+ *
656
+ * Only while MISSING: a loaded trust root is not re-fetched, so key rotation still needs the
657
+ * documented restart and a compromised plane cannot swap the trust root under a running
658
+ * instance mid-flight. This exists solely so a transient failure at boot is transient — the
659
+ * `unavailable` status is exactly what makes "transient" distinguishable from "not configured".
660
+ */
661
+ const retrySigningKeys = async () => {
662
+ if (packTrust.status !== 'configured') {
663
+ const next = await (0, signature_1.loadPackTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
664
+ if (next.status === 'configured') {
665
+ packTrust = next;
666
+ reportTrustRoot(next);
667
+ logger.info(`[Pack] signing keys now available (${Object.keys(next.keys).length}); re-verifying`);
668
+ verifyPackAt(activePackDir, activeVersion);
669
+ }
670
+ }
671
+ // The lease root gets the SAME rule, not a better one: retried only while missing, never
672
+ // re-fetched once configured. Improving one and not the other would leave two roots with two
673
+ // rotation stories, which is how the rotation runbook silently stops being true for half of it.
674
+ if (leaseTrust.status !== 'configured') {
675
+ const next = await (0, signature_1.loadLeaseTrustRoot)({ leaseUrl: process.env.AGENTHIPPO_LEASE_URL });
676
+ if (next.status === 'configured') {
677
+ leaseTrust = next;
678
+ if (next.explicitlyConfigured) {
679
+ reportTrustRoot(next, 'lease');
680
+ }
681
+ logger.info(`[Lease] lease-signing keys now available (${Object.keys(next.keys).length})`);
682
+ }
683
+ }
684
+ };
685
+ // Record the boot policy and watch for changes on the lease-poll cadence. The lease source
686
+ // caches, so this adds no extra fetches — it re-reads whatever the poll already trusts.
687
+ void leaseSource.get().then(notePolicy).catch(() => undefined);
688
+ {
689
+ const everyMs = leasePollSeconds * 1000;
690
+ const policyTimer = setInterval(() => {
691
+ void leaseSource.get().then(notePolicy).catch(() => undefined);
692
+ void retrySigningKeys().catch(() => undefined);
693
+ }, everyMs);
694
+ policyTimer.unref?.();
695
+ }
696
+ if (options.pullPacks) {
697
+ fs.mkdirSync(packsRoot, { recursive: true });
698
+ // Materialise whatever we boot with, so `current` is valid before any pin is considered and
699
+ // the drivers can capture one stable path.
700
+ const bootDir = path.join(packsRoot, activeVersion);
701
+ if (!fs.existsSync(bootDir)) {
702
+ fs.cpSync(pack.packPath, bootDir, { recursive: true });
703
+ }
704
+ (0, reconcile_1.promoteCurrent)(packsRoot, bootDir);
705
+ activePackDir = path.join(packsRoot, reconcile_1.CURRENT_LINK);
706
+ await reconcileToPin((await leaseSource.get())?.version);
707
+ // The pack is now read through `current`, which is a different path than the one verified
708
+ // above. Re-verify so `packSignature` describes the tree that will actually run.
709
+ verifyPackAt(activePackDir, activeVersion);
710
+ }
711
+ activeVersionRef.get = () => activeVersion;
712
+ inProcessRunner = (0, pack_runtime_1.createPackRunner)({
238
713
  storagePath,
239
714
  analyticsSessionsDir,
240
715
  packId,
241
- packVersion: pack?.manifest.metadata.version,
242
- bundledPromptsDir,
243
- apiKey: apiConfig.apiKey,
244
- baseUrl: apiConfig.baseUrl,
245
- useLiteLLM: apiConfig.useLiteLLM,
246
- // Per-destination keys so each engine authenticates to the endpoint it actually dials
247
- // (codex→OpenAI, claude→Anthropic, or the LiteLLM proxy) — no provider-blind collapse.
248
- litellmApiKey: apiConfig.litellmApiKey,
249
- openaiApiKey: apiConfig.openaiApiKey,
250
- anthropicApiKey: apiConfig.anthropicApiKey,
716
+ packVersion: activeVersion,
717
+ provenance: () => ({
718
+ policyVersion: lastPolicyVersion,
719
+ packDigest: packDigestValue,
720
+ engineVersion: (0, deps_1.installedEngineVersion)(defaultEngine),
721
+ tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
722
+ instanceId,
723
+ }),
724
+ workspaceRoot: options.workspace,
725
+ credentials: {
726
+ apiKey: apiConfig.apiKey,
727
+ baseUrl: apiConfig.baseUrl,
728
+ useLiteLLM: apiConfig.useLiteLLM,
729
+ litellmApiKey: apiConfig.litellmApiKey,
730
+ openaiApiKey: apiConfig.openaiApiKey,
731
+ anthropicApiKey: apiConfig.anthropicApiKey,
732
+ },
251
733
  logger,
252
- freshState: false,
253
- claudeCodeExecutablePath: claudeCodePath,
254
- codexExecutablePath: codexPath,
255
734
  permissionHandler,
256
- cliEnvPatch: (0, runtime_1.buildCliEnvPatch)({
257
- cliPath: process.execPath,
258
- workspaceRoot: options.workspace,
259
- agentId: packId,
260
- baseEnv: process.env,
261
- }),
262
- ...(claudeCheapModel ? { engineSettings: { claudeCheapModel } } : {}),
735
+ claudeCheapModel,
736
+ omitGatewayToken: true,
263
737
  });
738
+ // Converge on the same cadence the fleet already polls, so pinning a version in the console
739
+ // reaches instances without a redeploy. Errors are swallowed by reconcileToPin itself.
740
+ if (options.pullPacks) {
741
+ const every = leasePollSeconds * 1000;
742
+ const timer = setInterval(() => {
743
+ void leaseSource.get().then((l) => reconcileToPin(l?.version)).catch(() => undefined);
744
+ }, every);
745
+ timer.unref?.();
746
+ }
747
+ // Container sandboxes got NO filesystem mounts at all — the driver honours `fs`, but serve never
748
+ // passed it, so a sandbox's `/state/workspace` was container-private storage. Everything the
749
+ // agent wrote was invisible on the host and lost when the sandbox died. The process driver needs
750
+ // no mounts: tier 0 shares this host's filesystem already.
751
+ //
752
+ // The workspace is PER SESSION unless --shared-workspace says otherwise. One host directory
753
+ // mounted writable into every session's sandbox is a shared disk between users of the same
754
+ // agent: whatever session A writes (a query export, a downloaded file), session B reads on its
755
+ // next turn. That silently contradicts "one disposable sandbox per session" for everything that
756
+ // matters, since the container boundary does not extend to a directory both containers mount.
757
+ const sharedWorkspace = Boolean(options.sharedWorkspace);
758
+ // One sandbox per conversation unless told otherwise. Every key below derives from this, and
759
+ // setting it back to `session` is the whole rollback.
760
+ const sandboxScope = options.sandboxScope === 'user' ? 'user' : 'session';
761
+ const sandboxMounts = options.sandbox === 'container'
762
+ ? {
763
+ // serve's resolved pack, not the image's baked copy — the two cannot then disagree
764
+ // about which pack version is running.
765
+ readonly: [`${activePackDir}:${containerDriver_1.SANDBOX_PACK_DIR}`],
766
+ writable: [`${options.workspace}:${containerDriver_1.SANDBOX_WORKSPACE_DIR}`],
767
+ }
768
+ : undefined;
769
+ if (sandboxMounts && sharedWorkspace) {
770
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} --shared-workspace: every session's sandbox mounts ` +
771
+ `${options.workspace} read-write, so sessions can read and overwrite each other's files. ` +
772
+ `Only safe when all sessions belong to one trust domain.`);
773
+ }
774
+ // serve wrote that workspace as root; the sandbox runs non-root and must be able to write it too.
775
+ // Boot-time and once, before the first turn can arrive.
776
+ let sandboxImageUser;
777
+ if (sandboxMounts && options.sandboxImage) {
778
+ await (0, support_1.alignSandboxWorkspaceOwnership)({
779
+ exec: support_1.dockerExec,
780
+ image: options.sandboxImage,
781
+ writable: sandboxMounts.writable,
782
+ // A failure here is what an operator must see even without --verbose; success is noise.
783
+ logger: { info: (m) => logger.info(m), warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`) },
784
+ });
785
+ sandboxImageUser = await (0, support_1.resolveSandboxImageUser)(support_1.dockerExec, options.sandboxImage);
786
+ }
787
+ // Engine state must outlive the sandboxes that use it or the conversation restarts every time a
788
+ // sandbox is reaped (serve-v4-sandbox §8.4). Keyed at SANDBOX scope, under serve's own state
789
+ // dir: one directory per session by default, or one per user under `--sandbox-scope user`,
790
+ // where the sessions sharing it stay separate beneath in `provider-sessions/`.
791
+ const sandboxStateDirFor = (sandboxKey) => (0, support_1.ensureSandboxStateDir)(path.join(storagePath, 'sandbox-state', (0, state_sync_1.safeSessionId)(sandboxKey)), sandboxImageUser);
792
+ /**
793
+ * Mounts for ONE sandbox: the pack read-only, and a workspace only that sandbox sees.
794
+ *
795
+ * The workspace lives beside the engine state it belongs to, so the same reap/prune machinery
796
+ * already covers it and the agent's output stays visible on the host — the property the shared
797
+ * mount was added for — without being visible to every other sandbox.
798
+ *
799
+ * Keyed at sandbox scope, never narrower: the in-sandbox host chdir's here on every turn, and
800
+ * that is only safe while every session sharing a sandbox resolves the same directory (see the
801
+ * INVARIANT note in cli/commands/sandbox.ts). Under user scope this means a user's own sessions
802
+ * share a workspace — same trust domain, on the same terms their engine homes already shared.
803
+ */
804
+ const sandboxFsFor = async (sandboxKey) => {
805
+ if (!sandboxMounts) {
806
+ return { readonly: [], writable: [] };
807
+ }
808
+ if (sharedWorkspace) {
809
+ return sandboxMounts;
810
+ }
811
+ const dir = (0, support_1.ensureSandboxStateDir)(path.join(storagePath, 'sandbox-workspace', (0, state_sync_1.safeSessionId)(sandboxKey)), sandboxImageUser);
812
+ return { readonly: sandboxMounts.readonly, writable: [`${dir}:${containerDriver_1.SANDBOX_WORKSPACE_DIR}`] };
813
+ };
814
+ /**
815
+ * "May this pack run, at this version, for this session?" — signature requirement, version pin,
816
+ * image digest and scoped deny rules, in ONE place.
817
+ *
818
+ * Both gates call this: sandbox creation (where it decides whether a sandbox may be born) and,
819
+ * on deployments with no sandbox, the per-turn governor gate. Two copies of these four checks
820
+ * is exactly the drift this codebase keeps paying for, and the no-sandbox path having no copy
821
+ * at all is why `requireSignedPack` and a version pin could be switched on in the console and
822
+ * enforce nothing on the default deployment shape.
823
+ */
824
+ const assertPackAdmitted = (currentLease, sessionKey, userId) => {
825
+ const required = (0, signature_1.resolveRequireSignedPack)({
826
+ // The lease's demand is read FIRST and cannot be waived by any local setting — that is
827
+ // what keeps the requirement tighten-only now that the local default is "required".
828
+ lease: currentLease?.requireSignedPack,
829
+ localRequire: options.requireSignedPack,
830
+ localAllowUnsigned: allowUnsignedPack,
831
+ });
832
+ if (required) {
833
+ // RE-VERIFIED, not read off a boot-time boolean. In the default no-sandbox shape the
834
+ // engine runs in THIS process against a pack scaffolded with `fileAccess: full`, so the
835
+ // agent can edit its own pack directory between turns; a cached "verified once at boot"
836
+ // would pass every one of those later turns. `createPackVerifier` keeps this cheap — a
837
+ // stat-only probe in the common case, a full digest when the tree looks touched or the
838
+ // re-verify interval has elapsed.
839
+ const result = packVerifier.verify({
840
+ packDir: activePackDir, pack: packId, packVersion: activeVersion, keys: packTrust.keys,
841
+ });
842
+ if (result !== packSignature) {
843
+ packSignature = result;
844
+ packDigestValue = result.ok ? result.claims.digest : undefined;
845
+ }
846
+ if (!result.ok) {
847
+ throw new lease_1.LeaseDeniedError(packRefusalText(result));
848
+ }
849
+ }
850
+ // sessionKey makes admission SCOPED: a deny rule naming this session or its user refuses
851
+ // at the same gate that enforces the fleet-wide switch.
852
+ (0, lease_1.assertAdmitted)(currentLease, {
853
+ pack: packId,
854
+ version: activeVersion,
855
+ image: options.sandboxImage,
856
+ sessionKey,
857
+ userHash: serve_hardening_1.userNamespaceHash,
858
+ userId,
859
+ });
860
+ };
861
+ // Assigned just below; the identity closure above reads it lazily so a poll can report live
862
+ // sandbox count without forcing the runner to be constructed before the lease source.
863
+ // eslint-disable-next-line prefer-const
864
+ // Sandbox mode: the same HeadlessRunner shape, body relocated into a per-session sandbox.
865
+ // serve's turn loop below is untouched — see sandbox/sandboxRunner.ts.
866
+ const sandboxRunner = options.sandbox
867
+ ? (0, sandboxRunner_1.createSandboxRunner)({
868
+ driver: options.sandbox === 'container'
869
+ // A non-internal route network is a real weakening of the tier-2 claim, so its warning
870
+ // must reach an operator who did not pass --verbose.
871
+ ? new containerDriver_1.ContainerSandboxDriver({
872
+ exec: support_1.dockerExec,
873
+ socketOwner: sandboxImageUser,
874
+ logger: {
875
+ info: (m) => logger.info(m),
876
+ warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
877
+ },
878
+ })
879
+ : new processDriver_1.ProcessSandboxDriver({
880
+ command: process.execPath,
881
+ // --workspace is REQUIRED, not optional: a tier-0 sandbox shares this host's
882
+ // filesystem, and serve sends this same path as `workspaceRoot` on every run
883
+ // frame. Omitting it left the sandbox on the container-image default
884
+ // (/state/workspace), so a serve-launched process sandbox died at boot with
885
+ // ENOENT: mkdir '/state/workspace'.
886
+ // pack.packPath, not options.agent: serve has ALREADY resolved the pack (a
887
+ // name resolves against ~/.agent-hippo/agents and the workspace), and the
888
+ // sandbox must run that exact directory rather than re-resolve a name from a
889
+ // different cwd and possibly land on a different pack.
890
+ args: [
891
+ // Only forward the script path when the process actually HAS one. A
892
+ // single-file build reports `process.argv[1]` as a path inside its own
893
+ // virtual filesystem (`/$bunfs/root/index.js` for `bun build --compile`,
894
+ // `/snapshot/...` for pkg / node SEA). Passing that made the child parse
895
+ // it as the subcommand and die with `unknown command '/$bunfs/...'`, so
896
+ // every tier-0 sandbox failed with a blank "sandbox channel closed".
897
+ // An existence check is NOT enough to detect this: those runtimes mount
898
+ // the virtual path, so `fs.existsSync` cheerfully returns true for it.
899
+ ...(0, support_1.sandboxScriptArg)(),
900
+ 'sandbox', '--agent', activePackDir, '--stdio',
901
+ '--state-dir', storagePath, '--workspace', options.workspace,
902
+ ],
903
+ // PID/user-namespace wrap where the host allows it (residual P4: on a
904
+ // single-uid host, a concurrent tenant reads a turn's nonce from
905
+ // /proc/<pid>/environ — and the claude CLI's argv from /proc/<pid>/cmdline,
906
+ // which is world-readable). Default 'auto': hardened where possible, honest
907
+ // warning where not; 'require' makes multi-tenant hosts fail closed.
908
+ namespaceIsolation: (0, processDriver_1.namespaceIsolationFromEnv)(),
909
+ logger: {
910
+ info: (m) => logger.info(m),
911
+ warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`),
912
+ },
913
+ }),
914
+ image: options.sandboxImage ?? '',
915
+ tier: options.sandbox === 'container' ? 2 : 0,
916
+ ...(sandboxMounts ? { fs: sandboxMounts, fsFor: sandboxFsFor, stateDirFor: sandboxStateDirFor } : {}),
917
+ // The only routes a sandbox may reach. Everything else is unroutable, not merely denied.
918
+ routes: [
919
+ ...(process.env.AGENTHIPPO_DATA_MCP_URL ? [{ kind: 'data', target: process.env.AGENTHIPPO_DATA_MCP_URL }] : []),
920
+ ...(apiConfig.baseUrl ? [{ kind: 'inference', target: apiConfig.baseUrl }] : []),
921
+ ],
922
+ limits: { ttlSeconds: options.sandboxTtlSeconds ?? 30 * 60 },
923
+ provenanceEnv: () => (0, runtime_1.buildProvenanceEnv)({
924
+ policyVersion: lastPolicyVersion,
925
+ packDigest: packDigestValue,
926
+ engineVersion: (0, deps_1.installedEngineVersion)(defaultEngine),
927
+ tier: options.sandbox === 'container' ? 2 : options.sandbox ? 0 : undefined,
928
+ instanceId,
929
+ }),
930
+ // Kill switch + rollback, checked before a sandbox can exist. Revoking the lease stops
931
+ // new work fleet-wide immediately; TTL bounds the sandboxes already running.
932
+ // sessionKey, not sandboxKey: deny rules can name a SESSION, and under user scope the
933
+ // sandbox key names the user instead — checking that would stop matching them.
934
+ admit: async ({ sessionKey }) => {
935
+ const currentLease = await leaseSource.get();
936
+ assertPackAdmitted(currentLease, sessionKey);
937
+ },
938
+ inferenceBaseUrl: apiConfig.baseUrl,
939
+ // Per-sandbox inference warrant — signed, audience-bound, and naming the sandbox it was
940
+ // minted for. The provider key never enters a sandbox; the proxy verifies with a public
941
+ // key and can never mint, so a sandbox cannot re-issue itself more budget.
942
+ //
943
+ // `user` is the verified end user from the turn's credential scope, not the pack: the
944
+ // gateway strips caller-supplied x-litellm-metadata-user-id and re-stamps it from this
945
+ // claim, so the warrant is the only attribution channel that survives it — stamping
946
+ // packId here made every downstream user_id (LiteLLM, spend rows, evidence provenance)
947
+ // the pack. Every mint runs inside runWithCredential (create-time on first-turn
948
+ // acquire, then per-turn re-mints in RemoteEngine.stream), so the ambient credential is
949
+ // the turn's own. An unidentified turn gets a sandbox-scoped marker rather than
950
+ // silently borrowing the pack's name — the sandbox key is user-namespaced under either
951
+ // scope, so the marker cannot collide across identified users.
952
+ mintModelToken: (sandboxId, sandboxKey) => (0, support_1.mintSandboxInferenceWarrant)(warrantKey, {
953
+ user: (0, credential_context_1.currentCredential)()?.userId?.trim() || `anon:${sandboxKey}`,
954
+ pack: packId,
955
+ packVersion: activeVersion,
956
+ sandboxId,
957
+ tier: options.sandbox === 'container' ? 2 : 0,
958
+ // A sandbox outlives one turn, so its model warrant gets the full cap rather than a
959
+ // turn-length TTL; the spend cap, not the clock, is what bounds it.
960
+ ttlSeconds: 300,
961
+ }),
962
+ logger,
963
+ runtime: { logger, editTracker: undefined, configProvider: undefined, permissionHandler },
964
+ })
965
+ : undefined;
966
+ sandboxRunnerRef = sandboxRunner ?? undefined;
967
+ // Caps + kill switch. Enforcement is non-cooperative — it revokes a route or destroys the
968
+ // sandbox — so it exists only where there ARE sandboxes. Without --sandbox we say so rather than
969
+ // pretending a cap is in force.
970
+ // 0 / NaN mean "unlimited" for every one of these — `--max-turns-per-session` has documented
971
+ // that meaning since before caps existed, and passing its default 0 through as a real cap
972
+ // refused the very first turn (`turn cap reached (1/0)`, observed live).
973
+ const positive = (n) => typeof n === 'number' && Number.isFinite(n) && n > 0 ? n : undefined;
974
+ const localCaps = {
975
+ ...(0, caps_1.capsFromEnv)(),
976
+ ...(positive(options.maxTurnsPerSession) !== undefined ? { maxTurnsPerSession: options.maxTurnsPerSession } : {}),
977
+ ...(positive(options.maxSessionSeconds) !== undefined ? { maxSessionSeconds: options.maxSessionSeconds } : {}),
978
+ ...(positive(options.maxConcurrentSandboxes) !== undefined ? { maxConcurrentSandboxes: options.maxConcurrentSandboxes } : {}),
979
+ ...(options.capOnBreach ? { onBreach: options.capOnBreach } : {}),
980
+ };
981
+ const capsRequested = localCaps.maxTurnsPerSession !== undefined
982
+ || localCaps.maxSessionSeconds !== undefined
983
+ || localCaps.maxConcurrentSandboxes !== undefined;
984
+ // A deployment must be governable whether or not it runs sandboxes. Gating the governor on
985
+ // `sandboxRunner` meant a plain `serve` never polled its lease at all: invisible to the control
986
+ // plane AND immune to the kill switch, which is the opposite of what an operator configuring
987
+ // a lease URL is asking for. Without sandboxes the enforcement that needs one degrades (no route
988
+ // revocation, no concurrency bound) but the substance survives — the turn is still refused.
989
+ const nullSandboxes = {
990
+ sandboxes: () => [],
991
+ destroySession: async () => undefined,
992
+ revokeRoute: async () => undefined,
993
+ // There is nothing here that can stop a turn already executing in this process, so a
994
+ // `kill` deny rule degrades to `drain`. The governor now says so loudly instead of
995
+ // sweeping an empty list and reporting the rule as applied.
996
+ supportsInFlightTermination: false,
997
+ };
998
+ const governed = Boolean(sandboxRunner || process.env.AGENTHIPPO_LEASE_URL || capsRequested);
999
+ const governor = governed
1000
+ ? (0, governor_1.createGovernor)({
1001
+ // The lease's caps bind; local flags may tighten them and never loosen them.
1002
+ caps: (0, caps_1.mergeCaps)((await leaseSource.get())?.caps, localCaps),
1003
+ sandboxes: sandboxRunner ?? nullSandboxes,
1004
+ getLease: () => leaseSource.get(),
1005
+ pollSeconds: leasePollSeconds,
1006
+ // Scope for deny rules: the pack this instance runs, and the same hash that namespaces
1007
+ // session keys — so the console can name a user by email without serve holding a roster.
1008
+ pack: packId,
1009
+ userHash: serve_hardening_1.userNamespaceHash,
1010
+ // Only when there is no sandbox: with one, the same checks gate a sandbox's BIRTH and
1011
+ // running sandboxes are meant to drain on TTL. Without one, this is the only gate that
1012
+ // exists, so an unsigned pack or an unpinned version would otherwise keep serving.
1013
+ ...(sandboxRunner ? {} : {
1014
+ admitPack: (sessionKey, lease) => {
1015
+ try {
1016
+ // No userId here on purpose: the governor has ALREADY evaluated the deny rules for
1017
+ // this turn (with its followed-user map, which this call site cannot see), so the
1018
+ // only thing left for this gate is the signature/pin/digest half.
1019
+ assertPackAdmitted(lease, sessionKey);
1020
+ return { allowed: true };
1021
+ }
1022
+ catch (err) {
1023
+ return { allowed: false, reason: err instanceof Error ? err.message : String(err) };
1024
+ }
1025
+ },
1026
+ }),
1027
+ logger,
1028
+ audit: (event, fields) => auditEvent(String(fields.sessionKey ?? 'instance'), event, fields),
1029
+ })
1030
+ : undefined;
1031
+ if (capsRequested && !sandboxRunner) {
1032
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} caps are configured but --sandbox is not set. Turn and duration caps still refuse the turn; the concurrency cap and route revocation need sandboxes, so those are IGNORED.`);
1033
+ }
1034
+ const baseRunner = sandboxRunner ?? inProcessRunner;
1035
+ const runner = governor ? (0, governor_1.withGovernor)(baseRunner, governor) : baseRunner;
1036
+ if (governor) {
1037
+ governor.start();
1038
+ const p = governor.policy();
1039
+ // The banner prints the number an operator will be held to: worst-case kill latency is
1040
+ // poll + cache, not the poll alone. It used to print "lease poll 30s" while the cache
1041
+ // defaulted to 60, so a kill could take three times as long as advertised.
1042
+ const killLatency = (0, lease_1.effectiveKillLatencySeconds)(leasePollSeconds);
1043
+ logger.info(`[Serve] control plane: caps turns=${p.maxTurnsPerSession ?? '∞'} session=${p.maxSessionSeconds ?? '∞'}s sandboxes=${p.maxConcurrentSandboxes ?? '∞'} onBreach=${p.onBreach ?? 'revoke-inference'}; lease poll ${leasePollSeconds}s (worst-case kill latency ${killLatency}s; an admin refresh is immediate)`);
1044
+ if (!sandboxRunner) {
1045
+ logger.info('[Serve] no sandbox driver: a deny rule with mode=kill takes effect as DRAIN — the in-flight turn finishes and only the next one is refused.');
1046
+ }
1047
+ const leaseStatus = leaseSource.status();
1048
+ if (process.env.AGENTHIPPO_LEASE_URL && !leaseStatus.requireSignedLease) {
1049
+ logger.info('[Serve] lease signature is NOT required — set --require-signed-lease (or lease.requireSignedLease) once the control plane signs leases.');
1050
+ }
1051
+ }
1052
+ /**
1053
+ * Unattended runs (PLAN/agent-identity.md §5). Two gates, BOTH operator-side, plus a third
1054
+ * that needs no code here: the identity must be mapped in the connector's role map or the
1055
+ * connector fails the call closed exactly as it does for an unmapped human.
1056
+ *
1057
+ * Nothing in the pack participates. Enabling or disabling this never requires a re-sign.
1058
+ */
1059
+ const serviceIdentity = (0, serve_hardening_1.resolveServiceIdentity)();
1060
+ if (serviceIdentity.error) {
1061
+ // A malformed value must not degrade to "anonymous": that shape looks configured and runs
1062
+ // every scheduled job with no identity at all.
1063
+ console.error(`${utils_1.COLORS.red}Error:${utils_1.COLORS.reset} ${serviceIdentity.error}`);
1064
+ process.exit(1);
1065
+ }
1066
+ if (serviceIdentity.warning) {
1067
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${serviceIdentity.warning}`);
1068
+ }
1069
+ const serviceTurnsManaged = Boolean(process.env.AGENTHIPPO_LEASE_URL);
1070
+ /** Resolved per turn, not at boot: revoking in the console must take effect on the next poll. */
1071
+ const serviceUserIfPermitted = async () => {
1072
+ if (!serviceIdentity.identity) {
1073
+ return undefined;
1074
+ }
1075
+ const gate = (0, lease_2.resolveAllowServiceTurns)({
1076
+ lease: (await leaseSource.get())?.allowServiceTurns,
1077
+ managed: serviceTurnsManaged,
1078
+ });
1079
+ if (!gate.allowed) {
1080
+ // Named gate, so the operator reads which knob to turn rather than "denied".
1081
+ auditEvent('service-turn', 'service_turn_denied', {
1082
+ identity: (0, identity_1.hashIdentity)(serviceIdentity.identity),
1083
+ gate: gate.gate,
1084
+ detail: (0, lease_2.describeServiceTurnGate)(gate.gate),
1085
+ });
1086
+ return undefined;
1087
+ }
1088
+ return serviceIdentity.identity;
1089
+ };
1090
+ if (serviceIdentity.identity) {
1091
+ const boot = (0, lease_2.resolveAllowServiceTurns)({ lease: undefined, managed: serviceTurnsManaged });
1092
+ logger.info(`[Serve] unattended runs: identity configured (${serve_hardening_1.SERVICE_USER_ENV}); `
1093
+ + (serviceTurnsManaged
1094
+ ? 'the control plane decides per turn via lease.allowServiceTurns — absent means deny'
1095
+ : (0, lease_2.describeServiceTurnGate)(boot.gate)));
1096
+ }
1097
+ if (sandboxRunner) {
1098
+ logger.info(`[Serve] sandbox mode: ${options.sandbox} (tier ${options.sandbox === 'container' ? 2 : 0}) — engines run in per-session sandboxes`);
1099
+ // A sandbox's ONLY model credential is the per-sandbox warrant, which is meaningful only to an
1100
+ // inference proxy. With no proxy configured there is nothing to send it to, so the engine
1101
+ // dials the provider bare and the first turn dies with an opaque 401/reconnect loop. Say so
1102
+ // at boot — same reason the engine credential check above is a boot check, not a turn check.
1103
+ if (!apiConfig.baseUrl) {
1104
+ console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} --sandbox without an inference proxy: a sandbox holds no provider key, so model calls will fail 401. Set AGENTHIPPO_LITELLM_BASE_URL.`);
1105
+ }
1106
+ }
264
1107
  /** Set when `--gateway-bridge` is enabled; cleared on shutdown. */
265
1108
  let gatewayBridge = null;
266
1109
  /** True once the pack is synced and (if bridged) the gateway is connected. Drives GET /ready. */
267
1110
  let ready = false;
268
1111
  // Create HTTP server
1112
+ let evidenceShipHandle;
269
1113
  const server = http.createServer(async (req, res) => {
270
1114
  const cors = (0, serve_hardening_1.corsHeaders)();
271
1115
  for (const [k, v] of Object.entries(cors)) {
@@ -279,8 +1123,63 @@ async function serveCommand(options) {
279
1123
  try {
280
1124
  // Liveness — always open.
281
1125
  if (req.method === 'GET' && req.url === '/health') {
1126
+ const health = { status: 'ok' };
1127
+ // Reported unconditionally: with only `last_ok_at`, "shipping is off" and "shipping
1128
+ // is on but has never once succeeded" look identical to an operator and to alerting.
1129
+ health.evidence_ship_enabled = Boolean(evidenceShipHandle);
1130
+ const lastOk = evidenceShipHandle?.lastOkAt();
1131
+ if (lastOk !== undefined) {
1132
+ health.evidence_ship_last_ok_at = new Date(lastOk).toISOString();
1133
+ }
1134
+ // R-13 gap detection: evidence write failures and permanently-skipped files are
1135
+ // COUNTED now rather than swallowed, but a counter nobody can read is the same as no
1136
+ // counter. `disabledReason` matters most — "shipping is off because
1137
+ // AGENTHIPPO_EVIDENCE_SHIP=0" and "shipping is on and failing" produce the same
1138
+ // `enabled: false` shaped alert otherwise.
1139
+ const ship = (0, sessionShip_1.getEvidenceShipStatus)();
1140
+ health.evidence_ship_disabled_reason = ship.disabledReason ?? null;
1141
+ health.evidence_ship_consecutive_failures = ship.consecutiveFailures;
1142
+ health.evidence_records_staged = ship.recordsStaged;
1143
+ health.evidence_records_shipped = ship.recordsShipped;
1144
+ // Non-zero means the record is known to be INCOMPLETE, not merely late.
1145
+ health.evidence_records_skipped_too_old = ship.recordsSkippedTooOld;
1146
+ health.evidence_files_skipped_too_old = ship.filesSkippedTooOld;
1147
+ health.evidence_content_capture = ship.contentCapture;
1148
+ if (ship.lastHeartbeatAt !== undefined) {
1149
+ health.evidence_ship_last_heartbeat_at = new Date(ship.lastHeartbeatAt).toISOString();
1150
+ }
1151
+ const writes = (0, sessionEvents_1.getSessionEventWriterStats)();
1152
+ health.evidence_writes_ok = writes.written;
1153
+ health.evidence_writes_failed = writes.failed;
1154
+ if (writes.lastError) {
1155
+ health.evidence_write_last_error = {
1156
+ at: new Date(writes.lastError.at).toISOString(),
1157
+ event_type: writes.lastError.eventType,
1158
+ message: writes.lastError.message,
1159
+ };
1160
+ }
1161
+ // Governance surface: an operator asking "is this instance actually governed?" gets
1162
+ // the answer here rather than from boot logs that have long since scrolled away.
1163
+ const leaseState = leaseSource.status();
1164
+ health.lease_configured = Boolean(process.env.AGENTHIPPO_LEASE_URL);
1165
+ health.lease_signed = leaseState.signed;
1166
+ health.lease_signature_required = leaseState.requireSignedLease;
1167
+ health.lease_staleness_seconds = leaseState.stalenessSeconds ?? null;
1168
+ health.lease_fail_closed = leaseState.failClosed;
1169
+ health.pack_signature_ok = packSignature.ok;
1170
+ health.pack_signature_required = (0, signature_1.resolveRequireSignedPack)({
1171
+ localRequire: options.requireSignedPack, localAllowUnsigned: allowUnsignedPack,
1172
+ });
1173
+ health.pack_trust_root = packTrust.status;
1174
+ // Reported separately from the pack root: "signed leases are required and the lease
1175
+ // keyring is unavailable" is an outage in the making, and it is invisible in
1176
+ // `pack_trust_root`. `lease_trust_root_source` distinguishes an operator-named root
1177
+ // from the pack-keyring fallback, which is the state that stops working on rotation.
1178
+ health.lease_trust_root = leaseTrust.status;
1179
+ health.lease_trust_root_source = leaseTrust.explicitlyConfigured ? 'lease' : 'pack-fallback';
1180
+ health.kill_in_flight_supported = Boolean(sandboxRunner);
282
1181
  res.writeHead(200, { 'Content-Type': 'application/json' });
283
- res.end(JSON.stringify({ status: 'ok' }));
1182
+ res.end(JSON.stringify(health));
284
1183
  return;
285
1184
  }
286
1185
  // Readiness — pack synced (+ gateway connected when bridged).
@@ -316,11 +1215,11 @@ async function serveCommand(options) {
316
1215
  }
317
1216
  // User-namespaced session key (secure-obo-design §3.4): kills cross-user session
318
1217
  // attachment via a guessed session_id. Fails closed on pinned-identity deployments.
319
- const { sessionKey, rejected } = (0, serve_hardening_1.resolveServeSessionKey)(body.session_id, userId);
1218
+ const { sessionKey, sandboxKey, rejected } = (0, serve_hardening_1.resolveServeSessionKey)(body.session_id, userId, sandboxScope);
320
1219
  if (rejected) {
321
1220
  sessionEventWriter.append('unverified', (0, base_1.generateRequestId)(), 'warrant_denied', { reason: rejected });
322
1221
  res.writeHead(403, { 'Content-Type': 'application/json' });
323
- res.end(JSON.stringify({ error: 'session_id requires an authenticated user' }));
1222
+ res.end(JSON.stringify({ error: 'verified user identity required' }));
324
1223
  return;
325
1224
  }
326
1225
  // Acquire before turn-limiter so a capacity reject does not burn a turn slot.
@@ -346,15 +1245,15 @@ async function serveCommand(options) {
346
1245
  }
347
1246
  await prepareSession(sessionKey);
348
1247
  if (userId?.trim()) {
349
- turnReg = registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken });
1248
+ turnReg = await registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken, edgeNonce: edgeTurnNonce(req.headers) });
350
1249
  }
351
- const result = await (0, credential_context_1.runWithCredential)({ userId, dataToken }, () => handleChat(runner, body, sessionKey, defaultEngine, defaultModel, options.workspace, permissions, userId, turnReg?.env), { onDenied: logWarrantDenied(sessionKey) });
1250
+ const result = await (0, credential_context_1.runWithCredential)({ userId, dataToken }, () => handleChat(runner, body, sessionKey, sandboxKey, defaultEngine, defaultModel, options.workspace, permissions, userId, turnReg?.env), { onDenied: logWarrantDenied(sessionKey) });
352
1251
  res.writeHead(200, { 'Content-Type': 'application/json' });
353
1252
  res.end(JSON.stringify(result));
354
1253
  statePusher?.schedule(sessionKey); // per-session delta push, off the response path
355
1254
  }
356
1255
  finally {
357
- turnReg?.release();
1256
+ await turnReg?.release();
358
1257
  release();
359
1258
  }
360
1259
  return;
@@ -373,11 +1272,11 @@ async function serveCommand(options) {
373
1272
  return;
374
1273
  }
375
1274
  // Same user-namespaced session key + fail-closed rule as /v1/chat above.
376
- const { sessionKey, rejected } = (0, serve_hardening_1.resolveServeSessionKey)(body.session_id, userId);
1275
+ const { sessionKey, sandboxKey, rejected } = (0, serve_hardening_1.resolveServeSessionKey)(body.session_id, userId, sandboxScope);
377
1276
  if (rejected) {
378
1277
  sessionEventWriter.append('unverified', (0, base_1.generateRequestId)(), 'warrant_denied', { reason: rejected });
379
1278
  res.writeHead(403, { 'Content-Type': 'application/json' });
380
- res.end(JSON.stringify({ error: 'session_id requires an authenticated user' }));
1279
+ res.end(JSON.stringify({ error: 'verified user identity required' }));
381
1280
  return;
382
1281
  }
383
1282
  // Acquire BEFORE writeHead(200, SSE) so a reject still gets a clean 429.
@@ -401,7 +1300,7 @@ async function serveCommand(options) {
401
1300
  }
402
1301
  await prepareSession(sessionKey);
403
1302
  if (userId?.trim()) {
404
- turnReg = registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken });
1303
+ turnReg = await registerServeTurn({ userId: userId.trim(), sessionKey, inboundToken: dataToken, edgeNonce: edgeTurnNonce(req.headers) });
405
1304
  }
406
1305
  // Fail-closed credential check MUST run before SSE writeHead(200).
407
1306
  // Otherwise a warrant denial lands after headers are committed, the outer
@@ -413,12 +1312,12 @@ async function serveCommand(options) {
413
1312
  'Cache-Control': 'no-cache',
414
1313
  'Connection': 'keep-alive',
415
1314
  });
416
- await handleStreamingChat(runner, body, sessionKey, defaultEngine, defaultModel, options.workspace, permissions, res, userId, turnReg?.env);
1315
+ await handleStreamingChat(runner, body, sessionKey, sandboxKey, defaultEngine, defaultModel, options.workspace, permissions, res, userId, turnReg?.env);
417
1316
  }, { onDenied: logWarrantDenied(sessionKey) });
418
1317
  statePusher?.schedule(sessionKey); // per-session delta push, off the response path
419
1318
  }
420
1319
  finally {
421
- turnReg?.release();
1320
+ await turnReg?.release();
422
1321
  release();
423
1322
  }
424
1323
  return;
@@ -472,11 +1371,19 @@ async function serveCommand(options) {
472
1371
  console.log('');
473
1372
  console.log('Press Ctrl+C to stop.');
474
1373
  console.log('');
1374
+ const shipConfig = (0, sessionShip_1.resolveSessionShipConfig)(instanceId, analyticsSessionsDir, storagePath);
1375
+ if (shipConfig) {
1376
+ evidenceShipHandle = (0, sessionShip_1.startSessionShipper)({
1377
+ ...shipConfig,
1378
+ logger: { info: (m) => logger.info(m), warn: (m) => console.warn(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} ${m}`) },
1379
+ });
1380
+ }
475
1381
  // Bind channels before bridge connect so config.patch reload does not race registration.
476
1382
  const bindSpec = options.bind?.trim();
477
1383
  if (bindSpec) {
478
1384
  try {
479
- await bindToGatewayChannels(gatewayAgentId, bindSpec);
1385
+ // The captured token — the env var was stripped at boot (§10 hole 2).
1386
+ await bindToGatewayChannels(gatewayAgentId, bindSpec, brokerGatewayToken);
480
1387
  }
481
1388
  catch (err) {
482
1389
  console.error(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} Failed to bind channels: ${err instanceof Error ? err.message : String(err)}`);
@@ -500,13 +1407,15 @@ async function serveCommand(options) {
500
1407
  audit: auditEvent,
501
1408
  });
502
1409
  process.env[data_plane_1.EGRESS_URL_ENV] = egressBroker.url;
503
- // Now that the broker holds the token, strip it from the process env so no
504
- // engine child (all spread process.env) inherits it (§7.4 gap 1 close).
505
- delete process.env.OPENCLAW_GATEWAY_TOKEN;
1410
+ // The token itself was stripped from process.env at boot, broker or not.
506
1411
  console.log('OK: Egress broker on loopback — gateway token stays broker-side; agent sends via turn nonce');
507
1412
  }
508
1413
  catch (err) {
509
- console.error(`${utils_1.COLORS.yellow}Warning:${utils_1.COLORS.reset} Egress broker failed to start: ${err instanceof Error ? err.message : String(err)}`);
1414
+ // Fail CLOSED: without the broker, AGENTHIPPO_EGRESS_URL stays unset and the
1415
+ // runtime would bake the gateway token back into engine children (§7.6 undone
1416
+ // by a port-bind hiccup). Refuse to serve rather than degrade silently.
1417
+ console.error(`${utils_1.COLORS.red}Error:${utils_1.COLORS.reset} Egress broker failed to start: ${err instanceof Error ? err.message : String(err)}`);
1418
+ process.exit(1);
510
1419
  }
511
1420
  }
512
1421
  try {
@@ -525,8 +1434,10 @@ async function serveCommand(options) {
525
1434
  turnLimiter,
526
1435
  warnGateReject,
527
1436
  prepareSession,
1437
+ sandboxScope,
528
1438
  onTurnComplete: (sessionId) => statePusher?.schedule(sessionId),
529
1439
  logWarrantDenied,
1440
+ serviceUserIfPermitted,
530
1441
  });
531
1442
  }
532
1443
  catch (err) {
@@ -580,6 +1491,17 @@ async function serveCommand(options) {
580
1491
  // best-effort
581
1492
  }
582
1493
  }
1494
+ // After the planes are closed, so the tail includes whatever they just wrote. SIGTERM is
1495
+ // how every redeploy ends, and without this each one drops up to one interval of
1496
+ // session evidence — permanently, when the state dir (and its cursor) is ephemeral.
1497
+ if (evidenceShipHandle) {
1498
+ try {
1499
+ await evidenceShipHandle.flush();
1500
+ }
1501
+ catch {
1502
+ // best-effort — never block shutdown on the evidence server being reachable
1503
+ }
1504
+ }
583
1505
  server.close(() => {
584
1506
  runner.dispose();
585
1507
  process.exit(0);
@@ -603,7 +1525,7 @@ async function parseBody(req) {
603
1525
  throw new Error('Invalid JSON');
604
1526
  }
605
1527
  }
606
- async function handleChat(runner, body, sessionKey, defaultEngine, defaultModel, workspace, permissions, userId, turnEnv) {
1528
+ async function handleChat(runner, body, sessionKey, sandboxKey, defaultEngine, defaultModel, workspace, permissions, userId, turnEnv) {
607
1529
  const result = await runner.run({
608
1530
  engineId: resolveRequestedEngineId(body.engine, defaultEngine),
609
1531
  modelId: body.model || defaultModel,
@@ -612,6 +1534,9 @@ async function handleChat(runner, body, sessionKey, defaultEngine, defaultModel,
612
1534
  // The user-namespaced key (never raw body.session_id) so engine conversation
613
1535
  // state and per-session artifacts are per-user — see resolveServeSessionKey.
614
1536
  sessionId: sessionKey,
1537
+ // Which sandbox runs it. Equal to sessionKey unless --sandbox-scope user; ignored
1538
+ // entirely by the in-process runner.
1539
+ sandboxKey,
615
1540
  analyticsUserId: userId,
616
1541
  permissions,
617
1542
  turnEnv,
@@ -646,21 +1571,70 @@ async function handleChat(runner, body, sessionKey, defaultEngine, defaultModel,
646
1571
  * reply handles under it (bridge.ts upsertGatewayOutboundHandle), so the outbound-routing
647
1572
  * file must keep resolving from the un-namespaced key or reply-to-sender breaks.
648
1573
  */
649
- function resolveGatewaySessionKey(opts, userId) {
1574
+ function resolveGatewaySessionKey(opts, userId, scope = 'session',
1575
+ /**
1576
+ * This turn runs as the deployment's SERVICE identity — no human behind it
1577
+ * (PLAN/agent-identity.md §5.4). Passed explicitly rather than sniffed from a prefix on
1578
+ * `userId`: deriving behaviour from the shape of an identity string is how the `"false"` class
1579
+ * of bug happens, and §5.2 deliberately allows identities with no recognisable prefix at all.
1580
+ */
1581
+ isServiceIdentity = false) {
1582
+ if (isServiceIdentity && userId) {
1583
+ // Two independent reasons this branch cannot reuse the durable human shape.
1584
+ //
1585
+ // sessionKey FRESH: TurnLimiter is a monotonic per-session counter with no window, no
1586
+ // reset and no decay (serve-hardening.ts). A recurring job on a stable key therefore burns
1587
+ // a LIFETIME budget one run per night and, on night maxTurnsPerSession + 1, is rejected
1588
+ // with `turn_limit` forever — recovering only on a serve restart. A nightly job that dies
1589
+ // after N nights, where N is a cap nobody associates with cron.
1590
+ //
1591
+ // sandboxKey STABLE: under the default --sandbox-scope session it would otherwise track
1592
+ // sessionKey, so "fresh key per run" would silently mean a NEW CONTAINER every night —
1593
+ // cold start, a new governor entry per run, and maxSessionSeconds that never accumulates.
1594
+ // Holding it stable gives the bot one warm sandbox and, more importantly, ONE handle for
1595
+ // caps and the kill switch.
1596
+ //
1597
+ // Net shape: fresh conversation, warm sandbox.
1598
+ const stable = `gateway-${userId}`;
1599
+ return {
1600
+ sessionKey: `${stable}--${(0, node_crypto_1.randomUUID)()}`,
1601
+ sandboxKey: stable,
1602
+ // Replies must still route: the egress broker resolves this run's destinations from the
1603
+ // raw channel key, so a cron job that messages Slack keeps working.
1604
+ outboundKey: opts.sessionId?.trim() || opts.gatewaySessionKey?.trim() || stable,
1605
+ // No prior state to restore and no future turn to restore it for. A batch job that
1606
+ // silently accumulates context also silently accumulates cost; continuity across runs
1607
+ // stays available by passing a stable session_id, which takes the durable branch below.
1608
+ durable: false,
1609
+ };
1610
+ }
1611
+ // The SAME derivation resolveServeSessionKey uses. If the two disagreed, one person reaching
1612
+ // the same serve over HTTP and over a channel would get two sandboxes under user scope —
1613
+ // neither the old behaviour nor the new one.
1614
+ const userSandboxKey = userId ? `u${(0, serve_hardening_1.userNamespaceHash)(userId)}` : undefined;
650
1615
  const explicit = opts.sessionId?.trim() || opts.gatewaySessionKey?.trim();
651
1616
  if (explicit) {
652
1617
  const sessionKey = userId ? `u${(0, serve_hardening_1.userNamespaceHash)(userId)}--${explicit}` : explicit;
653
- return { sessionKey, outboundKey: explicit, durable: true };
1618
+ const sandboxKey = scope === 'user' && userSandboxKey ? userSandboxKey : sessionKey;
1619
+ return { sessionKey, sandboxKey, outboundKey: explicit, durable: true };
654
1620
  }
655
1621
  if (userId) {
656
1622
  // Already keyed on the sender's own identity — no cross-user attachment surface.
657
- return { sessionKey: `gateway-${userId}`, outboundKey: `gateway-${userId}`, durable: true };
1623
+ const sessionKey = `gateway-${userId}`;
1624
+ return {
1625
+ sessionKey,
1626
+ sandboxKey: scope === 'user' && userSandboxKey ? userSandboxKey : sessionKey,
1627
+ outboundKey: sessionKey,
1628
+ durable: true,
1629
+ };
658
1630
  }
659
1631
  // randomUUID (122 bits), not generateRequestId (~32 bits) — this is the sole thing
660
1632
  // standing between two unrelated anonymous senders and a real collision, so it needs to
661
1633
  // be architecturally collision-free, not just statistically unlikely.
662
1634
  const anon = `gateway-anon-${(0, node_crypto_1.randomUUID)()}`;
663
- return { sessionKey: anon, outboundKey: anon, durable: false };
1635
+ // Its own sandbox, under either scope: with no identity there is no user to group by, and
1636
+ // grouping unidentified senders would be inventing one.
1637
+ return { sessionKey: anon, sandboxKey: anon, outboundKey: anon, durable: false };
664
1638
  }
665
1639
  /**
666
1640
  * Gateway-channel turn admission: rate limit → turn gate (capacity) → per-session turn cap,
@@ -711,9 +1685,9 @@ async function startServeGatewayBridge(params) {
711
1685
  const { GatewayBridge: Bridge } = await import('../../gateway/bridge.js');
712
1686
  const DEFAULT_GATEWAY_PORT = 18789;
713
1687
  const gatewayUrl = process.env.OPENCLAW_GATEWAY_URL ?? `ws://127.0.0.1:${DEFAULT_GATEWAY_PORT}`;
714
- // Use the captured token — serve strips OPENCLAW_GATEWAY_TOKEN from process.env once the
715
- // egress broker holds it, so the bridge must not depend on the env var (§7.6).
716
- const token = params.brokerGatewayToken ?? process.env.OPENCLAW_GATEWAY_TOKEN;
1688
+ // The captured token — OPENCLAW_GATEWAY_TOKEN is stripped from process.env at boot (§7.6,
1689
+ // §10 hole 2), so the bridge must never depend on the env var.
1690
+ const token = params.brokerGatewayToken;
717
1691
  const bridge = new Bridge({ gatewayUrl, token });
718
1692
  await bridge.connect();
719
1693
  await bridge.registerAgent({
@@ -723,88 +1697,120 @@ async function startServeGatewayBridge(params) {
723
1697
  workspace: params.workspace,
724
1698
  engineId: params.engine,
725
1699
  syncOpenClawConfigAgent: true,
726
- messageHandler: async (message, opts) => {
727
- const requestedEngine = opts.engineId?.trim().toLowerCase();
728
- const effectiveEngine = requestedEngine === 'custom'
729
- ? params.engine
730
- : (requestedEngine || params.engine);
731
- // #4: the channel sender becomes the per-user user_id (analytics + per-user data scoping).
732
- const userId = (0, serve_hardening_1.peerToUserId)(opts.peerInfo);
733
- const { sessionKey, outboundKey, durable } = resolveGatewaySessionKey(opts, userId);
734
- return runGatewayTurn({
735
- rateLimiter: params.rateLimiter,
736
- turnLimiter: params.turnLimiter,
737
- turnGate: params.turnGate,
738
- warnGateReject: params.warnGateReject,
739
- // Same identity-keyed cap the HTTP endpoints enforce — a channel sender is already
740
- // trusted enough here to gate the OBO credential scope below, so it's trusted enough
741
- // to rate-limit on directly. Unidentified senders share one 'gateway' bucket rather
742
- // than being exempt from throttling entirely.
743
- rlKey: userId ?? 'gateway',
744
- sessionKey,
745
- trackSession: durable,
746
- emitter: opts.emitter,
747
- run: async () => {
748
- // A one-off anonymous sessionKey can never have prior state to restore, and
749
- // nothing will ever look it up again to push state for — skip both S3 round-trips.
750
- if (durable) {
751
- await params.prepareSession(sessionKey);
752
- }
753
- // Channel-originated turns carry no inbound platform token: with the
754
- // passthrough provider they fail closed at the gateway (an unlinked chat
755
- // peer gets NO data credential — secure-obo-design §7.5 bind-before-
756
- // credential); the identity provider maps the peer id server-side instead.
757
- // egressStoreKey is the RAW gateway key (outboundKey) the bridge records reply
758
- // handles under — it lets the egress broker resolve this session's bound
759
- // destinations server-side (§7.6). userId gates BOTH capabilities: an
760
- // unidentified peer gets neither a nonce nor an egress route.
761
- const turnReg = userId?.trim()
762
- ? params.registerTurn({ userId: userId.trim(), sessionKey, egressStoreKey: outboundKey })
763
- : null;
764
- try {
765
- const result = await (0, credential_context_1.runWithCredential)({ userId }, () => params.runner.run({
766
- engineId: effectiveEngine,
767
- modelId: params.defaultModel,
768
- message,
769
- workspaceRoot: params.workspace,
770
- // Use the resolved sessionKey (not the raw opts fields) so the engine's own
771
- // session state and the S3 restore/push above key off the same, user-namespaced
772
- // identity. The outbound-routing file (buildSessionEnvPatch) keeps the raw
773
- // gateway key — the bridge records reply handles under it, so namespacing it
774
- // here would break reply-to-sender. See resolveGatewaySessionKey.
775
- sessionId: sessionKey,
776
- gatewaySessionKey: outboundKey,
777
- analyticsUserId: userId,
778
- emitter: opts.emitter,
779
- permissions: params.permissions,
780
- turnEnv: turnReg?.env,
781
- }), { onDenied: params.logWarrantDenied(sessionKey) });
782
- if (durable) {
783
- params.onTurnComplete?.(sessionKey);
784
- }
785
- return { success: result.success, error: result.error };
786
- }
787
- finally {
788
- turnReg?.release();
789
- }
790
- },
791
- });
792
- },
1700
+ messageHandler: (message, opts) => runBridgeTurn(params, message, opts),
793
1701
  });
794
1702
  console.log(`${utils_1.COLORS.green}✓${utils_1.COLORS.reset} Gateway bridge: registered "${params.gatewayAgentId}" at ${gatewayUrl}`);
795
1703
  console.log(`${utils_1.COLORS.dim} Gateway outbound routes: ${params.workspace}/.agent-hippo/sessions/<sessionKey>/gateway-outbound.json${utils_1.COLORS.reset}`);
796
1704
  console.log(`${utils_1.COLORS.dim} openclaw agent --agent ${params.gatewayAgentId} --message "…"${utils_1.COLORS.reset}`);
797
1705
  return bridge;
798
1706
  }
1707
+ /**
1708
+ * One gateway-bridge turn, extracted from the message handler so the shipped path is directly
1709
+ * testable.
1710
+ *
1711
+ * It was previously an inline closure inside `bridge.registerAgent`, reachable only through a live
1712
+ * OpenClaw websocket — so the composition that actually delivers unattended runs (five identity
1713
+ * sites, four session-key values, the limiter, the registry) had no test that ran it. Transcribing
1714
+ * it into a test would have been a second implementation, free to drift from the one that ships.
1715
+ */
1716
+ async function runBridgeTurn(params, message, opts) {
1717
+ const requestedEngine = opts.engineId?.trim().toLowerCase();
1718
+ const effectiveEngine = requestedEngine === 'custom'
1719
+ ? params.engine
1720
+ : (requestedEngine || params.engine);
1721
+ // #4: the channel sender becomes the per-user user_id (analytics + per-user data scoping).
1722
+ const peerUser = (0, serve_hardening_1.peerToUserId)(opts.peerInfo);
1723
+ // No chat peer ⇒ this is an unattended run (cron, or a channel event with no sender).
1724
+ // It may act as the deployment's own service identity if BOTH operator-side gates
1725
+ // allow it (PLAN/agent-identity.md §5.6). Never on the anonymous HTTP lane — that is
1726
+ // externally reachable, and an unauthenticated caller must not be promotable to a
1727
+ // standing identity; this handler is the gateway-bridge lane only.
1728
+ const svcUser = peerUser ? undefined : await params.serviceUserIfPermitted?.();
1729
+ const effectiveUser = peerUser ?? svcUser;
1730
+ // effectiveUser flows to FIVE places, not two. Missing any of them degrades the design
1731
+ // silently rather than loudly — see each site below.
1732
+ const { sessionKey, sandboxKey, outboundKey, durable } = resolveGatewaySessionKey(opts, effectiveUser, params.sandboxScope, Boolean(svcUser));
1733
+ return runGatewayTurn({
1734
+ rateLimiter: params.rateLimiter,
1735
+ turnLimiter: params.turnLimiter,
1736
+ turnGate: params.turnGate,
1737
+ warnGateReject: params.warnGateReject,
1738
+ // (1) Same identity-keyed cap the HTTP endpoints enforce — a channel sender is already
1739
+ // trusted enough here to gate the OBO credential scope below, so it's trusted enough
1740
+ // to rate-limit on directly. Unidentified senders share one 'gateway' bucket rather
1741
+ // than being exempt from throttling entirely. A service turn gets its OWN bucket:
1742
+ // leaving it in the shared pool would let one bot's schedule throttle every
1743
+ // anonymous sender, and vice versa.
1744
+ rlKey: effectiveUser ?? 'gateway',
1745
+ sessionKey,
1746
+ trackSession: durable,
1747
+ emitter: opts.emitter,
1748
+ run: async () => {
1749
+ // A one-off anonymous sessionKey can never have prior state to restore, and
1750
+ // nothing will ever look it up again to push state for — skip both S3 round-trips.
1751
+ if (durable) {
1752
+ await params.prepareSession(sessionKey);
1753
+ }
1754
+ // Channel-originated turns carry no inbound platform token: with the
1755
+ // passthrough provider they fail closed at the gateway (an unlinked chat
1756
+ // peer gets NO data credential — secure-obo-design §7.5 bind-before-
1757
+ // credential); the identity provider maps the peer id server-side instead.
1758
+ // egressStoreKey is the RAW gateway key (outboundKey) the bridge records reply
1759
+ // handles under — it lets the egress broker resolve this session's bound
1760
+ // destinations server-side (§7.6). userId gates BOTH capabilities: an
1761
+ // unidentified peer gets neither a nonce nor an egress route.
1762
+ // (2) Data authority.
1763
+ const turnReg = effectiveUser?.trim()
1764
+ ? await params.registerTurn({ userId: effectiveUser.trim(), sessionKey, egressStoreKey: outboundKey })
1765
+ : null;
1766
+ try {
1767
+ // (3) Warrant + spend. The easy site to miss and the expensive one: the
1768
+ // warrant's `user` claim reads `currentCredential()?.userId ?? anon:<sandboxKey>`,
1769
+ // so registering without ALSO scoping here gives the bot data access under its
1770
+ // service identity while its SPEND attributes to `anon:` — outside the per-user
1771
+ // meter and unattributable in evidence.
1772
+ const result = await (0, credential_context_1.runWithCredential)({ userId: effectiveUser }, () => params.runner.run({
1773
+ engineId: effectiveEngine,
1774
+ modelId: params.defaultModel,
1775
+ message,
1776
+ workspaceRoot: params.workspace,
1777
+ // Use the resolved sessionKey (not the raw opts fields) so the engine's own
1778
+ // session state and the S3 restore/push above key off the same, user-namespaced
1779
+ // identity. The outbound-routing file (buildSessionEnvPatch) keeps the raw
1780
+ // gateway key — the bridge records reply handles under it, so namespacing it
1781
+ // here would break reply-to-sender. See resolveGatewaySessionKey.
1782
+ sessionId: sessionKey,
1783
+ // (4) Which sandbox runs it — stable across runs for a service
1784
+ // identity, so a nightly job reuses one warm container.
1785
+ sandboxKey,
1786
+ gatewaySessionKey: outboundKey,
1787
+ // (5) Attribution. Left unset, Spotlight rows would have no user
1788
+ // while the warrant claims the service identity — two attribution
1789
+ // channels disagreeing, which is worse than one being wrong.
1790
+ analyticsUserId: effectiveUser,
1791
+ emitter: opts.emitter,
1792
+ permissions: params.permissions,
1793
+ turnEnv: turnReg?.env,
1794
+ }), { onDenied: params.logWarrantDenied(sessionKey) });
1795
+ if (durable) {
1796
+ params.onTurnComplete?.(sessionKey);
1797
+ }
1798
+ return { success: result.success, error: result.error };
1799
+ }
1800
+ finally {
1801
+ await turnReg?.release();
1802
+ }
1803
+ },
1804
+ });
1805
+ }
799
1806
  /**
800
1807
  * Connect to the gateway and set channel bindings for this agent.
801
1808
  * Called when --bind is specified (e.g., --bind "telegram,whatsapp" or --bind "*").
802
1809
  */
803
- async function bindToGatewayChannels(agentId, bindSpec) {
1810
+ async function bindToGatewayChannels(agentId, bindSpec, token) {
804
1811
  const { createAgentIDEGatewayClient } = await import('../../gateway/client.js');
805
1812
  const DEFAULT_GATEWAY_PORT = 18789;
806
1813
  const gatewayUrl = process.env.OPENCLAW_GATEWAY_URL ?? `ws://127.0.0.1:${DEFAULT_GATEWAY_PORT}`;
807
- const token = process.env.OPENCLAW_GATEWAY_TOKEN;
808
1814
  const { client, clientInfo } = createAgentIDEGatewayClient(gatewayUrl, token);
809
1815
  try {
810
1816
  await client.connect(clientInfo);
@@ -852,7 +1858,7 @@ async function bindToGatewayChannels(agentId, bindSpec) {
852
1858
  client.close();
853
1859
  }
854
1860
  }
855
- async function handleStreamingChat(runner, body, sessionKey, defaultEngine, defaultModel, workspace, permissions, res, userId, turnEnv) {
1861
+ async function handleStreamingChat(runner, body, sessionKey, sandboxKey, defaultEngine, defaultModel, workspace, permissions, res, userId, turnEnv) {
856
1862
  let streamEnded = false;
857
1863
  const sendEvent = (type, data) => {
858
1864
  if (streamEnded || res.writableEnded) {
@@ -901,18 +1907,27 @@ async function handleStreamingChat(runner, body, sessionKey, defaultEngine, defa
901
1907
  },
902
1908
  };
903
1909
  try {
904
- await runner.run({
1910
+ const result = await runner.run({
905
1911
  engineId: resolveRequestedEngineId(body.engine, defaultEngine),
906
1912
  modelId: body.model || defaultModel,
907
1913
  message: body.message,
908
1914
  workspaceRoot: workspace,
909
1915
  // The user-namespaced key (never raw body.session_id) — see resolveServeSessionKey.
910
1916
  sessionId: sessionKey,
1917
+ sandboxKey,
911
1918
  analyticsUserId: userId,
912
1919
  emitter,
913
1920
  permissions,
914
1921
  turnEnv,
915
1922
  });
1923
+ // A policy refusal is a RESULT, not an exception: `withGovernor` returns
1924
+ // `{success:false, error}` without ever touching the emitter. Ignoring it sent the client
1925
+ // HTTP 200, zero events, then `done` — a kill switch, a cap breach or a deny rule looked
1926
+ // exactly like the agent choosing to say nothing, and the operator's reason reached nobody.
1927
+ // /v1/chat already returns the reason in its body; the two endpoints must not disagree.
1928
+ if (!result.success && !streamEnded) {
1929
+ sendEvent('error', { message: result.error ?? 'refused by policy' });
1930
+ }
916
1931
  if (!streamEnded) {
917
1932
  sendEvent('done', {});
918
1933
  endStream();