@agenthippo-ai/agenthippo 1.110.9 → 1.110.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/agentpack/index.js +3 -1
- package/out/agentpack/paths.js +25 -2
- package/out/analytics/dashboard/localFleetAdapter.js +82 -2
- package/out/analytics/service.js +6 -0
- package/out/analytics/sessionEvents.js +4 -3
- package/out/auth-broker/server.js +23 -19
- package/out/cli/commands/data-plane.js +84 -0
- package/out/cli/commands/sandbox.js +28 -9
- package/out/cli/commands/serve.js +289 -97
- package/out/cli/credential-context.js +17 -0
- package/out/cli/data-plane/index.js +2 -1
- package/out/cli/data-plane/sidecar.js +66 -2
- package/out/cli/deploy/layers/identity.js +5 -2
- package/out/cli/deploy-init.js +4 -4
- package/out/cli/index.js +16 -1
- package/out/cli/serve-hardening.js +181 -12
- package/out/engines/claude.js +1 -1
- package/out/engines/codex.js +1 -1
- package/out/engines/custom.js +7 -1
- package/out/evidence/sessionShip.js +50 -17
- package/out/headless/index.js +8 -1
- package/out/pack/governor.js +32 -15
- package/out/pack/lease-schema.js +4 -1
- package/out/pack/lease.js +42 -0
- package/out/runner/host.js +24 -14
- package/out/runner/protocol.js +24 -4
- package/out/runner/remoteEngine.js +53 -17
- package/out/sandbox/containerDriver.js +4 -2
- package/out/sandbox/processDriver.js +4 -1
- package/out/sandbox/sandboxRunner.js +148 -36
- package/out/scripts/agentide_litellm_hooks_v1.py +85 -1
- package/out/scripts/analytics/agentide-litellm-otel.config.yaml +4 -0
- package/out/scripts/analytics/setup-analytics.ps1 +8 -1
- package/out/scripts/analytics/setup-analytics.sh +13 -2
- package/out/test/caps-governor.test.js +6 -1
- package/out/test/data-plane-engine-wiring.test.js +32 -0
- package/out/test/data-plane.test.js +313 -0
- package/out/test/deploy-init.test.js +7 -1
- package/out/test/governor-hardening.test.js +6 -1
- package/out/test/lease-validation.test.js +55 -0
- package/out/test/localFleetAdapter.test.js +91 -0
- package/out/test/sandbox-driver.test.js +2 -2
- package/out/test/sandbox-hardening.test.js +12 -1
- package/out/test/sandbox-runner.test.js +1 -1
- package/out/test/sandbox-scope.test.js +449 -0
- package/out/test/serve-cron-acceptance.test.js +370 -0
- package/out/test/serve-gateway-turn.test.js +122 -0
- package/out/test/serve-hardening.test.js +182 -9
- package/out/test/sessionShip.test.js +58 -0
- package/package.json +1 -1
|
@@ -5,30 +5,58 @@
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.createSandboxRunner = createSandboxRunner;
|
|
7
7
|
const remoteEngine_1 = require("../runner/remoteEngine");
|
|
8
|
+
const paths_1 = require("../agentpack/paths");
|
|
9
|
+
const credential_context_1 = require("../cli/credential-context");
|
|
8
10
|
function createSandboxRunner(opts) {
|
|
9
11
|
const sandboxes = new Map();
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Creations in flight, keyed by sandbox key.
|
|
14
|
+
*
|
|
15
|
+
* Two turns arriving together must not each build a sandbox. `sandboxes.get` → `driver.create`
|
|
16
|
+
* → `sandboxes.set` spans two awaits, so without this both callers miss, both create, and the
|
|
17
|
+
* loser is a LIVE container that no longer appears in the map: destroy, revoke and the TTL
|
|
18
|
+
* bookkeeping all miss it, so it keeps its model warrant and answers turns nothing can stop.
|
|
19
|
+
* That is a kill-switch hole, not a leak. Reachable today — the turn gate caps concurrency
|
|
20
|
+
* process-wide but never serializes one key.
|
|
21
|
+
*/
|
|
22
|
+
const creating = new Map();
|
|
23
|
+
/**
|
|
24
|
+
* sessionKey → sandboxKey, so callers may keep naming a session while a sandbox holds several.
|
|
25
|
+
*
|
|
26
|
+
* Bounded for the reason governor.ts bounds `sessionUsers`: a long-lived serve sees unbounded
|
|
27
|
+
* distinct session keys (every anonymous gateway sender gets a fresh uuid), and this must never
|
|
28
|
+
* be the reason an instance falls over. Entries are never purged on teardown — a stale one
|
|
29
|
+
* resolves to a sandbox that is gone, which the lookups already treat as "nothing to do", and
|
|
30
|
+
* the next turn overwrites it.
|
|
31
|
+
*/
|
|
32
|
+
const sessionToSandbox = new Map();
|
|
33
|
+
const MAX_TRACKED_SESSIONS = 10_000;
|
|
34
|
+
const rememberSession = (sessionKey, sandboxKey) => {
|
|
35
|
+
if (sessionToSandbox.size >= MAX_TRACKED_SESSIONS && !sessionToSandbox.has(sessionKey)) {
|
|
36
|
+
const oldest = sessionToSandbox.keys().next();
|
|
37
|
+
if (!oldest.done) {
|
|
38
|
+
sessionToSandbox.delete(oldest.value);
|
|
39
|
+
}
|
|
17
40
|
}
|
|
41
|
+
sessionToSandbox.set(sessionKey, sandboxKey);
|
|
42
|
+
};
|
|
43
|
+
/** Accepts a session key or a sandbox key; unknown keys pass through as sandbox keys. */
|
|
44
|
+
const resolveSandboxKey = (key) => sessionToSandbox.get(key) ?? key;
|
|
45
|
+
const createSandbox = async (sandboxKey, sessionKey) => {
|
|
18
46
|
// Admission first: no lease ⇒ no sandbox. This is the kill switch and the commercial gate,
|
|
19
47
|
// deliberately the same mechanism.
|
|
20
|
-
await opts.admit?.(sessionKey);
|
|
48
|
+
await opts.admit?.({ sandboxKey, sessionKey });
|
|
21
49
|
const handle = await opts.driver.create({
|
|
22
|
-
|
|
50
|
+
sandboxKey,
|
|
23
51
|
image: opts.image,
|
|
24
52
|
tier: opts.tier,
|
|
25
53
|
routes: opts.routes,
|
|
26
|
-
fs: (await opts.fsFor?.(
|
|
27
|
-
...(opts.stateDirFor ? { stateDir: await opts.stateDirFor(
|
|
54
|
+
fs: (await opts.fsFor?.(sandboxKey)) ?? opts.fs ?? { readonly: [], writable: [] },
|
|
55
|
+
...(opts.stateDirFor ? { stateDir: await opts.stateDirFor(sandboxKey) } : {}),
|
|
28
56
|
limits: opts.limits ?? { ttlSeconds: 30 * 60 },
|
|
29
57
|
env: { ...(opts.env ?? {}), ...(opts.provenanceEnv?.() ?? {}) },
|
|
30
58
|
});
|
|
31
|
-
const modelToken = await opts.mintModelToken?.(handle.id,
|
|
59
|
+
const modelToken = await opts.mintModelToken?.(handle.id, sandboxKey);
|
|
32
60
|
const channel = handle.connect();
|
|
33
61
|
const engine = new remoteEngine_1.RemoteEngine({
|
|
34
62
|
id: 'sandbox',
|
|
@@ -41,24 +69,26 @@ function createSandboxRunner(opts) {
|
|
|
41
69
|
// its TTL. Binding the sandbox's identity here (handle.id) keeps the warrant
|
|
42
70
|
// non-transferable while letting its lifetime stay honest.
|
|
43
71
|
...(opts.mintModelToken
|
|
44
|
-
? { mintToken: async () => opts.mintModelToken(handle.id,
|
|
72
|
+
? { mintToken: async () => opts.mintModelToken(handle.id, sandboxKey) }
|
|
45
73
|
: {}),
|
|
46
74
|
},
|
|
47
75
|
logger: opts.logger,
|
|
48
76
|
});
|
|
49
|
-
const sandbox = {
|
|
77
|
+
const sandbox = {
|
|
78
|
+
handle, engine, dead: false, sessionKeys: new Set([sessionKey]), queue: Promise.resolve(),
|
|
79
|
+
};
|
|
50
80
|
// A sandbox dies on TTL reap, OOM, or node loss. Mark it so the next turn recreates rather
|
|
51
81
|
// than dialing a socket that is gone — otherwise the first message after a reap fails.
|
|
52
82
|
const onClosed = (err) => {
|
|
53
83
|
sandbox.dead = true;
|
|
54
84
|
// Still registered ⇒ the channel died on its own. destroySession, dispose and
|
|
55
85
|
// revoke('all') remove their entry first and have already torn the sandbox down.
|
|
56
|
-
const unexpected = sandboxes.get(
|
|
86
|
+
const unexpected = sandboxes.get(sandboxKey) === sandbox;
|
|
57
87
|
if (unexpected) {
|
|
58
|
-
sandboxes.delete(
|
|
88
|
+
sandboxes.delete(sandboxKey);
|
|
59
89
|
}
|
|
60
90
|
const why = err instanceof Error ? `: ${err.message}` : '';
|
|
61
|
-
opts.logger?.info(`[Sandbox] ${handle.id} closed${why};
|
|
91
|
+
opts.logger?.info(`[Sandbox] ${handle.id} closed${why}; ${sandboxKey} will rehydrate on next message`);
|
|
62
92
|
// Forgetting a sandbox is not the same as destroying it. A channel that dies while the
|
|
63
93
|
// CONTAINER is healthy (a failed dial, a dropped socket) used to leave that container
|
|
64
94
|
// running forever — observed live: four orphans still answering `ready` eight hours
|
|
@@ -70,12 +100,74 @@ function createSandboxRunner(opts) {
|
|
|
70
100
|
// Both arms: the channel REJECTS with the socket-level reason (see unixSocketChannel), and
|
|
71
101
|
// a one-armed `.then` would turn that into an unhandled rejection.
|
|
72
102
|
void channel.closed?.then(() => onClosed(), onClosed);
|
|
73
|
-
sandboxes.set(
|
|
74
|
-
opts.logger?.info(`[Sandbox] created ${handle.id} for
|
|
103
|
+
sandboxes.set(sandboxKey, sandbox);
|
|
104
|
+
opts.logger?.info(`[Sandbox] created ${handle.id} for ${sandboxKey} (tier ${handle.tier})`);
|
|
75
105
|
return sandbox;
|
|
76
106
|
};
|
|
107
|
+
/**
|
|
108
|
+
* The one place a sandbox is looked up or built. NOT async, deliberately: `creating.set` must
|
|
109
|
+
* run in the same synchronous step as the misses above it, and an `async` wrapper invites
|
|
110
|
+
* someone to add an await between them, which silently reopens the create race.
|
|
111
|
+
*/
|
|
112
|
+
const sandboxFor = (sandboxKey, sessionKey) => {
|
|
113
|
+
const existing = sandboxes.get(sandboxKey);
|
|
114
|
+
if (existing && !existing.dead) {
|
|
115
|
+
return Promise.resolve(existing);
|
|
116
|
+
}
|
|
117
|
+
if (existing?.dead) {
|
|
118
|
+
sandboxes.delete(sandboxKey);
|
|
119
|
+
}
|
|
120
|
+
const inFlight = creating.get(sandboxKey);
|
|
121
|
+
if (inFlight) {
|
|
122
|
+
return inFlight;
|
|
123
|
+
}
|
|
124
|
+
// `finally`, not `then`: a failed admission or a driver error must evict the entry, or the
|
|
125
|
+
// rejected promise is cached forever and every later turn replays a stale refusal.
|
|
126
|
+
const started = createSandbox(sandboxKey, sessionKey)
|
|
127
|
+
.finally(() => { creating.delete(sandboxKey); });
|
|
128
|
+
creating.set(sandboxKey, started);
|
|
129
|
+
return started;
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* Runs one turn, serialized when the peer cannot tag streamed events with their run id.
|
|
133
|
+
*
|
|
134
|
+
* An untagged peer's events are broadcast to every in-flight turn (see `RemoteEngine.dispatch`),
|
|
135
|
+
* so overlapping turns would write one session's tokens into another's transcript. The sandbox
|
|
136
|
+
* image is pinned by digest and upgrades independently of serve, so an older peer is a normal
|
|
137
|
+
* state, not an error — it just runs one turn at a time.
|
|
138
|
+
*/
|
|
139
|
+
const exclusive = async (sandbox, fn) => {
|
|
140
|
+
if (sandbox.engine.supportsDemux()) {
|
|
141
|
+
return fn(sandbox);
|
|
142
|
+
}
|
|
143
|
+
const prior = sandbox.queue;
|
|
144
|
+
let release;
|
|
145
|
+
sandbox.queue = new Promise((resolve) => { release = resolve; });
|
|
146
|
+
await prior;
|
|
147
|
+
// The sandbox can be reaped or killed while a turn waits its place in line; running now
|
|
148
|
+
// would dial a socket that is gone. Phrased so the caller's retry recognises it.
|
|
149
|
+
if (sandbox.dead) {
|
|
150
|
+
release();
|
|
151
|
+
throw new Error('sandbox channel closed');
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
return await fn(sandbox);
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
// In `finally` so a thrown turn cannot wedge every later turn on this sandbox.
|
|
158
|
+
release();
|
|
159
|
+
}
|
|
160
|
+
};
|
|
77
161
|
const run = async (options) => {
|
|
78
162
|
const sessionKey = options.sessionId ?? 'default';
|
|
163
|
+
// Which sandbox owns this turn. Absent ⇒ the session IS the sandbox, which is the historical
|
|
164
|
+
// behaviour and what any caller that does not thread the key keeps getting.
|
|
165
|
+
const sandboxKey = options.sandboxKey ?? sessionKey;
|
|
166
|
+
const join = (sandbox) => {
|
|
167
|
+
sandbox.sessionKeys.add(sessionKey);
|
|
168
|
+
rememberSession(sessionKey, sandboxKey);
|
|
169
|
+
return sandbox;
|
|
170
|
+
};
|
|
79
171
|
const collected = { text: '', thinking: '', toolCalls: [] };
|
|
80
172
|
// Collect for the JSON response while forwarding to serve's SSE emitter when present —
|
|
81
173
|
// both paths get the same events.
|
|
@@ -94,6 +186,11 @@ function createSandboxRunner(opts) {
|
|
|
94
186
|
// attribution (evidence), the per-turn data-plane nonce (OBO), and the
|
|
95
187
|
// bound gateway session.
|
|
96
188
|
analyticsUserId: options.analyticsUserId,
|
|
189
|
+
// Computed HERE, not in the child: this runs inside serve's own
|
|
190
|
+
// runWithCredential frame, and the child has none. Without it every user's
|
|
191
|
+
// engine home inside a sandbox resolved to the same unscoped directory, so
|
|
192
|
+
// tier 0 was strictly worse for transcript separation than no sandbox at all.
|
|
193
|
+
engineHomeUserKey: (0, paths_1.engineHomeUserKey)((0, credential_context_1.currentCredential)()?.userId),
|
|
97
194
|
turnEnv: options.turnEnv,
|
|
98
195
|
gatewaySessionKey: options.gatewaySessionKey,
|
|
99
196
|
// No nativeSessionId on purpose: the in-sandbox runner resumes from its own
|
|
@@ -104,21 +201,25 @@ function createSandboxRunner(opts) {
|
|
|
104
201
|
}, emitter, opts.runtime, options.signal);
|
|
105
202
|
};
|
|
106
203
|
try {
|
|
107
|
-
|
|
204
|
+
const sandbox = join(await sandboxFor(sandboxKey, sessionKey));
|
|
108
205
|
try {
|
|
109
|
-
await
|
|
206
|
+
await exclusive(sandbox, attempt);
|
|
110
207
|
}
|
|
111
208
|
catch (err) {
|
|
112
|
-
// The sandbox died mid-turn (TTL reap, OOM, kill)
|
|
113
|
-
//
|
|
209
|
+
// The sandbox died mid-turn (TTL reap, OOM, kill) or before this turn reached the
|
|
210
|
+
// front of its queue. Recreate and retry ONCE — but only if nothing has been emitted
|
|
211
|
+
// yet, so a streaming client never sees duplicates.
|
|
114
212
|
const message = errText(err);
|
|
115
213
|
if (!/channel closed/i.test(message) || !nothingEmitted()) {
|
|
116
214
|
throw err;
|
|
117
215
|
}
|
|
118
|
-
|
|
216
|
+
// Only if it is still the dead one: a concurrent turn may already have replaced it,
|
|
217
|
+
// and deleting that would strand a live container the same way the create race did.
|
|
218
|
+
if (sandboxes.get(sandboxKey) === sandbox) {
|
|
219
|
+
sandboxes.delete(sandboxKey);
|
|
220
|
+
}
|
|
119
221
|
opts.logger?.info(`[Sandbox] session ${sessionKey} lost its sandbox mid-turn; recreating and retrying once`);
|
|
120
|
-
|
|
121
|
-
await attempt(sandbox);
|
|
222
|
+
await exclusive(join(await sandboxFor(sandboxKey, sessionKey)), attempt);
|
|
122
223
|
}
|
|
123
224
|
return { success: true, text: collected.text, thinking: collected.thinking, toolCalls: collected.toolCalls };
|
|
124
225
|
}
|
|
@@ -134,28 +235,30 @@ function createSandboxRunner(opts) {
|
|
|
134
235
|
sandboxes.delete(key);
|
|
135
236
|
}
|
|
136
237
|
},
|
|
137
|
-
sandboxes: () => [...sandboxes.entries()].map(([
|
|
138
|
-
|
|
238
|
+
sandboxes: () => [...sandboxes.entries()].map(([sandboxKey, c]) => ({
|
|
239
|
+
sandboxKey, sandboxId: c.handle.id, tier: c.handle.tier, sessionKeys: [...c.sessionKeys],
|
|
139
240
|
})),
|
|
140
|
-
async destroySession(
|
|
141
|
-
const
|
|
241
|
+
async destroySession(key) {
|
|
242
|
+
const sandboxKey = resolveSandboxKey(key);
|
|
243
|
+
const sandbox = sandboxes.get(sandboxKey);
|
|
142
244
|
if (!sandbox) {
|
|
143
245
|
return;
|
|
144
246
|
}
|
|
145
|
-
sandboxes.delete(
|
|
247
|
+
sandboxes.delete(sandboxKey);
|
|
146
248
|
await opts.driver.destroy(sandbox.handle);
|
|
147
|
-
opts.logger?.info(`[Sandbox] destroyed ${sandbox.handle.id} (
|
|
249
|
+
opts.logger?.info(`[Sandbox] destroyed ${sandbox.handle.id} (${sandboxKey}${describeSessions(sandbox, key)})`);
|
|
148
250
|
},
|
|
149
|
-
async revokeRoute(
|
|
150
|
-
const
|
|
251
|
+
async revokeRoute(key, what) {
|
|
252
|
+
const sandboxKey = resolveSandboxKey(key);
|
|
253
|
+
const sandbox = sandboxes.get(sandboxKey);
|
|
151
254
|
if (!sandbox) {
|
|
152
255
|
return;
|
|
153
256
|
}
|
|
154
257
|
await opts.driver.revoke(sandbox.handle, what);
|
|
155
258
|
if (what === 'all') {
|
|
156
|
-
sandboxes.delete(
|
|
259
|
+
sandboxes.delete(sandboxKey);
|
|
157
260
|
}
|
|
158
|
-
opts.logger?.info(`[Sandbox] revoked ${what} on ${sandbox.handle.id} (
|
|
261
|
+
opts.logger?.info(`[Sandbox] revoked ${what} on ${sandbox.handle.id} (${sandboxKey}${describeSessions(sandbox, key)})`);
|
|
159
262
|
},
|
|
160
263
|
};
|
|
161
264
|
}
|
|
@@ -189,3 +292,12 @@ function teeEmitter(downstream, into) {
|
|
|
189
292
|
function errText(err) {
|
|
190
293
|
return err instanceof Error ? err.message : String(err);
|
|
191
294
|
}
|
|
295
|
+
/**
|
|
296
|
+
* Names the OTHER sessions a teardown ends. Under user scope an order aimed at one session stops
|
|
297
|
+
* that user's other conversations too, and a stop wider than the order that caused it must not be
|
|
298
|
+
* silent in the log.
|
|
299
|
+
*/
|
|
300
|
+
function describeSessions(sandbox, requested) {
|
|
301
|
+
const others = [...sandbox.sessionKeys].filter((s) => s !== requested);
|
|
302
|
+
return others.length === 0 ? '' : `, also ending ${others.length}: ${others.join(', ')}`;
|
|
303
|
+
}
|
|
@@ -24,6 +24,7 @@ PRIVACY DEFAULT (PLAN/compliance-req.md §6 "data minimization"):
|
|
|
24
24
|
|
|
25
25
|
from __future__ import annotations
|
|
26
26
|
from typing import Any
|
|
27
|
+
import ast
|
|
27
28
|
import hashlib
|
|
28
29
|
import json
|
|
29
30
|
import os
|
|
@@ -94,6 +95,72 @@ def _sha256(text: str) -> str:
|
|
|
94
95
|
return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()
|
|
95
96
|
|
|
96
97
|
|
|
98
|
+
# Correlation IDs Spotlight reads from the requester header blob (see FLEET_DATA_CONTRACT.md).
|
|
99
|
+
# These are minted by AgentHippo clients and carry no user content, so with capture off the
|
|
100
|
+
# blob is reduced to exactly these keys rather than dropped — unknown x-* headers from clients
|
|
101
|
+
# we do not control (e.g. x-api-key) still never reach the span in plaintext.
|
|
102
|
+
_METADATA_HEADERS_KEY = "metadata.requester_custom_headers"
|
|
103
|
+
|
|
104
|
+
_METADATA_HEADER_ALLOWLIST = frozenset({
|
|
105
|
+
"x-litellm-metadata-session-id",
|
|
106
|
+
"x-litellm-metadata-request-id",
|
|
107
|
+
"x-litellm-metadata-user-id",
|
|
108
|
+
"x-litellm-metadata-project-id",
|
|
109
|
+
"x-litellm-metadata-agent-id",
|
|
110
|
+
"x-litellm-metadata-agent-version",
|
|
111
|
+
"x-litellm-metadata-source",
|
|
112
|
+
"x-litellm-metadata-engine",
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _parse_attr_dict(value: Any) -> Any:
|
|
117
|
+
if not isinstance(value, str):
|
|
118
|
+
return value
|
|
119
|
+
try:
|
|
120
|
+
return ast.literal_eval(value)
|
|
121
|
+
except (ValueError, SyntaxError):
|
|
122
|
+
try:
|
|
123
|
+
return json.loads(value)
|
|
124
|
+
except ValueError:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _allowlisted_headers(headers: dict) -> dict:
|
|
129
|
+
return {
|
|
130
|
+
k.lower(): v for k, v in headers.items()
|
|
131
|
+
if isinstance(k, str) and k.lower() in _METADATA_HEADER_ALLOWLIST
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _filtered_metadata_headers(value: Any) -> str | None:
|
|
136
|
+
"""Reduce the header blob to allowlisted keys, keeping the str(dict) repr format that
|
|
137
|
+
the DuckDB extraction regexes parse. None when nothing allowlisted survives."""
|
|
138
|
+
headers = _parse_attr_dict(value)
|
|
139
|
+
if not isinstance(headers, dict):
|
|
140
|
+
return None
|
|
141
|
+
kept = _allowlisted_headers(headers)
|
|
142
|
+
return str(kept) if kept else None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# LiteLLM also embeds the raw request headers as metadata.requester_metadata['headers'] on some
|
|
146
|
+
# routes (observed on /v1/chat/completions; absent on /v1/messages). The attribute's scalar keys
|
|
147
|
+
# (session_id, request_id, …) are a fallback landing spot the extraction chain parses, so the
|
|
148
|
+
# attribute itself must survive — only its embedded header dict gets the allowlist treatment.
|
|
149
|
+
_REQUESTER_METADATA_KEY = "metadata.requester_metadata"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _scrub_requester_metadata(value: Any) -> Any:
|
|
153
|
+
"""Scrubbed value, or None (caller drops the attribute) when an embedded header dict
|
|
154
|
+
cannot be safely isolated — reprs with non-literal values defeat ast.literal_eval, and
|
|
155
|
+
passing those through unparsed would leak the very headers this exists to remove."""
|
|
156
|
+
md = _parse_attr_dict(value)
|
|
157
|
+
if isinstance(md, dict) and isinstance(md.get("headers"), dict):
|
|
158
|
+
return str({**md, "headers": _allowlisted_headers(md["headers"])})
|
|
159
|
+
if isinstance(md, dict):
|
|
160
|
+
return value
|
|
161
|
+
return None if "headers" in str(value) else value
|
|
162
|
+
|
|
163
|
+
|
|
97
164
|
class _RedactingSpan:
|
|
98
165
|
"""
|
|
99
166
|
Span proxy that drops content-bearing attributes and replaces them with a sha256 + length.
|
|
@@ -112,11 +179,20 @@ class _RedactingSpan:
|
|
|
112
179
|
return getattr(self._span, name)
|
|
113
180
|
|
|
114
181
|
def set_attribute(self, key: Any, value: Any) -> Any:
|
|
182
|
+
if key == _REQUESTER_METADATA_KEY:
|
|
183
|
+
scrubbed = _scrub_requester_metadata(value)
|
|
184
|
+
if scrubbed is None:
|
|
185
|
+
return None
|
|
186
|
+
return self._span.set_attribute(key, scrubbed)
|
|
115
187
|
if isinstance(key, str) and _CONTENT_ATTR_RE.match(key):
|
|
116
188
|
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
|
117
189
|
self._span.set_attribute(f"{key}.sha256", _sha256(text))
|
|
118
190
|
self._span.set_attribute(f"{key}.length", len(text))
|
|
119
191
|
self._span.set_attribute(f"{key}.redacted", True)
|
|
192
|
+
if key == _METADATA_HEADERS_KEY:
|
|
193
|
+
kept = _filtered_metadata_headers(value)
|
|
194
|
+
if kept is not None:
|
|
195
|
+
self._span.set_attribute(key, kept)
|
|
120
196
|
return None
|
|
121
197
|
return self._span.set_attribute(key, value)
|
|
122
198
|
|
|
@@ -131,10 +207,18 @@ class _RedactingSpan:
|
|
|
131
207
|
if isinstance(attributes, dict):
|
|
132
208
|
filtered = {}
|
|
133
209
|
for key, value in attributes.items():
|
|
134
|
-
if
|
|
210
|
+
if key == _REQUESTER_METADATA_KEY:
|
|
211
|
+
scrubbed = _scrub_requester_metadata(value)
|
|
212
|
+
if scrubbed is not None:
|
|
213
|
+
filtered[key] = scrubbed
|
|
214
|
+
elif isinstance(key, str) and _CONTENT_ATTR_RE.match(key):
|
|
135
215
|
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
|
136
216
|
filtered[f"{key}.sha256"] = _sha256(text)
|
|
137
217
|
filtered[f"{key}.length"] = len(text)
|
|
218
|
+
if key == _METADATA_HEADERS_KEY:
|
|
219
|
+
kept = _filtered_metadata_headers(value)
|
|
220
|
+
if kept is not None:
|
|
221
|
+
filtered[key] = kept
|
|
138
222
|
else:
|
|
139
223
|
filtered[key] = value
|
|
140
224
|
attributes = filtered
|
|
@@ -68,6 +68,10 @@ litellm_settings:
|
|
|
68
68
|
# 1. set turn_off_message_logging: false and callback_settings.otel.message_logging: true
|
|
69
69
|
# 2. export AGENTHIPPO_EVIDENCE_CONTENT_CAPTURE=1
|
|
70
70
|
# Step 2 is enforced in agentide_litellm_hooks_v1.py at span-write time and logs loudly when on.
|
|
71
|
+
#
|
|
72
|
+
# LANES: these hardened defaults are what Dockerfile.litellm bakes into the gateway image.
|
|
73
|
+
# setup-analytics.{sh,ps1} flips both flags ON when copying this template to a local
|
|
74
|
+
# ~/.agent-hippo/litellm-otel.yaml — the IDE/Spotlight lane records prompts by default.
|
|
71
75
|
turn_off_message_logging: true
|
|
72
76
|
|
|
73
77
|
callback_settings:
|
|
@@ -359,7 +359,14 @@ function Setup-Config {
|
|
|
359
359
|
|
|
360
360
|
if (-not (Test-Path $litellmDest)) {
|
|
361
361
|
Copy-Item $litellmSrc $litellmDest
|
|
362
|
-
|
|
362
|
+
# IDE/local lane: Spotlight needs prompt content, so message logging defaults ON here.
|
|
363
|
+
# The flip happens at copy time, not in the template, because Dockerfile.litellm bakes
|
|
364
|
+
# the same template into the gateway image, which must stay metadata-only.
|
|
365
|
+
$cfg = Get-Content $litellmDest -Raw
|
|
366
|
+
$cfg = $cfg -replace '(?m)^ turn_off_message_logging: true$', ' turn_off_message_logging: false'
|
|
367
|
+
$cfg = $cfg -replace '(?m)^ message_logging: false', ' message_logging: true'
|
|
368
|
+
Set-Content -Path $litellmDest -Value $cfg -NoNewline
|
|
369
|
+
Log-Success "Created LiteLLM OTEL config from template (local Spotlight: message logging ON)"
|
|
363
370
|
} else {
|
|
364
371
|
Log-Success "LiteLLM OTEL config already present (user-owned)"
|
|
365
372
|
}
|
|
@@ -456,7 +456,14 @@ setup_config() {
|
|
|
456
456
|
|
|
457
457
|
if [[ ! -f "$litellm_config" ]]; then
|
|
458
458
|
cp "$source_config" "$litellm_config"
|
|
459
|
-
|
|
459
|
+
# IDE/local lane: Spotlight needs prompt content, so message logging defaults ON here.
|
|
460
|
+
# The flip happens at copy time, not in the template, because Dockerfile.litellm bakes
|
|
461
|
+
# the same template into the gateway image, which must stay metadata-only.
|
|
462
|
+
sed -i.bak \
|
|
463
|
+
-e 's/^ turn_off_message_logging: true$/ turn_off_message_logging: false/' \
|
|
464
|
+
-e 's/^ message_logging: false/ message_logging: true/' \
|
|
465
|
+
"$litellm_config" && rm -f "${litellm_config}.bak"
|
|
466
|
+
log_success "Created LiteLLM OTEL config from template (local Spotlight: message logging ON)"
|
|
460
467
|
else
|
|
461
468
|
log_success "LiteLLM OTEL config already present (user-owned)"
|
|
462
469
|
fi
|
|
@@ -628,7 +635,11 @@ start_litellm_with_otel() {
|
|
|
628
635
|
log_info "Starting LiteLLM with OTEL config..."
|
|
629
636
|
log_info "Config: $otel_config"
|
|
630
637
|
# Start in background
|
|
631
|
-
|
|
638
|
+
# IDE/local Spotlight defaults content capture ON (operator can still export =0 to disable);
|
|
639
|
+
# hardened serve deployments do not launch through this script and stay metadata-only.
|
|
640
|
+
LITELLM_CONFIG_PATH="$otel_config" \
|
|
641
|
+
AGENTHIPPO_EVIDENCE_CONTENT_CAPTURE="${AGENTHIPPO_EVIDENCE_CONTENT_CAPTURE:-1}" \
|
|
642
|
+
nohup "$litellm_script" > /tmp/litellm-otel.log 2>&1 &
|
|
632
643
|
local litellm_pid=$!
|
|
633
644
|
|
|
634
645
|
# Write PID file with OTEL flag + base URL for fast status checks
|
|
@@ -54,7 +54,12 @@ function fakeSandboxes(initial = []) {
|
|
|
54
54
|
destroyed: [],
|
|
55
55
|
revoked: [],
|
|
56
56
|
failRevoke: false,
|
|
57
|
-
|
|
57
|
+
// Session scope: one sandbox per session, so its key IS the session key.
|
|
58
|
+
sandboxes() {
|
|
59
|
+
return state.live.map((sessionKey) => ({
|
|
60
|
+
sandboxKey: sessionKey, sandboxId: `sandbox-${sessionKey}`, sessionKeys: [sessionKey],
|
|
61
|
+
}));
|
|
62
|
+
},
|
|
58
63
|
async destroySession(sessionKey) {
|
|
59
64
|
state.destroyed.push(sessionKey);
|
|
60
65
|
state.live = state.live.filter((s) => s !== sessionKey);
|
|
@@ -247,4 +247,36 @@ suite('data-plane engine wiring: loadMcpServers + claude/custom shape (no API ke
|
|
|
247
247
|
fs.rmSync(workspace, { recursive: true, force: true });
|
|
248
248
|
}
|
|
249
249
|
});
|
|
250
|
+
test('the engine-home key crosses a process boundary that the credential deliberately does not', async () => {
|
|
251
|
+
const base = path.join(os.tmpdir(), 'ah-home-base');
|
|
252
|
+
// In serve: the credential frame is authoritative and no wire value is needed.
|
|
253
|
+
const inServe = await (0, credential_context_1.runWithCredential)({ userId: 'alice@corp.com' }, () => (0, paths_1.userScopedEngineHomeFromKey)(base, (0, credential_context_1.resolveEngineHomeUserKey)()));
|
|
254
|
+
assert.strictEqual(inServe, (0, paths_1.userScopedEngineHome)(base, 'alice@corp.com'));
|
|
255
|
+
// In a sandbox child: NO credential frame — which is the whole reason this regressed. The
|
|
256
|
+
// key arrives on the run frame and must land on the identical path.
|
|
257
|
+
const key = (0, paths_1.engineHomeUserKey)('alice@corp.com');
|
|
258
|
+
const inSandbox = (0, paths_1.userScopedEngineHomeFromKey)(base, (0, credential_context_1.resolveEngineHomeUserKey)(key));
|
|
259
|
+
assert.strictEqual(inSandbox, inServe, 'a sandboxed turn must resolve the same home as an in-process one');
|
|
260
|
+
// Distinct users stay distinct across the boundary — the property that was silently absent.
|
|
261
|
+
const bob = (0, paths_1.userScopedEngineHomeFromKey)(base, (0, credential_context_1.resolveEngineHomeUserKey)((0, paths_1.engineHomeUserKey)('bob@corp.com')));
|
|
262
|
+
assert.notStrictEqual(inSandbox, bob);
|
|
263
|
+
assert.ok(!inSandbox.includes('alice@corp.com'), 'the key is a hash, never the raw identity');
|
|
264
|
+
});
|
|
265
|
+
test('a credential frame outranks the wire, and a malformed key is ignored rather than trusted', async () => {
|
|
266
|
+
const base = path.join(os.tmpdir(), 'ah-home-base');
|
|
267
|
+
// A child cannot widen its own scope by sending someone else's key: where a frame exists,
|
|
268
|
+
// it wins.
|
|
269
|
+
const spoofed = await (0, credential_context_1.runWithCredential)({ userId: 'alice@corp.com' }, () => (0, credential_context_1.resolveEngineHomeUserKey)((0, paths_1.engineHomeUserKey)('bob@corp.com')));
|
|
270
|
+
assert.strictEqual(spoofed, (0, paths_1.engineHomeUserKey)('alice@corp.com'));
|
|
271
|
+
// The key is a path segment taken off the wire, so anything but the hex this produces
|
|
272
|
+
// falls back to the base dir instead of aiming an engine home elsewhere on disk.
|
|
273
|
+
for (const bad of ['../../etc', 'alice@corp.com', 'ZZZZ', '', 'abc']) {
|
|
274
|
+
assert.strictEqual((0, paths_1.userScopedEngineHomeFromKey)(base, bad), base, `rejected: ${JSON.stringify(bad)}`);
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
test('no verified user anywhere leaves the path untouched, so the IDE and single-tenant CLI are unaffected', () => {
|
|
278
|
+
const base = path.join(os.tmpdir(), 'ah-home-base');
|
|
279
|
+
assert.strictEqual((0, credential_context_1.resolveEngineHomeUserKey)(), undefined);
|
|
280
|
+
assert.strictEqual((0, paths_1.userScopedEngineHomeFromKey)(base, (0, credential_context_1.resolveEngineHomeUserKey)()), base);
|
|
281
|
+
});
|
|
250
282
|
});
|