@hue-run/sdk 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +62 -25
- package/README.md +10 -7
- package/dist/cli/eval.js +16 -10
- package/dist/cli/listen.js +21 -2
- package/dist/cli/login.js +2 -1
- package/dist/cli/mcp.d.ts +5 -4
- package/dist/cli/mcp.js +480 -48
- package/dist/environment/world.d.ts +4 -2
- package/dist/environment/world.js +13 -4
- package/dist/evals/checkpoint.d.ts +5 -0
- package/dist/evals/checkpoint.js +17 -1
- package/dist/evals/local-worker.js +9 -5
- package/dist/evals/simulation.js +7 -5
- package/dist/provider-tools.d.ts +3 -1
- package/dist/provider-tools.js +4 -2
- package/dist/tool-definitions.d.ts +14 -3
- package/dist/tool-definitions.js +635 -53
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -23,10 +23,12 @@ export interface AgentEnvironmentOptions {
|
|
|
23
23
|
* default; the canonical carriers are the world's own `env`. */
|
|
24
24
|
legacyMcpVariables?: boolean;
|
|
25
25
|
}
|
|
26
|
+
/** `environment` without any variable a world or its server owns. */
|
|
27
|
+
export declare function withoutWorldVariables(environment: Record<string, string>): Record<string, string>;
|
|
26
28
|
/**
|
|
27
29
|
* The environment for an agent child process running one case (the coordination briefs'
|
|
28
|
-
* recommended policy): the parent's variables minus Hue control-plane credentials
|
|
29
|
-
* world
|
|
30
|
+
* recommended policy): the parent's variables minus Hue control-plane credentials and any
|
|
31
|
+
* world-scoped variable, then the world's carriers. Nothing here is logged.
|
|
30
32
|
*/
|
|
31
33
|
export declare function agentEnvironment(world: WorldHandoff, options?: AgentEnvironmentOptions): Record<string, string>;
|
|
32
34
|
/** The `{ url, token, expiresAt }` shape the `hue_sim_` capability had, projected from the
|
|
@@ -53,16 +53,25 @@ export function stripHueControlPlaneCredentials(parent) {
|
|
|
53
53
|
}
|
|
54
54
|
return child;
|
|
55
55
|
}
|
|
56
|
+
/** Variables a world or its server owns: a parent's value belongs to another world (a runner
|
|
57
|
+
* started inside a case, a shell left from an earlier run) or to the server (a signing key), and
|
|
58
|
+
* would send this world's token to a mirror this world does not have. The whole `HUE_WORLD_`
|
|
59
|
+
* prefix is the world's. */
|
|
60
|
+
const WORLD_SCOPED = /^(?:HUE_WORLD_[A-Z0-9_]+|HUE_MCP_(?:CONFIG|URL|TOKEN|EXPIRES_AT)|HUE_SIM_[A-Z0-9_]+_(?:URL|ALIAS))$/;
|
|
61
|
+
/** `environment` without any variable a world or its server owns. */
|
|
62
|
+
export function withoutWorldVariables(environment) {
|
|
63
|
+
return Object.fromEntries(Object.entries(environment).filter(([name]) => !WORLD_SCOPED.test(name)));
|
|
64
|
+
}
|
|
56
65
|
/**
|
|
57
66
|
* The environment for an agent child process running one case (the coordination briefs'
|
|
58
|
-
* recommended policy): the parent's variables minus Hue control-plane credentials
|
|
59
|
-
* world
|
|
67
|
+
* recommended policy): the parent's variables minus Hue control-plane credentials and any
|
|
68
|
+
* world-scoped variable, then the world's carriers. Nothing here is logged.
|
|
60
69
|
*/
|
|
61
70
|
export function agentEnvironment(world, options = {}) {
|
|
62
71
|
const parent = options.parent ?? process.env;
|
|
63
|
-
const child = options.includeHueCredentials
|
|
72
|
+
const child = withoutWorldVariables(options.includeHueCredentials
|
|
64
73
|
? Object.fromEntries(Object.entries(parent).filter((entry) => entry[1] !== undefined))
|
|
65
|
-
: stripHueControlPlaneCredentials(parent);
|
|
74
|
+
: stripHueControlPlaneCredentials(parent));
|
|
66
75
|
Object.assign(child, world.env);
|
|
67
76
|
if (options.legacyMcpVariables ?? true) {
|
|
68
77
|
const legacy = legacyMcpCapability(world);
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/** A server- or user-provided identifier that is safe as a single checkpoint path component:
|
|
2
|
+
* no separators, no `.`/`..`, and it stays directly inside the directory it is joined onto. */
|
|
3
|
+
export declare function checkpointSegment(value: string, label: string): string;
|
|
4
|
+
/** `join(directory, ...segments)` after validating every segment. */
|
|
5
|
+
export declare function checkpointPath(directory: string, ...segments: [string, string][]): string;
|
|
1
6
|
/** The checkpoint belongs to a run started with a different identity. When only its content
|
|
2
7
|
* policy differs, `startedWith` holds the policy the unfinished run was started with. */
|
|
3
8
|
export declare class CheckpointIdentityError extends Error {
|
package/dist/evals/checkpoint.js
CHANGED
|
@@ -1,10 +1,26 @@
|
|
|
1
1
|
import { constants, rmSync } from "node:fs";
|
|
2
2
|
import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
|
|
3
|
-
import { join, resolve } from "node:path";
|
|
3
|
+
import { join, resolve, sep } from "node:path";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { onForcedExit } from "./exit-cleanup.js";
|
|
6
6
|
import { digest } from "./json.js";
|
|
7
7
|
const CONTENT_POLICY = ["persistResultContent", "captureContent"];
|
|
8
|
+
const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
9
|
+
/** A server- or user-provided identifier that is safe as a single checkpoint path component:
|
|
10
|
+
* no separators, no `.`/`..`, and it stays directly inside the directory it is joined onto. */
|
|
11
|
+
export function checkpointSegment(value, label) {
|
|
12
|
+
if (typeof value !== "string" || !SEGMENT.test(value))
|
|
13
|
+
throw new Error(`Refusing to use ${label} ${JSON.stringify(value)} in a checkpoint path`);
|
|
14
|
+
return value;
|
|
15
|
+
}
|
|
16
|
+
/** `join(directory, ...segments)` after validating every segment. */
|
|
17
|
+
export function checkpointPath(directory, ...segments) {
|
|
18
|
+
const root = resolve(directory);
|
|
19
|
+
const path = resolve(root, ...segments.map(([value, label]) => checkpointSegment(value, label)));
|
|
20
|
+
if (path !== root && !path.startsWith(root.endsWith(sep) ? root : root + sep))
|
|
21
|
+
throw new Error("Checkpoint path escapes its directory");
|
|
22
|
+
return path;
|
|
23
|
+
}
|
|
8
24
|
/** The checkpoint belongs to a run started with a different identity. When only its content
|
|
9
25
|
* policy differs, `startedWith` holds the policy the unfinished run was started with. */
|
|
10
26
|
export class CheckpointIdentityError extends Error {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolve } from "node:path";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { CheckpointStore } from "./checkpoint.js";
|
|
3
|
+
import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
|
|
4
4
|
import { pinRequestedAttemptV2, requestedAttemptV2, runEnvironmentTarget, } from "./environment-target.js";
|
|
5
5
|
import { runExperiment, OutcomeSerializationError, TargetOutcomeUncertainError, UncertainExecutionError, } from "./runner.js";
|
|
6
6
|
/** Capability strings a registration declares; Hue offers only matching cases. */
|
|
@@ -159,14 +159,18 @@ export async function runLocalAgent(options) {
|
|
|
159
159
|
}, Math.min(15_000, Math.max(1_000, interval)));
|
|
160
160
|
let experimentFinished = false;
|
|
161
161
|
try {
|
|
162
|
+
const experimentId = checkpointSegment(claim.experimentId, "experiment id");
|
|
162
163
|
const requested = requestedConfiguration
|
|
163
|
-
? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(
|
|
164
|
+
? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(experimentId)).config)
|
|
164
165
|
: undefined;
|
|
165
166
|
const report = await runExperiment({
|
|
166
167
|
client: options.client,
|
|
167
168
|
hue: options.hue,
|
|
168
|
-
experimentId
|
|
169
|
-
checkpointDirectory:
|
|
169
|
+
experimentId,
|
|
170
|
+
checkpointDirectory: checkpointPath(directory, [
|
|
171
|
+
`experiment-${experimentId}`,
|
|
172
|
+
"experiment id",
|
|
173
|
+
]),
|
|
170
174
|
persistResultContent: true,
|
|
171
175
|
// Worker dispatch uses the case pin: directTarget never receives a world.
|
|
172
176
|
environmentEvidence: options.directTarget ? "when_pinned" : "required",
|
package/dist/evals/simulation.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { join } from "node:path";
|
|
3
2
|
import { HueEnvironmentError } from "../environment/client.js";
|
|
4
3
|
import { HueApiError } from "./client.js";
|
|
5
|
-
import { CheckpointStore } from "./checkpoint.js";
|
|
4
|
+
import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
|
|
6
5
|
import { MAX_ENVIRONMENT_STEPS } from "./environment-evidence.js";
|
|
7
6
|
import { aggregateBounds, digest, json } from "./json.js";
|
|
8
7
|
import { normalizeScorerDefinitionForPublication } from "./scorer-publication.js";
|
|
@@ -404,7 +403,7 @@ export async function runSimulation(options) {
|
|
|
404
403
|
let bindings = options.localScorers ?? [];
|
|
405
404
|
if (!attempt.experimentId) {
|
|
406
405
|
const resolved = await resolveExperiment(options, definition, attempt.idempotencyKey);
|
|
407
|
-
attempt.experimentId = resolved.experimentId;
|
|
406
|
+
attempt.experimentId = checkpointSegment(resolved.experimentId, "experiment id");
|
|
408
407
|
bindings = resolved.bindings;
|
|
409
408
|
attempt.stage = "running";
|
|
410
409
|
await store.write("active-attempt", attempt);
|
|
@@ -417,7 +416,7 @@ export async function runSimulation(options) {
|
|
|
417
416
|
...(options.localScorers ?? []),
|
|
418
417
|
];
|
|
419
418
|
}
|
|
420
|
-
const experimentId = attempt.experimentId;
|
|
419
|
+
const experimentId = checkpointSegment(attempt.experimentId, "experiment id");
|
|
421
420
|
const runUrl = new URL(`/experiments/${experimentId}`, options.client.baseUrl).toString();
|
|
422
421
|
await options.onProgress?.({ type: "run_created", experimentId, runUrl });
|
|
423
422
|
const requested = requestedConfiguration
|
|
@@ -427,7 +426,10 @@ export async function runSimulation(options) {
|
|
|
427
426
|
client: options.client,
|
|
428
427
|
hue: options.hue,
|
|
429
428
|
experimentId,
|
|
430
|
-
checkpointDirectory:
|
|
429
|
+
checkpointDirectory: checkpointPath(store.directory, [
|
|
430
|
+
`experiment-${experimentId}`,
|
|
431
|
+
"experiment id",
|
|
432
|
+
]),
|
|
431
433
|
persistResultContent: options.persistResultContent,
|
|
432
434
|
traceEvidence: options.traceEvidence,
|
|
433
435
|
...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
|
package/dist/provider-tools.d.ts
CHANGED
|
@@ -31,7 +31,9 @@ export interface HostedToolActivity {
|
|
|
31
31
|
/**
|
|
32
32
|
* A provider's error text as a failed call exports it under content capture: credentials
|
|
33
33
|
* scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
|
|
34
|
-
* replaced and NUL removed.
|
|
34
|
+
* replaced and NUL removed. Only the first 16,384 code points are scrubbed, and what that cut
|
|
35
|
+
* interrupts (a quoted value, a URL, a prefixed token) is redacted to it. The Python SDK
|
|
36
|
+
* produces the same text.
|
|
35
37
|
*/
|
|
36
38
|
export declare function providerErrorDescription(value: string): string;
|
|
37
39
|
/**
|
package/dist/provider-tools.js
CHANGED
|
@@ -74,11 +74,13 @@ function codePoints(text, count) {
|
|
|
74
74
|
/**
|
|
75
75
|
* A provider's error text as a failed call exports it under content capture: credentials
|
|
76
76
|
* scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
|
|
77
|
-
* replaced and NUL removed.
|
|
77
|
+
* replaced and NUL removed. Only the first 16,384 code points are scrubbed, and what that cut
|
|
78
|
+
* interrupts (a quoted value, a URL, a prefixed token) is redacted to it. The Python SDK
|
|
79
|
+
* produces the same text.
|
|
78
80
|
*/
|
|
79
81
|
export function providerErrorDescription(value) {
|
|
80
82
|
const scanned = codePoints(value.toWellFormed().replaceAll("\u0000", ""), MAX_ERROR_SCAN);
|
|
81
|
-
const bounded = codePoints(scrubCredentialText(scanned.text), MAX_ERROR_TEXT);
|
|
83
|
+
const bounded = codePoints(scrubCredentialText(scanned.text, scanned.cut), MAX_ERROR_TEXT);
|
|
82
84
|
return bounded.cut || scanned.cut ? `${bounded.text}…` : bounded.text;
|
|
83
85
|
}
|
|
84
86
|
/** An OpenAI MCP call's `error`: its text, or an error object's string `message`. */
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
/** Whether a field name names a credential: one of `credentialKeys`, or a name ending in `token`,
|
|
2
|
+
* `secret`, `password`, `apikey` or `credential`, ignoring case, `-` and `_`. */
|
|
3
|
+
export declare function isCredentialKey(key: string): boolean;
|
|
1
4
|
/**
|
|
2
5
|
* Removes credentials from free text a provider returned, such as an MCP call's error message,
|
|
3
6
|
* with the rules tool definitions use, extended for text: each `http`, `https`, `ws`, `wss` or
|
|
@@ -6,10 +9,18 @@
|
|
|
6
9
|
* becomes `[redacted]` when it has an `@`, `?` or `#`; a token with a known credential prefix
|
|
7
10
|
* (`hue_sk_`, `sk-`, `xoxb-`, `ya29.` and others), the credential after an authorization scheme
|
|
8
11
|
* (`Bearer`, `Basic`, `Token`), the whole value of an `Authorization` header, and the value of a
|
|
9
|
-
* `key=value` or `key
|
|
10
|
-
* become `[redacted]`.
|
|
12
|
+
* `key=value`, `key: value` or `key => value` pair whose key names a credential (quoted,
|
|
13
|
+
* escaped-quoted, bare, or a whole `[…]` or `{…}`) become `[redacted]`. A JSON or `%` escape
|
|
14
|
+
* (`\n`, `\u0022`, `%20`) ends a word as a space does. A quote that does not close on its line
|
|
15
|
+
* runs to the end of the line, and a URL's quoted query value is replaced whole. `cut` says the
|
|
16
|
+
* text was cut from a longer one, so a URL or prefixed token that runs to its end is replaced
|
|
17
|
+
* whole, as is the word of scheme characters it ends in, which may start a URL's scheme. Only the
|
|
18
|
+
* first 16,384 code points are read: a longer text is cut there, and `…` marks it.
|
|
11
19
|
*/
|
|
12
|
-
export declare function scrubCredentialText(text: string): string;
|
|
20
|
+
export declare function scrubCredentialText(text: string, cut?: boolean): string;
|
|
21
|
+
/** `scrubCredentialText` without its bound on the text's length, for tests that time long texts;
|
|
22
|
+
* callers use `scrubCredentialText`. */
|
|
23
|
+
export declare function scrubCredentialTextUnbounded(text: string, cut?: boolean): string;
|
|
13
24
|
/**
|
|
14
25
|
* Removes hosted-tool credentials from an exported attribute value, before the caller's `redact`.
|
|
15
26
|
* Tool definitions come from OpenTelemetry GenAI (`gen_ai.tool.definitions`), AI SDK 6
|