@hue-run/sdk 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +32 -10
- package/README.md +4 -2
- package/dist/cli/eval.js +10 -6
- package/dist/cli/listen.js +21 -2
- package/dist/cli/mcp.js +239 -21
- package/dist/evals/checkpoint.d.ts +5 -0
- package/dist/evals/checkpoint.js +17 -1
- package/dist/evals/local-worker.js +9 -5
- package/dist/evals/simulation.js +7 -5
- package/dist/provider-tools.d.ts +3 -1
- package/dist/provider-tools.js +4 -2
- package/dist/tool-definitions.d.ts +14 -3
- package/dist/tool-definitions.js +635 -53
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/evals/simulation.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { join } from "node:path";
|
|
3
2
|
import { HueEnvironmentError } from "../environment/client.js";
|
|
4
3
|
import { HueApiError } from "./client.js";
|
|
5
|
-
import { CheckpointStore } from "./checkpoint.js";
|
|
4
|
+
import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
|
|
6
5
|
import { MAX_ENVIRONMENT_STEPS } from "./environment-evidence.js";
|
|
7
6
|
import { aggregateBounds, digest, json } from "./json.js";
|
|
8
7
|
import { normalizeScorerDefinitionForPublication } from "./scorer-publication.js";
|
|
@@ -404,7 +403,7 @@ export async function runSimulation(options) {
|
|
|
404
403
|
let bindings = options.localScorers ?? [];
|
|
405
404
|
if (!attempt.experimentId) {
|
|
406
405
|
const resolved = await resolveExperiment(options, definition, attempt.idempotencyKey);
|
|
407
|
-
attempt.experimentId = resolved.experimentId;
|
|
406
|
+
attempt.experimentId = checkpointSegment(resolved.experimentId, "experiment id");
|
|
408
407
|
bindings = resolved.bindings;
|
|
409
408
|
attempt.stage = "running";
|
|
410
409
|
await store.write("active-attempt", attempt);
|
|
@@ -417,7 +416,7 @@ export async function runSimulation(options) {
|
|
|
417
416
|
...(options.localScorers ?? []),
|
|
418
417
|
];
|
|
419
418
|
}
|
|
420
|
-
const experimentId = attempt.experimentId;
|
|
419
|
+
const experimentId = checkpointSegment(attempt.experimentId, "experiment id");
|
|
421
420
|
const runUrl = new URL(`/experiments/${experimentId}`, options.client.baseUrl).toString();
|
|
422
421
|
await options.onProgress?.({ type: "run_created", experimentId, runUrl });
|
|
423
422
|
const requested = requestedConfiguration
|
|
@@ -427,7 +426,10 @@ export async function runSimulation(options) {
|
|
|
427
426
|
client: options.client,
|
|
428
427
|
hue: options.hue,
|
|
429
428
|
experimentId,
|
|
430
|
-
checkpointDirectory:
|
|
429
|
+
checkpointDirectory: checkpointPath(store.directory, [
|
|
430
|
+
`experiment-${experimentId}`,
|
|
431
|
+
"experiment id",
|
|
432
|
+
]),
|
|
431
433
|
persistResultContent: options.persistResultContent,
|
|
432
434
|
traceEvidence: options.traceEvidence,
|
|
433
435
|
...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
|
package/dist/provider-tools.d.ts
CHANGED
|
@@ -31,7 +31,9 @@ export interface HostedToolActivity {
|
|
|
31
31
|
/**
|
|
32
32
|
* A provider's error text as a failed call exports it under content capture: credentials
|
|
33
33
|
* scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
|
|
34
|
-
* replaced and NUL removed.
|
|
34
|
+
* replaced and NUL removed. Only the first 16,384 code points are scrubbed, and what that cut
|
|
35
|
+
* interrupts (a quoted value, a URL, a prefixed token) is redacted to it. The Python SDK
|
|
36
|
+
* produces the same text.
|
|
35
37
|
*/
|
|
36
38
|
export declare function providerErrorDescription(value: string): string;
|
|
37
39
|
/**
|
package/dist/provider-tools.js
CHANGED
|
@@ -74,11 +74,13 @@ function codePoints(text, count) {
|
|
|
74
74
|
/**
|
|
75
75
|
* A provider's error text as a failed call exports it under content capture: credentials
|
|
76
76
|
* scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
|
|
77
|
-
* replaced and NUL removed.
|
|
77
|
+
* replaced and NUL removed. Only the first 16,384 code points are scrubbed, and what that cut
|
|
78
|
+
* interrupts (a quoted value, a URL, a prefixed token) is redacted to it. The Python SDK
|
|
79
|
+
* produces the same text.
|
|
78
80
|
*/
|
|
79
81
|
export function providerErrorDescription(value) {
|
|
80
82
|
const scanned = codePoints(value.toWellFormed().replaceAll("\u0000", ""), MAX_ERROR_SCAN);
|
|
81
|
-
const bounded = codePoints(scrubCredentialText(scanned.text), MAX_ERROR_TEXT);
|
|
83
|
+
const bounded = codePoints(scrubCredentialText(scanned.text, scanned.cut), MAX_ERROR_TEXT);
|
|
82
84
|
return bounded.cut || scanned.cut ? `${bounded.text}…` : bounded.text;
|
|
83
85
|
}
|
|
84
86
|
/** An OpenAI MCP call's `error`: its text, or an error object's string `message`. */
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
/** Whether a field name names a credential: one of `credentialKeys`, or a name ending in `token`,
|
|
2
|
+
* `secret`, `password`, `apikey` or `credential`, ignoring case, `-` and `_`. */
|
|
3
|
+
export declare function isCredentialKey(key: string): boolean;
|
|
1
4
|
/**
|
|
2
5
|
* Removes credentials from free text a provider returned, such as an MCP call's error message,
|
|
3
6
|
* with the rules tool definitions use, extended for text: each `http`, `https`, `ws`, `wss` or
|
|
@@ -6,10 +9,18 @@
|
|
|
6
9
|
* becomes `[redacted]` when it has an `@`, `?` or `#`; a token with a known credential prefix
|
|
7
10
|
* (`hue_sk_`, `sk-`, `xoxb-`, `ya29.` and others), the credential after an authorization scheme
|
|
8
11
|
* (`Bearer`, `Basic`, `Token`), the whole value of an `Authorization` header, and the value of a
|
|
9
|
-
* `key=value` or `key
|
|
10
|
-
* become `[redacted]`.
|
|
12
|
+
* `key=value`, `key: value` or `key => value` pair whose key names a credential (quoted,
|
|
13
|
+
* escaped-quoted, bare, or a whole `[…]` or `{…}`) become `[redacted]`. A JSON or `%` escape
|
|
14
|
+
* (`\n`, `\u0022`, `%20`) ends a word as a space does. A quote that does not close on its line
|
|
15
|
+
* runs to the end of the line, and a URL's quoted query value is replaced whole. `cut` says the
|
|
16
|
+
* text was cut from a longer one, so a URL or prefixed token that runs to its end is replaced
|
|
17
|
+
* whole, as is the word of scheme characters it ends in, which may start a URL's scheme. Only the
|
|
18
|
+
* first 16,384 code points are read: a longer text is cut there, and `…` marks it.
|
|
11
19
|
*/
|
|
12
|
-
export declare function scrubCredentialText(text: string): string;
|
|
20
|
+
export declare function scrubCredentialText(text: string, cut?: boolean): string;
|
|
21
|
+
/** `scrubCredentialText` without its bound on the text's length, for tests that time long texts;
|
|
22
|
+
* callers use `scrubCredentialText`. */
|
|
23
|
+
export declare function scrubCredentialTextUnbounded(text: string, cut?: boolean): string;
|
|
13
24
|
/**
|
|
14
25
|
* Removes hosted-tool credentials from an exported attribute value, before the caller's `redact`.
|
|
15
26
|
* Tool definitions come from OpenTelemetry GenAI (`gen_ai.tool.definitions`), AI SDK 6
|