@hue-run/sdk 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +14 -14
- package/EVALUATIONS.md +1 -1
- package/README.md +5 -5
- package/dist/cli/eval.js +9 -5
- package/dist/config.js +10 -3
- package/dist/types.d.ts +6 -4
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.14.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
469
469
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
470
470
|
project=/absolute/path/to/supported-fixture
|
|
471
471
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
472
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
472
|
+
--archive .artifacts/typescript/hue-run-sdk-0.14.0.tgz \
|
|
473
473
|
--origin https://STAGING_ORIGIN \
|
|
474
474
|
--project "$project" --command setup \
|
|
475
475
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
480
480
|
|
|
481
481
|
```sh
|
|
482
482
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
483
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
483
|
+
--archive .artifacts/typescript/hue-run-sdk-0.14.0.tgz \
|
|
484
484
|
--origin https://STAGING_ORIGIN \
|
|
485
485
|
--project "$project" --command claim \
|
|
486
486
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -523,10 +523,10 @@ production agent lacks. Pin the SDK and bring its `zod` peer along, so the agent
|
|
|
523
523
|
gains no dependency:
|
|
524
524
|
|
|
525
525
|
```sh
|
|
526
|
-
npx --yes --package @hue-run/sdk@0.
|
|
527
|
-
npx --yes --package @hue-run/sdk@0.
|
|
528
|
-
npx --yes --package @hue-run/sdk@0.
|
|
529
|
-
npx --yes --package @hue-run/sdk@0.
|
|
526
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" --command "node agent.js" --env-file .env.hue
|
|
527
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <experiment id>
|
|
528
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
529
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision> --env-file .env.hue
|
|
530
530
|
```
|
|
531
531
|
|
|
532
532
|
Without the second `--package`, `hue eval` exits with `hue eval needs zod`, even for `--help`;
|
|
@@ -562,7 +562,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
|
|
|
562
562
|
```
|
|
563
563
|
|
|
564
564
|
```sh
|
|
565
|
-
npx --yes --package @hue-run/sdk@0.
|
|
565
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
|
|
566
566
|
```
|
|
567
567
|
|
|
568
568
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
|
@@ -667,21 +667,21 @@ A Scenario or eval set whose dataset version is not saved cannot back an experim
|
|
|
667
667
|
exits 1 and asks for **Save eval-set version** in Hue or `--save-version`, which freezes that
|
|
668
668
|
version at its current revision. Connection settings are `HUE_API_KEY` and `HUE_BASE_URL`
|
|
669
669
|
(default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given; `--origin`
|
|
670
|
-
overrides the origin.
|
|
671
|
-
|
|
670
|
+
overrides the origin. Starting with TypeScript 0.14.0, telemetry content capture defaults on; `--no-content` opts out.
|
|
671
|
+
Earlier releases require `--content`; the examples include it for compatibility. Model and tool spans inside the agent come
|
|
672
672
|
only from the agent's own instrumentation. Case outputs, error messages and explanations are
|
|
673
|
-
stored in Hue
|
|
673
|
+
stored in Hue independently of the telemetry capture setting, so a case's answer can be graded and read on
|
|
674
674
|
its run page. A `--command`'s stdout is its answer unless it writes a result file, and is stored,
|
|
675
675
|
so it must not print credentials or debug logs there; before storing, `hue eval` replaces the
|
|
676
676
|
credentials it handed the case (the world token and MCP headers, a legacy MCP token, attempt
|
|
677
677
|
bearers) and every Hue control-plane credential in its environment with `[redacted]`, in the
|
|
678
678
|
answer, an adapter's thrown message and name, generated file names and the local checkpoint.
|
|
679
679
|
`--no-output` keeps outputs, error messages and explanations out of a one-shot run's stored
|
|
680
|
-
results;
|
|
680
|
+
results; captured telemetry still carries the output. Pass `--no-output --no-content` to omit both. `--worker` always stores them, because a run launched from Hue is read
|
|
681
681
|
on its run page (that is `runLocalAgent()`'s contract), and refuses `--no-output`. An interrupted
|
|
682
682
|
one-shot run keeps the choice it started with, so one run never mixes stored and unstored outputs:
|
|
683
|
-
rerunning it with
|
|
684
|
-
|
|
683
|
+
rerunning it with another output or telemetry capture policy is refused with the flags that resume it.
|
|
684
|
+
To resume a metadata-only trace from an earlier release, add `--no-content` and keep its original `--no-output` choice.
|
|
685
685
|
|
|
686
686
|
Files the agent writes to `output/` are uploaded as generated documents with or without
|
|
687
687
|
`--no-output`, and they are not redacted the way the answer is: `hue eval` replaces the same
|
package/EVALUATIONS.md
CHANGED
|
@@ -303,7 +303,7 @@ and require a Hue deployment that serves case `inputFiles` on experiment
|
|
|
303
303
|
items, subject `files`, and the artifact reservation, upload, completion and download APIs. The
|
|
304
304
|
Python SDK has no equivalent.
|
|
305
305
|
|
|
306
|
-
|
|
306
|
+
Since TypeScript 0.13.2, the `hue eval --worker --command "…"` wrapper supplies both callbacks, so one
|
|
307
307
|
command worker accepts direct and world cases, including mixed eval sets. It keeps the launch
|
|
308
308
|
directory for relative application entry points; private case inputs/output still use
|
|
309
309
|
`HUE_CASE_*`. An adapter-file worker opts into the direct callback with `--capability direct:v1`
|
package/README.md
CHANGED
|
@@ -35,7 +35,7 @@ import { createHue, HueExportError } from "@hue-run/sdk";
|
|
|
35
35
|
const hue = createHue({
|
|
36
36
|
apiKey: process.env.HUE_API_KEY!, // a project service key, on the server only
|
|
37
37
|
serviceName: "my-agent",
|
|
38
|
-
captureContent: true, //
|
|
38
|
+
captureContent: true, // Also works with older releases; false sends metadata only
|
|
39
39
|
onExportIssue: (issue) => console.error(issue), // sanitized counts, never server bodies
|
|
40
40
|
});
|
|
41
41
|
|
|
@@ -57,7 +57,7 @@ try {
|
|
|
57
57
|
await hue.shutdown(); // flushes and releases providers owned by this client
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
-
`captureContent`
|
|
60
|
+
Starting with TypeScript 0.14.0, `captureContent` defaults to `true`. Earlier releases require an explicit choice. The example sets `true`: trace inspection,
|
|
61
61
|
evaluations and judges read the prompts, responses and tool inputs/outputs it records. Choose
|
|
62
62
|
`false` for metadata only when a policy forbids sending that content; see
|
|
63
63
|
[Privacy and content](#privacy-and-content).
|
|
@@ -649,11 +649,11 @@ before), wraps `runSimulation()` and `runLocalAgent()` for an adapter file or a
|
|
|
649
649
|
`hue eval --case "<name>" ./hue-agent.ts --content` creates a fresh run from a published case's
|
|
650
650
|
immutable pins, runs the agent with an isolated world when the case pins one, waits
|
|
651
651
|
for Hue's outcome checks and prints the run URL and per-case PASS/FAIL verdicts with an exit code;
|
|
652
|
-
`--worker` registers the same adapter for runs launched from Hue.
|
|
652
|
+
`--worker` registers the same adapter for runs launched from Hue. Since TypeScript 0.13.2, a
|
|
653
653
|
command worker handles both world and answer-only cases; adapter workers opt into direct cases
|
|
654
654
|
with `--capability direct:v1`. Upgrade an existing command registration under a new `--revision`.
|
|
655
|
-
It needs a Read and write key in `HUE_API_KEY` (never printed). Content capture
|
|
656
|
-
|
|
655
|
+
It needs a Read and write key in `HUE_API_KEY` (never printed). Content capture defaults on; `--no-content` opts out. The example passes `--content` explicitly
|
|
656
|
+
for compatibility with older releases. Case outputs, error messages and
|
|
657
657
|
explanations are stored in Hue either way (`--no-output` keeps them out of a one-shot run's
|
|
658
658
|
results); a command's stdout is its stored answer, with the credentials `hue eval` handed the case
|
|
659
659
|
replaced by `[redacted]`.
|
package/dist/cli/eval.js
CHANGED
|
@@ -78,7 +78,8 @@ Output and limits:
|
|
|
78
78
|
--name <run name> Run name (default: <agent key> @ <revision>, hashes shortened)
|
|
79
79
|
--baseline <experiment id|url> Compare verdicts with a previous experiment
|
|
80
80
|
--json Print one JSON document on stdout; progress goes to stderr
|
|
81
|
-
--content Capture telemetry content (
|
|
81
|
+
--content Capture telemetry content (the default)
|
|
82
|
+
--no-content Omit telemetry inputs, outputs and messages
|
|
82
83
|
--no-output One-shot: do not store case outputs, error messages and
|
|
83
84
|
explanations (stored by default; --worker always stores); with
|
|
84
85
|
--content the case span still carries the output
|
|
@@ -141,7 +142,8 @@ function parse(argv) {
|
|
|
141
142
|
name: { type: "string" },
|
|
142
143
|
baseline: { type: "string" },
|
|
143
144
|
json: { type: "boolean", default: false },
|
|
144
|
-
content: { type: "boolean"
|
|
145
|
+
content: { type: "boolean" },
|
|
146
|
+
"no-content": { type: "boolean", default: false },
|
|
145
147
|
"no-output": { type: "boolean", default: false },
|
|
146
148
|
"trace-not-accepted": { type: "string" },
|
|
147
149
|
"save-version": { type: "boolean", default: false },
|
|
@@ -817,9 +819,9 @@ export function explain(error) {
|
|
|
817
819
|
if (error instanceof CheckpointIdentityError && error.startedWith) {
|
|
818
820
|
const flags = [
|
|
819
821
|
error.startedWith.persistResultContent === false ? "--no-output" : "",
|
|
820
|
-
error.startedWith.captureContent ===
|
|
822
|
+
error.startedWith.captureContent === false ? "--no-content" : "",
|
|
821
823
|
].filter(Boolean);
|
|
822
|
-
return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
|
|
824
|
+
return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --no-content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
|
|
823
825
|
}
|
|
824
826
|
// The error's own message points at an in-memory report the user cannot reach.
|
|
825
827
|
if (error instanceof HueExportError)
|
|
@@ -1446,6 +1448,8 @@ export async function runEvalCommand(argv) {
|
|
|
1446
1448
|
let json = false;
|
|
1447
1449
|
try {
|
|
1448
1450
|
const { values, positionals } = parse(argv);
|
|
1451
|
+
if (values.content && values["no-content"])
|
|
1452
|
+
throw new UsageError("Choose either --content or --no-content");
|
|
1449
1453
|
// A usage error before anything is prepared or created: no run is left to recover from it.
|
|
1450
1454
|
traceNotAcceptedPolicy(values["trace-not-accepted"]);
|
|
1451
1455
|
json = values.json;
|
|
@@ -1522,7 +1526,7 @@ export async function runEvalCommand(argv) {
|
|
|
1522
1526
|
apiKey,
|
|
1523
1527
|
baseUrl,
|
|
1524
1528
|
serviceName: key,
|
|
1525
|
-
captureContent: values.content,
|
|
1529
|
+
captureContent: values.content ?? !values["no-content"],
|
|
1526
1530
|
timeoutMillis: CASE_TRACE_EXPORT_MILLIS,
|
|
1527
1531
|
});
|
|
1528
1532
|
return values.worker
|
package/dist/config.js
CHANGED
|
@@ -37,8 +37,8 @@ export function validateOptions(options) {
|
|
|
37
37
|
liveSpans: false,
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
if (typeof options.captureContent !== "boolean")
|
|
41
|
-
throw new TypeError("
|
|
40
|
+
if (options.captureContent !== undefined && typeof options.captureContent !== "boolean")
|
|
41
|
+
throw new TypeError("captureContent must be a boolean");
|
|
42
42
|
if (typeof options.apiKey !== "string" ||
|
|
43
43
|
!options.apiKey ||
|
|
44
44
|
options.apiKey.length > 4096 ||
|
|
@@ -83,5 +83,12 @@ export function validateOptions(options) {
|
|
|
83
83
|
throw new TypeError("liveSpans must be a boolean");
|
|
84
84
|
// Setup credentials send installer telemetry only, never in-progress placeholders.
|
|
85
85
|
const liveSpans = options.liveSpans !== false && !options.apiKey.startsWith("hue_setup_");
|
|
86
|
-
return {
|
|
86
|
+
return {
|
|
87
|
+
...options,
|
|
88
|
+
captureContent: options.captureContent ?? true,
|
|
89
|
+
baseUrl: url.origin,
|
|
90
|
+
timeoutMillis,
|
|
91
|
+
maxQueueBytes,
|
|
92
|
+
liveSpans,
|
|
93
|
+
};
|
|
87
94
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -13,10 +13,11 @@ export type Signal = "traces" | "logs";
|
|
|
13
13
|
/** Options accepted by every Hue client and transport, whether enabled or disabled. */
|
|
14
14
|
export interface SharedHueOptions {
|
|
15
15
|
/**
|
|
16
|
-
* Whether helpers record prompts, responses, tool arguments and results.
|
|
17
|
-
*
|
|
16
|
+
* Whether helpers record prompts, responses, tool arguments and results. Defaults to `true`
|
|
17
|
+
* for an enabled client; set `false` explicitly for metadata-only export. A disabled client
|
|
18
|
+
* (`enabled: false`) exports nothing and defaults it to `false`.
|
|
18
19
|
*/
|
|
19
|
-
captureContent
|
|
20
|
+
captureContent?: boolean;
|
|
20
21
|
/** Hue origin, `https://app.hue.run` by default. An origin only: no path, query, fragment or credentials. */
|
|
21
22
|
baseUrl?: string;
|
|
22
23
|
/** Recorded as the `service.version` resource attribute of an owned client. */
|
|
@@ -52,7 +53,8 @@ export interface SharedHueOptions {
|
|
|
52
53
|
}
|
|
53
54
|
/**
|
|
54
55
|
* Options for a client that owns its OpenTelemetry providers. An enabled client needs a project
|
|
55
|
-
* key
|
|
56
|
+
* key and a service name. Content capture defaults to `true`; `captureContent: false` opts out.
|
|
57
|
+
* The kill switch (`enabled: false`)
|
|
56
58
|
* exports nothing while helpers keep running application code, so it needs no key and
|
|
57
59
|
* `captureContent` defaults to `false`.
|
|
58
60
|
*/
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.14.0";
|
package/dist/version.js
CHANGED