@hue-run/sdk 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +33 -18
- package/EVALUATIONS.md +8 -1
- package/README.md +8 -6
- package/dist/cli/eval.js +40 -10
- package/dist/config.js +10 -3
- package/dist/evals/client.d.ts +1 -1
- package/dist/evals/client.js +1 -1
- package/dist/evals/scorer-publication.js +7 -2
- package/dist/evals/types.d.ts +19 -0
- package/dist/types.d.ts +6 -4
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.14.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
469
469
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
470
470
|
project=/absolute/path/to/supported-fixture
|
|
471
471
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
472
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
472
|
+
--archive .artifacts/typescript/hue-run-sdk-0.14.0.tgz \
|
|
473
473
|
--origin https://STAGING_ORIGIN \
|
|
474
474
|
--project "$project" --command setup \
|
|
475
475
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
480
480
|
|
|
481
481
|
```sh
|
|
482
482
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
483
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
483
|
+
--archive .artifacts/typescript/hue-run-sdk-0.14.0.tgz \
|
|
484
484
|
--origin https://STAGING_ORIGIN \
|
|
485
485
|
--project "$project" --command claim \
|
|
486
486
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -523,10 +523,10 @@ production agent lacks. Pin the SDK and bring its `zod` peer along, so the agent
|
|
|
523
523
|
gains no dependency:
|
|
524
524
|
|
|
525
525
|
```sh
|
|
526
|
-
npx --yes --package @hue-run/sdk@0.
|
|
527
|
-
npx --yes --package @hue-run/sdk@0.
|
|
528
|
-
npx --yes --package @hue-run/sdk@0.
|
|
529
|
-
npx --yes --package @hue-run/sdk@0.
|
|
526
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" --command "node agent.js" --env-file .env.hue
|
|
527
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <experiment id>
|
|
528
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
529
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision> --env-file .env.hue
|
|
530
530
|
```
|
|
531
531
|
|
|
532
532
|
Without the second `--package`, `hue eval` exits with `hue eval needs zod`, even for `--help`;
|
|
@@ -562,7 +562,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
|
|
|
562
562
|
```
|
|
563
563
|
|
|
564
564
|
```sh
|
|
565
|
-
npx --yes --package @hue-run/sdk@0.
|
|
565
|
+
npx --yes --package @hue-run/sdk@0.14.0 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
|
|
566
566
|
```
|
|
567
567
|
|
|
568
568
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
|
@@ -581,9 +581,11 @@ Failing cases print the scorer explanation. An evaluator Hue records as not appl
|
|
|
581
581
|
that case's columns and neither passes nor fails it, so in a mixed eval set a case is decided by
|
|
582
582
|
the evaluators that apply to it; the pass line counts the not-applicable results. A case that no
|
|
583
583
|
pinned evaluator applies to is an error that names what they need. A result of a pinned Hue judge
|
|
584
|
-
(an evaluator of kind `world_judge`) that Hue marks advisory
|
|
584
|
+
(an evaluator of kind `world_judge`) that Hue marks advisory is
|
|
585
585
|
shown in its column, marked `(advisory)`, but never decides a case or the exit code; the pass line
|
|
586
|
-
counts advisory failures as not counted, and a case only advisory results ran for is an error.
|
|
586
|
+
counts advisory failures as not counted, and a case only advisory results ran for is an error.
|
|
587
|
+
The Answer outcome evaluator uses its required judge to decide answer-only cases; if that judge
|
|
588
|
+
cannot run, the case is inconclusive. Any
|
|
587
589
|
other evaluator's result, an error, or a metric that carries `passed` is never advisory, whatever
|
|
588
590
|
its evidence says. A metric name that
|
|
589
591
|
several evaluators report, such as two judges' `verdict`, gets a column per evaluator, labelled
|
|
@@ -629,8 +631,21 @@ script name; `--revision` is sent to Hue as the agent revision of every world.
|
|
|
629
631
|
|
|
630
632
|
`--worker` registers the adapter through `runLocalAgent()` with key `--agent-key` (default: the
|
|
631
633
|
adapter filename slug), name `--agent-name` (default: the key), revision `--revision` (default:
|
|
632
|
-
`AGENT_REVISION`, then the Git `HEAD` short hash, then `dev`)
|
|
633
|
-
plus any repeated `--capability <value
|
|
634
|
+
`AGENT_REVISION`, then the Git `HEAD` short hash, then `dev`). Command workers register both
|
|
635
|
+
`environment:v1` and `direct:v1`, plus any repeated `--capability <value>`: the same worker runs
|
|
636
|
+
world cases, answer-only cases and mixed eval sets launched from Hue. Commands keep the launch
|
|
637
|
+
directory for both case kinds, so relative application entry points and configuration work.
|
|
638
|
+
Direct cases receive the task/config on stdin and private files/output through `HUE_CASE_*`,
|
|
639
|
+
without a world token, MCP connection or `HUE_ENVIRONMENT_RUN_ID`. App calls that require a world
|
|
640
|
+
must stay disabled for this handoff; never fall back to live app credentials.
|
|
641
|
+
|
|
642
|
+
Adapter-file workers register `environment:v1` by default. Add `--capability direct:v1` only when
|
|
643
|
+
the adapter also handles `context.mode === "direct"`; its direct context carries `config`, case
|
|
644
|
+
identity, `executionId`, `files`, `outputDirectory` and `signal`, with no world or tools. These
|
|
645
|
+
worker behaviors are added in TypeScript 0.13.2. Upgrading an existing command worker requires a
|
|
646
|
+
new `--revision`, because adding a capability to a previously registered revision is refused.
|
|
647
|
+
|
|
648
|
+
The worker prints the registration and each claimed run with its
|
|
634
649
|
URL, executes runs launched from Hue until Ctrl+C or `--max-runs <n>`, and prints the verdict table
|
|
635
650
|
after each run. Selection flags do not apply; Hue chooses the pinned experiment. The worker exits 0
|
|
636
651
|
when it stops normally. Hue offers a case whose world comes with agent-visible files only to a
|
|
@@ -652,21 +667,21 @@ A Scenario or eval set whose dataset version is not saved cannot back an experim
|
|
|
652
667
|
exits 1 and asks for **Save eval-set version** in Hue or `--save-version`, which freezes that
|
|
653
668
|
version at its current revision. Connection settings are `HUE_API_KEY` and `HUE_BASE_URL`
|
|
654
669
|
(default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given; `--origin`
|
|
655
|
-
overrides the origin.
|
|
656
|
-
|
|
670
|
+
overrides the origin. Starting with TypeScript 0.14.0, telemetry content capture defaults on; `--no-content` opts out.
|
|
671
|
+
Earlier releases require `--content`; the examples include it for compatibility. Model and tool spans inside the agent come
|
|
657
672
|
only from the agent's own instrumentation. Case outputs, error messages and explanations are
|
|
658
|
-
stored in Hue
|
|
673
|
+
stored in Hue independently of the telemetry capture setting, so a case's answer can be graded and read on
|
|
659
674
|
its run page. A `--command`'s stdout is its answer unless it writes a result file, and is stored,
|
|
660
675
|
so it must not print credentials or debug logs there; before storing, `hue eval` replaces the
|
|
661
676
|
credentials it handed the case (the world token and MCP headers, a legacy MCP token, attempt
|
|
662
677
|
bearers) and every Hue control-plane credential in its environment with `[redacted]`, in the
|
|
663
678
|
answer, an adapter's thrown message and name, generated file names and the local checkpoint.
|
|
664
679
|
`--no-output` keeps outputs, error messages and explanations out of a one-shot run's stored
|
|
665
|
-
results;
|
|
680
|
+
results; captured telemetry still carries the output. Pass `--no-output --no-content` to omit both. `--worker` always stores them, because a run launched from Hue is read
|
|
666
681
|
on its run page (that is `runLocalAgent()`'s contract), and refuses `--no-output`. An interrupted
|
|
667
682
|
one-shot run keeps the choice it started with, so one run never mixes stored and unstored outputs:
|
|
668
|
-
rerunning it with
|
|
669
|
-
|
|
683
|
+
rerunning it with another output or telemetry capture policy is refused with the flags that resume it.
|
|
684
|
+
To resume a metadata-only trace from an earlier release, add `--no-content` and keep its original `--no-output` choice.
|
|
670
685
|
|
|
671
686
|
Files the agent writes to `output/` are uploaded as generated documents with or without
|
|
672
687
|
`--no-output`, and they are not redacted the way the answer is: `hue eval` replaces the same
|
package/EVALUATIONS.md
CHANGED
|
@@ -297,12 +297,19 @@ acceptance remains a post-publication Fern gate.
|
|
|
297
297
|
|
|
298
298
|
## Direct cases and files
|
|
299
299
|
|
|
300
|
-
Cases without a simulated world —
|
|
300
|
+
Cases without a simulated world — answer-only tasks, or document workflows with pinned files —
|
|
301
301
|
run on the same runner under the same checkpoint rules. They shipped in `@hue-run/sdk` `0.5.0`
|
|
302
302
|
and require a Hue deployment that serves case `inputFiles` on experiment
|
|
303
303
|
items, subject `files`, and the artifact reservation, upload, completion and download APIs. The
|
|
304
304
|
Python SDK has no equivalent.
|
|
305
305
|
|
|
306
|
+
Since TypeScript 0.13.2, the `hue eval --worker --command "…"` wrapper supplies both callbacks, so one
|
|
307
|
+
command worker accepts direct and world cases, including mixed eval sets. It keeps the launch
|
|
308
|
+
directory for relative application entry points; private case inputs/output still use
|
|
309
|
+
`HUE_CASE_*`. An adapter-file worker opts into the direct callback with `--capability direct:v1`
|
|
310
|
+
and must handle `context.mode === "direct"`. Restart an existing environment-only command
|
|
311
|
+
registration with a new `--revision` when upgrading.
|
|
312
|
+
|
|
306
313
|
On the outbound worker, supply `directTarget` beside or instead of `target`. Supplying
|
|
307
314
|
`directTarget` registers the `direct:v1` capability and supplying `target` registers
|
|
308
315
|
`environment:v1` (both strings are exported as `localAgentCapabilities`). Declare the file
|
package/README.md
CHANGED
|
@@ -35,7 +35,7 @@ import { createHue, HueExportError } from "@hue-run/sdk";
|
|
|
35
35
|
const hue = createHue({
|
|
36
36
|
apiKey: process.env.HUE_API_KEY!, // a project service key, on the server only
|
|
37
37
|
serviceName: "my-agent",
|
|
38
|
-
captureContent: true, //
|
|
38
|
+
captureContent: true, // Also works with older releases; false sends metadata only
|
|
39
39
|
onExportIssue: (issue) => console.error(issue), // sanitized counts, never server bodies
|
|
40
40
|
});
|
|
41
41
|
|
|
@@ -57,7 +57,7 @@ try {
|
|
|
57
57
|
await hue.shutdown(); // flushes and releases providers owned by this client
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
-
`captureContent`
|
|
60
|
+
Starting with TypeScript 0.14.0, `captureContent` defaults to `true`. Earlier releases require an explicit choice. The example sets `true`: trace inspection,
|
|
61
61
|
evaluations and judges read the prompts, responses and tool inputs/outputs it records. Choose
|
|
62
62
|
`false` for metadata only when a policy forbids sending that content; see
|
|
63
63
|
[Privacy and content](#privacy-and-content).
|
|
@@ -647,11 +647,13 @@ their agents with `runLocalAgent()`; setup does not register workers or launch s
|
|
|
647
647
|
The `hue eval` command, shipped in TypeScript `0.5.0` (`--case` since `0.6.0`, `--scenario`
|
|
648
648
|
before), wraps `runSimulation()` and `runLocalAgent()` for an adapter file or a shell command:
|
|
649
649
|
`hue eval --case "<name>" ./hue-agent.ts --content` creates a fresh run from a published case's
|
|
650
|
-
immutable pins, runs the agent
|
|
650
|
+
immutable pins, runs the agent with an isolated world when the case pins one, waits
|
|
651
651
|
for Hue's outcome checks and prints the run URL and per-case PASS/FAIL verdicts with an exit code;
|
|
652
|
-
`--worker` registers the same adapter for runs launched from Hue.
|
|
653
|
-
|
|
654
|
-
|
|
652
|
+
`--worker` registers the same adapter for runs launched from Hue. Since TypeScript 0.13.2, a
|
|
653
|
+
command worker handles both world and answer-only cases; adapter workers opt into direct cases
|
|
654
|
+
with `--capability direct:v1`. Upgrade an existing command registration under a new `--revision`.
|
|
655
|
+
It needs a Read and write key in `HUE_API_KEY` (never printed). Content capture defaults on; `--no-content` opts out. The example passes `--content` explicitly
|
|
656
|
+
for compatibility with older releases. Case outputs, error messages and
|
|
655
657
|
explanations are stored in Hue either way (`--no-output` keeps them out of a one-shot run's
|
|
656
658
|
results); a command's stdout is its stored answer, with the credentials `hue eval` handed the case
|
|
657
659
|
replaced by `[redacted]`.
|
package/dist/cli/eval.js
CHANGED
|
@@ -60,6 +60,8 @@ Agent (exactly one):
|
|
|
60
60
|
|
|
61
61
|
Modes:
|
|
62
62
|
--worker Register the agent and poll for runs launched from Hue
|
|
63
|
+
Commands support cases with or without a world; adapter files
|
|
64
|
+
opt into direct cases with --capability direct:v1
|
|
63
65
|
--max-runs <n> Stop the worker after n completed runs
|
|
64
66
|
--agent-key <key> Agent key (default: slug of the adapter filename)
|
|
65
67
|
--agent-name <name> Agent display name (default: the key)
|
|
@@ -76,7 +78,8 @@ Output and limits:
|
|
|
76
78
|
--name <run name> Run name (default: <agent key> @ <revision>, hashes shortened)
|
|
77
79
|
--baseline <experiment id|url> Compare verdicts with a previous experiment
|
|
78
80
|
--json Print one JSON document on stdout; progress goes to stderr
|
|
79
|
-
--content Capture telemetry content (
|
|
81
|
+
--content Capture telemetry content (the default)
|
|
82
|
+
--no-content Omit telemetry inputs, outputs and messages
|
|
80
83
|
--no-output One-shot: do not store case outputs, error messages and
|
|
81
84
|
explanations (stored by default; --worker always stores); with
|
|
82
85
|
--content the case span still carries the output
|
|
@@ -139,7 +142,8 @@ function parse(argv) {
|
|
|
139
142
|
name: { type: "string" },
|
|
140
143
|
baseline: { type: "string" },
|
|
141
144
|
json: { type: "boolean", default: false },
|
|
142
|
-
content: { type: "boolean"
|
|
145
|
+
content: { type: "boolean" },
|
|
146
|
+
"no-content": { type: "boolean", default: false },
|
|
143
147
|
"no-output": { type: "boolean", default: false },
|
|
144
148
|
"trace-not-accepted": { type: "string" },
|
|
145
149
|
"save-version": { type: "boolean", default: false },
|
|
@@ -600,9 +604,11 @@ function redacting(adapter) {
|
|
|
600
604
|
function parentEnvironment(allowHueCredentials) {
|
|
601
605
|
// A world variable left in this process's environment belongs to another world: only the
|
|
602
606
|
// current case's world sets them, whether or not the case has one.
|
|
603
|
-
|
|
607
|
+
const parent = withoutWorldVariables(allowHueCredentials
|
|
604
608
|
? Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined))
|
|
605
609
|
: stripHueControlPlaneCredentials(process.env));
|
|
610
|
+
delete parent.HUE_ENVIRONMENT_RUN_ID;
|
|
611
|
+
return parent;
|
|
606
612
|
}
|
|
607
613
|
function commandAdapter(command, timeoutSeconds, options) {
|
|
608
614
|
return async (inputs, context) => {
|
|
@@ -683,7 +689,7 @@ function directCommandAdapter(command, timeoutSeconds, options) {
|
|
|
683
689
|
files: context.files,
|
|
684
690
|
});
|
|
685
691
|
const stdout = await spawnAgentCommand(command, {
|
|
686
|
-
cwd: layout.caseDirectory,
|
|
692
|
+
cwd: options.workingDirectory ?? layout.caseDirectory,
|
|
687
693
|
env: {
|
|
688
694
|
...parentEnvironment(options.allowHueCredentials),
|
|
689
695
|
HUE_CASE_DIR: layout.caseDirectory,
|
|
@@ -813,9 +819,9 @@ export function explain(error) {
|
|
|
813
819
|
if (error instanceof CheckpointIdentityError && error.startedWith) {
|
|
814
820
|
const flags = [
|
|
815
821
|
error.startedWith.persistResultContent === false ? "--no-output" : "",
|
|
816
|
-
error.startedWith.captureContent ===
|
|
822
|
+
error.startedWith.captureContent === false ? "--no-content" : "",
|
|
817
823
|
].filter(Boolean);
|
|
818
|
-
return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
|
|
824
|
+
return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --no-content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
|
|
819
825
|
}
|
|
820
826
|
// The error's own message points at an in-memory report the user cannot reach.
|
|
821
827
|
if (error instanceof HueExportError)
|
|
@@ -1286,7 +1292,7 @@ async function runDirect(run, connection) {
|
|
|
1286
1292
|
telemetryNotAccepted: report.telemetryNotAccepted,
|
|
1287
1293
|
}, run.baselineId, output, signal);
|
|
1288
1294
|
}
|
|
1289
|
-
async function runWorker(values, connection,
|
|
1295
|
+
async function runWorker(values, connection, agents, agent, hue, output, signal) {
|
|
1290
1296
|
const client = new ObservedClient(connection);
|
|
1291
1297
|
const environmentClient = createEnvironmentClient(connection);
|
|
1292
1298
|
const wait = integer("wait", values.wait, 300, 0, 86_400);
|
|
@@ -1305,6 +1311,9 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
1305
1311
|
output.log(`Claimed run ${claim.runId}: ${new URL(`/experiments/${claim.experimentId}`, connection.baseUrl).toString()}`);
|
|
1306
1312
|
};
|
|
1307
1313
|
const telemetry = telemetryReporter(output);
|
|
1314
|
+
// A command can use the same stdin contract for both case kinds. Existing adapter files
|
|
1315
|
+
// may require a world, so their direct context remains an explicit capability opt-in.
|
|
1316
|
+
const direct = values.command !== undefined || values.capability?.includes("direct:v1");
|
|
1308
1317
|
await runLocalAgent({
|
|
1309
1318
|
client,
|
|
1310
1319
|
environmentClient,
|
|
@@ -1339,7 +1348,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
1339
1348
|
},
|
|
1340
1349
|
target(inputs, tools, context) {
|
|
1341
1350
|
output.log(`[${context.item.externalKey}] agent started`);
|
|
1342
|
-
return
|
|
1351
|
+
return agents.simulation(inputs, {
|
|
1343
1352
|
config: context.config,
|
|
1344
1353
|
item: context.item,
|
|
1345
1354
|
executionId: context.executionId,
|
|
@@ -1355,6 +1364,22 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
1355
1364
|
signal: context.signal ?? signal,
|
|
1356
1365
|
});
|
|
1357
1366
|
},
|
|
1367
|
+
...(direct
|
|
1368
|
+
? {
|
|
1369
|
+
directTarget(inputs, context) {
|
|
1370
|
+
output.log(`[${context.item.externalKey}] agent started`);
|
|
1371
|
+
return agents.direct(inputs, {
|
|
1372
|
+
mode: "direct",
|
|
1373
|
+
config: context.config,
|
|
1374
|
+
item: context.item,
|
|
1375
|
+
executionId: context.executionId,
|
|
1376
|
+
files: context.files,
|
|
1377
|
+
outputDirectory: context.outputDirectory,
|
|
1378
|
+
signal: context.signal ?? signal,
|
|
1379
|
+
});
|
|
1380
|
+
},
|
|
1381
|
+
}
|
|
1382
|
+
: {}),
|
|
1358
1383
|
async onCompleted(report) {
|
|
1359
1384
|
output.log(`Run ${report.runId} completed: ${report.subjectIds.length} case${report.subjectIds.length === 1 ? "" : "s"}`);
|
|
1360
1385
|
telemetry.rest(report.telemetryNotAccepted);
|
|
@@ -1423,6 +1448,8 @@ export async function runEvalCommand(argv) {
|
|
|
1423
1448
|
let json = false;
|
|
1424
1449
|
try {
|
|
1425
1450
|
const { values, positionals } = parse(argv);
|
|
1451
|
+
if (values.content && values["no-content"])
|
|
1452
|
+
throw new UsageError("Choose either --content or --no-content");
|
|
1426
1453
|
// A usage error before anything is prepared or created: no run is left to recover from it.
|
|
1427
1454
|
traceNotAcceptedPolicy(values["trace-not-accepted"]);
|
|
1428
1455
|
json = values.json;
|
|
@@ -1490,17 +1517,20 @@ export async function runEvalCommand(argv) {
|
|
|
1490
1517
|
direct: redacting(loaded ??
|
|
1491
1518
|
directCommandAdapter(values.command, timeout, {
|
|
1492
1519
|
allowHueCredentials: values["allow-hue-credentials"],
|
|
1520
|
+
// A worker's single command must resolve relative entry points the same way for
|
|
1521
|
+
// direct and world cases. One-shot direct runs keep their private-directory cwd.
|
|
1522
|
+
...(values.worker ? { workingDirectory: process.cwd() } : {}),
|
|
1493
1523
|
})),
|
|
1494
1524
|
};
|
|
1495
1525
|
hue = createHue({
|
|
1496
1526
|
apiKey,
|
|
1497
1527
|
baseUrl,
|
|
1498
1528
|
serviceName: key,
|
|
1499
|
-
captureContent: values.content,
|
|
1529
|
+
captureContent: values.content ?? !values["no-content"],
|
|
1500
1530
|
timeoutMillis: CASE_TRACE_EXPORT_MILLIS,
|
|
1501
1531
|
});
|
|
1502
1532
|
return values.worker
|
|
1503
|
-
? await runWorker(values, connection, agents
|
|
1533
|
+
? await runWorker(values, connection, agents, agent, hue, output, controller.signal)
|
|
1504
1534
|
: await runOnce(values, connection, agents, agent, hue, output, controller.signal);
|
|
1505
1535
|
}
|
|
1506
1536
|
catch (error) {
|
package/dist/config.js
CHANGED
|
@@ -37,8 +37,8 @@ export function validateOptions(options) {
|
|
|
37
37
|
liveSpans: false,
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
if (typeof options.captureContent !== "boolean")
|
|
41
|
-
throw new TypeError("
|
|
40
|
+
if (options.captureContent !== undefined && typeof options.captureContent !== "boolean")
|
|
41
|
+
throw new TypeError("captureContent must be a boolean");
|
|
42
42
|
if (typeof options.apiKey !== "string" ||
|
|
43
43
|
!options.apiKey ||
|
|
44
44
|
options.apiKey.length > 4096 ||
|
|
@@ -83,5 +83,12 @@ export function validateOptions(options) {
|
|
|
83
83
|
throw new TypeError("liveSpans must be a boolean");
|
|
84
84
|
// Setup credentials send installer telemetry only, never in-progress placeholders.
|
|
85
85
|
const liveSpans = options.liveSpans !== false && !options.apiKey.startsWith("hue_setup_");
|
|
86
|
-
return {
|
|
86
|
+
return {
|
|
87
|
+
...options,
|
|
88
|
+
captureContent: options.captureContent ?? true,
|
|
89
|
+
baseUrl: url.origin,
|
|
90
|
+
timeoutMillis,
|
|
91
|
+
maxQueueBytes,
|
|
92
|
+
liveSpans,
|
|
93
|
+
};
|
|
87
94
|
}
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -110,7 +110,7 @@ export declare class EvaluationClient {
|
|
|
110
110
|
createScorer(input: TaggedIdentity): Promise<Scorer>;
|
|
111
111
|
/** Renames a scorer or replaces its tags by name; the same values again change nothing. */
|
|
112
112
|
updateScorer(id: string, input: IdentityUpdate): Promise<Scorer>;
|
|
113
|
-
/** Lists the project's tags in
|
|
113
|
+
/** Lists the project's tags in its order: as people arranged them in Hue, then newer tags. */
|
|
114
114
|
listTags(): Promise<{
|
|
115
115
|
/** The project's tags. */
|
|
116
116
|
items: Tag[];
|
package/dist/evals/client.js
CHANGED
|
@@ -451,7 +451,7 @@ export class EvaluationClient {
|
|
|
451
451
|
updateScorer(id, input) {
|
|
452
452
|
return this.request("PATCH", `/scorers/${uuid(id)}`, withTagNames(input), undefined, undefined, true);
|
|
453
453
|
}
|
|
454
|
-
/** Lists the project's tags in
|
|
454
|
+
/** Lists the project's tags in its order: as people arranged them in Hue, then newer tags. */
|
|
455
455
|
listTags() {
|
|
456
456
|
return this.request("GET", "/tags");
|
|
457
457
|
}
|
|
@@ -142,12 +142,17 @@ const sdkScorerPublication = z.union([
|
|
|
142
142
|
}),
|
|
143
143
|
z.strictObject({
|
|
144
144
|
kind: z.literal("world_outcome"),
|
|
145
|
-
entry: z.
|
|
145
|
+
entry: z.enum([
|
|
146
|
+
"hue.outcome_assertions.v2",
|
|
147
|
+
"hue.outcome_assertions.v4",
|
|
148
|
+
"hue.outcome_assertions.v5",
|
|
149
|
+
"hue.outcome_assertions.v6",
|
|
150
|
+
]),
|
|
146
151
|
metrics: fixedMetrics(outcomeAssertionMetrics),
|
|
147
152
|
}),
|
|
148
153
|
z.strictObject({
|
|
149
154
|
kind: z.literal("world_outcome"),
|
|
150
|
-
entry: z.
|
|
155
|
+
entry: z.enum(["hue.outcome_assertions.v3", "hue.answer_outcome.v1", "hue.answer_outcome.v2"]),
|
|
151
156
|
metrics: fixedMetrics(outcomeAssertionV3Metrics),
|
|
152
157
|
config: z.strictObject({ judge: outcomeJudgeConfig }),
|
|
153
158
|
}),
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -275,6 +275,25 @@ export type ScorerDefinition = {
|
|
|
275
275
|
/** The judge every case scored by this version uses. */
|
|
276
276
|
judge: OutcomeJudgeConfig;
|
|
277
277
|
};
|
|
278
|
+
} | {
|
|
279
|
+
/** A case's reviewed assertions, scored inside Hue from gateway evidence. */
|
|
280
|
+
kind: "world_outcome";
|
|
281
|
+
/** Versioned gateway-ledger assertion semantics. */
|
|
282
|
+
entry: "hue.outcome_assertions.v4" | "hue.outcome_assertions.v5" | "hue.outcome_assertions.v6";
|
|
283
|
+
/** `task_success` and the fixed assertion counts defined by the entry. */
|
|
284
|
+
metrics: MetricDefinition[];
|
|
285
|
+
} | {
|
|
286
|
+
/** An answer-only case, scored inside Hue without world evidence. */
|
|
287
|
+
kind: "world_outcome";
|
|
288
|
+
/** Versioned answer semantics; the pinned judge is required to decide the case. */
|
|
289
|
+
entry: "hue.answer_outcome.v1" | "hue.answer_outcome.v2";
|
|
290
|
+
/** The assertion and judge counts defined by the entry. */
|
|
291
|
+
metrics: MetricDefinition[];
|
|
292
|
+
/** The judge pin used to score the answer. */
|
|
293
|
+
config: {
|
|
294
|
+
/** The required judge every case scored by this version uses. */
|
|
295
|
+
judge: OutcomeJudgeConfig;
|
|
296
|
+
};
|
|
278
297
|
} | {
|
|
279
298
|
/** Scored by a person in Hue; the local runner defers it. */
|
|
280
299
|
kind: "manual";
|
package/dist/types.d.ts
CHANGED
|
@@ -13,10 +13,11 @@ export type Signal = "traces" | "logs";
|
|
|
13
13
|
/** Options accepted by every Hue client and transport, whether enabled or disabled. */
|
|
14
14
|
export interface SharedHueOptions {
|
|
15
15
|
/**
|
|
16
|
-
* Whether helpers record prompts, responses, tool arguments and results.
|
|
17
|
-
*
|
|
16
|
+
* Whether helpers record prompts, responses, tool arguments and results. Defaults to `true`
|
|
17
|
+
* for an enabled client; set `false` explicitly for metadata-only export. A disabled client
|
|
18
|
+
* (`enabled: false`) exports nothing and defaults it to `false`.
|
|
18
19
|
*/
|
|
19
|
-
captureContent
|
|
20
|
+
captureContent?: boolean;
|
|
20
21
|
/** Hue origin, `https://app.hue.run` by default. An origin only: no path, query, fragment or credentials. */
|
|
21
22
|
baseUrl?: string;
|
|
22
23
|
/** Recorded as the `service.version` resource attribute of an owned client. */
|
|
@@ -52,7 +53,8 @@ export interface SharedHueOptions {
|
|
|
52
53
|
}
|
|
53
54
|
/**
|
|
54
55
|
* Options for a client that owns its OpenTelemetry providers. An enabled client needs a project
|
|
55
|
-
* key
|
|
56
|
+
* key and a service name. Content capture defaults to `true`; `captureContent: false` opts out.
|
|
57
|
+
* The kill switch (`enabled: false`)
|
|
56
58
|
* exports nothing while helpers keep running application code, so it needs no key and
|
|
57
59
|
* `captureContent` defaults to `false`.
|
|
58
60
|
*/
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.14.0";
|
package/dist/version.js
CHANGED