@hue-run/sdk 0.20.1 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +102 -10
- package/ENVIRONMENTS.md +1 -1
- package/EVALUATIONS.md +3 -1
- package/README.md +30 -2
- package/dist/cli/eval.js +90 -34
- package/dist/cli/traces-import-convert.d.ts +262 -0
- package/dist/cli/traces-import-convert.js +1779 -0
- package/dist/cli/traces-import-prepare.d.ts +64 -0
- package/dist/cli/traces-import-prepare.js +443 -0
- package/dist/cli/traces-import-upload.d.ts +129 -0
- package/dist/cli/traces-import-upload.js +838 -0
- package/dist/cli/traces-import-zip.d.ts +52 -0
- package/dist/cli/traces-import-zip.js +278 -0
- package/dist/cli/traces-import.d.ts +18 -0
- package/dist/cli/traces-import.js +231 -0
- package/dist/evals/runner.d.ts +5 -2
- package/dist/evals/runner.js +31 -5
- package/dist/evals/simulation.d.ts +4 -0
- package/dist/evals/simulation.js +24 -26
- package/dist/setup/cli.js +7 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/world/world.d.ts +3 -2
- package/dist/world/world.js +4 -2
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.20.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.20.3`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -115,6 +115,8 @@ hue login # validate keys created in Hue and store them in ./.en
|
|
|
115
115
|
hue mcp install --client claude-code # write the Hue MCP configuration for a coding agent
|
|
116
116
|
hue listen --subscription <id> --forward-to http://localhost:3000/slack/events # deliver events to a local bot
|
|
117
117
|
hue import langfuse --from 2026-01-01 --dry-run # count a Langfuse project's traces, then import them
|
|
118
|
+
hue traces import prepare ./exports --out ./exports.hue-import --json # convert exports into a bundle
|
|
119
|
+
hue traces import upload ./exports.hue-import --authorization-file ./authorization.json --json # deliver it
|
|
118
120
|
```
|
|
119
121
|
|
|
120
122
|
Hosted setup accepts HTTPS origins only. HTTP is accepted solely for `localhost`, `127.0.0.1` and
|
|
@@ -379,8 +381,8 @@ and Hue never receives or stores them. The command is part of the `hue` executab
|
|
|
379
381
|
# LANGFUSE_PUBLIC_KEY=pk-lf-...
|
|
380
382
|
# LANGFUSE_SECRET_KEY=sk-lf-...
|
|
381
383
|
# LANGFUSE_BASE_URL=https://cloud.langfuse.com
|
|
382
|
-
npx -y @hue-run/sdk@0.20.
|
|
383
|
-
npx -y @hue-run/sdk@0.20.
|
|
384
|
+
npx -y @hue-run/sdk@0.20.3 import langfuse --env-path .env.langfuse --from 2026-01-01 --dry-run
|
|
385
|
+
npx -y @hue-run/sdk@0.20.3 import langfuse --env-path .env.langfuse --from 2026-01-01 --media
|
|
384
386
|
```
|
|
385
387
|
|
|
386
388
|
`LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` come from the environment or `--env-path`
|
|
@@ -491,6 +493,96 @@ once, abandoning exports still in flight for the next run to send. Exit codes: `
|
|
|
491
493
|
trace was imported, `1` a trace, span or file was not (or a key was refused), `2` usage error, `130`
|
|
492
494
|
stopped.
|
|
493
495
|
|
|
496
|
+
## Import a bundle of existing traces
|
|
497
|
+
|
|
498
|
+
`hue traces import` brings historical traces into Hue from exports you already hold: OTLP/JSON
|
|
499
|
+
trace exports (a `{"resourceSpans": […]}` document, a list of them, or one per line), complete
|
|
500
|
+
Langfuse observation exports (the `data` or native `observations` envelope) and text ATIF v1.7 and
|
|
501
|
+
v1.8 exports, embedded subagents included. It needs no project key. `prepare` runs on the machine
|
|
502
|
+
that holds the exports and sends nothing; a Hue MCP connection then authorizes the delivery
|
|
503
|
+
(`prepare_trace_import`); `upload` delivers the bundle with that authorization, and
|
|
504
|
+
`get_trace_import` reports what arrived.
|
|
505
|
+
|
|
506
|
+
```sh
|
|
507
|
+
npx -y @hue-run/sdk@latest traces import prepare ./exports --out ./exports.hue-import --json
|
|
508
|
+
npx -y @hue-run/sdk@latest traces import upload ./exports.hue-import --authorization-file ./authorization.json --json
|
|
509
|
+
```
|
|
510
|
+
|
|
511
|
+
**Prepare.** `prepare <path>` reads a file, a directory tree or a ZIP archive (members are inflated
|
|
512
|
+
in memory; encrypted members, unsafe paths and nested archives are listed, not read) and
|
|
513
|
+
recognizes each file by its content, never by its name. Every trace becomes gzip OTLP/JSON batches
|
|
514
|
+
of at most 3 MiB decoded and 900 KiB on the wire, written to `--out` (default `<path>.hue-import`,
|
|
515
|
+
created with mode 0700) beside `manifest.json`, which declares each trace's id, span count, batches
|
|
516
|
+
(by the SHA-256 of each gzip body), large values, history owner and origin. A text value over 1 MiB
|
|
517
|
+
is stored as a value of its own and listed in `hue.blobs`, as the exporter lists the values it
|
|
518
|
+
uploads. OTLP spans are kept as recorded; a trace or span id that is not an OpenTelemetry id (an
|
|
519
|
+
exporter's UUID, for example) is mapped to one derived from it by SHA-256, with the original kept
|
|
520
|
+
as `hue.import.original_trace_id` or `hue.import.original_span_id`. A Langfuse observation becomes
|
|
521
|
+
one span keeping its whole record (`hue.import.source_record`) beside the `langfuse.*` and GenAI
|
|
522
|
+
attributes Hue reads; an id that is no OpenTelemetry id is mapped as `hue import langfuse` maps it,
|
|
523
|
+
so an export and a live import of the same trace agree. An ATIF export becomes one trace: an agent
|
|
524
|
+
span for the run, a model span per agent step with the conversation so far, a tool span per call,
|
|
525
|
+
each embedded subagent as its own agent span under the call that spawned it, and the export's
|
|
526
|
+
`extra.user_id` as the trace's `user.id`. ATIF records instants, not durations, so model and tool
|
|
527
|
+
spans are points in time, and a tool's result is kept as text with no error status of the
|
|
528
|
+
importer's making. The same trace in two files is one trace when the copies agree and a refusal
|
|
529
|
+
when they differ; a trace with more than 100,000 spans, 64 batches or 1,000 values is refused and
|
|
530
|
+
named. Nothing about where a file sat or what it was called reaches a span: a renamed, reordered
|
|
531
|
+
or re-zipped copy of the bundle prepares to byte-identical batches and the same manifest digest.
|
|
532
|
+
|
|
533
|
+
The report (`--json` prints it as one object) names `manifest_digest`, `totals` (traces, batches,
|
|
534
|
+
spans, blobs, bytes) and `source`, which `prepare_trace_import` takes as given; the formats
|
|
535
|
+
recognized and the traces each file held; the files it could not read, with the reason; the traces
|
|
536
|
+
and spans it refused; the traces without a history owner (a root `user.id`, a Langfuse `userId`, an
|
|
537
|
+
ATIF `extra.user_id`, or an explicit `hue.world.entity.namespace` and `hue.world.entity.id`), which
|
|
538
|
+
import like any other trace but leave a gap in continuous history; the origin each source recorded
|
|
539
|
+
(`production`, `evaluation`, `simulation` or `unknown`; evaluation and simulation traffic is
|
|
540
|
+
imported and kept out of production history); and the traces whose vendor ids were mapped. An
|
|
541
|
+
existing `--out` directory is accepted only while it holds nothing but an earlier preparation and
|
|
542
|
+
no upload has started in it.
|
|
543
|
+
|
|
544
|
+
**Authorize.** Call `prepare_trace_import` over the Hue MCP with the report's `manifest_digest`,
|
|
545
|
+
`totals` and `source`. It answers the import id, `upload_url` and an `authorization` (`hue_it_` and
|
|
546
|
+
43 characters) valid for an hour. Save its answer, or the authorization alone, in a private file
|
|
547
|
+
(mode 0600; never in a transcript, a message or a shell history). The same digest in the same
|
|
548
|
+
project is the same import: calling again renews the authorization for an interrupted upload and
|
|
549
|
+
revokes the earlier one.
|
|
550
|
+
|
|
551
|
+
**Upload.** `upload <directory> --authorization-file <file>` reads the authorization from the file
|
|
552
|
+
(its JSON answer, or the authorization on the first line and the `upload_url` on the second;
|
|
553
|
+
`--import-url` names the URL otherwise, and `--origin` refuses a URL on any other Hue origin),
|
|
554
|
+
never from the command line or the environment, and never prints it. It registers the manifest's
|
|
555
|
+
exact bytes, then, trace by trace in declared order and two requests at a time, reserves each large
|
|
556
|
+
value, sends it to the URL Hue answers and completes it, posts each batch with its SHA-256 in
|
|
557
|
+
`Hue-Import-Batch`, and reads the trace's receipt for every span id: a trace counts as verified only
|
|
558
|
+
when Hue confirms every span. It then reconciles and completes the import. Progress is written to
|
|
559
|
+
`progress.json` in the bundle as it goes; running the same command again resumes there, and what
|
|
560
|
+
Hue already holds is not sent twice (a batch sent again after a lost acknowledgement is taken as
|
|
561
|
+
duplicates). Connection failures, timeouts, `408`, `429` and `5xx` answers are retried after 1, 2,
|
|
562
|
+
4, 8, 16 and 30 seconds, waiting for `Retry-After`; after eight retries the command stops and the
|
|
563
|
+
next run resumes. Ctrl+C or `SIGTERM` finishes the requests in flight, saves progress and exits
|
|
564
|
+
`130`; a second signal stops at once.
|
|
565
|
+
|
|
566
|
+
**What Hue refuses.** Every refusal carries a fixed code, which the report names in `error.code`.
|
|
567
|
+
An expired or replaced authorization (`401`), a lapsed connection (`403` `authority_lapsed`) and an
|
|
568
|
+
expired import (`410`) stop the upload: call `prepare_trace_import` again with the same
|
|
569
|
+
`manifest_digest` and `totals`, write the new authorization to the file and run the same command.
|
|
570
|
+
A manifest whose digest or totals are not the import's (`409`), an authorization of another import
|
|
571
|
+
(`403`), an unknown import (`404`) and a canceled import (`410`) are final. A batch or value Hue
|
|
572
|
+
refuses for the trace's own sake (undeclared, another digest, another trace's records, fewer
|
|
573
|
+
finished spans than declared, a size or type the manifest did not declare), or a record Hue already
|
|
574
|
+
holds under the same id with other content, is a conflict: it is recorded, the trace is reported,
|
|
575
|
+
none of its later records are sent, and the same bytes are never sent again. An import Hue reports
|
|
576
|
+
already completed is done: the command exits `0` and points to `get_trace_import`.
|
|
577
|
+
|
|
578
|
+
**Output and exit codes.** Without `--json`, the command prints the registration, every hundredth
|
|
579
|
+
verified trace, each conflict and a summary. With `--json`, one progress object per line goes to
|
|
580
|
+
stderr and the report to stdout: the state (`completed`, `incomplete`, `stopped` or `failed`), the
|
|
581
|
+
traces verified and conflicted, the batches landed and values stored, Hue's own note and the traces
|
|
582
|
+
it still lists when completion was refused, the conflicts with their codes, and `next_action`. Exit
|
|
583
|
+
codes: `0` prepared or completed, `1` not completed (the report says what to do next), `2` usage
|
|
584
|
+
error, `130` stopped.
|
|
585
|
+
|
|
494
586
|
## Local state and conflicts
|
|
495
587
|
|
|
496
588
|
Before its first network write, setup creates a lowercase UUIDv4 and a 32-byte random installation
|
|
@@ -595,7 +687,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
595
687
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
596
688
|
project=/absolute/path/to/supported-fixture
|
|
597
689
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
598
|
-
--archive .artifacts/typescript/hue-run-sdk-0.20.
|
|
690
|
+
--archive .artifacts/typescript/hue-run-sdk-0.20.3.tgz \
|
|
599
691
|
--origin https://STAGING_ORIGIN \
|
|
600
692
|
--project "$project" --command setup \
|
|
601
693
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -606,7 +698,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
606
698
|
|
|
607
699
|
```sh
|
|
608
700
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
609
|
-
--archive .artifacts/typescript/hue-run-sdk-0.20.
|
|
701
|
+
--archive .artifacts/typescript/hue-run-sdk-0.20.3.tgz \
|
|
610
702
|
--origin https://STAGING_ORIGIN \
|
|
611
703
|
--project "$project" --command claim \
|
|
612
704
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -648,10 +740,10 @@ otherwise. Never hand the agent Hue-native tools (`context.tools`, `context.mcp`
|
|
|
648
740
|
production agent lacks. Pin the SDK, so the agent's repository gains no dependency:
|
|
649
741
|
|
|
650
742
|
```sh
|
|
651
|
-
npx -y @hue-run/sdk@0.20.
|
|
652
|
-
npx -y @hue-run/sdk@0.20.
|
|
653
|
-
npx -y @hue-run/sdk@0.20.
|
|
654
|
-
npx -y @hue-run/sdk@0.20.
|
|
743
|
+
npx -y @hue-run/sdk@0.20.3 eval --case "Refund an eligible charge" --command "node agent.js"
|
|
744
|
+
npx -y @hue-run/sdk@0.20.3 eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <run id>
|
|
745
|
+
npx -y @hue-run/sdk@0.20.3 eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
746
|
+
npx -y @hue-run/sdk@0.20.3 eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision>
|
|
655
747
|
```
|
|
656
748
|
|
|
657
749
|
One invocation creates one run holding every selected case; it prints the run URL
|
|
@@ -689,7 +781,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
|
|
|
689
781
|
```
|
|
690
782
|
|
|
691
783
|
```sh
|
|
692
|
-
npx -y @hue-run/sdk@0.20.
|
|
784
|
+
npx -y @hue-run/sdk@0.20.3 eval --case "Refund an eligible charge" ./hue-agent.ts --content
|
|
693
785
|
```
|
|
694
786
|
|
|
695
787
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -332,7 +332,7 @@ A refusal such as 409 `simulation_gateway_required` exposes its validated server
|
|
|
332
332
|
|
|
333
333
|
`agentEnvironment` removes Hue control-plane credentials from the child by default: `HUE_API_KEY`,
|
|
334
334
|
`HUE_MCP_KEY`, and any value carrying a Hue control-plane credential (`hue_sk_`, `hue_setup_`,
|
|
335
|
-
`hue_attempt_`, `hue_mcp_`, `hue_inv_`, `hue_install_`, `hue_at_`, `hue_rt_`, `hue_oauth_`)
|
|
335
|
+
`hue_attempt_`, `hue_mcp_`, `hue_inv_`, `hue_install_`, `hue_at_`, `hue_rt_`, `hue_it_`, `hue_oauth_`)
|
|
336
336
|
anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. World tokens and
|
|
337
337
|
`hue_sim_` capabilities pass. Pass `includeHueCredentials: true` only for an agent that must call
|
|
338
338
|
Hue's own API.
|
package/EVALUATIONS.md
CHANGED
|
@@ -362,7 +362,9 @@ Python SDK's `run_experiment` and `rescore` hand pinned input files to targets a
|
|
|
362
362
|
generated files and the outbound worker are TypeScript-only.
|
|
363
363
|
|
|
364
364
|
Since TypeScript 0.13.2, the `hue eval --worker --command "…"` wrapper supplies both callbacks, so one
|
|
365
|
-
command worker accepts direct and world cases, including mixed eval sets.
|
|
365
|
+
command worker accepts direct and world cases, including mixed eval sets. Since 0.20.3, a one-shot
|
|
366
|
+
`hue eval --set` over such a set runs it as one `mixed` run: each case takes its own kind's path
|
|
367
|
+
under one experiment and its own checkpoint identity (`--mode simulation` over it is refused). It keeps the launch
|
|
366
368
|
directory for relative application entry points; private case inputs/output still use
|
|
367
369
|
`HUE_CASE_*`. An adapter-file worker opts into the direct callback with `--capability direct:v1`
|
|
368
370
|
and must handle `context.mode === "direct"`. Restart an existing environment-only command
|
package/README.md
CHANGED
|
@@ -536,8 +536,8 @@ the Langfuse project's own keys (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, a
|
|
|
536
536
|
`HUE_API_KEY`:
|
|
537
537
|
|
|
538
538
|
```sh
|
|
539
|
-
npx -y @hue-run/sdk@0.20.
|
|
540
|
-
npx -y @hue-run/sdk@0.20.
|
|
539
|
+
npx -y @hue-run/sdk@0.20.3 import langfuse --env-path .env.langfuse --from 2026-01-01 --dry-run
|
|
540
|
+
npx -y @hue-run/sdk@0.20.3 import langfuse --env-path .env.langfuse --from 2026-01-01 --media
|
|
541
541
|
```
|
|
542
542
|
|
|
543
543
|
- It reads Langfuse Cloud and self-hosted v4 through the Observations API v2, and self-hosted v3
|
|
@@ -558,6 +558,34 @@ The command has no Python counterpart; it runs with `npx` for an application in
|
|
|
558
558
|
[Import traces from Langfuse](CLI.md#import-traces-from-langfuse) for every option, the attribute
|
|
559
559
|
mapping, media handling, limits and exit codes.
|
|
560
560
|
|
|
561
|
+
## Import a bundle of existing traces
|
|
562
|
+
|
|
563
|
+
`hue traces import` brings exports you already hold into Hue in two steps, with no project key.
|
|
564
|
+
`prepare` converts the bundle on your machine and sends nothing; a Hue MCP connection authorizes
|
|
565
|
+
the delivery with `prepare_trace_import`; `upload` delivers it with that authorization. Sources
|
|
566
|
+
are files, directories or ZIP archives of OTLP/JSON trace exports, complete Langfuse observation
|
|
567
|
+
exports and text ATIF v1.7 and v1.8 exports, recognized by content:
|
|
568
|
+
|
|
569
|
+
```sh
|
|
570
|
+
npx -y @hue-run/sdk@latest traces import prepare ./exports --out ./exports.hue-import --json
|
|
571
|
+
# Call prepare_trace_import with the report's manifest_digest, totals and source, and save its
|
|
572
|
+
# answer (or the authorization alone) in a private file.
|
|
573
|
+
npx -y @hue-run/sdk@latest traces import upload ./exports.hue-import --authorization-file ./authorization.json --json
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
- `prepare` writes each trace as gzip OTLP/JSON batches beside a manifest and reports the
|
|
577
|
+
manifest's digest and totals, the formats recognized, the files it could not read, invalid vendor
|
|
578
|
+
ids it mapped to OpenTelemetry ids, and the traces without a history owner (a root `user.id` or an
|
|
579
|
+
explicit entity). Nothing about a file's name or place reaches a span.
|
|
580
|
+
- `upload` registers the manifest, sends each trace's large values and batches in order, two
|
|
581
|
+
requests at a time, checks every span's receipt, reconciles and completes the import. Interrupted,
|
|
582
|
+
run it again: it resumes from `progress.json` in the bundle. An expired authorization is renewed
|
|
583
|
+
by calling `prepare_trace_import` again with the same digest and totals.
|
|
584
|
+
- The authorization (`hue_it_…`) is read only from the file and never printed.
|
|
585
|
+
|
|
586
|
+
See [Import a bundle of existing traces](CLI.md#import-a-bundle-of-existing-traces) for the
|
|
587
|
+
formats, the report, what Hue refuses and the exit codes.
|
|
588
|
+
|
|
561
589
|
## Vercel AI SDK 6
|
|
562
590
|
|
|
563
591
|
AI SDK 6 accepts a per-call tracer through `experimental_telemetry`. Pass
|
package/dist/cli/eval.js
CHANGED
|
@@ -20,7 +20,8 @@ import { describeTelemetryIssues, runExperiment, TargetCancelledError, TELEMETRY
|
|
|
20
20
|
import { HueExportError } from "../transport.js";
|
|
21
21
|
import { matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenarioPins, } from "../evals/scenarios.js";
|
|
22
22
|
import { collectDirectOutputs, stageDirectCase } from "./eval-direct.js";
|
|
23
|
-
import {
|
|
23
|
+
import { runEnvironmentTarget } from "../evals/environment-target.js";
|
|
24
|
+
import { runSimulation, simulationTargetContext, } from "../evals/simulation.js";
|
|
24
25
|
import { collectExperimentVerdicts, compareVerdicts, metricPassed, } from "../evals/verdicts.js";
|
|
25
26
|
import { DEFAULT_ENV_FILE, envFileArgument, envFileOptions } from "./env-file.js";
|
|
26
27
|
const USAGE = `Usage: hue eval [adapter-file] [options]
|
|
@@ -35,7 +36,9 @@ Selection (exactly one, not used with --worker):
|
|
|
35
36
|
--dataset-version <id> Frozen dataset version; requires --scorer or --scorer-version
|
|
36
37
|
--scorer <slug|name|id> Evaluator to pin at its latest published version (repeatable)
|
|
37
38
|
--scorer-version <id> Scorer version to pin (repeatable)
|
|
38
|
-
--mode <auto|direct|simulation> Case kind; auto picks direct for sets whose cases pin no
|
|
39
|
+
--mode <auto|direct|simulation> Case kind; auto picks direct for sets whose cases pin no
|
|
40
|
+
world and runs a set mixing both kinds as one run, each
|
|
41
|
+
case through its own kind's path
|
|
39
42
|
|
|
40
43
|
Agent (exactly one):
|
|
41
44
|
<adapter-file> Module exporting default or runMyAgent(inputs, context)
|
|
@@ -979,9 +982,11 @@ function parseMode(value) {
|
|
|
979
982
|
return value;
|
|
980
983
|
throw new UsageError("--mode must be auto, direct or simulation");
|
|
981
984
|
}
|
|
982
|
-
/** The selection's cases
|
|
983
|
-
*
|
|
984
|
-
*
|
|
985
|
+
/** The selection's cases and how they run. When `--mode` leaves it open, a version mixing cases
|
|
986
|
+
* that pin a world with cases that pin none runs as one mixed run; direct means nothing pins a
|
|
987
|
+
* simulated world, not a Scenario and no case of the version; anything else is a simulation. A
|
|
988
|
+
* simulation run cannot execute a case that pins no world, so `--mode simulation` over such
|
|
989
|
+
* cases is refused. */
|
|
985
990
|
async function inspectSelection(client, pins, mode) {
|
|
986
991
|
let cases = 0;
|
|
987
992
|
let worldCases = 0;
|
|
@@ -992,14 +997,18 @@ async function inspectSelection(client, pins, mode) {
|
|
|
992
997
|
worldCases += page.items.filter((item) => item.environmentVersionId).length;
|
|
993
998
|
after = page.nextCursor ?? undefined;
|
|
994
999
|
} while (after);
|
|
995
|
-
const direct = mode === "auto"
|
|
996
|
-
? !pins.scenarioId && !pins.environmentVersionId && worldCases === 0
|
|
997
|
-
: mode === "direct";
|
|
998
1000
|
const answerOnly = cases - worldCases;
|
|
999
|
-
const
|
|
1000
|
-
?
|
|
1001
|
+
const kind = mode !== "auto"
|
|
1002
|
+
? mode
|
|
1003
|
+
: worldCases > 0 && answerOnly > 0
|
|
1004
|
+
? "mixed"
|
|
1005
|
+
: worldCases === 0 && !pins.scenarioId && !pins.environmentVersionId
|
|
1006
|
+
? "direct"
|
|
1007
|
+
: "simulation";
|
|
1008
|
+
const refusal = kind === "simulation" && answerOnly > 0
|
|
1009
|
+
? `${answerOnly} of the ${cases} ${cases === 1 ? "case pins" : "cases pin"} no world, so a simulation run would fail ${answerOnly === 1 ? "it" : "them"} before the agent starts. ${mode === "simulation" ? "Drop --mode simulation: --mode auto runs the world cases and the answer-only cases together as one run" : "Put the answer-only cases in their own eval set"}.`
|
|
1001
1010
|
: undefined;
|
|
1002
|
-
return {
|
|
1011
|
+
return { kind, cases, worldCases, ...(refusal ? { refusal } : {}) };
|
|
1003
1012
|
}
|
|
1004
1013
|
function toJson(verdicts, runUrl, baseline, extra = {}, telemetry = []) {
|
|
1005
1014
|
const notAccepted = new Map(telemetry.map((entry) => [entry.caseId, entry.issues]));
|
|
@@ -1255,7 +1264,9 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1255
1264
|
}
|
|
1256
1265
|
const runName = values.name ?? defaultRunName(agent);
|
|
1257
1266
|
const project = await client.checkConnection();
|
|
1258
|
-
|
|
1267
|
+
// Each kind of run has its own checkpoint directory and identity: a mixed run never resumes
|
|
1268
|
+
// from, or is resumed by, a direct or simulation run's checkpoint.
|
|
1269
|
+
const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, selection.kind);
|
|
1259
1270
|
const judging = values.judges
|
|
1260
1271
|
? {
|
|
1261
1272
|
store: await CheckpointStore.acquire(checkpointPath(checkpointDirectory, ["judge-dispatch", "judge checkpoint"]), { kind: "cli-judge-dispatch", projectId: project.id, baseUrl: connection.baseUrl }),
|
|
@@ -1267,7 +1278,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1267
1278
|
source: {
|
|
1268
1279
|
directory: checkpointDirectory,
|
|
1269
1280
|
identity: {
|
|
1270
|
-
kind: selection.
|
|
1281
|
+
kind: selection.kind,
|
|
1271
1282
|
projectId: project.id,
|
|
1272
1283
|
baseUrl: connection.baseUrl,
|
|
1273
1284
|
},
|
|
@@ -1283,8 +1294,9 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1283
1294
|
await completeJudgeHandoff(client, judging, pending);
|
|
1284
1295
|
return await reportVerdicts(client, values, pending.run, baselineId, output, signal, judging);
|
|
1285
1296
|
}
|
|
1286
|
-
if (selection.
|
|
1287
|
-
return await
|
|
1297
|
+
if (selection.kind !== "simulation")
|
|
1298
|
+
return await runExperimentAttempt({
|
|
1299
|
+
kind: selection.kind,
|
|
1288
1300
|
client,
|
|
1289
1301
|
values,
|
|
1290
1302
|
pins,
|
|
@@ -1298,6 +1310,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1298
1310
|
concurrency,
|
|
1299
1311
|
baselineId,
|
|
1300
1312
|
judging,
|
|
1313
|
+
checkpointDirectory,
|
|
1301
1314
|
}, connection);
|
|
1302
1315
|
const environmentClient = createEnvironmentClient(connection);
|
|
1303
1316
|
const caseKeys = new Map();
|
|
@@ -1393,8 +1406,11 @@ async function checkRun(client, values, pins, selection, agent, output) {
|
|
|
1393
1406
|
return 1;
|
|
1394
1407
|
}
|
|
1395
1408
|
}
|
|
1396
|
-
const {
|
|
1409
|
+
const { kind, cases, worldCases, refusal } = selection;
|
|
1397
1410
|
const unsaved = !pins.saved && !values["save-version"];
|
|
1411
|
+
// What each kind of run gives its cases: a direct run gives no case a world, a simulation
|
|
1412
|
+
// run gives every case one (and is refused over cases that pin none), a mixed run gives one
|
|
1413
|
+
// to each case that pins one.
|
|
1398
1414
|
const check = {
|
|
1399
1415
|
ok: !unsaved && !refusal,
|
|
1400
1416
|
project: { id: project.id, name: project.name },
|
|
@@ -1406,8 +1422,9 @@ async function checkRun(client, values, pins, selection, agent, output) {
|
|
|
1406
1422
|
cases,
|
|
1407
1423
|
evaluators: pins.scorerVersionIds.length,
|
|
1408
1424
|
},
|
|
1409
|
-
mode:
|
|
1410
|
-
worldCases: direct ? 0 : worldCases,
|
|
1425
|
+
mode: kind,
|
|
1426
|
+
worldCases: kind === "direct" ? 0 : worldCases,
|
|
1427
|
+
directCases: kind === "direct" ? cases : cases - worldCases,
|
|
1411
1428
|
agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "1" },
|
|
1412
1429
|
};
|
|
1413
1430
|
if (values.json)
|
|
@@ -1415,9 +1432,12 @@ async function checkRun(client, values, pins, selection, agent, output) {
|
|
|
1415
1432
|
else {
|
|
1416
1433
|
output.log(`Project: ${project.name} (${project.id})`);
|
|
1417
1434
|
output.log(`Selection: ${check.selection.kind === "case" ? "case" : "eval set"} "${pins.name}", ${cases} ${cases === 1 ? "case" : "cases"}, ${pins.scorerVersionIds.length} ${pins.scorerVersionIds.length === 1 ? "evaluator" : "evaluators"}`);
|
|
1418
|
-
|
|
1435
|
+
const worlds = `${worldCases} ${worldCases === 1 ? "case gets" : "cases get"} a fresh world, reached through the HUE_SIM_*_URL mirrors and HUE_WORLD_TOKEN`;
|
|
1436
|
+
output.log(kind === "direct"
|
|
1419
1437
|
? "Mode: direct; the agent answers without a simulated world or app mirrors"
|
|
1420
|
-
:
|
|
1438
|
+
: kind === "simulation"
|
|
1439
|
+
? `Mode: simulation; ${worlds}`
|
|
1440
|
+
: `Mode: mixed; ${worlds}, and ${check.directCases} ${check.directCases === 1 ? "case answers" : "cases answer"} without one`);
|
|
1421
1441
|
output.log(`Agent: ${agent.key} @ ${agent.revision}`);
|
|
1422
1442
|
}
|
|
1423
1443
|
if (refusal) {
|
|
@@ -1432,21 +1452,26 @@ async function checkRun(client, values, pins, selection, agent, output) {
|
|
|
1432
1452
|
return 0;
|
|
1433
1453
|
}
|
|
1434
1454
|
/**
|
|
1435
|
-
*
|
|
1436
|
-
* pinned files and returns generated documents
|
|
1437
|
-
*
|
|
1438
|
-
*
|
|
1439
|
-
*
|
|
1455
|
+
* One ordinary experiment through `runExperiment`, for a direct run and for a mixed one. A
|
|
1456
|
+
* direct run's agent receives the case's pinned files and returns generated documents. A mixed
|
|
1457
|
+
* run routes each case by its pin: a case pinning no world goes to the direct adapter, a case
|
|
1458
|
+
* pinning one gets a fresh world through the environment target and the simulation adapter, as
|
|
1459
|
+
* a simulation run gives it. Code evaluators pinned to the run stay deferred for Hue's executor
|
|
1460
|
+
* (`deferUnboundLocalScorers`), so no evaluator source runs on this machine. The attempt is
|
|
1461
|
+
* checkpointed as in simulation mode, under the run kind's own identity: rerunning the same
|
|
1462
|
+
* selection resumes the saved experiment without invoking the agent again for its finished
|
|
1463
|
+
* cases.
|
|
1440
1464
|
*/
|
|
1441
|
-
async function
|
|
1442
|
-
const { client, values, pins, output, signal } = run;
|
|
1465
|
+
async function runExperimentAttempt(run, connection) {
|
|
1466
|
+
const { client, kind, values, pins, output, signal } = run;
|
|
1443
1467
|
const config = { agentKey: run.agent.key, agentRevision: run.agent.revision };
|
|
1444
|
-
const
|
|
1445
|
-
|
|
1446
|
-
kind: "direct",
|
|
1468
|
+
const store = await CheckpointStore.acquire(run.checkpointDirectory, {
|
|
1469
|
+
kind,
|
|
1447
1470
|
projectId: run.projectId,
|
|
1448
1471
|
baseUrl: connection.baseUrl,
|
|
1449
1472
|
});
|
|
1473
|
+
// Only a mixed run creates worlds; its world cases reach Hue through the environment client.
|
|
1474
|
+
const environmentClient = kind === "mixed" ? createEnvironmentClient(connection) : undefined;
|
|
1450
1475
|
let experimentId = "";
|
|
1451
1476
|
let runUrl = "";
|
|
1452
1477
|
let report;
|
|
@@ -1459,7 +1484,7 @@ async function runDirect(run, connection) {
|
|
|
1459
1484
|
});
|
|
1460
1485
|
let attempt = await store.read("active-attempt");
|
|
1461
1486
|
if (attempt && attempt.stage !== "completed" && attempt.selectionDigest !== selectionDigest)
|
|
1462
|
-
throw new Error(
|
|
1487
|
+
throw new Error(`Recover the unfinished ${kind} run before running a changed selection: run the unchanged command again to finish it, or pass another --checkpoint-dir`);
|
|
1463
1488
|
if (!attempt || attempt.stage === "completed") {
|
|
1464
1489
|
attempt = { selectionDigest, idempotencyKey: randomUUID(), stage: "preparing" };
|
|
1465
1490
|
await store.write("active-attempt", attempt);
|
|
@@ -1496,10 +1521,41 @@ async function runDirect(run, connection) {
|
|
|
1496
1521
|
scorers: [],
|
|
1497
1522
|
deferUnboundLocalScorers: true,
|
|
1498
1523
|
environmentEvidence: "when_pinned",
|
|
1524
|
+
// A world case that a process which died here left running runs again in a fresh world;
|
|
1525
|
+
// an answer-only case it left running stays uncertain, since its agent may still be
|
|
1526
|
+
// acting. A direct run gives no case a world, so none of its cases reruns.
|
|
1527
|
+
...(environmentClient
|
|
1528
|
+
? { rerunCrashedCases: (item) => Boolean(item.environmentVersionId) }
|
|
1529
|
+
: {}),
|
|
1499
1530
|
async target(inputs, context) {
|
|
1500
1531
|
if (signal.aborted)
|
|
1501
1532
|
throw new TargetCancelledError();
|
|
1502
|
-
|
|
1533
|
+
const label = context.item.externalKey;
|
|
1534
|
+
if (environmentClient && context.item.environmentVersionId)
|
|
1535
|
+
return runEnvironmentTarget({
|
|
1536
|
+
client,
|
|
1537
|
+
environmentClient,
|
|
1538
|
+
hue: run.hue,
|
|
1539
|
+
inputs,
|
|
1540
|
+
context,
|
|
1541
|
+
agentRevision: run.agent.revision,
|
|
1542
|
+
// The CLI adapts to whatever the deployment serves; the library warning is for
|
|
1543
|
+
// code that still reads the legacy capability itself.
|
|
1544
|
+
deprecationWarnings: false,
|
|
1545
|
+
signal,
|
|
1546
|
+
onProgress(event) {
|
|
1547
|
+
if (event.type === "world_created")
|
|
1548
|
+
output.log(`[${label}] world created`);
|
|
1549
|
+
else if (event.type === "target_started")
|
|
1550
|
+
output.log(`[${label}] agent started`);
|
|
1551
|
+
else if (event.type === "world_sealed")
|
|
1552
|
+
output.log(`[${label}] world sealed`);
|
|
1553
|
+
else if (event.type === "attempt_prepared")
|
|
1554
|
+
output.log(`[${label}] attempt prepared: ${event.status}`);
|
|
1555
|
+
},
|
|
1556
|
+
target: (targetInputs, targetContext) => run.agents.simulation(structuredClone(targetInputs), simulationTargetContext(targetContext)),
|
|
1557
|
+
});
|
|
1558
|
+
output.log(`[${label}] agent started`);
|
|
1503
1559
|
const result = await run.agents.direct(inputs, {
|
|
1504
1560
|
mode: "direct",
|
|
1505
1561
|
config: structuredClone(context.config),
|
|
@@ -1522,7 +1578,7 @@ async function runDirect(run, connection) {
|
|
|
1522
1578
|
experimentId,
|
|
1523
1579
|
subjectIds: report.subjectIds,
|
|
1524
1580
|
runUrl,
|
|
1525
|
-
extra: { mode:
|
|
1581
|
+
extra: { mode: kind, deferredScorerVersionIds: report.deferredScorerVersionIds },
|
|
1526
1582
|
telemetryNotAccepted: report.telemetryNotAccepted,
|
|
1527
1583
|
}, { kind: "attempt" });
|
|
1528
1584
|
attempt.stage = "completed";
|
|
@@ -1538,7 +1594,7 @@ async function runDirect(run, connection) {
|
|
|
1538
1594
|
experimentId,
|
|
1539
1595
|
subjectIds: report.subjectIds,
|
|
1540
1596
|
runUrl,
|
|
1541
|
-
extra: { mode:
|
|
1597
|
+
extra: { mode: kind, deferredScorerVersionIds: report.deferredScorerVersionIds },
|
|
1542
1598
|
telemetryNotAccepted: report.telemetryNotAccepted,
|
|
1543
1599
|
}, run.baselineId, output, signal, run.judging);
|
|
1544
1600
|
}
|