@hue-run/sdk 0.20.1 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +102 -10
- package/ENVIRONMENTS.md +1 -1
- package/EVALUATIONS.md +3 -1
- package/README.md +30 -2
- package/dist/cli/eval.js +90 -34
- package/dist/cli/traces-import-convert.d.ts +262 -0
- package/dist/cli/traces-import-convert.js +1779 -0
- package/dist/cli/traces-import-prepare.d.ts +64 -0
- package/dist/cli/traces-import-prepare.js +443 -0
- package/dist/cli/traces-import-upload.d.ts +129 -0
- package/dist/cli/traces-import-upload.js +838 -0
- package/dist/cli/traces-import-zip.d.ts +52 -0
- package/dist/cli/traces-import-zip.js +278 -0
- package/dist/cli/traces-import.d.ts +18 -0
- package/dist/cli/traces-import.js +231 -0
- package/dist/evals/runner.d.ts +5 -2
- package/dist/evals/runner.js +31 -5
- package/dist/evals/simulation.d.ts +4 -0
- package/dist/evals/simulation.js +24 -26
- package/dist/setup/cli.js +7 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/world/world.d.ts +3 -2
- package/dist/world/world.js +4 -2
- package/package.json +1 -1
|
@@ -4,6 +4,7 @@ import type { EnvironmentTool } from "../world/tools.js";
|
|
|
4
4
|
import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../world/types.js";
|
|
5
5
|
import { type EvaluationClient } from "./client.js";
|
|
6
6
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
7
|
+
import { type EnvironmentTargetContext } from "./environment-target.js";
|
|
7
8
|
import { type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
|
|
8
9
|
import type { ExperimentCase, Identity, JsonValue, LocalFile, LocalScorer, ScorerDefinition, SimulationMcpCapability, TargetResult } from "./types.js";
|
|
9
10
|
/** Repository-authored scorer identity and its public definition or local binding. */
|
|
@@ -222,6 +223,9 @@ export interface SimulationReport extends RunnerReport {
|
|
|
222
223
|
* @deprecated Use {@link SimulationDefinition}.
|
|
223
224
|
*/
|
|
224
225
|
export type SimulationScenario = SimulationDefinition;
|
|
226
|
+
/** The candidate-visible context of a world case, cloned from the environment target's: the
|
|
227
|
+
* frozen case's identity alone, never its expected outcome, metadata or source pins. */
|
|
228
|
+
export declare function simulationTargetContext(context: EnvironmentTargetContext): SimulationTargetContext;
|
|
225
229
|
/** Run an existing agent callback against one fresh hosted world per case. The helper
|
|
226
230
|
* owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
|
|
227
231
|
*/
|
package/dist/evals/simulation.js
CHANGED
|
@@ -344,6 +344,29 @@ async function resolveExperiment(options, definition, idempotencyKey) {
|
|
|
344
344
|
};
|
|
345
345
|
}
|
|
346
346
|
let scenarioDeprecationWarned = false;
|
|
347
|
+
/** The candidate-visible context of a world case, cloned from the environment target's: the
|
|
348
|
+
* frozen case's identity alone, never its expected outcome, metadata or source pins. */
|
|
349
|
+
export function simulationTargetContext(context) {
|
|
350
|
+
return {
|
|
351
|
+
config: structuredClone(context.config),
|
|
352
|
+
item: { id: context.item.id, externalKey: context.item.externalKey },
|
|
353
|
+
executionId: context.executionId,
|
|
354
|
+
environmentRunId: context.environmentRunId,
|
|
355
|
+
tools: context.tools,
|
|
356
|
+
...(context.world ? { world: structuredClone(context.world) } : {}),
|
|
357
|
+
...(context.mcp
|
|
358
|
+
? {
|
|
359
|
+
mcp: { url: context.mcp.url, token: context.mcp.token, expiresAt: context.mcp.expiresAt },
|
|
360
|
+
}
|
|
361
|
+
: {}),
|
|
362
|
+
...(context.connectionBundle
|
|
363
|
+
? { connectionBundle: structuredClone(context.connectionBundle) }
|
|
364
|
+
: {}),
|
|
365
|
+
files: structuredClone(context.files),
|
|
366
|
+
outputDirectory: context.outputDirectory,
|
|
367
|
+
signal: context.signal,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
347
370
|
/** Run an existing agent callback against one fresh hosted world per case. The helper
|
|
348
371
|
* owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
|
|
349
372
|
*/
|
|
@@ -462,32 +485,7 @@ export async function runSimulation(options) {
|
|
|
462
485
|
executionId: context.executionId,
|
|
463
486
|
caseId: context.item.id,
|
|
464
487
|
}),
|
|
465
|
-
target: (targetInputs, targetContext) => options.target(structuredClone(targetInputs),
|
|
466
|
-
config: structuredClone(targetContext.config),
|
|
467
|
-
item: {
|
|
468
|
-
id: targetContext.item.id,
|
|
469
|
-
externalKey: targetContext.item.externalKey,
|
|
470
|
-
},
|
|
471
|
-
executionId: targetContext.executionId,
|
|
472
|
-
environmentRunId: targetContext.environmentRunId,
|
|
473
|
-
tools: targetContext.tools,
|
|
474
|
-
...(targetContext.world ? { world: structuredClone(targetContext.world) } : {}),
|
|
475
|
-
...(targetContext.mcp
|
|
476
|
-
? {
|
|
477
|
-
mcp: {
|
|
478
|
-
url: targetContext.mcp.url,
|
|
479
|
-
token: targetContext.mcp.token,
|
|
480
|
-
expiresAt: targetContext.mcp.expiresAt,
|
|
481
|
-
},
|
|
482
|
-
}
|
|
483
|
-
: {}),
|
|
484
|
-
...(targetContext.connectionBundle
|
|
485
|
-
? { connectionBundle: structuredClone(targetContext.connectionBundle) }
|
|
486
|
-
: {}),
|
|
487
|
-
files: structuredClone(targetContext.files),
|
|
488
|
-
outputDirectory: targetContext.outputDirectory,
|
|
489
|
-
signal: targetContext.signal,
|
|
490
|
-
}),
|
|
488
|
+
target: (targetInputs, targetContext) => options.target(structuredClone(targetInputs), simulationTargetContext(targetContext)),
|
|
491
489
|
}),
|
|
492
490
|
});
|
|
493
491
|
const complete = { ...report, experimentId, runUrl };
|
package/dist/setup/cli.js
CHANGED
|
@@ -32,6 +32,10 @@ const extensions = new Map([
|
|
|
32
32
|
"listen",
|
|
33
33
|
async () => (await loadCommand("listen", () => import("../cli/listen.js"))).runListenCommand(process.argv.slice(3)),
|
|
34
34
|
],
|
|
35
|
+
[
|
|
36
|
+
"traces",
|
|
37
|
+
async () => (await loadCommand("traces", () => import("../cli/traces-import.js"))).runTracesCommand(process.argv.slice(3)),
|
|
38
|
+
],
|
|
35
39
|
]);
|
|
36
40
|
function writeEvent(event, mode, width) {
|
|
37
41
|
const line = mode === "jsonl"
|
|
@@ -74,7 +78,7 @@ async function main() {
|
|
|
74
78
|
}
|
|
75
79
|
catch {
|
|
76
80
|
if (!agentRequested) {
|
|
77
|
-
process.stderr.write("Usage: hue <setup|resume|status|claim|login|eval|mcp|listen|import> [--agent|--format human|plain|jsonl] [--project PATH] [--origin URL] [--restart]\n");
|
|
81
|
+
process.stderr.write("Usage: hue <setup|resume|status|claim|login|eval|mcp|listen|import|traces> [--agent|--format human|plain|jsonl] [--project PATH] [--origin URL] [--restart]\n");
|
|
78
82
|
return 2;
|
|
79
83
|
}
|
|
80
84
|
const event = {
|
|
@@ -105,7 +109,7 @@ async function main() {
|
|
|
105
109
|
process.stdout.write(`${renderJsonlEvent(event)}\n`);
|
|
106
110
|
return 2;
|
|
107
111
|
}
|
|
108
|
-
process.stdout.write("Usage: hue <setup|resume|status|claim|login|eval|mcp|listen|import> [--agent|--format human|plain|jsonl] [--project PATH] [--origin URL] [--restart]\n");
|
|
112
|
+
process.stdout.write("Usage: hue <setup|resume|status|claim|login|eval|mcp|listen|import|traces> [--agent|--format human|plain|jsonl] [--project PATH] [--origin URL] [--restart]\n");
|
|
109
113
|
return 0;
|
|
110
114
|
}
|
|
111
115
|
const command = parsed.positionals[0];
|
|
@@ -137,7 +141,7 @@ async function main() {
|
|
|
137
141
|
process.stdout.write(`${renderJsonlEvent(event)}\n`);
|
|
138
142
|
}
|
|
139
143
|
else
|
|
140
|
-
process.stderr.write("Usage: hue <setup|resume|status|claim|login|eval|mcp|listen|import> [--agent|--format human|plain|jsonl] [--project PATH] [--origin URL] [--restart]\n");
|
|
144
|
+
process.stderr.write("Usage: hue <setup|resume|status|claim|login|eval|mcp|listen|import|traces> [--agent|--format human|plain|jsonl] [--project PATH] [--origin URL] [--restart]\n");
|
|
141
145
|
return 2;
|
|
142
146
|
}
|
|
143
147
|
const mode = selectSetupOutputMode({
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.20.
|
|
2
|
+
export declare const sdkVersion = "0.20.3";
|
package/dist/version.js
CHANGED
package/dist/world/world.d.ts
CHANGED
|
@@ -16,8 +16,9 @@ export declare function worldNow(source?: WorldHandoff | Record<string, string |
|
|
|
16
16
|
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
17
17
|
export declare const HUE_CONTROL_PLANE_VARIABLES: readonly string[];
|
|
18
18
|
/** Prefixes of the Hue control-plane credentials the server issues: project and setup keys,
|
|
19
|
-
* attempt grants, the project MCP key, invocation and installation tokens,
|
|
20
|
-
* secrets. World tokens (`hue_world_`) and MCP capabilities
|
|
19
|
+
* attempt grants, the project MCP key, invocation and installation tokens, OAuth tokens, trace
|
|
20
|
+
* import authorizations and secrets. World tokens (`hue_world_`) and MCP capabilities
|
|
21
|
+
* (`hue_sim_`) are what the agent holds. */
|
|
21
22
|
export declare const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES: readonly string[];
|
|
22
23
|
/** True for a control-plane variable by name, or for a value carrying a Hue control-plane
|
|
23
24
|
* credential anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. */
|
package/dist/world/world.js
CHANGED
|
@@ -53,8 +53,9 @@ export const HUE_CONTROL_PLANE_VARIABLES = [
|
|
|
53
53
|
"HUE_SERVICE_KEY",
|
|
54
54
|
];
|
|
55
55
|
/** Prefixes of the Hue control-plane credentials the server issues: project and setup keys,
|
|
56
|
-
* attempt grants, the project MCP key, invocation and installation tokens,
|
|
57
|
-
* secrets. World tokens (`hue_world_`) and MCP capabilities
|
|
56
|
+
* attempt grants, the project MCP key, invocation and installation tokens, OAuth tokens, trace
|
|
57
|
+
* import authorizations and secrets. World tokens (`hue_world_`) and MCP capabilities
|
|
58
|
+
* (`hue_sim_`) are what the agent holds. */
|
|
58
59
|
export const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES = [
|
|
59
60
|
"hue_sk_",
|
|
60
61
|
"hue_attempt_",
|
|
@@ -64,6 +65,7 @@ export const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES = [
|
|
|
64
65
|
"hue_install_",
|
|
65
66
|
"hue_at_",
|
|
66
67
|
"hue_rt_",
|
|
68
|
+
"hue_it_",
|
|
67
69
|
"hue_oauth_",
|
|
68
70
|
];
|
|
69
71
|
/** A Hue credential anywhere in a value, including JSON Unicode escapes, not inside a longer word. */
|