@hue-run/sdk 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +5 -5
- package/ENVIRONMENTS.md +1 -1
- package/dist/evals/simulation.d.ts +12 -4
- package/dist/evals/simulation.js +34 -24
- package/dist/evals.d.ts +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,8 +42,8 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.4.
|
|
46
|
-
`@opentelemetry/context-async-hooks@2.11.0`; Python uses `hue-run==0.2.2`.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.4.2`, `@opentelemetry/api@1.9.1` and
|
|
46
|
+
`@opentelemetry/context-async-hooks@2.11.0`; Python uses published `hue-run==0.2.2`.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
49
49
|
The generated bootstrap supplies standard active SERVER-span context across asynchronous/streaming
|
|
@@ -172,7 +172,7 @@ capability `setup_telemetry_write`.
|
|
|
172
172
|
Normal project credentials and unknown token shapes are refused in setup responses and private
|
|
173
173
|
managed state. These credentials authorize metadata-only OTLP at `/api/v1/otlp/v1/traces` and exact
|
|
174
174
|
content-free receipt verification at `/api/v1/setup/traces/{traceId}/receipt`. They do not authorize
|
|
175
|
-
generic project, receipt, evaluation, log or browsing APIs. Python `0.2.2` can export with the setup
|
|
175
|
+
generic project, receipt, evaluation, log or browsing APIs. Published Python `0.2.2` can export with the setup
|
|
176
176
|
credential; the CLI verifies the dedicated setup receipt instead of Python's generic receipt helper.
|
|
177
177
|
|
|
178
178
|
Each command uses bounded timeouts and retries. Provisioning records at most five attempts per local
|
|
@@ -241,7 +241,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
241
241
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
242
242
|
project=/absolute/path/to/supported-fixture
|
|
243
243
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
244
|
-
--archive .artifacts/typescript/hue-run-sdk-0.4.
|
|
244
|
+
--archive .artifacts/typescript/hue-run-sdk-0.4.2.tgz \
|
|
245
245
|
--origin https://STAGING_ORIGIN \
|
|
246
246
|
--project "$project" --command setup \
|
|
247
247
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -252,7 +252,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
252
252
|
|
|
253
253
|
```sh
|
|
254
254
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
255
|
-
--archive .artifacts/typescript/hue-run-sdk-0.4.
|
|
255
|
+
--archive .artifacts/typescript/hue-run-sdk-0.4.2.tgz \
|
|
256
256
|
--origin https://STAGING_ORIGIN \
|
|
257
257
|
--project "$project" --command claim \
|
|
258
258
|
--evidence .context/setup-staging-after-claim.json
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -30,7 +30,7 @@ try {
|
|
|
30
30
|
environmentClient: createEnvironmentClient(connection),
|
|
31
31
|
hue,
|
|
32
32
|
checkpointDirectory: ".hue-checkpoints/refund-scenario",
|
|
33
|
-
|
|
33
|
+
definition: { kind: "experiment", experimentId: process.env.HUE_EXPERIMENT_ID! },
|
|
34
34
|
persistResultContent: true,
|
|
35
35
|
traceEvidence: { mode: "required" },
|
|
36
36
|
target: (inputs, { tools, mcp, config, signal }) =>
|
|
@@ -22,8 +22,8 @@ export interface RepositorySimulationCase {
|
|
|
22
22
|
/** Optional caller-owned case metadata. */
|
|
23
23
|
metadata?: Record<string, JsonValue>;
|
|
24
24
|
}
|
|
25
|
-
/** Immutable app-authored experiment reference or repository-authored
|
|
26
|
-
export type
|
|
25
|
+
/** Immutable app-authored experiment reference or repository-authored simulation definition. */
|
|
26
|
+
export type SimulationDefinition = {
|
|
27
27
|
/** Select an existing app-authored immutable experiment template. */
|
|
28
28
|
kind: "experiment";
|
|
29
29
|
/** Experiment to clone into a fresh attempt. */
|
|
@@ -118,8 +118,12 @@ export interface RunSimulationOptions {
|
|
|
118
118
|
hue: HueClient;
|
|
119
119
|
/** Dedicated private directory for resumable checkpoints. */
|
|
120
120
|
checkpointDirectory: string;
|
|
121
|
-
/** App-authored reference or repository-authored
|
|
122
|
-
|
|
121
|
+
/** App-authored reference or repository-authored definition. */
|
|
122
|
+
definition?: SimulationDefinition;
|
|
123
|
+
/**
|
|
124
|
+
* @deprecated Use `definition`.
|
|
125
|
+
*/
|
|
126
|
+
scenario?: SimulationDefinition;
|
|
123
127
|
/** Required privacy decision for saved target/scorer content. */
|
|
124
128
|
persistResultContent: boolean;
|
|
125
129
|
/** Required trace receipt policy for every target attempt. */
|
|
@@ -178,6 +182,10 @@ export interface SimulationReport extends RunnerReport {
|
|
|
178
182
|
/** Browser URL joining task, trace, world evidence and scoring. */
|
|
179
183
|
runUrl: string;
|
|
180
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* @deprecated Use {@link SimulationDefinition}.
|
|
187
|
+
*/
|
|
188
|
+
export type SimulationScenario = SimulationDefinition;
|
|
181
189
|
/** Run an existing agent callback against one fresh hosted world per case. The helper
|
|
182
190
|
* owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
|
|
183
191
|
*/
|
package/dist/evals/simulation.js
CHANGED
|
@@ -46,24 +46,24 @@ function normalizedEnvironmentDefinition(definition) {
|
|
|
46
46
|
metadata: definition.metadata ?? {},
|
|
47
47
|
}, aggregateBounds(240_000));
|
|
48
48
|
}
|
|
49
|
-
function
|
|
50
|
-
if (
|
|
51
|
-
return
|
|
49
|
+
function definitionIdentity(definition) {
|
|
50
|
+
if (definition.kind === "experiment")
|
|
51
|
+
return definition;
|
|
52
52
|
return json({
|
|
53
|
-
kind:
|
|
54
|
-
name:
|
|
55
|
-
slug:
|
|
56
|
-
description:
|
|
53
|
+
kind: definition.kind,
|
|
54
|
+
name: definition.name,
|
|
55
|
+
slug: definition.slug,
|
|
56
|
+
description: definition.description ?? "",
|
|
57
57
|
environment: {
|
|
58
|
-
...
|
|
59
|
-
definition: normalizedEnvironmentDefinition(
|
|
58
|
+
...definition.environment,
|
|
59
|
+
definition: normalizedEnvironmentDefinition(definition.environment.definition),
|
|
60
60
|
},
|
|
61
|
-
cases:
|
|
62
|
-
scorers:
|
|
61
|
+
cases: definition.cases,
|
|
62
|
+
scorers: definition.scorers.map(({ scorer, ...identity }) => ({
|
|
63
63
|
...identity,
|
|
64
64
|
definition: normalizeScorerDefinitionForPublication("definition" in scorer ? scorer.definition : scorer),
|
|
65
65
|
})),
|
|
66
|
-
config:
|
|
66
|
+
config: definition.config ?? {},
|
|
67
67
|
}, aggregateBounds(8 * 1024 * 1024));
|
|
68
68
|
}
|
|
69
69
|
async function findBySlug(page, slug) {
|
|
@@ -283,9 +283,9 @@ async function allCases(client, versionId) {
|
|
|
283
283
|
after = page.nextCursor;
|
|
284
284
|
}
|
|
285
285
|
}
|
|
286
|
-
async function resolveExperiment(options, idempotencyKey) {
|
|
287
|
-
if (
|
|
288
|
-
const source = await options.client.getExperiment(
|
|
286
|
+
async function resolveExperiment(options, definition, idempotencyKey) {
|
|
287
|
+
if (definition.kind === "experiment") {
|
|
288
|
+
const source = await options.client.getExperiment(definition.experimentId);
|
|
289
289
|
const created = await options.client.createExperiment({
|
|
290
290
|
idempotencyKey,
|
|
291
291
|
name: options.runName ?? source.name,
|
|
@@ -295,25 +295,35 @@ async function resolveExperiment(options, idempotencyKey) {
|
|
|
295
295
|
});
|
|
296
296
|
return { experimentId: created.id, bindings: options.localScorers ?? [] };
|
|
297
297
|
}
|
|
298
|
-
const environmentVersionId = await resolveEnvironment(options.environmentClient,
|
|
299
|
-
const datasetVersionId = await resolveDataset(options.client,
|
|
300
|
-
const scorers = await resolveScorers(options.client,
|
|
298
|
+
const environmentVersionId = await resolveEnvironment(options.environmentClient, definition.environment);
|
|
299
|
+
const datasetVersionId = await resolveDataset(options.client, definition, environmentVersionId);
|
|
300
|
+
const scorers = await resolveScorers(options.client, definition.scorers);
|
|
301
301
|
const created = await options.client.createExperiment({
|
|
302
302
|
idempotencyKey,
|
|
303
|
-
name: options.runName ??
|
|
303
|
+
name: options.runName ?? definition.name,
|
|
304
304
|
datasetVersionId,
|
|
305
305
|
scorerVersionIds: scorers.versionIds,
|
|
306
|
-
config:
|
|
306
|
+
config: definition.config ?? {},
|
|
307
307
|
});
|
|
308
308
|
return {
|
|
309
309
|
experimentId: created.id,
|
|
310
310
|
bindings: [...scorers.bindings, ...(options.localScorers ?? [])],
|
|
311
311
|
};
|
|
312
312
|
}
|
|
313
|
+
let scenarioDeprecationWarned = false;
|
|
313
314
|
/** Run an existing agent callback against one fresh hosted world per case. The helper
|
|
314
315
|
* owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
|
|
315
316
|
*/
|
|
316
317
|
export async function runSimulation(options) {
|
|
318
|
+
const definition = options.definition ?? options.scenario;
|
|
319
|
+
if (!definition)
|
|
320
|
+
throw new TypeError("runSimulation requires a definition");
|
|
321
|
+
if (options.definition && options.scenario && options.definition !== options.scenario)
|
|
322
|
+
throw new TypeError("Pass either definition or scenario, not both");
|
|
323
|
+
if (!options.definition && !scenarioDeprecationWarned) {
|
|
324
|
+
scenarioDeprecationWarned = true;
|
|
325
|
+
process.emitWarning("runSimulation option scenario is deprecated; use definition", "DeprecationWarning");
|
|
326
|
+
}
|
|
317
327
|
const requestedConfiguration = requestedAttemptV2(options);
|
|
318
328
|
if (options.maxSteps !== undefined &&
|
|
319
329
|
(!Number.isInteger(options.maxSteps) ||
|
|
@@ -332,7 +342,7 @@ export async function runSimulation(options) {
|
|
|
332
342
|
baseUrl: options.client.baseUrl,
|
|
333
343
|
});
|
|
334
344
|
try {
|
|
335
|
-
const scenarioDigest = digest(
|
|
345
|
+
const scenarioDigest = digest(definitionIdentity(definition));
|
|
336
346
|
let attempt = await store.read("active-attempt");
|
|
337
347
|
if (attempt && attempt.stage !== "completed" && attempt.scenarioDigest !== scenarioDigest)
|
|
338
348
|
throw new Error("Recover the unfinished simulation before running a changed scenario");
|
|
@@ -346,15 +356,15 @@ export async function runSimulation(options) {
|
|
|
346
356
|
}
|
|
347
357
|
let bindings = options.localScorers ?? [];
|
|
348
358
|
if (!attempt.experimentId) {
|
|
349
|
-
const resolved = await resolveExperiment(options, attempt.idempotencyKey);
|
|
359
|
+
const resolved = await resolveExperiment(options, definition, attempt.idempotencyKey);
|
|
350
360
|
attempt.experimentId = resolved.experimentId;
|
|
351
361
|
bindings = resolved.bindings;
|
|
352
362
|
attempt.stage = "running";
|
|
353
363
|
await store.write("active-attempt", attempt);
|
|
354
364
|
}
|
|
355
|
-
else if (
|
|
365
|
+
else if (definition.kind === "repository") {
|
|
356
366
|
bindings = [
|
|
357
|
-
...
|
|
367
|
+
...definition.scorers
|
|
358
368
|
.filter((item) => "definition" in item.scorer)
|
|
359
369
|
.map((item) => item.scorer),
|
|
360
370
|
...(options.localScorers ?? []),
|
package/dist/evals.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export type { EvaluationClientOptions } from "./evals/client.js";
|
|
|
3
3
|
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
|
|
4
4
|
export type { RunExperimentOptions, RescoreOptions, RunnerReport } from "./evals/runner.js";
|
|
5
5
|
export { runSimulation } from "./evals/simulation.js";
|
|
6
|
-
export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
|
|
6
|
+
export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationDefinition, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
|
|
7
7
|
export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
|
|
8
8
|
export type { ActualAgentManifestInputV2, ActualAgentManifestV2, AttemptBaselineV2, AttemptBindingRead, AttemptConnectionBundleV2, AttemptIdentityV2, DependencyManifestV2, DependencyProviderV2, ExpectedAgentManifestV2, ParityEvidenceV2, PreflightFindingV2, PreflightReportV2, PrepareAttemptIncompleteV2, PrepareAttemptInputV2, PrepareAttemptReadyV2, PrepareAttemptRequestV2, PrepareAttemptResultV2, RefreshAttemptResultV2, RequestedAttemptProviderV2, RevokeAttemptResult, SurfaceBindingV2, } from "./evals/attempt.js";
|
|
9
9
|
export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.4.
|
|
2
|
+
export declare const sdkVersion = "0.4.2";
|
package/dist/version.js
CHANGED