@hue-run/sdk 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +52 -0
- package/ENVIRONMENTS.md +3 -4
- package/EVALUATIONS.md +10 -7
- package/README.md +10 -4
- package/dist/evals/environment-target.js +1 -1
- package/dist/evals/runner.d.ts +1 -1
- package/dist/evals/runner.js +22 -12
- package/dist/evals/scorer-publication.js +17 -1
- package/dist/evals/scorers.d.ts +10 -4
- package/dist/evals/scorers.js +11 -8
- package/dist/evals/types.d.ts +8 -1
- package/dist/setup/checkpoint.d.ts +14 -0
- package/dist/setup/checkpoint.js +186 -0
- package/dist/setup/cli.d.ts +2 -0
- package/dist/setup/cli.js +150 -0
- package/dist/setup/detect.d.ts +3 -0
- package/dist/setup/detect.js +146 -0
- package/dist/setup/machine.d.ts +109 -0
- package/dist/setup/machine.js +43 -0
- package/dist/setup/render.d.ts +16 -0
- package/dist/setup/render.js +111 -0
- package/dist/setup/runner.d.ts +101 -0
- package/dist/setup/runner.js +117 -0
- package/dist/setup/types.d.ts +145 -0
- package/dist/setup/types.js +2 -0
- package/dist/setup.d.ts +6 -0
- package/dist/setup.js +6 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +12 -1
- package/setup-events.schema.json +134 -0
package/CLI.md
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Hue setup-session CLI contract
|
|
2
|
+
|
|
3
|
+
The local setup-session CLI core shipped in TypeScript `0.3.1` and is available from npm:
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
hue setup
|
|
7
|
+
hue setup --agent
|
|
8
|
+
hue resume
|
|
9
|
+
hue status
|
|
10
|
+
hue claim # reports account attachment unavailable in this local core
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
These commands belong only to an installer setup session. They do not create or launch a Hue Run,
|
|
14
|
+
Scenario, evaluation, or worker. `claim` only means attaching the anonymous setup project to an
|
|
15
|
+
account; there is no generic `connect` command or local-agent connection in this CLI.
|
|
16
|
+
|
|
17
|
+
`setup` only inspects bounded manifest and lockfile metadata. It does not execute repository code,
|
|
18
|
+
change project files, open a browser, ask a question, create a trial, or contact Hue. `claim` also
|
|
19
|
+
makes no network request in this build and reports that account attachment is unavailable. `resume`
|
|
20
|
+
deterministically continues the same setup-session checkpoint; `status` reads it without changing
|
|
21
|
+
it.
|
|
22
|
+
|
|
23
|
+
Checkpoints are secret-free JSON files outside the project, under the operating system's user state
|
|
24
|
+
directory. On POSIX, directories use mode `0700` and files use mode `0600`; Windows uses its
|
|
25
|
+
per-user local state directory without interpreting POSIX mode bits. Writes are atomic, and a
|
|
26
|
+
configured state location inside the project is rejected. Checkpoints contain project categories
|
|
27
|
+
and hashes, never environment values, credentials, source contents, or claim URLs.
|
|
28
|
+
|
|
29
|
+
`--agent` is explicitly noninteractive JSONL. It never uses ANSI, stdin, or a browser, and each
|
|
30
|
+
invocation emits exactly one terminal `run.completed` or `run.failed` installer event. Those names
|
|
31
|
+
describe the setup-session lifecycle, not a Hue Run. Human output is an append-only inline
|
|
32
|
+
transcript. Plain and JSONL output contain no ANSI; `NO_COLOR`, `TERM=dumb`, CI, and non-TTY output
|
|
33
|
+
select plain mode automatically.
|
|
34
|
+
|
|
35
|
+
Every JSONL record carries `contractVersion: 1`. The TypeScript union is exported from
|
|
36
|
+
`@hue-run/sdk/setup`; the JSON Schema is exported as
|
|
37
|
+
`@hue-run/sdk/setup-events.schema.json`. Consumers must ignore neither unknown versions nor terminal
|
|
38
|
+
failures.
|
|
39
|
+
|
|
40
|
+
The future Fern implementation plugs into `SetupBackendAdapter`. Its three installer operations
|
|
41
|
+
create an anonymous setup trial hard-pinned to `trial_metadata_v1`, verify instrumentation-only
|
|
42
|
+
receipt evidence, and read account-claim state. Inputs carry deterministic idempotency keys and an
|
|
43
|
+
optional abort signal. Adapter results must use bounded, non-secret IDs; claim URLs may be sensitive
|
|
44
|
+
and therefore must never be checkpointed. Receipt verification does not prove that task or
|
|
45
|
+
environment content was captured and must never authorize Scenario publication. The adapter must
|
|
46
|
+
not create a Scenario, evaluation, worker, or Hue Run. No live implementation ships in this slice.
|
|
47
|
+
|
|
48
|
+
The V1 handoff is deliberately staged: setup verifies the anonymous instrumentation trace; `hue claim`
|
|
49
|
+
preserves the project and trace history; then the user performs an explicit content-approved capture
|
|
50
|
+
or rerun, with a prepared tester as the first golden path. Only that content-approved trace passes to
|
|
51
|
+
the separate review/publication flow for a Scenario. The actual URL and that handoff contract remain
|
|
52
|
+
deferred. Setup itself does not capture content, create, publish, or run a Scenario.
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
# Simulated environments
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
the exact reviewed archive rather than requesting `0.3.0` from npm:
|
|
3
|
+
The provider-aware environment APIs shipped in TypeScript `0.3.0` and are publicly available:
|
|
5
4
|
|
|
6
5
|
```bash
|
|
7
|
-
npm install
|
|
6
|
+
npm install @hue-run/sdk zod
|
|
8
7
|
```
|
|
9
8
|
|
|
10
9
|
Your agent runs in your process while a disposable simulated world runs in Hue. The world is
|
|
@@ -192,7 +191,7 @@ arbitrary-step diffs are outside this interface.
|
|
|
192
191
|
|
|
193
192
|
## Candidate context migration
|
|
194
193
|
|
|
195
|
-
This candidate-context restriction
|
|
194
|
+
This candidate-context restriction shipped in `@hue-run/sdk@0.3.0`.
|
|
196
195
|
|
|
197
196
|
`runSimulation` now supplies `context.item` as `{ id, externalKey }`. Read candidate inputs
|
|
198
197
|
from the callback's first argument. Expected outcomes, case metadata and environment-version
|
package/EVALUATIONS.md
CHANGED
|
@@ -120,7 +120,11 @@ All declared metrics must appear exactly once and satisfy pinned types, bounds a
|
|
|
120
120
|
|
|
121
121
|
### Hosted and manual scorer pins
|
|
122
122
|
|
|
123
|
-
|
|
123
|
+
Scorer deferral shipped in TypeScript `0.3.1`: the local runner executes only the three known built-in entries and bound `local_code` scorers. It leaves every other pin pending and reports its ID in `deferredScorerVersionIds`, including kinds and built-in entries introduced by a newer server. It never uploads a placeholder result that would occupy the immutable result slot, including placeholders already saved in an older SDK's checkpoint. Direct `scoreLocally()` calls reject pins that require another executor.
|
|
124
|
+
|
|
125
|
+
`world_outcome` pins run inside Hue and need no local callback or executable source digest. A supporting server owns their execution from saved world evidence. Legacy `local_code` pins still require the exact registered callback; changing the worker cannot convert those immutable pins into hosted ones.
|
|
126
|
+
|
|
127
|
+
Manual results require a human session. Hosted model-judge dispatch is an explicit separate API operation: inspect `getJudgeBudget()`, then call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. `listJudgeJobs`, `getJudgeJob` and `cancelJudgeJob` expose job progress and cancellation requests. These methods never claim that local execution has hosted provenance. Hosted job endpoints are covered by HTTP contract tests here; live hosted model execution is a separate platform acceptance phase. `listResults` and `getResult` read recorded local or hosted results.
|
|
124
128
|
|
|
125
129
|
When present, the budget's `authentication` reports credential resolution only. An
|
|
126
130
|
`available` status or `configured: true` does not prove that a provider accepted the
|
|
@@ -156,12 +160,10 @@ The runner stops scheduling more cases after an operational failure and waits fo
|
|
|
156
160
|
|
|
157
161
|
## Outbound local agent worker
|
|
158
162
|
|
|
159
|
-
`runLocalAgent`
|
|
160
|
-
claims, scoped MCP capabilities and sealed
|
|
161
|
-
access; the package version alone does not
|
|
162
|
-
|
|
163
|
-
peer. `node packages/sdk-typescript/scripts/verify-package.mjs` creates and verifies that archive;
|
|
164
|
-
do not request `0.3.0` from npm yet.
|
|
163
|
+
`runLocalAgent` shipped in `@hue-run/sdk@0.3.0` and is publicly available. Install
|
|
164
|
+
`npm install @hue-run/sdk zod`. Queue registration, claims, scoped MCP capabilities and sealed
|
|
165
|
+
evidence require a supporting Hue server and project access; the package version alone does not
|
|
166
|
+
establish hosted provider availability.
|
|
165
167
|
|
|
166
168
|
`runLocalAgent` registers one fixed application callback and polls for queued runs. Hue selects
|
|
167
169
|
the registered key/revision; it does not send executable code or shell commands. Keep the
|
|
@@ -183,6 +185,7 @@ try {
|
|
|
183
185
|
hue,
|
|
184
186
|
agent: { key: "support-agent", name: "Support agent", revision: "1" },
|
|
185
187
|
checkpointDirectory: ".hue-checkpoints/support-agent",
|
|
188
|
+
scorers: [], // Hue-executed scorers require no local callback registration.
|
|
186
189
|
target: (inputs, tools, context) => runMyAgent({ inputs, tools, config: context.config }),
|
|
187
190
|
});
|
|
188
191
|
} finally {
|
package/README.md
CHANGED
|
@@ -397,10 +397,9 @@ The optional `@hue-run/sdk/evals` entry point supports dataset/scorer registrati
|
|
|
397
397
|
|
|
398
398
|
### App-launched local workers
|
|
399
399
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
`hue-run-sdk-0.3.0.tgz` archive rather than requesting `0.3.0` from npm. It shares
|
|
400
|
+
`runLocalAgent()` shipped in TypeScript `0.3.0` and is available from npm. It connects a fixed local
|
|
401
|
+
callback to app-launched simulation work while keeping the agent and provider orchestration in
|
|
402
|
+
the developer's process. It shares
|
|
404
403
|
`runSimulation()`'s provider-aware world lifecycle, keeps scoped credentials in callback memory,
|
|
405
404
|
skips target/scorer execution for incomplete environments, and never reacquires or replays after
|
|
406
405
|
an uncertain preparation. See the
|
|
@@ -409,6 +408,13 @@ an uncertain preparation. See the
|
|
|
409
408
|
an issued facade endpoint or the official Gmail service, and do not claim universal provider
|
|
410
409
|
parity.
|
|
411
410
|
|
|
411
|
+
Scorer deferral shipped in TypeScript `0.3.1`. Only built-ins
|
|
412
|
+
and bound `local_code` callbacks run here; other pins remain pending for their authorized executor.
|
|
413
|
+
See [scorer execution](EVALUATIONS.md#hosted-and-manual-scorer-pins).
|
|
414
|
+
|
|
415
|
+
The [setup CLI](CLI.md) is a resumable local inspection core. Existing customers connect their
|
|
416
|
+
agents with `runLocalAgent()`; setup does not register workers or launch Scenarios.
|
|
417
|
+
|
|
412
418
|
## Managed targets
|
|
413
419
|
|
|
414
420
|
Start a frozen dataset run in Hue while your agent stays in your application. Expose a
|
|
@@ -50,7 +50,7 @@ async function seal(client, runId, executionId, status) {
|
|
|
50
50
|
}
|
|
51
51
|
catch (error) {
|
|
52
52
|
const recovered = await client.getRun(runId).catch(() => undefined);
|
|
53
|
-
if (recovered?.status !== status)
|
|
53
|
+
if (recovered?.status !== status && recovered?.status !== "expired")
|
|
54
54
|
throw new TargetOutcomeUncertainError(executionId, { cause: error });
|
|
55
55
|
}
|
|
56
56
|
}
|
package/dist/evals/runner.d.ts
CHANGED
|
@@ -90,7 +90,7 @@ export interface RunnerReport {
|
|
|
90
90
|
subjectIds: string[];
|
|
91
91
|
/** Result IDs uploaded by this call. */
|
|
92
92
|
resultIds: string[];
|
|
93
|
-
/**
|
|
93
|
+
/** Pins without a local implementation, left pending for their authorized executor. */
|
|
94
94
|
deferredScorerVersionIds: string[];
|
|
95
95
|
}
|
|
96
96
|
/**
|
package/dist/evals/runner.js
CHANGED
|
@@ -4,7 +4,7 @@ import { HueExportError } from "../transport.js";
|
|
|
4
4
|
import { loadEnvironmentEvidence } from "./environment-evidence.js";
|
|
5
5
|
import { CheckpointStore } from "./checkpoint.js";
|
|
6
6
|
import { json, uuid } from "./json.js";
|
|
7
|
-
import { persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
|
|
7
|
+
import { persistedScore, scoreLocally, validateScorerBindings, isLocallyExecutable, } from "./scorers.js";
|
|
8
8
|
/**
|
|
9
9
|
* Thrown when a case has a started attempt without a saved outcome. The runner never reruns the
|
|
10
10
|
* target; inspect the execution and authorize a new attempt explicitly through `startExecution`.
|
|
@@ -105,7 +105,8 @@ async function pool(items, concurrency, execute) {
|
|
|
105
105
|
async function scoresFor(versions, context, options, executionId) {
|
|
106
106
|
const scores = [];
|
|
107
107
|
let environmentUnavailable = false;
|
|
108
|
-
if (options.environmentEvidence === "required"
|
|
108
|
+
if (options.environmentEvidence === "required" &&
|
|
109
|
+
versions.some((version) => isLocallyExecutable(version.definition))) {
|
|
109
110
|
try {
|
|
110
111
|
context = {
|
|
111
112
|
...context,
|
|
@@ -117,8 +118,8 @@ async function scoresFor(versions, context, options, executionId) {
|
|
|
117
118
|
}
|
|
118
119
|
}
|
|
119
120
|
for (const version of versions) {
|
|
120
|
-
//
|
|
121
|
-
if (version.definition
|
|
121
|
+
// Every pin without a local implementation belongs to another executor.
|
|
122
|
+
if (!isLocallyExecutable(version.definition))
|
|
122
123
|
continue;
|
|
123
124
|
const score = persistedScore(environmentUnavailable && version.definition.kind === "local_code"
|
|
124
125
|
? { state: "error", error: { type: "EnvironmentEvidenceUnavailable" } }
|
|
@@ -136,8 +137,16 @@ async function scoresFor(versions, context, options, executionId) {
|
|
|
136
137
|
}
|
|
137
138
|
return scores;
|
|
138
139
|
}
|
|
139
|
-
async function uploadScores(options, runId, scores, save) {
|
|
140
|
-
|
|
140
|
+
async function uploadScores(options, runId, scores, save, versions) {
|
|
141
|
+
// A previous SDK may have checkpointed a placeholder for an unknown kind.
|
|
142
|
+
// Keep its evidence intact, but never upload or report it as a local result.
|
|
143
|
+
const local = scores.filter((score) => {
|
|
144
|
+
const pin = versions.find((version) => version.id === score.payload.scorerVersionId);
|
|
145
|
+
if (!pin)
|
|
146
|
+
throw new Error("Saved result references an unpinned scorer version");
|
|
147
|
+
return isLocallyExecutable(pin.definition);
|
|
148
|
+
});
|
|
149
|
+
for (const score of local) {
|
|
141
150
|
if (score.receipt)
|
|
142
151
|
continue;
|
|
143
152
|
if (!score.payload.evaluationItemId)
|
|
@@ -149,6 +158,7 @@ async function uploadScores(options, runId, scores, save) {
|
|
|
149
158
|
score.receipt = result.ids;
|
|
150
159
|
await save();
|
|
151
160
|
}
|
|
161
|
+
return local.flatMap((score) => score.receipt ?? []);
|
|
152
162
|
}
|
|
153
163
|
/**
|
|
154
164
|
* Runs every case of a frozen experiment through `target` on this machine, completes each
|
|
@@ -204,7 +214,7 @@ export async function runExperiment(options) {
|
|
|
204
214
|
subjectIds: [],
|
|
205
215
|
resultIds: [],
|
|
206
216
|
deferredScorerVersionIds: versions
|
|
207
|
-
.filter((version) =>
|
|
217
|
+
.filter((version) => !isLocallyExecutable(version.definition))
|
|
208
218
|
.map((version) => version.id),
|
|
209
219
|
};
|
|
210
220
|
try {
|
|
@@ -352,9 +362,9 @@ export async function runExperiment(options) {
|
|
|
352
362
|
score.payload.evaluationItemId = prepared.completion.evaluationItemId;
|
|
353
363
|
await save();
|
|
354
364
|
}
|
|
355
|
-
await uploadScores(options, experiment.evaluation.id, prepared.scores, save);
|
|
365
|
+
const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
|
|
356
366
|
report.subjectIds.push(prepared.completion.subjectId);
|
|
357
|
-
report.resultIds.push(...
|
|
367
|
+
report.resultIds.push(...results);
|
|
358
368
|
});
|
|
359
369
|
let finish = await store.read("finish");
|
|
360
370
|
if (!finish) {
|
|
@@ -395,7 +405,7 @@ export async function rescore(options) {
|
|
|
395
405
|
subjectIds: [],
|
|
396
406
|
resultIds: [],
|
|
397
407
|
deferredScorerVersionIds: run.scorerVersions
|
|
398
|
-
.filter((version) =>
|
|
408
|
+
.filter((version) => !isLocallyExecutable(version.definition))
|
|
399
409
|
.map((version) => version.id),
|
|
400
410
|
};
|
|
401
411
|
try {
|
|
@@ -419,9 +429,9 @@ export async function rescore(options) {
|
|
|
419
429
|
await store.write(file, saved);
|
|
420
430
|
}
|
|
421
431
|
const current = saved;
|
|
422
|
-
await uploadScores(options, run.id, current.scores, () => store.write(file, current));
|
|
432
|
+
const results = await uploadScores(options, run.id, current.scores, () => store.write(file, current), run.scorerVersions);
|
|
423
433
|
report.subjectIds.push(item.subjectId);
|
|
424
|
-
report.resultIds.push(...
|
|
434
|
+
report.resultIds.push(...results);
|
|
425
435
|
});
|
|
426
436
|
return report;
|
|
427
437
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import { json } from "./json.js";
|
|
2
|
+
import { json, digest } from "./json.js";
|
|
3
3
|
const metricName = z
|
|
4
4
|
.string()
|
|
5
5
|
.min(1)
|
|
@@ -21,6 +21,15 @@ const metric = z.discriminatedUnion("type", [
|
|
|
21
21
|
}),
|
|
22
22
|
]);
|
|
23
23
|
const metrics = z.array(metric).min(1);
|
|
24
|
+
const worldOutcomeMetrics = [
|
|
25
|
+
"completed_run",
|
|
26
|
+
"saved_draft",
|
|
27
|
+
"correct_destination",
|
|
28
|
+
"content",
|
|
29
|
+
"unrelated_preserved",
|
|
30
|
+
"process_constraints",
|
|
31
|
+
"task_success",
|
|
32
|
+
].map((name) => ({ name, type: "boolean" }));
|
|
24
33
|
const jsonValue = z.custom((value) => {
|
|
25
34
|
try {
|
|
26
35
|
json(value);
|
|
@@ -73,6 +82,13 @@ const sdkScorerPublication = z.union([
|
|
|
73
82
|
sourceDigest: z.string(),
|
|
74
83
|
metrics,
|
|
75
84
|
}),
|
|
85
|
+
z.strictObject({
|
|
86
|
+
kind: z.literal("world_outcome"),
|
|
87
|
+
entry: z.literal("hue.conversion_outcome.v1"),
|
|
88
|
+
metrics: metrics
|
|
89
|
+
.default(worldOutcomeMetrics)
|
|
90
|
+
.refine((value) => digest(value) === digest(worldOutcomeMetrics), "World outcome metrics are fixed by the pinned entry"),
|
|
91
|
+
}),
|
|
76
92
|
z.strictObject({ kind: z.literal("manual"), metrics }),
|
|
77
93
|
z.strictObject({ kind: z.literal("llm_judge"), config: judgeConfig, metrics }),
|
|
78
94
|
]);
|
package/dist/evals/scorers.d.ts
CHANGED
|
@@ -17,12 +17,18 @@ export declare function defineLocalScorer(options: {
|
|
|
17
17
|
}): LocalScorer;
|
|
18
18
|
/** Check local callbacks before target invocation; never execute downloaded source code. */
|
|
19
19
|
export declare function validateScorerBindings(versions: ScorerVersion[], scorers?: LocalScorer[]): void;
|
|
20
|
+
/** Only implementations this SDK owns may produce local results. Unknown pins are deferred. */
|
|
21
|
+
export declare function isLocallyExecutable(definition: {
|
|
22
|
+
kind: string;
|
|
23
|
+
entry?: string;
|
|
24
|
+
}): definition is Extract<ScorerDefinition, {
|
|
25
|
+
kind: "builtin" | "local_code";
|
|
26
|
+
}>;
|
|
20
27
|
/**
|
|
21
|
-
* Scores one subject
|
|
22
|
-
*
|
|
23
|
-
* returned as sanitized error scores.
|
|
28
|
+
* Scores one subject locally using a known built-in or a bound `local_code` callback.
|
|
29
|
+
* Scorer failures are returned as sanitized error scores.
|
|
24
30
|
*
|
|
25
|
-
* @throws TypeError for
|
|
31
|
+
* @throws TypeError for pins without a local implementation; their authorized executor owns scoring.
|
|
26
32
|
*/
|
|
27
33
|
export declare function scoreLocally(version: ScorerVersion, context: ScoreContext, options?: {
|
|
28
34
|
scorers?: LocalScorer[];
|
package/dist/evals/scorers.js
CHANGED
|
@@ -99,17 +99,22 @@ export function validateScorerBindings(versions, scorers = []) {
|
|
|
99
99
|
throw new Error("A pinned local scorer has no matching language/source/entrypoint/metric binding");
|
|
100
100
|
}
|
|
101
101
|
}
|
|
102
|
+
/** Only implementations this SDK owns may produce local results. Unknown pins are deferred. */
|
|
103
|
+
export function isLocallyExecutable(definition) {
|
|
104
|
+
return (definition.kind === "local_code" ||
|
|
105
|
+
(definition.kind === "builtin" &&
|
|
106
|
+
["hue.exact_match.v1", "hue.includes.v1", "hue.json_schema.v1"].includes(definition.entry ?? "")));
|
|
107
|
+
}
|
|
102
108
|
/**
|
|
103
|
-
* Scores one subject
|
|
104
|
-
*
|
|
105
|
-
* returned as sanitized error scores.
|
|
109
|
+
* Scores one subject locally using a known built-in or a bound `local_code` callback.
|
|
110
|
+
* Scorer failures are returned as sanitized error scores.
|
|
106
111
|
*
|
|
107
|
-
* @throws TypeError for
|
|
112
|
+
* @throws TypeError for pins without a local implementation; their authorized executor owns scoring.
|
|
108
113
|
*/
|
|
109
114
|
export async function scoreLocally(version, context, options = {}) {
|
|
110
115
|
const definition = version.definition;
|
|
111
|
-
if (definition
|
|
112
|
-
throw new TypeError("
|
|
116
|
+
if (!isLocallyExecutable(definition))
|
|
117
|
+
throw new TypeError("This scorer must be deferred to its authorized executor; do not submit a local result");
|
|
113
118
|
const timeout = options.schemaTimeoutMillis ?? 2000;
|
|
114
119
|
if (!Number.isInteger(timeout) || timeout < 100 || timeout > 60_000)
|
|
115
120
|
throw new RangeError("schemaTimeoutMillis must be 100–60000");
|
|
@@ -133,8 +138,6 @@ export async function scoreLocally(version, context, options = {}) {
|
|
|
133
138
|
if (environment !== undefined)
|
|
134
139
|
validateEnvironmentEvidence(environment);
|
|
135
140
|
const owned = structuredClone(context);
|
|
136
|
-
if (definition.kind === "manual")
|
|
137
|
-
return skip("Manual scoring requires a human session");
|
|
138
141
|
if (definition.kind === "local_code") {
|
|
139
142
|
const binding = options.scorers?.find((local) => digest(local.definition) === digest(definition));
|
|
140
143
|
if (!binding)
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -119,7 +119,7 @@ export type MetricDefinition = {
|
|
|
119
119
|
/** Allowed values. */
|
|
120
120
|
categories: string[];
|
|
121
121
|
};
|
|
122
|
-
/** A pinned scorer definition
|
|
122
|
+
/** A pinned scorer definition executed locally, by a person, or by Hue. */
|
|
123
123
|
export type ScorerDefinition = {
|
|
124
124
|
/** Hue built-in scorer. */
|
|
125
125
|
kind: "builtin";
|
|
@@ -158,6 +158,13 @@ export type ScorerDefinition = {
|
|
|
158
158
|
sourceDigest: string;
|
|
159
159
|
/** Metrics the callback reports. */
|
|
160
160
|
metrics: MetricDefinition[];
|
|
161
|
+
} | {
|
|
162
|
+
/** Scored inside Hue using immutable world evidence; the local runner defers it. */
|
|
163
|
+
kind: "world_outcome";
|
|
164
|
+
/** Pinned Hue-executed outcome evaluator. */
|
|
165
|
+
entry: "hue.conversion_outcome.v1";
|
|
166
|
+
/** The seven fixed boolean metrics defined by the entry. */
|
|
167
|
+
metrics: MetricDefinition[];
|
|
161
168
|
} | {
|
|
162
169
|
/** Scored by a person in Hue; the local runner defers it. */
|
|
163
170
|
kind: "manual";
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SetupCheckpointAdapter } from "./runner.js";
|
|
2
|
+
import type { SetupMachineState } from "./machine.js";
|
|
3
|
+
export declare function defaultSetupStateDirectory(env?: NodeJS.ProcessEnv): string;
|
|
4
|
+
/** Stable installer-session identifier; it is not a Hue Run and reveals no path or file contents. */
|
|
5
|
+
export declare function setupRunId(projectRoot: string): string;
|
|
6
|
+
export declare class FileSetupCheckpointAdapter implements SetupCheckpointAdapter {
|
|
7
|
+
readonly directory: string;
|
|
8
|
+
private readonly runtimePlatform;
|
|
9
|
+
constructor(directory?: string, runtimePlatform?: NodeJS.Platform);
|
|
10
|
+
private get enforcesPosixPermissions();
|
|
11
|
+
private pathFor;
|
|
12
|
+
load(runId: string, projectRoot: string): Promise<SetupMachineState | undefined>;
|
|
13
|
+
save(state: SetupMachineState): Promise<void>;
|
|
14
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { constants } from "node:fs";
|
|
3
|
+
import { chmod, lstat, mkdir, open, realpath, rename } from "node:fs/promises";
|
|
4
|
+
import { homedir, platform } from "node:os";
|
|
5
|
+
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
6
|
+
const MAX_CHECKPOINT_BYTES = 256 * 1024;
|
|
7
|
+
const STEPS = ["detect-project", "configure-telemetry", "verify-receipt", "claim-project"];
|
|
8
|
+
function isInside(parent, child) {
|
|
9
|
+
const path = relative(parent, child);
|
|
10
|
+
return path === "" || (!path.startsWith("..") && !isAbsolute(path));
|
|
11
|
+
}
|
|
12
|
+
export function defaultSetupStateDirectory(env = process.env) {
|
|
13
|
+
if (platform() === "darwin")
|
|
14
|
+
return join(homedir(), "Library", "Application Support", "Hue", "setup");
|
|
15
|
+
if (platform() === "win32")
|
|
16
|
+
return join(env.LOCALAPPDATA ?? join(homedir(), "AppData", "Local"), "Hue", "setup");
|
|
17
|
+
return join(env.XDG_STATE_HOME ?? join(homedir(), ".local", "state"), "hue", "setup");
|
|
18
|
+
}
|
|
19
|
+
/** Stable installer-session identifier; it is not a Hue Run and reveals no path or file contents. */
|
|
20
|
+
export function setupRunId(projectRoot) {
|
|
21
|
+
return `setup_${createHash("sha256")
|
|
22
|
+
.update(`hue-setup-v1\0${resolve(projectRoot)}`)
|
|
23
|
+
.digest("hex")
|
|
24
|
+
.slice(0, 24)}`;
|
|
25
|
+
}
|
|
26
|
+
function digest(value) {
|
|
27
|
+
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
28
|
+
}
|
|
29
|
+
function hasExactKeys(value, keys) {
|
|
30
|
+
return Object.keys(value).sort().join("\0") === [...keys].sort().join("\0");
|
|
31
|
+
}
|
|
32
|
+
function validDetection(value, projectRoot) {
|
|
33
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
34
|
+
return false;
|
|
35
|
+
const item = value;
|
|
36
|
+
if (!hasExactKeys(item, [
|
|
37
|
+
"root",
|
|
38
|
+
"fingerprint",
|
|
39
|
+
"languages",
|
|
40
|
+
"packageManagers",
|
|
41
|
+
"frameworks",
|
|
42
|
+
"hue",
|
|
43
|
+
"openTelemetry",
|
|
44
|
+
]))
|
|
45
|
+
return false;
|
|
46
|
+
const allowed = (items, values, maximum) => Array.isArray(items) &&
|
|
47
|
+
items.length <= maximum &&
|
|
48
|
+
new Set(items).size === items.length &&
|
|
49
|
+
items.every((entry) => typeof entry === "string" && values.includes(entry));
|
|
50
|
+
return (item.root === projectRoot &&
|
|
51
|
+
typeof item.fingerprint === "string" &&
|
|
52
|
+
/^[a-f0-9]{64}$/u.test(item.fingerprint) &&
|
|
53
|
+
allowed(item.languages, ["typescript", "python"], 2) &&
|
|
54
|
+
allowed(item.packageManagers, ["bun", "npm", "pnpm", "yarn", "uv", "poetry", "pip"], 7) &&
|
|
55
|
+
allowed(item.frameworks, ["nextjs", "nestjs", "express", "fastapi", "django", "flask", "vercel-ai-sdk"], 7) &&
|
|
56
|
+
["absent", "typescript", "python", "multiple"].includes(item.hue) &&
|
|
57
|
+
["absent", "typescript", "python", "multiple"].includes(item.openTelemetry));
|
|
58
|
+
}
|
|
59
|
+
function validState(value, runId, projectRoot) {
|
|
60
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
61
|
+
return false;
|
|
62
|
+
const state = value;
|
|
63
|
+
if (state.format !== 1 || state.runId !== runId || state.projectRoot !== projectRoot)
|
|
64
|
+
return false;
|
|
65
|
+
if (state.phase === "created" || state.phase === "detecting")
|
|
66
|
+
return hasExactKeys(state, ["format", "phase", "runId", "projectRoot"]);
|
|
67
|
+
if (state.phase !== "local-ready" ||
|
|
68
|
+
!hasExactKeys(state, ["format", "phase", "runId", "projectRoot", "project", "plan"]) ||
|
|
69
|
+
!validDetection(state.project, projectRoot))
|
|
70
|
+
return false;
|
|
71
|
+
const plan = state.plan;
|
|
72
|
+
return (!!plan &&
|
|
73
|
+
typeof plan === "object" &&
|
|
74
|
+
!Array.isArray(plan) &&
|
|
75
|
+
hasExactKeys(plan, ["steps", "mutatesProject", "backendRequired"]) &&
|
|
76
|
+
JSON.stringify(plan.steps) === JSON.stringify(STEPS) &&
|
|
77
|
+
plan.mutatesProject === false &&
|
|
78
|
+
plan.backendRequired === true);
|
|
79
|
+
}
|
|
80
|
+
export class FileSetupCheckpointAdapter {
|
|
81
|
+
directory;
|
|
82
|
+
runtimePlatform;
|
|
83
|
+
constructor(directory = defaultSetupStateDirectory(), runtimePlatform = platform()) {
|
|
84
|
+
this.directory = directory;
|
|
85
|
+
this.runtimePlatform = runtimePlatform;
|
|
86
|
+
}
|
|
87
|
+
get enforcesPosixPermissions() {
|
|
88
|
+
return this.runtimePlatform !== "win32";
|
|
89
|
+
}
|
|
90
|
+
async pathFor(runId, projectRoot) {
|
|
91
|
+
if (!/^setup_[a-f0-9]{24}$/u.test(runId))
|
|
92
|
+
throw new Error("Invalid setup run identifier");
|
|
93
|
+
const root = resolve(this.directory);
|
|
94
|
+
const project = await realpath(projectRoot);
|
|
95
|
+
if (isInside(project, root))
|
|
96
|
+
throw new Error("Setup checkpoints must be outside the project repository");
|
|
97
|
+
let info;
|
|
98
|
+
try {
|
|
99
|
+
info = await lstat(root);
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (error.code !== "ENOENT")
|
|
103
|
+
throw error;
|
|
104
|
+
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
105
|
+
info = await lstat(root);
|
|
106
|
+
}
|
|
107
|
+
if (!info.isDirectory() || info.isSymbolicLink())
|
|
108
|
+
throw new Error("Setup checkpoint directory must be private (mode 0700, no symlink)");
|
|
109
|
+
if (this.enforcesPosixPermissions) {
|
|
110
|
+
await chmod(root, 0o700);
|
|
111
|
+
info = await lstat(root);
|
|
112
|
+
if ((info.mode & 0o077) !== 0)
|
|
113
|
+
throw new Error("Setup checkpoint directory must be private (mode 0700, no symlink)");
|
|
114
|
+
}
|
|
115
|
+
return join(root, `${runId}.json`);
|
|
116
|
+
}
|
|
117
|
+
async load(runId, projectRoot) {
|
|
118
|
+
const path = await this.pathFor(runId, projectRoot);
|
|
119
|
+
let handle;
|
|
120
|
+
try {
|
|
121
|
+
if (this.runtimePlatform === "win32") {
|
|
122
|
+
const entry = await lstat(path);
|
|
123
|
+
if (entry.isSymbolicLink())
|
|
124
|
+
throw new Error("Unsafe setup checkpoint symlink");
|
|
125
|
+
}
|
|
126
|
+
const noFollow = this.runtimePlatform === "win32" ? 0 : constants.O_NOFOLLOW;
|
|
127
|
+
handle = await open(path, constants.O_RDONLY | noFollow);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
if (error.code === "ENOENT")
|
|
131
|
+
return undefined;
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
const info = await handle.stat();
|
|
136
|
+
if (!info.isFile() ||
|
|
137
|
+
info.size > MAX_CHECKPOINT_BYTES ||
|
|
138
|
+
(this.enforcesPosixPermissions && (info.mode & 0o077) !== 0))
|
|
139
|
+
throw new Error("Unsafe or oversized setup checkpoint");
|
|
140
|
+
const envelope = JSON.parse(await handle.readFile("utf8"));
|
|
141
|
+
if (!envelope ||
|
|
142
|
+
typeof envelope !== "object" ||
|
|
143
|
+
Array.isArray(envelope) ||
|
|
144
|
+
!hasExactKeys(envelope, ["state", "digest"]))
|
|
145
|
+
throw new Error("Invalid setup checkpoint envelope");
|
|
146
|
+
const { state, digest: expected } = envelope;
|
|
147
|
+
if (typeof expected !== "string" || expected !== digest(state))
|
|
148
|
+
throw new Error("Setup checkpoint integrity check failed");
|
|
149
|
+
if (!validState(state, runId, await realpath(projectRoot)))
|
|
150
|
+
throw new Error("Setup checkpoint identity or shape does not match this project");
|
|
151
|
+
return state;
|
|
152
|
+
}
|
|
153
|
+
finally {
|
|
154
|
+
await handle.close();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
async save(state) {
|
|
158
|
+
const path = await this.pathFor(state.runId, state.projectRoot);
|
|
159
|
+
const encoded = `${JSON.stringify({ state, digest: digest(state) })}\n`;
|
|
160
|
+
if (Buffer.byteLength(encoded) > MAX_CHECKPOINT_BYTES)
|
|
161
|
+
throw new Error("Setup checkpoint exceeds 256 KiB");
|
|
162
|
+
const temporary = join(dirname(path), `.${state.runId}.${randomUUID()}.tmp`);
|
|
163
|
+
const handle = await open(temporary, "wx", 0o600);
|
|
164
|
+
try {
|
|
165
|
+
await handle.writeFile(encoded);
|
|
166
|
+
await handle.sync();
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
await handle.close();
|
|
170
|
+
}
|
|
171
|
+
await rename(temporary, path);
|
|
172
|
+
// Windows cannot open a directory as a file handle for fsync. The atomic rename and
|
|
173
|
+
// per-user state directory still provide resumability there; POSIX additionally fsyncs
|
|
174
|
+
// the containing directory so the rename survives a sudden interruption.
|
|
175
|
+
if (this.runtimePlatform !== "win32") {
|
|
176
|
+
await chmod(path, 0o600);
|
|
177
|
+
const directory = await open(dirname(path), "r");
|
|
178
|
+
try {
|
|
179
|
+
await directory.sync();
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
await directory.close();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|