@skill-harness/core 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/adapters/types.d.ts +0 -12
- package/dist/capture-trace-types.d.ts +0 -96
- package/dist/capture-trace-types.js +0 -4
- package/dist/command-cost.d.ts +1 -1
- package/dist/command-cost.js +0 -5
- package/dist/execution-trace.js +1 -1
- package/dist/index.d.ts +3 -23
- package/dist/index.js +3 -23
- package/dist/instruction-coverage.d.ts +0 -7
- package/dist/instruction-coverage.js +5 -7
- package/dist/journal.d.ts +0 -8
- package/dist/metrics.d.ts +18 -0
- package/dist/metrics.js +28 -0
- package/dist/redaction.d.ts +6 -0
- package/dist/redaction.js +55 -0
- package/dist/regate.js +6 -71
- package/dist/regrade.d.ts +1 -3
- package/dist/regrade.js +1 -21
- package/dist/report.d.ts +1 -15
- package/dist/report.js +1 -13
- package/dist/reps.d.ts +1 -2
- package/dist/reps.js +0 -2
- package/dist/results.d.ts +0 -9
- package/dist/results.js +2 -19
- package/dist/run.js +11 -72
- package/dist/seeded.d.ts +1 -7
- package/dist/seeded.js +5 -11
- package/dist/sources.js +4 -9
- package/dist/spec.d.ts +0 -13
- package/dist/spec.js +6 -48
- package/dist/trajectory-events.d.ts +58 -0
- package/dist/trajectory-events.js +119 -0
- package/package.json +1 -1
- package/schemas/results-v2.schema.json +376 -75
- package/schemas/specification-v1.schema.json +128 -174
- package/dist/adjudication.d.ts +0 -219
- package/dist/adjudication.js +0 -422
- package/dist/adoption.d.ts +0 -128
- package/dist/adoption.js +0 -102
- package/dist/affected.d.ts +0 -88
- package/dist/affected.js +0 -222
- package/dist/capture.d.ts +0 -193
- package/dist/capture.js +0 -344
- package/dist/comparison.d.ts +0 -97
- package/dist/comparison.js +0 -189
- package/dist/factory-calibration.d.ts +0 -63
- package/dist/factory-calibration.js +0 -115
- package/dist/intervention-results.d.ts +0 -6
- package/dist/intervention-results.js +0 -52
- package/dist/intervention.d.ts +0 -123
- package/dist/intervention.js +0 -218
- package/dist/investigation.d.ts +0 -94
- package/dist/investigation.js +0 -148
- package/dist/judge-agreement.d.ts +0 -19
- package/dist/judge-agreement.js +0 -31
- package/dist/qualification-capture.d.ts +0 -44
- package/dist/qualification-capture.js +0 -163
- package/dist/qualification-config.d.ts +0 -233
- package/dist/qualification-config.js +0 -694
- package/dist/qualification-lock.d.ts +0 -15
- package/dist/qualification-lock.js +0 -87
- package/dist/qualification-oauth-directory.d.ts +0 -79
- package/dist/qualification-oauth-directory.js +0 -444
- package/dist/qualification-panel-store.d.ts +0 -14
- package/dist/qualification-panel-store.js +0 -188
- package/dist/qualification-panels.d.ts +0 -99
- package/dist/qualification-panels.js +0 -91
- package/dist/qualification-process.d.ts +0 -13
- package/dist/qualification-process.js +0 -190
- package/dist/qualification-runner.d.ts +0 -203
- package/dist/qualification-runner.js +0 -2037
- package/dist/qualification-store.d.ts +0 -187
- package/dist/qualification-store.js +0 -833
- package/dist/qualification-validation.d.ts +0 -12
- package/dist/qualification-validation.js +0 -8
- package/dist/screen.d.ts +0 -31
- package/dist/screen.js +0 -105
- package/dist/trajectory-gates.d.ts +0 -137
- package/dist/trajectory-gates.js +0 -576
- package/dist/trusted-host-supervision.d.ts +0 -25
- package/dist/trusted-host-supervision.js +0 -94
- package/dist/work-capture-schema.d.ts +0 -38
- package/dist/work-capture-schema.js +0 -33
- package/dist/work-capture.d.ts +0 -43
- package/dist/work-capture.js +0 -81
- package/dist/work-signals.d.ts +0 -55
- package/dist/work-signals.js +0 -71
- package/schemas/qualification-cell-result-v1.schema.json +0 -32
- package/schemas/qualification-config-v1.schema.json +0 -244
- package/schemas/qualification-invocation-request-v1.schema.json +0 -59
- package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
- package/schemas/trajectory-event-v1.schema.json +0 -80
package/README.md
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# @skill-harness/core
|
|
2
2
|
|
|
3
3
|
The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
|
|
4
|
-
spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results
|
|
5
|
-
and the separately versioned durable qualification-runner-v1 primitives.
|
|
4
|
+
spec parsing, discovery, run orchestration, LLM-judge grading, scoring, and results.
|
|
6
5
|
|
|
7
6
|
Internal API — use the `skill-harness` CLI. See the repo for docs.
|
|
8
7
|
|
package/dist/adapters/types.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import type { ExecutionTraceV1 } from "../capture-trace-types.js";
|
|
2
|
-
import type { TrajectoryEventSource } from "../spec.js";
|
|
3
|
-
import type { TrajectoryEventV1 } from "../trajectory-gates.js";
|
|
4
2
|
export type RunMode = "red" | "green" | "force";
|
|
5
3
|
/** A provider+model pair, e.g. { provider: "fireworks", model: "accounts/.../deepseek-v4-pro" }. */
|
|
6
4
|
export interface ModelRef {
|
|
@@ -57,16 +55,12 @@ export interface RunReq {
|
|
|
57
55
|
* extension the developer happens to have installed cannot join the test.
|
|
58
56
|
*/
|
|
59
57
|
extensions?: string[];
|
|
60
|
-
/** Workspace-local native ledgers to normalize after the subject finishes. */
|
|
61
|
-
eventSources?: TrajectoryEventSource[];
|
|
62
58
|
/**
|
|
63
59
|
* Extra env for the subject process, from the arm. Merged over `process.env` by
|
|
64
60
|
* the adapter — an arm must be able to add `PI_GRANTS_*` without the harness
|
|
65
61
|
* inheriting a caller's copy of them.
|
|
66
62
|
*/
|
|
67
63
|
armEnv?: Record<string, string>;
|
|
68
|
-
/** Adapter-produced observation callback. Callers receive evidence; they never supply delivery status. */
|
|
69
|
-
onPromptObservation?: (observation: PromptProvenance) => void;
|
|
70
64
|
}
|
|
71
65
|
/** A judge request: single prompt, no skills, no session. */
|
|
72
66
|
export interface JudgeReq {
|
|
@@ -79,10 +73,6 @@ export interface StructuredRun {
|
|
|
79
73
|
transcript: string;
|
|
80
74
|
/** One trace per turn — each `pi` invocation emits an independent event stream. */
|
|
81
75
|
traces: ExecutionTraceV1[];
|
|
82
|
-
/** Adapter-neutral workflow/tool events. Additive; absent on older adapters. */
|
|
83
|
-
events?: TrajectoryEventV1[];
|
|
84
|
-
/** Native sources that were required but missing/malformed. Never treated as an empty success. */
|
|
85
|
-
eventErrors?: string[];
|
|
86
76
|
/**
|
|
87
77
|
* Set when pi failed provider-side (auth, transport) rather than the model
|
|
88
78
|
* answering badly. `run.ts` turns this into ERROR — never a model verdict.
|
|
@@ -91,8 +81,6 @@ export interface StructuredRun {
|
|
|
91
81
|
}
|
|
92
82
|
export interface HarnessAdapter {
|
|
93
83
|
name: string;
|
|
94
|
-
/** True only when the adapter computes final provider-payload observations. */
|
|
95
|
-
observesPrompts?: boolean;
|
|
96
84
|
available(): Promise<boolean>;
|
|
97
85
|
run(req: RunReq): Promise<string>;
|
|
98
86
|
/**
|
|
@@ -151,99 +151,3 @@ export interface ExecutionTraceV1 {
|
|
|
151
151
|
/** SHA-256 over the deterministic serialization of this trace, minus this field. */
|
|
152
152
|
trace_sha256?: string;
|
|
153
153
|
}
|
|
154
|
-
export declare const CAPTURE_SCHEMA_VERSION = 1;
|
|
155
|
-
/** Why a conversation was captured. */
|
|
156
|
-
export type CaptureClassification = "failure" | "good_example";
|
|
157
|
-
/** Where a capture is in the human review pipeline. */
|
|
158
|
-
export type CapturePromotionStatus = "pending" | "promoted" | "rejected";
|
|
159
|
-
/** What the human confirmed as responsible for the behavior. */
|
|
160
|
-
export interface CaptureTarget {
|
|
161
|
-
kind: "skill" | "subagent";
|
|
162
|
-
/** Path relative to the skills root or repo root — never absolute. */
|
|
163
|
-
path: string;
|
|
164
|
-
/** SHA-256 of the target's content at capture time, so drift is detectable. */
|
|
165
|
-
content_sha256: string;
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Provenance for a capture.
|
|
169
|
-
*
|
|
170
|
-
* Hashed, not absolute: an absolute session path identifies a machine and a
|
|
171
|
-
* user, and the capture file is meant to be committed. The hash is enough to
|
|
172
|
-
* recognize the same session again locally, which is all provenance needs to do.
|
|
173
|
-
*/
|
|
174
|
-
export interface CaptureProvenance {
|
|
175
|
-
session_sha256: string;
|
|
176
|
-
/** Indices of the selected turn range within the active branch, inclusive. */
|
|
177
|
-
turn_range: {
|
|
178
|
-
start: number;
|
|
179
|
-
end: number;
|
|
180
|
-
};
|
|
181
|
-
subject?: ModelRef;
|
|
182
|
-
git_commit?: string;
|
|
183
|
-
/** True when the working tree was dirty at capture time. */
|
|
184
|
-
git_dirty?: boolean;
|
|
185
|
-
}
|
|
186
|
-
/**
|
|
187
|
-
* A reviewed conversation awaiting promotion into a scenario.
|
|
188
|
-
*
|
|
189
|
-
* Lives under `<skill>/tests/captures/`, NOT in `specification.yaml`. A pending
|
|
190
|
-
* capture is not a test: putting it in the spec would drag it into ship-bar
|
|
191
|
-
* totals, staleness, lift, and stability, and the alternative — a `draft: true`
|
|
192
|
-
* flag every runner, scorer and linter has to honor — is a state that only has
|
|
193
|
-
* to be forgotten once to either corrupt a grade or silently drop a real
|
|
194
|
-
* scenario from a release run.
|
|
195
|
-
*/
|
|
196
|
-
export interface CaptureCaseV1 {
|
|
197
|
-
capture_schema: typeof CAPTURE_SCHEMA_VERSION;
|
|
198
|
-
id: string;
|
|
199
|
-
created: string;
|
|
200
|
-
classification: CaptureClassification;
|
|
201
|
-
/** Sanitized user turns only. Assistant prose is evidence, never an oracle. */
|
|
202
|
-
turns: string[];
|
|
203
|
-
/** Human-written, in their own words. Required — a capture with no expectation is not reviewable. */
|
|
204
|
-
expected_behavior: string;
|
|
205
|
-
/** Editable draft checklist. Offline-derived by default; LLM drafting is opt-in and costs tokens. */
|
|
206
|
-
checklist: string[];
|
|
207
|
-
target: CaptureTarget;
|
|
208
|
-
provenance: CaptureProvenance;
|
|
209
|
-
status: CapturePromotionStatus;
|
|
210
|
-
/** Set once promoted: the scenario id appended to `specification.yaml`. */
|
|
211
|
-
scenario_id?: string;
|
|
212
|
-
/**
|
|
213
|
-
* `covers` refs for this pending case, so `coverage` can park it against the
|
|
214
|
-
* instructions it is about before anyone promotes it.
|
|
215
|
-
*
|
|
216
|
-
* Derived from `target` — the author has already been made to choose which
|
|
217
|
-
* instructions are responsible, and asking again in different words would be
|
|
218
|
-
* asking the same question twice. File granularity, not `#section`: the target
|
|
219
|
-
* choice attributes a file, and inventing a section would be a guess the
|
|
220
|
-
* session cannot support.
|
|
221
|
-
*
|
|
222
|
-
* Written relative to the SPEC dir (`<skill>/tests`), matching how a scenario's
|
|
223
|
-
* own `covers` resolve. `coverage` read this field from the start; nothing ever
|
|
224
|
-
* wrote it, so every pending case parked against nothing and the feature was
|
|
225
|
-
* inert.
|
|
226
|
-
*/
|
|
227
|
-
covers?: string[];
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Local-only evidence sidecar for a capture.
|
|
231
|
-
*
|
|
232
|
-
* Written to `<skill>/tests/captures/.local/` and git-ignored by default. It may
|
|
233
|
-
* hold a sanitized assistant excerpt and tool-call summaries to help a human
|
|
234
|
-
* review the capture, and must NEVER hold hidden thinking, complete tool-result
|
|
235
|
-
* bodies, or the effective system prompt.
|
|
236
|
-
*
|
|
237
|
-
* pi's stream carries thinking in `message_end`, `turn_end` AND `agent_end`, so
|
|
238
|
-
* dropping it is an explicit filter at every one of those, not a side effect of
|
|
239
|
-
* reading the convenient field.
|
|
240
|
-
*/
|
|
241
|
-
export interface CaptureEvidenceV1 {
|
|
242
|
-
capture_id: string;
|
|
243
|
-
/** Sanitized, truncated excerpt of the assistant text of every selected turn, joined. */
|
|
244
|
-
assistant_excerpt: string;
|
|
245
|
-
/** Tool name, error state and redacted arguments — no bodies. */
|
|
246
|
-
tool_calls: Array<Pick<TraceToolCall, "name" | "isError"> & {
|
|
247
|
-
args: SanitizedArgs;
|
|
248
|
-
}>;
|
|
249
|
-
}
|
|
@@ -16,8 +16,4 @@
|
|
|
16
16
|
// Execution trace
|
|
17
17
|
// ---------------------------------------------------------------------------
|
|
18
18
|
export const EXECUTION_TRACE_VERSION = 2;
|
|
19
|
-
// ---------------------------------------------------------------------------
|
|
20
|
-
// Capture case
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
export const CAPTURE_SCHEMA_VERSION = 1;
|
|
23
19
|
//# sourceMappingURL=capture-trace-types.js.map
|
package/dist/command-cost.d.ts
CHANGED
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
* Keep this list narrow: commands that can conditionally spend (notably `regate`)
|
|
5
5
|
* do not belong here even when their common path only replays saved artifacts.
|
|
6
6
|
*/
|
|
7
|
-
export declare const FREE_OFFLINE_COMMANDS: readonly ["
|
|
7
|
+
export declare const FREE_OFFLINE_COMMANDS: readonly ["coverage", "init", "lint", "list", "rescore", "restamp", "stability"];
|
|
8
8
|
export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
|
|
9
9
|
export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;
|
package/dist/command-cost.js
CHANGED
|
@@ -5,17 +5,12 @@
|
|
|
5
5
|
* do not belong here even when their common path only replays saved artifacts.
|
|
6
6
|
*/
|
|
7
7
|
export const FREE_OFFLINE_COMMANDS = [
|
|
8
|
-
"affected",
|
|
9
|
-
"archive",
|
|
10
8
|
"coverage",
|
|
11
9
|
"init",
|
|
12
|
-
"judge-agreement",
|
|
13
|
-
"learning",
|
|
14
10
|
"lint",
|
|
15
11
|
"list",
|
|
16
12
|
"rescore",
|
|
17
13
|
"restamp",
|
|
18
|
-
"screen",
|
|
19
14
|
"stability",
|
|
20
15
|
];
|
|
21
16
|
export function isFreeOfflineCommand(command) {
|
package/dist/execution-trace.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { normalizeSubagentCall } from "./trace-gates.js";
|
|
3
3
|
import { EXECUTION_TRACE_VERSION } from "./capture-trace-types.js";
|
|
4
|
-
import { redactArgs, redactText } from "./
|
|
4
|
+
import { redactArgs, redactText } from "./redaction.js";
|
|
5
5
|
/**
|
|
6
6
|
* Parse pi's `--mode json` event stream into an `ExecutionTraceV1`.
|
|
7
7
|
*
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
export * from "./trusted-host-supervision.js";
|
|
2
1
|
export * from "./spec.js";
|
|
3
2
|
export * from "./discover.js";
|
|
4
3
|
export * from "./run.js";
|
|
@@ -13,6 +12,7 @@ export * from "./rescore.js";
|
|
|
13
12
|
export * from "./workspace.js";
|
|
14
13
|
export * from "./seeded.js";
|
|
15
14
|
export * from "./report.js";
|
|
15
|
+
export * from "./metrics.js";
|
|
16
16
|
export * from "./trends.js";
|
|
17
17
|
export * from "./lint.js";
|
|
18
18
|
export * from "./restamp.js";
|
|
@@ -26,39 +26,19 @@ export * from "./version.js";
|
|
|
26
26
|
export * from "./defaults.js";
|
|
27
27
|
export * from "./command-cost.js";
|
|
28
28
|
export * from "./judge-policy.js";
|
|
29
|
-
export * from "./judge-agreement.js";
|
|
30
29
|
export * from "./regate.js";
|
|
31
|
-
export * from "./screen.js";
|
|
32
30
|
export * from "./prompt-normalization.js";
|
|
33
31
|
export * from "./downgrade.js";
|
|
34
32
|
export * from "./canary.js";
|
|
35
33
|
export * from "./stability.js";
|
|
36
34
|
export * from "./capture-trace-types.js";
|
|
37
|
-
export * from "./work-capture.js";
|
|
38
|
-
export * from "./work-signals.js";
|
|
39
|
-
export * from "./work-capture-schema.js";
|
|
40
|
-
export * from "./factory-calibration.js";
|
|
41
|
-
export * from "./adoption.js";
|
|
42
|
-
export * from "./investigation.js";
|
|
43
|
-
export * from "./intervention.js";
|
|
44
|
-
export * from "./intervention-results.js";
|
|
45
35
|
export * from "./spec-write.js";
|
|
46
|
-
export * from "./
|
|
36
|
+
export * from "./redaction.js";
|
|
47
37
|
export * from "./execution-trace.js";
|
|
48
38
|
export * from "./trace-gates.js";
|
|
49
|
-
export * from "./trajectory-
|
|
39
|
+
export * from "./trajectory-events.js";
|
|
50
40
|
export * from "./instruction-coverage.js";
|
|
51
|
-
export * from "./affected.js";
|
|
52
|
-
export * from "./adjudication.js";
|
|
53
41
|
export * from "./vote-panel.js";
|
|
54
|
-
export * from "./qualification-panels.js";
|
|
55
|
-
export * from "./qualification-panel-store.js";
|
|
56
|
-
export * from "./qualification-validation.js";
|
|
57
|
-
export * from "./comparison.js";
|
|
58
42
|
export * from "./sandbox.js";
|
|
59
43
|
export * from "./provider-failure.js";
|
|
60
44
|
export * from "./arms.js";
|
|
61
|
-
export * from "./qualification-config.js";
|
|
62
|
-
export * from "./qualification-oauth-directory.js";
|
|
63
|
-
export * from "./qualification-store.js";
|
|
64
|
-
export * from "./qualification-runner.js";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
export * from "./trusted-host-supervision.js";
|
|
2
1
|
export * from "./spec.js";
|
|
3
2
|
export * from "./discover.js";
|
|
4
3
|
export * from "./run.js";
|
|
@@ -13,6 +12,7 @@ export * from "./rescore.js";
|
|
|
13
12
|
export * from "./workspace.js";
|
|
14
13
|
export * from "./seeded.js";
|
|
15
14
|
export * from "./report.js";
|
|
15
|
+
export * from "./metrics.js";
|
|
16
16
|
export * from "./trends.js";
|
|
17
17
|
export * from "./lint.js";
|
|
18
18
|
export * from "./restamp.js";
|
|
@@ -26,40 +26,20 @@ export * from "./version.js";
|
|
|
26
26
|
export * from "./defaults.js";
|
|
27
27
|
export * from "./command-cost.js";
|
|
28
28
|
export * from "./judge-policy.js";
|
|
29
|
-
export * from "./judge-agreement.js";
|
|
30
29
|
export * from "./regate.js";
|
|
31
|
-
export * from "./screen.js";
|
|
32
30
|
export * from "./prompt-normalization.js";
|
|
33
31
|
export * from "./downgrade.js";
|
|
34
32
|
export * from "./canary.js";
|
|
35
33
|
export * from "./stability.js";
|
|
36
34
|
export * from "./capture-trace-types.js";
|
|
37
|
-
export * from "./work-capture.js";
|
|
38
|
-
export * from "./work-signals.js";
|
|
39
|
-
export * from "./work-capture-schema.js";
|
|
40
|
-
export * from "./factory-calibration.js";
|
|
41
|
-
export * from "./adoption.js";
|
|
42
|
-
export * from "./investigation.js";
|
|
43
|
-
export * from "./intervention.js";
|
|
44
|
-
export * from "./intervention-results.js";
|
|
45
35
|
export * from "./spec-write.js";
|
|
46
|
-
export * from "./
|
|
36
|
+
export * from "./redaction.js";
|
|
47
37
|
export * from "./execution-trace.js";
|
|
48
38
|
export * from "./trace-gates.js";
|
|
49
|
-
export * from "./trajectory-
|
|
39
|
+
export * from "./trajectory-events.js";
|
|
50
40
|
export * from "./instruction-coverage.js";
|
|
51
|
-
export * from "./affected.js";
|
|
52
|
-
export * from "./adjudication.js";
|
|
53
41
|
export * from "./vote-panel.js";
|
|
54
|
-
export * from "./qualification-panels.js";
|
|
55
|
-
export * from "./qualification-panel-store.js";
|
|
56
|
-
export * from "./qualification-validation.js";
|
|
57
|
-
export * from "./comparison.js";
|
|
58
42
|
export * from "./sandbox.js";
|
|
59
43
|
export * from "./provider-failure.js";
|
|
60
44
|
export * from "./arms.js";
|
|
61
|
-
export * from "./qualification-config.js";
|
|
62
|
-
export * from "./qualification-oauth-directory.js";
|
|
63
|
-
export * from "./qualification-store.js";
|
|
64
|
-
export * from "./qualification-runner.js";
|
|
65
45
|
//# sourceMappingURL=index.js.map
|
|
@@ -61,8 +61,6 @@ export interface SectionCoverage {
|
|
|
61
61
|
section: Section;
|
|
62
62
|
/** Scenario ids that declare a reference to this section. */
|
|
63
63
|
scenarios: string[];
|
|
64
|
-
/** Capture ids parked against this section but not yet promoted. */
|
|
65
|
-
pendingCaptures: string[];
|
|
66
64
|
}
|
|
67
65
|
export interface BrokenRef {
|
|
68
66
|
scenarioId: string;
|
|
@@ -87,11 +85,6 @@ export interface CoverageOptions {
|
|
|
87
85
|
scenarios: Scenario[];
|
|
88
86
|
/** Instruction files to report on even if nothing references them. */
|
|
89
87
|
baseFiles?: string[];
|
|
90
|
-
/** capture id → covers refs, for parking a pending case against a section. */
|
|
91
|
-
pendingCaptures?: {
|
|
92
|
-
id: string;
|
|
93
|
-
covers: string[];
|
|
94
|
-
}[];
|
|
95
88
|
}
|
|
96
89
|
/**
|
|
97
90
|
* Build the coverage report.
|
|
@@ -130,7 +130,7 @@ export function computeCoverage(opts) {
|
|
|
130
130
|
const k = key(file, section.slug);
|
|
131
131
|
let entry = bySection.get(k);
|
|
132
132
|
if (!entry) {
|
|
133
|
-
entry = { file, section, scenarios: []
|
|
133
|
+
entry = { file, section, scenarios: [] };
|
|
134
134
|
bySection.set(k, entry);
|
|
135
135
|
}
|
|
136
136
|
return entry;
|
|
@@ -142,7 +142,7 @@ export function computeCoverage(opts) {
|
|
|
142
142
|
ensure(file, s);
|
|
143
143
|
const broken = [];
|
|
144
144
|
const unmapped = [];
|
|
145
|
-
const attach = (id, refs
|
|
145
|
+
const attach = (id, refs) => {
|
|
146
146
|
for (const raw of refs) {
|
|
147
147
|
const ref = parseCoversRef(raw);
|
|
148
148
|
const sections = readSections(ref.file);
|
|
@@ -155,7 +155,7 @@ export function computeCoverage(opts) {
|
|
|
155
155
|
if (ref.slug === undefined) {
|
|
156
156
|
// A whole-file reference covers every section in it.
|
|
157
157
|
for (const s of sections)
|
|
158
|
-
ensure(ref.file, s)
|
|
158
|
+
ensure(ref.file, s).scenarios.push(id);
|
|
159
159
|
continue;
|
|
160
160
|
}
|
|
161
161
|
const match = sections.find((s) => s.slug === ref.slug);
|
|
@@ -168,7 +168,7 @@ export function computeCoverage(opts) {
|
|
|
168
168
|
});
|
|
169
169
|
continue;
|
|
170
170
|
}
|
|
171
|
-
ensure(ref.file, match)
|
|
171
|
+
ensure(ref.file, match).scenarios.push(id);
|
|
172
172
|
}
|
|
173
173
|
};
|
|
174
174
|
for (const s of opts.scenarios) {
|
|
@@ -176,10 +176,8 @@ export function computeCoverage(opts) {
|
|
|
176
176
|
unmapped.push(s.id);
|
|
177
177
|
continue;
|
|
178
178
|
}
|
|
179
|
-
attach(s.id, s.covers
|
|
179
|
+
attach(s.id, s.covers);
|
|
180
180
|
}
|
|
181
|
-
for (const c of opts.pendingCaptures ?? [])
|
|
182
|
-
attach(c.id, c.covers, "pendingCaptures");
|
|
183
181
|
const sections = [...bySection.values()].sort((a, b) => a.file.localeCompare(b.file) || a.section.startLine - b.section.startLine);
|
|
184
182
|
const covered = sections.filter((s) => s.scenarios.length > 0);
|
|
185
183
|
const uncovered = sections.filter((s) => s.scenarios.length === 0);
|
package/dist/journal.d.ts
CHANGED
|
@@ -63,14 +63,6 @@ export type JournalEvent = {
|
|
|
63
63
|
ok: boolean;
|
|
64
64
|
detail: string;
|
|
65
65
|
rep?: number;
|
|
66
|
-
}
|
|
67
|
-
/** One adjudication pass: which cells were re-judged, what it cost in CALLS, what stayed unresolved. */
|
|
68
|
-
| {
|
|
69
|
-
event: "adjudication";
|
|
70
|
-
ts: string;
|
|
71
|
-
triggered: string[];
|
|
72
|
-
judge_calls: number;
|
|
73
|
-
unresolved: string[];
|
|
74
66
|
} | {
|
|
75
67
|
event: "judge-verdict";
|
|
76
68
|
ts: string;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ScenarioResult } from "./results.js";
|
|
2
|
+
export interface AggregateMetrics {
|
|
3
|
+
wall_time_ms: number;
|
|
4
|
+
judge_calls: number;
|
|
5
|
+
judge_rejudge_calls: number;
|
|
6
|
+
subject_metrics_reps: number;
|
|
7
|
+
total_reps: number;
|
|
8
|
+
input_tokens: number | null;
|
|
9
|
+
output_tokens: number | null;
|
|
10
|
+
cache_read_tokens: number | null;
|
|
11
|
+
cache_write_tokens: number | null;
|
|
12
|
+
subject_cost_usd: number | null;
|
|
13
|
+
cost_source: "provider-reported" | "subscription" | "unreported" | null;
|
|
14
|
+
tool_calls: number | null;
|
|
15
|
+
delegated_children: number | null;
|
|
16
|
+
max_concurrency: number | null;
|
|
17
|
+
}
|
|
18
|
+
export declare function aggregateMetrics(scenarios: ScenarioResult[]): AggregateMetrics;
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export function aggregateMetrics(scenarios) {
|
|
2
|
+
const metrics = scenarios.map((scenario) => scenario.metrics).filter((value) => value !== undefined);
|
|
3
|
+
const sumOptional = (field) => {
|
|
4
|
+
const values = metrics.map((value) => value[field]).filter((value) => typeof value === "number");
|
|
5
|
+
return values.length ? values.reduce((sum, value) => sum + value, 0) : null;
|
|
6
|
+
};
|
|
7
|
+
const maxValues = metrics.map((value) => value.max_concurrency).filter((value) => typeof value === "number");
|
|
8
|
+
return {
|
|
9
|
+
wall_time_ms: metrics.reduce((sum, value) => sum + value.wall_time_ms, 0),
|
|
10
|
+
judge_calls: metrics.reduce((sum, value) => sum + value.judge_calls, 0),
|
|
11
|
+
judge_rejudge_calls: metrics.reduce((sum, value) => sum + value.judge_rejudge_calls, 0),
|
|
12
|
+
subject_metrics_reps: metrics.reduce((sum, value) => sum + value.subject_metrics_reps, 0),
|
|
13
|
+
total_reps: metrics.reduce((sum, value) => sum + value.total_reps, 0),
|
|
14
|
+
input_tokens: sumOptional("input_tokens"),
|
|
15
|
+
output_tokens: sumOptional("output_tokens"),
|
|
16
|
+
cache_read_tokens: sumOptional("cache_read_tokens"),
|
|
17
|
+
cache_write_tokens: sumOptional("cache_write_tokens"),
|
|
18
|
+
subject_cost_usd: sumOptional("subject_cost_usd"),
|
|
19
|
+
cost_source: (() => {
|
|
20
|
+
const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
|
|
21
|
+
return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
|
|
22
|
+
})(),
|
|
23
|
+
tool_calls: sumOptional("tool_calls"),
|
|
24
|
+
delegated_children: sumOptional("delegated_children"),
|
|
25
|
+
max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SanitizedArgs } from "./capture-trace-types.js";
|
|
2
|
+
/** Values longer than this are truncated rather than persisted whole. */
|
|
3
|
+
export declare const MAX_VALUE_CHARS = 2000;
|
|
4
|
+
export declare function redactText(input: string, homeDir?: string): string;
|
|
5
|
+
export declare function truncate(input: string, max?: number): string;
|
|
6
|
+
export declare function redactArgs(args: unknown, homeDir?: string, depth?: number): SanitizedArgs;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** Values longer than this are truncated rather than persisted whole. */
|
|
2
|
+
export const MAX_VALUE_CHARS = 2000;
|
|
3
|
+
const REDACTED = "[redacted]";
|
|
4
|
+
const SECRET_KEY = /^(.*[-_])?(password|passwd|secret|token|api[-_]?key|apikey|auth|authorization|credential|private[-_]?key|access[-_]?key|session[-_]?key)([-_].*)?$/i;
|
|
5
|
+
const SECRET_VALUE = [
|
|
6
|
+
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
7
|
+
/\bBearer\s+[A-Za-z0-9._~+/-]{16,}=*/g,
|
|
8
|
+
/\bsk-[A-Za-z0-9]{16,}\b/g,
|
|
9
|
+
/\bgh[pousr]_[A-Za-z0-9]{16,}\b/g,
|
|
10
|
+
/\bxox[abposr]-[A-Za-z0-9-]{10,}\b/g,
|
|
11
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
12
|
+
/\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
|
|
13
|
+
/\b([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s@]+@/gi,
|
|
14
|
+
];
|
|
15
|
+
export function redactText(input, homeDir) {
|
|
16
|
+
let out = input;
|
|
17
|
+
out = out.replace(SECRET_VALUE[SECRET_VALUE.length - 1], `$1${REDACTED}@`);
|
|
18
|
+
for (const re of SECRET_VALUE.slice(0, -1))
|
|
19
|
+
out = out.replace(re, REDACTED);
|
|
20
|
+
if (homeDir && homeDir.length > 1)
|
|
21
|
+
out = out.split(homeDir).join("~");
|
|
22
|
+
return out;
|
|
23
|
+
}
|
|
24
|
+
export function truncate(input, max = MAX_VALUE_CHARS) {
|
|
25
|
+
if (input.length <= max)
|
|
26
|
+
return input;
|
|
27
|
+
return `${input.slice(0, max)}… [truncated ${input.length - max} chars]`;
|
|
28
|
+
}
|
|
29
|
+
export function redactArgs(args, homeDir, depth = 0) {
|
|
30
|
+
if (args === null || typeof args !== "object" || Array.isArray(args))
|
|
31
|
+
return {};
|
|
32
|
+
const out = {};
|
|
33
|
+
for (const [key, value] of Object.entries(args)) {
|
|
34
|
+
if (SECRET_KEY.test(key)) {
|
|
35
|
+
out[key] = REDACTED;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
out[key] = redactValue(value, homeDir, depth);
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
function redactValue(value, homeDir, depth) {
|
|
43
|
+
if (typeof value === "string")
|
|
44
|
+
return truncate(redactText(value, homeDir));
|
|
45
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null)
|
|
46
|
+
return value;
|
|
47
|
+
if (depth >= 3)
|
|
48
|
+
return "[nested]";
|
|
49
|
+
if (Array.isArray(value))
|
|
50
|
+
return value.slice(0, 20).map((v) => redactValue(v, homeDir, depth + 1));
|
|
51
|
+
if (typeof value === "object")
|
|
52
|
+
return redactArgs(value, homeDir, depth + 1);
|
|
53
|
+
return String(value);
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=redaction.js.map
|
package/dist/regate.js
CHANGED
|
@@ -4,9 +4,8 @@ import { parseVerdict, detectMisfire } from "./grade.js";
|
|
|
4
4
|
import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
|
|
5
5
|
import { evaluateTraceGates } from "./trace-gates.js";
|
|
6
6
|
import { mergeTraces, deserializeTrace } from "./execution-trace.js";
|
|
7
|
-
import { deserializeTrajectoryEvents, evaluateTrajectoryGates } from "./trajectory-gates.js";
|
|
8
7
|
import { judgeOneRep } from "./regrade.js";
|
|
9
|
-
import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles,
|
|
8
|
+
import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
|
|
10
9
|
import { outcomesToResult } from "./reps.js";
|
|
11
10
|
import { appendJournal } from "./journal.js";
|
|
12
11
|
import { gatesDigest, GATES_PREFIX } from "./sources.js";
|
|
@@ -89,8 +88,7 @@ export async function regateRun(opts) {
|
|
|
89
88
|
const s = specById.get(rec.id);
|
|
90
89
|
const needles = hasNeedleGates(s ?? {});
|
|
91
90
|
const traceGated = Boolean(s?.traceAssert);
|
|
92
|
-
|
|
93
|
-
if (!s || (!needles && !traceGated && !trajectoryGated))
|
|
91
|
+
if (!s || (!needles && !traceGated))
|
|
94
92
|
continue; // nothing for regate to re-decide
|
|
95
93
|
if (s.assert?.vitest || s.assert?.post_test) {
|
|
96
94
|
blocked.push(`${s.id}: declares ${s.assert.vitest ? "assert.vitest" : "assert.post_test"}, which needs the workspace — ` +
|
|
@@ -112,16 +110,11 @@ export async function regateRun(opts) {
|
|
|
112
110
|
`(traces are gitignored, or capture failed) — it needs a re-run`);
|
|
113
111
|
continue;
|
|
114
112
|
}
|
|
115
|
-
if (trajectoryGated && !complete(findTrajectoryFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => trajectoryPath(opts.runDir, s.id, mode, rep)))) {
|
|
116
|
-
blocked.push(`${s.id}: assert.trajectory artifacts are incomplete for ${expectedReps} recorded rep(s) ` +
|
|
117
|
-
`(events are gitignored, or required native evidence was missing) — it needs a re-run`);
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
113
|
targets.push(s);
|
|
121
114
|
}
|
|
122
115
|
if (targets.length === 0) {
|
|
123
116
|
throw new Error(`nothing to regate in ${opts.runDir}` +
|
|
124
|
-
(blocked.length > 0 ? `:\n ${blocked.join("\n ")}` : " — no scenario declares diff_contains/diff_excludes
|
|
117
|
+
(blocked.length > 0 ? `:\n ${blocked.join("\n ")}` : " — no scenario declares diff_contains/diff_excludes or assert.trace"));
|
|
125
118
|
}
|
|
126
119
|
const changes = [];
|
|
127
120
|
let judgeCalls = 0;
|
|
@@ -134,15 +127,9 @@ export async function regateRun(opts) {
|
|
|
134
127
|
}
|
|
135
128
|
const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
|
|
136
129
|
const traceFiles = findTraceFiles(opts.runDir, scenario.id, mode);
|
|
137
|
-
const eventFiles = findTrajectoryFiles(opts.runDir, scenario.id, mode);
|
|
138
|
-
// Reps come from whichever artifact this scenario actually has. A trajectory-only
|
|
139
|
-
// scenario has neither diff nor trace, so falling back only once would regate
|
|
140
|
-
// nothing and report success.
|
|
141
130
|
const repKeys = diffFiles.length > 0
|
|
142
131
|
? diffFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: f }))
|
|
143
|
-
: traceFiles.
|
|
144
|
-
? traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }))
|
|
145
|
-
: eventFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
|
|
132
|
+
: traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
|
|
146
133
|
const outcomes = [];
|
|
147
134
|
// Per scenario, not run-wide: with several regated scenarios, a global counter
|
|
148
135
|
// would report every change as "re-judged" because some other scenario was.
|
|
@@ -192,61 +179,9 @@ export async function regateRun(opts) {
|
|
|
192
179
|
}
|
|
193
180
|
}
|
|
194
181
|
}
|
|
195
|
-
// Adapter-neutral trajectory gate, replayed from the saved normalized
|
|
196
|
-
// event artifact. Native ledgers are deliberately not re-read here: the
|
|
197
|
-
// artifact is the immutable evidence this run actually captured.
|
|
198
|
-
let trajectoryFailure = null;
|
|
199
|
-
if (scenario.trajectoryAssert) {
|
|
200
|
-
const ep = trajectoryPath(opts.runDir, scenario.id, mode, rep);
|
|
201
|
-
const events = existsSync(ep) ? deserializeTrajectoryEvents(readFileSync(ep, "utf8")) : null;
|
|
202
|
-
// The normalized artifact contains only events that were successfully captured.
|
|
203
|
-
// Source-read/normalization failures are persisted on the original objective;
|
|
204
|
-
// dropping them here would let a partial ledger become PASS on replay.
|
|
205
|
-
const priorEvidenceErrors = (rec.objective?.assertions ?? [])
|
|
206
|
-
.filter((result) => result.kind === "trajectory_evidence" && result.status === "ERROR");
|
|
207
|
-
if (!events) {
|
|
208
|
-
trajectoryFailure = "objective: saved normalized events are missing, malformed, or from an unsupported version";
|
|
209
|
-
objective = {
|
|
210
|
-
...(objective ?? { assertions: [] }),
|
|
211
|
-
status: "ERROR",
|
|
212
|
-
assertions: [...(objective?.assertions ?? []), { kind: "trajectory_evidence", status: "ERROR", detail: trajectoryFailure }],
|
|
213
|
-
};
|
|
214
|
-
}
|
|
215
|
-
else {
|
|
216
|
-
const g = evaluateTrajectoryGates(scenario.trajectoryAssert, events);
|
|
217
|
-
const expectedHash = rec.objective?.rep_events_sha256?.[rep ?? 0] ?? (rec.reps === undefined ? rec.objective?.events_sha256 : undefined);
|
|
218
|
-
const digestMismatch = expectedHash !== undefined && expectedHash !== g.events_sha256;
|
|
219
|
-
const integrityAssertion = digestMismatch
|
|
220
|
-
? [{ kind: "trajectory_evidence", status: "ERROR", detail: "saved normalized events no longer match the hash recorded by the run" }]
|
|
221
|
-
: [];
|
|
222
|
-
const priorStatus = objective?.status ?? "PASS";
|
|
223
|
-
const status = priorStatus === "ERROR" || g.status === "ERROR" || priorEvidenceErrors.length || digestMismatch
|
|
224
|
-
? "ERROR"
|
|
225
|
-
: priorStatus === "NOT-MEASURED"
|
|
226
|
-
? "NOT-MEASURED"
|
|
227
|
-
: priorStatus === "FAIL" || g.status === "FAIL"
|
|
228
|
-
? "FAIL"
|
|
229
|
-
: "PASS";
|
|
230
|
-
objective = {
|
|
231
|
-
...(objective ?? {}),
|
|
232
|
-
status,
|
|
233
|
-
trajectory_version: g.event_version,
|
|
234
|
-
events_sha256: g.events_sha256,
|
|
235
|
-
assertions: [...(objective?.assertions ?? []), ...priorEvidenceErrors, ...integrityAssertion, ...g.assertions],
|
|
236
|
-
};
|
|
237
|
-
if (g.status !== "PASS" || priorEvidenceErrors.length || digestMismatch) {
|
|
238
|
-
const details = [
|
|
239
|
-
...priorEvidenceErrors.map((result) => result.detail),
|
|
240
|
-
...integrityAssertion.map((result) => result.detail),
|
|
241
|
-
...g.assertions.filter((result) => result.status === g.status && g.status !== "PASS").map((result) => result.detail),
|
|
242
|
-
];
|
|
243
|
-
trajectoryFailure = `objective: ${details.join("; ")}`;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
182
|
// Prompt delivery was observed during the subject invocation and cannot be
|
|
248
183
|
// replayed by regate. Carry its objective finding while refreshing only
|
|
249
|
-
// trace/
|
|
184
|
+
// trace/needle assertions.
|
|
250
185
|
const retainedDelivery = rec.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
|
|
251
186
|
const repObservations = prev.subject_invocations?.filter(observation => observation.scenario_id === scenario.id && observation.repetition === (rep ?? 0)) ?? [];
|
|
252
187
|
const delivery = prev.schema === 3
|
|
@@ -259,7 +194,7 @@ export async function regateRun(opts) {
|
|
|
259
194
|
if (delivery.status !== "PASS")
|
|
260
195
|
traceFailure = `objective: ${delivery.detail}`;
|
|
261
196
|
}
|
|
262
|
-
const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure
|
|
197
|
+
const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure };
|
|
263
198
|
const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
|
|
264
199
|
const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
|
|
265
200
|
// Two sources, because the two gate kinds record their prior state
|
package/dist/regrade.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Spec, Scenario } from "./spec.js";
|
|
2
2
|
import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
|
|
3
|
-
import { type
|
|
3
|
+
import { type ScenarioResult, type ResultsFile } from "./results.js";
|
|
4
4
|
import { type RepOutcome } from "./reps.js";
|
|
5
5
|
/**
|
|
6
6
|
* Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
|
|
@@ -84,6 +84,4 @@ export interface RegradeRunOptions {
|
|
|
84
84
|
* results.yaml, emits the `score` journal event, and returns the new
|
|
85
85
|
* ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
|
|
86
86
|
*/
|
|
87
|
-
/** Preserve prior panel votes and append a full-cell grade without writing invalid one-vote history. */
|
|
88
|
-
export declare function appendJudgeHistory(prior: ScenarioResult | undefined, priorJudge: ModelRef | undefined, fresh: Omit<Judgment, "ordinal">): Judgment[] | undefined;
|
|
89
87
|
export declare function regradeRun(opts: RegradeRunOptions): Promise<ResultsFile>;
|