@skill-harness/core 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -2
  2. package/dist/adapters/types.d.ts +0 -12
  3. package/dist/capture-trace-types.d.ts +0 -96
  4. package/dist/capture-trace-types.js +0 -4
  5. package/dist/command-cost.d.ts +1 -1
  6. package/dist/command-cost.js +0 -5
  7. package/dist/execution-trace.js +1 -1
  8. package/dist/index.d.ts +3 -23
  9. package/dist/index.js +3 -23
  10. package/dist/instruction-coverage.d.ts +0 -7
  11. package/dist/instruction-coverage.js +5 -7
  12. package/dist/journal.d.ts +0 -8
  13. package/dist/metrics.d.ts +18 -0
  14. package/dist/metrics.js +28 -0
  15. package/dist/redaction.d.ts +6 -0
  16. package/dist/redaction.js +55 -0
  17. package/dist/regate.js +6 -71
  18. package/dist/regrade.d.ts +1 -3
  19. package/dist/regrade.js +1 -21
  20. package/dist/report.d.ts +1 -15
  21. package/dist/report.js +1 -13
  22. package/dist/reps.d.ts +1 -2
  23. package/dist/reps.js +0 -2
  24. package/dist/results.d.ts +0 -9
  25. package/dist/results.js +2 -19
  26. package/dist/run.js +11 -72
  27. package/dist/seeded.d.ts +1 -7
  28. package/dist/seeded.js +5 -11
  29. package/dist/sources.js +4 -9
  30. package/dist/spec.d.ts +0 -13
  31. package/dist/spec.js +6 -48
  32. package/dist/trajectory-events.d.ts +58 -0
  33. package/dist/trajectory-events.js +119 -0
  34. package/package.json +1 -1
  35. package/schemas/results-v2.schema.json +376 -75
  36. package/schemas/specification-v1.schema.json +128 -174
  37. package/dist/adjudication.d.ts +0 -219
  38. package/dist/adjudication.js +0 -422
  39. package/dist/adoption.d.ts +0 -128
  40. package/dist/adoption.js +0 -102
  41. package/dist/affected.d.ts +0 -88
  42. package/dist/affected.js +0 -222
  43. package/dist/capture.d.ts +0 -193
  44. package/dist/capture.js +0 -344
  45. package/dist/comparison.d.ts +0 -97
  46. package/dist/comparison.js +0 -189
  47. package/dist/factory-calibration.d.ts +0 -63
  48. package/dist/factory-calibration.js +0 -115
  49. package/dist/intervention-results.d.ts +0 -6
  50. package/dist/intervention-results.js +0 -52
  51. package/dist/intervention.d.ts +0 -123
  52. package/dist/intervention.js +0 -218
  53. package/dist/investigation.d.ts +0 -94
  54. package/dist/investigation.js +0 -148
  55. package/dist/judge-agreement.d.ts +0 -19
  56. package/dist/judge-agreement.js +0 -31
  57. package/dist/qualification-capture.d.ts +0 -44
  58. package/dist/qualification-capture.js +0 -163
  59. package/dist/qualification-config.d.ts +0 -233
  60. package/dist/qualification-config.js +0 -694
  61. package/dist/qualification-lock.d.ts +0 -15
  62. package/dist/qualification-lock.js +0 -87
  63. package/dist/qualification-oauth-directory.d.ts +0 -79
  64. package/dist/qualification-oauth-directory.js +0 -444
  65. package/dist/qualification-panel-store.d.ts +0 -14
  66. package/dist/qualification-panel-store.js +0 -188
  67. package/dist/qualification-panels.d.ts +0 -99
  68. package/dist/qualification-panels.js +0 -91
  69. package/dist/qualification-process.d.ts +0 -13
  70. package/dist/qualification-process.js +0 -190
  71. package/dist/qualification-runner.d.ts +0 -203
  72. package/dist/qualification-runner.js +0 -2037
  73. package/dist/qualification-store.d.ts +0 -187
  74. package/dist/qualification-store.js +0 -833
  75. package/dist/qualification-validation.d.ts +0 -12
  76. package/dist/qualification-validation.js +0 -8
  77. package/dist/screen.d.ts +0 -31
  78. package/dist/screen.js +0 -105
  79. package/dist/trajectory-gates.d.ts +0 -137
  80. package/dist/trajectory-gates.js +0 -576
  81. package/dist/trusted-host-supervision.d.ts +0 -25
  82. package/dist/trusted-host-supervision.js +0 -94
  83. package/dist/work-capture-schema.d.ts +0 -38
  84. package/dist/work-capture-schema.js +0 -33
  85. package/dist/work-capture.d.ts +0 -43
  86. package/dist/work-capture.js +0 -81
  87. package/dist/work-signals.d.ts +0 -55
  88. package/dist/work-signals.js +0 -71
  89. package/schemas/qualification-cell-result-v1.schema.json +0 -32
  90. package/schemas/qualification-config-v1.schema.json +0 -244
  91. package/schemas/qualification-invocation-request-v1.schema.json +0 -59
  92. package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
  93. package/schemas/trajectory-event-v1.schema.json +0 -80
package/README.md CHANGED
@@ -1,8 +1,7 @@
1
1
  # @skill-harness/core
2
2
 
3
3
  The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
4
- spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results,
5
- and the separately versioned durable qualification-runner-v1 primitives.
4
+ spec parsing, discovery, run orchestration, LLM-judge grading, scoring, and results.
6
5
 
7
6
  Internal API — use the `skill-harness` CLI. See the repo for docs.
8
7
 
@@ -1,6 +1,4 @@
1
1
  import type { ExecutionTraceV1 } from "../capture-trace-types.js";
2
- import type { TrajectoryEventSource } from "../spec.js";
3
- import type { TrajectoryEventV1 } from "../trajectory-gates.js";
4
2
  export type RunMode = "red" | "green" | "force";
5
3
  /** A provider+model pair, e.g. { provider: "fireworks", model: "accounts/.../deepseek-v4-pro" }. */
6
4
  export interface ModelRef {
@@ -57,16 +55,12 @@ export interface RunReq {
57
55
  * extension the developer happens to have installed cannot join the test.
58
56
  */
59
57
  extensions?: string[];
60
- /** Workspace-local native ledgers to normalize after the subject finishes. */
61
- eventSources?: TrajectoryEventSource[];
62
58
  /**
63
59
  * Extra env for the subject process, from the arm. Merged over `process.env` by
64
60
  * the adapter — an arm must be able to add `PI_GRANTS_*` without the harness
65
61
  * inheriting a caller's copy of them.
66
62
  */
67
63
  armEnv?: Record<string, string>;
68
- /** Adapter-produced observation callback. Callers receive evidence; they never supply delivery status. */
69
- onPromptObservation?: (observation: PromptProvenance) => void;
70
64
  }
71
65
  /** A judge request: single prompt, no skills, no session. */
72
66
  export interface JudgeReq {
@@ -79,10 +73,6 @@ export interface StructuredRun {
79
73
  transcript: string;
80
74
  /** One trace per turn — each `pi` invocation emits an independent event stream. */
81
75
  traces: ExecutionTraceV1[];
82
- /** Adapter-neutral workflow/tool events. Additive; absent on older adapters. */
83
- events?: TrajectoryEventV1[];
84
- /** Native sources that were required but missing/malformed. Never treated as an empty success. */
85
- eventErrors?: string[];
86
76
  /**
87
77
  * Set when pi failed provider-side (auth, transport) rather than the model
88
78
  * answering badly. `run.ts` turns this into ERROR — never a model verdict.
@@ -91,8 +81,6 @@ export interface StructuredRun {
91
81
  }
92
82
  export interface HarnessAdapter {
93
83
  name: string;
94
- /** True only when the adapter computes final provider-payload observations. */
95
- observesPrompts?: boolean;
96
84
  available(): Promise<boolean>;
97
85
  run(req: RunReq): Promise<string>;
98
86
  /**
@@ -151,99 +151,3 @@ export interface ExecutionTraceV1 {
151
151
  /** SHA-256 over the deterministic serialization of this trace, minus this field. */
152
152
  trace_sha256?: string;
153
153
  }
154
- export declare const CAPTURE_SCHEMA_VERSION = 1;
155
- /** Why a conversation was captured. */
156
- export type CaptureClassification = "failure" | "good_example";
157
- /** Where a capture is in the human review pipeline. */
158
- export type CapturePromotionStatus = "pending" | "promoted" | "rejected";
159
- /** What the human confirmed as responsible for the behavior. */
160
- export interface CaptureTarget {
161
- kind: "skill" | "subagent";
162
- /** Path relative to the skills root or repo root — never absolute. */
163
- path: string;
164
- /** SHA-256 of the target's content at capture time, so drift is detectable. */
165
- content_sha256: string;
166
- }
167
- /**
168
- * Provenance for a capture.
169
- *
170
- * Hashed, not absolute: an absolute session path identifies a machine and a
171
- * user, and the capture file is meant to be committed. The hash is enough to
172
- * recognize the same session again locally, which is all provenance needs to do.
173
- */
174
- export interface CaptureProvenance {
175
- session_sha256: string;
176
- /** Indices of the selected turn range within the active branch, inclusive. */
177
- turn_range: {
178
- start: number;
179
- end: number;
180
- };
181
- subject?: ModelRef;
182
- git_commit?: string;
183
- /** True when the working tree was dirty at capture time. */
184
- git_dirty?: boolean;
185
- }
186
- /**
187
- * A reviewed conversation awaiting promotion into a scenario.
188
- *
189
- * Lives under `<skill>/tests/captures/`, NOT in `specification.yaml`. A pending
190
- * capture is not a test: putting it in the spec would drag it into ship-bar
191
- * totals, staleness, lift, and stability, and the alternative — a `draft: true`
192
- * flag every runner, scorer and linter has to honor — is a state that only has
193
- * to be forgotten once to either corrupt a grade or silently drop a real
194
- * scenario from a release run.
195
- */
196
- export interface CaptureCaseV1 {
197
- capture_schema: typeof CAPTURE_SCHEMA_VERSION;
198
- id: string;
199
- created: string;
200
- classification: CaptureClassification;
201
- /** Sanitized user turns only. Assistant prose is evidence, never an oracle. */
202
- turns: string[];
203
- /** Human-written, in their own words. Required — a capture with no expectation is not reviewable. */
204
- expected_behavior: string;
205
- /** Editable draft checklist. Offline-derived by default; LLM drafting is opt-in and costs tokens. */
206
- checklist: string[];
207
- target: CaptureTarget;
208
- provenance: CaptureProvenance;
209
- status: CapturePromotionStatus;
210
- /** Set once promoted: the scenario id appended to `specification.yaml`. */
211
- scenario_id?: string;
212
- /**
213
- * `covers` refs for this pending case, so `coverage` can park it against the
214
- * instructions it is about before anyone promotes it.
215
- *
216
- * Derived from `target` — the author has already been made to choose which
217
- * instructions are responsible, and asking again in different words would be
218
- * asking the same question twice. File granularity, not `#section`: the target
219
- * choice attributes a file, and inventing a section would be a guess the
220
- * session cannot support.
221
- *
222
- * Written relative to the SPEC dir (`<skill>/tests`), matching how a scenario's
223
- * own `covers` resolve. `coverage` read this field from the start; nothing ever
224
- * wrote it, so every pending case parked against nothing and the feature was
225
- * inert.
226
- */
227
- covers?: string[];
228
- }
229
- /**
230
- * Local-only evidence sidecar for a capture.
231
- *
232
- * Written to `<skill>/tests/captures/.local/` and git-ignored by default. It may
233
- * hold a sanitized assistant excerpt and tool-call summaries to help a human
234
- * review the capture, and must NEVER hold hidden thinking, complete tool-result
235
- * bodies, or the effective system prompt.
236
- *
237
- * pi's stream carries thinking in `message_end`, `turn_end` AND `agent_end`, so
238
- * dropping it is an explicit filter at every one of those, not a side effect of
239
- * reading the convenient field.
240
- */
241
- export interface CaptureEvidenceV1 {
242
- capture_id: string;
243
- /** Sanitized, truncated excerpt of the assistant text of every selected turn, joined. */
244
- assistant_excerpt: string;
245
- /** Tool name, error state and redacted arguments — no bodies. */
246
- tool_calls: Array<Pick<TraceToolCall, "name" | "isError"> & {
247
- args: SanitizedArgs;
248
- }>;
249
- }
@@ -16,8 +16,4 @@
16
16
  // Execution trace
17
17
  // ---------------------------------------------------------------------------
18
18
  export const EXECUTION_TRACE_VERSION = 2;
19
- // ---------------------------------------------------------------------------
20
- // Capture case
21
- // ---------------------------------------------------------------------------
22
- export const CAPTURE_SCHEMA_VERSION = 1;
23
19
  //# sourceMappingURL=capture-trace-types.js.map
@@ -4,6 +4,6 @@
4
4
  * Keep this list narrow: commands that can conditionally spend (notably `regate`)
5
5
  * do not belong here even when their common path only replays saved artifacts.
6
6
  */
7
- export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "archive", "coverage", "init", "judge-agreement", "learning", "lint", "list", "rescore", "restamp", "screen", "stability"];
7
+ export declare const FREE_OFFLINE_COMMANDS: readonly ["coverage", "init", "lint", "list", "rescore", "restamp", "stability"];
8
8
  export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
9
9
  export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;
@@ -5,17 +5,12 @@
5
5
  * do not belong here even when their common path only replays saved artifacts.
6
6
  */
7
7
  export const FREE_OFFLINE_COMMANDS = [
8
- "affected",
9
- "archive",
10
8
  "coverage",
11
9
  "init",
12
- "judge-agreement",
13
- "learning",
14
10
  "lint",
15
11
  "list",
16
12
  "rescore",
17
13
  "restamp",
18
- "screen",
19
14
  "stability",
20
15
  ];
21
16
  export function isFreeOfflineCommand(command) {
@@ -1,7 +1,7 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { normalizeSubagentCall } from "./trace-gates.js";
3
3
  import { EXECUTION_TRACE_VERSION } from "./capture-trace-types.js";
4
- import { redactArgs, redactText } from "./capture.js";
4
+ import { redactArgs, redactText } from "./redaction.js";
5
5
  /**
6
6
  * Parse pi's `--mode json` event stream into an `ExecutionTraceV1`.
7
7
  *
package/dist/index.d.ts CHANGED
@@ -1,4 +1,3 @@
1
- export * from "./trusted-host-supervision.js";
2
1
  export * from "./spec.js";
3
2
  export * from "./discover.js";
4
3
  export * from "./run.js";
@@ -13,6 +12,7 @@ export * from "./rescore.js";
13
12
  export * from "./workspace.js";
14
13
  export * from "./seeded.js";
15
14
  export * from "./report.js";
15
+ export * from "./metrics.js";
16
16
  export * from "./trends.js";
17
17
  export * from "./lint.js";
18
18
  export * from "./restamp.js";
@@ -26,39 +26,19 @@ export * from "./version.js";
26
26
  export * from "./defaults.js";
27
27
  export * from "./command-cost.js";
28
28
  export * from "./judge-policy.js";
29
- export * from "./judge-agreement.js";
30
29
  export * from "./regate.js";
31
- export * from "./screen.js";
32
30
  export * from "./prompt-normalization.js";
33
31
  export * from "./downgrade.js";
34
32
  export * from "./canary.js";
35
33
  export * from "./stability.js";
36
34
  export * from "./capture-trace-types.js";
37
- export * from "./work-capture.js";
38
- export * from "./work-signals.js";
39
- export * from "./work-capture-schema.js";
40
- export * from "./factory-calibration.js";
41
- export * from "./adoption.js";
42
- export * from "./investigation.js";
43
- export * from "./intervention.js";
44
- export * from "./intervention-results.js";
45
35
  export * from "./spec-write.js";
46
- export * from "./capture.js";
36
+ export * from "./redaction.js";
47
37
  export * from "./execution-trace.js";
48
38
  export * from "./trace-gates.js";
49
- export * from "./trajectory-gates.js";
39
+ export * from "./trajectory-events.js";
50
40
  export * from "./instruction-coverage.js";
51
- export * from "./affected.js";
52
- export * from "./adjudication.js";
53
41
  export * from "./vote-panel.js";
54
- export * from "./qualification-panels.js";
55
- export * from "./qualification-panel-store.js";
56
- export * from "./qualification-validation.js";
57
- export * from "./comparison.js";
58
42
  export * from "./sandbox.js";
59
43
  export * from "./provider-failure.js";
60
44
  export * from "./arms.js";
61
- export * from "./qualification-config.js";
62
- export * from "./qualification-oauth-directory.js";
63
- export * from "./qualification-store.js";
64
- export * from "./qualification-runner.js";
package/dist/index.js CHANGED
@@ -1,4 +1,3 @@
1
- export * from "./trusted-host-supervision.js";
2
1
  export * from "./spec.js";
3
2
  export * from "./discover.js";
4
3
  export * from "./run.js";
@@ -13,6 +12,7 @@ export * from "./rescore.js";
13
12
  export * from "./workspace.js";
14
13
  export * from "./seeded.js";
15
14
  export * from "./report.js";
15
+ export * from "./metrics.js";
16
16
  export * from "./trends.js";
17
17
  export * from "./lint.js";
18
18
  export * from "./restamp.js";
@@ -26,40 +26,20 @@ export * from "./version.js";
26
26
  export * from "./defaults.js";
27
27
  export * from "./command-cost.js";
28
28
  export * from "./judge-policy.js";
29
- export * from "./judge-agreement.js";
30
29
  export * from "./regate.js";
31
- export * from "./screen.js";
32
30
  export * from "./prompt-normalization.js";
33
31
  export * from "./downgrade.js";
34
32
  export * from "./canary.js";
35
33
  export * from "./stability.js";
36
34
  export * from "./capture-trace-types.js";
37
- export * from "./work-capture.js";
38
- export * from "./work-signals.js";
39
- export * from "./work-capture-schema.js";
40
- export * from "./factory-calibration.js";
41
- export * from "./adoption.js";
42
- export * from "./investigation.js";
43
- export * from "./intervention.js";
44
- export * from "./intervention-results.js";
45
35
  export * from "./spec-write.js";
46
- export * from "./capture.js";
36
+ export * from "./redaction.js";
47
37
  export * from "./execution-trace.js";
48
38
  export * from "./trace-gates.js";
49
- export * from "./trajectory-gates.js";
39
+ export * from "./trajectory-events.js";
50
40
  export * from "./instruction-coverage.js";
51
- export * from "./affected.js";
52
- export * from "./adjudication.js";
53
41
  export * from "./vote-panel.js";
54
- export * from "./qualification-panels.js";
55
- export * from "./qualification-panel-store.js";
56
- export * from "./qualification-validation.js";
57
- export * from "./comparison.js";
58
42
  export * from "./sandbox.js";
59
43
  export * from "./provider-failure.js";
60
44
  export * from "./arms.js";
61
- export * from "./qualification-config.js";
62
- export * from "./qualification-oauth-directory.js";
63
- export * from "./qualification-store.js";
64
- export * from "./qualification-runner.js";
65
45
  //# sourceMappingURL=index.js.map
@@ -61,8 +61,6 @@ export interface SectionCoverage {
61
61
  section: Section;
62
62
  /** Scenario ids that declare a reference to this section. */
63
63
  scenarios: string[];
64
- /** Capture ids parked against this section but not yet promoted. */
65
- pendingCaptures: string[];
66
64
  }
67
65
  export interface BrokenRef {
68
66
  scenarioId: string;
@@ -87,11 +85,6 @@ export interface CoverageOptions {
87
85
  scenarios: Scenario[];
88
86
  /** Instruction files to report on even if nothing references them. */
89
87
  baseFiles?: string[];
90
- /** capture id → covers refs, for parking a pending case against a section. */
91
- pendingCaptures?: {
92
- id: string;
93
- covers: string[];
94
- }[];
95
88
  }
96
89
  /**
97
90
  * Build the coverage report.
@@ -130,7 +130,7 @@ export function computeCoverage(opts) {
130
130
  const k = key(file, section.slug);
131
131
  let entry = bySection.get(k);
132
132
  if (!entry) {
133
- entry = { file, section, scenarios: [], pendingCaptures: [] };
133
+ entry = { file, section, scenarios: [] };
134
134
  bySection.set(k, entry);
135
135
  }
136
136
  return entry;
@@ -142,7 +142,7 @@ export function computeCoverage(opts) {
142
142
  ensure(file, s);
143
143
  const broken = [];
144
144
  const unmapped = [];
145
- const attach = (id, refs, into) => {
145
+ const attach = (id, refs) => {
146
146
  for (const raw of refs) {
147
147
  const ref = parseCoversRef(raw);
148
148
  const sections = readSections(ref.file);
@@ -155,7 +155,7 @@ export function computeCoverage(opts) {
155
155
  if (ref.slug === undefined) {
156
156
  // A whole-file reference covers every section in it.
157
157
  for (const s of sections)
158
- ensure(ref.file, s)[into].push(id);
158
+ ensure(ref.file, s).scenarios.push(id);
159
159
  continue;
160
160
  }
161
161
  const match = sections.find((s) => s.slug === ref.slug);
@@ -168,7 +168,7 @@ export function computeCoverage(opts) {
168
168
  });
169
169
  continue;
170
170
  }
171
- ensure(ref.file, match)[into].push(id);
171
+ ensure(ref.file, match).scenarios.push(id);
172
172
  }
173
173
  };
174
174
  for (const s of opts.scenarios) {
@@ -176,10 +176,8 @@ export function computeCoverage(opts) {
176
176
  unmapped.push(s.id);
177
177
  continue;
178
178
  }
179
- attach(s.id, s.covers, "scenarios");
179
+ attach(s.id, s.covers);
180
180
  }
181
- for (const c of opts.pendingCaptures ?? [])
182
- attach(c.id, c.covers, "pendingCaptures");
183
181
  const sections = [...bySection.values()].sort((a, b) => a.file.localeCompare(b.file) || a.section.startLine - b.section.startLine);
184
182
  const covered = sections.filter((s) => s.scenarios.length > 0);
185
183
  const uncovered = sections.filter((s) => s.scenarios.length === 0);
package/dist/journal.d.ts CHANGED
@@ -63,14 +63,6 @@ export type JournalEvent = {
63
63
  ok: boolean;
64
64
  detail: string;
65
65
  rep?: number;
66
- }
67
- /** One adjudication pass: which cells were re-judged, what it cost in CALLS, what stayed unresolved. */
68
- | {
69
- event: "adjudication";
70
- ts: string;
71
- triggered: string[];
72
- judge_calls: number;
73
- unresolved: string[];
74
66
  } | {
75
67
  event: "judge-verdict";
76
68
  ts: string;
@@ -0,0 +1,18 @@
1
+ import type { ScenarioResult } from "./results.js";
2
+ export interface AggregateMetrics {
3
+ wall_time_ms: number;
4
+ judge_calls: number;
5
+ judge_rejudge_calls: number;
6
+ subject_metrics_reps: number;
7
+ total_reps: number;
8
+ input_tokens: number | null;
9
+ output_tokens: number | null;
10
+ cache_read_tokens: number | null;
11
+ cache_write_tokens: number | null;
12
+ subject_cost_usd: number | null;
13
+ cost_source: "provider-reported" | "subscription" | "unreported" | null;
14
+ tool_calls: number | null;
15
+ delegated_children: number | null;
16
+ max_concurrency: number | null;
17
+ }
18
+ export declare function aggregateMetrics(scenarios: ScenarioResult[]): AggregateMetrics;
@@ -0,0 +1,28 @@
1
+ export function aggregateMetrics(scenarios) {
2
+ const metrics = scenarios.map((scenario) => scenario.metrics).filter((value) => value !== undefined);
3
+ const sumOptional = (field) => {
4
+ const values = metrics.map((value) => value[field]).filter((value) => typeof value === "number");
5
+ return values.length ? values.reduce((sum, value) => sum + value, 0) : null;
6
+ };
7
+ const maxValues = metrics.map((value) => value.max_concurrency).filter((value) => typeof value === "number");
8
+ return {
9
+ wall_time_ms: metrics.reduce((sum, value) => sum + value.wall_time_ms, 0),
10
+ judge_calls: metrics.reduce((sum, value) => sum + value.judge_calls, 0),
11
+ judge_rejudge_calls: metrics.reduce((sum, value) => sum + value.judge_rejudge_calls, 0),
12
+ subject_metrics_reps: metrics.reduce((sum, value) => sum + value.subject_metrics_reps, 0),
13
+ total_reps: metrics.reduce((sum, value) => sum + value.total_reps, 0),
14
+ input_tokens: sumOptional("input_tokens"),
15
+ output_tokens: sumOptional("output_tokens"),
16
+ cache_read_tokens: sumOptional("cache_read_tokens"),
17
+ cache_write_tokens: sumOptional("cache_write_tokens"),
18
+ subject_cost_usd: sumOptional("subject_cost_usd"),
19
+ cost_source: (() => {
20
+ const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
21
+ return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
22
+ })(),
23
+ tool_calls: sumOptional("tool_calls"),
24
+ delegated_children: sumOptional("delegated_children"),
25
+ max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
26
+ };
27
+ }
28
+ //# sourceMappingURL=metrics.js.map
@@ -0,0 +1,6 @@
1
+ import type { SanitizedArgs } from "./capture-trace-types.js";
2
+ /** Values longer than this are truncated rather than persisted whole. */
3
+ export declare const MAX_VALUE_CHARS = 2000;
4
+ export declare function redactText(input: string, homeDir?: string): string;
5
+ export declare function truncate(input: string, max?: number): string;
6
+ export declare function redactArgs(args: unknown, homeDir?: string, depth?: number): SanitizedArgs;
@@ -0,0 +1,55 @@
1
+ /** Values longer than this are truncated rather than persisted whole. */
2
+ export const MAX_VALUE_CHARS = 2000;
3
+ const REDACTED = "[redacted]";
4
+ const SECRET_KEY = /^(.*[-_])?(password|passwd|secret|token|api[-_]?key|apikey|auth|authorization|credential|private[-_]?key|access[-_]?key|session[-_]?key)([-_].*)?$/i;
5
+ const SECRET_VALUE = [
6
+ /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
7
+ /\bBearer\s+[A-Za-z0-9._~+/-]{16,}=*/g,
8
+ /\bsk-[A-Za-z0-9]{16,}\b/g,
9
+ /\bgh[pousr]_[A-Za-z0-9]{16,}\b/g,
10
+ /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/g,
11
+ /\bAKIA[0-9A-Z]{16}\b/g,
12
+ /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
13
+ /\b([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s@]+@/gi,
14
+ ];
15
+ export function redactText(input, homeDir) {
16
+ let out = input;
17
+ out = out.replace(SECRET_VALUE[SECRET_VALUE.length - 1], `$1${REDACTED}@`);
18
+ for (const re of SECRET_VALUE.slice(0, -1))
19
+ out = out.replace(re, REDACTED);
20
+ if (homeDir && homeDir.length > 1)
21
+ out = out.split(homeDir).join("~");
22
+ return out;
23
+ }
24
+ export function truncate(input, max = MAX_VALUE_CHARS) {
25
+ if (input.length <= max)
26
+ return input;
27
+ return `${input.slice(0, max)}… [truncated ${input.length - max} chars]`;
28
+ }
29
+ export function redactArgs(args, homeDir, depth = 0) {
30
+ if (args === null || typeof args !== "object" || Array.isArray(args))
31
+ return {};
32
+ const out = {};
33
+ for (const [key, value] of Object.entries(args)) {
34
+ if (SECRET_KEY.test(key)) {
35
+ out[key] = REDACTED;
36
+ continue;
37
+ }
38
+ out[key] = redactValue(value, homeDir, depth);
39
+ }
40
+ return out;
41
+ }
42
+ function redactValue(value, homeDir, depth) {
43
+ if (typeof value === "string")
44
+ return truncate(redactText(value, homeDir));
45
+ if (typeof value === "number" || typeof value === "boolean" || value === null)
46
+ return value;
47
+ if (depth >= 3)
48
+ return "[nested]";
49
+ if (Array.isArray(value))
50
+ return value.slice(0, 20).map((v) => redactValue(v, homeDir, depth + 1));
51
+ if (typeof value === "object")
52
+ return redactArgs(value, homeDir, depth + 1);
53
+ return String(value);
54
+ }
55
+ //# sourceMappingURL=redaction.js.map
package/dist/regate.js CHANGED
@@ -4,9 +4,8 @@ import { parseVerdict, detectMisfire } from "./grade.js";
4
4
  import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
5
5
  import { evaluateTraceGates } from "./trace-gates.js";
6
6
  import { mergeTraces, deserializeTrace } from "./execution-trace.js";
7
- import { deserializeTrajectoryEvents, evaluateTrajectoryGates } from "./trajectory-gates.js";
8
7
  import { judgeOneRep } from "./regrade.js";
9
- import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, findTrajectoryFiles, tracePath, trajectoryPath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
8
+ import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
10
9
  import { outcomesToResult } from "./reps.js";
11
10
  import { appendJournal } from "./journal.js";
12
11
  import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -89,8 +88,7 @@ export async function regateRun(opts) {
89
88
  const s = specById.get(rec.id);
90
89
  const needles = hasNeedleGates(s ?? {});
91
90
  const traceGated = Boolean(s?.traceAssert);
92
- const trajectoryGated = Boolean(s?.trajectoryAssert);
93
- if (!s || (!needles && !traceGated && !trajectoryGated))
91
+ if (!s || (!needles && !traceGated))
94
92
  continue; // nothing for regate to re-decide
95
93
  if (s.assert?.vitest || s.assert?.post_test) {
96
94
  blocked.push(`${s.id}: declares ${s.assert.vitest ? "assert.vitest" : "assert.post_test"}, which needs the workspace — ` +
@@ -112,16 +110,11 @@ export async function regateRun(opts) {
112
110
  `(traces are gitignored, or capture failed) — it needs a re-run`);
113
111
  continue;
114
112
  }
115
- if (trajectoryGated && !complete(findTrajectoryFiles(opts.runDir, s.id, mode), expectedSuffixes.map((rep) => trajectoryPath(opts.runDir, s.id, mode, rep)))) {
116
- blocked.push(`${s.id}: assert.trajectory artifacts are incomplete for ${expectedReps} recorded rep(s) ` +
117
- `(events are gitignored, or required native evidence was missing) — it needs a re-run`);
118
- continue;
119
- }
120
113
  targets.push(s);
121
114
  }
122
115
  if (targets.length === 0) {
123
116
  throw new Error(`nothing to regate in ${opts.runDir}` +
124
- (blocked.length > 0 ? `:\n ${blocked.join("\n ")}` : " — no scenario declares diff_contains/diff_excludes, assert.trace, or assert.trajectory"));
117
+ (blocked.length > 0 ? `:\n ${blocked.join("\n ")}` : " — no scenario declares diff_contains/diff_excludes or assert.trace"));
125
118
  }
126
119
  const changes = [];
127
120
  let judgeCalls = 0;
@@ -134,15 +127,9 @@ export async function regateRun(opts) {
134
127
  }
135
128
  const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
136
129
  const traceFiles = findTraceFiles(opts.runDir, scenario.id, mode);
137
- const eventFiles = findTrajectoryFiles(opts.runDir, scenario.id, mode);
138
- // Reps come from whichever artifact this scenario actually has. A trajectory-only
139
- // scenario has neither diff nor trace, so falling back only once would regate
140
- // nothing and report success.
141
130
  const repKeys = diffFiles.length > 0
142
131
  ? diffFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: f }))
143
- : traceFiles.length > 0
144
- ? traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }))
145
- : eventFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
132
+ : traceFiles.map((f) => ({ rep: repIndexOf(f) ?? undefined, diffFile: undefined }));
146
133
  const outcomes = [];
147
134
  // Per scenario, not run-wide: with several regated scenarios, a global counter
148
135
  // would report every change as "re-judged" because some other scenario was.
@@ -192,61 +179,9 @@ export async function regateRun(opts) {
192
179
  }
193
180
  }
194
181
  }
195
- // Adapter-neutral trajectory gate, replayed from the saved normalized
196
- // event artifact. Native ledgers are deliberately not re-read here: the
197
- // artifact is the immutable evidence this run actually captured.
198
- let trajectoryFailure = null;
199
- if (scenario.trajectoryAssert) {
200
- const ep = trajectoryPath(opts.runDir, scenario.id, mode, rep);
201
- const events = existsSync(ep) ? deserializeTrajectoryEvents(readFileSync(ep, "utf8")) : null;
202
- // The normalized artifact contains only events that were successfully captured.
203
- // Source-read/normalization failures are persisted on the original objective;
204
- // dropping them here would let a partial ledger become PASS on replay.
205
- const priorEvidenceErrors = (rec.objective?.assertions ?? [])
206
- .filter((result) => result.kind === "trajectory_evidence" && result.status === "ERROR");
207
- if (!events) {
208
- trajectoryFailure = "objective: saved normalized events are missing, malformed, or from an unsupported version";
209
- objective = {
210
- ...(objective ?? { assertions: [] }),
211
- status: "ERROR",
212
- assertions: [...(objective?.assertions ?? []), { kind: "trajectory_evidence", status: "ERROR", detail: trajectoryFailure }],
213
- };
214
- }
215
- else {
216
- const g = evaluateTrajectoryGates(scenario.trajectoryAssert, events);
217
- const expectedHash = rec.objective?.rep_events_sha256?.[rep ?? 0] ?? (rec.reps === undefined ? rec.objective?.events_sha256 : undefined);
218
- const digestMismatch = expectedHash !== undefined && expectedHash !== g.events_sha256;
219
- const integrityAssertion = digestMismatch
220
- ? [{ kind: "trajectory_evidence", status: "ERROR", detail: "saved normalized events no longer match the hash recorded by the run" }]
221
- : [];
222
- const priorStatus = objective?.status ?? "PASS";
223
- const status = priorStatus === "ERROR" || g.status === "ERROR" || priorEvidenceErrors.length || digestMismatch
224
- ? "ERROR"
225
- : priorStatus === "NOT-MEASURED"
226
- ? "NOT-MEASURED"
227
- : priorStatus === "FAIL" || g.status === "FAIL"
228
- ? "FAIL"
229
- : "PASS";
230
- objective = {
231
- ...(objective ?? {}),
232
- status,
233
- trajectory_version: g.event_version,
234
- events_sha256: g.events_sha256,
235
- assertions: [...(objective?.assertions ?? []), ...priorEvidenceErrors, ...integrityAssertion, ...g.assertions],
236
- };
237
- if (g.status !== "PASS" || priorEvidenceErrors.length || digestMismatch) {
238
- const details = [
239
- ...priorEvidenceErrors.map((result) => result.detail),
240
- ...integrityAssertion.map((result) => result.detail),
241
- ...g.assertions.filter((result) => result.status === g.status && g.status !== "PASS").map((result) => result.detail),
242
- ];
243
- trajectoryFailure = `objective: ${details.join("; ")}`;
244
- }
245
- }
246
- }
247
182
  // Prompt delivery was observed during the subject invocation and cannot be
248
183
  // replayed by regate. Carry its objective finding while refreshing only
249
- // trace/trajectory/needle assertions.
184
+ // trace/needle assertions.
250
185
  const retainedDelivery = rec.objective?.assertions.find(assertion => assertion.kind === "skill_delivered");
251
186
  const repObservations = prev.subject_invocations?.filter(observation => observation.scenario_id === scenario.id && observation.repetition === (rep ?? 0)) ?? [];
252
187
  const delivery = prev.schema === 3
@@ -259,7 +194,7 @@ export async function regateRun(opts) {
259
194
  if (delivery.status !== "PASS")
260
195
  traceFailure = `objective: ${delivery.detail}`;
261
196
  }
262
- const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure ?? trajectoryFailure };
197
+ const gate = { lines: needleGate.lines, failure: needleGate.failure ?? traceFailure };
263
198
  const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
264
199
  const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
265
200
  // Two sources, because the two gate kinds record their prior state
package/dist/regrade.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { Spec, Scenario } from "./spec.js";
2
2
  import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
3
- import { type Judgment, type ScenarioResult, type ResultsFile } from "./results.js";
3
+ import { type ScenarioResult, type ResultsFile } from "./results.js";
4
4
  import { type RepOutcome } from "./reps.js";
5
5
  /**
6
6
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
@@ -84,6 +84,4 @@ export interface RegradeRunOptions {
84
84
  * results.yaml, emits the `score` journal event, and returns the new
85
85
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
86
86
  */
87
- /** Preserve prior panel votes and append a full-cell grade without writing invalid one-vote history. */
88
- export declare function appendJudgeHistory(prior: ScenarioResult | undefined, priorJudge: ModelRef | undefined, fresh: Omit<Judgment, "ordinal">): Judgment[] | undefined;
89
87
  export declare function regradeRun(opts: RegradeRunOptions): Promise<ResultsFile>;