@skill-harness/core 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +1 -2
  2. package/dist/adapters/types.d.ts +2 -12
  3. package/dist/capture-trace-types.d.ts +8 -102
  4. package/dist/capture-trace-types.js +0 -4
  5. package/dist/command-cost.d.ts +1 -1
  6. package/dist/command-cost.js +0 -5
  7. package/dist/execution-trace.d.ts +0 -16
  8. package/dist/execution-trace.js +31 -42
  9. package/dist/index.d.ts +3 -23
  10. package/dist/index.js +3 -23
  11. package/dist/instruction-coverage.d.ts +0 -7
  12. package/dist/instruction-coverage.js +5 -7
  13. package/dist/journal.d.ts +0 -8
  14. package/dist/metrics.d.ts +19 -0
  15. package/dist/metrics.js +28 -0
  16. package/dist/redaction.d.ts +6 -0
  17. package/dist/redaction.js +55 -0
  18. package/dist/regate.js +6 -71
  19. package/dist/regrade.d.ts +1 -3
  20. package/dist/regrade.js +1 -21
  21. package/dist/report.d.ts +1 -15
  22. package/dist/report.js +1 -13
  23. package/dist/reps.d.ts +1 -2
  24. package/dist/reps.js +33 -10
  25. package/dist/results.d.ts +10 -9
  26. package/dist/results.js +3 -19
  27. package/dist/run.js +15 -74
  28. package/dist/seeded.d.ts +1 -7
  29. package/dist/seeded.js +5 -11
  30. package/dist/sources.js +4 -9
  31. package/dist/spec.d.ts +0 -13
  32. package/dist/spec.js +6 -48
  33. package/dist/trajectory-events.d.ts +58 -0
  34. package/dist/trajectory-events.js +119 -0
  35. package/package.json +1 -1
  36. package/schemas/results-v2.schema.json +394 -75
  37. package/schemas/results-v3.schema.json +18 -0
  38. package/schemas/specification-v1.schema.json +128 -174
  39. package/dist/adjudication.d.ts +0 -219
  40. package/dist/adjudication.js +0 -422
  41. package/dist/adoption.d.ts +0 -128
  42. package/dist/adoption.js +0 -102
  43. package/dist/affected.d.ts +0 -88
  44. package/dist/affected.js +0 -222
  45. package/dist/capture.d.ts +0 -193
  46. package/dist/capture.js +0 -344
  47. package/dist/comparison.d.ts +0 -97
  48. package/dist/comparison.js +0 -189
  49. package/dist/factory-calibration.d.ts +0 -63
  50. package/dist/factory-calibration.js +0 -115
  51. package/dist/intervention-results.d.ts +0 -6
  52. package/dist/intervention-results.js +0 -52
  53. package/dist/intervention.d.ts +0 -123
  54. package/dist/intervention.js +0 -218
  55. package/dist/investigation.d.ts +0 -94
  56. package/dist/investigation.js +0 -148
  57. package/dist/judge-agreement.d.ts +0 -19
  58. package/dist/judge-agreement.js +0 -31
  59. package/dist/qualification-capture.d.ts +0 -44
  60. package/dist/qualification-capture.js +0 -163
  61. package/dist/qualification-config.d.ts +0 -233
  62. package/dist/qualification-config.js +0 -694
  63. package/dist/qualification-lock.d.ts +0 -15
  64. package/dist/qualification-lock.js +0 -87
  65. package/dist/qualification-oauth-directory.d.ts +0 -79
  66. package/dist/qualification-oauth-directory.js +0 -444
  67. package/dist/qualification-panel-store.d.ts +0 -14
  68. package/dist/qualification-panel-store.js +0 -188
  69. package/dist/qualification-panels.d.ts +0 -99
  70. package/dist/qualification-panels.js +0 -91
  71. package/dist/qualification-process.d.ts +0 -13
  72. package/dist/qualification-process.js +0 -190
  73. package/dist/qualification-runner.d.ts +0 -203
  74. package/dist/qualification-runner.js +0 -2037
  75. package/dist/qualification-store.d.ts +0 -187
  76. package/dist/qualification-store.js +0 -833
  77. package/dist/qualification-validation.d.ts +0 -12
  78. package/dist/qualification-validation.js +0 -8
  79. package/dist/screen.d.ts +0 -31
  80. package/dist/screen.js +0 -105
  81. package/dist/trajectory-gates.d.ts +0 -137
  82. package/dist/trajectory-gates.js +0 -576
  83. package/dist/trusted-host-supervision.d.ts +0 -25
  84. package/dist/trusted-host-supervision.js +0 -94
  85. package/dist/work-capture-schema.d.ts +0 -38
  86. package/dist/work-capture-schema.js +0 -33
  87. package/dist/work-capture.d.ts +0 -43
  88. package/dist/work-capture.js +0 -81
  89. package/dist/work-signals.d.ts +0 -55
  90. package/dist/work-signals.js +0 -71
  91. package/schemas/qualification-cell-result-v1.schema.json +0 -32
  92. package/schemas/qualification-config-v1.schema.json +0 -244
  93. package/schemas/qualification-invocation-request-v1.schema.json +0 -59
  94. package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
  95. package/schemas/trajectory-event-v1.schema.json +0 -80
package/dist/run.js CHANGED
@@ -1,22 +1,21 @@
1
1
  import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
2
2
  import { dirname, join, resolve } from "node:path";
3
- import { sourceHashes, PROMPT_NORMALIZATION_SOURCE_KEY, PROMPT_NORMALIZATION_SOURCE_DIGEST } from "./sources.js";
3
+ import { sourceHashes } from "./sources.js";
4
4
  import { judgeResemblesSubject } from "./grade.js";
5
5
  import { NONE_ARM, seedArmDefinitions } from "./arms.js";
6
- import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, deliveryStatusForObservations, } from "./results.js";
6
+ import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
7
7
  import { appendJournal } from "./journal.js";
8
8
  import { liftHeadline } from "./lift.js";
9
9
  import { runSeeded } from "./seeded.js";
10
10
  import { serializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
11
11
  import { evaluateTraceGates } from "./trace-gates.js";
12
- import { evaluateTrajectoryGates, serializeTrajectoryEvents, } from "./trajectory-gates.js";
13
12
  import { snapshotPaths, diffSnapshots, createWorkspace } from "./workspace.js";
14
13
  import { runPool } from "./scheduler.js";
15
14
  import { outcomesToResult } from "./reps.js";
16
15
  import { judgeOneRep } from "./regrade.js";
17
16
  import { runDeliveryCanary, canaryFailure } from "./canary.js";
18
17
  import { boundaryCells, stabilityNote } from "./stability.js";
19
- import { aggregateMetrics } from "./comparison.js";
18
+ import { aggregateMetrics } from "./metrics.js";
20
19
  import { providerFailureFromTranscript } from "./provider-failure.js";
21
20
  /**
22
21
  * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
@@ -139,20 +138,13 @@ export async function runSkillModel(opts) {
139
138
  }
140
139
  });
141
140
  const flat = await runPool(tasks, opts.concurrency ?? 1);
142
- const subjectInvocations = flat.flatMap(outcome => outcome.subject_invocations ?? []);
143
141
  const grouped = scenarios.map(() => []);
144
142
  flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
145
143
  const scenarioResults = scenarios.map((scenario, si) => {
146
144
  const threshold = scenario.critical
147
145
  ? effectiveThreshold(undefined, scenario)
148
146
  : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
149
- const result = outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
150
- if (adapter.observesPrompts)
151
- result.criterion_count = scenario.checklist.length;
152
- if (adapter.observesPrompts && !result.rep_judgments) {
153
- result.rep_judgments = grouped[si].map((outcome, repetition) => ({ repetition, judgments: [], recorded_verdict: outcome.verdict, ...(outcome.objective ? { objective: outcome.objective } : {}) }));
154
- }
155
- return result;
147
+ return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
156
148
  });
157
149
  const ctx = scoreContextFor({ mode, partial }, spec);
158
150
  const results = writeResults(runDir, {
@@ -168,12 +160,8 @@ export async function runSkillModel(opts) {
168
160
  ...(partial ? { partial: true } : {}),
169
161
  // Only the scenarios this run actually measured: a --only run must not claim
170
162
  // coverage of scenarios it skipped.
171
- source_hashes: {
172
- ...sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
173
- ...(adapter.observesPrompts ? { [PROMPT_NORMALIZATION_SOURCE_KEY]: PROMPT_NORMALIZATION_SOURCE_DIGEST } : {}),
174
- },
163
+ source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
175
164
  scenarios: scenarioResults,
176
- ...(adapter.observesPrompts ? { schema: 3, subject_invocations: subjectInvocations } : {}),
177
165
  ...(arm.name === NONE_ARM.name ? {} : {
178
166
  arm: {
179
167
  name: arm.name,
@@ -240,7 +228,6 @@ async function runRep(scenario, rep, repCount, ctx) {
240
228
  // produces no diff, and writing an empty artifact there would misreport "the
241
229
  // model changed nothing" for a rep that never ran.
242
230
  let stagedDiff = null;
243
- const subjectInvocations = [];
244
231
  try {
245
232
  try {
246
233
  ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
@@ -254,8 +241,6 @@ async function runRep(scenario, rep, repCount, ctx) {
254
241
  }
255
242
  let noResponse = false;
256
243
  let traces = [];
257
- let events = [];
258
- let eventErrors = [];
259
244
  let unobservablePaths = false;
260
245
  // The pre-run state, captured AFTER `createWorkspace` has applied the
261
246
  // fixture's `_staged/` and `_uncommitted/` trees. Those land after the
@@ -286,18 +271,17 @@ async function runRep(scenario, rep, repCount, ctx) {
286
271
  // used to pass `trace: {...}` unconditionally and let `runSeeded` throw
287
272
  // ``scenario `X` declares `assert.trace`…`` at a scenario that declares no
288
273
  // such thing.
289
- const needsStructuredEvidence = Boolean(scenario.traceAssert || scenario.trajectoryAssert);
274
+ const needsStructuredEvidence = Boolean(scenario.traceAssert);
290
275
  if (needsStructuredEvidence && !ctx.adapter.runStructured) {
291
276
  throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
292
- ` cannot produce execution traces/events — the gate would have no evidence to read.`);
277
+ ` cannot produce execution traces — the gate would have no evidence to read.`);
293
278
  }
294
- const useStructured = (Boolean(ctx.structured) || needsStructuredEvidence) && Boolean(ctx.adapter.runStructured);
279
+ const useStructured = (Boolean(ctx.structured) || needsStructuredEvidence || Boolean(ctx.adapter.preferStructured)) && Boolean(ctx.adapter.runStructured);
295
280
  // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
296
281
  // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
297
282
  // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
298
283
  // An adapter that THREW is the same class of event and takes the same retry.
299
284
  for (let attempt = 0; attempt < 2; attempt++) {
300
- const observe = (prompt) => subjectInvocations.push({ scenario_id: scenario.id, repetition: rep, attempt, prompt });
301
285
  if (attempt > 0) {
302
286
  const why = adapterFailure ? `adapter failed (${adapterFailure})` : "empty response";
303
287
  appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, reason: why, ...repField });
@@ -337,15 +321,12 @@ async function runRep(scenario, rep, repCount, ctx) {
337
321
  // already substituted) both must reach pi.
338
322
  armExtensions: arm.extensions,
339
323
  ...(armEnvFor(ws.cwd) ? { armEnv: armEnvFor(ws.cwd) } : {}),
340
- ...(ctx.adapter.observesPrompts ? { onPromptObservation: observe } : {}),
341
324
  });
342
325
  transcript = r.transcript;
343
326
  gatePrefix = r.gateFailure;
344
327
  infrastructureFailure = r.gateError;
345
328
  stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
346
329
  traces = r.traces;
347
- events = r.events;
348
- eventErrors = r.eventErrors;
349
330
  }
350
331
  else {
351
332
  const req = {
@@ -361,16 +342,12 @@ async function runRep(scenario, rep, repCount, ctx) {
361
342
  ...(scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)) ?? []),
362
343
  ...arm.extensions,
363
344
  ],
364
- eventSources: scenario.eventSources,
365
345
  ...(armEnvFor(ws.cwd) ? { armEnv: armEnvFor(ws.cwd) } : {}),
366
- ...(ctx.adapter.observesPrompts ? { onPromptObservation: observe } : {}),
367
346
  };
368
347
  if (useStructured) {
369
348
  const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
370
349
  transcript = structured.transcript;
371
350
  traces = structured.traces;
372
- events = structured.events ?? [];
373
- eventErrors = structured.eventErrors ?? [];
374
351
  if (structured.providerFailure)
375
352
  infrastructureFailure = `provider failure — ${structured.providerFailure}`;
376
353
  }
@@ -399,8 +376,6 @@ async function runRep(scenario, rep, repCount, ctx) {
399
376
  gatePrefix = null;
400
377
  stagedDiff = null;
401
378
  traces = [];
402
- events = [];
403
- eventErrors = [];
404
379
  }
405
380
  // A provider outage is infrastructure, never a model verdict. Checked on
406
381
  // every path: the text path carries the marker in the transcript, the
@@ -457,16 +432,6 @@ async function runRep(scenario, rep, repCount, ctx) {
457
432
  });
458
433
  }
459
434
  }
460
- // Delivery is an objective gate on every Pi subject invocation. It is derived
461
- // from captured provider payload bytes, never from mode/argv supplied by a caller.
462
- let deliveryObjective;
463
- if (ctx.adapter.observesPrompts) {
464
- const statuses = subjectInvocations.map(observation => observation.prompt.status);
465
- const status = deliveryStatusForObservations(subjectInvocations);
466
- deliveryObjective = { status, assertions: [{ kind: "skill_delivered", status, detail: statuses.length === 0 ? "no model-visible prompt observation was retained" : `${statuses.length} provider request(s): ${statuses.join(", ")}` }] };
467
- if (status !== "PASS")
468
- gatePrefix = `objective: skill_delivered ${status.toLowerCase()} — ${deliveryObjective.assertions[0].detail}`;
469
- }
470
435
  // Objective evidence is persisted for every rep, pass or fail — a failing gate
471
436
  // is exactly when someone wants to read what the model actually did.
472
437
  // `!adapterFailure`: a rep whose adapter threw never ran, so there is nothing
@@ -474,12 +439,11 @@ async function runRep(scenario, rep, repCount, ctx) {
474
439
  // which is the symptom — it names the missing artifact instead of the crash or
475
440
  // timeout that caused it, and it is that text, not the real reason, that would
476
441
  // reach the results record. The verdict is ERROR either way.
477
- let objective = deliveryObjective;
478
- if ((scenario.traceAssert || scenario.trajectoryAssert) && !adapterFailure) {
442
+ let objective;
443
+ if (scenario.traceAssert && !adapterFailure) {
479
444
  const assertionResults = [];
480
445
  let status = "PASS";
481
446
  let traceMeta = {};
482
- let trajectoryMeta = {};
483
447
  if (scenario.traceAssert) {
484
448
  if (traces.length > 0) {
485
449
  writeFileSync(tracePath(runDir, scenario.id, mode, repSuffix), traces.map(serializeTrace).join(""), "utf8");
@@ -500,32 +464,7 @@ async function runRep(scenario, rep, repCount, ctx) {
500
464
  traceMeta = { trace_version: merged.trace_version, trace_sha256: merged.trace_sha256 };
501
465
  }
502
466
  }
503
- if (scenario.trajectoryAssert) {
504
- if (events.length > 0) {
505
- writeFileSync(trajectoryPath(runDir, scenario.id, mode, repSuffix), serializeTrajectoryEvents(events), "utf8");
506
- }
507
- if (eventErrors.length > 0) {
508
- status = "ERROR";
509
- assertionResults.push(...eventErrors.map((detail) => ({ kind: "trajectory_evidence", status: "ERROR", detail })));
510
- }
511
- else if (events.length === 0) {
512
- status = "ERROR";
513
- assertionResults.push({ kind: "trajectory_evidence", status: "ERROR", detail: "no normalized workflow events were produced" });
514
- }
515
- else {
516
- const gate = evaluateTrajectoryGates(scenario.trajectoryAssert, events);
517
- if (gate.status === "ERROR" || (gate.status === "FAIL" && status === "PASS"))
518
- status = gate.status;
519
- assertionResults.push(...gate.assertions);
520
- trajectoryMeta = { trajectory_version: gate.event_version, events_sha256: gate.events_sha256 };
521
- }
522
- }
523
- if (deliveryObjective) {
524
- if (deliveryObjective.status === "ERROR" || (deliveryObjective.status === "NOT-MEASURED" && status !== "ERROR") || (deliveryObjective.status === "FAIL" && status === "PASS"))
525
- status = deliveryObjective.status;
526
- assertionResults.unshift(...deliveryObjective.assertions);
527
- }
528
- objective = { status, ...traceMeta, ...trajectoryMeta, assertions: assertionResults };
467
+ objective = { status, ...traceMeta, assertions: assertionResults };
529
468
  if (status !== "PASS") {
530
469
  const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
531
470
  gatePrefix = `objective: ${details.join("; ") || "structured evidence could not be evaluated"}`;
@@ -580,7 +519,7 @@ async function runRep(scenario, rep, repCount, ctx) {
580
519
  log(` → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? " ⚠ suspect" : ""}`);
581
520
  const subject = mergeTraces(traces)?.metrics;
582
521
  return {
583
- verdict, reason, suspect, objective, judgment, subject_invocations: subjectInvocations,
522
+ verdict, reason, suspect, objective, judgment,
584
523
  metrics: {
585
524
  wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
586
525
  judge_calls: judgeCalls,
@@ -642,7 +581,9 @@ export function formatScorecard(summary, lift, stability) {
642
581
  ? " · WARNING: subject tokens were used but the provider reported $0 cost"
643
582
  : metrics.subject_cost_usd === null
644
583
  ? " · subject cost unavailable"
645
- : ` · $${metrics.subject_cost_usd.toFixed(6)} provider-reported`;
584
+ : metrics.cost_source === "price-table"
585
+ ? ` · $${metrics.subject_cost_usd.toFixed(6)} price table`
586
+ : ` · $${metrics.subject_cost_usd.toFixed(6)} provider-reported`;
646
587
  lines.push(` COST: ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported)${cost} · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
647
588
  }
648
589
  // Lift is a statement about a skill-delivered run (green or force). On a red run
package/dist/seeded.d.ts CHANGED
@@ -1,7 +1,6 @@
1
1
  import type { Scenario } from "./spec.js";
2
2
  import type { ExecutionTraceV1 } from "./capture-trace-types.js";
3
- import type { TrajectoryEventV1 } from "./trajectory-gates.js";
4
- import type { HarnessAdapter, ModelRef, RunMode, PromptProvenance } from "./adapters/types.js";
3
+ import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
5
4
  import { type ExecResult } from "./util/exec.js";
6
5
  interface SeededOpts {
7
6
  skillDir: string;
@@ -37,7 +36,6 @@ interface SeededOpts {
37
36
  armExtensions?: string[];
38
37
  /** The arm's env for the subject process; `<run-dir>` already substituted by the caller. */
39
38
  armEnv?: Record<string, string>;
40
- onPromptObservation?: (observation: PromptProvenance) => void;
41
39
  }
42
40
  /**
43
41
  * Result of one vitest invocation — deliberately `ExecResult`, not a narrower
@@ -58,10 +56,6 @@ export interface SeededOutcome {
58
56
  diff: string;
59
57
  /** One per turn; empty unless structured execution was requested. */
60
58
  traces: ExecutionTraceV1[];
61
- /** Adapter-neutral workflow/tool events for assert.trajectory. */
62
- events: TrajectoryEventV1[];
63
- /** Required native sources that could not be read or normalized. */
64
- eventErrors: string[];
65
59
  }
66
60
  /**
67
61
  * The added/removed lines of a unified diff — what the model actually *changed*,
package/dist/seeded.js CHANGED
@@ -173,17 +173,13 @@ export async function runSeeded(scenario, opts) {
173
173
  ...(scenario.extensions?.map((e) => resolve(opts.specDir, e)) ?? []),
174
174
  ...(opts.armExtensions ?? []),
175
175
  ],
176
- eventSources: scenario.eventSources,
177
176
  ...(opts.armEnv ? { armEnv: opts.armEnv } : {}),
178
- ...(opts.onPromptObservation ? { onPromptObservation: opts.onPromptObservation } : {}),
179
177
  };
180
178
  // A trace-gated seeded scenario runs through the structured path so the tool
181
179
  // calls are recorded; everything downstream (gates, diff, transcript) is
182
180
  // identical, because the rebuilt transcript is what print mode would have
183
181
  // emitted anyway.
184
182
  let traces = [];
185
- let events = [];
186
- let eventErrors = [];
187
183
  let harnessOut;
188
184
  if (opts.trace) {
189
185
  if (!opts.adapter.runStructured) {
@@ -197,8 +193,6 @@ export async function runSeeded(scenario, opts) {
197
193
  });
198
194
  harnessOut = structured.transcript;
199
195
  traces = structured.traces;
200
- events = structured.events ?? [];
201
- eventErrors = structured.eventErrors ?? [];
202
196
  }
203
197
  else {
204
198
  harnessOut = await opts.adapter.run(req);
@@ -227,7 +221,7 @@ export async function runSeeded(scenario, opts) {
227
221
  parts.push(` staged diff: ERROR (${msg})`);
228
222
  gateFailure = msg;
229
223
  gateError = msg;
230
- return finish(parts, gateFailure, gateError, diff, traces, events, eventErrors);
224
+ return finish(parts, gateFailure, gateError, diff, traces);
231
225
  }
232
226
  // BOTH needle gates read the changed lines only, never context. A unified diff
233
227
  // carries three lines of context per hunk, so an untouched symbol near the edit
@@ -301,7 +295,7 @@ export async function runSeeded(scenario, opts) {
301
295
  if (!gateFailure)
302
296
  gateFailure = msg;
303
297
  gateError = msg;
304
- return finish(parts, gateFailure, gateError, diff, traces, events, eventErrors);
298
+ return finish(parts, gateFailure, gateError, diff, traces);
305
299
  }
306
300
  const v = await runVitest([POST_TEST_BASE], repo);
307
301
  const out = `${v.stdout}\n${v.stderr}`;
@@ -358,7 +352,7 @@ export async function runSeeded(scenario, opts) {
358
352
  gateError = problem;
359
353
  }
360
354
  }
361
- return finish(parts, gateFailure, gateError, diff, traces, events, eventErrors);
355
+ return finish(parts, gateFailure, gateError, diff, traces);
362
356
  }
363
357
  function git(cwd, args) {
364
358
  return exec("git", args, { cwd, timeoutMs: 30_000 });
@@ -392,13 +386,13 @@ function bothStreams(v) {
392
386
  return o || e;
393
387
  }
394
388
  /** Append the staged diff and return the outcome. Every exit path goes through here, so the judge always sees the same sections in the same order. */
395
- function finish(parts, gateFailure, gateError, diff, traces = [], events = [], eventErrors = []) {
389
+ function finish(parts, gateFailure, gateError, diff, traces = []) {
396
390
  // The code itself, last — the gates above only prove that keywords appeared.
397
391
  // Without this section a seeded checklist item about what the code *does* is
398
392
  // graded from the model's own description of its work.
399
393
  parts.push("", "=== STAGED DIFF ===");
400
394
  parts.push(diff.trim() === "" ? " (empty — the model left no staged changes)" : capDiff(diff));
401
- return { transcript: parts.join("\n"), gateFailure, gateError, diff, traces, events, eventErrors };
395
+ return { transcript: parts.join("\n"), gateFailure, gateError, diff, traces };
402
396
  }
403
397
  /**
404
398
  * Parse vitest's `Tests N passed | M skipped (T)` summary line.
package/dist/sources.js CHANGED
@@ -170,13 +170,13 @@ function walk(dir, prefix = "") {
170
170
  * transcripts are actually invalid.
171
171
  */
172
172
  function facets(s) {
173
- const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, trajectoryAssert, workspace, remote, systemPromptFile, extensions, eventSources, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
173
+ const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
174
174
  const _scenarioExhaustive = restScenario;
175
175
  void _scenarioExhaustive;
176
176
  // `covers` is destructured into a discard on purpose, and this comment is the
177
177
  // decision the guard demanded: it belongs to NO digest. It records which
178
- // instruction sections a scenario is declared to exercise, which changes what
179
- // `--affected` selects next time — not what any past run measured. Bucketing it
178
+ // instruction sections a scenario is declared to exercise; it does not change
179
+ // what any past run measured. Bucketing it
180
180
  // anywhere would charge a re-run (or at best a re-score) for editing a label,
181
181
  // which is the exact trap the facet split was built to remove.
182
182
  void _coversIsMetadata;
@@ -187,7 +187,7 @@ function facets(s) {
187
187
  // already saved, so `regate` can re-answer it without re-running the subject model.
188
188
  // Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
189
189
  // changes what gets executed, the other only what we conclude from it.
190
- const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined || trajectoryAssert !== undefined;
190
+ const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined;
191
191
  return {
192
192
  // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
193
193
  // run executes in the workspace, and neither can be re-evaluated from a saved
@@ -206,10 +206,6 @@ function facets(s) {
206
206
  id, mode, turns, workspace, remote, systemPromptFile ?? null,
207
207
  fixture ?? null, vitest ?? null, post_test ?? null,
208
208
  ...(extensions ? [extensions] : []),
209
- // Event-source paths choose which native ledgers are captured. A saved
210
- // normalized artifact cannot answer for a source that was never collected,
211
- // so changing this is stimulus and needs a re-run.
212
- ...(eventSources ? [eventSources] : []),
213
209
  ]),
214
210
  rubric: JSON.stringify([id, title, checklist]),
215
211
  policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
@@ -219,7 +215,6 @@ function facets(s) {
219
215
  ? JSON.stringify([
220
216
  id, diff_contains ?? null, diff_excludes ?? null,
221
217
  ...(traceAssert ? [traceAssert] : []),
222
- ...(trajectoryAssert ? [trajectoryAssert] : []),
223
218
  ])
224
219
  : null,
225
220
  };
package/dist/spec.d.ts CHANGED
@@ -1,6 +1,5 @@
1
1
  import type { WorkspaceKind } from "./workspace.js";
2
2
  import { type TraceAssert } from "./trace-gates.js";
3
- import { type TrajectoryAssert } from "./trajectory-gates.js";
4
3
  export type ScenarioMode = "inline" | "seeded";
5
4
  export interface SeededAssert {
6
5
  vitest?: boolean;
@@ -21,14 +20,6 @@ export interface SeededAssert {
21
20
  */
22
21
  post_test?: string;
23
22
  }
24
- export interface TrajectoryEventSource {
25
- /** Native format adapter registered by the harness adapter. */
26
- adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1" | "pi-daddy-ledger-v3";
27
- /** Workspace-relative file/glob. Never an absolute path or traversal. */
28
- path: string;
29
- /** Missing evidence is an error by default; false is for optional supplemental ledgers. */
30
- required: boolean;
31
- }
32
23
  export interface Scenario {
33
24
  id: string;
34
25
  title: string;
@@ -46,10 +37,6 @@ export interface Scenario {
46
37
  * Declaring it opts the scenario into structured (`--mode json`) execution.
47
38
  */
48
39
  traceAssert?: TraceAssert;
49
- /** Objective assertions over adapter-neutral, versioned workflow events. */
50
- trajectoryAssert?: TrajectoryAssert;
51
- /** Native event/ledger files the harness normalizes after execution. */
52
- eventSources?: TrajectoryEventSource[];
53
40
  workspace: WorkspaceKind;
54
41
  remote: boolean;
55
42
  systemPromptFile?: string;
package/dist/spec.js CHANGED
@@ -1,7 +1,6 @@
1
1
  import { readFileSync } from "node:fs";
2
2
  import yaml from "js-yaml";
3
3
  import { parseTraceAssert } from "./trace-gates.js";
4
- import { parseTrajectoryAssert } from "./trajectory-gates.js";
5
4
  /** Thrown on any validation failure. Message always carries the spec file path. */
6
5
  export class SpecError extends Error {
7
6
  constructor(message, file) {
@@ -89,46 +88,6 @@ function resolveExtensions(env, hasSystemPrompt, id, file) {
89
88
  * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
90
89
  * not something to silently ignore.
91
90
  */
92
- function resolveEventSources(env, id, file) {
93
- const raw = env && typeof env === "object" ? env.event_sources : undefined;
94
- if (raw === undefined)
95
- return undefined;
96
- if (!Array.isArray(raw) || raw.length === 0) {
97
- throw new SpecError(`scenario \`${id}\` env.event_sources must be a non-empty list`, file);
98
- }
99
- const allowedAdapters = new Set([
100
- "normalized-v1", "principal-assurance-v1", "pi-daddy-v1", "pi-daddy-ledger-v3",
101
- ]);
102
- return raw.map((entry, index) => {
103
- if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
104
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] must be a mapping`, file);
105
- }
106
- const source = entry;
107
- for (const key of Object.keys(source)) {
108
- if (!["adapter", "path", "required"].includes(key)) {
109
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] has unknown key \`${key}\``, file);
110
- }
111
- }
112
- if (!allowedAdapters.has(source.adapter)) {
113
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].adapter is unsupported`, file);
114
- }
115
- if (typeof source.path !== "string" || !source.path.trim()) {
116
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].path must be non-empty`, file);
117
- }
118
- const path = source.path.trim();
119
- if (path.startsWith("/") || /^[A-Za-z]:[\\/]/.test(path) || path.includes("\\") || path.split("/").some((part) => part === ".." || part === "." || part === "")) {
120
- throw new SpecError(`scenario \`${id}\` env.event_sources paths must be workspace-relative without traversal (got \`${path}\`)`, file);
121
- }
122
- if (source.required !== undefined && typeof source.required !== "boolean") {
123
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].required must be true or false`, file);
124
- }
125
- return {
126
- adapter: source.adapter,
127
- path,
128
- required: source.required !== false,
129
- };
130
- });
131
- }
132
91
  function resolveRemote(env, workspace, id, file) {
133
92
  const raw = env && typeof env === "object" ? env.remote : undefined;
134
93
  if (raw === undefined)
@@ -220,12 +179,12 @@ export function parseSpec(text, file) {
220
179
  // `assert.trace` is legal for inline AND seeded scenarios — it reads the
221
180
  // execution trace, which every run produces, not a staged diff.
222
181
  const rawAssert = s.assert;
182
+ if (rawAssert?.trajectory !== undefined) {
183
+ throw new SpecError(`scenario \`${id}\` uses removed \`assert.trajectory\`; delete it or replace it with an active objective gate`, file);
184
+ }
223
185
  if (rawAssert?.trace !== undefined) {
224
186
  scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
225
187
  }
226
- if (rawAssert?.trajectory !== undefined) {
227
- scenario.trajectoryAssert = parseTrajectoryAssert(rawAssert.trajectory, `${file}: scenario \`${id}\``);
228
- }
229
188
  if (mode === "seeded") {
230
189
  if (typeof s.fixture !== "string" || s.fixture.length === 0) {
231
190
  throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
@@ -276,12 +235,11 @@ export function parseSpec(text, file) {
276
235
  scenario.assert = assertObj;
277
236
  }
278
237
  }
238
+ if (s.env && typeof s.env === "object" && Object.hasOwn(s.env, "event_sources")) {
239
+ throw new SpecError(`scenario \`${id}\` uses removed \`env.event_sources\`; it is no longer collected`, file);
240
+ }
279
241
  scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
280
242
  scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
281
- scenario.eventSources = resolveEventSources(s.env, id, file);
282
- if (scenario.eventSources && !scenario.trajectoryAssert) {
283
- throw new SpecError(`scenario \`${id}\` declares env.event_sources without assert.trajectory`, file);
284
- }
285
243
  if (s.system_prompt_file !== undefined) {
286
244
  if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
287
245
  throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
@@ -0,0 +1,58 @@
1
+ export declare const LEGACY_TRAJECTORY_EVENT_VERSION: "1.0";
2
+ export declare const TRAJECTORY_EVENT_VERSION: "1.1";
3
+ export type TrajectoryEventVersion = typeof LEGACY_TRAJECTORY_EVENT_VERSION | typeof TRAJECTORY_EVENT_VERSION;
4
+ export interface TrajectoryEventSource {
5
+ adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1" | "pi-daddy-ledger-v3";
6
+ path: string;
7
+ required: boolean;
8
+ }
9
+ export interface TrajectoryApproval {
10
+ id?: string;
11
+ capability?: string;
12
+ subject?: string;
13
+ source?: string;
14
+ scope?: string;
15
+ approved_at?: string;
16
+ expires_at?: string;
17
+ used_at?: string;
18
+ }
19
+ export interface TrajectoryDigests {
20
+ plan?: string;
21
+ task?: string;
22
+ definition?: string;
23
+ head?: string;
24
+ tree?: string;
25
+ [key: string]: string | undefined;
26
+ }
27
+ /** Normalized pi-daddy/principal event retained by adapter consumers. */
28
+ export interface TrajectoryEventV1 {
29
+ event_version: TrajectoryEventVersion;
30
+ seq: number;
31
+ type: string;
32
+ source: string;
33
+ at?: string;
34
+ run_id?: string;
35
+ task_id?: string;
36
+ workspace_id?: string;
37
+ context_id?: string;
38
+ finding_id?: string;
39
+ parent_id?: string;
40
+ child_id?: string;
41
+ execution_id?: string;
42
+ parent_execution_id?: string | null;
43
+ task_from_execution_id?: string;
44
+ workflow_fact_id?: string;
45
+ deadline_at?: string;
46
+ phase?: string;
47
+ tool?: string;
48
+ capability?: string;
49
+ requested_capabilities?: string[];
50
+ effective_capabilities?: string[];
51
+ refusal_code?: string;
52
+ exit_code?: number;
53
+ digests?: TrajectoryDigests;
54
+ approval?: TrajectoryApproval;
55
+ requirements?: string[];
56
+ attributes?: Record<string, unknown>;
57
+ }
58
+ export declare function deserializeTrajectoryEvents(text: string): TrajectoryEventV1[] | null;