@skill-harness/core 0.8.0 → 0.10.0

package/dist/seeded.js

@@ -168,12 +168,15 @@ export async function runSeeded(scenario, opts) {
         cwd: repo,
         // Resolved against the spec dir, exactly like fixtures and post-tests.
         extensions: scenario.extensions?.map((e) => resolve(opts.specDir, e)),
+        eventSources: scenario.eventSources,
     };
     // A trace-gated seeded scenario runs through the structured path so the tool
     // calls are recorded; everything downstream (gates, diff, transcript) is
     // identical, because the rebuilt transcript is what print mode would have
     // emitted anyway.
     let traces = [];
+    let events = [];
+    let eventErrors = [];
     let harnessOut;
     if (opts.trace) {
         if (!opts.adapter.runStructured) {
@@ -187,12 +190,15 @@ export async function runSeeded(scenario, opts) {
         });
         harnessOut = structured.transcript;
         traces = structured.traces;
+        events = structured.events ?? [];
+        eventErrors = structured.eventErrors ?? [];
     }
     else {
         harnessOut = await opts.adapter.run(req);
     }
     const parts = [harnessOut, "", "=== SEEDED GATES ==="];
     let gateFailure = null;
+    let gateError = null;
     const runVitest = opts.runVitest ??
         ((args, cwd) => exec("npx", ["vitest", "run", ...args], { cwd, timeoutMs: VITEST_TIMEOUT_MS }));
     // `exec` never throws on a non-zero exit and, on timeout, SIGKILLs the child and
@@ -213,7 +219,8 @@ export async function runSeeded(scenario, opts) {
             (gitFailure.stderr.trim() ? `: ${gitFailure.stderr.trim().split("\n")[0]}` : "");
         parts.push(`  staged diff: ERROR (${msg})`);
         gateFailure = msg;
-        return finish(parts, gateFailure, diff, traces);
+        gateError = msg;
+        return finish(parts, gateFailure, gateError, diff, traces, events, eventErrors);
     }
     // BOTH needle gates read the changed lines only, never context. A unified diff
     // carries three lines of context per hunk, so an untouched symbol near the edit
@@ -241,10 +248,14 @@ export async function runSeeded(scenario, opts) {
             ? `  vitest run: ERROR (timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior)`
             : `  vitest run: ${passed ? "PASS" : `FAIL (exit ${v.code})`}`);
         parts.push(indent(bothStreams(v)));
-        if (!passed && !gateFailure) {
-            gateFailure = killed
+        if (!passed) {
+            const problem = killed
                 ? `vitest timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior`
                 : `vitest failed (exit ${v.code})`;
+            if (!gateFailure)
+                gateFailure = problem;
+            if (killed)
+                gateError = problem;
         }
     }
     const postTest = scenario.assert?.post_test;
@@ -263,6 +274,7 @@ export async function runSeeded(scenario, opts) {
             parts.push(`  post_test: ERROR (${msg})`);
             if (!gateFailure)
                 gateFailure = msg;
+            gateError = msg;
         }
         else {
             // `.test` is part of the name we build, never taken from the source: extname()
@@ -281,7 +293,8 @@ export async function runSeeded(scenario, opts) {
                 parts.push(`  post_test: ERROR (${msg})`);
                 if (!gateFailure)
                     gateFailure = msg;
-                return finish(parts, gateFailure, diff, traces);
+                gateError = msg;
+                return finish(parts, gateFailure, gateError, diff, traces, events, eventErrors);
             }
             const v = await runVitest([POST_TEST_BASE], repo);
             const out = `${v.stdout}\n${v.stderr}`;
@@ -300,13 +313,17 @@ export async function runSeeded(scenario, opts) {
             const notCollected = /^\s*No test files found/im.test(out);
             const killed = v.code === null;
             let problem = null;
+            let problemIsError = false;
             if (killed) {
+                problemIsError = true;
                 problem = `post_test ${JSON.stringify(postTest)} timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior`;
             }
             else if (notCollected) {
+                problemIsError = true;
                 problem = `post_test ${JSON.stringify(postTest)} was never collected by vitest — spec/fixture error, not model behavior`;
             }
             else if (tally === null) {
+                problemIsError = true;
                 problem = `post_test ${JSON.stringify(postTest)} produced no parseable vitest summary (exit ${v.code}) — cannot confirm it ran`;
             }
             else if (v.code !== 0 || tally.failed > 0) {
@@ -317,20 +334,24 @@ export async function runSeeded(scenario, opts) {
                 problem = `post_test ${JSON.stringify(postTest)} failed (exit ${v.code})`;
             }
             else if (tally.skipped > 0 || tally.todo > 0) {
+                problemIsError = true;
                 problem = `post_test ${JSON.stringify(postTest)} has ${tally.skipped + tally.todo} skipped/todo test(s) — a hidden gate must actually run; spec error, not model behavior`;
             }
             else if (tally.passed === 0) {
+                problemIsError = true;
                 problem = `post_test ${JSON.stringify(postTest)} ran no assertions — spec error, not model behavior`;
             }
             parts.push(problem === null
                 ? `  post_test ${JSON.stringify(postTest)}: PASS (${tally.passed} assertion-bearing test(s))`
-                : `  post_test ${JSON.stringify(postTest)}: ${v.code === 0 && !killed ? "ERROR" : "FAIL"} (${problem})`);
+                : `  post_test ${JSON.stringify(postTest)}: ${problemIsError ? "ERROR" : "FAIL"} (${problem})`);
             parts.push(indent(bothStreams(v)));
             if (problem && !gateFailure)
                 gateFailure = problem;
+            if (problem && problemIsError)
+                gateError = problem;
         }
     }
-    return finish(parts, gateFailure, diff, traces);
+    return finish(parts, gateFailure, gateError, diff, traces, events, eventErrors);
 }
 function git(cwd, args) {
     return exec("git", args, { cwd, timeoutMs: 30_000 });
@@ -364,13 +385,13 @@ function bothStreams(v) {
     return o || e;
 }
 /** Append the staged diff and return the outcome. Every exit path goes through here, so the judge always sees the same sections in the same order. */
-function finish(parts, gateFailure, diff, traces = []) {
+function finish(parts, gateFailure, gateError, diff, traces = [], events = [], eventErrors = []) {
     // The code itself, last — the gates above only prove that keywords appeared.
     // Without this section a seeded checklist item about what the code *does* is
     // graded from the model's own description of its work.
     parts.push("", "=== STAGED DIFF ===");
     parts.push(diff.trim() === "" ? "  (empty — the model left no staged changes)" : capDiff(diff));
-    return { transcript: parts.join("\n"), gateFailure, diff, traces };
+    return { transcript: parts.join("\n"), gateFailure, gateError, diff, traces, events, eventErrors };
 }
 /**
  * Parse vitest's `Tests  N passed | M skipped (T)` summary line.

package/dist/sources.js

@@ -168,7 +168,7 @@ function walk(dir, prefix = "") {
  * transcripts are actually invalid.
  */
 function facets(s) {
-    const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
+    const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, trajectoryAssert, workspace, remote, systemPromptFile, extensions, eventSources, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
     const _scenarioExhaustive = restScenario;
     void _scenarioExhaustive;
     // `covers` is destructured into a discard on purpose, and this comment is the
@@ -185,7 +185,7 @@ function facets(s) {
     // already saved, so `regate` (free) can re-answer it without re-running the model.
     // Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
     // changes what gets executed, the other only what we conclude from it.
-    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined;
+    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined || trajectoryAssert !== undefined;
     return {
         // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
         // run executes in the workspace, and neither can be re-evaluated from a saved
@@ -204,13 +204,21 @@ function facets(s) {
             id, mode, turns, workspace, remote, systemPromptFile ?? null,
             fixture ?? null, vitest ?? null, post_test ?? null,
             ...(extensions ? [extensions] : []),
+            // Event-source paths choose which native ledgers are captured. A saved
+            // normalized artifact cannot answer for a source that was never collected,
+            // so changing this is stimulus and needs a re-run.
+            ...(eventSources ? [eventSources] : []),
         ]),
         rubric: JSON.stringify([id, title, checklist]),
         policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
         // Same rule as `stimulus` above: conditional, so a needle-gated scenario that
         // declares no trace assertions keeps the digest it was published with.
         gates: hasGates
-            ? JSON.stringify([id, diff_contains ?? null, diff_excludes ?? null, ...(traceAssert ? [traceAssert] : [])])
+            ? JSON.stringify([
+                id, diff_contains ?? null, diff_excludes ?? null,
+                ...(traceAssert ? [traceAssert] : []),
+                ...(trajectoryAssert ? [trajectoryAssert] : []),
+            ])
             : null,
     };
 }

package/dist/spec.d.ts

@@ -1,5 +1,6 @@
 import type { WorkspaceKind } from "./workspace.js";
 import { type TraceAssert } from "./trace-gates.js";
+import { type TrajectoryAssert } from "./trajectory-gates.js";
 export type ScenarioMode = "inline" | "seeded";
 export interface SeededAssert {
     vitest?: boolean;
@@ -20,6 +21,14 @@ export interface SeededAssert {
      */
     post_test?: string;
 }
+export interface TrajectoryEventSource {
+    /** Native format adapter registered by the harness adapter. */
+    adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1";
+    /** Workspace-relative file/glob. Never an absolute path or traversal. */
+    path: string;
+    /** Missing evidence is an error by default; false is for optional supplemental ledgers. */
+    required: boolean;
+}
 export interface Scenario {
     id: string;
     title: string;
@@ -37,6 +46,10 @@ export interface Scenario {
      * Declaring it opts the scenario into structured (`--mode json`) execution.
      */
     traceAssert?: TraceAssert;
+    /** Objective assertions over adapter-neutral, versioned workflow events. */
+    trajectoryAssert?: TrajectoryAssert;
+    /** Native event/ledger files the harness normalizes after execution. */
+    eventSources?: TrajectoryEventSource[];
     workspace: WorkspaceKind;
     remote: boolean;
     systemPromptFile?: string;
@@ -69,6 +82,8 @@ export interface ShipBar {
     no_critical_fail: boolean;
 }
 export interface Spec {
+    /** Specification schema. Omitted YAML is read as v1 for 0.8 compatibility. */
+    schema?: 1;
     skill: string;
     judge_persona: string;
     ship_bar: ShipBar;

package/dist/spec.js

@@ -1,6 +1,7 @@
 import { readFileSync } from "node:fs";
 import yaml from "js-yaml";
 import { parseTraceAssert } from "./trace-gates.js";
+import { parseTrajectoryAssert } from "./trajectory-gates.js";
 /** Thrown on any validation failure. Message always carries the spec file path. */
 export class SpecError extends Error {
     constructor(message, file) {
@@ -88,6 +89,46 @@ function resolveExtensions(env, hasSystemPrompt, id, file) {
  * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
  * not something to silently ignore.
  */
+function resolveEventSources(env, id, file) {
+    const raw = env && typeof env === "object" ? env.event_sources : undefined;
+    if (raw === undefined)
+        return undefined;
+    if (!Array.isArray(raw) || raw.length === 0) {
+        throw new SpecError(`scenario \`${id}\` env.event_sources must be a non-empty list`, file);
+    }
+    const allowedAdapters = new Set([
+        "normalized-v1", "principal-assurance-v1", "pi-daddy-v1",
+    ]);
+    return raw.map((entry, index) => {
+        if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] must be a mapping`, file);
+        }
+        const source = entry;
+        for (const key of Object.keys(source)) {
+            if (!["adapter", "path", "required"].includes(key)) {
+                throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] has unknown key \`${key}\``, file);
+            }
+        }
+        if (!allowedAdapters.has(source.adapter)) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].adapter is unsupported`, file);
+        }
+        if (typeof source.path !== "string" || !source.path.trim()) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].path must be non-empty`, file);
+        }
+        const path = source.path.trim();
+        if (path.startsWith("/") || /^[A-Za-z]:[\\/]/.test(path) || path.includes("\\") || path.split("/").some((part) => part === ".." || part === "." || part === "")) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources paths must be workspace-relative without traversal (got \`${path}\`)`, file);
+        }
+        if (source.required !== undefined && typeof source.required !== "boolean") {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].required must be true or false`, file);
+        }
+        return {
+            adapter: source.adapter,
+            path,
+            required: source.required !== false,
+        };
+    });
+}
 function resolveRemote(env, workspace, id, file) {
     const raw = env && typeof env === "object" ? env.remote : undefined;
     if (raw === undefined)
@@ -114,6 +155,9 @@ export function parseSpec(text, file) {
         throw new SpecError("spec must be a YAML mapping", file);
     }
     const o = doc;
+    if (o.schema !== undefined && o.schema !== 1) {
+        throw new SpecError(`unsupported \`schema\` ${JSON.stringify(o.schema)} (expected 1)`, file);
+    }
     if (typeof o.skill !== "string" || o.skill.length === 0) {
         throw new SpecError("missing or invalid `skill` (string)", file);
     }
@@ -179,6 +223,9 @@ export function parseSpec(text, file) {
         if (rawAssert?.trace !== undefined) {
             scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
         }
+        if (rawAssert?.trajectory !== undefined) {
+            scenario.trajectoryAssert = parseTrajectoryAssert(rawAssert.trajectory, `${file}: scenario \`${id}\``);
+        }
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
                 throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
@@ -231,6 +278,10 @@ export function parseSpec(text, file) {
         }
         scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
         scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
+        scenario.eventSources = resolveEventSources(s.env, id, file);
+        if (scenario.eventSources && !scenario.trajectoryAssert) {
+            throw new SpecError(`scenario \`${id}\` declares env.event_sources without assert.trajectory`, file);
+        }
         if (s.system_prompt_file !== undefined) {
             if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
                 throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
@@ -277,7 +328,8 @@ export function parseSpec(text, file) {
         }
         return scenario;
     });
-    return { skill: o.skill, judge_persona: o.judge_persona, ship_bar, critical, scenarios };
+    const effectiveCritical = [...new Set([...critical, ...scenarios.filter((scenario) => scenario.critical).map((scenario) => scenario.id)])];
+    return { schema: 1, skill: o.skill, judge_persona: o.judge_persona, ship_bar, critical: effectiveCritical, scenarios };
 }
 /** Load + validate a specification.yaml from disk. */
 export function loadSpec(file) {

package/dist/trace-gates.d.ts

@@ -95,7 +95,7 @@ export declare function normalizeSubagentCall(args: Record<string, unknown>): Su
  */
 export type AssertionStatus = "PASS" | "FAIL" | "ERROR";
 export interface AssertionResult {
-    kind: "require_call" | "require_subagent" | "forbid_call" | "unchanged_path";
+    kind: "require_call" | "require_subagent" | "forbid_call" | "unchanged_path" | "trace_evidence";
     status: AssertionStatus;
     detail: string;
 }
@@ -131,3 +131,4 @@ export declare function matchesGlob(pattern: string, path: string): boolean;
  * possible failure for a safety check.
  */
 export declare function parseTraceAssert(raw: unknown, ctx: string): TraceAssert;
+export declare function parsePredicate(raw: unknown, ctx: string): ArgPredicate;

package/dist/trace-gates.js

@@ -39,6 +39,9 @@ export function normalizeSubagentCall(args) {
  */
 export function evaluateTraceGates(assert, trace) {
     const assertions = [];
+    for (const error of trace.capture_errors ?? []) {
+        assertions.push({ kind: "trace_evidence", status: "ERROR", detail: error });
+    }
     for (const req of assert.require_calls ?? []) {
         const matched = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
         const min = req.count?.min ?? 1;
@@ -469,7 +472,7 @@ function parseArgs(raw, ctx) {
     }
     return out;
 }
-function parsePredicate(raw, ctx) {
+export function parsePredicate(raw, ctx) {
     // `agent: plan` is shorthand for `agent: { equals: plan }` — the common case
     // should not require the author to know the operator vocabulary.
     if (typeof raw === "string" || typeof raw === "number" || typeof raw === "boolean") {

package/dist/trajectory-gates.d.ts

@@ -0,0 +1,137 @@
+import { type ArgPredicate, type CountConstraint } from "./trace-gates.js";
+/** Adapter-neutral event contract used by objective trajectory assertions. */
+export declare const TRAJECTORY_EVENT_VERSION: "1.0";
+export declare const TRAJECTORY_ASSERT_VERSION: "1.0";
+export interface TrajectoryApproval {
+    id?: string;
+    capability?: string;
+    subject?: string;
+    source?: string;
+    scope?: string;
+    approved_at?: string;
+    expires_at?: string;
+    used_at?: string;
+}
+export interface TrajectoryDigests {
+    plan?: string;
+    task?: string;
+    definition?: string;
+    head?: string;
+    tree?: string;
+    [key: string]: string | undefined;
+}
+/**
+ * One normalized workflow event. Native field names remain available in
+ * `attributes`; common correlation fields are promoted so the evaluator never
+ * needs to know whether the source was pi, pi-daddy, or principal assurance.
+ */
+export interface TrajectoryEventV1 {
+    event_version: typeof TRAJECTORY_EVENT_VERSION;
+    seq: number;
+    type: string;
+    source: string;
+    at?: string;
+    run_id?: string;
+    task_id?: string;
+    workspace_id?: string;
+    context_id?: string;
+    finding_id?: string;
+    parent_id?: string;
+    child_id?: string;
+    phase?: string;
+    tool?: string;
+    capability?: string;
+    requested_capabilities?: string[];
+    effective_capabilities?: string[];
+    refusal_code?: string;
+    exit_code?: number;
+    digests?: TrajectoryDigests;
+    approval?: TrajectoryApproval;
+    requirements?: string[];
+    attributes?: Record<string, unknown>;
+}
+export interface EventSelector {
+    event: string;
+    where?: Record<string, ArgPredicate>;
+    select?: "first" | "last" | "all";
+}
+export interface RequiredEvent extends EventSelector {
+    count?: CountConstraint;
+}
+export interface CorrelationAssert {
+    left: EventSelector;
+    right: EventSelector;
+    same?: string[];
+    different?: string[];
+    order?: "before" | "after";
+}
+export interface FreshnessAssert {
+    subject: EventSelector;
+    after: EventSelector[];
+    same?: string[];
+}
+export interface UniqueAssert {
+    events: EventSelector;
+    fields: string[];
+}
+export interface ForbidAfterAssert {
+    anchor: EventSelector;
+    forbidden: EventSelector[];
+    same?: string[];
+    anchor_optional?: boolean;
+}
+export interface ApprovalAssert {
+    grant: EventSelector;
+    use: EventSelector;
+    same?: string[];
+    scopes?: string[];
+    sources?: string[];
+    unexpired?: boolean;
+    max_uses?: number;
+}
+export interface CoverageAssert {
+    requirements: string[];
+    events?: EventSelector;
+}
+export interface TrajectoryAssert {
+    version: typeof TRAJECTORY_ASSERT_VERSION;
+    require?: RequiredEvent[];
+    forbid?: EventSelector[];
+    ordered?: EventSelector[][];
+    correlate?: CorrelationAssert[];
+    freshness?: FreshnessAssert[];
+    unique?: UniqueAssert[];
+    forbid_after?: ForbidAfterAssert[];
+    approvals?: ApprovalAssert[];
+    coverage?: CoverageAssert[];
+}
+export type TrajectoryAssertionStatus = "PASS" | "FAIL" | "ERROR";
+export interface TrajectoryAssertionResult {
+    kind: "require_event" | "forbid_event" | "ordered_events" | "correlation" | "freshness" | "unique" | "forbid_after" | "approval" | "coverage" | "evidence";
+    status: TrajectoryAssertionStatus;
+    detail: string;
+}
+export interface TrajectoryGateResult {
+    status: TrajectoryAssertionStatus;
+    event_version: typeof TRAJECTORY_EVENT_VERSION;
+    events_sha256: string;
+    assertions: TrajectoryAssertionResult[];
+}
+export declare function trajectoryEventsSha256(events: TrajectoryEventV1[]): string;
+export declare function serializeTrajectoryEvents(events: TrajectoryEventV1[]): string;
+export declare function deserializeTrajectoryEvents(text: string): TrajectoryEventV1[] | null;
+/** Evaluate a closed, declarative assertion set over saved normalized events. */
+export declare function evaluateTrajectoryGates(assert: TrajectoryAssert, input: TrajectoryEventV1[]): TrajectoryGateResult;
+export declare function parseTrajectoryAssert(raw: unknown, ctx: string): TrajectoryAssert;
+export interface MutationSelfTestCase {
+    id: string;
+    detected: boolean;
+    status: TrajectoryAssertionStatus;
+    detail: string;
+}
+export interface MutationSelfTestReport {
+    baseline: "PASS";
+    cases: MutationSelfTestCase[];
+}
+/** Fast, deterministic proof that every trajectory assertion class can turn red. */
+export declare function runTrajectoryMutationSelfTest(): MutationSelfTestReport;