@skill-harness/core 0.7.0 → 0.9.0

package/dist/spec.js

@@ -1,6 +1,7 @@
 import { readFileSync } from "node:fs";
 import yaml from "js-yaml";
 import { parseTraceAssert } from "./trace-gates.js";
+import { parseTrajectoryAssert } from "./trajectory-gates.js";
 /** Thrown on any validation failure. Message always carries the spec file path. */
 export class SpecError extends Error {
     constructor(message, file) {
@@ -88,6 +89,46 @@ function resolveExtensions(env, hasSystemPrompt, id, file) {
  * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
  * not something to silently ignore.
  */
+function resolveEventSources(env, id, file) {
+    const raw = env && typeof env === "object" ? env.event_sources : undefined;
+    if (raw === undefined)
+        return undefined;
+    if (!Array.isArray(raw) || raw.length === 0) {
+        throw new SpecError(`scenario \`${id}\` env.event_sources must be a non-empty list`, file);
+    }
+    const allowedAdapters = new Set([
+        "normalized-v1", "principal-assurance-v1", "pi-daddy-v1",
+    ]);
+    return raw.map((entry, index) => {
+        if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] must be a mapping`, file);
+        }
+        const source = entry;
+        for (const key of Object.keys(source)) {
+            if (!["adapter", "path", "required"].includes(key)) {
+                throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] has unknown key \`${key}\``, file);
+            }
+        }
+        if (!allowedAdapters.has(source.adapter)) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].adapter is unsupported`, file);
+        }
+        if (typeof source.path !== "string" || !source.path.trim()) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].path must be non-empty`, file);
+        }
+        const path = source.path.trim();
+        if (path.startsWith("/") || /^[A-Za-z]:[\\/]/.test(path) || path.includes("\\") || path.split("/").some((part) => part === ".." || part === "." || part === "")) {
+            throw new SpecError(`scenario \`${id}\` env.event_sources paths must be workspace-relative without traversal (got \`${path}\`)`, file);
+        }
+        if (source.required !== undefined && typeof source.required !== "boolean") {
+            throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].required must be true or false`, file);
+        }
+        return {
+            adapter: source.adapter,
+            path,
+            required: source.required !== false,
+        };
+    });
+}
 function resolveRemote(env, workspace, id, file) {
     const raw = env && typeof env === "object" ? env.remote : undefined;
     if (raw === undefined)
@@ -114,6 +155,9 @@ export function parseSpec(text, file) {
         throw new SpecError("spec must be a YAML mapping", file);
     }
     const o = doc;
+    if (o.schema !== undefined && o.schema !== 1) {
+        throw new SpecError(`unsupported \`schema\` ${JSON.stringify(o.schema)} (expected 1)`, file);
+    }
     if (typeof o.skill !== "string" || o.skill.length === 0) {
         throw new SpecError("missing or invalid `skill` (string)", file);
     }
@@ -179,6 +223,9 @@ export function parseSpec(text, file) {
         if (rawAssert?.trace !== undefined) {
             scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
         }
+        if (rawAssert?.trajectory !== undefined) {
+            scenario.trajectoryAssert = parseTrajectoryAssert(rawAssert.trajectory, `${file}: scenario \`${id}\``);
+        }
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
                 throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
@@ -231,6 +278,10 @@ export function parseSpec(text, file) {
         }
         scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
         scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
+        scenario.eventSources = resolveEventSources(s.env, id, file);
+        if (scenario.eventSources && !scenario.trajectoryAssert) {
+            throw new SpecError(`scenario \`${id}\` declares env.event_sources without assert.trajectory`, file);
+        }
         if (s.system_prompt_file !== undefined) {
             if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
                 throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
@@ -277,7 +328,8 @@ export function parseSpec(text, file) {
         }
         return scenario;
     });
-    return { skill: o.skill, judge_persona: o.judge_persona, ship_bar, critical, scenarios };
+    const effectiveCritical = [...new Set([...critical, ...scenarios.filter((scenario) => scenario.critical).map((scenario) => scenario.id)])];
+    return { schema: 1, skill: o.skill, judge_persona: o.judge_persona, ship_bar, critical: effectiveCritical, scenarios };
 }
 /** Load + validate a specification.yaml from disk. */
 export function loadSpec(file) {

package/dist/stability.js

@@ -2,7 +2,7 @@ import { join } from "node:path";
 import { loadSpec } from "./spec.js";
 import { effectiveVerdicts } from "./results.js";
 import { collectScoredRuns } from "./trends.js";
-import { describeSourceKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY } from "./sources.js";
+import { describeSourceKey, isSupersededKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY, SKILL_PROMPT_KEY } from "./sources.js";
 const DEFAULT_WINDOW = 5;
 /** A verdict that carries evidence about the task rather than about the harness or judge. */
 function conclusive(v) {
@@ -23,6 +23,25 @@ function pointFor(r, s, verdict) {
         unanimous: reps > 1 && s.flakiness === 0,
     };
 }
+/**
+ * Did the skill text the MODEL RECEIVES differ between two runs?
+ *
+ * Prefers `skill:prompt` when both runs recorded it, and falls back to the raw-bytes
+ * `SKILL.md` key for runs that predate it. A frontmatter-only edit moves the raw bytes and
+ * not the prompt, and labelling a flip "SKILL.md changed across that step" on that basis
+ * would point a reader at an edit no model could have seen — the note's whole job is to
+ * say which of two explanations the record supports, so it must not invent a third.
+ */
+function skillTextChanged(prev, cur) {
+    for (const key of [SKILL_PROMPT_KEY, SKILL_KEY]) {
+        const a = prev?.[key];
+        const b = cur?.[key];
+        if (a === undefined || b === undefined)
+            continue;
+        return a !== b;
+    }
+    return false;
+}
 /** reps + threshold, normalised the way lift.ts normalises it (a lone rep has no threshold). */
 function shapeOf(s) {
     const reps = s.reps ?? 1;
@@ -43,6 +62,11 @@ function compareSources(a, b, keys) {
     let shared = 0;
     const changed = [];
     for (const key of keys) {
+        // Same rule as lint's: when both runs recorded the model-visible digest of a prompt
+        // document, THAT is the comparison. Leaving the raw-bytes key in would call two runs
+        // "different questions" over an `allowed-tools:` line neither model could read.
+        if (isSupersededKey(key, a) && isSupersededKey(key, b))
+            continue;
         const va = a[key];
         const vb = b[key];
         if (va === undefined || vb === undefined)
@@ -77,9 +101,7 @@ function stabilityForScenario(group, scenario, window) {
         const cur = raw[i];
         const from = points[i - 1];
         const to = points[i];
-        const skillChanged = prev.r.source_hashes?.[SKILL_KEY] !== undefined &&
-            cur.r.source_hashes?.[SKILL_KEY] !== undefined &&
-            prev.r.source_hashes[SKILL_KEY] !== cur.r.source_hashes[SKILL_KEY];
+        const skillChanged = skillTextChanged(prev.r.source_hashes, cur.r.source_hashes);
         const base = { from, to, flipped: false, skillChanged, changedSources: [], unanimousFlip: false };
         if (!prev.ok || !cur.ok) {
             pairs.push({ ...base, status: "inconclusive" });

package/dist/trace-gates.d.ts

@@ -95,7 +95,7 @@ export declare function normalizeSubagentCall(args: Record<string, unknown>): Su
  */
 export type AssertionStatus = "PASS" | "FAIL" | "ERROR";
 export interface AssertionResult {
-    kind: "require_call" | "require_subagent" | "forbid_call" | "unchanged_path";
+    kind: "require_call" | "require_subagent" | "forbid_call" | "unchanged_path" | "trace_evidence";
     status: AssertionStatus;
     detail: string;
 }
@@ -131,3 +131,4 @@ export declare function matchesGlob(pattern: string, path: string): boolean;
  * possible failure for a safety check.
  */
 export declare function parseTraceAssert(raw: unknown, ctx: string): TraceAssert;
+export declare function parsePredicate(raw: unknown, ctx: string): ArgPredicate;

package/dist/trace-gates.js

@@ -39,6 +39,9 @@ export function normalizeSubagentCall(args) {
  */
 export function evaluateTraceGates(assert, trace) {
     const assertions = [];
+    for (const error of trace.capture_errors ?? []) {
+        assertions.push({ kind: "trace_evidence", status: "ERROR", detail: error });
+    }
     for (const req of assert.require_calls ?? []) {
         const matched = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
         const min = req.count?.min ?? 1;
@@ -469,7 +472,7 @@ function parseArgs(raw, ctx) {
     }
     return out;
 }
-function parsePredicate(raw, ctx) {
+export function parsePredicate(raw, ctx) {
     // `agent: plan` is shorthand for `agent: { equals: plan }` — the common case
     // should not require the author to know the operator vocabulary.
     if (typeof raw === "string" || typeof raw === "number" || typeof raw === "boolean") {

package/dist/trajectory-gates.d.ts

@@ -0,0 +1,137 @@
+import { type ArgPredicate, type CountConstraint } from "./trace-gates.js";
+/** Adapter-neutral event contract used by objective trajectory assertions. */
+export declare const TRAJECTORY_EVENT_VERSION: "1.0";
+export declare const TRAJECTORY_ASSERT_VERSION: "1.0";
+export interface TrajectoryApproval {
+    id?: string;
+    capability?: string;
+    subject?: string;
+    source?: string;
+    scope?: string;
+    approved_at?: string;
+    expires_at?: string;
+    used_at?: string;
+}
+export interface TrajectoryDigests {
+    plan?: string;
+    task?: string;
+    definition?: string;
+    head?: string;
+    tree?: string;
+    [key: string]: string | undefined;
+}
+/**
+ * One normalized workflow event. Native field names remain available in
+ * `attributes`; common correlation fields are promoted so the evaluator never
+ * needs to know whether the source was pi, pi-daddy, or principal assurance.
+ */
+export interface TrajectoryEventV1 {
+    event_version: typeof TRAJECTORY_EVENT_VERSION;
+    seq: number;
+    type: string;
+    source: string;
+    at?: string;
+    run_id?: string;
+    task_id?: string;
+    workspace_id?: string;
+    context_id?: string;
+    finding_id?: string;
+    parent_id?: string;
+    child_id?: string;
+    phase?: string;
+    tool?: string;
+    capability?: string;
+    requested_capabilities?: string[];
+    effective_capabilities?: string[];
+    refusal_code?: string;
+    exit_code?: number;
+    digests?: TrajectoryDigests;
+    approval?: TrajectoryApproval;
+    requirements?: string[];
+    attributes?: Record<string, unknown>;
+}
+export interface EventSelector {
+    event: string;
+    where?: Record<string, ArgPredicate>;
+    select?: "first" | "last" | "all";
+}
+export interface RequiredEvent extends EventSelector {
+    count?: CountConstraint;
+}
+export interface CorrelationAssert {
+    left: EventSelector;
+    right: EventSelector;
+    same?: string[];
+    different?: string[];
+    order?: "before" | "after";
+}
+export interface FreshnessAssert {
+    subject: EventSelector;
+    after: EventSelector[];
+    same?: string[];
+}
+export interface UniqueAssert {
+    events: EventSelector;
+    fields: string[];
+}
+export interface ForbidAfterAssert {
+    anchor: EventSelector;
+    forbidden: EventSelector[];
+    same?: string[];
+    anchor_optional?: boolean;
+}
+export interface ApprovalAssert {
+    grant: EventSelector;
+    use: EventSelector;
+    same?: string[];
+    scopes?: string[];
+    sources?: string[];
+    unexpired?: boolean;
+    max_uses?: number;
+}
+export interface CoverageAssert {
+    requirements: string[];
+    events?: EventSelector;
+}
+export interface TrajectoryAssert {
+    version: typeof TRAJECTORY_ASSERT_VERSION;
+    require?: RequiredEvent[];
+    forbid?: EventSelector[];
+    ordered?: EventSelector[][];
+    correlate?: CorrelationAssert[];
+    freshness?: FreshnessAssert[];
+    unique?: UniqueAssert[];
+    forbid_after?: ForbidAfterAssert[];
+    approvals?: ApprovalAssert[];
+    coverage?: CoverageAssert[];
+}
+export type TrajectoryAssertionStatus = "PASS" | "FAIL" | "ERROR";
+export interface TrajectoryAssertionResult {
+    kind: "require_event" | "forbid_event" | "ordered_events" | "correlation" | "freshness" | "unique" | "forbid_after" | "approval" | "coverage" | "evidence";
+    status: TrajectoryAssertionStatus;
+    detail: string;
+}
+export interface TrajectoryGateResult {
+    status: TrajectoryAssertionStatus;
+    event_version: typeof TRAJECTORY_EVENT_VERSION;
+    events_sha256: string;
+    assertions: TrajectoryAssertionResult[];
+}
+export declare function trajectoryEventsSha256(events: TrajectoryEventV1[]): string;
+export declare function serializeTrajectoryEvents(events: TrajectoryEventV1[]): string;
+export declare function deserializeTrajectoryEvents(text: string): TrajectoryEventV1[] | null;
+/** Evaluate a closed, declarative assertion set over saved normalized events. */
+export declare function evaluateTrajectoryGates(assert: TrajectoryAssert, input: TrajectoryEventV1[]): TrajectoryGateResult;
+export declare function parseTrajectoryAssert(raw: unknown, ctx: string): TrajectoryAssert;
+export interface MutationSelfTestCase {
+    id: string;
+    detected: boolean;
+    status: TrajectoryAssertionStatus;
+    detail: string;
+}
+export interface MutationSelfTestReport {
+    baseline: "PASS";
+    cases: MutationSelfTestCase[];
+}
+/** Fast, deterministic proof that every trajectory assertion class can turn red. */
+export declare function runTrajectoryMutationSelfTest(): MutationSelfTestReport;