@skill-harness/core 0.29.0 → 0.31.0

package/README.md

@@ -1,8 +1,22 @@
 # @skill-harness/core
 
 The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
-spec parsing, discovery, run orchestration, LLM-judge grading, scoring, and results.
+skill-authoring drafts and previews, spec parsing, eval preflight, selected judge
+references, calibration, run orchestration, grading, scoring, and result diagnosis.
 
-Internal API — use the `skill-harness` CLI. See the repo for docs.
+Most users should install `skill-harness` in Pi, select a model, and start with
+`/skill-harness create <idea>`. Authoring requires Pi and its selected model;
+generated skills require only Pi. A pi-daddy wrapper is optional. Evaluation
+files can live with the skill or in a separate repository.
+
+The CLI exposes `preflight`, offline `calibrate` validation, explicit
+`calibrate --run --judge ...`, and `diagnose`. Core helpers preserve selected
+references and reviewed calibration expectations without treating them as a
+model-quality score. See
+[EVAL-AUTHORING.md](https://github.com/mojomanyana/skill-harness/blob/main/docs/EVAL-AUTHORING.md)
+and [JUDGE-EVIDENCE.md](https://github.com/mojomanyana/skill-harness/blob/main/docs/JUDGE-EVIDENCE.md).
+
+Fresh Pi execution is qualified for exact versions **1.0.4 and 1.1.0**.
+Other versions require separate transport qualification.
 
 ESM, Node ≥ 20.

package/dist/adapters/types.d.ts

@@ -55,6 +55,8 @@ export interface RunReq {
      * extension the developer happens to have installed cannot join the test.
      */
     extensions?: string[];
+    /** Explicit Pi tool filter; an empty list disables tools. Not OS isolation. */
+    tools?: string[];
     /**
      * Extra env for the subject process, from the arm. Merged over `process.env` by
      * the adapter — an arm must be able to add `PI_GRANTS_*` without the harness

package/dist/authoring-preflight.d.ts

@@ -0,0 +1,22 @@
+import type { Spec } from "./spec.js";
+export interface AuthoringFinding {
+    scenario: string;
+    code: "INPUT_DISCOVERY" | "READ_COUNT" | "PATH_PORTABILITY" | "COMMAND_SPELLING" | "TEST_SYNTAX" | "CITATION_FORMAT" | "UNAVAILABLE_FORBIDDEN_TOOL" | "UNAVAILABLE_REQUIRED_TOOL" | "JUDGE_REFERENCE_GAP" | "MISSING_COVERAGE_MAP";
+    severity?: "error";
+    message: string;
+}
+/** Review hints only. A valid exact-format contract may intentionally trigger one. */
+export declare function inspectEvalAuthoring(spec: Spec, profile?: {
+    tools?: readonly string[];
+    fixtureFiles?: (scenario: Spec["scenarios"][number]) => readonly string[];
+    /** Explicitly selected fixture/reference text only. No host search or automatic judge disclosure. */
+    fixtureText?: (scenario: Spec["scenarios"][number], file: string) => string | null;
+    referenceText?: (file: string) => string | null;
+}): AuthoringFinding[];
+/** Names only, never file contents. Do not follow symlinks or inspect private host directories. */
+export declare function fixtureInventory(root: string): string[];
+/** Optional text inventory for offline lint. Bounded; never expands the judge's input. */
+export declare function authoringEvidenceProfile(specDir: string): {
+    fixtureText: (s: Spec["scenarios"][number], file: string) => string | null;
+    referenceText: (file: string) => string | null;
+};

package/dist/authoring-preflight.js

@@ -0,0 +1,111 @@
+import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { join, relative, isAbsolute, resolve } from "node:path";
+import { MARKERS } from "./workspace.js";
+/** Review hints only. A valid exact-format contract may intentionally trigger one. */
+export function inspectEvalAuthoring(spec, profile = {}) {
+    const findings = [];
+    for (const s of spec.scenarios) {
+        const tools = profile.tools ?? s.tools;
+        const add = (code, message) => findings.push({ scenario: s.id, code, message });
+        if (!s.covers?.length)
+            add("MISSING_COVERAGE_MAP", "No instruction-section coverage is declared. Map the behavior this scenario can detect; a map is not proof of coverage.");
+        if (s.judgeEvidence === "response" && profile.fixtureText && profile.referenceText) {
+            const references = (s.judgeReferences ?? []).map(profile.referenceText).filter((text) => text !== null);
+            const normalize = (text) => text.replace(/\r\n/g, "\n").trim();
+            const visible = [...references, s.judgeContext ?? ""].map(normalize);
+            for (const file of profile.fixtureFiles?.(s) ?? []) {
+                if (!/\.(?:md|txt|json|ya?ml|csv)$/i.test(file))
+                    continue;
+                const source = profile.fixtureText(s, file);
+                if (source !== null && normalize(source) && !visible.some(ref => ref.includes(normalize(source)))) {
+                    add("JUDGE_REFERENCE_GAP", `Fixture ${file} is not present verbatim in judge references/context. Verify whether its claims are needed to judge valid answers, including brand, goals and conflicting facts. Deliberately private sources may remain excluded; document the reason. This is an inventory warning, not a semantic verdict.`);
+                }
+            }
+        }
+        for (const call of s.traceAssert?.require_calls ?? []) {
+            const pattern = call.args?.path?.matches;
+            if (tools && !tools.includes(call.tool)) {
+                findings.push({ scenario: s.id, code: "UNAVAILABLE_REQUIRED_TOOL", severity: "error", message: `Required tool ${call.tool} is excluded by the explicit Pi tool filter.` });
+            }
+            if (tools?.length === 1 && tools[0] === "read" && call.tool === "read" && pattern) {
+                const matches = (profile.fixtureFiles?.(s) ?? []).filter(file => new RegExp(pattern).test(file));
+                if (matches.length && !matches.some(file => s.turns.join("\n").includes(file))) {
+                    add("INPUT_DISCOVERY", "A required source is not named in the prompt and the supplied profile exposes only read. Provide an inventory or discovery capability.");
+                }
+            }
+            if (call.tool === "read" && (call.count?.min ?? 1) > 1) {
+                add("READ_COUNT", "Repeated reads do not prove content coverage. Calibrate a single complete read and repeated first chunks.");
+            }
+            if (pattern?.includes("(^|/)"))
+                add("PATH_PORTABILITY", "The path predicate rejects Windows separators. Check whether that restriction is intentional.");
+            if (call.tool === "bash" && call.args?.command?.matches === String.raw `\bnpm test\b`) {
+                add("COMMAND_SPELLING", "The command predicate rejects npm run test, which invokes the same package script.");
+            }
+        }
+        if (s.assert?.diff_contains?.includes("expect("))
+            add("TEST_SYNTAX", "An assertion spelling does not establish that a regression test detects the defect.");
+        if (s.checklist.some(item => /\bCite\s+\S+\s+line\s+\d/i.test(item))) {
+            add("CITATION_FORMAT", "A mandatory line citation can reject an accurate attributed answer. Justify the format requirement or accept equivalent evidence.");
+        }
+        if (tools && s.traceAssert?.forbid_calls?.some(call => !tools.includes(call.tool))) {
+            add("UNAVAILABLE_FORBIDDEN_TOOL", "A forbidden tool is absent from the supplied profile. Non-use does not demonstrate refusal of an available capability.");
+        }
+    }
+    return findings;
+}
+/** Names only, never file contents. Do not follow symlinks or inspect private host directories. */
+export function fixtureInventory(root) {
+    const files = [];
+    let entries = 0;
+    const visit = (directory, prefix, depth = 0) => {
+        if (depth > 32)
+            throw new Error("Fixture inventory exceeds 32 directory levels; select a smaller fixture");
+        for (const entry of readdirSync(directory, { withFileTypes: true })) {
+            if (++entries > 4096)
+                throw new Error("Fixture inventory exceeds 4096 entries; select a smaller fixture");
+            if (entry.name === ".git" || entry.isSymbolicLink())
+                continue;
+            const name = prefix + entry.name;
+            if (entry.isDirectory())
+                visit(join(directory, entry.name), name + "/", depth + 1);
+            else if (entry.isFile())
+                files.push(MARKERS.some(marker => name.startsWith(marker + "/")) ? name.slice(name.indexOf("/") + 1) : name);
+        }
+    };
+    visit(root, "");
+    return [...new Set(files)].sort();
+}
+/** Optional text inventory for offline lint. Bounded; never expands the judge's input. */
+export function authoringEvidenceProfile(specDir) {
+    let remaining = 2 * 1024 * 1024;
+    const cache = new Map();
+    const text = (file) => {
+        const path = resolve(specDir, file);
+        if (cache.has(path))
+            return cache.get(path);
+        let value = null;
+        try {
+            const rel = relative(realpathSync(specDir), realpathSync(path));
+            const stat = statSync(path), size = stat.size;
+            if (!stat.isFile() || isAbsolute(rel) || rel === ".." || rel.startsWith("../") || rel.startsWith("..\\") || size > 65536 || size > remaining)
+                return null;
+            remaining -= size;
+            value = new TextDecoder("utf-8", { fatal: true }).decode(readFileSync(path));
+        }
+        catch { /* Unreadable/binary source is not invented as empty evidence. */ }
+        cache.set(path, value);
+        return value;
+    };
+    const fixture = (s) => typeof s.workspace === "object" ? s.workspace.fixture : s.fixture;
+    return {
+        fixtureText: (s, file) => {
+            const root = fixture(s);
+            if (!root)
+                return null;
+            // Overlay order is the same as workspace creation: later pending content wins.
+            return text(join(root, "_uncommitted", file)) ?? text(join(root, "_staged", file)) ?? text(join(root, file));
+        },
+        referenceText: text,
+    };
+}
+//# sourceMappingURL=authoring-preflight.js.map

package/dist/authoring.d.ts

@@ -0,0 +1,96 @@
+export interface AuthoringAnswers {
+    purpose: string;
+    outcomes: string;
+    boundaries: string;
+    tools: string[];
+    placement: "same-repo" | "separate-repo";
+    delivery: "skill" | "skill-and-agent";
+    evalRoot?: string;
+    /** Additional operator-agreed context/delegation capabilities for the optional wrapper. */
+    agentCapabilities?: string[];
+}
+export interface AuthoringQuestion {
+    key: keyof AuthoringAnswers;
+    prompt: string;
+    options?: string[];
+}
+export interface AuthoringFile {
+    area: "skill" | "evals" | "agent";
+    path: string;
+    content: string;
+}
+export interface AuthoringDraft {
+    files: AuthoringFile[];
+    requirements: {
+        id: string;
+        description: string;
+        scenarios: string[];
+    }[];
+    /** An author's assertion about proposed control labels, never calibration or ship approval. */
+    labelStatus: "proposed" | "reviewed";
+}
+export interface AuthoringChange {
+    area: AuthoringFile["area"];
+    path: string;
+    destination: string;
+    beforeSha256: string | null;
+    afterSha256: string;
+    kind: "create" | "update" | "unchanged";
+    before: string | null;
+    after: string;
+}
+export interface AuthoringState {
+    schema: 1;
+    name: string;
+    idea: string;
+    root: string;
+    skillDir: string;
+    skillsRoot: string;
+    agentRoot: string;
+    answers: Partial<AuthoringAnswers>;
+    imported: {
+        path: string;
+        sha256: string;
+    } | null;
+    revision: number;
+    draft?: {
+        data: AuthoringDraft;
+        digest: string;
+        changes: AuthoringChange[];
+    };
+    appliedDigest?: string;
+}
+export interface AuthoringPreview {
+    state: AuthoringState;
+    nextQuestion: AuthoringQuestion | null;
+    draftDigest?: string;
+    changes: AuthoringChange[];
+    issues: string[];
+    warnings: string[];
+    readyToApply: boolean;
+}
+/** Create a local brief, without running a model, scanning examples or editing a skill. */
+export declare function startAuthoring(options: {
+    root: string;
+    name: string;
+    idea: string;
+    answers?: Partial<AuthoringAnswers>;
+    skillRoot?: string;
+    evalRoot?: string;
+    agentRoot?: string;
+    skillDir?: string;
+}): AuthoringState;
+export declare function readAuthoring(root: string, name: string): AuthoringState;
+/** Known answers can be supplied together; the UI asks only the next unresolved question. */
+export declare function answerAuthoring(root: string, name: string, answers: Partial<AuthoringAnswers>): AuthoringState;
+/** Stage model-authored content and capture exact existing bytes. This does not apply or certify it. */
+export declare function stageAuthoringDraft(root: string, name: string, draft: AuthoringDraft): AuthoringPreview;
+export declare function inspectAuthoring(root: string, name: string): AuthoringPreview;
+/** Explicit, digest-bound apply. No grants, installation, provider calls, labels or training authorization are inferred. */
+export declare function applyAuthoring(root: string, name: string, options: {
+    digest: string;
+}): {
+    state: AuthoringState;
+    files: string[];
+    status: "applied" | "already-applied";
+};