pactwright 0.0.1

package/dist/eval/runner.d.ts

@@ -0,0 +1,75 @@
+import { type ResolvedPack } from "../pack/resolve.js";
+import type { CandidateRunner, EvalSuite, SemanticJudge } from "./case.js";
+export interface DeterministicResult {
+    readonly id: string;
+    readonly description: string;
+    readonly passed: boolean;
+    readonly detail: string;
+}
+export interface SemanticResult {
+    readonly id: string;
+    readonly question: string;
+    readonly judged: boolean;
+    readonly verdict?: string;
+    readonly rationale?: string;
+    /** Why the dimension was not judged. */
+    readonly reason?: string;
+}
+export interface EvalCaseResult {
+    readonly id: string;
+    readonly title: string;
+    readonly capability: string;
+    /** The pack agent key that implemented the capability, when the pack provides it. */
+    readonly agent?: string;
+    /** The case could not be evaluated (missing capability, candidate error). */
+    readonly error?: string;
+    readonly deterministic: readonly DeterministicResult[];
+    /** Reported separately from the deterministic results, never merged or scored. */
+    readonly semantic: readonly SemanticResult[];
+}
+/**
+ * One evaluation run: per-case, per-assertion and per-dimension results
+ * only. Deliberately no aggregate quality score (Distribution §16); the
+ * report is a generated artefact, never Project Graph state.
+ */
+export interface EvalReport {
+    readonly suite: string;
+    readonly runtime: string;
+    readonly pack: {
+        readonly name: string;
+        readonly version: string;
+        readonly hash: string;
+    };
+    readonly cases: readonly EvalCaseResult[];
+}
+export interface EvalOptions {
+    /** The agent pack supplying the implementation being evaluated. */
+    readonly pack: ResolvedPack;
+    readonly suite: EvalSuite;
+    /** Overrides each case's scripted reference candidate (model-backed runs plug in here). */
+    readonly candidate?: CandidateRunner;
+    /** Judges semantic dimensions; absent means they are reported unjudged. */
+    readonly judge?: SemanticJudge;
+    /** Directory sandboxes are created under; defaults to the OS temp directory. */
+    readonly workDir?: string;
+    /**
+     * Per-case candidate deadline in ms; `undefined` disables. Scripted
+     * reference candidates are synchronous, so only model-backed runners need
+     * this. On timeout the case is reported as an error and its sandbox is
+     * removed immediately; a still-running candidate writes into the void.
+     */
+    readonly candidateTimeoutMs?: number;
+    /** Per-dimension judge deadline in ms; `undefined` disables. */
+    readonly judgeTimeoutMs?: number;
+}
+/**
+ * Runs a suite against a resolved pack, one throw-away sandbox per case.
+ * Never throws for a failing case: failures are data in the report.
+ */
+export declare function runEval(options: EvalOptions): Promise<EvalReport>;
+/**
+ * Whether every case was evaluated and every deterministic assertion
+ * passed — the CLI's exit-code gate. This is not a quality score: semantic
+ * dimensions never enter it, and no aggregate is calculated anywhere.
+ */
+export declare function evalPassed(report: EvalReport): boolean;

package/dist/eval/runner.js

@@ -0,0 +1,159 @@
+import { rmSync } from "node:fs";
+import { agentFor } from "../pack/resolve.js";
+import { runtimeVersion } from "../version.js";
+import { createSandbox, diffSnapshots, sandboxRevision, snapshotFiles } from "./sandbox.js";
+const message = (error) => error instanceof Error ? error.message : String(error);
+async function withTimeout(work, ms, label) {
+    if (ms === undefined)
+        return work;
+    let timer;
+    try {
+        return await Promise.race([
+            work,
+            new Promise((_, reject) => {
+                // The timer stays referenced so the deadline fires even when the
+                // hung work holds nothing else on the event loop; it is cleared as
+                // soon as the race settles.
+                timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+        void work.catch(() => { }); // the losing promise must not become an unhandled rejection
+    }
+}
+async function runCase(options, evalCase) {
+    const base = { id: evalCase.id, title: evalCase.title, capability: evalCase.capability };
+    const agent = agentFor(options.pack, evalCase.capability);
+    if (agent === undefined) {
+        return {
+            ...base,
+            error: `pack "${options.pack.manifest.name}@${options.pack.manifest.version}" does not provide capability "${evalCase.capability}"; the case was not evaluated`,
+            deterministic: [],
+            semantic: [],
+        };
+    }
+    const withAgent = { ...base, agent: agent.key };
+    let root;
+    try {
+        root = createSandbox(options.pack, options.workDir);
+        evalCase.setup(root);
+        const filesBefore = snapshotFiles(root);
+        const revisionBefore = sandboxRevision(root);
+        let output;
+        try {
+            const run = options.candidate ?? ((task) => evalCase.reference.run(task.root));
+            output = await withTimeout(Promise.resolve(run({
+                caseId: evalCase.id,
+                capability: evalCase.capability,
+                instruction: evalCase.instruction,
+                root,
+                agent,
+            })), options.candidateTimeoutMs, "candidate");
+        }
+        catch (error) {
+            return {
+                ...withAgent,
+                error: `candidate failed: ${message(error)}`,
+                deterministic: [],
+                semantic: [],
+            };
+        }
+        const observation = {
+            root,
+            changedFiles: diffSnapshots(filesBefore, snapshotFiles(root)),
+            revisionBefore,
+            revisionAfter: sandboxRevision(root),
+            output,
+        };
+        const deterministic = evalCase.deterministic.map((assertion) => {
+            try {
+                const result = assertion.check(observation);
+                return {
+                    id: assertion.id,
+                    description: assertion.description,
+                    passed: result.passed,
+                    detail: result.detail,
+                };
+            }
+            catch (error) {
+                return {
+                    id: assertion.id,
+                    description: assertion.description,
+                    passed: false,
+                    detail: `assertion threw: ${message(error)}`,
+                };
+            }
+        });
+        const semantic = [];
+        for (const dimension of evalCase.semantic) {
+            const entry = { id: dimension.id, question: dimension.question };
+            if (options.judge === undefined) {
+                semantic.push({
+                    ...entry,
+                    judged: false,
+                    reason: "no semantic judge configured; semantic quality is never decided deterministically",
+                });
+                continue;
+            }
+            try {
+                const judgement = await withTimeout(Promise.resolve(options.judge({ caseId: evalCase.id, dimension, observation })), options.judgeTimeoutMs, "judge");
+                semantic.push({
+                    ...entry,
+                    judged: true,
+                    verdict: judgement.verdict,
+                    ...(judgement.rationale === undefined ? {} : { rationale: judgement.rationale }),
+                });
+            }
+            catch (error) {
+                semantic.push({ ...entry, judged: false, reason: `judge failed: ${message(error)}` });
+            }
+        }
+        return { ...withAgent, deterministic, semantic };
+    }
+    catch (error) {
+        // Sandbox creation, case setup or observation building failed: the
+        // failure is data in the report, never a thrown run (Distribution §16).
+        return {
+            ...withAgent,
+            error: `case failed: ${message(error)}`,
+            deterministic: [],
+            semantic: [],
+        };
+    }
+    finally {
+        if (root !== undefined)
+            rmSync(root, { recursive: true, force: true });
+    }
+}
+/**
+ * Runs a suite against a resolved pack, one throw-away sandbox per case.
+ * Never throws for a failing case: failures are data in the report.
+ */
+export async function runEval(options) {
+    const cases = [];
+    for (const evalCase of options.suite.cases)
+        cases.push(await runCase(options, evalCase));
+    return {
+        suite: options.suite.name,
+        runtime: runtimeVersion(),
+        pack: {
+            name: options.pack.manifest.name,
+            version: options.pack.manifest.version,
+            hash: options.pack.hashes.pack,
+        },
+        cases,
+    };
+}
+/**
+ * Whether every case was evaluated and every deterministic assertion
+ * passed — the CLI's exit-code gate. This is not a quality score: semantic
+ * dimensions never enter it, and no aggregate is calculated anywhere.
+ */
+export function evalPassed(report) {
+    // An empty run proves nothing; the gate must not pass vacuously.
+    if (report.cases.length === 0)
+        return false;
+    return report.cases.every((entry) => entry.error === undefined && entry.deterministic.every((a) => a.passed));
+}

package/dist/eval/sandbox.d.ts

@@ -0,0 +1,39 @@
+import type { Edge } from "../graph/edges.js";
+import { type ResolvedPack } from "../pack/resolve.js";
+/** Every seeded node carries the same fixed date: fixtures are deterministic. */
+export declare const SEED_CREATED = "2026-08-20";
+/**
+ * A throw-away sandbox project for one evaluation case: a complete,
+ * loadable Pactwright project whose config selects the pack under
+ * evaluation by its absolute path and whose graph starts empty. The caller
+ * removes the directory when the case is done.
+ */
+export declare function createSandbox(pack: ResolvedPack, workDir?: string): string;
+export interface SeedNode {
+    readonly id: string;
+    readonly type: string;
+    readonly title: string;
+    readonly body: string;
+    /** Extra frontmatter fields, e.g. a decision's `decided_by`/`outcome`. */
+    readonly fields?: Readonly<Record<string, string>>;
+}
+/** Serialises a seed node in the canonical node-file shape. */
+export declare function seedNodeFile(node: SeedNode): string;
+export declare function seedNode(root: string, node: SeedNode): void;
+export declare function seedEdges(root: string, edges: readonly Edge[]): void;
+/** Writes an ordinary repository file into the sandbox. */
+export declare function seedFile(root: string, relativePath: string, content: string): void;
+/** Sandbox-relative POSIX path → sha256 of the file's bytes. */
+export type FileSnapshot = ReadonlyMap<string, string>;
+/** Hashes every file under `root`; the observation diff comes from two snapshots. */
+export declare function snapshotFiles(root: string): FileSnapshot;
+/** Paths whose bytes changed, appeared or disappeared between two snapshots, sorted. */
+export declare function diffSnapshots(before: FileSnapshot, after: FileSnapshot): readonly string[];
+/**
+ * The sandbox's deterministic Project Graph revision (Delivery Graph §5)
+ * over whatever canonical state currently parses. Load problems are
+ * deliberately ignored here: a candidate that corrupts the graph is caught
+ * by the changed-files observation, while the revision proves whether the
+ * canonical state it left behind still equals the seeded state.
+ */
+export declare function sandboxRevision(root: string): string;

package/dist/eval/sandbox.js

@@ -0,0 +1,143 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, sep } from "node:path";
+import { loadEdges } from "../graph/edges.js";
+import { serialiseEdges } from "../graph/mutations.js";
+import { loadNodes } from "../graph/nodes.js";
+import { graphRevision } from "../graph/revision.js";
+import { lockEntriesFor, serialiseLock } from "../pack/resolve.js";
+import { CONFIG_FILE, EDGES_FILE, LIFECYCLE_FILE, LOCK_FILE, NODES_DIR } from "../project.js";
+/** Every seeded node carries the same fixed date: fixtures are deterministic. */
+export const SEED_CREATED = "2026-08-20";
+/** The §17 default lifecycle configuration seeded into every sandbox. */
+const SANDBOX_LIFECYCLE = `version: 1
+
+stages:
+  capture-intent:
+    execution: manual
+  propose-contracts:
+    execution: automatic
+  approve-contract:
+    execution: manual
+    actor: human
+  write-brief:
+    execution: automatic
+  deliver-brief:
+    execution: automatic
+  review:
+    execution: automatic
+  prepare-evidence:
+    execution: automatic
+`;
+function write(root, relativePath, content) {
+    const target = join(root, relativePath);
+    mkdirSync(dirname(target), { recursive: true });
+    writeFileSync(target, content, "utf8");
+}
+/**
+ * A throw-away sandbox project for one evaluation case: a complete,
+ * loadable Pactwright project whose config selects the pack under
+ * evaluation by its absolute path and whose graph starts empty. The caller
+ * removes the directory when the case is done.
+ */
+export function createSandbox(pack, workDir = tmpdir()) {
+    const root = mkdtempSync(join(workDir, "pactwright-eval-"));
+    write(root, CONFIG_FILE, [
+        "version: 1",
+        "",
+        "agent_pack:",
+        `  source: ${JSON.stringify(pack.dir)}`,
+        "",
+        "adapter:",
+        "  type: claude-code",
+        "",
+        "extensions: {}",
+        "",
+        "github:",
+        "  enabled: false",
+        "",
+    ].join("\n"));
+    write(root, LIFECYCLE_FILE, SANDBOX_LIFECYCLE);
+    write(root, LOCK_FILE, serialiseLock(lockEntriesFor(pack)));
+    mkdirSync(join(root, NODES_DIR), { recursive: true });
+    write(root, EDGES_FILE, "edges: []\n");
+    return root;
+}
+/** Serialises a seed node in the canonical node-file shape. */
+export function seedNodeFile(node) {
+    const fields = Object.entries(node.fields ?? {})
+        .map(([key, value]) => `${key}: ${value}\n`)
+        .join("");
+    return `---\nid: ${node.id}\ntype: ${node.type}\ntitle: ${node.title}\ncreated: ${SEED_CREATED}\n${fields}---\n\n${node.body}\n`;
+}
+export function seedNode(root, node) {
+    write(root, `${NODES_DIR}/${node.id}.md`, seedNodeFile(node));
+}
+export function seedEdges(root, edges) {
+    write(root, EDGES_FILE, serialiseEdges(edges));
+}
+/** Writes an ordinary repository file into the sandbox. */
+export function seedFile(root, relativePath, content) {
+    write(root, relativePath, content);
+}
+function walk(dir, base, into) {
+    for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) {
+        const absolute = join(dir, entry.name);
+        const key = relative(base, absolute).split(sep).join("/");
+        // Symlinks first, and never followed: a link is recorded by its target
+        // path, so creating, deleting or retargeting one is a visible change,
+        // while the tree behind a directory link stays outside the snapshot.
+        if (entry.isSymbolicLink()) {
+            into.set(key, createHash("sha256")
+                .update(`symlink:${readlinkSync(absolute)}`)
+                .digest("hex"));
+        }
+        else if (entry.isDirectory()) {
+            walk(absolute, base, into);
+        }
+        else if (entry.isFile()) {
+            into.set(key, createHash("sha256").update(readFileSync(absolute)).digest("hex"));
+        }
+        else {
+            // FIFOs, sockets, devices: content is meaningless, presence is not.
+            into.set(key, createHash("sha256").update("special").digest("hex"));
+        }
+    }
+}
+/** Hashes every file under `root`; the observation diff comes from two snapshots. */
+export function snapshotFiles(root) {
+    const snapshot = new Map();
+    walk(root, root, snapshot);
+    return snapshot;
+}
+/** Paths whose bytes changed, appeared or disappeared between two snapshots, sorted. */
+export function diffSnapshots(before, after) {
+    const changed = new Set();
+    for (const [key, hash] of before)
+        if (after.get(key) !== hash)
+            changed.add(key);
+    for (const [key, hash] of after)
+        if (before.get(key) !== hash)
+            changed.add(key);
+    return [...changed].sort();
+}
+/**
+ * The sandbox's deterministic Project Graph revision (Delivery Graph §5)
+ * over whatever canonical state currently parses. Load problems are
+ * deliberately ignored here: a candidate that corrupts the graph is caught
+ * by the changed-files observation, while the revision proves whether the
+ * canonical state it left behind still equals the seeded state.
+ */
+export function sandboxRevision(root) {
+    try {
+        const nodes = loadNodes(join(root, NODES_DIR));
+        const edges = loadEdges(join(root, EDGES_FILE));
+        return graphRevision({ nodes: nodes.nodes, edges: edges.edges });
+    }
+    catch {
+        // Never equals a real sha256: revision, so revision-comparing
+        // assertions fail instead of the case erroring out.
+        return "revision-unavailable";
+    }
+}

package/dist/extension/manage.d.ts

@@ -0,0 +1,65 @@
+import type { Problem } from "../errors.js";
+/** One extension the operation touched. */
+export interface ExtensionChange {
+    readonly id: string;
+    readonly action: "added" | "removed" | "upgraded" | "unchanged";
+    readonly version?: string;
+    readonly previousVersion?: string;
+}
+/** Result of `pactwright extension add|remove|upgrade`. */
+export interface ExtensionChangeReport {
+    readonly ok: boolean;
+    readonly root: string;
+    readonly changes: readonly ExtensionChange[];
+    /**
+     * GitHub profiles declared by newly added extensions. Reported only:
+     * provisioning does not exist in this checkpoint, and `github.enabled`
+     * controls whether it ever runs.
+     */
+    readonly githubProfiles: readonly string[];
+    /**
+     * Paths of canonical records that stayed on disk after a removal. Removal
+     * never deletes user-authored extension graph data; the user chooses
+     * separately whether to delete it.
+     */
+    readonly preserved: readonly string[];
+    readonly problems: readonly Problem[];
+}
+/**
+ * Enables an extension (Distribution §4): resolve the package, resolve and
+ * enable missing dependencies first, validate the complete required
+ * capability union, then record the exact state in config and lock. Fails
+ * before any write when compatibility is incomplete; rolls both files back
+ * if the resulting project state does not validate.
+ *
+ * "Missing" means not enabled, not merely absent: a dependency the
+ * configuration already names but has disabled is enabled too. So adding an
+ * extension that is itself already enabled still repairs a disabled
+ * dependency underneath it, at any depth, and reports what it enabled rather
+ * than `unchanged`.
+ *
+ * Because the walk crosses already-enabled dependencies to reach what is
+ * below them, an add whose enabled dependency is itself broken reports that
+ * problem rather than `unchanged`. The write would have failed on it anyway;
+ * saying so up front is the more truthful answer.
+ */
+export declare function addExtension(root: string, spec: string): ExtensionChangeReport;
+/**
+ * Removes an extension (Distribution §4). Blocked while an enabled
+ * extension still depends on it. Canonical graph data owned by the
+ * extension is never deleted — it is reported as preserved, and the user
+ * chooses separately whether to delete it.
+ *
+ * Removal is the remedy for a broken extension set, so nothing about that set
+ * being broken may block it: the dependant scan runs best effort, and when the
+ * remaining configuration still does not resolve the lock is derived from its
+ * previous contents rather than re-resolved. Both degradations are reported.
+ */
+export declare function removeExtension(root: string, id: string): ExtensionChangeReport;
+/**
+ * Upgrades an extension (Distribution §15): re-resolves the configured
+ * package, validates the complete dependency graph and capability union,
+ * and updates the lock. The configuration is desired state and does not
+ * change; canonical Project Graph state is never reinterpreted.
+ */
+export declare function upgradeExtension(root: string, id: string): ExtensionChangeReport;