@skill-test/sdk 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nick DeRobertis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,33 @@
+# @skill-test/sdk
+
+The TypeScript SDK for the
+[`skilltest`](https://github.com/nickderobertis/skilltest) CLI. A thin, typed
+wrapper and nothing else: it runs the CLI as a subprocess and types the stable
+`--format json` contract with declarations generated from the CLI's own JSON
+Schemas. Test-framework integrations build on it — use
+[`@skill-test/vitest`](../../plugins/vitest) if you want the vitest helpers; use
+this package directly from any other TypeScript/JavaScript code.
+
+```ts
+import { runSkill, validateSkill, assistantText, describeFailures } from "@skill-test/sdk";
+
+const report = await runSkill("cases/greet.yaml");
+if (!report.passed) throw new Error(describeFailures(report));
+// Mix in deterministic checks on the transcript:
+const text = assistantText(report.runs[0]!.transcript);
+
+const result = await validateSkill("skills/greeter");
+```
+
+The `skilltest` binary is resolved from the `bin` option, the `SKILLTEST_BIN`
+env var, or `PATH`; a provider override comes from `provider` or
+`SKILLTEST_PROVIDER`. A failing eval is *reported* (`report.passed` is false),
+not thrown; bad input throws `SkilltestUsageError` (CLI exit 2) and provider
+problems throw `SkilltestProviderError` (exit 3).
+
+The types in `src/generated/` are **generated** from
+`schemas/report.schema.json` / `schemas/validation.schema.json` — themselves
+generated from the CLI's own types — via `just gen-contract`, and a drift gate
+in CI fails if anything is stale, so the types cannot diverge from the binary.
+They are types only: the runner trusts the shape after `JSON.parse`, because
+the gate (not runtime re-validation) is what guarantees it.

package/dist/errors.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Errors mirroring the CLI's exit-code contract. A *test failure* (exit 1) is
+ * returned as a Report with `passed === false`, not thrown; *bad input* (exit 2)
+ * and *provider failure* (exit 3) are thrown because the author must fix them.
+ */
+export declare class SkilltestError extends Error {
+    constructor(message: string);
+}
+export declare class SkilltestUsageError extends SkilltestError {
+    constructor(message: string);
+}
+export declare class SkilltestProviderError extends SkilltestError {
+    constructor(message: string);
+}

package/dist/errors.js

@@ -0,0 +1,23 @@
+/**
+ * Errors mirroring the CLI's exit-code contract. A *test failure* (exit 1) is
+ * returned as a Report with `passed === false`, not thrown; *bad input* (exit 2)
+ * and *provider failure* (exit 3) are thrown because the author must fix them.
+ */
+export class SkilltestError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "SkilltestError";
+    }
+}
+export class SkilltestUsageError extends SkilltestError {
+    constructor(message) {
+        super(message);
+        this.name = "SkilltestUsageError";
+    }
+}
+export class SkilltestProviderError extends SkilltestError {
+    constructor(message) {
+        super(message);
+        this.name = "SkilltestProviderError";
+    }
+}

package/dist/generated/report.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Generated from the golden JSON Schemas in schemas/ by `just gen-contract`.
+ * DO NOT MODIFY BY HAND — change the Rust report types and regenerate; the
+ * contract drift gate fails while this file is stale.
+ */
+/**
+ * The kind-specific detail of an eval outcome, for reporting.
+ *
+ * The variant titles name the generated SDK model for each union arm, so keep
+ * them stable: they are part of the SDK API surface.
+ */
+export type EvalDetail = BooleanDetail | NumericDetail;
+/**
+ * How a numeric score is compared to its threshold.
+ */
+export type Comparator = "gte" | "gt" | "lte" | "lt";
+/**
+ * Who produced a message.
+ */
+export type Role = "user" | "assistant" | "system";
+/**
+ * The top-level report for a `skilltest run` invocation.
+ */
+export interface Report {
+    /**
+     * True iff every run passed.
+     */
+    passed: boolean;
+    /**
+     * Every individual run.
+     */
+    runs: CaseRun[];
+    /**
+     * Aggregate counts.
+     */
+    summary: Summary;
+}
+/**
+ * The result of running one test case on one (platform, model) pair.
+ */
+export interface CaseRun {
+    /**
+     * The test case name.
+     */
+    case: string;
+    /**
+     * Per-eval outcomes, in declaration order.
+     */
+    evals: EvalOutcome[];
+    /**
+     * The model this run used.
+     */
+    model: string;
+    /**
+     * True iff every eval in this run passed.
+     */
+    passed: boolean;
+    /**
+     * The harness platform this run used.
+     */
+    platform: string;
+    /**
+     * Absolute-ish path to the skill that was exercised.
+     */
+    skill: string;
+    /**
+     * The full conversation, for debugging and deterministic mix-in checks.
+     */
+    transcript: Transcript;
+    /**
+     * Number of assistant turns produced.
+     */
+    turns: number;
+    /**
+     * Aggregated token/cost usage across every provider call in this run
+     * (skill turns + simulated-user turns + judge calls). Omitted when no
+     * usage was reported (e.g. the fake provider or a harness that doesn't
+     * surface usage).
+     */
+    usage?: Usage | null;
+}
+/**
+ * The result of running one eval against a transcript.
+ */
+export interface EvalOutcome {
+    /**
+     * Kind-specific verdict detail.
+     */
+    detail: EvalDetail;
+    /**
+     * The eval's label (name or criterion).
+     */
+    label: string;
+    /**
+     * Whether the eval passed.
+     */
+    passed: boolean;
+    /**
+     * The judge's stated reason.
+     */
+    reason: string;
+}
+export interface BooleanDetail {
+    expected: boolean;
+    kind: "boolean";
+    value: boolean;
+}
+export interface NumericDetail {
+    comparator: Comparator;
+    kind: "numeric";
+    threshold: number;
+    value: number;
+}
+/**
+ * An ordered list of messages. Thin wrapper so the type reads clearly at call
+ * sites and so we can grow conversation-level helpers without churn.
+ */
+export interface Transcript {
+    messages: Message[];
+}
+/**
+ * A single turn in the conversation.
+ */
+export interface Message {
+    content: string;
+    role: Role;
+}
+/**
+ * Token / cost usage for one provider call.
+ *
+ * Each field is independently optional because not every harness reports every
+ * signal (cost is commonly absent on subscription auth; some harnesses report
+ * no usage at all). The whole struct is `Option<Usage>` on a turn — `None`
+ * means "no signal," not "zero."
+ */
+export interface Usage {
+    cost_usd?: number | null;
+    input_tokens?: number | null;
+    output_tokens?: number | null;
+}
+/**
+ * Aggregate pass/fail counts for a report.
+ */
+export interface Summary {
+    /**
+     * Distinct test cases represented.
+     */
+    cases: number;
+    /**
+     * Runs that failed.
+     */
+    failed: number;
+    /**
+     * Runs that passed.
+     */
+    passed: number;
+    /**
+     * Total (case × platform × model) runs.
+     */
+    runs: number;
+    /**
+     * Aggregated token/cost usage across every run in the report. Omitted
+     * when no run reported usage.
+     */
+    usage?: Usage | null;
+}

package/dist/generated/report.js

@@ -0,0 +1,7 @@
+/* eslint-disable */
+/**
+ * Generated from the golden JSON Schemas in schemas/ by `just gen-contract`.
+ * DO NOT MODIFY BY HAND — change the Rust report types and regenerate; the
+ * contract drift gate fails while this file is stale.
+ */
+export {};

package/dist/generated/validation.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Generated from the golden JSON Schemas in schemas/ by `just gen-contract`.
+ * DO NOT MODIFY BY HAND — change the Rust report types and regenerate; the
+ * contract drift gate fails while this file is stale.
+ */
+/**
+ * The top-level report for a `skilltest validate` invocation.
+ */
+export interface ValidationReport {
+    /**
+     * Every finding, in discovery order.
+     */
+    findings: ValidationFinding[];
+    /**
+     * True iff no findings were produced.
+     */
+    valid: boolean;
+}
+/**
+ * One problem found while validating a skill, as serialized in the
+ * `skilltest validate --format json` output.
+ */
+export interface ValidationFinding {
+    /**
+     * What is wrong and how to fix it.
+     */
+    message: string;
+    /**
+     * The skill directory the finding is about.
+     */
+    skill: string;
+}

package/dist/generated/validation.js

@@ -0,0 +1,7 @@
+/* eslint-disable */
+/**
+ * Generated from the golden JSON Schemas in schemas/ by `just gen-contract`.
+ * DO NOT MODIFY BY HAND — change the Rust report types and regenerate; the
+ * contract drift gate fails while this file is stale.
+ */
+export {};

package/dist/helpers.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Hand-written conveniences over the generated contract types. The type
+ * checker keeps these honest against `generated/` — a renamed or removed
+ * field fails `tsc`, not a user's test.
+ */
+import type { Report, Transcript } from "./generated/report.js";
+/** The assistant turns of a transcript joined — handy for mix-in checks. */
+export declare function assistantText(transcript: Transcript): string;
+/** A one-line-per-failed-eval summary, for assertion messages. */
+export declare function describeFailures(report: Report): string;

package/dist/helpers.js

@@ -0,0 +1,21 @@
+/** The assistant turns of a transcript joined — handy for mix-in checks. */
+export function assistantText(transcript) {
+    return transcript.messages
+        .filter((m) => m.role === "assistant")
+        .map((m) => m.content)
+        .join("\n");
+}
+/** A one-line-per-failed-eval summary, for assertion messages. */
+export function describeFailures(report) {
+    const lines = [];
+    for (const run of report.runs) {
+        if (run.passed)
+            continue;
+        for (const outcome of run.evals) {
+            if (!outcome.passed) {
+                lines.push(`${run.case} [${run.platform}/${run.model}] ${outcome.label}: ${outcome.reason}`);
+            }
+        }
+    }
+    return lines.join("\n");
+}

package/dist/index.d.ts

@@ -0,0 +1,21 @@
+/**
+ * `@skill-test/sdk` — the TypeScript SDK for the `skilltest` CLI.
+ *
+ * A thin, typed wrapper around the CLI and nothing else: run test cases,
+ * validate skills, and get back objects typed by declarations **generated from
+ * the CLI's own JSON Schemas** (`just gen-contract`), so the types cannot
+ * drift from the binary. Test frameworks build on this — `@skill-test/vitest`
+ * adds the vitest helpers on top.
+ *
+ * ```ts
+ * import { runSkill, assistantText, describeFailures } from "@skill-test/sdk";
+ *
+ * const report = await runSkill("cases/greet.yaml");
+ * if (!report.passed) throw new Error(describeFailures(report));
+ * ```
+ */
+export { runSkill, validateSkill, ENV_BIN, ENV_PROVIDER, type RunOptions } from "./runner.js";
+export { SkilltestError, SkilltestProviderError, SkilltestUsageError, } from "./errors.js";
+export { assistantText, describeFailures } from "./helpers.js";
+export type { BooleanDetail, CaseRun, Comparator, EvalDetail, EvalOutcome, Message, NumericDetail, Report, Role, Summary, Transcript, Usage, } from "./generated/report.js";
+export type { ValidationFinding, ValidationReport } from "./generated/validation.js";

package/dist/index.js

@@ -0,0 +1,19 @@
+/**
+ * `@skill-test/sdk` — the TypeScript SDK for the `skilltest` CLI.
+ *
+ * A thin, typed wrapper around the CLI and nothing else: run test cases,
+ * validate skills, and get back objects typed by declarations **generated from
+ * the CLI's own JSON Schemas** (`just gen-contract`), so the types cannot
+ * drift from the binary. Test frameworks build on this — `@skill-test/vitest`
+ * adds the vitest helpers on top.
+ *
+ * ```ts
+ * import { runSkill, assistantText, describeFailures } from "@skill-test/sdk";
+ *
+ * const report = await runSkill("cases/greet.yaml");
+ * if (!report.passed) throw new Error(describeFailures(report));
+ * ```
+ */
+export { runSkill, validateSkill, ENV_BIN, ENV_PROVIDER } from "./runner.js";
+export { SkilltestError, SkilltestProviderError, SkilltestUsageError, } from "./errors.js";
+export { assistantText, describeFailures } from "./helpers.js";

package/dist/runner.d.ts

@@ -0,0 +1,32 @@
+import type { Report } from "./generated/report.js";
+import type { ValidationReport } from "./generated/validation.js";
+/** Environment variables supplying defaults for the binary and provider. */
+export declare const ENV_BIN = "SKILLTEST_BIN";
+export declare const ENV_PROVIDER = "SKILLTEST_PROVIDER";
+export interface RunOptions {
+    /** Path to the `skilltest` binary (default: `$SKILLTEST_BIN` or `skilltest`). */
+    bin?: string;
+    /** Provider command (default: `$SKILLTEST_PROVIDER`). A string or argv array. */
+    provider?: string | string[];
+    /** Harness platforms to run on (overrides config). */
+    platforms?: string[];
+    /** Models to run on (overrides config). */
+    models?: string[];
+    /** Model used for evals and the simulated user. */
+    judgeModel?: string;
+    /** Cap on assistant turns for multi-turn cases. */
+    maxTurns?: number;
+    /** Path to a config file. */
+    config?: string;
+    /** Working directory for the subprocess. */
+    cwd?: string;
+}
+/**
+ * Run one or more test cases and return the parsed {@link Report}. A failing
+ * eval is reported in `report.passed`, not thrown; only bad input
+ * ({@link SkilltestUsageError}) and provider failures
+ * ({@link SkilltestProviderError}) throw.
+ */
+export declare function runSkill(casePath: string, options?: RunOptions): Promise<Report>;
+/** Validate a skill directory (or a folder of them) and return findings. */
+export declare function validateSkill(path: string, options?: Pick<RunOptions, "bin" | "cwd">): Promise<ValidationReport>;

package/dist/runner.js

@@ -0,0 +1,92 @@
+/**
+ * Run the `skilltest` CLI as a subprocess and parse its JSON contract.
+ *
+ * This is the code-level API: call {@link runSkill}, get a typed
+ * {@link Report}, assert on `report.passed`, and mix in deterministic checks
+ * against the transcript.
+ */
+import { spawn } from "node:child_process";
+import { SkilltestError, SkilltestProviderError, SkilltestUsageError } from "./errors.js";
+/** Environment variables supplying defaults for the binary and provider. */
+export const ENV_BIN = "SKILLTEST_BIN";
+export const ENV_PROVIDER = "SKILLTEST_PROVIDER";
+function resolveBin(bin) {
+    return bin ?? process.env[ENV_BIN] ?? "skilltest";
+}
+function resolveProvider(provider) {
+    const value = provider ?? process.env[ENV_PROVIDER];
+    if (value === undefined)
+        return undefined;
+    return Array.isArray(value) ? value.join(" ") : value;
+}
+function capture(bin, args, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(bin, args, { cwd });
+        let stdout = "";
+        let stderr = "";
+        child.stdout.on("data", (chunk) => {
+            stdout += chunk.toString();
+        });
+        child.stderr.on("data", (chunk) => {
+            stderr += chunk.toString();
+        });
+        child.on("error", (err) => reject(new SkilltestProviderError(`could not run skilltest binary \`${bin}\`: ${err.message}. Set ${ENV_BIN} or pass bin.`)));
+        child.on("close", (status) => resolve({ status, stdout, stderr }));
+    });
+}
+// Exit codes that still produce a JSON report (0 = all passed, 1 = some failed).
+function raiseForStatus(result) {
+    if (result.status === 0 || result.status === 1)
+        return;
+    const detail = result.stderr.trim() || result.stdout.trim();
+    if (result.status === 2)
+        throw new SkilltestUsageError(detail);
+    if (result.status === 3)
+        throw new SkilltestProviderError(detail);
+    throw new SkilltestError(`skilltest exited ${result.status}: ${detail}`);
+}
+// The cast is sound by construction: the SDK's types are generated from the
+// CLI's own JSON Schemas and the contract drift gate (`just gen-contract
+// --check` in CI) fails when they diverge, so the shape is not re-validated
+// here at runtime.
+function parse(stdout) {
+    try {
+        return JSON.parse(stdout);
+    }
+    catch (err) {
+        throw new SkilltestError(`skilltest did not emit JSON: ${err.message}`);
+    }
+}
+/**
+ * Run one or more test cases and return the parsed {@link Report}. A failing
+ * eval is reported in `report.passed`, not thrown; only bad input
+ * ({@link SkilltestUsageError}) and provider failures
+ * ({@link SkilltestProviderError}) throw.
+ */
+export async function runSkill(casePath, options = {}) {
+    const args = [];
+    if (options.config)
+        args.push("--config", options.config);
+    args.push("run", casePath, "--format", "json");
+    const provider = resolveProvider(options.provider);
+    if (provider !== undefined)
+        args.push("--provider", provider);
+    for (const platform of options.platforms ?? [])
+        args.push("--platform", platform);
+    for (const model of options.models ?? [])
+        args.push("--model", model);
+    if (options.judgeModel)
+        args.push("--judge-model", options.judgeModel);
+    if (options.maxTurns !== undefined)
+        args.push("--max-turns", String(options.maxTurns));
+    const result = await capture(resolveBin(options.bin), args, options.cwd);
+    raiseForStatus(result);
+    return parse(result.stdout);
+}
+/** Validate a skill directory (or a folder of them) and return findings. */
+export async function validateSkill(path, options = {}) {
+    const args = ["validate", path, "--format", "json"];
+    const result = await capture(resolveBin(options.bin), args, options.cwd);
+    raiseForStatus(result);
+    return parse(result.stdout);
+}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@skill-test/sdk",
+  "version": "0.1.1",
+  "description": "TypeScript SDK for the skilltest CLI: run AI-skill tests and natural-language evals from TypeScript, with a typed report contract generated from the CLI's JSON Schemas.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nickderobertis/skilltest.git",
+    "directory": "sdks/typescript"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "devDependencies": {
+    "@types/node": "^22.7.5",
+    "json-schema-to-typescript": "^15.0.4",
+    "typescript": "^5.6.3",
+    "vitest": "^2.1.3"
+  },
+  "nx": {
+    "includedScripts": []
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.json",
+    "test": "vitest run"
+  }
+}