@skill-test/vitest 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nick DeRobertis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,68 @@
+# @skill-test/vitest
+
+A [vitest](https://vitest.dev) plugin for [skilltest](../../README.md): run
+AI-skill tests and natural-language evals from vitest, and mix in your own
+deterministic checks. Built on
+[`@skill-test/sdk`](../../sdks/typescript/README.md) — the SDK's API is
+re-exported here, so a vitest suite needs only this one dependency.
+
+## As code
+
+```ts
+import { runSkill, assistantText } from "@skill-test/vitest";
+
+test("greeter names the patient", async () => {
+  const report = await runSkill("cases/greet.yaml", {
+    platforms: ["claude-code"],
+    models: ["claude-opus-4-8"],
+  });
+  expect(report.passed, report.runs[0]?.case).toBe(true);
+  expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
+});
+```
+
+## One-liner
+
+```ts
+import { skillTest } from "@skill-test/vitest";
+
+skillTest("greeter confirms the appointment", "cases/greet.yaml");
+```
+
+## Recommended: auto-discover a tree of cases
+
+When vitest is your primary test runner, keep your cases as data and let one
+test module collect them. Name each case `*.skilltest.yaml` (or `.yml`) and add
+a single `skills.test.ts`:
+
+```ts
+// skills.test.ts
+import { discover } from "@skill-test/vitest";
+
+discover("cases"); // registers one vitest test per *.skilltest.yaml under cases/
+```
+
+```yaml
+# cases/greet.skilltest.yaml
+skill: ./skills/greeter
+input: "Greet Dr. Smith."
+evals:
+  - type: boolean
+    criterion: "the reply greets Dr. Smith by name"
+```
+
+This is the closest vitest equivalent to pytest's auto-collection: vitest only
+collects its own test modules, so the one-line `discover()` call stands in for a
+file collector. Adding a case is then just dropping in a YAML file — no code
+change. Pass run options as the second argument (`discover("cases", { platforms:
+["claude-code"] })`); for matrices or deterministic mix-in assertions, reach for
+`runSkill` in an ordinary `test()` instead.
+
+## Configuration
+
+The plugin shells out to the `skilltest` binary. Point at one with the
+`SKILLTEST_BIN` env var (or the `bin` option) and the provider with
+`SKILLTEST_PROVIDER` (or the `provider` option). A failing eval is returned in
+`report.passed`; bad input and provider failures throw `SkilltestUsageError` /
+`SkilltestProviderError`. See the repository root for the provider protocol and
+full schema.

package/dist/index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * `@skill-test/vitest` — run AI-skill tests and natural-language evals in vitest.
+ *
+ * The vitest integration on top of `@skill-test/sdk`, whose API is re-exported
+ * here so a vitest suite needs only this one dependency:
+ *
+ * ```ts
+ * import { skillTest, discover, runSkill, assistantText } from "@skill-test/vitest";
+ *
+ * skillTest("greeter names the patient", "cases/greet.yaml");
+ * // or auto-discover a tree of *.skilltest.yaml cases:
+ * discover("cases");
+ *
+ * // For matrices or deterministic mix-in checks, use the SDK API directly:
+ * test("greeter", async () => {
+ *   const report = await runSkill("cases/greet.yaml");
+ *   expect(report.passed).toBe(true);
+ *   expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
+ * });
+ * ```
+ *
+ * This module (via the helpers) imports `vitest`, so only load it inside a
+ * vitest run. `@skill-test/vitest/vitest` remains as an alias for the helpers.
+ */
+export { skillTest, discover, CASE_SUFFIXES } from "./vitest.js";
+export * from "@skill-test/sdk";

package/dist/index.js

@@ -0,0 +1,26 @@
+/**
+ * `@skill-test/vitest` — run AI-skill tests and natural-language evals in vitest.
+ *
+ * The vitest integration on top of `@skill-test/sdk`, whose API is re-exported
+ * here so a vitest suite needs only this one dependency:
+ *
+ * ```ts
+ * import { skillTest, discover, runSkill, assistantText } from "@skill-test/vitest";
+ *
+ * skillTest("greeter names the patient", "cases/greet.yaml");
+ * // or auto-discover a tree of *.skilltest.yaml cases:
+ * discover("cases");
+ *
+ * // For matrices or deterministic mix-in checks, use the SDK API directly:
+ * test("greeter", async () => {
+ *   const report = await runSkill("cases/greet.yaml");
+ *   expect(report.passed).toBe(true);
+ *   expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
+ * });
+ * ```
+ *
+ * This module (via the helpers) imports `vitest`, so only load it inside a
+ * vitest run. `@skill-test/vitest/vitest` remains as an alias for the helpers.
+ */
+export { skillTest, discover, CASE_SUFFIXES } from "./vitest.js";
+export * from "@skill-test/sdk";

package/dist/vitest.d.ts

@@ -0,0 +1,16 @@
+import { type RunOptions } from "@skill-test/sdk";
+/** Filename suffixes a case file must carry to be auto-discovered. */
+export declare const CASE_SUFFIXES: readonly [".skilltest.yaml", ".skilltest.yml"];
+/** Register a vitest test that runs `casePath` and asserts every eval passed. */
+export declare function skillTest(name: string, casePath: string, options?: RunOptions): void;
+/**
+ * Recursively find every `*.skilltest.yaml` / `*.skilltest.yml` case under `dir`
+ * and register each as a vitest test (named by its path relative to `dir`).
+ *
+ * vitest only collects its own test modules, so it can't pick up bare YAML the
+ * way pytest's collector does. Calling `discover` from a single `*.test.ts` is
+ * the closest equivalent: one line gives you pytest-style auto-collection. Cases
+ * are sorted for a stable order, and discovery is synchronous so the tests are
+ * registered before vitest collects the file.
+ */
+export declare function discover(dir?: string, options?: RunOptions): void;

package/dist/vitest.js

@@ -0,0 +1,54 @@
+/**
+ * The vitest helpers: register a skill case as a vitest test in one line with
+ * {@link skillTest}, or auto-discover a tree of cases (the recommended setup
+ * when vitest is your primary runner) with {@link discover}:
+ *
+ * ```ts
+ * import { skillTest, discover } from "@skill-test/vitest";
+ * skillTest("greeter names the patient", "cases/greet.yaml");
+ * discover("cases");
+ * ```
+ *
+ * For matrices or deterministic mix-in checks, call the SDK's `runSkill` from
+ * an ordinary `test()` instead. This module imports `vitest`, so only load it
+ * inside a vitest run.
+ */
+import { readdirSync } from "node:fs";
+import { join, relative } from "node:path";
+import { describeFailures, runSkill } from "@skill-test/sdk";
+import { expect, test } from "vitest";
+/** Filename suffixes a case file must carry to be auto-discovered. */
+export const CASE_SUFFIXES = [".skilltest.yaml", ".skilltest.yml"];
+/** Register a vitest test that runs `casePath` and asserts every eval passed. */
+export function skillTest(name, casePath, options = {}) {
+    test(name, async () => {
+        const report = await runSkill(casePath, options);
+        expect(report.passed, describeFailures(report)).toBe(true);
+    });
+}
+/**
+ * Recursively find every `*.skilltest.yaml` / `*.skilltest.yml` case under `dir`
+ * and register each as a vitest test (named by its path relative to `dir`).
+ *
+ * vitest only collects its own test modules, so it can't pick up bare YAML the
+ * way pytest's collector does. Calling `discover` from a single `*.test.ts` is
+ * the closest equivalent: one line gives you pytest-style auto-collection. Cases
+ * are sorted for a stable order, and discovery is synchronous so the tests are
+ * registered before vitest collects the file.
+ */
+export function discover(dir = ".", options = {}) {
+    let entries;
+    try {
+        entries = readdirSync(dir, { recursive: true, withFileTypes: true });
+    }
+    catch (err) {
+        throw new Error(`skilltest discover: cannot read directory \`${dir}\`: ${err.message}`);
+    }
+    const cases = entries
+        .filter((entry) => entry.isFile() && CASE_SUFFIXES.some((suffix) => entry.name.endsWith(suffix)))
+        .map((entry) => join(entry.parentPath, entry.name))
+        .sort();
+    for (const casePath of cases) {
+        skillTest(relative(dir, casePath) || casePath, casePath, options);
+    }
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@skill-test/vitest",
+  "version": "0.1.1",
+  "description": "vitest integration for skilltest: register AI-skill test cases as vitest tests, built on @skill-test/sdk.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nickderobertis/skilltest.git",
+    "directory": "plugins/vitest"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./vitest": {
+      "types": "./dist/vitest.d.ts",
+      "default": "./dist/vitest.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@skill-test/sdk": "0.1.1"
+  },
+  "peerDependencies": {
+    "vitest": ">=1.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.7.5",
+    "typescript": "^5.6.3",
+    "vitest": "^2.1.3"
+  },
+  "nx": {
+    "includedScripts": []
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.json",
+    "test": "vitest run"
+  }
+}