@skill-test/vitest 0.5.0 → 0.6.0

package/README.md

@@ -6,29 +6,51 @@ deterministic checks. Built on
 [`@skill-test/sdk`](../../sdks/typescript/README.md) — the SDK's API is
 re-exported here, so a vitest suite needs only this one dependency.
 
-## As code
+## Define the whole case in code (recommended)
+
+Build the case — skill, input, evals, an optional simulated user, mocks — right
+in the test, and register it in one line with `skillTest`. Everything the YAML
+carries has a typed builder:
 
 ```ts
-import { runSkill, assistantText } from "@skill-test/vitest";
-
-test("greeter names the patient", async () => {
-  const report = await runSkill("cases/greet.yaml", {
-    platforms: ["claude-code"],
-    models: ["claude-opus-4-8"],
-  });
-  expect(report.passed, report.runs[0]?.case).toBe(true);
-  expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
-});
+import { skillTest, testCase, boolean, numeric } from "@skill-test/vitest";
+
+skillTest(
+  "greeter names the patient",
+  testCase({
+    skill: "skills/greeter",            // resolved relative to the working dir
+    input: "Greet Dr. Smith, who has an appointment today.",
+    evals: [
+      boolean("the reply greets Dr. Smith by name"),
+      numeric("how warm is the tone", { min: 0, max: 10, threshold: 7 }),
+    ],
+  }),
+);
 ```
 
-## One-liner
+Multi-turn cases add `user(...)`; call-count checks use `called` / `notCalled`
+referencing a named `stub`/`spy`. For a matrix or extra deterministic checks,
+call `runSkill` from an ordinary `test` — it takes the same case object:
 
 ```ts
-import { skillTest } from "@skill-test/vitest";
-
-skillTest("greeter confirms the appointment", "cases/greet.yaml");
+import { runSkill, testCase, boolean, assistantText } from "@skill-test/vitest";
+
+test("greeter across the matrix", async () => {
+  const report = await runSkill(
+    testCase({ skill: "skills/greeter", input: "Greet Dr. Smith", evals: [boolean("greets by name")] }),
+    { platforms: ["claude-code"], models: ["claude-opus-4-8"] },
+  );
+  expect(report.passed).toBe(true);
+  expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
+});
 ```
 
+## Or point at a YAML file
+
+`skillTest` and `runSkill` accept a path just as well
+(`skillTest("greeter", "cases/greet.yaml")`). The full field reference for both
+forms is [`docs/schema.md`](../../docs/schema.md).
+
 ## Assert on tool use, and stream
 
 The SDK's tool-event and streaming surfaces are re-exported too. `toolCalls`
@@ -38,27 +60,33 @@ test can **short-circuit** on bad behavior:
 
 ```ts
 import { it, expect } from "vitest";
-import { runSkill, toolCalls, streamSkill } from "@skill-test/vitest";
+import { runSkill, streamSkill, testCase, toolCalls, boolean } from "@skill-test/vitest";
+
+const editCase = testCase({
+  skill: "skills/editor",
+  input: "Update the config and commit it.",
+  evals: [boolean("the change was committed")],
+});
 
 it("commits without deleting", async () => {
-  const report = await runSkill("cases/edit.skilltest.yaml");
+  const report = await runSkill(editCase);
   const calls = toolCalls(report.runs[0]!.transcript);
   expect(calls.some((c) => String(c.input?.command).includes("git commit"))).toBe(true);
   expect(calls.some((c) => String(c.input?.command).includes("rm -rf"))).toBe(false);
 });
 
 it("makes no network call", async () => {
-  for await (const ev of streamSkill("cases/edit.skilltest.yaml")) {
+  for await (const ev of streamSkill(editCase)) {
     expect(ev.event.name).not.toBe("curl"); // break to abort early
   }
 });
 ```
 
-## Recommended: auto-discover a tree of cases
+## Auto-discover existing YAML cases
 
-When vitest is your primary test runner, keep your cases as data and let one
-test module collect them. Name each case `*.skilltest.yaml` (or `.yml`) and add
-a single `skills.test.ts`:
+Cases can also live as data: name each `*.skilltest.yaml` (or `.yml`) and let
+one test module collect the whole tree — useful when a suite already has YAML
+cases, or when non-engineers author them:
 
 ```ts
 // skills.test.ts
@@ -80,8 +108,7 @@ This is the closest vitest equivalent to pytest's auto-collection: vitest only
 collects its own test modules, so the one-line `discover()` call stands in for a
 file collector. Adding a case is then just dropping in a YAML file — no code
 change. Pass run options as the second argument (`discover("cases", { platforms:
-["claude-code"] })`); for matrices or deterministic mix-in assertions, reach for
-`runSkill` in an ordinary `test()` instead.
+["claude-code"] })`). YAML cases and code-defined ones mix freely in one suite.
 
 ## Configuration
 

package/dist/index.d.ts

@@ -2,23 +2,27 @@
  * `@skill-test/vitest` — run AI-skill tests and natural-language evals in vitest.
  *
  * The vitest integration on top of `@skill-test/sdk`, whose API is re-exported
- * here so a vitest suite needs only this one dependency:
+ * here so a vitest suite needs only this one dependency. Define the whole case
+ * in code (the recommended form) and register it in one line:
  *
  * ```ts
- * import { skillTest, discover, runSkill, assistantText } from "@skill-test/vitest";
+ * import { skillTest, testCase, boolean } from "@skill-test/vitest";
  *
- * skillTest("greeter names the patient", "cases/greet.yaml");
- * // or auto-discover a tree of *.skilltest.yaml cases:
- * discover("cases");
- *
- * // For matrices or deterministic mix-in checks, use the SDK API directly:
- * test("greeter", async () => {
- *   const report = await runSkill("cases/greet.yaml");
- *   expect(report.passed).toBe(true);
- *   expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
- * });
+ * skillTest(
+ *   "greeter names the patient",
+ *   testCase({
+ *     skill: "skills/greeter",
+ *     input: "Greet Dr. Smith.",
+ *     evals: [boolean("the reply greets Dr. Smith by name")],
+ *   }),
+ * );
  * ```
  *
+ * Existing YAML cases stay first-class: `skillTest("greeter", "cases/greet.yaml")`
+ * runs a file, and `discover("cases")` auto-collects a whole tree of
+ * `*.skilltest.yaml` cases. For matrices or deterministic mix-in checks, call
+ * the SDK's `runSkill` from an ordinary `test()`.
+ *
  * This module (via the helpers) imports `vitest`, so only load it inside a
  * vitest run. `@skill-test/vitest/vitest` remains as an alias for the helpers.
  */

package/dist/index.js

@@ -2,23 +2,27 @@
  * `@skill-test/vitest` — run AI-skill tests and natural-language evals in vitest.
  *
  * The vitest integration on top of `@skill-test/sdk`, whose API is re-exported
- * here so a vitest suite needs only this one dependency:
+ * here so a vitest suite needs only this one dependency. Define the whole case
+ * in code (the recommended form) and register it in one line:
  *
  * ```ts
- * import { skillTest, discover, runSkill, assistantText } from "@skill-test/vitest";
+ * import { skillTest, testCase, boolean } from "@skill-test/vitest";
  *
- * skillTest("greeter names the patient", "cases/greet.yaml");
- * // or auto-discover a tree of *.skilltest.yaml cases:
- * discover("cases");
- *
- * // For matrices or deterministic mix-in checks, use the SDK API directly:
- * test("greeter", async () => {
- *   const report = await runSkill("cases/greet.yaml");
- *   expect(report.passed).toBe(true);
- *   expect(assistantText(report.runs[0]!.transcript)).toContain("Dr. Smith");
- * });
+ * skillTest(
+ *   "greeter names the patient",
+ *   testCase({
+ *     skill: "skills/greeter",
+ *     input: "Greet Dr. Smith.",
+ *     evals: [boolean("the reply greets Dr. Smith by name")],
+ *   }),
+ * );
  * ```
  *
+ * Existing YAML cases stay first-class: `skillTest("greeter", "cases/greet.yaml")`
+ * runs a file, and `discover("cases")` auto-collects a whole tree of
+ * `*.skilltest.yaml` cases. For matrices or deterministic mix-in checks, call
+ * the SDK's `runSkill` from an ordinary `test()`.
+ *
  * This module (via the helpers) imports `vitest`, so only load it inside a
  * vitest run. `@skill-test/vitest/vitest` remains as an alias for the helpers.
  */

package/dist/vitest.d.ts

@@ -1,8 +1,13 @@
-import { type RunOptions } from "@skill-test/sdk";
+import { type RunOptions, type TestCaseInput } from "@skill-test/sdk";
 /** Filename suffixes a case file must carry to be auto-discovered. */
 export declare const CASE_SUFFIXES: readonly [".skilltest.yaml", ".skilltest.yml"];
-/** Register a vitest test that runs `casePath` and asserts every eval passed. */
-export declare function skillTest(name: string, casePath: string, options?: RunOptions): void;
+/**
+ * Register a vitest test that runs a case and asserts every eval passed. The
+ * case may be a code-defined {@link TestCaseInput} (the recommended form —
+ * build it with `testCase`/`boolean`/`numeric`/… re-exported here) or a path to
+ * a YAML file.
+ */
+export declare function skillTest(name: string, caseInput: string | TestCaseInput, options?: RunOptions): void;
 /**
  * Recursively find every `*.skilltest.yaml` / `*.skilltest.yml` case under `dir`
  * and register each as a vitest test (named by its path relative to `dir`).

package/dist/vitest.js

@@ -19,10 +19,15 @@ import { describeFailures, runSkill } from "@skill-test/sdk";
 import { expect, test } from "vitest";
 /** Filename suffixes a case file must carry to be auto-discovered. */
 export const CASE_SUFFIXES = [".skilltest.yaml", ".skilltest.yml"];
-/** Register a vitest test that runs `casePath` and asserts every eval passed. */
-export function skillTest(name, casePath, options = {}) {
+/**
+ * Register a vitest test that runs a case and asserts every eval passed. The
+ * case may be a code-defined {@link TestCaseInput} (the recommended form —
+ * build it with `testCase`/`boolean`/`numeric`/… re-exported here) or a path to
+ * a YAML file.
+ */
+export function skillTest(name, caseInput, options = {}) {
     test(name, async () => {
-        const report = await runSkill(casePath, options);
+        const report = await runSkill(caseInput, options);
         expect(report.passed, describeFailures(report)).toBe(true);
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skill-test/vitest",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "vitest integration for skilltest: register AI-skill test cases as vitest tests, built on @skill-test/sdk.",
   "license": "MIT",
   "repository": {
@@ -28,7 +28,7 @@
     "dist"
   ],
   "dependencies": {
-    "@skill-test/sdk": "0.5.0"
+    "@skill-test/sdk": "0.6.0"
   },
   "peerDependencies": {
     "vitest": ">=1.0.0"