katt 0.0.7 → 0.0.8

package/.nvmrc

@@ -0,0 +1 @@
+24

package/README.md

@@ -68,6 +68,7 @@ describe("Greeting agent", () => {
 - **Classification Matcher**: Built-in `toBeClassifiedAs()` matcher to grade a response against a target label on a 1-5 scale
 - **Concurrent Execution**: Runs eval files concurrently for faster test execution
 - **Model Selection**: Support for specifying custom AI models
+- **Runtime Selection**: Run prompts through GitHub Copilot (default) or Codex
 - **Configurable Timeouts**: Override prompt wait time per test or via `katt.json`
 
 ## Usage
@@ -127,11 +128,14 @@ describe("Model selection", () => {
 });
 ```
 
-You can also set a default model for the project by adding a `katt.json` file in the project root:
+You can also set runtime defaults in `katt.json`.
+
+Copilot (default runtime):
 
 ```json
 {
-  "copilot": {
+  "agent": "gh-copilot",
+  "agentOptions": {
     "model": "gpt-5-mini"
   },
   "prompt": {
@@ -140,10 +144,29 @@ You can also set a default model for the project by adding a `katt.json` file in
 }
 ```
 
+Codex:
+
+```json
+{
+  "agent": "codex",
+  "agentOptions": {
+    "model": "gpt-5-codex",
+    "profile": "default",
+    "sandbox": "workspace-write"
+  },
+  "prompt": {
+    "timeoutMs": 240000
+  }
+}
+```
+
 When this file exists:
 
-- `prompt("...")` and `promptFile("...")` use `copilot.model` by default
-- `prompt("...", { model: "..." })` still overrides the config value
+- Supported agents are:
+  - `gh-copilot` (default when `agent` is missing or unsupported)
+  - `codex`
+- `prompt("...")` and `promptFile("...")` merge `agentOptions` with call-time options
+- `prompt("...", { model: "..." })` overrides the model from config
 - `prompt.timeoutMs` sets the default wait timeout for long-running prompts
 
 ## Development
@@ -201,8 +224,8 @@ katt/
 ## Requirements
 
 - Node.js
-- GitHub Copilot CLI installed (see [GitHub Copilot CLI installation docs](https://docs.github.com/en/copilot/how-tos/copilot-cli/install-copilot-cli))
-- Access to AI models (e.g., OpenAI API key for Codex)
+- For `gh-copilot` runtime: access to GitHub Copilot with a logged-in user
+- For `codex` runtime: Codex CLI installed and authenticated (`codex login`)
 
 ## License
 

package/build-tests/__snapshots__/check1.snap.md

@@ -0,0 +1 @@
+heeey

package/build-tests/__snapshots__/check1__Hello_World__should_return_the_date_in_a_json_format.snap.md

@@ -0,0 +1 @@
+{ year: 2026 }

package/build-tests/__snapshots__/check1__root.snap.md

@@ -0,0 +1 @@
+heeey

package/build-tests/check1.eval.js

@@ -0,0 +1,19 @@
+import { describe, expect, it, prompt } from "katt";
+
+describe('Hello World', () => {
+    it('should return the date in a json format', async () => {
+        const currentData = new Date(Date.now());
+
+        const result = await prompt('Return the current year in the format "{ year: YYYY }"');
+        expect(result).toContain(`{ year: ${currentData.getFullYear()} }`);
+    });
+
+    it('should classify a response as helpful', async () => {
+        const response = await prompt('You are a helpful assistant. Give one short tip for learning JavaScript.');
+        await expect(response).toBeClassifiedAs('helpful', { threshold: 3 });
+    });
+});
+
+
+const result2 = await prompt('If you read this just say heeey');
+expect(result2.toLowerCase()).toMatchSnapshot();

package/build-tests/check2.eval.js

@@ -0,0 +1,15 @@
+import { describe, expect, it, prompt, promptFile } from "katt";
+
+describe('Working with files', () => {
+    it('It should load the file and compare', async () => {
+        const result = await promptFile('./customPrompt.md');
+        expect(result.toLowerCase()).toContain('hola');
+    });
+});
+
+describe('Working with prompt as expectation', () => {
+    it('It should be friendly', async () => {
+        const result = await prompt('You are a friendly assistant. If you read this, say "Hola"!', { model: 'gpt-5.2' });
+        expect(result).promptCheck('To be friendly, the response should contain a greeting.');
+    });
+});

package/build-tests/customPrompt.md

@@ -0,0 +1 @@
+If you read this, say "Hola"!

package/dist/index.js

@@ -1,191 +1,249 @@
-import { r as E, c as N, a as O, p as I, b as m, l as d, d as w, g as S, e as U, f as C, s as z, h as R, i as A, j as B, k as W, m as K, n as H, o as Y, q } from "./runCli-C7uxWavX.js";
-import { t as Et } from "./runCli-C7uxWavX.js";
-import { CopilotClient as J } from "@github/copilot-sdk";
-import { readFile as j } from "node:fs/promises";
-import { resolve as P, dirname as v, isAbsolute as G, basename as Q, join as V } from "node:path";
-import { readFileSync as X, writeFileSync as F, mkdirSync as Z } from "node:fs";
-function Tt(t, e) {
-  E(() => {
-    O(), I(t);
-    const n = S(), o = Date.now(), i = () => S() === n, r = () => Date.now() - o;
+import { r as N, c as I, a as j, p as z, b as C, l as h, d as T, g as D, e as B, f as $, h as H, s as A, i as W, j as L, k as G, m as X, n as Y, o as q, q as K, t as J } from "./runCli-425rgVp8.js";
+import { u as jt } from "./runCli-425rgVp8.js";
+import { CopilotClient as Q } from "@github/copilot-sdk";
+import { mkdtemp as V, rm as Z, readFile as R } from "node:fs/promises";
+import { join as S, dirname as v, isAbsolute as tt, resolve as et, basename as nt } from "node:path";
+import { spawn as ot } from "node:child_process";
+import { tmpdir as rt } from "node:os";
+import { readFileSync as st, writeFileSync as M, mkdirSync as it } from "node:fs";
+function Pt(t, e) {
+  N(() => {
+    j(), z(t);
+    const n = D(), o = Date.now(), i = () => D() === n, s = () => Date.now() - o;
     try {
-      const s = e();
-      if (s && typeof s.then == "function") {
-        m(
-          s.then(() => {
-            d(!0, r());
+      const r = e();
+      if (r && typeof r.then == "function") {
+        C(
+          r.then(() => {
+            h(!0, s());
           }).catch((a) => {
-            throw d(!1, r()), a;
+            throw h(!1, s()), a;
           }).finally(() => {
-            w();
+            T();
           })
         );
         return;
       }
-    } catch (s) {
-      throw d(!1, r()), w(), s;
+    } catch (r) {
+      throw h(!1, s()), T(), r;
     }
-    d(i(), r()), w();
-  }, N());
+    h(i(), s()), T();
+  }, I());
 }
-function St(t, e) {
-  E(() => {
-    U(t);
+function Nt(t, e) {
+  N(() => {
+    B(t);
     try {
       const n = e();
       if (n && typeof n.then == "function") {
-        m(
+        C(
           n.finally(() => {
-            C();
+            $();
           })
         );
         return;
       }
     } catch (n) {
-      throw C(), n;
+      throw $(), n;
     }
-    C();
-  }, N());
+    $();
+  }, I());
 }
-function tt(t, e) {
+const at = "katt-codex-", ct = "last-message.txt";
+function ut(t, e) {
   return typeof t == "object" && t !== null && "code" in t && t.code === e;
 }
-function et(t) {
-  try {
-    const e = JSON.parse(t);
-    return typeof e == "object" && e !== null ? e : void 0;
-  } catch (e) {
-    console.warn(`Failed to parse katt.json: ${String(e)}`);
-    return;
-  }
+function m(t) {
+  return typeof t == "string" && t.length > 0;
 }
-async function nt() {
-  const t = P(process.cwd(), "katt.json");
-  try {
-    const e = await j(t, "utf8");
-    return et(e);
-  } catch (e) {
-    if (tt(e, "ENOENT"))
-      return;
-    console.warn(`Failed to read katt.json: ${String(e)}`);
-    return;
-  }
+function dt(t) {
+  return m(t) ? [t] : Array.isArray(t) ? t.filter(m) : [];
 }
-function ot(t) {
-  const e = t?.copilot;
-  if (typeof e != "object" || e === null || Array.isArray(e))
-    return;
-  const n = {
-    ...e
-  }, o = n.model;
-  return (typeof o != "string" || o.length === 0) && delete n.model, Object.keys(n).length > 0 ? n : void 0;
+function lt(t, e) {
+  const n = e ?? {}, o = [
+    "exec",
+    "--color",
+    "never",
+    "--output-last-message",
+    t
+  ];
+  m(n.model) && o.push("--model", n.model), m(n.profile) && o.push("--profile", n.profile), m(n.sandbox) && o.push("--sandbox", n.sandbox), n.fullAuto === !0 && o.push("--full-auto"), n.skipGitRepoCheck === !0 && o.push("--skip-git-repo-check"), n.dangerouslyBypassApprovalsAndSandbox === !0 && o.push("--dangerously-bypass-approvals-and-sandbox");
+  for (const i of dt(n.config))
+    o.push("--config", i);
+  return o.push("-"), o;
 }
-function rt(t) {
-  if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
-    return Math.floor(t);
+function ft(t, e, n, o) {
+  return new Promise((i, s) => {
+    const r = ot("codex", e, {
+      cwd: o,
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    let a = "", c = "", p = !1;
+    r.stdout.setEncoding("utf8"), r.stderr.setEncoding("utf8"), r.stdout.on("data", (u) => {
+      a += u;
+    }), r.stderr.on("data", (u) => {
+      c += u;
+    }), r.stdin.on("error", () => {
+    });
+    const g = setTimeout(() => {
+      p = !0, r.kill("SIGTERM");
+    }, n);
+    r.once("error", (u) => {
+      clearTimeout(g), s(
+        new Error(
+          `Failed to start Codex CLI. Ensure codex is installed and available on PATH. ${String(
+            u
+          )}`
+        )
+      );
+    }), r.once("close", (u, x) => {
+      clearTimeout(g), i({
+        exitCode: u,
+        signal: x,
+        stdout: a.trim(),
+        stderr: c.trim(),
+        timedOut: p
+      });
+    }), r.stdin.end(t);
+  });
 }
-function it(t) {
-  const e = t?.prompt;
-  if (!(typeof e != "object" || e === null || Array.isArray(e)))
-    return rt(e.timeoutMs);
+async function pt(t, e) {
+  try {
+    return await R(t, "utf8");
+  } catch (n) {
+    if (!ut(n, "ENOENT"))
+      throw n;
+    return e;
+  }
 }
-async function st() {
-  const t = await nt();
-  return {
-    copilot: ot(t),
-    promptTimeoutMs: it(t)
-  };
+function ht(t) {
+  if (t.timedOut)
+    return "Codex timed out before returning a response.";
+  if (t.exitCode === null)
+    return `Codex exited due to signal ${t.signal ?? "unknown"}.`;
+  const e = t.stderr.length > 0 ? ` ${t.stderr}` : "";
+  return `Codex exited with code ${t.exitCode}.${e}`;
 }
-const at = 6e5;
-function _(t) {
+async function mt(t, e, n) {
+  const o = n ?? {}, i = m(o.workingDirectory) ? o.workingDirectory : process.cwd(), s = await V(S(rt(), at)), r = S(s, ct);
+  try {
+    const a = lt(r, n), c = await ft(
+      t,
+      a,
+      e,
+      i
+    );
+    if (c.timedOut)
+      throw new Error(`Codex timed out after ${e}ms.`);
+    if (c.exitCode !== 0)
+      throw new Error(ht(c));
+    const p = await pt(r, c.stdout);
+    if (p.length === 0)
+      throw new Error("Codex did not return a response.");
+    return p;
+  } finally {
+    await Z(s, { recursive: !0, force: !0 });
+  }
+}
+const gt = 6e5;
+function U(t) {
   return typeof t == "string" && t.length > 0 ? t : void 0;
 }
-function b(t) {
+function E(t) {
   if (!t)
     return;
   const e = { ...t };
   if (e.model !== void 0) {
-    const n = _(e.model);
+    const n = U(
+      typeof e.model == "string" ? e.model : void 0
+    );
     n ? e.model = n : delete e.model;
   }
   return Object.keys(e).length > 0 ? e : void 0;
 }
-function M(t) {
+function O(t) {
   if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
     return Math.floor(t);
 }
-function h(t) {
+function y(t) {
   return !Number.isFinite(t) || (t ?? 0) <= 0 ? 0 : Math.floor(t ?? 0);
 }
-function ct(t) {
-  return h(t.inputTokens) + h(t.outputTokens) + h(t.cacheReadTokens) + h(t.cacheWriteTokens);
+function xt(t) {
+  return y(t.inputTokens) + y(t.outputTokens) + y(t.cacheReadTokens) + y(t.cacheWriteTokens);
 }
-async function $(t, e = {}) {
-  const { timeoutMs: n, ...o } = e, i = await st(), r = b(i.copilot), s = b(
+async function F(t, e = {}) {
+  const { timeoutMs: n, ...o } = e, i = await H(), s = E(i.agentOptions), r = E(
     o
-  ), a = b({
-    ...r ?? {},
-    ...s ?? {}
-  }), c = M(i.promptTimeoutMs), L = M(n) ?? c ?? at, x = _(a?.model), g = new J({ useLoggedInUser: !0 });
-  let p, T, y = 0;
+  ), a = E({
+    ...s ?? {},
+    ...r ?? {}
+  }), c = O(i.promptTimeoutMs), g = O(n) ?? c ?? gt, u = U(
+    typeof a?.model == "string" ? a.model : void 0
+  );
+  if (i.agent === "codex") {
+    const d = await mt(t, g, a);
+    return u && A(u), d;
+  }
+  const x = new Q({ useLoggedInUser: !0 });
+  let w, k, b = 0;
   try {
-    await g.start(), p = await g.createSession(a), T = p.on("assistant.usage", (f) => {
-      y += ct(f.data);
+    await x.start(), w = await x.createSession(a), k = w.on("assistant.usage", (f) => {
+      b += xt(f.data);
     });
-    const l = await p.sendAndWait({ prompt: t }, L);
-    if (!l?.data?.content)
+    const d = await w.sendAndWait({ prompt: t }, g);
+    if (!d?.data?.content)
       throw new Error("Copilot did not return a response.");
-    return x && z(x), l.data.content;
+    return u && A(u), d.data.content;
   } finally {
-    const l = [];
-    if (T?.(), y > 0 && R(y), p)
+    const d = [];
+    if (k?.(), b > 0 && W(b), w)
       try {
-        await p.destroy();
+        await w.destroy();
       } catch (f) {
-        l.push(f);
+        d.push(f);
       }
     try {
-      const f = await g.stop();
-      l.push(...f);
+      const f = await x.stop();
+      d.push(...f);
     } catch (f) {
-      l.push(f);
+      d.push(f);
     }
-    l.length > 0 && console.error(
-      `Copilot cleanup encountered ${l.length} error(s).`
+    d.length > 0 && console.error(
+      `Copilot cleanup encountered ${d.length} error(s).`
     );
   }
 }
-async function Ft(t, e = {}) {
-  const n = A.getStore(), o = n?.evalFile ? v(n.evalFile) : process.cwd(), i = G(t) ? t : P(o, t), r = await j(i, "utf8");
-  return $(r, e);
+async function It(t, e = {}) {
+  const n = L.getStore(), o = n?.evalFile ? v(n.evalFile) : process.cwd(), i = tt(t) ? t : et(o, t), s = await R(i, "utf8");
+  return F(s, e);
 }
-function u(t) {
-  B({
-    describePath: K(),
-    itPath: W(),
+function l(t) {
+  G({
+    describePath: Y(),
+    itPath: X(),
     message: t
   });
 }
-async function ut(t, e) {
-  const n = Date.now(), o = `expected '${t}' to satisfy '${e}'`, i = $(`Evaluate if the expectation is fulfiled in by the input.
+async function wt(t, e) {
+  const n = Date.now(), o = `expected '${t}' to satisfy '${e}'`, i = F(`Evaluate if the expectation is fulfiled in by the input.
         Expectation: "${e}".
         Input:
         ---
         ${t}
         ---
         Important: Answer with "Yes" or "No" only, without any additional text.
-        `).then((r) => {
-    r.includes("Yes") ? d(!0, Date.now() - n, "promptCheck") : r.includes("No") ? u(o) : u(`failed to evaluate expectation '${e}'`);
+        `).then((s) => {
+    s.includes("Yes") ? h(!0, Date.now() - n, "promptCheck") : s.includes("No") ? l(o) : l(`failed to evaluate expectation '${e}'`);
   });
-  return m(i), i;
+  return C(i), i;
 }
-const lt = 3;
-function ft(t) {
+const yt = 3;
+function Ct(t) {
   const e = t.match(/\b([1-5])\b/);
   return !e || !e[1] ? null : Number(e[1]);
 }
-async function dt(t, e, n) {
-  const o = Date.now(), i = n?.threshold ?? lt, r = $(
+async function bt(t, e, n) {
+  const o = Date.now(), i = n?.threshold ?? yt, s = F(
     `Classify the input by how "${e}" it is on a scale of 1 to 5.
 1 means "not ${e}" and 5 means "very ${e}".
 Return only a single number: 1, 2, 3, 4, or 5.
@@ -195,94 +253,94 @@ Input:
 ${t}
 ---`,
     n?.model ? { model: n.model } : void 0
-  ).then((s) => {
-    const a = ft(s);
+  ).then((r) => {
+    const a = Ct(r);
     if (a === null) {
-      u(
-        `failed to classify as '${e}'. Evaluator returned '${s}'`
+      l(
+        `failed to classify as '${e}'. Evaluator returned '${r}'`
       );
       return;
     }
     const c = `expected response to be classified as '${e}' with score >= ${i}, got ${a}`;
     if (a < i) {
-      u(c);
+      l(c);
       return;
     }
-    d(
+    h(
       !0,
       Date.now() - o,
       "toBeClassifiedAs"
     );
   });
-  return m(r), r;
+  return C(s), s;
 }
-function pt(t, e) {
+function Tt(t, e) {
   const n = `expected '${t}' to include '${e}'`;
-  t.includes(e) || u(n);
+  t.includes(e) || l(n);
 }
-function k(t) {
+function _(t) {
   const e = t.trim().replace(/[<>:"/\\|?*\x00-\x1f]/g, "_").replace(/\s+/g, "_");
   return e.length > 0 ? e : "unnamed";
 }
-function ht() {
-  const t = Y().map(
-    (o) => k(o.description)
-  ), e = q().map(
-    (o) => k(o.description)
+function $t() {
+  const t = K().map(
+    (o) => _(o.description)
+  ), e = J().map(
+    (o) => _(o.description)
   ), n = [...t, ...e];
   return n.length === 0 ? "root" : n.join("__");
 }
-function mt(t) {
-  const n = Q(t).replace(/\.eval\.[^./\\]+$/, ""), o = ht();
-  return V(
+function Et(t) {
+  const n = nt(t).replace(/\.eval\.[^./\\]+$/, ""), o = $t();
+  return S(
     v(t),
     "__snapshots__",
     `${n}__${o}.snap.md`
   );
 }
-function D(t) {
+function P(t) {
   return t.split(/\r?\n/);
 }
-function gt(t, e) {
+function St(t, e) {
   if (t === e)
     return "  (no diff)";
-  const n = D(t), o = D(e), i = Math.max(n.length, o.length), r = [];
-  for (let s = 0; s < i; s += 1) {
-    const a = n[s], c = o[s];
+  const n = P(t), o = P(e), i = Math.max(n.length, o.length), s = [];
+  for (let r = 0; r < i; r += 1) {
+    const a = n[r], c = o[r];
     if (a !== c) {
       if (a === void 0 && c !== void 0) {
-        r.push(`+ ${c}`);
+        s.push(`+ ${c}`);
         continue;
       }
       if (a !== void 0 && c === void 0) {
-        r.push(`- ${a}`);
+        s.push(`- ${a}`);
         continue;
       }
-      r.push(`- ${a ?? ""}`), r.push(`+ ${c ?? ""}`);
+      s.push(`- ${a ?? ""}`), s.push(`+ ${c ?? ""}`);
     }
   }
-  return r.join(`
+  return s.join(`
 `);
 }
-function yt(t) {
-  const e = A.getStore()?.evalFile;
+function vt(t) {
+  const e = L.getStore()?.evalFile;
   if (!e) {
-    u(
+    l(
       "toMatchSnapshot can only be used while running an eval file."
     );
     return;
   }
-  const n = mt(e);
+  const n = Et(e);
   try {
-    const o = X(n, "utf8");
+    const o = st(n, "utf8");
     if (o === t)
       return;
-    if (H()) {
-      F(n, t, "utf8");
+    if (q()) {
+      M(n, t, "utf8");
       return;
     }
-    const i = gt(o, t);
-    u(
+    const i = St(o, t);
+    l(
       [
         `Snapshot mismatch at ${n}`,
         "",
@@ -295,41 +353,41 @@ function yt(t) {
     );
   } catch (o) {
     if (o.code !== "ENOENT") {
-      u(
+      l(
         `Failed to read snapshot at ${n}: ${String(o)}`
       );
       return;
     }
     try {
-      Z(v(n), { recursive: !0 }), F(n, t, "utf8");
-    } catch (r) {
-      u(
-        `Failed to write snapshot at ${n}: ${String(r)}`
+      it(v(n), { recursive: !0 }), M(n, t, "utf8");
+    } catch (s) {
+      l(
+        `Failed to write snapshot at ${n}: ${String(s)}`
       );
     }
   }
 }
-function Mt(t) {
+function Lt(t) {
   return {
     toContain: (e) => {
-      pt(t, e);
+      Tt(t, e);
     },
     toMatchSnapshot: () => {
-      yt(t);
+      vt(t);
     },
     promptCheck: async (e) => {
-      await ut(t, e);
+      await wt(t, e);
     },
     toBeClassifiedAs: async (e, n) => {
-      await dt(t, e, n);
+      await bt(t, e, n);
     }
   };
 }
 export {
-  St as describe,
-  Mt as expect,
-  Tt as it,
-  $ as prompt,
-  Ft as promptFile,
-  Et as runCli
+  Nt as describe,
+  Lt as expect,
+  Pt as it,
+  F as prompt,
+  It as promptFile,
+  jt as runCli
 };