katt 0.0.2 → 0.0.4

package/README.md

@@ -2,7 +2,7 @@
 
 Katt is a lightweight testing framework for running AI Evals, inspired by [Jest](https://github.com/jestjs/jest).
 
-<img src="docs/logo.png" alt="Katt logo" width="250" />
+<img src="https://raw.githubusercontent.com/raphaelpor/katt/main/docs/logo.png" alt="Katt logo" width="250" />
 
 ## Overview
 
@@ -37,6 +37,7 @@ describe("Greeting agent", () => {
 - **Classification Matcher**: Built-in `toBeClassifiedAs()` matcher to grade a response against a target label on a 1-5 scale
 - **Concurrent Execution**: Runs eval files concurrently for faster test execution
 - **Model Selection**: Support for specifying custom AI models
+- **Configurable Timeouts**: Override prompt wait time per test or via `katt.json`
 
 ## Usage
 
@@ -89,6 +90,9 @@ You can also set a default model for the project by adding a `katt.json` file in
 {
   "copilot": {
     "model": "gpt-5-mini"
+  },
+  "prompt": {
+    "timeoutMs": 240000
   }
 }
 ```
@@ -97,6 +101,7 @@ When this file exists:
 
 - `prompt("...")` and `promptFile("...")` use `copilot.model` by default
 - `prompt("...", { model: "..." })` still overrides the config value
+- `prompt.timeoutMs` sets the default wait timeout for long-running prompts
 
 ## Development
 

package/dist/index.js

@@ -1,23 +1,23 @@
 #!/usr/bin/env node
-import { AsyncLocalStorage as E } from "node:async_hooks";
-import { CopilotClient as et } from "@github/copilot-sdk";
-import { readFile as M, readdir as nt } from "node:fs/promises";
-import { resolve as x, dirname as T, isAbsolute as ot, basename as it, join as st } from "node:path";
-import { readFileSync as Y, writeFileSync as Z, mkdirSync as rt } from "node:fs";
-import { fileURLToPath as ct, pathToFileURL as at } from "node:url";
-const D = new E(), lt = {
+import { AsyncLocalStorage as z } from "node:async_hooks";
+import { CopilotClient as it } from "@github/copilot-sdk";
+import { readFile as X, readdir as st } from "node:fs/promises";
+import { resolve as J, dirname as R, isAbsolute as rt, basename as ct, join as at } from "node:path";
+import { readFileSync as V, writeFileSync as W, mkdirSync as lt } from "node:fs";
+import { fileURLToPath as ut, pathToFileURL as dt } from "node:url";
+const Y = new z(), ft = {
   describeStack: [],
   itStack: [],
   tokenUsageStack: [],
   modelStack: []
 };
-let U = 0, W = 0;
-const w = [], A = [];
-let J = 0;
+let Z = 0, E = 0;
+const w = [], y = [];
+let U = 0;
 function u() {
-  return D.getStore() ?? lt;
+  return Y.getStore() ?? ft;
 }
-function H(t) {
+function Q(t) {
   return {
     describeStack: [...t.describeStack],
     itStack: [...t.itStack],
@@ -25,83 +25,83 @@ function H(t) {
     modelStack: [...t.modelStack]
   };
 }
-function ut() {
-  return U += 1, `d${U}`;
+function gt() {
+  return Z += 1, `d${Z}`;
 }
-function dt() {
-  return W += 1, `i${W}`;
+function pt() {
+  return E += 1, `i${E}`;
 }
-function X(t, e) {
-  const n = e ?? H(u());
-  return D.run(n, t);
+function K(t, e) {
+  const n = e ?? Q(u());
+  return Y.run(n, t);
 }
-function z() {
-  return H(u());
+function _() {
+  return Q(u());
 }
-function gt(t) {
-  u().describeStack.push({ id: ut(), description: t });
+function ht(t) {
+  u().describeStack.push({ id: gt(), description: t });
 }
-function b() {
+function x() {
   u().describeStack.pop();
 }
-function P() {
+function q() {
   return u().describeStack.map((t) => t.description).join(" > ");
 }
-function ft(t) {
-  u().itStack.push({ id: dt(), description: t }), u().tokenUsageStack.push(0), u().modelStack.push(void 0);
+function It(t) {
+  u().itStack.push({ id: pt(), description: t }), u().tokenUsageStack.push(0), u().modelStack.push(void 0);
 }
-function v() {
+function F() {
   u().itStack.pop(), u().tokenUsageStack.pop(), u().modelStack.pop();
 }
-function V() {
+function tt() {
   return u().itStack.map((t) => t.description).join(" > ");
 }
-function pt(t) {
+function mt(t) {
   if (!Number.isFinite(t) || t <= 0)
     return;
   const e = u(), n = e.tokenUsageStack.length - 1;
   n < 0 || (e.tokenUsageStack[n] += t);
 }
-function ht() {
+function Ct() {
   const t = u(), e = t.tokenUsageStack.length - 1;
   return e < 0 ? 0 : t.tokenUsageStack[e] ?? 0;
 }
-function It(t) {
+function St(t) {
   if (t.length === 0)
     return;
   const e = u(), n = e.modelStack.length - 1;
   n < 0 || (e.modelStack[n] = t);
 }
-function Ct() {
+function At() {
   const t = u(), e = t.modelStack.length - 1;
   if (!(e < 0))
     return t.modelStack[e];
 }
-function $(t) {
+function v(t) {
   w.push(t);
 }
-function mt() {
-  J += 1;
+function bt() {
+  U += 1;
 }
-function St() {
-  return J;
+function $t() {
+  return U;
 }
-function wt() {
-  J = 0;
+function kt() {
+  U = 0;
 }
-function At(t) {
-  A.push(t);
+function wt(t) {
+  y.push(t);
 }
-function $t() {
-  return [...A];
+function yt() {
+  return [...y];
 }
-function B() {
-  return A.length;
+function O() {
+  return y.length;
 }
-function kt() {
-  A.length = 0;
+function vt() {
+  y.length = 0;
 }
-async function yt() {
+async function Lt() {
   const t = [];
   for (; w.length > 0; ) {
     const e = w.splice(0, w.length), n = await Promise.allSettled(e);
@@ -109,41 +109,41 @@ async function yt() {
   }
   return t;
 }
-function bt(t, e) {
-  X(() => {
-    gt(t);
+function jt(t, e) {
+  K(() => {
+    ht(t);
     try {
       const n = e();
       if (n && typeof n.then == "function") {
-        $(
+        v(
           n.finally(() => {
-            b();
+            x();
           })
         );
         return;
       }
     } catch (n) {
-      throw b(), n;
+      throw x(), n;
     }
-    b();
-  }, z());
+    x();
+  }, _());
 }
-const vt = "\x1B[1;36m", Lt = "\x1B[33m", jt = "\x1B[38;5;208m", Ft = "\x1B[1;38;5;208m", k = "\x1B[0m";
+const Tt = "\x1B[1;36m", xt = "\x1B[33m", Ft = "\x1B[38;5;208m", Mt = "\x1B[1;38;5;208m", L = "\x1B[0m";
 function f(t) {
-  return `${vt}${t}${k}`;
+  return `${Tt}${t}${L}`;
 }
-function m(t) {
-  return `${Lt}${t}${k}`;
+function $(t) {
+  return `${xt}${t}${L}`;
 }
-function G(t) {
-  return `${jt}${t}${k}`;
+function D(t) {
+  return `${Ft}${t}${L}`;
 }
-function xt(t) {
-  return `${Ft}${t}${k}`;
+function Nt(t) {
+  return `${Mt}${t}${L}`;
 }
-let F = "";
-function Tt() {
-  F = "";
+let B = "";
+function Bt() {
+  B = "";
 }
 function Jt({
   suitePath: t,
@@ -153,31 +153,31 @@ function Jt({
   model: s,
   tokenUsage: i
 }) {
-  const r = t.length > 0 ? t : "(root)", a = e.length > 0 ? e : "(root)";
-  F !== r && (console.log(`Suite "${f(r)}"`), F = r);
-  const l = n ? "✅ Passed in" : "❌ Failed in", d = [
-    `Test "${f(a)}"`,
+  const r = t.length > 0 ? t : "(root)", c = e.length > 0 ? e : "(root)";
+  B !== r && (console.log(`Suite "${f(r)}"`), B = r);
+  const l = n ? "✅ Passed in" : "❌ Failed in", I = [
+    `Test "${f(c)}"`,
     `- ${l} ${f(`${o}ms`)}`
   ];
-  s && d.push(`- Model ${f(s)}`), (i ?? 0) > 0 && d.push(`- Tokens used ${f(String(i))}`), d.push("---"), console.log(d.join(`
+  s && I.push(`- Model ${f(s)}`), (i ?? 0) > 0 && I.push(`- Tokens used ${f(String(i))}`), I.push("---"), console.log(I.join(`
 `));
 }
-function I(t, e, n = "(root)") {
-  const o = V();
+function C(t, e, n = "(root)") {
+  const o = tt();
   Jt({
-    suitePath: P(),
+    suitePath: q(),
     casePath: o.length > 0 ? o : n,
     didPass: t,
     durationMs: e,
-    model: Ct(),
-    tokenUsage: ht()
+    model: At(),
+    tokenUsage: Ct()
   });
 }
-const N = new E();
-function Nt(t, e) {
+const G = new z();
+function Rt(t, e) {
   return typeof t == "object" && t !== null && "code" in t && t.code === e;
 }
-function Rt(t) {
+function Ut(t) {
   try {
     const e = JSON.parse(t);
     return typeof e == "object" && e !== null ? e : void 0;
@@ -186,7 +186,19 @@ function Rt(t) {
     return;
   }
 }
-function Zt(t) {
+async function Gt() {
+  const t = J(process.cwd(), "katt.json");
+  try {
+    const e = await X(t, "utf8");
+    return Ut(e);
+  } catch (e) {
+    if (Rt(e, "ENOENT"))
+      return;
+    console.warn(`Failed to read katt.json: ${String(e)}`);
+    return;
+  }
+}
+function Wt(t) {
   const e = t?.copilot;
   if (typeof e != "object" || e === null || Array.isArray(e))
     return;
@@ -195,83 +207,94 @@ function Zt(t) {
   }, o = n.model;
   return (typeof o != "string" || o.length === 0) && delete n.model, Object.keys(n).length > 0 ? n : void 0;
 }
-async function Ut() {
-  const t = x(process.cwd(), "katt.json");
-  try {
-    const e = await M(t, "utf8");
-    return Zt(Rt(e));
-  } catch (e) {
-    if (Nt(e, "ENOENT"))
-      return;
-    console.warn(`Failed to read katt.json: ${String(e)}`);
-    return;
-  }
+function Zt(t) {
+  if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
+    return Math.floor(t);
+}
+function Et(t) {
+  const e = t?.prompt;
+  if (!(typeof e != "object" || e === null || Array.isArray(e)))
+    return Zt(e.timeoutMs);
+}
+async function Ot() {
+  const t = await Gt();
+  return {
+    copilot: Wt(t),
+    promptTimeoutMs: Et(t)
+  };
 }
-function K(t) {
+const Dt = 6e5;
+function et(t) {
   return typeof t == "string" && t.length > 0 ? t : void 0;
 }
-function L(t) {
+function M(t) {
   if (!t)
     return;
   const e = { ...t };
   if (e.model !== void 0) {
-    const n = K(e.model);
+    const n = et(e.model);
     n ? e.model = n : delete e.model;
   }
   return Object.keys(e).length > 0 ? e : void 0;
 }
-function S(t) {
+function H(t) {
+  if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
+    return Math.floor(t);
+}
+function k(t) {
   return !Number.isFinite(t) || (t ?? 0) <= 0 ? 0 : Math.floor(t ?? 0);
 }
-function Wt(t) {
-  return S(t.inputTokens) + S(t.outputTokens) + S(t.cacheReadTokens) + S(t.cacheWriteTokens);
-}
-async function y(t, e = {}) {
-  const n = L(await Ut()), o = L(e), s = L({
-    ...n ?? {},
-    ...o ?? {}
-  }), i = K(s?.model), r = new et({ useLoggedInUser: !0 });
-  let a, l, d = 0;
+function Ht(t) {
+  return k(t.inputTokens) + k(t.outputTokens) + k(t.cacheReadTokens) + k(t.cacheWriteTokens);
+}
+async function j(t, e = {}) {
+  const { timeoutMs: n, ...o } = e, s = await Ot(), i = M(s.copilot), r = M(
+    o
+  ), c = M({
+    ...i ?? {},
+    ...r ?? {}
+  }), l = H(s.promptTimeoutMs), T = H(n) ?? l ?? Dt, a = et(c?.model), d = new it({ useLoggedInUser: !0 });
+  let p, b, S = 0;
   try {
-    await r.start(), a = await r.createSession(s), l = a.on("assistant.usage", (c) => {
-      d += Wt(c.data);
+    await d.start(), p = await d.createSession(c), b = p.on("assistant.usage", (m) => {
+      S += Ht(m.data);
     });
-    const g = await a.sendAndWait({ prompt: t });
-    if (!g?.data?.content)
+    const h = await p.sendAndWait({ prompt: t }, T);
+    if (!h?.data?.content)
       throw new Error("Copilot did not return a response.");
-    return i && It(i), g.data.content;
+    return a && St(a), h.data.content;
   } finally {
-    const g = [];
-    if (l?.(), d > 0 && pt(d), a)
+    const h = [];
+    if (b?.(), S > 0 && mt(S), p)
       try {
-        await a.destroy();
-      } catch (c) {
-        g.push(c);
+        await p.destroy();
+      } catch (m) {
+        h.push(m);
       }
     try {
-      const c = await r.stop();
-      g.push(...c);
-    } catch (c) {
-      g.push(c);
+      const m = await d.stop();
+      h.push(...m);
+    } catch (m) {
+      h.push(m);
     }
-    g.length > 0 && console.error(
-      `Copilot cleanup encountered ${g.length} error(s).`
+    h.length > 0 && console.error(
+      `Copilot cleanup encountered ${h.length} error(s).`
     );
   }
 }
-async function Bt(t, e = {}) {
-  const n = N.getStore(), o = n?.evalFile ? T(n.evalFile) : process.cwd(), s = ot(t) ? t : x(o, t), i = await M(s, "utf8");
-  return y(i, e);
+async function Pt(t, e = {}) {
+  const n = G.getStore(), o = n?.evalFile ? R(n.evalFile) : process.cwd(), s = rt(t) ? t : J(o, t), i = await X(s, "utf8");
+  return j(i, e);
 }
-function p(t) {
-  At({
-    describePath: P(),
-    itPath: V(),
+function g(t) {
+  wt({
+    describePath: q(),
+    itPath: tt(),
     message: t
   });
 }
-async function Gt(t, e) {
-  const n = Date.now(), o = `expected '${t}' to satisfy '${e}'`, s = y(`Evaluate if the expectation is fulfiled in by the input.
+async function zt(t, e) {
+  const n = Date.now(), o = `expected '${t}' to satisfy '${e}'`, s = j(`Evaluate if the expectation is fulfiled in by the input.
         Expectation: "${e}".
         Input:
         ---
@@ -279,17 +302,17 @@ async function Gt(t, e) {
         ---
         Important: Answer with "Yes" or "No" only, without any additional text.
         `).then((i) => {
-    i.includes("Yes") ? I(!0, Date.now() - n, "promptCheck") : i.includes("No") ? p(o) : p(`failed to evaluate expectation '${e}'`);
+    i.includes("Yes") ? C(!0, Date.now() - n, "promptCheck") : i.includes("No") ? g(o) : g(`failed to evaluate expectation '${e}'`);
   });
-  return $(s), s;
+  return v(s), s;
 }
-const Ot = 3;
-function Et(t) {
+const Xt = 3;
+function Vt(t) {
   const e = t.match(/\b([1-5])\b/);
   return !e || !e[1] ? null : Number(e[1]);
 }
-async function Mt(t, e, n) {
-  const o = Date.now(), s = n?.threshold ?? Ot, i = y(
+async function Yt(t, e, n) {
+  const o = Date.now(), s = n?.threshold ?? Xt, i = j(
     `Classify the input by how "${e}" it is on a scale of 1 to 5.
 1 means "not ${e}" and 5 means "very ${e}".
 Return only a single number: 1, 2, 3, 4, or 5.
@@ -300,88 +323,88 @@ ${t}
 ---`,
     n?.model ? { model: n.model } : void 0
   ).then((r) => {
-    const a = Et(r);
-    if (a === null) {
-      p(
+    const c = Vt(r);
+    if (c === null) {
+      g(
         `failed to classify as '${e}'. Evaluator returned '${r}'`
       );
       return;
     }
-    const l = `expected response to be classified as '${e}' with score >= ${s}, got ${a}`;
-    if (a < s) {
-      p(l);
+    const l = `expected response to be classified as '${e}' with score >= ${s}, got ${c}`;
+    if (c < s) {
+      g(l);
       return;
     }
-    I(
+    C(
       !0,
       Date.now() - o,
       "toBeClassifiedAs"
     );
   });
-  return $(i), i;
+  return v(i), i;
 }
-function Yt(t, e) {
+function Qt(t, e) {
   const n = `expected '${t}' to include '${e}'`;
-  t.includes(e) || p(n);
+  t.includes(e) || g(n);
 }
-let _ = !1;
-function Dt(t) {
-  _ = t;
+let nt = !1;
+function Kt(t) {
+  nt = t;
 }
-function Ht() {
-  return _;
+function _t() {
+  return nt;
 }
-function Xt(t) {
-  const n = it(t).replace(/\.eval\.[^./\\]+$/, "");
-  return st(
-    T(t),
+function qt(t) {
+  const n = ct(t).replace(/\.eval\.[^./\\]+$/, "");
+  return at(
+    R(t),
     "__snapshots__",
     `${n}.snap.md`
   );
 }
-function O(t) {
+function P(t) {
   return t.split(/\r?\n/);
 }
-function zt(t, e) {
+function te(t, e) {
   if (t === e)
     return "  (no diff)";
-  const n = O(t), o = O(e), s = Math.max(n.length, o.length), i = [];
+  const n = P(t), o = P(e), s = Math.max(n.length, o.length), i = [];
   for (let r = 0; r < s; r += 1) {
-    const a = n[r], l = o[r];
-    if (a !== l) {
-      if (a === void 0 && l !== void 0) {
+    const c = n[r], l = o[r];
+    if (c !== l) {
+      if (c === void 0 && l !== void 0) {
         i.push(`+ ${l}`);
         continue;
       }
-      if (a !== void 0 && l === void 0) {
-        i.push(`- ${a}`);
+      if (c !== void 0 && l === void 0) {
+        i.push(`- ${c}`);
         continue;
       }
-      i.push(`- ${a ?? ""}`), i.push(`+ ${l ?? ""}`);
+      i.push(`- ${c ?? ""}`), i.push(`+ ${l ?? ""}`);
     }
   }
   return i.join(`
 `);
 }
-function Pt(t) {
-  const e = N.getStore()?.evalFile;
+function ee(t) {
+  const e = G.getStore()?.evalFile;
   if (!e) {
-    p(
+    g(
       "toMatchSnapshot can only be used while running an eval file."
     );
     return;
   }
-  const n = Xt(e);
+  const n = qt(e);
   try {
-    const o = Y(n, "utf8");
+    const o = V(n, "utf8");
     if (o === t)
       return;
-    if (Ht()) {
-      Z(n, t, "utf8");
+    if (_t()) {
+      W(n, t, "utf8");
       return;
     }
-    const s = zt(o, t);
-    p(
+    const s = te(o, t);
+    g(
       [
         `Snapshot mismatch at ${n}`,
         "",
@@ -394,173 +417,173 @@ function Pt(t) {
     );
   } catch (o) {
     if (o.code !== "ENOENT") {
-      p(
+      g(
         `Failed to read snapshot at ${n}: ${String(o)}`
       );
       return;
     }
     try {
-      rt(T(n), { recursive: !0 }), Z(n, t, "utf8");
+      lt(R(n), { recursive: !0 }), W(n, t, "utf8");
     } catch (i) {
-      p(
+      g(
         `Failed to write snapshot at ${n}: ${String(i)}`
       );
     }
   }
 }
-function Vt(t) {
+function ne(t) {
   return {
     toContain: (e) => {
-      Yt(t, e);
+      Qt(t, e);
     },
     toMatchSnapshot: () => {
-      Pt(t);
+      ee(t);
     },
     promptCheck: async (e) => {
-      await Gt(t, e);
+      await zt(t, e);
     },
     toBeClassifiedAs: async (e, n) => {
-      await Mt(t, e, n);
+      await Yt(t, e, n);
     }
   };
 }
-function Kt(t, e) {
-  X(() => {
-    mt(), ft(t);
-    const n = B(), o = Date.now(), s = () => B() === n, i = () => Date.now() - o;
+function oe(t, e) {
+  K(() => {
+    bt(), It(t);
+    const n = O(), o = Date.now(), s = () => O() === n, i = () => Date.now() - o;
     try {
       const r = e();
       if (r && typeof r.then == "function") {
-        $(
+        v(
           r.then(() => {
-            I(!0, i());
-          }).catch((a) => {
-            throw I(!1, i()), a;
+            C(!0, i());
+          }).catch((c) => {
+            throw C(!1, i()), c;
           }).finally(() => {
-            v();
+            F();
           })
         );
         return;
       }
     } catch (r) {
-      throw I(!1, i()), v(), r;
+      throw C(!1, i()), F(), r;
     }
-    I(s(), i()), v();
-  }, z());
+    C(s(), i()), F();
+  }, _());
 }
-const _t = /\.eval\.(js|ts)$/, Qt = /* @__PURE__ */ new Set([".git", "node_modules"]);
-async function Q(t) {
-  const e = await nt(t, { withFileTypes: !0 }), n = [];
+const ie = /\.eval\.(js|ts)$/, se = /* @__PURE__ */ new Set([".git", "node_modules"]);
+async function ot(t) {
+  const e = await st(t, { withFileTypes: !0 }), n = [];
   return await Promise.all(
     e.map(async (o) => {
-      const s = x(t, o.name);
+      const s = J(t, o.name);
       if (o.isDirectory()) {
-        if (Qt.has(o.name))
+        if (se.has(o.name))
           return;
-        n.push(...await Q(s));
+        n.push(...await ot(s));
         return;
       }
-      o.isFile() && _t.test(o.name) && n.push(s);
+      o.isFile() && ie.test(o.name) && n.push(s);
     })
   ), n;
 }
-const j = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMS4wIiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInByaXZhdGUiOiB0cnVlLAogICJ0eXBlIjogIm1vZHVsZSIsCiAgIm1haW4iOiAiZGlzdC9pbmRleC5qcyIsCiAgImJpbiI6IHsKICAgICJrYXR0IjogImRpc3QvaW5kZXguanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3QvaW5kZXguanMiCiAgfSwKICAidHlwZXMiOiAiZGlzdC9pbmRleC5kLnRzIiwKICAiZGV2RGVwZW5kZW5jaWVzIjogewogICAgIkBiaW9tZWpzL2Jpb21lIjogIjEuOS40IiwKICAgICJAdHlwZXMvbm9kZSI6ICIyNS4yLjAiLAogICAgInRzeCI6ICI0LjIxLjAiLAogICAgInR5cGVzY3JpcHQiOiAiNS44LjIiLAogICAgInZpdGUiOiAiNy4zLjEiLAogICAgInZpdGUtcGx1Z2luLWR0cyI6ICI0LjUuNCIsCiAgICAidml0ZXN0IjogIjMuMi40IiwKICAgICJ2c2NvZGUtanNvbnJwYyI6ICJeOC4yLjEiCiAgfSwKICAiZGVwZW5kZW5jaWVzIjogewogICAgIkBnaXRodWIvY29waWxvdC1zZGsiOiAiXjAuMS4yMSIKICB9LAogICJidWdzIjogewogICAgInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0L2lzc3VlcyIKICB9LAogICJob21lcGFnZSI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0Igp9Cg==", import.meta.url);
-let C;
-function qt() {
-  if (C !== void 0)
-    return C;
+const N = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC4zIiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiYmluIjogewogICAgImthdHQiOiAiZGlzdC9pbmRleC5qcyIKICB9LAogICJzY3JpcHRzIjogewogICAgImJ1aWxkIjogInZpdGUgYnVpbGQiLAogICAgImRldiI6ICJ0c3ggc3JjL2luZGV4LnRzIiwKICAgICJsaW50IjogImJpb21lIGxpbnQgLi9zcmMiLAogICAgImZvcm1hdCI6ICJiaW9tZSBmb3JtYXQgLS13cml0ZSAuL3NyYyIsCiAgICAidGVzdCI6ICJ2aXRlc3QiLAogICAgInR5cGVjaGVjayI6ICJ0c2MgLXAgdHNjb25maWcuanNvbiAtLW5vRW1pdCIsCiAgICAidGVzdDpidWlsZCI6ICJub2RlIC4vZGlzdC9pbmRleC5qcyIKICB9LAogICJ0eXBlcyI6ICJkaXN0L2luZGV4LmQudHMiLAogICJkZXZEZXBlbmRlbmNpZXMiOiB7CiAgICAiQGJpb21lanMvYmlvbWUiOiAiMS45LjQiLAogICAgIkB0eXBlcy9ub2RlIjogIjI1LjIuMCIsCiAgICAidHN4IjogIjQuMjEuMCIsCiAgICAidHlwZXNjcmlwdCI6ICI1LjguMiIsCiAgICAidml0ZSI6ICI3LjMuMSIsCiAgICAidml0ZS1wbHVnaW4tZHRzIjogIjQuNS40IiwKICAgICJ2aXRlc3QiOiAiMy4yLjQiLAogICAgInZzY29kZS1qc29ucnBjIjogIl44LjIuMSIKICB9LAogICJkZXBlbmRlbmNpZXMiOiB7CiAgICAiQGdpdGh1Yi9jb3BpbG90LXNkayI6ICJeMC4xLjIxIgogIH0sCiAgImJ1Z3MiOiB7CiAgICAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQvaXNzdWVzIgogIH0sCiAgImhvbWVwYWdlIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQiCn0K", import.meta.url);
+let A;
+function re() {
+  if (A !== void 0)
+    return A;
   try {
-    const t = j.protocol === "data:" ? te(j) : Y(ct(j), "utf8"), e = JSON.parse(t);
-    C = typeof e.version == "string" ? e.version : "unknown";
+    const t = N.protocol === "data:" ? ce(N) : V(ut(N), "utf8"), e = JSON.parse(t);
+    A = typeof e.version == "string" ? e.version : "unknown";
   } catch {
-    C = "unknown";
+    A = "unknown";
   }
-  return C;
+  return A;
 }
-function te(t) {
+function ce(t) {
   const e = t.pathname.indexOf(",");
   if (e < 0)
     throw new Error("Invalid data URL.");
   const n = t.pathname.slice(0, e), o = t.pathname.slice(e + 1);
   return n.includes(";base64") ? Buffer.from(o, "base64").toString("utf8") : decodeURIComponent(o);
 }
-function ee() {
-  const t = " ██╗  ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║   ██║      ██║", o = " ██╔═██╗ ██╔══██║   ██║      ██║", s = " ██║  ██╗██║  ██║   ██║      ██║", i = " ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝      ╚═╝", r = `v${qt()}`, a = Math.max(
+function ae() {
+  const t = " ██╗  ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║   ██║      ██║", o = " ██╔═██╗ ██╔══██║   ██║      ██║", s = " ██║  ██╗██║  ██║   ██║      ██║", i = " ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝      ╚═╝", r = `v${re()}`, c = Math.max(
     0,
     Math.floor((t.length - r.length) / 2)
-  ), l = `${" ".repeat(a)}${r}`;
+  ), l = `${" ".repeat(c)}${r}`;
   console.log(`
-${m(t)}
-${m(e)}
-${m(n)}
-${G(o)}
-${G(s)}
-${xt(i)}
-${m(l)}
+${$(t)}
+${$(e)}
+${$(n)}
+${D(o)}
+${D(s)}
+${Nt(i)}
+${$(l)}
 `);
 }
-function ne(t) {
+function le(t) {
   const e = String(t.getHours()).padStart(2, "0"), n = String(t.getMinutes()).padStart(2, "0"), o = String(t.getSeconds()).padStart(2, "0");
   return `${e}:${n}:${o}`;
 }
-async function oe() {
+async function ue() {
   const t = process.argv.slice(2), e = t.includes("--update-snapshots") || t.includes("-u");
-  Dt(e), ee();
+  Kt(e), ae();
   const n = /* @__PURE__ */ new Date();
-  Tt(), kt(), wt();
-  const o = await Q(process.cwd());
+  Bt(), vt(), kt();
+  const o = await ot(process.cwd());
   if (o.length === 0)
     return console.log("No .eval.js or .eval.ts files found."), 1;
   const i = (await Promise.allSettled(
     o.map(
-      (c) => N.run(
-        { evalFile: c },
-        () => import(at(c).href)
+      (a) => G.run(
+        { evalFile: a },
+        () => import(dt(a).href)
       )
     )
-  )).map((c, h) => ({ result: c, file: o[h] })).filter(({ result: c }) => c.status === "rejected");
+  )).map((a, d) => ({ result: a, file: o[d] })).filter(({ result: a }) => a.status === "rejected");
   if (i.length > 0) {
-    for (const c of i) {
-      const h = c.result.status === "rejected" ? c.result.reason : void 0;
-      console.error(`Error executing ${c.file}: ${String(h)}`);
+    for (const a of i) {
+      const d = a.result.status === "rejected" ? a.result.reason : void 0;
+      console.error(`Error executing ${a.file}: ${String(d)}`);
     }
     return 1;
   }
-  const a = (await yt()).filter(
-    (c) => c.status === "rejected"
+  const c = (await Lt()).filter(
+    (a) => a.status === "rejected"
   );
-  if (a.length > 0) {
-    for (const c of a)
-      c.status === "rejected" && console.error(`Error executing async test: ${String(c.reason)}`);
+  if (c.length > 0) {
+    for (const a of c)
+      a.status === "rejected" && console.error(`Error executing async test: ${String(a.reason)}`);
     return 1;
   }
-  const l = $t();
+  const l = yt();
   if (l.length > 0) {
     console.error("❌ Failed tests:");
-    for (const [c, h] of l.entries()) {
-      const R = [h.describePath, h.itPath].filter((tt) => tt.length > 0).join(" > "), q = R.length > 0 ? `${R}: ` : "";
-      console.error(`${c + 1}. ${q}${h.message}`);
+    for (const [a, d] of l.entries()) {
+      const p = [d.describePath, d.itPath].filter((S) => S.length > 0).join(" > "), b = p.length > 0 ? `${p}: ` : "";
+      console.error(`${a + 1}. ${b}${d.message}`);
     }
     return 1;
   }
-  const d = St(), g = Date.now() - n.getTime();
+  const I = $t(), T = Date.now() - n.getTime();
   return console.log(
     [
       "---",
       `${f("Files")}  ${o.length} passed`,
-      `${f("Evals")}  ${d} passed`,
-      `${f("Start at")}  ${ne(n)}`,
-      `${f("Duration")}  ${g}ms`
+      `${f("Evals")}  ${I} passed`,
+      `${f("Start at")}  ${le(n)}`,
+      `${f("Duration")}  ${T}ms`
     ].join(`
 `)
   ), 0;
 }
-Object.assign(globalThis, { describe: bt, it: Kt, expect: Vt, prompt: y, promptFile: Bt });
-oe().then((t) => {
+Object.assign(globalThis, { describe: jt, it: oe, expect: ne, prompt: j, promptFile: Pt });
+ue().then((t) => {
   process.exit(t);
 }).catch((t) => {
   console.error(`Unexpected error: ${String(t)}`), process.exit(1);
 });
 export {
-  oe as runCli
+  ue as runCli
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "katt",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "CLI tool that tests the output of agentic AI tools",
   "keywords": [
     "cli",