katt 0.0.6 → 0.0.7

package/README.md

@@ -9,6 +9,7 @@ Katt is a lightweight testing framework for running AI Evals, inspired by [Jest]
 
 - [Overview](#overview)
 - [API Documentation](#api-documentation)
+- [Articles](#articles)
 - [Hello World - Example](#hello-world---example)
 - [Main Features](#main-features)
 - [Usage](#usage)
@@ -34,6 +35,10 @@ Katt is designed to evaluate and validate the behavior of AI agents like **Claud
 
 For a complete list of features and usage examples, see [docs/api-documentation.md](https://github.com/raphaelpor/katt/blob/main/docs/api-documentation.md).
 
+## Articles
+
+- [Introducing Katt](https://github.com/raphaelpor/katt/blob/main/docs/articles/introduction-to-katt.md)
+
 ## Hello World - Example
 
 ```typescript
@@ -86,7 +91,7 @@ expect(result).toContain("hello world");
 2. Run Katt from your project directory:
 
 ```bash
-katt
+npx katt
 ```
 
 ### Using promptFile

package/dist/index.js

@@ -1,56 +1,56 @@
-import { r as $, c as x, p as E, a as d, l, b as p, g as w, d as N, e as P, f as h, s as A, h as S, i as j, j as _, k as O, m as L, n as I, o as U, q as R } from "./runCli-B3oIBxOl.js";
-import { t as xt } from "./runCli-B3oIBxOl.js";
-import { CopilotClient as z } from "@github/copilot-sdk";
-import { readFile as F } from "node:fs/promises";
-import { resolve as k, dirname as g, isAbsolute as B, basename as W, join as K } from "node:path";
-import { readFileSync as H, writeFileSync as C, mkdirSync as Y } from "node:fs";
-function wt(t, e) {
-  $(() => {
-    N(), E(t);
-    const n = w(), o = Date.now(), r = () => w() === n, s = () => Date.now() - o;
+import { r as E, c as N, a as O, p as I, b as m, l as d, d as w, g as S, e as U, f as C, s as z, h as R, i as A, j as B, k as W, m as K, n as H, o as Y, q } from "./runCli-C7uxWavX.js";
+import { t as Et } from "./runCli-C7uxWavX.js";
+import { CopilotClient as J } from "@github/copilot-sdk";
+import { readFile as j } from "node:fs/promises";
+import { resolve as P, dirname as v, isAbsolute as G, basename as Q, join as V } from "node:path";
+import { readFileSync as X, writeFileSync as F, mkdirSync as Z } from "node:fs";
+function Tt(t, e) {
+  E(() => {
+    O(), I(t);
+    const n = S(), o = Date.now(), i = () => S() === n, r = () => Date.now() - o;
     try {
-      const i = e();
-      if (i && typeof i.then == "function") {
-        d(
-          i.then(() => {
-            l(!0, s());
+      const s = e();
+      if (s && typeof s.then == "function") {
+        m(
+          s.then(() => {
+            d(!0, r());
           }).catch((a) => {
-            throw l(!1, s()), a;
+            throw d(!1, r()), a;
           }).finally(() => {
-            p();
+            w();
           })
         );
         return;
       }
-    } catch (i) {
-      throw l(!1, s()), p(), i;
+    } catch (s) {
+      throw d(!1, r()), w(), s;
     }
-    l(r(), s()), p();
-  }, x());
+    d(i(), r()), w();
+  }, N());
 }
-function Ct(t, e) {
-  $(() => {
-    P(t);
+function St(t, e) {
+  E(() => {
+    U(t);
     try {
       const n = e();
       if (n && typeof n.then == "function") {
-        d(
+        m(
           n.finally(() => {
-            h();
+            C();
           })
         );
         return;
       }
     } catch (n) {
-      throw h(), n;
+      throw C(), n;
     }
-    h();
-  }, x());
+    C();
+  }, N());
 }
-function q(t, e) {
+function tt(t, e) {
   return typeof t == "object" && t !== null && "code" in t && t.code === e;
 }
-function J(t) {
+function et(t) {
   try {
     const e = JSON.parse(t);
     return typeof e == "object" && e !== null ? e : void 0;
@@ -59,19 +59,19 @@ function J(t) {
     return;
   }
 }
-async function G() {
-  const t = k(process.cwd(), "katt.json");
+async function nt() {
+  const t = P(process.cwd(), "katt.json");
   try {
-    const e = await F(t, "utf8");
-    return J(e);
+    const e = await j(t, "utf8");
+    return et(e);
   } catch (e) {
-    if (q(e, "ENOENT"))
+    if (tt(e, "ENOENT"))
       return;
     console.warn(`Failed to read katt.json: ${String(e)}`);
     return;
   }
 }
-function Q(t) {
+function ot(t) {
   const e = t?.copilot;
   if (typeof e != "object" || e === null || Array.isArray(e))
     return;
@@ -80,134 +80,112 @@ function Q(t) {
   }, o = n.model;
   return (typeof o != "string" || o.length === 0) && delete n.model, Object.keys(n).length > 0 ? n : void 0;
 }
-function V(t) {
+function rt(t) {
   if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
     return Math.floor(t);
 }
-function X(t) {
+function it(t) {
   const e = t?.prompt;
   if (!(typeof e != "object" || e === null || Array.isArray(e)))
-    return V(e.timeoutMs);
+    return rt(e.timeoutMs);
 }
-async function Z() {
-  const t = await G();
+async function st() {
+  const t = await nt();
   return {
-    copilot: Q(t),
-    promptTimeoutMs: X(t)
+    copilot: ot(t),
+    promptTimeoutMs: it(t)
   };
 }
-const tt = 6e5;
-function M(t) {
+const at = 6e5;
+function _(t) {
   return typeof t == "string" && t.length > 0 ? t : void 0;
 }
-function m(t) {
+function b(t) {
   if (!t)
     return;
   const e = { ...t };
   if (e.model !== void 0) {
-    const n = M(e.model);
+    const n = _(e.model);
     n ? e.model = n : delete e.model;
   }
   return Object.keys(e).length > 0 ? e : void 0;
 }
-function b(t) {
+function M(t) {
   if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
     return Math.floor(t);
 }
-function f(t) {
+function h(t) {
   return !Number.isFinite(t) || (t ?? 0) <= 0 ? 0 : Math.floor(t ?? 0);
 }
-function et(t) {
-  return f(t.inputTokens) + f(t.outputTokens) + f(t.cacheReadTokens) + f(t.cacheWriteTokens);
+function ct(t) {
+  return h(t.inputTokens) + h(t.outputTokens) + h(t.cacheReadTokens) + h(t.cacheWriteTokens);
 }
-function nt(t, e) {
-  const { timeoutMs: n, ...o } = t, r = m(e.copilot), s = m(
+async function $(t, e = {}) {
+  const { timeoutMs: n, ...o } = e, i = await st(), r = b(i.copilot), s = b(
     o
-  ), i = m({
+  ), a = b({
     ...r ?? {},
     ...s ?? {}
-  }), a = b(e.promptTimeoutMs), D = b(n) ?? a ?? tt;
-  return {
-    sessionOptions: i,
-    model: M(i?.model),
-    timeoutMs: D
-  };
-}
-async function ot(t, e) {
-  const n = [];
-  if (e.unsubscribeUsage?.(), e.usedTokens > 0 && j(e.usedTokens), e.session)
-    try {
-      await e.session.destroy();
-    } catch (o) {
-      n.push(o);
-    }
-  try {
-    const o = await t.stop();
-    n.push(...o);
-  } catch (o) {
-    n.push(o);
-  }
-  n.length > 0 && console.error(
-    `Copilot cleanup encountered ${n.length} error(s).`
-  );
-}
-async function y(t, e = {}) {
-  const n = await Z(), o = nt(e, n), r = new z({ useLoggedInUser: !0 }), s = {
-    session: void 0,
-    unsubscribeUsage: void 0,
-    usedTokens: 0
-  };
+  }), c = M(i.promptTimeoutMs), L = M(n) ?? c ?? at, x = _(a?.model), g = new J({ useLoggedInUser: !0 });
+  let p, T, y = 0;
   try {
-    await r.start(), s.session = await r.createSession(
-      o.sessionOptions
-    ), s.unsubscribeUsage = s.session.on(
-      "assistant.usage",
-      (a) => {
-        s.usedTokens += et(a.data);
-      }
-    );
-    const i = await s.session.sendAndWait(
-      { prompt: t },
-      o.timeoutMs
-    );
-    if (!i?.data?.content)
+    await g.start(), p = await g.createSession(a), T = p.on("assistant.usage", (f) => {
+      y += ct(f.data);
+    });
+    const l = await p.sendAndWait({ prompt: t }, L);
+    if (!l?.data?.content)
       throw new Error("Copilot did not return a response.");
-    return o.model && A(o.model), i.data.content;
+    return x && z(x), l.data.content;
   } finally {
-    await ot(r, s);
+    const l = [];
+    if (T?.(), y > 0 && R(y), p)
+      try {
+        await p.destroy();
+      } catch (f) {
+        l.push(f);
+      }
+    try {
+      const f = await g.stop();
+      l.push(...f);
+    } catch (f) {
+      l.push(f);
+    }
+    l.length > 0 && console.error(
+      `Copilot cleanup encountered ${l.length} error(s).`
+    );
   }
 }
-async function bt(t, e = {}) {
-  const n = S.getStore(), o = n?.evalFile ? g(n.evalFile) : process.cwd(), r = B(t) ? t : k(o, t), s = await F(r, "utf8");
-  return y(s, e);
+async function Ft(t, e = {}) {
+  const n = A.getStore(), o = n?.evalFile ? v(n.evalFile) : process.cwd(), i = G(t) ? t : P(o, t), r = await j(i, "utf8");
+  return $(r, e);
 }
-function c(t) {
-  _({
-    describePath: L(),
-    itPath: O(),
+function u(t) {
+  B({
+    describePath: K(),
+    itPath: W(),
     message: t
   });
 }
-async function st(t, e) {
-  const n = Date.now(), o = `expected '${t}' to satisfy '${e}'`, r = y(`Evaluate if the expectation is fulfiled in by the input.
+async function ut(t, e) {
+  const n = Date.now(), o = `expected '${t}' to satisfy '${e}'`, i = $(`Evaluate if the expectation is fulfiled in by the input.
         Expectation: "${e}".
         Input:
         ---
         ${t}
         ---
         Important: Answer with "Yes" or "No" only, without any additional text.
-        `).then((s) => {
-    s.includes("Yes") ? l(!0, Date.now() - n, "promptCheck") : s.includes("No") ? c(o) : c(`failed to evaluate expectation '${e}'`);
+        `).then((r) => {
+    r.includes("Yes") ? d(!0, Date.now() - n, "promptCheck") : r.includes("No") ? u(o) : u(`failed to evaluate expectation '${e}'`);
   });
-  return d(r), r;
+  return m(i), i;
 }
-const rt = 3;
-function it(t) {
+const lt = 3;
+function ft(t) {
   const e = t.match(/\b([1-5])\b/);
   return !e || !e[1] ? null : Number(e[1]);
 }
-async function at(t, e, n) {
-  const o = Date.now(), r = n?.threshold ?? rt, s = y(
+async function dt(t, e, n) {
+  const o = Date.now(), i = n?.threshold ?? lt, r = $(
     `Classify the input by how "${e}" it is on a scale of 1 to 5.
 1 means "not ${e}" and 5 means "very ${e}".
 Return only a single number: 1, 2, 3, 4, or 5.
@@ -217,141 +195,141 @@ Input:
 ${t}
 ---`,
     n?.model ? { model: n.model } : void 0
-  ).then((i) => {
-    const a = it(i);
+  ).then((s) => {
+    const a = ft(s);
     if (a === null) {
-      c(
-        `failed to classify as '${e}'. Evaluator returned '${i}'`
+      u(
+        `failed to classify as '${e}'. Evaluator returned '${s}'`
       );
       return;
     }
-    const u = `expected response to be classified as '${e}' with score >= ${r}, got ${a}`;
-    if (a < r) {
-      c(u);
+    const c = `expected response to be classified as '${e}' with score >= ${i}, got ${a}`;
+    if (a < i) {
+      u(c);
       return;
     }
-    l(
+    d(
       !0,
       Date.now() - o,
       "toBeClassifiedAs"
     );
   });
-  return d(s), s;
+  return m(r), r;
 }
-function ct(t, e) {
+function pt(t, e) {
   const n = `expected '${t}' to include '${e}'`;
-  t.includes(e) || c(n);
+  t.includes(e) || u(n);
 }
-function v(t) {
+function k(t) {
   const e = t.trim().replace(/[<>:"/\\|?*\x00-\x1f]/g, "_").replace(/\s+/g, "_");
   return e.length > 0 ? e : "unnamed";
 }
-function ut() {
-  const t = U().map(
-    (o) => v(o.description)
-  ), e = R().map(
-    (o) => v(o.description)
+function ht() {
+  const t = Y().map(
+    (o) => k(o.description)
+  ), e = q().map(
+    (o) => k(o.description)
   ), n = [...t, ...e];
   return n.length === 0 ? "root" : n.join("__");
 }
-function lt(t) {
-  const n = W(t).replace(/\.eval\.[^./\\]+$/, ""), o = ut();
-  return K(
-    g(t),
+function mt(t) {
+  const n = Q(t).replace(/\.eval\.[^./\\]+$/, ""), o = ht();
+  return V(
+    v(t),
     "__snapshots__",
     `${n}__${o}.snap.md`
   );
 }
-function T(t) {
+function D(t) {
   return t.split(/\r?\n/);
 }
-function ft(t, e) {
+function gt(t, e) {
   if (t === e)
     return "  (no diff)";
-  const n = T(t), o = T(e), r = Math.max(n.length, o.length), s = [];
-  for (let i = 0; i < r; i += 1) {
-    const a = n[i], u = o[i];
-    if (a !== u) {
-      if (a === void 0 && u !== void 0) {
-        s.push(`+ ${u}`);
+  const n = D(t), o = D(e), i = Math.max(n.length, o.length), r = [];
+  for (let s = 0; s < i; s += 1) {
+    const a = n[s], c = o[s];
+    if (a !== c) {
+      if (a === void 0 && c !== void 0) {
+        r.push(`+ ${c}`);
         continue;
       }
-      if (a !== void 0 && u === void 0) {
-        s.push(`- ${a}`);
+      if (a !== void 0 && c === void 0) {
+        r.push(`- ${a}`);
         continue;
       }
-      s.push(`- ${a ?? ""}`), s.push(`+ ${u ?? ""}`);
+      r.push(`- ${a ?? ""}`), r.push(`+ ${c ?? ""}`);
     }
   }
-  return s.join(`
+  return r.join(`
 `);
 }
-function dt(t) {
-  const e = S.getStore()?.evalFile;
+function yt(t) {
+  const e = A.getStore()?.evalFile;
   if (!e) {
-    c(
+    u(
       "toMatchSnapshot can only be used while running an eval file."
     );
     return;
   }
-  const n = lt(e);
+  const n = mt(e);
   try {
-    const o = H(n, "utf8");
+    const o = X(n, "utf8");
     if (o === t)
       return;
-    if (I()) {
-      C(n, t, "utf8");
+    if (H()) {
+      F(n, t, "utf8");
       return;
     }
-    const r = ft(o, t);
-    c(
+    const i = gt(o, t);
+    u(
       [
         `Snapshot mismatch at ${n}`,
         "",
         "Diff:",
-        r,
+        i,
         "",
-        "Run katt with --update-snapshots (or -u) to accept this change."
+        "Run 'npx katt --update-snapshots' (or -u) to accept this change."
       ].join(`
 `)
     );
   } catch (o) {
     if (o.code !== "ENOENT") {
-      c(
+      u(
         `Failed to read snapshot at ${n}: ${String(o)}`
       );
       return;
     }
     try {
-      Y(g(n), { recursive: !0 }), C(n, t, "utf8");
-    } catch (s) {
-      c(
-        `Failed to write snapshot at ${n}: ${String(s)}`
+      Z(v(n), { recursive: !0 }), F(n, t, "utf8");
+    } catch (r) {
+      u(
+        `Failed to write snapshot at ${n}: ${String(r)}`
       );
     }
   }
 }
-function vt(t) {
+function Mt(t) {
   return {
     toContain: (e) => {
-      ct(t, e);
+      pt(t, e);
     },
     toMatchSnapshot: () => {
-      dt(t);
+      yt(t);
     },
     promptCheck: async (e) => {
-      await st(t, e);
+      await ut(t, e);
     },
     toBeClassifiedAs: async (e, n) => {
-      await at(t, e, n);
+      await dt(t, e, n);
     }
   };
 }
 export {
-  Ct as describe,
-  vt as expect,
-  wt as it,
-  y as prompt,
-  bt as promptFile,
-  xt as runCli
+  St as describe,
+  Mt as expect,
+  Tt as it,
+  $ as prompt,
+  Ft as promptFile,
+  Et as runCli
 };

package/dist/katt.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as r } from "./runCli-B3oIBxOl.js";
+import { t as r } from "./runCli-C7uxWavX.js";
 r().then((e) => {
   process.exit(e);
 }).catch((e) => {

package/dist/runCli-C7uxWavX.js

@@ -0,0 +1,312 @@
+import { fileURLToPath as T, pathToFileURL as X } from "node:url";
+import { readdir as M } from "node:fs/promises";
+import { resolve as R } from "node:path";
+import { AsyncLocalStorage as v } from "node:async_hooks";
+import { readFileSync as U } from "node:fs";
+const W = /\.eval\.(js|ts)$/, G = /* @__PURE__ */ new Set([".git", "node_modules"]);
+async function J(t) {
+  const e = await M(t, { withFileTypes: !0 }), n = [];
+  return await Promise.all(
+    e.map(async (o) => {
+      const a = R(t, o.name);
+      if (o.isDirectory()) {
+        if (G.has(o.name))
+          return;
+        n.push(...await J(a));
+        return;
+      }
+      o.isFile() && W.test(o.name) && n.push(a);
+    })
+  ), n;
+}
+const B = new v(), O = {
+  describeStack: [],
+  itStack: [],
+  tokenUsageStack: [],
+  modelStack: []
+};
+let L = 0, $ = 0;
+const f = [], h = [];
+let k = 0;
+function i() {
+  return B.getStore() ?? O;
+}
+function Z(t) {
+  return {
+    describeStack: [...t.describeStack],
+    itStack: [...t.itStack],
+    tokenUsageStack: [...t.tokenUsageStack],
+    modelStack: [...t.modelStack]
+  };
+}
+function Y() {
+  return L += 1, `d${L}`;
+}
+function Q() {
+  return $ += 1, `i${$}`;
+}
+function St(t, e) {
+  const n = e ?? Z(i());
+  return B.run(n, t);
+}
+function At() {
+  return Z(i());
+}
+function kt(t) {
+  i().describeStack.push({ id: Y(), description: t });
+}
+function bt() {
+  i().describeStack.pop();
+}
+function V() {
+  return i().describeStack.map((t) => t.description).join(" > ");
+}
+function Lt() {
+  return [...i().describeStack];
+}
+function $t(t) {
+  i().itStack.push({ id: Q(), description: t }), i().tokenUsageStack.push(0), i().modelStack.push(void 0);
+}
+function wt() {
+  i().itStack.pop(), i().tokenUsageStack.pop(), i().modelStack.pop();
+}
+function K() {
+  return i().itStack.map((t) => t.description).join(" > ");
+}
+function jt() {
+  return [...i().itStack];
+}
+function vt(t) {
+  if (!Number.isFinite(t) || t <= 0)
+    return;
+  const e = i(), n = e.tokenUsageStack.length - 1;
+  n < 0 || (e.tokenUsageStack[n] += t);
+}
+function P() {
+  const t = i(), e = t.tokenUsageStack.length - 1;
+  return e < 0 ? 0 : t.tokenUsageStack[e] ?? 0;
+}
+function Jt(t) {
+  if (t.length === 0)
+    return;
+  const e = i(), n = e.modelStack.length - 1;
+  n < 0 || (e.modelStack[n] = t);
+}
+function z() {
+  const t = i(), e = t.modelStack.length - 1;
+  if (!(e < 0))
+    return t.modelStack[e];
+}
+function Bt(t) {
+  f.push(t);
+}
+function Zt() {
+  k += 1;
+}
+function E() {
+  return k;
+}
+function D() {
+  k = 0;
+}
+function xt(t) {
+  h.push(t);
+}
+function _() {
+  return [...h];
+}
+function Ht() {
+  return h.length;
+}
+function q() {
+  h.length = 0;
+}
+async function tt() {
+  const t = [];
+  for (; f.length > 0; ) {
+    const e = f.splice(0, f.length), n = await Promise.allSettled(e);
+    t.push(...n);
+  }
+  return t;
+}
+const et = "\x1B[1;36m", nt = "\x1B[33m", ot = "\x1B[38;5;208m", it = "\x1B[1;38;5;208m", m = "\x1B[0m";
+function r(t) {
+  return `${et}${t}${m}`;
+}
+function p(t) {
+  return `${nt}${t}${m}`;
+}
+function w(t) {
+  return `${ot}${t}${m}`;
+}
+function st(t) {
+  return `${it}${t}${m}`;
+}
+let A = "";
+function at() {
+  A = "";
+}
+function ct({
+  suitePath: t,
+  casePath: e,
+  didPass: n,
+  durationMs: o,
+  model: a,
+  tokenUsage: I
+}) {
+  const c = t.length > 0 ? t : "(root)", C = e.length > 0 ? e : "(root)";
+  A !== c && (console.log(`Suite "${r(c)}"`), A = c);
+  const g = n ? "✅ Passed in" : "❌ Failed in", l = [
+    `Test "${r(C)}"`,
+    `- ${g} ${r(`${o}ms`)}`
+  ];
+  a && l.push(`- Model ${r(a)}`), (I ?? 0) > 0 && l.push(`- Tokens used ${r(String(I))}`), l.push("---"), console.log(l.join(`
+`));
+}
+function yt(t, e, n = "(root)") {
+  const o = K();
+  ct({
+    suitePath: V(),
+    casePath: o.length > 0 ? o : n,
+    didPass: t,
+    durationMs: e,
+    model: z(),
+    tokenUsage: P()
+  });
+}
+const rt = new v(), S = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC42IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIKICB9LAogICJ0eXBlcyI6ICJkaXN0L2luZGV4LmQudHMiLAogICJkZXZEZXBlbmRlbmNpZXMiOiB7CiAgICAiQGJpb21lanMvYmlvbWUiOiAiMS45LjQiLAogICAgIkB0eXBlcy9ub2RlIjogIjI1LjIuMCIsCiAgICAidHN4IjogIjQuMjEuMCIsCiAgICAidHlwZXNjcmlwdCI6ICI1LjguMiIsCiAgICAidml0ZSI6ICI3LjMuMSIsCiAgICAidml0ZS1wbHVnaW4tZHRzIjogIjQuNS40IiwKICAgICJ2aXRlc3QiOiAiMy4yLjQiLAogICAgInZzY29kZS1qc29ucnBjIjogIl44LjIuMSIKICB9LAogICJkZXBlbmRlbmNpZXMiOiB7CiAgICAiQGdpdGh1Yi9jb3BpbG90LXNkayI6ICJeMC4xLjIxIgogIH0sCiAgImJ1Z3MiOiB7CiAgICAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQvaXNzdWVzIgogIH0sCiAgImhvbWVwYWdlIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQiCn0K", import.meta.url);
+let d;
+function lt() {
+  if (d !== void 0)
+    return d;
+  try {
+    const t = S.protocol === "data:" ? ut(S) : U(T(S), "utf8"), e = JSON.parse(t);
+    d = typeof e.version == "string" ? e.version : "unknown";
+  } catch {
+    d = "unknown";
+  }
+  return d;
+}
+function ut(t) {
+  const e = t.pathname.indexOf(",");
+  if (e < 0)
+    throw new Error("Invalid data URL.");
+  const n = t.pathname.slice(0, e), o = t.pathname.slice(e + 1);
+  return n.includes(";base64") ? Buffer.from(o, "base64").toString("utf8") : decodeURIComponent(o);
+}
+function j() {
+  const t = " ██╗  ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║   ██║      ██║", o = " ██╔═██╗ ██╔══██║   ██║      ██║", a = " ██║  ██╗██║  ██║   ██║      ██║", I = " ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝      ╚═╝", c = `v${lt()}`, C = Math.max(
+    0,
+    Math.floor((t.length - c.length) / 2)
+  ), g = `${" ".repeat(C)}${c}`;
+  console.log(`
+${p(t)}
+${p(e)}
+${p(n)}
+${w(o)}
+${w(a)}
+${st(I)}
+${p(g)}
+`);
+}
+let x = !1;
+function gt(t) {
+  x = t;
+}
+function Ft() {
+  return x;
+}
+function It(t) {
+  const e = String(t.getHours()).padStart(2, "0"), n = String(t.getMinutes()).padStart(2, "0"), o = String(t.getSeconds()).padStart(2, "0");
+  return `${e}:${n}:${o}`;
+}
+function dt() {
+  console.log(
+    [
+      "Usage:",
+      "  katt [options]",
+      "",
+      "Options:",
+      "  -h, --help              Show CLI usage information",
+      "  -u, --update-snapshots  Update snapshot files on mismatch"
+    ].join(`
+`)
+  );
+}
+async function Nt() {
+  const t = process.argv.slice(2);
+  if (t.includes("--help") || t.includes("-h"))
+    return j(), dt(), 0;
+  const n = t.includes("--update-snapshots") || t.includes("-u");
+  gt(n), j();
+  const o = /* @__PURE__ */ new Date();
+  at(), q(), D();
+  const a = await J(process.cwd());
+  if (a.length === 0)
+    return console.log("No .eval.js or .eval.ts files found."), 1;
+  const c = (await Promise.allSettled(
+    a.map(
+      (s) => rt.run(
+        { evalFile: s },
+        () => import(X(s).href)
+      )
+    )
+  )).map((s, u) => ({ result: s, file: a[u] })).filter(({ result: s }) => s.status === "rejected");
+  if (c.length > 0) {
+    for (const s of c) {
+      const u = s.result.status === "rejected" ? s.result.reason : void 0;
+      console.error(`Error executing ${s.file}: ${String(u)}`);
+    }
+    return 1;
+  }
+  const g = (await tt()).filter(
+    (s) => s.status === "rejected"
+  );
+  if (g.length > 0) {
+    for (const s of g)
+      s.status === "rejected" && console.error(`Error executing async test: ${String(s.reason)}`);
+    return 1;
+  }
+  const l = _();
+  if (l.length > 0) {
+    console.error("❌ Failed tests:");
+    for (const [s, u] of l.entries()) {
+      const b = [u.describePath, u.itPath].filter((N) => N.length > 0).join(" > "), F = b.length > 0 ? `${b}: ` : "";
+      console.error(`${s + 1}. ${F}${u.message}`);
+    }
+    return 1;
+  }
+  const H = E(), y = Date.now() - o.getTime();
+  return console.log(
+    [
+      "---",
+      `${r("Files")}  ${a.length} passed`,
+      `${r("Evals")}  ${H} passed`,
+      `${r("Start at")}  ${It(o)}`,
+      `${r("Duration")}  ${y}ms`
+    ].join(`
+`)
+  ), 0;
+}
+export {
+  Zt as a,
+  Bt as b,
+  At as c,
+  wt as d,
+  kt as e,
+  bt as f,
+  Ht as g,
+  vt as h,
+  rt as i,
+  xt as j,
+  K as k,
+  yt as l,
+  V as m,
+  Ft as n,
+  Lt as o,
+  $t as p,
+  jt as q,
+  St as r,
+  Jt as s,
+  Nt as t
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "katt",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "CLI tool that tests the output of agentic AI tools",
   "keywords": [
     "cli",

package/dist/runCli-B3oIBxOl.js

@@ -1,317 +0,0 @@
-import { fileURLToPath as Z, pathToFileURL as y } from "node:url";
-import { readdir as N } from "node:fs/promises";
-import { resolve as M } from "node:path";
-import { AsyncLocalStorage as v } from "node:async_hooks";
-import { readFileSync as X } from "node:fs";
-const H = /\.eval\.(js|ts)$/, R = /* @__PURE__ */ new Set([".git", "node_modules"]);
-async function w(t) {
-  const e = await N(t, { withFileTypes: !0 }), n = [];
-  return await Promise.all(
-    e.map(async (i) => {
-      const s = M(t, i.name);
-      if (i.isDirectory()) {
-        if (R.has(i.name))
-          return;
-        n.push(...await w(s));
-        return;
-      }
-      i.isFile() && H.test(i.name) && n.push(s);
-    })
-  ), n;
-}
-const j = new v(), W = {
-  describeStack: [],
-  itStack: [],
-  tokenUsageStack: [],
-  modelStack: []
-};
-let b = 0, L = 0;
-const p = [], f = [];
-let k = 0;
-function o() {
-  return j.getStore() ?? W;
-}
-function J(t) {
-  return {
-    describeStack: [...t.describeStack],
-    itStack: [...t.itStack],
-    tokenUsageStack: [...t.tokenUsageStack],
-    modelStack: [...t.modelStack]
-  };
-}
-function U() {
-  return b += 1, `d${b}`;
-}
-function G() {
-  return L += 1, `i${L}`;
-}
-function B(t) {
-  return t.tokenUsageStack.length - 1;
-}
-function F(t) {
-  return t.modelStack.length - 1;
-}
-function bt(t, e) {
-  const n = e ?? J(o());
-  return j.run(n, t);
-}
-function Lt() {
-  return J(o());
-}
-function $t(t) {
-  o().describeStack.push({ id: U(), description: t });
-}
-function vt() {
-  o().describeStack.pop();
-}
-function O() {
-  return o().describeStack.map((t) => t.description).join(" > ");
-}
-function wt() {
-  return [...o().describeStack];
-}
-function jt(t) {
-  const e = o();
-  e.itStack.push({ id: G(), description: t }), e.tokenUsageStack.push(0), e.modelStack.push(void 0);
-}
-function Jt() {
-  const t = o();
-  t.itStack.pop(), t.tokenUsageStack.pop(), t.modelStack.pop();
-}
-function Y() {
-  return o().itStack.map((t) => t.description).join(" > ");
-}
-function Bt() {
-  return [...o().itStack];
-}
-function Ft(t) {
-  if (!Number.isFinite(t) || t <= 0)
-    return;
-  const e = o(), n = B(e);
-  n < 0 || (e.tokenUsageStack[n] += t);
-}
-function Q() {
-  const t = o(), e = B(t);
-  return e < 0 ? 0 : t.tokenUsageStack[e] ?? 0;
-}
-function xt(t) {
-  if (t.length === 0)
-    return;
-  const e = o(), n = F(e);
-  n < 0 || (e.modelStack[n] = t);
-}
-function V() {
-  const t = o(), e = F(t);
-  if (!(e < 0))
-    return t.modelStack[e];
-}
-function Tt(t) {
-  p.push(t);
-}
-function Zt() {
-  k += 1;
-}
-function K() {
-  return k;
-}
-function P() {
-  k = 0;
-}
-function yt(t) {
-  f.push(t);
-}
-function E() {
-  return [...f];
-}
-function Nt() {
-  return f.length;
-}
-function z() {
-  f.length = 0;
-}
-async function D() {
-  const t = [];
-  for (; p.length > 0; ) {
-    const e = p.splice(0, p.length), n = await Promise.allSettled(e);
-    t.push(...n);
-  }
-  return t;
-}
-const _ = "\x1B[1;36m", q = "\x1B[33m", tt = "\x1B[38;5;208m", et = "\x1B[1;38;5;208m", m = "\x1B[0m";
-function r(t) {
-  return `${_}${t}${m}`;
-}
-function C(t) {
-  return `${q}${t}${m}`;
-}
-function $(t) {
-  return `${tt}${t}${m}`;
-}
-function nt(t) {
-  return `${et}${t}${m}`;
-}
-let A = "";
-function it() {
-  A = "";
-}
-function ot({
-  suitePath: t,
-  casePath: e,
-  didPass: n,
-  durationMs: i,
-  model: s,
-  tokenUsage: c
-}) {
-  const l = t.length > 0 ? t : "(root)", u = e.length > 0 ? e : "(root)";
-  A !== l && (console.log(`Suite "${r(l)}"`), A = l);
-  const g = n ? "✅ Passed in" : "❌ Failed in", I = [
-    `Test "${r(u)}"`,
-    `- ${g} ${r(`${i}ms`)}`
-  ];
-  s && I.push(`- Model ${r(s)}`), (c ?? 0) > 0 && I.push(`- Tokens used ${r(String(c))}`), I.push("---"), console.log(I.join(`
-`));
-}
-function Mt(t, e, n = "(root)") {
-  const i = Y();
-  ot({
-    suitePath: O(),
-    casePath: i.length > 0 ? i : n,
-    didPass: t,
-    durationMs: e,
-    model: V(),
-    tokenUsage: Q()
-  });
-}
-const st = new v(), h = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC41IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIKICB9LAogICJ0eXBlcyI6ICJkaXN0L2luZGV4LmQudHMiLAogICJkZXZEZXBlbmRlbmNpZXMiOiB7CiAgICAiQGJpb21lanMvYmlvbWUiOiAiMS45LjQiLAogICAgIkB0eXBlcy9ub2RlIjogIjI1LjIuMCIsCiAgICAidHN4IjogIjQuMjEuMCIsCiAgICAidHlwZXNjcmlwdCI6ICI1LjguMiIsCiAgICAidml0ZSI6ICI3LjMuMSIsCiAgICAidml0ZS1wbHVnaW4tZHRzIjogIjQuNS40IiwKICAgICJ2aXRlc3QiOiAiMy4yLjQiLAogICAgInZzY29kZS1qc29ucnBjIjogIl44LjIuMSIKICB9LAogICJkZXBlbmRlbmNpZXMiOiB7CiAgICAiQGdpdGh1Yi9jb3BpbG90LXNkayI6ICJeMC4xLjIxIgogIH0sCiAgImJ1Z3MiOiB7CiAgICAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQvaXNzdWVzIgogIH0sCiAgImhvbWVwYWdlIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQiCn0K", import.meta.url);
-let d;
-function at() {
-  if (d !== void 0)
-    return d;
-  try {
-    const t = h.protocol === "data:" ? rt(h) : X(Z(h), "utf8"), e = JSON.parse(t);
-    d = typeof e.version == "string" ? e.version : "unknown";
-  } catch {
-    d = "unknown";
-  }
-  return d;
-}
-function rt(t) {
-  const e = t.pathname.indexOf(",");
-  if (e < 0)
-    throw new Error("Invalid data URL.");
-  const n = t.pathname.slice(0, e), i = t.pathname.slice(e + 1);
-  return n.includes(";base64") ? Buffer.from(i, "base64").toString("utf8") : decodeURIComponent(i);
-}
-function ct() {
-  const t = " ██╗  ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║   ██║      ██║", i = " ██╔═██╗ ██╔══██║   ██║      ██║", s = " ██║  ██╗██║  ██║   ██║      ██║", c = " ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝      ╚═╝", l = `v${at()}`, u = Math.max(
-    0,
-    Math.floor((t.length - l.length) / 2)
-  ), g = `${" ".repeat(u)}${l}`;
-  console.log(`
-${C(t)}
-${C(e)}
-${C(n)}
-${$(i)}
-${$(s)}
-${nt(c)}
-${C(g)}
-`);
-}
-let x = !1;
-function lt(t) {
-  x = t;
-}
-function Xt() {
-  return x;
-}
-function ut(t) {
-  const e = String(t.getHours()).padStart(2, "0"), n = String(t.getMinutes()).padStart(2, "0"), i = String(t.getSeconds()).padStart(2, "0");
-  return `${e}:${n}:${i}`;
-}
-function gt(t) {
-  return t.includes("--update-snapshots") || t.includes("-u");
-}
-function It(t) {
-  return [t.describePath, t.itPath].filter((e) => e.length > 0).join(" > ");
-}
-function dt(t) {
-  for (const e of t)
-    console.error(`Error executing ${e.file}: ${String(e.reason)}`);
-}
-function Ct(t) {
-  for (const e of t)
-    console.error(`Error executing async test: ${String(e.reason)}`);
-}
-function pt(t) {
-  console.error("❌ Failed tests:");
-  for (const [e, n] of t.entries()) {
-    const i = It(n), s = i.length > 0 ? `${i}: ` : "";
-    console.error(`${e + 1}. ${s}${n.message}`);
-  }
-}
-function ft(t, e, n, i) {
-  return [
-    "---",
-    `${r("Files")}  ${t} passed`,
-    `${r("Evals")}  ${e} passed`,
-    `${r("Start at")}  ${ut(n)}`,
-    `${r("Duration")}  ${i}ms`
-  ].join(`
-`);
-}
-async function Ht() {
-  const t = process.argv.slice(2), e = gt(t);
-  lt(e), ct();
-  const n = /* @__PURE__ */ new Date();
-  it(), z(), P();
-  const i = await w(process.cwd());
-  if (i.length === 0)
-    return console.log("No .eval.js or .eval.ts files found."), 1;
-  const c = (await Promise.allSettled(
-    i.map(
-      (a) => st.run(
-        { evalFile: a },
-        () => import(y(a).href)
-      )
-    )
-  )).map((a, S) => ({ result: a, file: i[S] })).filter(({ result: a }) => a.status === "rejected").map(({ result: a, file: S }) => ({
-    file: S,
-    reason: a.status === "rejected" ? a.reason : void 0
-  }));
-  if (c.length > 0)
-    return dt(c), 1;
-  const u = (await D()).filter(
-    (a) => a.status === "rejected"
-  );
-  if (u.length > 0)
-    return Ct(u), 1;
-  const g = E();
-  if (g.length > 0)
-    return pt(g), 1;
-  const I = K(), T = Date.now() - n.getTime();
-  return console.log(
-    ft(i.length, I, n, T)
-  ), 0;
-}
-export {
-  Tt as a,
-  Jt as b,
-  Lt as c,
-  Zt as d,
-  $t as e,
-  vt as f,
-  Nt as g,
-  st as h,
-  Ft as i,
-  yt as j,
-  Y as k,
-  Mt as l,
-  O as m,
-  Xt as n,
-  wt as o,
-  jt as p,
-  Bt as q,
-  bt as r,
-  xt as s,
-  Ht as t
-};