katt 0.0.6 → 0.0.8

package/dist/runCli-425rgVp8.js

@@ -0,0 +1,424 @@
+import { fileURLToPath as X, pathToFileURL as Y } from "node:url";
+import { readdir as V, readFile as D } from "node:fs/promises";
+import { resolve as m, matchesGlob as E, dirname as Q } from "node:path";
+import { AsyncLocalStorage as N } from "node:async_hooks";
+import { readFileSync as _ } from "node:fs";
+const q = /\.eval\.(js|ts)$/, tt = /* @__PURE__ */ new Set([".git", "node_modules"]);
+function j(t, e) {
+  return e.some((n) => t === n || E(t, n));
+}
+async function Z(t, e = []) {
+  const n = await V(t, { withFileTypes: !0 }), o = [];
+  return await Promise.all(
+    n.map(async (i) => {
+      const a = m(t, i.name);
+      if (i.isDirectory()) {
+        if (tt.has(i.name) || j(a, e))
+          return;
+        o.push(...await Z(a, e));
+        return;
+      }
+      i.isFile() && q.test(i.name) && !j(a, e) && o.push(a);
+    })
+  ), o;
+}
+const x = new N(), et = {
+  describeStack: [],
+  itStack: [],
+  tokenUsageStack: [],
+  modelStack: []
+};
+let F = 0, G = 0;
+const h = [], A = [];
+let v = 0;
+function s() {
+  return x.getStore() ?? et;
+}
+function U(t) {
+  return {
+    describeStack: [...t.describeStack],
+    itStack: [...t.itStack],
+    tokenUsageStack: [...t.tokenUsageStack],
+    modelStack: [...t.modelStack]
+  };
+}
+function nt() {
+  return F += 1, `d${F}`;
+}
+function ot() {
+  return G += 1, `i${G}`;
+}
+function Mt(t, e) {
+  const n = e ?? U(s());
+  return x.run(n, t);
+}
+function Ot() {
+  return U(s());
+}
+function Bt(t) {
+  s().describeStack.push({ id: nt(), description: t });
+}
+function zt() {
+  s().describeStack.pop();
+}
+function it() {
+  return s().describeStack.map((t) => t.description).join(" > ");
+}
+function Kt() {
+  return [...s().describeStack];
+}
+function Xt(t) {
+  s().itStack.push({ id: ot(), description: t }), s().tokenUsageStack.push(0), s().modelStack.push(void 0);
+}
+function Yt() {
+  s().itStack.pop(), s().tokenUsageStack.pop(), s().modelStack.pop();
+}
+function st() {
+  return s().itStack.map((t) => t.description).join(" > ");
+}
+function Vt() {
+  return [...s().itStack];
+}
+function Dt(t) {
+  if (!Number.isFinite(t) || t <= 0)
+    return;
+  const e = s(), n = e.tokenUsageStack.length - 1;
+  n < 0 || (e.tokenUsageStack[n] += t);
+}
+function rt() {
+  const t = s(), e = t.tokenUsageStack.length - 1;
+  return e < 0 ? 0 : t.tokenUsageStack[e] ?? 0;
+}
+function Et(t) {
+  if (t.length === 0)
+    return;
+  const e = s(), n = e.modelStack.length - 1;
+  n < 0 || (e.modelStack[n] = t);
+}
+function at() {
+  const t = s(), e = t.modelStack.length - 1;
+  if (!(e < 0))
+    return t.modelStack[e];
+}
+function Qt(t) {
+  h.push(t);
+}
+function _t() {
+  v += 1;
+}
+function ct() {
+  return v;
+}
+function lt() {
+  v = 0;
+}
+function qt(t) {
+  A.push(t);
+}
+function ut() {
+  return [...A];
+}
+function te() {
+  return A.length;
+}
+function gt() {
+  A.length = 0;
+}
+async function dt() {
+  const t = [];
+  for (; h.length > 0; ) {
+    const e = h.splice(0, h.length), n = await Promise.allSettled(e);
+    t.push(...n);
+  }
+  return t;
+}
+const ft = "\x1B[1;36m", It = "\x1B[33m", pt = "\x1B[38;5;208m", Ct = "\x1B[1;38;5;208m", S = "\x1B[0m";
+function l(t) {
+  return `${ft}${t}${S}`;
+}
+function C(t) {
+  return `${It}${t}${S}`;
+}
+function J(t) {
+  return `${pt}${t}${S}`;
+}
+function ht(t) {
+  return `${Ct}${t}${S}`;
+}
+let w = "";
+function mt() {
+  w = "";
+}
+function At({
+  suitePath: t,
+  casePath: e,
+  didPass: n,
+  durationMs: o,
+  model: i,
+  tokenUsage: a
+}) {
+  const c = t.length > 0 ? t : "(root)", p = e.length > 0 ? e : "(root)";
+  w !== c && (console.log(`Suite "${l(c)}"`), w = c);
+  const g = n ? "✅ Passed in" : "❌ Failed in", d = [
+    `Test "${l(p)}"`,
+    `- ${g} ${l(`${o}ms`)}`
+  ];
+  i && d.push(`- Model ${l(i)}`), (a ?? 0) > 0 && d.push(`- Tokens used ${l(String(a))}`), d.push("---"), console.log(d.join(`
+`));
+}
+function ee(t, e, n = "(root)") {
+  const o = st();
+  At({
+    suitePath: it(),
+    casePath: o.length > 0 ? o : n,
+    didPass: t,
+    durationMs: e,
+    model: at(),
+    tokenUsage: rt()
+  });
+}
+const St = new N(), b = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC43IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIsCiAgICAidGVzdDpidWlsZDpjb2RleCI6ICJub2RlIC4vZGlzdC9rYXR0LmpzIC0tY29uZmlnLWZpbGU9Li9rYXR0LWNvZGV4Lmpzb24iCiAgfSwKICAidHlwZXMiOiAiZGlzdC9pbmRleC5kLnRzIiwKICAiZGV2RGVwZW5kZW5jaWVzIjogewogICAgIkBiaW9tZWpzL2Jpb21lIjogIjIuNC4yIiwKICAgICJAdHlwZXMvbm9kZSI6ICIyNS4yLjMiLAogICAgInRzeCI6ICI0LjIxLjAiLAogICAgInR5cGVzY3JpcHQiOiAiNS45LjMiLAogICAgInZpdGUiOiAiNy4zLjEiLAogICAgInZpdGUtcGx1Z2luLWR0cyI6ICI0LjUuNCIsCiAgICAidml0ZXN0IjogIjQuMC4xOCIsCiAgICAidnNjb2RlLWpzb25ycGMiOiAiXjguMi4xIgogIH0sCiAgImRlcGVuZGVuY2llcyI6IHsKICAgICJAZ2l0aHViL2NvcGlsb3Qtc2RrIjogIjAuMS4yMyIKICB9LAogICJidWdzIjogewogICAgInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0L2lzc3VlcyIKICB9LAogICJob21lcGFnZSI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0Igp9Cg==", import.meta.url);
+let f;
+function bt() {
+  if (f !== void 0)
+    return f;
+  try {
+    const t = b.protocol === "data:" ? kt(b) : _(X(b), "utf8"), e = JSON.parse(t);
+    f = typeof e.version == "string" ? e.version : "unknown";
+  } catch {
+    f = "unknown";
+  }
+  return f;
+}
+function kt(t) {
+  const e = t.pathname.indexOf(",");
+  if (e < 0)
+    throw new Error("Invalid data URL.");
+  const n = t.pathname.slice(0, e), o = t.pathname.slice(e + 1);
+  return n.includes(";base64") ? Buffer.from(o, "base64").toString("utf8") : decodeURIComponent(o);
+}
+function k() {
+  const t = " ██╗  ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║   ██║      ██║", o = " ██╔═██╗ ██╔══██║   ██║      ██║", i = " ██║  ██╗██║  ██║   ██║      ██║", a = " ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝      ╚═╝", c = `v${bt()}`, p = Math.max(
+    0,
+    Math.floor((t.length - c.length) / 2)
+  ), g = `${" ".repeat(p)}${c}`;
+  console.log(`
+${C(t)}
+${C(e)}
+${C(n)}
+${J(o)}
+${J(i)}
+${ht(a)}
+${C(g)}
+`);
+}
+let W = !1;
+function wt(t) {
+  W = t;
+}
+function ne() {
+  return W;
+}
+const vt = "gh-copilot", R = "katt.json";
+let I;
+function P() {
+  return typeof I == "string" && I.length > 0 ? {
+    resolvedPath: m(process.cwd(), I),
+    label: I
+  } : {
+    resolvedPath: m(process.cwd(), R),
+    label: R
+  };
+}
+function T(t) {
+  I = t;
+}
+function Lt(t, e) {
+  return typeof t == "object" && t !== null && "code" in t && t.code === e;
+}
+function yt(t, e) {
+  try {
+    const n = JSON.parse(t);
+    return typeof n == "object" && n !== null ? n : void 0;
+  } catch (n) {
+    console.warn(`Failed to parse ${e}: ${String(n)}`);
+    return;
+  }
+}
+function $t(t, e) {
+  t && "copilot" in t && console.warn(
+    `Deprecated config property "copilot" found in ${e}. Use "agent" and "agentOptions" instead.`
+  );
+}
+async function H() {
+  const { resolvedPath: t, label: e } = P();
+  try {
+    const n = await D(t, "utf8"), o = yt(n, e);
+    return $t(o, e), o;
+  } catch (n) {
+    if (Lt(n, "ENOENT"))
+      return;
+    console.warn(`Failed to read ${e}: ${String(n)}`);
+    return;
+  }
+}
+function M(t) {
+  if (t === "gh-copilot" || t === "codex")
+    return t;
+}
+function jt(t, e) {
+  if (!t || M(t?.agent) !== e)
+    return;
+  const n = t.agentOptions;
+  if (typeof n != "object" || n === null || Array.isArray(n))
+    return;
+  const o = { ...n }, i = o.model;
+  return (typeof i != "string" || i.length === 0) && delete o.model, Object.keys(o).length > 0 ? o : void 0;
+}
+function Ft(t) {
+  if (!(typeof t != "number" || !Number.isFinite(t)) && !(t <= 0))
+    return Math.floor(t);
+}
+function Gt(t) {
+  const e = t?.prompt;
+  if (!(typeof e != "object" || e === null || Array.isArray(e)))
+    return Ft(e.timeoutMs);
+}
+function Jt(t) {
+  if (!t || !Array.isArray(t.ignorePatterns))
+    return [];
+  const e = Q(P().resolvedPath);
+  return t.ignorePatterns.filter((n) => typeof n == "string" && n.length > 0).map((n) => m(e, n));
+}
+async function oe() {
+  const t = await H(), e = M(t?.agent) ?? vt;
+  return {
+    agent: e,
+    agentOptions: jt(t, e),
+    promptTimeoutMs: Gt(t)
+  };
+}
+async function Rt() {
+  const t = await H();
+  return Jt(t);
+}
+function Tt(t) {
+  const e = String(t.getHours()).padStart(2, "0"), n = String(t.getMinutes()).padStart(2, "0"), o = String(t.getSeconds()).padStart(2, "0");
+  return `${e}:${n}:${o}`;
+}
+function Nt() {
+  console.log(
+    [
+      "Usage:",
+      "  katt [options]",
+      "",
+      "Options:",
+      "  -h, --help              Show CLI usage information",
+      "  -u, --update-snapshots  Update snapshot files on mismatch",
+      "      --config-file PATH  Use a custom config file instead of katt.json"
+    ].join(`
+`)
+  );
+}
+function Zt(t) {
+  let e;
+  for (let n = 0; n < t.length; n += 1) {
+    const o = t[n];
+    if (o === "--config-file") {
+      const i = t[n + 1];
+      if (i === void 0 || i.length === 0)
+        return { error: "Missing value for --config-file." };
+      e = i, n += 1;
+      continue;
+    }
+    if (o.startsWith("--config-file=")) {
+      const i = o.slice(14);
+      if (i.length === 0)
+        return { error: "Missing value for --config-file." };
+      e = i;
+    }
+  }
+  return { configFilePath: e };
+}
+async function ie() {
+  const t = process.argv.slice(2);
+  if (T(void 0), t.includes("--help") || t.includes("-h"))
+    return k(), Nt(), 0;
+  const n = Zt(t);
+  if (n.error)
+    return k(), console.error(n.error), 1;
+  T(n.configFilePath);
+  const o = t.includes("--update-snapshots") || t.includes("-u");
+  wt(o), k();
+  const i = /* @__PURE__ */ new Date();
+  mt(), gt(), lt();
+  const a = await Rt(), c = await Z(process.cwd(), a);
+  if (c.length === 0)
+    return console.log("No .eval.js or .eval.ts files found."), 1;
+  const g = (await Promise.allSettled(
+    c.map(
+      (r) => St.run(
+        { evalFile: r },
+        () => import(Y(r).href)
+      )
+    )
+  )).map((r, u) => ({ result: r, file: c[u] })).filter(({ result: r }) => r.status === "rejected");
+  if (g.length > 0) {
+    for (const r of g) {
+      const u = r.result.status === "rejected" ? r.result.reason : void 0;
+      console.error(`Error executing ${r.file}: ${String(u)}`);
+    }
+    return 1;
+  }
+  const L = (await dt()).filter(
+    (r) => r.status === "rejected"
+  );
+  if (L.length > 0) {
+    for (const r of L)
+      r.status === "rejected" && console.error(`Error executing async test: ${String(r.reason)}`);
+    return 1;
+  }
+  const y = ut();
+  if (y.length > 0) {
+    console.error("❌ Failed tests:");
+    for (const [r, u] of y.entries()) {
+      const $ = [u.describePath, u.itPath].filter((K) => K.length > 0).join(" > "), z = $.length > 0 ? `${$}: ` : "";
+      console.error(`${r + 1}. ${z}${u.message}`);
+    }
+    return 1;
+  }
+  const O = ct(), B = Date.now() - i.getTime();
+  return console.log(
+    [
+      "---",
+      `${l("Files")}  ${c.length} passed`,
+      `${l("Evals")}  ${O} passed`,
+      `${l("Start at")}  ${Tt(i)}`,
+      `${l("Duration")}  ${B}ms`
+    ].join(`
+`)
+  ), 0;
+}
+export {
+  _t as a,
+  Qt as b,
+  Ot as c,
+  Yt as d,
+  Bt as e,
+  zt as f,
+  te as g,
+  oe as h,
+  Dt as i,
+  St as j,
+  qt as k,
+  ee as l,
+  st as m,
+  it as n,
+  ne as o,
+  Xt as p,
+  Kt as q,
+  Mt as r,
+  Et as s,
+  Vt as t,
+  ie as u
+};

package/katt-codex.json

@@ -0,0 +1,4 @@
+{
+  "agent": "codex",
+  "ignorePatterns": ["./examples/**"]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "katt",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "CLI tool that tests the output of agentic AI tools",
   "keywords": [
     "cli",
@@ -29,21 +29,22 @@
     "format": "biome format --write ./src",
     "test": "vitest",
     "typecheck": "tsc -p tsconfig.json --noEmit",
-    "test:build": "node ./dist/katt.js"
+    "test:build": "node ./dist/katt.js",
+    "test:build:codex": "node ./dist/katt.js --config-file=./katt-codex.json"
   },
   "types": "dist/index.d.ts",
   "devDependencies": {
-    "@biomejs/biome": "1.9.4",
-    "@types/node": "25.2.0",
+    "@biomejs/biome": "2.4.2",
+    "@types/node": "25.2.3",
     "tsx": "4.21.0",
-    "typescript": "5.8.2",
+    "typescript": "5.9.3",
     "vite": "7.3.1",
     "vite-plugin-dts": "4.5.4",
-    "vitest": "3.2.4",
+    "vitest": "4.0.18",
     "vscode-jsonrpc": "^8.2.1"
   },
   "dependencies": {
-    "@github/copilot-sdk": "^0.1.21"
+    "@github/copilot-sdk": "0.1.23"
   },
   "bugs": {
     "url": "https://github.com/raphaelpor/katt/issues"

package/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}