katt 0.0.8 → 0.0.9

package/CODE_OF_CONDUCT.MD

@@ -0,0 +1,92 @@
+
+# Contributor Covenant 3.0 Code of Conduct
+
+## Our Pledge
+
+We pledge to make our community welcoming, safe, and equitable for all.
+
+We are committed to fostering an environment that respects and promotes the dignity, rights, and contributions of all individuals, regardless of characteristics including race, ethnicity, caste, color, age, physical characteristics, neurodiversity, disability, sex or gender, gender identity or expression, sexual orientation, language, philosophy or religion, national or social origin, socio-economic position, level of education, or other status. The same privileges of participation are extended to everyone who participates in good faith and in accordance with this Covenant.
+
+## Encouraged Behaviors
+
+While acknowledging differences in social norms, we all strive to meet our community's expectations for positive behavior. We also understand that our words and actions may be interpreted differently than we intend based on culture, background, or native language.
+
+With these considerations in mind, we agree to behave mindfully toward each other and act in ways that center our shared values, including:
+
+1. Respecting the **purpose of our community**, our activities, and our ways of gathering.
+2. Engaging **kindly and honestly** with others.
+3. Respecting **different viewpoints** and experiences.
+4. **Taking responsibility** for our actions and contributions.
+5. Gracefully giving and accepting **constructive feedback**.
+6. Committing to **repairing harm** when it occurs.
+7. Behaving in other ways that promote and sustain the **well-being of our community**.
+
+
+## Restricted Behaviors
+
+We agree to restrict the following behaviors in our community. Instances, threats, and promotion of these behaviors are violations of this Code of Conduct.
+
+1. **Harassment.** Violating explicitly expressed boundaries or engaging in unnecessary personal attention after any clear request to stop.
+2. **Character attacks.** Making insulting, demeaning, or pejorative comments directed at a community member or group of people.
+3. **Stereotyping or discrimination.** Characterizing anyone’s personality or behavior on the basis of immutable identities or traits.
+4. **Sexualization.** Behaving in a way that would generally be considered inappropriately intimate in the context or purpose of the community.
+5. **Violating confidentiality**. Sharing or acting on someone's personal or private information without their permission.
+6. **Endangerment.** Causing, encouraging, or threatening violence or other harm toward any person or group.
+7. Behaving in other ways that **threaten the well-being** of our community.
+
+### Other Restrictions
+
+1. **Misleading identity.** Impersonating someone else for any reason, or pretending to be someone else to evade enforcement actions.
+2. **Failing to credit sources.** Not properly crediting the sources of content you contribute.
+3. **Promotional materials**. Sharing marketing or other commercial content in a way that is outside the norms of the community.
+4. **Irresponsible communication.** Failing to responsibly present content which includes, links or describes any other restricted behaviors.
+
+
+## Reporting an Issue
+
+Tensions can occur between community members even when they are trying their best to collaborate. Not every conflict represents a code of conduct violation, and this Code of Conduct reinforces encouraged behaviors and norms that can help avoid conflicts and minimize harm.
+
+When an incident does occur, it is important to report it promptly. To report a possible violation, **Open a discussion in this repository.**
+
+Community Moderators take reports of violations seriously and will make every effort to respond in a timely manner. They will investigate all reports of code of conduct violations, reviewing messages, logs, and recordings, or interviewing witnesses and other participants. Community Moderators will keep investigation and enforcement actions as transparent as possible while prioritizing safety and confidentiality. In order to honor these values, enforcement actions are carried out in private with the involved parties, but communicating to the whole community may be part of a mutually agreed upon resolution.
+
+
+## Addressing and Repairing Harm
+
+****
+
+If an investigation by the Community Moderators finds that this Code of Conduct has been violated, the following enforcement ladder may be used to determine how best to repair harm, based on the incident's impact on the individuals involved and the community as a whole. Depending on the severity of a violation, lower rungs on the ladder may be skipped.
+
+1) Warning
+   1) Event: A violation involving a single incident or series of incidents.
+   2) Consequence: A private, written warning from the Community Moderators.
+   3) Repair: Examples of repair include a private written apology, acknowledgement of responsibility, and seeking clarification on expectations.
+2) Temporarily Limited Activities
+   1) Event: A repeated incidence of a violation that previously resulted in a warning, or the first incidence of a more serious violation.
+   2) Consequence: A private, written warning with a time-limited cooldown period designed to underscore the seriousness of the situation and give the community members involved time to process the incident. The cooldown period may be limited to particular communication channels or interactions with particular community members.
+   3) Repair: Examples of repair may include making an apology, using the cooldown period to reflect on actions and impact, and being thoughtful about re-entering community spaces after the period is over.
+3) Temporary Suspension
+   1) Event: A pattern of repeated violation which the Community Moderators have tried to address with warnings, or a single serious violation.
+   2) Consequence: A private written warning with conditions for return from suspension. In general, temporary suspensions give the person being suspended time to reflect upon their behavior and possible corrective actions.
+   3) Repair: Examples of repair include respecting the spirit of the suspension, meeting the specified conditions for return, and being thoughtful about how to reintegrate with the community when the suspension is lifted.
+4) Permanent Ban
+   1) Event: A pattern of repeated code of conduct violations that other steps on the ladder have failed to resolve, or a violation so serious that the Community Moderators determine there is no way to keep the community safe with this person as a member.
+   2) Consequence: Access to all community spaces, tools, and communication channels is removed. In general, permanent bans should be rarely used, should have strong reasoning behind them, and should only be resorted to if working through other remedies has failed to change the behavior.
+   3) Repair: There is no possible repair in cases of this severity.
+
+This enforcement ladder is intended as a guideline. It does not limit the ability of Community Managers to use their discretion and judgment, in keeping with the best interests of our community.
+
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public or other spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+
+## Attribution
+
+This Code of Conduct is adapted from the Contributor Covenant, version 3.0, permanently available at [https://www.contributor-covenant.org/version/3/0/](https://www.contributor-covenant.org/version/3/0/).
+
+Contributor Covenant is stewarded by the Organization for Ethical Source and licensed under CC BY-SA 4.0. To view a copy of this license, visit [https://creativecommons.org/licenses/by-sa/4.0/](https://creativecommons.org/licenses/by-sa/4.0/)
+
+For answers to common questions about Contributor Covenant, see the FAQ at [https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). Translations are provided at [https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). Additional enforcement and community guideline resources can be found at [https://www.contributor-covenant.org/resources](https://www.contributor-covenant.org/resources). The enforcement ladder was inspired by the work of [Mozilla’s code of conduct team](https://github.com/mozilla/inclusion).
+

package/README.md

@@ -12,16 +12,10 @@ Katt is a lightweight testing framework for running AI Evals, inspired by [Jest]
 - [Articles](#articles)
 - [Hello World - Example](#hello-world---example)
 - [Main Features](#main-features)
-- [Usage](#usage)
 - [Installation](#installation)
 - [Basic Usage](#basic-usage)
-- [Using promptFile](#using-promptfile)
 - [Specifying AI Models](#specifying-ai-models)
 - [Development](#development)
-- [Setup](#setup)
-- [Available Scripts](#available-scripts)
-- [Verification Process](#verification-process)
-- [Project Structure](#project-structure)
 - [How It Works](#how-it-works)
 - [Requirements](#requirements)
 - [License](#license)
@@ -29,7 +23,8 @@ Katt is a lightweight testing framework for running AI Evals, inspired by [Jest]
 
 ## Overview
 
-Katt is designed to evaluate and validate the behavior of AI agents like **Claude Code**, **GitHub Copilot**, **OpenAI Codex** and more. It provides a simple, intuitive API for writing tests that interact with AI models and assert their responses.
+#### ✨ Run your own benchmarks and evaluations ✨
+**Katt** is designed to evaluate and validate the behavior of AI agents like **Claude Code**, **GitHub Copilot**, **OpenAI Codex** and more. It provides a simple, intuitive API for writing tests that interact with AI models and assert their responses.
 
 ## API Documentation
 

package/SECURITY.md

@@ -0,0 +1,10 @@
+# Security Policy
+
+## Supported Versions
+
+Since Katt is under development, only the latest version will be supported.
+
+## Reporting a Vulnerability
+
+- Create an issue on this repository.
+- Describe the vulnerability and the level of it.

package/dist/index.js

@@ -1,5 +1,5 @@
-import { r as N, c as I, a as j, p as z, b as C, l as h, d as T, g as D, e as B, f as $, h as H, s as A, i as W, j as L, k as G, m as X, n as Y, o as q, q as K, t as J } from "./runCli-425rgVp8.js";
-import { u as jt } from "./runCli-425rgVp8.js";
+import { r as N, c as I, a as j, p as z, b as C, l as h, d as T, g as D, e as B, f as $, h as H, s as A, i as W, j as L, k as G, m as X, n as Y, o as q, q as K, t as J } from "./runCli-j5xhVCdB.js";
+import { u as jt } from "./runCli-j5xhVCdB.js";
 import { CopilotClient as Q } from "@github/copilot-sdk";
 import { mkdtemp as V, rm as Z, readFile as R } from "node:fs/promises";
 import { join as S, dirname as v, isAbsolute as tt, resolve as et, basename as nt } from "node:path";

package/dist/katt.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { u as r } from "./runCli-425rgVp8.js";
+import { u as r } from "./runCli-j5xhVCdB.js";
 r().then((e) => {
   process.exit(e);
 }).catch((e) => {

package/dist/{runCli-425rgVp8.js → runCli-j5xhVCdB.js}

@@ -1,6 +1,6 @@
 import { fileURLToPath as X, pathToFileURL as Y } from "node:url";
 import { readdir as V, readFile as D } from "node:fs/promises";
-import { resolve as m, matchesGlob as E, dirname as Q } from "node:path";
+import { resolve as A, matchesGlob as E, dirname as Q } from "node:path";
 import { AsyncLocalStorage as N } from "node:async_hooks";
 import { readFileSync as _ } from "node:fs";
 const q = /\.eval\.(js|ts)$/, tt = /* @__PURE__ */ new Set([".git", "node_modules"]);
@@ -11,7 +11,7 @@ async function Z(t, e = []) {
   const n = await V(t, { withFileTypes: !0 }), o = [];
   return await Promise.all(
     n.map(async (i) => {
-      const a = m(t, i.name);
+      const a = A(t, i.name);
       if (i.isDirectory()) {
         if (tt.has(i.name) || j(a, e))
           return;
@@ -29,7 +29,7 @@ const x = new N(), et = {
   modelStack: []
 };
 let F = 0, G = 0;
-const h = [], A = [];
+const h = [], m = [];
 let v = 0;
 function s() {
   return x.getStore() ?? et;
@@ -48,11 +48,11 @@ function nt() {
 function ot() {
   return G += 1, `i${G}`;
 }
-function Mt(t, e) {
+function Ot(t, e) {
   const n = e ?? U(s());
   return x.run(n, t);
 }
-function Ot() {
+function Mt() {
   return U(s());
 }
 function Bt(t) {
@@ -113,16 +113,16 @@ function lt() {
   v = 0;
 }
 function qt(t) {
-  A.push(t);
+  m.push(t);
 }
 function ut() {
-  return [...A];
+  return [...m];
 }
 function te() {
-  return A.length;
+  return m.length;
 }
 function gt() {
-  A.length = 0;
+  m.length = 0;
 }
 async function dt() {
   const t = [];
@@ -146,10 +146,10 @@ function ht(t) {
   return `${Ct}${t}${S}`;
 }
 let w = "";
-function mt() {
+function At() {
   w = "";
 }
-function At({
+function mt({
   suitePath: t,
   casePath: e,
   didPass: n,
@@ -168,7 +168,7 @@ function At({
 }
 function ee(t, e, n = "(root)") {
   const o = st();
-  At({
+  mt({
     suitePath: it(),
     casePath: o.length > 0 ? o : n,
     didPass: t,
@@ -177,7 +177,7 @@ function ee(t, e, n = "(root)") {
     tokenUsage: rt()
   });
 }
-const St = new N(), b = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC43IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIsCiAgICAidGVzdDpidWlsZDpjb2RleCI6ICJub2RlIC4vZGlzdC9rYXR0LmpzIC0tY29uZmlnLWZpbGU9Li9rYXR0LWNvZGV4Lmpzb24iCiAgfSwKICAidHlwZXMiOiAiZGlzdC9pbmRleC5kLnRzIiwKICAiZGV2RGVwZW5kZW5jaWVzIjogewogICAgIkBiaW9tZWpzL2Jpb21lIjogIjIuNC4yIiwKICAgICJAdHlwZXMvbm9kZSI6ICIyNS4yLjMiLAogICAgInRzeCI6ICI0LjIxLjAiLAogICAgInR5cGVzY3JpcHQiOiAiNS45LjMiLAogICAgInZpdGUiOiAiNy4zLjEiLAogICAgInZpdGUtcGx1Z2luLWR0cyI6ICI0LjUuNCIsCiAgICAidml0ZXN0IjogIjQuMC4xOCIsCiAgICAidnNjb2RlLWpzb25ycGMiOiAiXjguMi4xIgogIH0sCiAgImRlcGVuZGVuY2llcyI6IHsKICAgICJAZ2l0aHViL2NvcGlsb3Qtc2RrIjogIjAuMS4yMyIKICB9LAogICJidWdzIjogewogICAgInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0L2lzc3VlcyIKICB9LAogICJob21lcGFnZSI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0Igp9Cg==", import.meta.url);
+const St = new N(), b = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC45IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIsCiAgICAidGVzdDpidWlsZDpjb2RleCI6ICJub2RlIC4vZGlzdC9rYXR0LmpzIC0tY29uZmlnLWZpbGU9Li9rYXR0LWNvZGV4Lmpzb24iCiAgfSwKICAidHlwZXMiOiAiZGlzdC9pbmRleC5kLnRzIiwKICAiZGV2RGVwZW5kZW5jaWVzIjogewogICAgIkBiaW9tZWpzL2Jpb21lIjogIjIuNC4yIiwKICAgICJAdHlwZXMvbm9kZSI6ICIyNS4zLjAiLAogICAgInRzeCI6ICI0LjIxLjAiLAogICAgInR5cGVzY3JpcHQiOiAiNS45LjMiLAogICAgInZpdGUiOiAiNy4zLjEiLAogICAgInZpdGUtcGx1Z2luLWR0cyI6ICI0LjUuNCIsCiAgICAidml0ZXN0IjogIjQuMC4xOCIsCiAgICAidnNjb2RlLWpzb25ycGMiOiAiXjguMi4xIgogIH0sCiAgImRlcGVuZGVuY2llcyI6IHsKICAgICJAZ2l0aHViL2NvcGlsb3Qtc2RrIjogIjAuMS4yMyIKICB9LAogICJidWdzIjogewogICAgInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0L2lzc3VlcyIKICB9LAogICJob21lcGFnZSI6ICJodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvci9rYXR0Igp9Cg==", import.meta.url);
 let f;
 function bt() {
   if (f !== void 0)
@@ -223,10 +223,10 @@ const vt = "gh-copilot", R = "katt.json";
 let I;
 function P() {
   return typeof I == "string" && I.length > 0 ? {
-    resolvedPath: m(process.cwd(), I),
+    resolvedPath: A(process.cwd(), I),
     label: I
   } : {
-    resolvedPath: m(process.cwd(), R),
+    resolvedPath: A(process.cwd(), R),
     label: R
   };
 }
@@ -262,12 +262,12 @@ async function H() {
     return;
   }
 }
-function M(t) {
+function O(t) {
   if (t === "gh-copilot" || t === "codex")
     return t;
 }
 function jt(t, e) {
-  if (!t || M(t?.agent) !== e)
+  if (!t || O(t?.agent) !== e)
     return;
   const n = t.agentOptions;
   if (typeof n != "object" || n === null || Array.isArray(n))
@@ -288,10 +288,10 @@ function Jt(t) {
   if (!t || !Array.isArray(t.ignorePatterns))
     return [];
   const e = Q(P().resolvedPath);
-  return t.ignorePatterns.filter((n) => typeof n == "string" && n.length > 0).map((n) => m(e, n));
+  return t.ignorePatterns.filter((n) => typeof n == "string" && n.length > 0).map((n) => A(e, n));
 }
 async function oe() {
-  const t = await H(), e = M(t?.agent) ?? vt;
+  const t = await H(), e = O(t?.agent) ?? vt;
   return {
     agent: e,
     agentOptions: jt(t, e),
@@ -351,7 +351,7 @@ async function ie() {
   const o = t.includes("--update-snapshots") || t.includes("-u");
   wt(o), k();
   const i = /* @__PURE__ */ new Date();
-  mt(), gt(), lt();
+  At(), gt(), lt();
   const a = await Rt(), c = await Z(process.cwd(), a);
   if (c.length === 0)
     return console.log("No .eval.js or .eval.ts files found."), 1;
@@ -387,12 +387,12 @@ async function ie() {
     }
     return 1;
   }
-  const O = ct(), B = Date.now() - i.getTime();
+  const M = ct(), B = Date.now() - i.getTime();
   return console.log(
     [
       "---",
       `${l("Files")}  ${c.length} passed`,
-      `${l("Evals")}  ${O} passed`,
+      `${l("Evals")}  ${M} passed`,
       `${l("Start at")}  ${Tt(i)}`,
       `${l("Duration")}  ${B}ms`
     ].join(`
@@ -402,7 +402,7 @@ async function ie() {
 export {
   _t as a,
   Qt as b,
-  Ot as c,
+  Mt as c,
   Yt as d,
   Bt as e,
   zt as f,
@@ -417,7 +417,7 @@ export {
   ne as o,
   Xt as p,
   Kt as q,
-  Mt as r,
+  Ot as r,
   Et as s,
   Vt as t,
   ie as u

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "katt",
-  "version": "0.0.8",
+  "version": "0.0.9",
   "description": "CLI tool that tests the output of agentic AI tools",
   "keywords": [
     "cli",
@@ -35,7 +35,7 @@
   "types": "dist/index.d.ts",
   "devDependencies": {
     "@biomejs/biome": "2.4.2",
-    "@types/node": "25.2.3",
+    "@types/node": "25.3.0",
     "tsx": "4.21.0",
     "typescript": "5.9.3",
     "vite": "7.3.1",