katt 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nvmrc +1 -0
- package/CODE_OF_CONDUCT.MD +92 -0
- package/README.md +31 -13
- package/SECURITY.md +10 -0
- package/build-tests/__snapshots__/check1.snap.md +1 -0
- package/build-tests/__snapshots__/check1__Hello_World__should_return_the_date_in_a_json_format.snap.md +1 -0
- package/build-tests/__snapshots__/check1__root.snap.md +1 -0
- package/build-tests/check1.eval.js +19 -0
- package/build-tests/check2.eval.js +15 -0
- package/build-tests/customPrompt.md +1 -0
- package/dist/index.js +225 -167
- package/dist/katt.js +1 -1
- package/dist/runCli-j5xhVCdB.js +424 -0
- package/katt-codex.json +4 -0
- package/package.json +8 -7
- package/renovate.json +6 -0
- package/dist/runCli-C7uxWavX.js +0 -312
package/dist/runCli-C7uxWavX.js
DELETED
|
@@ -1,312 +0,0 @@
|
|
|
1
|
-
import { fileURLToPath as T, pathToFileURL as X } from "node:url";
|
|
2
|
-
import { readdir as M } from "node:fs/promises";
|
|
3
|
-
import { resolve as R } from "node:path";
|
|
4
|
-
import { AsyncLocalStorage as v } from "node:async_hooks";
|
|
5
|
-
import { readFileSync as U } from "node:fs";
|
|
6
|
-
const W = /\.eval\.(js|ts)$/, G = /* @__PURE__ */ new Set([".git", "node_modules"]);
|
|
7
|
-
async function J(t) {
|
|
8
|
-
const e = await M(t, { withFileTypes: !0 }), n = [];
|
|
9
|
-
return await Promise.all(
|
|
10
|
-
e.map(async (o) => {
|
|
11
|
-
const a = R(t, o.name);
|
|
12
|
-
if (o.isDirectory()) {
|
|
13
|
-
if (G.has(o.name))
|
|
14
|
-
return;
|
|
15
|
-
n.push(...await J(a));
|
|
16
|
-
return;
|
|
17
|
-
}
|
|
18
|
-
o.isFile() && W.test(o.name) && n.push(a);
|
|
19
|
-
})
|
|
20
|
-
), n;
|
|
21
|
-
}
|
|
22
|
-
const B = new v(), O = {
|
|
23
|
-
describeStack: [],
|
|
24
|
-
itStack: [],
|
|
25
|
-
tokenUsageStack: [],
|
|
26
|
-
modelStack: []
|
|
27
|
-
};
|
|
28
|
-
let L = 0, $ = 0;
|
|
29
|
-
const f = [], h = [];
|
|
30
|
-
let k = 0;
|
|
31
|
-
function i() {
|
|
32
|
-
return B.getStore() ?? O;
|
|
33
|
-
}
|
|
34
|
-
function Z(t) {
|
|
35
|
-
return {
|
|
36
|
-
describeStack: [...t.describeStack],
|
|
37
|
-
itStack: [...t.itStack],
|
|
38
|
-
tokenUsageStack: [...t.tokenUsageStack],
|
|
39
|
-
modelStack: [...t.modelStack]
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
function Y() {
|
|
43
|
-
return L += 1, `d${L}`;
|
|
44
|
-
}
|
|
45
|
-
function Q() {
|
|
46
|
-
return $ += 1, `i${$}`;
|
|
47
|
-
}
|
|
48
|
-
function St(t, e) {
|
|
49
|
-
const n = e ?? Z(i());
|
|
50
|
-
return B.run(n, t);
|
|
51
|
-
}
|
|
52
|
-
function At() {
|
|
53
|
-
return Z(i());
|
|
54
|
-
}
|
|
55
|
-
function kt(t) {
|
|
56
|
-
i().describeStack.push({ id: Y(), description: t });
|
|
57
|
-
}
|
|
58
|
-
function bt() {
|
|
59
|
-
i().describeStack.pop();
|
|
60
|
-
}
|
|
61
|
-
function V() {
|
|
62
|
-
return i().describeStack.map((t) => t.description).join(" > ");
|
|
63
|
-
}
|
|
64
|
-
function Lt() {
|
|
65
|
-
return [...i().describeStack];
|
|
66
|
-
}
|
|
67
|
-
function $t(t) {
|
|
68
|
-
i().itStack.push({ id: Q(), description: t }), i().tokenUsageStack.push(0), i().modelStack.push(void 0);
|
|
69
|
-
}
|
|
70
|
-
function wt() {
|
|
71
|
-
i().itStack.pop(), i().tokenUsageStack.pop(), i().modelStack.pop();
|
|
72
|
-
}
|
|
73
|
-
function K() {
|
|
74
|
-
return i().itStack.map((t) => t.description).join(" > ");
|
|
75
|
-
}
|
|
76
|
-
function jt() {
|
|
77
|
-
return [...i().itStack];
|
|
78
|
-
}
|
|
79
|
-
function vt(t) {
|
|
80
|
-
if (!Number.isFinite(t) || t <= 0)
|
|
81
|
-
return;
|
|
82
|
-
const e = i(), n = e.tokenUsageStack.length - 1;
|
|
83
|
-
n < 0 || (e.tokenUsageStack[n] += t);
|
|
84
|
-
}
|
|
85
|
-
function P() {
|
|
86
|
-
const t = i(), e = t.tokenUsageStack.length - 1;
|
|
87
|
-
return e < 0 ? 0 : t.tokenUsageStack[e] ?? 0;
|
|
88
|
-
}
|
|
89
|
-
function Jt(t) {
|
|
90
|
-
if (t.length === 0)
|
|
91
|
-
return;
|
|
92
|
-
const e = i(), n = e.modelStack.length - 1;
|
|
93
|
-
n < 0 || (e.modelStack[n] = t);
|
|
94
|
-
}
|
|
95
|
-
function z() {
|
|
96
|
-
const t = i(), e = t.modelStack.length - 1;
|
|
97
|
-
if (!(e < 0))
|
|
98
|
-
return t.modelStack[e];
|
|
99
|
-
}
|
|
100
|
-
function Bt(t) {
|
|
101
|
-
f.push(t);
|
|
102
|
-
}
|
|
103
|
-
function Zt() {
|
|
104
|
-
k += 1;
|
|
105
|
-
}
|
|
106
|
-
function E() {
|
|
107
|
-
return k;
|
|
108
|
-
}
|
|
109
|
-
function D() {
|
|
110
|
-
k = 0;
|
|
111
|
-
}
|
|
112
|
-
function xt(t) {
|
|
113
|
-
h.push(t);
|
|
114
|
-
}
|
|
115
|
-
function _() {
|
|
116
|
-
return [...h];
|
|
117
|
-
}
|
|
118
|
-
function Ht() {
|
|
119
|
-
return h.length;
|
|
120
|
-
}
|
|
121
|
-
function q() {
|
|
122
|
-
h.length = 0;
|
|
123
|
-
}
|
|
124
|
-
async function tt() {
|
|
125
|
-
const t = [];
|
|
126
|
-
for (; f.length > 0; ) {
|
|
127
|
-
const e = f.splice(0, f.length), n = await Promise.allSettled(e);
|
|
128
|
-
t.push(...n);
|
|
129
|
-
}
|
|
130
|
-
return t;
|
|
131
|
-
}
|
|
132
|
-
const et = "\x1B[1;36m", nt = "\x1B[33m", ot = "\x1B[38;5;208m", it = "\x1B[1;38;5;208m", m = "\x1B[0m";
|
|
133
|
-
function r(t) {
|
|
134
|
-
return `${et}${t}${m}`;
|
|
135
|
-
}
|
|
136
|
-
function p(t) {
|
|
137
|
-
return `${nt}${t}${m}`;
|
|
138
|
-
}
|
|
139
|
-
function w(t) {
|
|
140
|
-
return `${ot}${t}${m}`;
|
|
141
|
-
}
|
|
142
|
-
function st(t) {
|
|
143
|
-
return `${it}${t}${m}`;
|
|
144
|
-
}
|
|
145
|
-
let A = "";
|
|
146
|
-
function at() {
|
|
147
|
-
A = "";
|
|
148
|
-
}
|
|
149
|
-
function ct({
|
|
150
|
-
suitePath: t,
|
|
151
|
-
casePath: e,
|
|
152
|
-
didPass: n,
|
|
153
|
-
durationMs: o,
|
|
154
|
-
model: a,
|
|
155
|
-
tokenUsage: I
|
|
156
|
-
}) {
|
|
157
|
-
const c = t.length > 0 ? t : "(root)", C = e.length > 0 ? e : "(root)";
|
|
158
|
-
A !== c && (console.log(`Suite "${r(c)}"`), A = c);
|
|
159
|
-
const g = n ? "✅ Passed in" : "❌ Failed in", l = [
|
|
160
|
-
`Test "${r(C)}"`,
|
|
161
|
-
`- ${g} ${r(`${o}ms`)}`
|
|
162
|
-
];
|
|
163
|
-
a && l.push(`- Model ${r(a)}`), (I ?? 0) > 0 && l.push(`- Tokens used ${r(String(I))}`), l.push("---"), console.log(l.join(`
|
|
164
|
-
`));
|
|
165
|
-
}
|
|
166
|
-
function yt(t, e, n = "(root)") {
|
|
167
|
-
const o = K();
|
|
168
|
-
ct({
|
|
169
|
-
suitePath: V(),
|
|
170
|
-
casePath: o.length > 0 ? o : n,
|
|
171
|
-
didPass: t,
|
|
172
|
-
durationMs: e,
|
|
173
|
-
model: z(),
|
|
174
|
-
tokenUsage: P()
|
|
175
|
-
});
|
|
176
|
-
}
|
|
177
|
-
const rt = new v(), S = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC42IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIKICB9LAogICJ0eXBlcyI6ICJkaXN0L2luZGV4LmQudHMiLAogICJkZXZEZXBlbmRlbmNpZXMiOiB7CiAgICAiQGJpb21lanMvYmlvbWUiOiAiMS45LjQiLAogICAgIkB0eXBlcy9ub2RlIjogIjI1LjIuMCIsCiAgICAidHN4IjogIjQuMjEuMCIsCiAgICAidHlwZXNjcmlwdCI6ICI1LjguMiIsCiAgICAidml0ZSI6ICI3LjMuMSIsCiAgICAidml0ZS1wbHVnaW4tZHRzIjogIjQuNS40IiwKICAgICJ2aXRlc3QiOiAiMy4yLjQiLAogICAgInZzY29kZS1qc29ucnBjIjogIl44LjIuMSIKICB9LAogICJkZXBlbmRlbmNpZXMiOiB7CiAgICAiQGdpdGh1Yi9jb3BpbG90LXNkayI6ICJeMC4xLjIxIgogIH0sCiAgImJ1Z3MiOiB7CiAgICAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQvaXNzdWVzIgogIH0sCiAgImhvbWVwYWdlIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQiCn0K", import.meta.url);
|
|
178
|
-
let d;
|
|
179
|
-
function lt() {
|
|
180
|
-
if (d !== void 0)
|
|
181
|
-
return d;
|
|
182
|
-
try {
|
|
183
|
-
const t = S.protocol === "data:" ? ut(S) : U(T(S), "utf8"), e = JSON.parse(t);
|
|
184
|
-
d = typeof e.version == "string" ? e.version : "unknown";
|
|
185
|
-
} catch {
|
|
186
|
-
d = "unknown";
|
|
187
|
-
}
|
|
188
|
-
return d;
|
|
189
|
-
}
|
|
190
|
-
function ut(t) {
|
|
191
|
-
const e = t.pathname.indexOf(",");
|
|
192
|
-
if (e < 0)
|
|
193
|
-
throw new Error("Invalid data URL.");
|
|
194
|
-
const n = t.pathname.slice(0, e), o = t.pathname.slice(e + 1);
|
|
195
|
-
return n.includes(";base64") ? Buffer.from(o, "base64").toString("utf8") : decodeURIComponent(o);
|
|
196
|
-
}
|
|
197
|
-
function j() {
|
|
198
|
-
const t = " ██╗ ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║ ██║ ██║", o = " ██╔═██╗ ██╔══██║ ██║ ██║", a = " ██║ ██╗██║ ██║ ██║ ██║", I = " ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝", c = `v${lt()}`, C = Math.max(
|
|
199
|
-
0,
|
|
200
|
-
Math.floor((t.length - c.length) / 2)
|
|
201
|
-
), g = `${" ".repeat(C)}${c}`;
|
|
202
|
-
console.log(`
|
|
203
|
-
${p(t)}
|
|
204
|
-
${p(e)}
|
|
205
|
-
${p(n)}
|
|
206
|
-
${w(o)}
|
|
207
|
-
${w(a)}
|
|
208
|
-
${st(I)}
|
|
209
|
-
${p(g)}
|
|
210
|
-
`);
|
|
211
|
-
}
|
|
212
|
-
let x = !1;
|
|
213
|
-
function gt(t) {
|
|
214
|
-
x = t;
|
|
215
|
-
}
|
|
216
|
-
function Ft() {
|
|
217
|
-
return x;
|
|
218
|
-
}
|
|
219
|
-
function It(t) {
|
|
220
|
-
const e = String(t.getHours()).padStart(2, "0"), n = String(t.getMinutes()).padStart(2, "0"), o = String(t.getSeconds()).padStart(2, "0");
|
|
221
|
-
return `${e}:${n}:${o}`;
|
|
222
|
-
}
|
|
223
|
-
function dt() {
|
|
224
|
-
console.log(
|
|
225
|
-
[
|
|
226
|
-
"Usage:",
|
|
227
|
-
" katt [options]",
|
|
228
|
-
"",
|
|
229
|
-
"Options:",
|
|
230
|
-
" -h, --help Show CLI usage information",
|
|
231
|
-
" -u, --update-snapshots Update snapshot files on mismatch"
|
|
232
|
-
].join(`
|
|
233
|
-
`)
|
|
234
|
-
);
|
|
235
|
-
}
|
|
236
|
-
async function Nt() {
|
|
237
|
-
const t = process.argv.slice(2);
|
|
238
|
-
if (t.includes("--help") || t.includes("-h"))
|
|
239
|
-
return j(), dt(), 0;
|
|
240
|
-
const n = t.includes("--update-snapshots") || t.includes("-u");
|
|
241
|
-
gt(n), j();
|
|
242
|
-
const o = /* @__PURE__ */ new Date();
|
|
243
|
-
at(), q(), D();
|
|
244
|
-
const a = await J(process.cwd());
|
|
245
|
-
if (a.length === 0)
|
|
246
|
-
return console.log("No .eval.js or .eval.ts files found."), 1;
|
|
247
|
-
const c = (await Promise.allSettled(
|
|
248
|
-
a.map(
|
|
249
|
-
(s) => rt.run(
|
|
250
|
-
{ evalFile: s },
|
|
251
|
-
() => import(X(s).href)
|
|
252
|
-
)
|
|
253
|
-
)
|
|
254
|
-
)).map((s, u) => ({ result: s, file: a[u] })).filter(({ result: s }) => s.status === "rejected");
|
|
255
|
-
if (c.length > 0) {
|
|
256
|
-
for (const s of c) {
|
|
257
|
-
const u = s.result.status === "rejected" ? s.result.reason : void 0;
|
|
258
|
-
console.error(`Error executing ${s.file}: ${String(u)}`);
|
|
259
|
-
}
|
|
260
|
-
return 1;
|
|
261
|
-
}
|
|
262
|
-
const g = (await tt()).filter(
|
|
263
|
-
(s) => s.status === "rejected"
|
|
264
|
-
);
|
|
265
|
-
if (g.length > 0) {
|
|
266
|
-
for (const s of g)
|
|
267
|
-
s.status === "rejected" && console.error(`Error executing async test: ${String(s.reason)}`);
|
|
268
|
-
return 1;
|
|
269
|
-
}
|
|
270
|
-
const l = _();
|
|
271
|
-
if (l.length > 0) {
|
|
272
|
-
console.error("❌ Failed tests:");
|
|
273
|
-
for (const [s, u] of l.entries()) {
|
|
274
|
-
const b = [u.describePath, u.itPath].filter((N) => N.length > 0).join(" > "), F = b.length > 0 ? `${b}: ` : "";
|
|
275
|
-
console.error(`${s + 1}. ${F}${u.message}`);
|
|
276
|
-
}
|
|
277
|
-
return 1;
|
|
278
|
-
}
|
|
279
|
-
const H = E(), y = Date.now() - o.getTime();
|
|
280
|
-
return console.log(
|
|
281
|
-
[
|
|
282
|
-
"---",
|
|
283
|
-
`${r("Files")} ${a.length} passed`,
|
|
284
|
-
`${r("Evals")} ${H} passed`,
|
|
285
|
-
`${r("Start at")} ${It(o)}`,
|
|
286
|
-
`${r("Duration")} ${y}ms`
|
|
287
|
-
].join(`
|
|
288
|
-
`)
|
|
289
|
-
), 0;
|
|
290
|
-
}
|
|
291
|
-
export {
|
|
292
|
-
Zt as a,
|
|
293
|
-
Bt as b,
|
|
294
|
-
At as c,
|
|
295
|
-
wt as d,
|
|
296
|
-
kt as e,
|
|
297
|
-
bt as f,
|
|
298
|
-
Ht as g,
|
|
299
|
-
vt as h,
|
|
300
|
-
rt as i,
|
|
301
|
-
xt as j,
|
|
302
|
-
K as k,
|
|
303
|
-
yt as l,
|
|
304
|
-
V as m,
|
|
305
|
-
Ft as n,
|
|
306
|
-
Lt as o,
|
|
307
|
-
$t as p,
|
|
308
|
-
jt as q,
|
|
309
|
-
St as r,
|
|
310
|
-
Jt as s,
|
|
311
|
-
Nt as t
|
|
312
|
-
};
|