katt 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nvmrc +1 -0
- package/README.md +35 -7
- package/build-tests/__snapshots__/check1.snap.md +1 -0
- package/build-tests/__snapshots__/check1__Hello_World__should_return_the_date_in_a_json_format.snap.md +1 -0
- package/build-tests/__snapshots__/check1__root.snap.md +1 -0
- package/build-tests/check1.eval.js +19 -0
- package/build-tests/check2.eval.js +15 -0
- package/build-tests/customPrompt.md +1 -0
- package/dist/index.js +235 -199
- package/dist/katt.js +1 -1
- package/dist/runCli-425rgVp8.js +424 -0
- package/katt-codex.json +4 -0
- package/package.json +8 -7
- package/renovate.json +6 -0
- package/dist/runCli-B3oIBxOl.js +0 -317
package/dist/runCli-B3oIBxOl.js
DELETED
|
@@ -1,317 +0,0 @@
|
|
|
1
|
-
import { fileURLToPath as Z, pathToFileURL as y } from "node:url";
|
|
2
|
-
import { readdir as N } from "node:fs/promises";
|
|
3
|
-
import { resolve as M } from "node:path";
|
|
4
|
-
import { AsyncLocalStorage as v } from "node:async_hooks";
|
|
5
|
-
import { readFileSync as X } from "node:fs";
|
|
6
|
-
const H = /\.eval\.(js|ts)$/, R = /* @__PURE__ */ new Set([".git", "node_modules"]);
|
|
7
|
-
async function w(t) {
|
|
8
|
-
const e = await N(t, { withFileTypes: !0 }), n = [];
|
|
9
|
-
return await Promise.all(
|
|
10
|
-
e.map(async (i) => {
|
|
11
|
-
const s = M(t, i.name);
|
|
12
|
-
if (i.isDirectory()) {
|
|
13
|
-
if (R.has(i.name))
|
|
14
|
-
return;
|
|
15
|
-
n.push(...await w(s));
|
|
16
|
-
return;
|
|
17
|
-
}
|
|
18
|
-
i.isFile() && H.test(i.name) && n.push(s);
|
|
19
|
-
})
|
|
20
|
-
), n;
|
|
21
|
-
}
|
|
22
|
-
const j = new v(), W = {
|
|
23
|
-
describeStack: [],
|
|
24
|
-
itStack: [],
|
|
25
|
-
tokenUsageStack: [],
|
|
26
|
-
modelStack: []
|
|
27
|
-
};
|
|
28
|
-
let b = 0, L = 0;
|
|
29
|
-
const p = [], f = [];
|
|
30
|
-
let k = 0;
|
|
31
|
-
function o() {
|
|
32
|
-
return j.getStore() ?? W;
|
|
33
|
-
}
|
|
34
|
-
function J(t) {
|
|
35
|
-
return {
|
|
36
|
-
describeStack: [...t.describeStack],
|
|
37
|
-
itStack: [...t.itStack],
|
|
38
|
-
tokenUsageStack: [...t.tokenUsageStack],
|
|
39
|
-
modelStack: [...t.modelStack]
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
function U() {
|
|
43
|
-
return b += 1, `d${b}`;
|
|
44
|
-
}
|
|
45
|
-
function G() {
|
|
46
|
-
return L += 1, `i${L}`;
|
|
47
|
-
}
|
|
48
|
-
function B(t) {
|
|
49
|
-
return t.tokenUsageStack.length - 1;
|
|
50
|
-
}
|
|
51
|
-
function F(t) {
|
|
52
|
-
return t.modelStack.length - 1;
|
|
53
|
-
}
|
|
54
|
-
function bt(t, e) {
|
|
55
|
-
const n = e ?? J(o());
|
|
56
|
-
return j.run(n, t);
|
|
57
|
-
}
|
|
58
|
-
function Lt() {
|
|
59
|
-
return J(o());
|
|
60
|
-
}
|
|
61
|
-
function $t(t) {
|
|
62
|
-
o().describeStack.push({ id: U(), description: t });
|
|
63
|
-
}
|
|
64
|
-
function vt() {
|
|
65
|
-
o().describeStack.pop();
|
|
66
|
-
}
|
|
67
|
-
function O() {
|
|
68
|
-
return o().describeStack.map((t) => t.description).join(" > ");
|
|
69
|
-
}
|
|
70
|
-
function wt() {
|
|
71
|
-
return [...o().describeStack];
|
|
72
|
-
}
|
|
73
|
-
function jt(t) {
|
|
74
|
-
const e = o();
|
|
75
|
-
e.itStack.push({ id: G(), description: t }), e.tokenUsageStack.push(0), e.modelStack.push(void 0);
|
|
76
|
-
}
|
|
77
|
-
function Jt() {
|
|
78
|
-
const t = o();
|
|
79
|
-
t.itStack.pop(), t.tokenUsageStack.pop(), t.modelStack.pop();
|
|
80
|
-
}
|
|
81
|
-
function Y() {
|
|
82
|
-
return o().itStack.map((t) => t.description).join(" > ");
|
|
83
|
-
}
|
|
84
|
-
function Bt() {
|
|
85
|
-
return [...o().itStack];
|
|
86
|
-
}
|
|
87
|
-
function Ft(t) {
|
|
88
|
-
if (!Number.isFinite(t) || t <= 0)
|
|
89
|
-
return;
|
|
90
|
-
const e = o(), n = B(e);
|
|
91
|
-
n < 0 || (e.tokenUsageStack[n] += t);
|
|
92
|
-
}
|
|
93
|
-
function Q() {
|
|
94
|
-
const t = o(), e = B(t);
|
|
95
|
-
return e < 0 ? 0 : t.tokenUsageStack[e] ?? 0;
|
|
96
|
-
}
|
|
97
|
-
function xt(t) {
|
|
98
|
-
if (t.length === 0)
|
|
99
|
-
return;
|
|
100
|
-
const e = o(), n = F(e);
|
|
101
|
-
n < 0 || (e.modelStack[n] = t);
|
|
102
|
-
}
|
|
103
|
-
function V() {
|
|
104
|
-
const t = o(), e = F(t);
|
|
105
|
-
if (!(e < 0))
|
|
106
|
-
return t.modelStack[e];
|
|
107
|
-
}
|
|
108
|
-
function Tt(t) {
|
|
109
|
-
p.push(t);
|
|
110
|
-
}
|
|
111
|
-
function Zt() {
|
|
112
|
-
k += 1;
|
|
113
|
-
}
|
|
114
|
-
function K() {
|
|
115
|
-
return k;
|
|
116
|
-
}
|
|
117
|
-
function P() {
|
|
118
|
-
k = 0;
|
|
119
|
-
}
|
|
120
|
-
function yt(t) {
|
|
121
|
-
f.push(t);
|
|
122
|
-
}
|
|
123
|
-
function E() {
|
|
124
|
-
return [...f];
|
|
125
|
-
}
|
|
126
|
-
function Nt() {
|
|
127
|
-
return f.length;
|
|
128
|
-
}
|
|
129
|
-
function z() {
|
|
130
|
-
f.length = 0;
|
|
131
|
-
}
|
|
132
|
-
async function D() {
|
|
133
|
-
const t = [];
|
|
134
|
-
for (; p.length > 0; ) {
|
|
135
|
-
const e = p.splice(0, p.length), n = await Promise.allSettled(e);
|
|
136
|
-
t.push(...n);
|
|
137
|
-
}
|
|
138
|
-
return t;
|
|
139
|
-
}
|
|
140
|
-
const _ = "\x1B[1;36m", q = "\x1B[33m", tt = "\x1B[38;5;208m", et = "\x1B[1;38;5;208m", m = "\x1B[0m";
|
|
141
|
-
function r(t) {
|
|
142
|
-
return `${_}${t}${m}`;
|
|
143
|
-
}
|
|
144
|
-
function C(t) {
|
|
145
|
-
return `${q}${t}${m}`;
|
|
146
|
-
}
|
|
147
|
-
function $(t) {
|
|
148
|
-
return `${tt}${t}${m}`;
|
|
149
|
-
}
|
|
150
|
-
function nt(t) {
|
|
151
|
-
return `${et}${t}${m}`;
|
|
152
|
-
}
|
|
153
|
-
let A = "";
|
|
154
|
-
function it() {
|
|
155
|
-
A = "";
|
|
156
|
-
}
|
|
157
|
-
function ot({
|
|
158
|
-
suitePath: t,
|
|
159
|
-
casePath: e,
|
|
160
|
-
didPass: n,
|
|
161
|
-
durationMs: i,
|
|
162
|
-
model: s,
|
|
163
|
-
tokenUsage: c
|
|
164
|
-
}) {
|
|
165
|
-
const l = t.length > 0 ? t : "(root)", u = e.length > 0 ? e : "(root)";
|
|
166
|
-
A !== l && (console.log(`Suite "${r(l)}"`), A = l);
|
|
167
|
-
const g = n ? "✅ Passed in" : "❌ Failed in", I = [
|
|
168
|
-
`Test "${r(u)}"`,
|
|
169
|
-
`- ${g} ${r(`${i}ms`)}`
|
|
170
|
-
];
|
|
171
|
-
s && I.push(`- Model ${r(s)}`), (c ?? 0) > 0 && I.push(`- Tokens used ${r(String(c))}`), I.push("---"), console.log(I.join(`
|
|
172
|
-
`));
|
|
173
|
-
}
|
|
174
|
-
function Mt(t, e, n = "(root)") {
|
|
175
|
-
const i = Y();
|
|
176
|
-
ot({
|
|
177
|
-
suitePath: O(),
|
|
178
|
-
casePath: i.length > 0 ? i : n,
|
|
179
|
-
didPass: t,
|
|
180
|
-
durationMs: e,
|
|
181
|
-
model: V(),
|
|
182
|
-
tokenUsage: Q()
|
|
183
|
-
});
|
|
184
|
-
}
|
|
185
|
-
const st = new v(), h = new URL("data:application/json;base64,ewogICJuYW1lIjogImthdHQiLAogICJ2ZXJzaW9uIjogIjAuMC41IiwKICAiZGVzY3JpcHRpb24iOiAiQ0xJIHRvb2wgdGhhdCB0ZXN0cyB0aGUgb3V0cHV0IG9mIGFnZW50aWMgQUkgdG9vbHMiLAogICJrZXl3b3JkcyI6IFsKICAgICJjbGkiLAogICAgImFpIiwKICAgICJhZ2VudGljLWFpIiwKICAgICJ0ZXN0aW5nIiwKICAgICJldmFsdWF0aW9uIgogIF0sCiAgImF1dGhvciI6ICJSYXBoYWVsIFBvcnRvIChodHRwczovL2dpdGh1Yi5jb20vcmFwaGFlbHBvcikiLAogICJsaWNlbnNlIjogIk1JVCIsCiAgInR5cGUiOiAibW9kdWxlIiwKICAibWFpbiI6ICJkaXN0L2luZGV4LmpzIiwKICAiZXhwb3J0cyI6IHsKICAgICIuIjogewogICAgICAidHlwZXMiOiAiLi9kaXN0L2luZGV4LmQudHMiLAogICAgICAiaW1wb3J0IjogIi4vZGlzdC9pbmRleC5qcyIKICAgIH0KICB9LAogICJiaW4iOiB7CiAgICAia2F0dCI6ICJkaXN0L2thdHQuanMiCiAgfSwKICAic2NyaXB0cyI6IHsKICAgICJidWlsZCI6ICJ2aXRlIGJ1aWxkIiwKICAgICJkZXYiOiAidHN4IHNyYy9pbmRleC50cyIsCiAgICAibGludCI6ICJiaW9tZSBsaW50IC4vc3JjIiwKICAgICJmb3JtYXQiOiAiYmlvbWUgZm9ybWF0IC0td3JpdGUgLi9zcmMiLAogICAgInRlc3QiOiAidml0ZXN0IiwKICAgICJ0eXBlY2hlY2siOiAidHNjIC1wIHRzY29uZmlnLmpzb24gLS1ub0VtaXQiLAogICAgInRlc3Q6YnVpbGQiOiAibm9kZSAuL2Rpc3Qva2F0dC5qcyIKICB9LAogICJ0eXBlcyI6ICJkaXN0L2luZGV4LmQudHMiLAogICJkZXZEZXBlbmRlbmNpZXMiOiB7CiAgICAiQGJpb21lanMvYmlvbWUiOiAiMS45LjQiLAogICAgIkB0eXBlcy9ub2RlIjogIjI1LjIuMCIsCiAgICAidHN4IjogIjQuMjEuMCIsCiAgICAidHlwZXNjcmlwdCI6ICI1LjguMiIsCiAgICAidml0ZSI6ICI3LjMuMSIsCiAgICAidml0ZS1wbHVnaW4tZHRzIjogIjQuNS40IiwKICAgICJ2aXRlc3QiOiAiMy4yLjQiLAogICAgInZzY29kZS1qc29ucnBjIjogIl44LjIuMSIKICB9LAogICJkZXBlbmRlbmNpZXMiOiB7CiAgICAiQGdpdGh1Yi9jb3BpbG90LXNkayI6ICJeMC4xLjIxIgogIH0sCiAgImJ1Z3MiOiB7CiAgICAidXJsIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQvaXNzdWVzIgogIH0sCiAgImhvbWVwYWdlIjogImh0dHBzOi8vZ2l0aHViLmNvbS9yYXBoYWVscG9yL2thdHQiCn0K", import.meta.url);
|
|
186
|
-
let d;
|
|
187
|
-
function at() {
|
|
188
|
-
if (d !== void 0)
|
|
189
|
-
return d;
|
|
190
|
-
try {
|
|
191
|
-
const t = h.protocol === "data:" ? rt(h) : X(Z(h), "utf8"), e = JSON.parse(t);
|
|
192
|
-
d = typeof e.version == "string" ? e.version : "unknown";
|
|
193
|
-
} catch {
|
|
194
|
-
d = "unknown";
|
|
195
|
-
}
|
|
196
|
-
return d;
|
|
197
|
-
}
|
|
198
|
-
function rt(t) {
|
|
199
|
-
const e = t.pathname.indexOf(",");
|
|
200
|
-
if (e < 0)
|
|
201
|
-
throw new Error("Invalid data URL.");
|
|
202
|
-
const n = t.pathname.slice(0, e), i = t.pathname.slice(e + 1);
|
|
203
|
-
return n.includes(";base64") ? Buffer.from(i, "base64").toString("utf8") : decodeURIComponent(i);
|
|
204
|
-
}
|
|
205
|
-
function ct() {
|
|
206
|
-
const t = " ██╗ ██╗ █████╗ ████████╗████████╗", e = " ██║ ██╔╝██╔══██╗╚══██╔══╝╚══██╔══╝", n = " █████╔╝ ███████║ ██║ ██║", i = " ██╔═██╗ ██╔══██║ ██║ ██║", s = " ██║ ██╗██║ ██║ ██║ ██║", c = " ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝", l = `v${at()}`, u = Math.max(
|
|
207
|
-
0,
|
|
208
|
-
Math.floor((t.length - l.length) / 2)
|
|
209
|
-
), g = `${" ".repeat(u)}${l}`;
|
|
210
|
-
console.log(`
|
|
211
|
-
${C(t)}
|
|
212
|
-
${C(e)}
|
|
213
|
-
${C(n)}
|
|
214
|
-
${$(i)}
|
|
215
|
-
${$(s)}
|
|
216
|
-
${nt(c)}
|
|
217
|
-
${C(g)}
|
|
218
|
-
`);
|
|
219
|
-
}
|
|
220
|
-
let x = !1;
|
|
221
|
-
function lt(t) {
|
|
222
|
-
x = t;
|
|
223
|
-
}
|
|
224
|
-
function Xt() {
|
|
225
|
-
return x;
|
|
226
|
-
}
|
|
227
|
-
function ut(t) {
|
|
228
|
-
const e = String(t.getHours()).padStart(2, "0"), n = String(t.getMinutes()).padStart(2, "0"), i = String(t.getSeconds()).padStart(2, "0");
|
|
229
|
-
return `${e}:${n}:${i}`;
|
|
230
|
-
}
|
|
231
|
-
function gt(t) {
|
|
232
|
-
return t.includes("--update-snapshots") || t.includes("-u");
|
|
233
|
-
}
|
|
234
|
-
function It(t) {
|
|
235
|
-
return [t.describePath, t.itPath].filter((e) => e.length > 0).join(" > ");
|
|
236
|
-
}
|
|
237
|
-
function dt(t) {
|
|
238
|
-
for (const e of t)
|
|
239
|
-
console.error(`Error executing ${e.file}: ${String(e.reason)}`);
|
|
240
|
-
}
|
|
241
|
-
function Ct(t) {
|
|
242
|
-
for (const e of t)
|
|
243
|
-
console.error(`Error executing async test: ${String(e.reason)}`);
|
|
244
|
-
}
|
|
245
|
-
function pt(t) {
|
|
246
|
-
console.error("❌ Failed tests:");
|
|
247
|
-
for (const [e, n] of t.entries()) {
|
|
248
|
-
const i = It(n), s = i.length > 0 ? `${i}: ` : "";
|
|
249
|
-
console.error(`${e + 1}. ${s}${n.message}`);
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
function ft(t, e, n, i) {
|
|
253
|
-
return [
|
|
254
|
-
"---",
|
|
255
|
-
`${r("Files")} ${t} passed`,
|
|
256
|
-
`${r("Evals")} ${e} passed`,
|
|
257
|
-
`${r("Start at")} ${ut(n)}`,
|
|
258
|
-
`${r("Duration")} ${i}ms`
|
|
259
|
-
].join(`
|
|
260
|
-
`);
|
|
261
|
-
}
|
|
262
|
-
async function Ht() {
|
|
263
|
-
const t = process.argv.slice(2), e = gt(t);
|
|
264
|
-
lt(e), ct();
|
|
265
|
-
const n = /* @__PURE__ */ new Date();
|
|
266
|
-
it(), z(), P();
|
|
267
|
-
const i = await w(process.cwd());
|
|
268
|
-
if (i.length === 0)
|
|
269
|
-
return console.log("No .eval.js or .eval.ts files found."), 1;
|
|
270
|
-
const c = (await Promise.allSettled(
|
|
271
|
-
i.map(
|
|
272
|
-
(a) => st.run(
|
|
273
|
-
{ evalFile: a },
|
|
274
|
-
() => import(y(a).href)
|
|
275
|
-
)
|
|
276
|
-
)
|
|
277
|
-
)).map((a, S) => ({ result: a, file: i[S] })).filter(({ result: a }) => a.status === "rejected").map(({ result: a, file: S }) => ({
|
|
278
|
-
file: S,
|
|
279
|
-
reason: a.status === "rejected" ? a.reason : void 0
|
|
280
|
-
}));
|
|
281
|
-
if (c.length > 0)
|
|
282
|
-
return dt(c), 1;
|
|
283
|
-
const u = (await D()).filter(
|
|
284
|
-
(a) => a.status === "rejected"
|
|
285
|
-
);
|
|
286
|
-
if (u.length > 0)
|
|
287
|
-
return Ct(u), 1;
|
|
288
|
-
const g = E();
|
|
289
|
-
if (g.length > 0)
|
|
290
|
-
return pt(g), 1;
|
|
291
|
-
const I = K(), T = Date.now() - n.getTime();
|
|
292
|
-
return console.log(
|
|
293
|
-
ft(i.length, I, n, T)
|
|
294
|
-
), 0;
|
|
295
|
-
}
|
|
296
|
-
export {
|
|
297
|
-
Tt as a,
|
|
298
|
-
Jt as b,
|
|
299
|
-
Lt as c,
|
|
300
|
-
Zt as d,
|
|
301
|
-
$t as e,
|
|
302
|
-
vt as f,
|
|
303
|
-
Nt as g,
|
|
304
|
-
st as h,
|
|
305
|
-
Ft as i,
|
|
306
|
-
yt as j,
|
|
307
|
-
Y as k,
|
|
308
|
-
Mt as l,
|
|
309
|
-
O as m,
|
|
310
|
-
Xt as n,
|
|
311
|
-
wt as o,
|
|
312
|
-
jt as p,
|
|
313
|
-
Bt as q,
|
|
314
|
-
bt as r,
|
|
315
|
-
xt as s,
|
|
316
|
-
Ht as t
|
|
317
|
-
};
|