@siftline/cli 0.0.2 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -8
- package/dist/cli.mjs +18 -5
- package/dist/index.d.mts +33 -20
- package/dist/index.mjs +2 -2
- package/dist/src-Ci51FAD3.mjs +456 -0
- package/package.json +4 -2
- package/dist/src-DYu5d4UT.mjs +0 -51
package/README.md
CHANGED
|
@@ -1,19 +1,36 @@
|
|
|
1
1
|
# @siftline/cli
|
|
2
2
|
|
|
3
|
-
The Siftline command line: measure a
|
|
4
|
-
|
|
5
|
-
This package is a walking skeleton. The `siftline` bin prints its version and a usage
|
|
6
|
-
block; `test` and `label` are named there but not implemented, which is enough to prove
|
|
7
|
-
the bin, shebang, build, type, test, pack and publish path. The commands land next.
|
|
3
|
+
The Siftline command line: measure a Recipe against its Fixtures, then label for real.
|
|
8
4
|
|
|
9
5
|
```sh
|
|
10
|
-
|
|
6
|
+
npm install --save-dev @siftline/cli
|
|
7
|
+
export TYPESAFE_API_KEY=...
|
|
8
|
+
|
|
9
|
+
npx siftline test recipe.json fixtures.jsonl
|
|
10
|
+
npx siftline label recipe.json records.jsonl --rules rules.json > decisions.jsonl
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
```
|
|
14
|
+
support-inbox v1 · jev-1.13.0
|
|
15
|
+
|
|
16
|
+
category 4/5 0.80
|
|
17
|
+
wants_human 5/5 1.00
|
|
18
|
+
|
|
19
|
+
accuracy 0.80 (lowest question)
|
|
20
|
+
unsure 1 of 5 would go to Review
|
|
21
|
+
|
|
22
|
+
fixture:4 category: expected complaint, got question
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
`test` reports one accuracy per Question and fails CI with `--min-accuracy`. `label` writes
|
|
26
|
+
one Decision line per Record, routed through Rules when `--rules` is given. `npx siftline --help`
|
|
27
|
+
prints the rest.
|
|
28
|
+
|
|
29
|
+
The guide, the options and the exit codes live at
|
|
30
|
+
[docs.siftline.dev](https://docs.siftline.dev/docs/packages/cli).
|
|
14
31
|
|
|
15
32
|
ESM only. Node 22.14 or newer.
|
|
16
33
|
|
|
17
34
|
## Licence
|
|
18
35
|
|
|
19
|
-
MIT
|
|
36
|
+
MIT. See [LICENSE](./LICENSE).
|
package/dist/cli.mjs
CHANGED
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import { i as API_KEY_ENV, t as run } from "./src-Ci51FAD3.mjs";
|
|
3
|
+
import { TypeSafeClient } from "@typesafe-ai/sdk";
|
|
3
4
|
//#region src/cli.ts
|
|
4
|
-
const
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
const controller = new AbortController();
|
|
6
|
+
process.on("SIGINT", () => {
|
|
7
|
+
controller.abort();
|
|
8
|
+
});
|
|
9
|
+
let client;
|
|
10
|
+
process.exitCode = await run(process.argv.slice(2), {
|
|
11
|
+
client: { systemOne: (request, options) => {
|
|
12
|
+
client ??= new TypeSafeClient({ apiKey: process.env[API_KEY_ENV] });
|
|
13
|
+
return client.systemOne(request, options);
|
|
14
|
+
} },
|
|
15
|
+
stdin: process.stdin,
|
|
16
|
+
stdout: process.stdout,
|
|
17
|
+
stderr: process.stderr,
|
|
18
|
+
env: process.env,
|
|
19
|
+
signal: controller.signal
|
|
20
|
+
});
|
|
8
21
|
//#endregion
|
|
9
22
|
export {};
|
package/dist/index.d.mts
CHANGED
|
@@ -1,24 +1,37 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
import { SystemOneClient } from "@siftline/core";
|
|
2
|
+
//#region src/deps.d.ts
|
|
3
|
+
/** `process.stdout` and `process.stderr` satisfy this, and so does a string buffer. */
|
|
4
|
+
interface OutputStream {
|
|
5
|
+
write: (chunk: string) => void;
|
|
6
|
+
readonly isTTY?: boolean;
|
|
7
|
+
}
|
|
8
|
+
/** `process.stdin`, or any async iterable of chunks. */
|
|
9
|
+
type InputStream = AsyncIterable<string | Uint8Array>;
|
|
10
|
+
/** The CLI's only seam. The bin wires the process to it; tests wire strings. */
|
|
11
|
+
interface RunDeps {
|
|
12
|
+
client: SystemOneClient;
|
|
13
|
+
stdin: InputStream;
|
|
14
|
+
stdout: OutputStream;
|
|
15
|
+
stderr: OutputStream;
|
|
16
|
+
env: {
|
|
17
|
+
readonly [name: string]: string | undefined;
|
|
18
|
+
};
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
}
|
|
21
|
+
//#endregion
|
|
22
|
+
//#region src/options.d.ts
|
|
23
|
+
export declare const API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
24
|
+
//#endregion
|
|
25
|
+
//#region src/usage.d.ts
|
|
26
|
+
/** The published version of `@siftline/cli`, baked in at build time. */
|
|
7
27
|
export declare const VERSION: string;
|
|
8
|
-
/**
|
|
9
|
-
* The usage block. `test` and `label` are named before they exist so that the bin
|
|
10
|
-
* wiring is proven — and so that `siftline --help` never lies about what it does.
|
|
11
|
-
*/
|
|
12
28
|
export declare const USAGE: string;
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
readonly code: 0 | 1;
|
|
16
|
-
readonly stdout: string;
|
|
17
|
-
readonly stderr: string;
|
|
18
|
-
}
|
|
29
|
+
//#endregion
|
|
30
|
+
//#region src/index.d.ts
|
|
19
31
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
32
|
+
* The library door. Every command runs through it, so a test drives the whole CLI with an
|
|
33
|
+
* injected client and string streams and asserts on the exit code and what was written.
|
|
22
34
|
*/
|
|
23
|
-
export declare function run(argv: readonly string[]):
|
|
24
|
-
//#endregion
|
|
35
|
+
export declare function run(argv: readonly string[], deps: RunDeps): Promise<number>;
|
|
36
|
+
//#endregion
|
|
37
|
+
export type { InputStream, OutputStream, RunDeps };
|
package/dist/index.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { n as
|
|
2
|
-
export { USAGE, VERSION, run };
|
|
1
|
+
import { i as API_KEY_ENV, n as USAGE, r as VERSION, t as run } from "./src-Ci51FAD3.mjs";
|
|
2
|
+
export { API_KEY_ENV, USAGE, VERSION, run };
|
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
import { DEFAULT_MAX_IN_FLIGHT, JudgeError, SiftlineError, VERSION, createJudge, parseFixtures, parseRecipe, recordSchema, routeDecision, ruleSchema, serializeDecision, testRecipe, validateFixtures, validateRules } from "@siftline/core";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { parseArgs } from "node:util";
|
|
5
|
+
//#region src/deps.ts
|
|
6
|
+
/** A bad invocation. Exit 2, with the usage block. */
|
|
7
|
+
var UsageError = class extends Error {};
|
|
8
|
+
/** A file the arguments named that could not be read or parsed. Exit 2, no usage block. */
|
|
9
|
+
var InputError = class extends Error {};
|
|
10
|
+
function writeLine(stream, line) {
|
|
11
|
+
stream.write(`${line}\n`);
|
|
12
|
+
}
|
|
13
|
+
//#endregion
|
|
14
|
+
//#region src/input.ts
|
|
15
|
+
async function readTextFile(path, what) {
|
|
16
|
+
try {
|
|
17
|
+
return await readFile(path, "utf8");
|
|
18
|
+
} catch (cause) {
|
|
19
|
+
throw new InputError(`cannot read the ${what} ${path}: ${reason(cause)}`, { cause });
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
async function readRecipeFile(path) {
|
|
23
|
+
const text = await readTextFile(path, "recipe");
|
|
24
|
+
try {
|
|
25
|
+
return parseRecipe(text);
|
|
26
|
+
} catch (cause) {
|
|
27
|
+
throw new InputError(`${path} is not a valid Recipe: ${reason(cause)}`, { cause });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async function readFixturesFile(path, recipe) {
|
|
31
|
+
return await readChecked(path, "fixtures", "Fixtures", parseFixtures, (fixtures) => validateFixtures(fixtures, recipe).map((problem) => `fixture ${problem.fixture}: ${problem.problem}`));
|
|
32
|
+
}
|
|
33
|
+
async function readRulesFile(path, recipe) {
|
|
34
|
+
return await readChecked(path, "rules", "Rules", (text) => ruleSchema.array().parse(JSON.parse(text)), (rules) => validateRules(rules, recipe).map((problem) => `rule ${problem.rule}: ${problem.problem}`));
|
|
35
|
+
}
|
|
36
|
+
/** Parse the whole file, then refuse it on the first line that no longer fits the Recipe. */
|
|
37
|
+
async function readChecked(path, what, kind, parse, stale) {
|
|
38
|
+
const text = await readTextFile(path, what);
|
|
39
|
+
let items;
|
|
40
|
+
try {
|
|
41
|
+
items = parse(text);
|
|
42
|
+
} catch (cause) {
|
|
43
|
+
throw new InputError(`${path} does not hold valid ${kind}: ${reason(cause)}`, { cause });
|
|
44
|
+
}
|
|
45
|
+
const problems = stale(items);
|
|
46
|
+
const first = problems[0];
|
|
47
|
+
if (first) throw new InputError(`${path} does not fit the Recipe: ${first}${andMore(problems.length)}`);
|
|
48
|
+
return items;
|
|
49
|
+
}
|
|
50
|
+
/** `-` and an absent path both mean stdin. Every line is parsed before the caller judges any. */
|
|
51
|
+
async function readRecords(path, stdin) {
|
|
52
|
+
const fromStdin = path === void 0 || path === "-";
|
|
53
|
+
const source = fromStdin ? "stdin" : path;
|
|
54
|
+
const text = fromStdin ? await readStdin(stdin) : await readTextFile(path, "records");
|
|
55
|
+
const records = [];
|
|
56
|
+
for (const [offset, line] of text.split("\n").entries()) {
|
|
57
|
+
if (line.trim() === "") continue;
|
|
58
|
+
try {
|
|
59
|
+
records.push(recordSchema.parse(JSON.parse(line)));
|
|
60
|
+
} catch (cause) {
|
|
61
|
+
throw new InputError(`${source} line ${offset + 1} is not a valid Record: ${reason(cause)}`, { cause });
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return records;
|
|
65
|
+
}
|
|
66
|
+
async function readStdin(stream) {
|
|
67
|
+
const decoder = new TextDecoder();
|
|
68
|
+
let text = "";
|
|
69
|
+
for await (const chunk of stream) text += chunk instanceof Uint8Array ? decoder.decode(chunk, { stream: true }) : chunk;
|
|
70
|
+
return text + decoder.decode();
|
|
71
|
+
}
|
|
72
|
+
const issueSchema = z.object({
|
|
73
|
+
path: z.array(z.union([
|
|
74
|
+
z.string(),
|
|
75
|
+
z.number(),
|
|
76
|
+
z.symbol()
|
|
77
|
+
])),
|
|
78
|
+
message: z.string()
|
|
79
|
+
});
|
|
80
|
+
const issuesHolder = z.object({ issues: z.array(z.unknown()) });
|
|
81
|
+
/** Zod's issues by shape, not by class, so a second copy of Zod still reads. */
|
|
82
|
+
function issuesOf(cause) {
|
|
83
|
+
const holder = issuesHolder.safeParse(cause);
|
|
84
|
+
if (!holder.success) return void 0;
|
|
85
|
+
const issues = holder.data.issues.flatMap((candidate) => {
|
|
86
|
+
const issue = issueSchema.safeParse(candidate);
|
|
87
|
+
return issue.success ? [issue.data] : [];
|
|
88
|
+
});
|
|
89
|
+
return issues.length === 0 ? void 0 : issues;
|
|
90
|
+
}
|
|
91
|
+
function reason(cause) {
|
|
92
|
+
const issues = issuesOf(cause);
|
|
93
|
+
if (issues) return firstIssue(issues);
|
|
94
|
+
if (cause instanceof Error) {
|
|
95
|
+
const nested = issuesOf(cause.cause);
|
|
96
|
+
return nested ? `${cause.message}: ${firstIssue(nested)}` : cause.message;
|
|
97
|
+
}
|
|
98
|
+
return String(cause);
|
|
99
|
+
}
|
|
100
|
+
/** The first issue with its path, and how many stand behind it. */
|
|
101
|
+
function firstIssue(issues) {
|
|
102
|
+
const first = issues[0];
|
|
103
|
+
if (!first) return "invalid";
|
|
104
|
+
return `${first.path.length === 0 ? "" : `${first.path.join(".")}: `}${first.message}${andMore(issues.length)}`;
|
|
105
|
+
}
|
|
106
|
+
/** The tail every "here is the first of them" sentence in this file ends with. */
|
|
107
|
+
function andMore(count) {
|
|
108
|
+
return count === 1 ? "" : ` (and ${count - 1} more)`;
|
|
109
|
+
}
|
|
110
|
+
//#endregion
|
|
111
|
+
//#region src/options.ts
|
|
112
|
+
const API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
113
|
+
/** Each command hands `parseArgs` only its own flags, so a foreign one is an unknown option. */
|
|
114
|
+
const TEST_FLAGS = {
|
|
115
|
+
"max-in-flight": { type: "string" },
|
|
116
|
+
"min-accuracy": { type: "string" },
|
|
117
|
+
json: { type: "boolean" },
|
|
118
|
+
quiet: { type: "boolean" }
|
|
119
|
+
};
|
|
120
|
+
const LABEL_FLAGS = {
|
|
121
|
+
"max-in-flight": { type: "string" },
|
|
122
|
+
quiet: { type: "boolean" },
|
|
123
|
+
rules: { type: "string" }
|
|
124
|
+
};
|
|
125
|
+
function parseTestArgs(args) {
|
|
126
|
+
const { values, positionals } = parseWith(args, TEST_FLAGS);
|
|
127
|
+
return {
|
|
128
|
+
positionals,
|
|
129
|
+
options: {
|
|
130
|
+
maxInFlight: gateWidth(values["max-in-flight"]),
|
|
131
|
+
minAccuracy: ratio$1(values["min-accuracy"]),
|
|
132
|
+
json: values.json === true,
|
|
133
|
+
quiet: values.quiet === true
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
function parseLabelArgs(args) {
|
|
138
|
+
const { values, positionals } = parseWith(args, LABEL_FLAGS);
|
|
139
|
+
return {
|
|
140
|
+
positionals,
|
|
141
|
+
options: {
|
|
142
|
+
maxInFlight: gateWidth(values["max-in-flight"]),
|
|
143
|
+
quiet: values.quiet === true,
|
|
144
|
+
rules: values.rules ?? null
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
function parseWith(args, flags) {
|
|
149
|
+
try {
|
|
150
|
+
return parseArgs({
|
|
151
|
+
args: [...args],
|
|
152
|
+
options: flags,
|
|
153
|
+
allowPositionals: true,
|
|
154
|
+
strict: true
|
|
155
|
+
});
|
|
156
|
+
} catch (cause) {
|
|
157
|
+
throw new UsageError(firstSentence(cause), { cause });
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/** The key is read here and nowhere else, so it never reaches argv, usage or a log line. */
|
|
161
|
+
function requireApiKey(env) {
|
|
162
|
+
const key = env[API_KEY_ENV];
|
|
163
|
+
if (key === void 0 || key.trim() === "") throw new UsageError(`${API_KEY_ENV} is not set`);
|
|
164
|
+
}
|
|
165
|
+
function gateWidth(raw) {
|
|
166
|
+
if (raw === void 0) return DEFAULT_MAX_IN_FLIGHT;
|
|
167
|
+
const value = parseNumber(raw);
|
|
168
|
+
if (value === void 0 || !Number.isInteger(value) || value < 1) throw new UsageError(`--max-in-flight takes an integer of 1 or more, not ${JSON.stringify(raw)}`);
|
|
169
|
+
return value;
|
|
170
|
+
}
|
|
171
|
+
function ratio$1(raw) {
|
|
172
|
+
if (raw === void 0) return null;
|
|
173
|
+
const value = parseNumber(raw);
|
|
174
|
+
if (value === void 0 || value < 0 || value > 1) throw new UsageError(`--min-accuracy takes a ratio from 0 to 1, not ${JSON.stringify(raw)}`);
|
|
175
|
+
return value;
|
|
176
|
+
}
|
|
177
|
+
function parseNumber(raw) {
|
|
178
|
+
if (raw.trim() === "") return void 0;
|
|
179
|
+
const value = Number(raw);
|
|
180
|
+
return Number.isFinite(value) ? value : void 0;
|
|
181
|
+
}
|
|
182
|
+
function firstSentence(cause) {
|
|
183
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
184
|
+
return message.split(". ")[0] ?? message;
|
|
185
|
+
}
|
|
186
|
+
//#endregion
|
|
187
|
+
//#region src/report.ts
|
|
188
|
+
const NA = "n/a";
|
|
189
|
+
/** Wide enough for `0.00` plus the gap before the note beside it. */
|
|
190
|
+
const RATIO_COLUMN = 6;
|
|
191
|
+
function watchDrift(recipeModel, stderr) {
|
|
192
|
+
let message = null;
|
|
193
|
+
return {
|
|
194
|
+
note: (decisionModel) => {
|
|
195
|
+
if (message !== null || decisionModel === recipeModel) return;
|
|
196
|
+
message = `model drift: the Recipe asks for ${recipeModel}, the Decisions came back from ${decisionModel}`;
|
|
197
|
+
writeLine(stderr, `siftline: ${message}`);
|
|
198
|
+
},
|
|
199
|
+
get message() {
|
|
200
|
+
return message;
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
/** One line per result, or one per miss. `order` is the Recipe's Question order. */
|
|
205
|
+
function progressLines(result, order) {
|
|
206
|
+
if (result.mismatches.length === 0) return [`ok ${result.id}`];
|
|
207
|
+
return inOrder(result.mismatches, order).map((miss) => `miss ${result.id} ${describeMismatch(miss)}`);
|
|
208
|
+
}
|
|
209
|
+
function renderReport(report, order) {
|
|
210
|
+
const rows = Object.entries(report.questions).map(([question, tally]) => ({
|
|
211
|
+
question,
|
|
212
|
+
counts: tally.asserted === 0 ? NA : `${tally.matched}/${tally.asserted}`,
|
|
213
|
+
accuracy: ratio(tally.accuracy)
|
|
214
|
+
}));
|
|
215
|
+
const labels = Math.max(...rows.map((row) => row.question.length), 8, 6) + 3;
|
|
216
|
+
const counts = Math.max(...rows.map((row) => row.counts.length), 3) + 3;
|
|
217
|
+
const lines = [
|
|
218
|
+
`${report.recipe.name} v${report.recipe.version} · ${report.model === "" ? NA : report.model}`,
|
|
219
|
+
"",
|
|
220
|
+
...rows.map((row) => `${row.question.padEnd(labels)}${row.counts.padEnd(counts)}${row.accuracy}`),
|
|
221
|
+
"",
|
|
222
|
+
`${"accuracy".padEnd(labels)}${ratio(report.accuracy).padEnd(RATIO_COLUMN)}(lowest question)`,
|
|
223
|
+
`${"unsure".padEnd(labels)}${unsure(report)} of ${report.fixtures.length} would go to Review`
|
|
224
|
+
];
|
|
225
|
+
const misses = report.fixtures.flatMap((result) => inOrder(result.mismatches, order).map((miss) => `${result.id} ${describeMismatch(miss)}`));
|
|
226
|
+
if (misses.length > 0) lines.push("", ...misses);
|
|
227
|
+
return lines.join("\n");
|
|
228
|
+
}
|
|
229
|
+
function unsure(report) {
|
|
230
|
+
return report.fixtures.filter((result) => result.review).length;
|
|
231
|
+
}
|
|
232
|
+
function ratio(value) {
|
|
233
|
+
return value === null ? NA : value.toFixed(2);
|
|
234
|
+
}
|
|
235
|
+
function describeMismatch(miss) {
|
|
236
|
+
return `${miss.question}: expected ${show(miss.expected)}, got ${show(miss.actual)}`;
|
|
237
|
+
}
|
|
238
|
+
function show(value) {
|
|
239
|
+
return value === void 0 ? "nothing" : String(value);
|
|
240
|
+
}
|
|
241
|
+
function inOrder(mismatches, order) {
|
|
242
|
+
return mismatches.toSorted((a, b) => order.indexOf(a.question) - order.indexOf(b.question));
|
|
243
|
+
}
|
|
244
|
+
//#endregion
|
|
245
|
+
//#region src/label.ts
|
|
246
|
+
async function runLabel(args, deps) {
|
|
247
|
+
const { positionals, options } = parseLabelArgs(args);
|
|
248
|
+
const [recipePath, recordsPath] = positionals;
|
|
249
|
+
if (recipePath === void 0) throw new UsageError("label needs a recipe");
|
|
250
|
+
if (positionals.length > 2) throw new UsageError(`label takes at most two positionals, not ${positionals.length}`);
|
|
251
|
+
const recipe = await readRecipeFile(recipePath);
|
|
252
|
+
const rules = options.rules === null ? null : await readRulesFile(options.rules, recipe);
|
|
253
|
+
return await label(await readRecords(recordsPath, deps.stdin), recipe, rules, deps, options);
|
|
254
|
+
}
|
|
255
|
+
async function label(records, recipe, rules, deps, options) {
|
|
256
|
+
const { maxInFlight } = options;
|
|
257
|
+
const judge = createJudge({
|
|
258
|
+
client: deps.client,
|
|
259
|
+
retry: "patient",
|
|
260
|
+
maxInFlight
|
|
261
|
+
});
|
|
262
|
+
const progress = createProgress(deps.stderr, records.length, options.quiet);
|
|
263
|
+
const drift = watchDrift(recipe.model, deps.stderr);
|
|
264
|
+
const buffered = /* @__PURE__ */ new Map();
|
|
265
|
+
let written = 0;
|
|
266
|
+
const flush = () => {
|
|
267
|
+
while (buffered.has(written)) {
|
|
268
|
+
const line = buffered.get(written) ?? null;
|
|
269
|
+
buffered.delete(written);
|
|
270
|
+
written += 1;
|
|
271
|
+
if (line !== null) writeLine(deps.stdout, line);
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
const controller = new AbortController();
|
|
275
|
+
const signal = deps.signal ? AbortSignal.any([deps.signal, controller.signal]) : controller.signal;
|
|
276
|
+
let fatal;
|
|
277
|
+
let failed = false;
|
|
278
|
+
let taken = 0;
|
|
279
|
+
let done = 0;
|
|
280
|
+
const worker = async () => {
|
|
281
|
+
while (fatal === void 0) {
|
|
282
|
+
const index = taken;
|
|
283
|
+
const record = records[index];
|
|
284
|
+
if (record === void 0) return;
|
|
285
|
+
taken += 1;
|
|
286
|
+
try {
|
|
287
|
+
const decision = await judge(record, recipe, { signal });
|
|
288
|
+
drift.note(decision.model);
|
|
289
|
+
buffered.set(index, serializeDecision(rules === null ? decision : routeDecision(decision, rules)));
|
|
290
|
+
} catch (error) {
|
|
291
|
+
if (!(error instanceof JudgeError)) {
|
|
292
|
+
fatal ??= error;
|
|
293
|
+
controller.abort(error);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
failed = true;
|
|
297
|
+
writeLine(deps.stderr, `${record.id}: ${sentence(error)}`);
|
|
298
|
+
buffered.set(index, null);
|
|
299
|
+
}
|
|
300
|
+
done += 1;
|
|
301
|
+
progress.tick(done);
|
|
302
|
+
flush();
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
const width = Math.min(maxInFlight, records.length);
|
|
306
|
+
await Promise.all(Array.from({ length: width }, () => worker()));
|
|
307
|
+
progress.finish();
|
|
308
|
+
if (fatal !== void 0) throw fatal;
|
|
309
|
+
return failed ? 1 : 0;
|
|
310
|
+
}
|
|
311
|
+
const REASONS = {
|
|
312
|
+
max_tokens_exceeded: "state over the token budget",
|
|
313
|
+
api_usage_error: "the API refused the request",
|
|
314
|
+
network: "the call never reached the API",
|
|
315
|
+
timeout: "the call timed out"
|
|
316
|
+
};
|
|
317
|
+
function sentence(error) {
|
|
318
|
+
if (error.reason === "unknown") return error.message;
|
|
319
|
+
if (error.reason === "invalid_answers") return `the answers did not fit the Recipe: ${error.message}`;
|
|
320
|
+
return REASONS[error.reason];
|
|
321
|
+
}
|
|
322
|
+
/** One line per this many Records where the counter cannot be rewritten in place. */
|
|
323
|
+
const PROGRESS_EVERY = 50;
|
|
324
|
+
const SILENT = {
|
|
325
|
+
tick: () => void 0,
|
|
326
|
+
finish: () => void 0
|
|
327
|
+
};
|
|
328
|
+
function createProgress(stream, total, quiet) {
|
|
329
|
+
if (quiet || total === 0) return SILENT;
|
|
330
|
+
if (stream.isTTY === true) return {
|
|
331
|
+
tick: (done) => {
|
|
332
|
+
stream.write(`\r${done}/${total}`);
|
|
333
|
+
},
|
|
334
|
+
finish: () => {
|
|
335
|
+
stream.write("\n");
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
return {
|
|
339
|
+
tick: (done) => {
|
|
340
|
+
if (done % PROGRESS_EVERY === 0) writeLine(stream, `${done}/${total}`);
|
|
341
|
+
},
|
|
342
|
+
finish: () => void 0
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
//#endregion
|
|
346
|
+
//#region src/test.ts
|
|
347
|
+
async function runTest(args, deps) {
|
|
348
|
+
const { positionals, options } = parseTestArgs(args);
|
|
349
|
+
const [recipePath, fixturesPath] = positionals;
|
|
350
|
+
if (recipePath === void 0 || fixturesPath === void 0) throw new UsageError("test needs a recipe and a fixtures file");
|
|
351
|
+
if (positionals.length > 2) throw new UsageError(`test takes two positionals, not ${positionals.length}`);
|
|
352
|
+
const recipe = await readRecipeFile(recipePath);
|
|
353
|
+
const fixtures = await readFixturesFile(fixturesPath, recipe);
|
|
354
|
+
const order = Object.keys(recipe.questions);
|
|
355
|
+
const judge = createJudge({
|
|
356
|
+
client: deps.client,
|
|
357
|
+
retry: "patient",
|
|
358
|
+
maxInFlight: options.maxInFlight
|
|
359
|
+
});
|
|
360
|
+
const drift = watchDrift(recipe.model, deps.stderr);
|
|
361
|
+
const report = await testRecipe(judge, recipe, fixtures, {
|
|
362
|
+
signal: deps.signal,
|
|
363
|
+
onResult: (result) => {
|
|
364
|
+
drift.note(result.decision.model);
|
|
365
|
+
if (options.quiet) return;
|
|
366
|
+
for (const line of progressLines(result, order)) writeLine(deps.stderr, line);
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
if (options.json) writeLine(deps.stdout, JSON.stringify({
|
|
370
|
+
...report,
|
|
371
|
+
drift: drift.message
|
|
372
|
+
}, null, 2));
|
|
373
|
+
else writeLine(deps.stdout, renderReport(report, order));
|
|
374
|
+
const floor = options.minAccuracy;
|
|
375
|
+
if (floor === null) return 0;
|
|
376
|
+
if (report.accuracy === null) {
|
|
377
|
+
writeLine(deps.stderr, `siftline: no Question was asserted, so there is no accuracy to meet --min-accuracy ${floor}`);
|
|
378
|
+
return 1;
|
|
379
|
+
}
|
|
380
|
+
if (report.accuracy < floor) {
|
|
381
|
+
writeLine(deps.stderr, `siftline: accuracy ${report.accuracy.toFixed(2)} is below --min-accuracy ${floor}`);
|
|
382
|
+
return 1;
|
|
383
|
+
}
|
|
384
|
+
return 0;
|
|
385
|
+
}
|
|
386
|
+
//#endregion
|
|
387
|
+
//#region src/usage.ts
|
|
388
|
+
/** The published version of `@siftline/cli`, baked in at build time. */
|
|
389
|
+
const VERSION$1 = "0.1.1";
|
|
390
|
+
const USAGE = [
|
|
391
|
+
`siftline ${VERSION$1} (engine ${VERSION})`,
|
|
392
|
+
"",
|
|
393
|
+
"Usage",
|
|
394
|
+
" siftline test <recipe.json> <fixtures.jsonl> [--max-in-flight <n>] [--min-accuracy <ratio>] [--json] [--quiet]",
|
|
395
|
+
" siftline label <recipe.json> [records.jsonl | -] [--rules <rules.json>] [--max-in-flight <n>] [--quiet]",
|
|
396
|
+
"",
|
|
397
|
+
"Options",
|
|
398
|
+
" --max-in-flight <n> Records judged at once (default 8)",
|
|
399
|
+
" --min-accuracy <ratio> test: exit 1 when the lowest question accuracy is below this",
|
|
400
|
+
" --json test: print the report as JSON",
|
|
401
|
+
" --rules <rules.json> label: select an Action per Decision with these Rules",
|
|
402
|
+
" --quiet Silence progress (never warnings or errors)",
|
|
403
|
+
" --help Print this help and exit",
|
|
404
|
+
" --version Print the version and exit",
|
|
405
|
+
"",
|
|
406
|
+
"Environment",
|
|
407
|
+
" TYPESAFE_API_KEY Required. Read from the environment only."
|
|
408
|
+
].join("\n");
|
|
409
|
+
//#endregion
|
|
410
|
+
//#region src/index.ts
|
|
411
|
+
/**
|
|
412
|
+
* The library door. Every command runs through it, so a test drives the whole CLI with an
|
|
413
|
+
* injected client and string streams and asserts on the exit code and what was written.
|
|
414
|
+
*/
|
|
415
|
+
async function run(argv, deps) {
|
|
416
|
+
try {
|
|
417
|
+
if (argv.includes("--help")) {
|
|
418
|
+
writeLine(deps.stdout, USAGE);
|
|
419
|
+
return 0;
|
|
420
|
+
}
|
|
421
|
+
if (argv.includes("--version")) {
|
|
422
|
+
writeLine(deps.stdout, VERSION$1);
|
|
423
|
+
return 0;
|
|
424
|
+
}
|
|
425
|
+
const [command, ...rest] = argv;
|
|
426
|
+
if (command === void 0) throw new UsageError("no command given");
|
|
427
|
+
if (command !== "test" && command !== "label") throw new UsageError(`unknown command: ${command}`);
|
|
428
|
+
requireApiKey(deps.env);
|
|
429
|
+
return command === "label" ? await runLabel(rest, deps) : await runTest(rest, deps);
|
|
430
|
+
} catch (cause) {
|
|
431
|
+
return exitFor(cause, deps);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
/** Every failure's exit code and its one stderr line. */
|
|
435
|
+
function exitFor(cause, deps) {
|
|
436
|
+
if (deps.signal?.aborted) {
|
|
437
|
+
writeLine(deps.stderr, "siftline: interrupted");
|
|
438
|
+
return 130;
|
|
439
|
+
}
|
|
440
|
+
if (cause instanceof UsageError) {
|
|
441
|
+
writeLine(deps.stderr, `siftline: ${cause.message}\n\n${USAGE}`);
|
|
442
|
+
return 2;
|
|
443
|
+
}
|
|
444
|
+
if (cause instanceof InputError) {
|
|
445
|
+
writeLine(deps.stderr, `siftline: ${cause.message}`);
|
|
446
|
+
return 2;
|
|
447
|
+
}
|
|
448
|
+
if (cause instanceof SiftlineError) {
|
|
449
|
+
writeLine(deps.stderr, `siftline: ${cause.message}`);
|
|
450
|
+
return cause.code === "fixture_invalid" ? 2 : 1;
|
|
451
|
+
}
|
|
452
|
+
writeLine(deps.stderr, `siftline: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
453
|
+
return 1;
|
|
454
|
+
}
|
|
455
|
+
//#endregion
|
|
456
|
+
export { API_KEY_ENV as i, USAGE as n, VERSION$1 as r, run as t };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@siftline/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "The Siftline command line: measure a recipe against its fixtures, then label for real.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"classification",
|
|
@@ -45,7 +45,9 @@
|
|
|
45
45
|
"typecheck": "tsc --noEmit"
|
|
46
46
|
},
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@siftline/core": "^0.
|
|
48
|
+
"@siftline/core": "^0.1.1",
|
|
49
|
+
"@typesafe-ai/sdk": "0.6.0",
|
|
50
|
+
"zod": "4.6.5"
|
|
49
51
|
},
|
|
50
52
|
"devDependencies": {
|
|
51
53
|
"@arethetypeswrong/core": "0.18.5",
|
package/dist/src-DYu5d4UT.mjs
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { VERSION } from "@siftline/core";
|
|
2
|
-
//#endregion
|
|
3
|
-
//#region src/index.ts
|
|
4
|
-
/**
|
|
5
|
-
* The published version of `@siftline/cli`, baked in at build time.
|
|
6
|
-
*
|
|
7
|
-
* Changesets bumps `package.json`; this constant follows it without a second edit.
|
|
8
|
-
*/
|
|
9
|
-
const VERSION$1 = "0.0.2";
|
|
10
|
-
/**
|
|
11
|
-
* The usage block. `test` and `label` are named before they exist so that the bin
|
|
12
|
-
* wiring is proven — and so that `siftline --help` never lies about what it does.
|
|
13
|
-
*/
|
|
14
|
-
const USAGE = [
|
|
15
|
-
`siftline ${VERSION$1} (engine ${VERSION})`,
|
|
16
|
-
"",
|
|
17
|
-
"Usage",
|
|
18
|
-
" siftline <command> [options]",
|
|
19
|
-
"",
|
|
20
|
-
"Commands",
|
|
21
|
-
" test Measure a recipe against its fixtures (not yet implemented)",
|
|
22
|
-
" label Label inputs with a recipe (not yet implemented)",
|
|
23
|
-
"",
|
|
24
|
-
"Options",
|
|
25
|
-
" --help Print this help and exit",
|
|
26
|
-
" --version Print the version and exit"
|
|
27
|
-
].join("\n");
|
|
28
|
-
/**
|
|
29
|
-
* Interpret command line arguments. Pure on purpose: the bin is the only place that
|
|
30
|
-
* touches the process, so the behaviour stays testable without one.
|
|
31
|
-
*/
|
|
32
|
-
function run(argv) {
|
|
33
|
-
const [first, ...rest] = argv;
|
|
34
|
-
if (rest.length === 0 && first === "--version") return {
|
|
35
|
-
code: 0,
|
|
36
|
-
stdout: `${VERSION$1}\n`,
|
|
37
|
-
stderr: ""
|
|
38
|
-
};
|
|
39
|
-
if (rest.length === 0 && first === "--help") return {
|
|
40
|
-
code: 0,
|
|
41
|
-
stdout: `${USAGE}\n`,
|
|
42
|
-
stderr: ""
|
|
43
|
-
};
|
|
44
|
-
return {
|
|
45
|
-
code: 1,
|
|
46
|
-
stdout: "",
|
|
47
|
-
stderr: `siftline: ${first === void 0 ? "no command given" : `unknown command: ${argv.join(" ")}`}\n\n${USAGE}\n`
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
//#endregion
|
|
51
|
-
export { VERSION$1 as n, run as r, USAGE as t };
|