aitelier 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/add.d.ts +3 -0
- package/dist/commands/add.d.ts.map +1 -0
- package/dist/commands/add.js +141 -0
- package/dist/commands/add.js.map +1 -0
- package/dist/commands/add.test.d.ts +2 -0
- package/dist/commands/add.test.d.ts.map +1 -0
- package/dist/commands/add.test.js +203 -0
- package/dist/commands/add.test.js.map +1 -0
- package/dist/commands/eval.d.ts +3 -0
- package/dist/commands/eval.d.ts.map +1 -0
- package/dist/commands/eval.js +505 -0
- package/dist/commands/eval.js.map +1 -0
- package/dist/commands/eval.test.d.ts +2 -0
- package/dist/commands/eval.test.d.ts.map +1 -0
- package/dist/commands/eval.test.js +804 -0
- package/dist/commands/eval.test.js.map +1 -0
- package/dist/commands/format.d.ts +3 -0
- package/dist/commands/format.d.ts.map +1 -0
- package/dist/commands/format.js +198 -0
- package/dist/commands/format.js.map +1 -0
- package/dist/commands/format.test.d.ts +2 -0
- package/dist/commands/format.test.d.ts.map +1 -0
- package/dist/commands/format.test.js +568 -0
- package/dist/commands/format.test.js.map +1 -0
- package/dist/commands/init.d.ts +3 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +94 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/init.test.d.ts +2 -0
- package/dist/commands/init.test.d.ts.map +1 -0
- package/dist/commands/init.test.js +123 -0
- package/dist/commands/init.test.js.map +1 -0
- package/dist/commands/rate.d.ts +3 -0
- package/dist/commands/rate.d.ts.map +1 -0
- package/dist/commands/rate.js +209 -0
- package/dist/commands/rate.js.map +1 -0
- package/dist/commands/rate.test.d.ts +2 -0
- package/dist/commands/rate.test.d.ts.map +1 -0
- package/dist/commands/rate.test.js +393 -0
- package/dist/commands/rate.test.js.map +1 -0
- package/dist/commands/split.d.ts +3 -0
- package/dist/commands/split.d.ts.map +1 -0
- package/dist/commands/split.js +210 -0
- package/dist/commands/split.js.map +1 -0
- package/dist/commands/split.test.d.ts +2 -0
- package/dist/commands/split.test.d.ts.map +1 -0
- package/dist/commands/split.test.js +503 -0
- package/dist/commands/split.test.js.map +1 -0
- package/dist/commands/stats.d.ts +3 -0
- package/dist/commands/stats.d.ts.map +1 -0
- package/dist/commands/stats.js +209 -0
- package/dist/commands/stats.js.map +1 -0
- package/dist/commands/stats.test.d.ts +2 -0
- package/dist/commands/stats.test.d.ts.map +1 -0
- package/dist/commands/stats.test.js +476 -0
- package/dist/commands/stats.test.js.map +1 -0
- package/dist/commands/status.d.ts +3 -0
- package/dist/commands/status.d.ts.map +1 -0
- package/dist/commands/status.js +132 -0
- package/dist/commands/status.js.map +1 -0
- package/dist/commands/status.test.d.ts +2 -0
- package/dist/commands/status.test.d.ts.map +1 -0
- package/dist/commands/status.test.js +443 -0
- package/dist/commands/status.test.js.map +1 -0
- package/dist/commands/train.d.ts +3 -0
- package/dist/commands/train.d.ts.map +1 -0
- package/dist/commands/train.js +139 -0
- package/dist/commands/train.js.map +1 -0
- package/dist/commands/train.test.d.ts +2 -0
- package/dist/commands/train.test.d.ts.map +1 -0
- package/dist/commands/train.test.js +288 -0
- package/dist/commands/train.test.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/providers/openai.d.ts +9 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +20 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/together.d.ts +11 -0
- package/dist/providers/together.d.ts.map +1 -0
- package/dist/providers/together.js +118 -0
- package/dist/providers/together.js.map +1 -0
- package/dist/providers/types.d.ts +28 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/storage/config.d.ts +24 -0
- package/dist/storage/config.d.ts.map +1 -0
- package/dist/storage/config.js +11 -0
- package/dist/storage/config.js.map +1 -0
- package/dist/storage/dataset.d.ts +14 -0
- package/dist/storage/dataset.d.ts.map +1 -0
- package/dist/storage/dataset.js +18 -0
- package/dist/storage/dataset.js.map +1 -0
- package/package.json +58 -0
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
2
|
+
import { writeFile, readFile, mkdtemp, rm, mkdir } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import inquirer from 'inquirer';
|
|
6
|
+
import { Command } from 'commander';
|
|
7
|
+
import { registerRate } from './rate.js';
|
|
8
|
+
describe('ft rate', () => {
|
|
9
|
+
let testDir;
|
|
10
|
+
let originalCwd;
|
|
11
|
+
beforeEach(async () => {
|
|
12
|
+
// Create a temporary directory for each test
|
|
13
|
+
testDir = await mkdtemp(join(tmpdir(), 'ft-rate-test-'));
|
|
14
|
+
originalCwd = process.cwd();
|
|
15
|
+
process.chdir(testDir);
|
|
16
|
+
// Create basic project structure
|
|
17
|
+
const config = {
|
|
18
|
+
name: 'test-project',
|
|
19
|
+
provider: 'together',
|
|
20
|
+
model: 'meta-llama/Llama-3.3-70B-Instruct',
|
|
21
|
+
qualityThreshold: 8,
|
|
22
|
+
runs: [],
|
|
23
|
+
};
|
|
24
|
+
await writeFile('.ftpipeline.json', JSON.stringify(config, null, 2));
|
|
25
|
+
await mkdir('data', { recursive: true });
|
|
26
|
+
});
|
|
27
|
+
afterEach(async () => {
|
|
28
|
+
process.chdir(originalCwd);
|
|
29
|
+
await rm(testDir, { recursive: true, force: true });
|
|
30
|
+
vi.restoreAllMocks();
|
|
31
|
+
});
|
|
32
|
+
it('should rate an unrated example', async () => {
|
|
33
|
+
// Create an unrated example
|
|
34
|
+
const example = {
|
|
35
|
+
id: 1,
|
|
36
|
+
messages: [
|
|
37
|
+
{ role: 'user', content: 'What is 2 + 2?' },
|
|
38
|
+
{ role: 'assistant', content: '2 + 2 equals 4.' },
|
|
39
|
+
],
|
|
40
|
+
rating: null,
|
|
41
|
+
createdAt: new Date().toISOString(),
|
|
42
|
+
version: 1,
|
|
43
|
+
};
|
|
44
|
+
await writeFile('data/examples.jsonl', JSON.stringify(example) + '\n');
|
|
45
|
+
// Mock inquirer to rate the example then quit
|
|
46
|
+
vi.spyOn(inquirer, 'prompt')
|
|
47
|
+
.mockResolvedValueOnce({ action: 'rate' })
|
|
48
|
+
.mockResolvedValueOnce({ rating: '8' });
|
|
49
|
+
const program = new Command();
|
|
50
|
+
registerRate(program);
|
|
51
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
52
|
+
// Verify the rating was saved
|
|
53
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
54
|
+
const updated = JSON.parse(content.trim());
|
|
55
|
+
expect(updated.rating).toBe(8);
|
|
56
|
+
});
|
|
57
|
+
it('should rewrite and rate an example', async () => {
|
|
58
|
+
const example = {
|
|
59
|
+
id: 1,
|
|
60
|
+
messages: [
|
|
61
|
+
{ role: 'user', content: 'What is AI?' },
|
|
62
|
+
{ role: 'assistant', content: 'AI is computers.' },
|
|
63
|
+
],
|
|
64
|
+
rating: null,
|
|
65
|
+
createdAt: new Date().toISOString(),
|
|
66
|
+
version: 1,
|
|
67
|
+
};
|
|
68
|
+
await writeFile('data/examples.jsonl', JSON.stringify(example) + '\n');
|
|
69
|
+
// Mock inquirer: rewrite → provide new output → rate
|
|
70
|
+
vi.spyOn(inquirer, 'prompt')
|
|
71
|
+
.mockResolvedValueOnce({ action: 'rewrite' })
|
|
72
|
+
.mockResolvedValueOnce({
|
|
73
|
+
output: 'AI is artificial intelligence, a field of computer science.',
|
|
74
|
+
})
|
|
75
|
+
.mockResolvedValueOnce({ rating: '9' });
|
|
76
|
+
const program = new Command();
|
|
77
|
+
registerRate(program);
|
|
78
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
79
|
+
// Verify the rewrite and rating
|
|
80
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
81
|
+
const updated = JSON.parse(content.trim());
|
|
82
|
+
expect(updated.rating).toBe(9);
|
|
83
|
+
expect(updated.messages[1].content).toBe('AI is artificial intelligence, a field of computer science.');
|
|
84
|
+
expect(updated.originalOutput).toBe('AI is computers.');
|
|
85
|
+
});
|
|
86
|
+
it('should preserve originalOutput on subsequent rewrites', async () => {
|
|
87
|
+
const example = {
|
|
88
|
+
id: 1,
|
|
89
|
+
messages: [
|
|
90
|
+
{ role: 'user', content: 'What is AI?' },
|
|
91
|
+
{ role: 'assistant', content: 'AI is computers.' },
|
|
92
|
+
],
|
|
93
|
+
rating: null,
|
|
94
|
+
originalOutput: 'Original answer.',
|
|
95
|
+
createdAt: new Date().toISOString(),
|
|
96
|
+
version: 1,
|
|
97
|
+
};
|
|
98
|
+
await writeFile('data/examples.jsonl', JSON.stringify(example) + '\n');
|
|
99
|
+
// Mock inquirer: rewrite again
|
|
100
|
+
vi.spyOn(inquirer, 'prompt')
|
|
101
|
+
.mockResolvedValueOnce({ action: 'rewrite' })
|
|
102
|
+
.mockResolvedValueOnce({ output: 'Third version of the answer.' })
|
|
103
|
+
.mockResolvedValueOnce({ rating: '7' });
|
|
104
|
+
const program = new Command();
|
|
105
|
+
registerRate(program);
|
|
106
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
107
|
+
// Verify originalOutput is unchanged
|
|
108
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
109
|
+
const updated = JSON.parse(content.trim());
|
|
110
|
+
expect(updated.originalOutput).toBe('Original answer.');
|
|
111
|
+
expect(updated.messages[1].content).toBe('Third version of the answer.');
|
|
112
|
+
});
|
|
113
|
+
it('should skip an example without changes', async () => {
|
|
114
|
+
const example = {
|
|
115
|
+
id: 1,
|
|
116
|
+
messages: [
|
|
117
|
+
{ role: 'user', content: 'Hello' },
|
|
118
|
+
{ role: 'assistant', content: 'Hi there!' },
|
|
119
|
+
],
|
|
120
|
+
rating: null,
|
|
121
|
+
createdAt: new Date().toISOString(),
|
|
122
|
+
version: 1,
|
|
123
|
+
};
|
|
124
|
+
await writeFile('data/examples.jsonl', JSON.stringify(example) + '\n');
|
|
125
|
+
// Mock inquirer: skip
|
|
126
|
+
vi.spyOn(inquirer, 'prompt').mockResolvedValueOnce({ action: 'skip' });
|
|
127
|
+
const program = new Command();
|
|
128
|
+
registerRate(program);
|
|
129
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
130
|
+
// Verify no changes
|
|
131
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
132
|
+
const updated = JSON.parse(content.trim());
|
|
133
|
+
expect(updated.rating).toBeNull();
|
|
134
|
+
expect(updated.messages[1].content).toBe('Hi there!');
|
|
135
|
+
});
|
|
136
|
+
it('should quit early and save progress', async () => {
|
|
137
|
+
const examples = [
|
|
138
|
+
{
|
|
139
|
+
id: 1,
|
|
140
|
+
messages: [
|
|
141
|
+
{ role: 'user', content: 'Question 1' },
|
|
142
|
+
{ role: 'assistant', content: 'Answer 1' },
|
|
143
|
+
],
|
|
144
|
+
rating: null,
|
|
145
|
+
createdAt: new Date().toISOString(),
|
|
146
|
+
version: 1,
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
id: 2,
|
|
150
|
+
messages: [
|
|
151
|
+
{ role: 'user', content: 'Question 2' },
|
|
152
|
+
{ role: 'assistant', content: 'Answer 2' },
|
|
153
|
+
],
|
|
154
|
+
rating: null,
|
|
155
|
+
createdAt: new Date().toISOString(),
|
|
156
|
+
version: 1,
|
|
157
|
+
},
|
|
158
|
+
];
|
|
159
|
+
await writeFile('data/examples.jsonl', examples.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
|
160
|
+
// Mock inquirer: rate first, then quit before second
|
|
161
|
+
vi.spyOn(inquirer, 'prompt')
|
|
162
|
+
.mockResolvedValueOnce({ action: 'rate' })
|
|
163
|
+
.mockResolvedValueOnce({ rating: '7' })
|
|
164
|
+
.mockResolvedValueOnce({ action: 'quit' });
|
|
165
|
+
const program = new Command();
|
|
166
|
+
registerRate(program);
|
|
167
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
168
|
+
// Verify first was rated, second was not
|
|
169
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
170
|
+
const lines = content.trim().split('\n');
|
|
171
|
+
expect(lines).toHaveLength(2);
|
|
172
|
+
const updated1 = JSON.parse(lines[0]);
|
|
173
|
+
const updated2 = JSON.parse(lines[1]);
|
|
174
|
+
expect(updated1.rating).toBe(7);
|
|
175
|
+
expect(updated2.rating).toBeNull();
|
|
176
|
+
});
|
|
177
|
+
it('should filter by --min flag', async () => {
|
|
178
|
+
const examples = [
|
|
179
|
+
{
|
|
180
|
+
id: 1,
|
|
181
|
+
messages: [
|
|
182
|
+
{ role: 'user', content: 'Q1' },
|
|
183
|
+
{ role: 'assistant', content: 'A1' },
|
|
184
|
+
],
|
|
185
|
+
rating: 5,
|
|
186
|
+
createdAt: new Date().toISOString(),
|
|
187
|
+
version: 1,
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
id: 2,
|
|
191
|
+
messages: [
|
|
192
|
+
{ role: 'user', content: 'Q2' },
|
|
193
|
+
{ role: 'assistant', content: 'A2' },
|
|
194
|
+
],
|
|
195
|
+
rating: 9,
|
|
196
|
+
createdAt: new Date().toISOString(),
|
|
197
|
+
version: 1,
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
id: 3,
|
|
201
|
+
messages: [
|
|
202
|
+
{ role: 'user', content: 'Q3' },
|
|
203
|
+
{ role: 'assistant', content: 'A3' },
|
|
204
|
+
],
|
|
205
|
+
rating: null,
|
|
206
|
+
createdAt: new Date().toISOString(),
|
|
207
|
+
version: 1,
|
|
208
|
+
},
|
|
209
|
+
];
|
|
210
|
+
await writeFile('data/examples.jsonl', examples.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
|
211
|
+
// Mock inquirer: rate the first low-rated example then quit
|
|
212
|
+
vi.spyOn(inquirer, 'prompt')
|
|
213
|
+
.mockResolvedValueOnce({ action: 'rate' })
|
|
214
|
+
.mockResolvedValueOnce({ rating: '8' })
|
|
215
|
+
.mockResolvedValueOnce({ action: 'quit' });
|
|
216
|
+
const program = new Command();
|
|
217
|
+
registerRate(program);
|
|
218
|
+
await program.parseAsync(['node', 'test', 'rate', '--min', '8']);
|
|
219
|
+
// Verify only examples with rating < 8 or null were shown
|
|
220
|
+
// ID 1 (rating 5) and ID 3 (null) should be shown
|
|
221
|
+
// ID 2 (rating 9) should not be shown
|
|
222
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
223
|
+
const lines = content.trim().split('\n');
|
|
224
|
+
const updated = lines.map((line) => JSON.parse(line));
|
|
225
|
+
// First example should be updated
|
|
226
|
+
expect(updated[0].rating).toBe(8);
|
|
227
|
+
// Second example should be unchanged
|
|
228
|
+
expect(updated[1].rating).toBe(9);
|
|
229
|
+
// Third example should be unchanged (quit before reaching it)
|
|
230
|
+
expect(updated[2].rating).toBeNull();
|
|
231
|
+
});
|
|
232
|
+
it('should display message when no examples to rate', async () => {
|
|
233
|
+
const example = {
|
|
234
|
+
id: 1,
|
|
235
|
+
messages: [
|
|
236
|
+
{ role: 'user', content: 'Test' },
|
|
237
|
+
{ role: 'assistant', content: 'Response' },
|
|
238
|
+
],
|
|
239
|
+
rating: 9,
|
|
240
|
+
createdAt: new Date().toISOString(),
|
|
241
|
+
version: 1,
|
|
242
|
+
};
|
|
243
|
+
await writeFile('data/examples.jsonl', JSON.stringify(example) + '\n');
|
|
244
|
+
const consoleLogSpy = vi.spyOn(console, 'log');
|
|
245
|
+
const program = new Command();
|
|
246
|
+
registerRate(program);
|
|
247
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
248
|
+
expect(consoleLogSpy).toHaveBeenCalledWith('No unrated examples found. All examples have been rated.');
|
|
249
|
+
});
|
|
250
|
+
it('should display message when no examples exist', async () => {
|
|
251
|
+
await writeFile('data/examples.jsonl', '');
|
|
252
|
+
const consoleLogSpy = vi.spyOn(console, 'log');
|
|
253
|
+
const program = new Command();
|
|
254
|
+
registerRate(program);
|
|
255
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
256
|
+
expect(consoleLogSpy).toHaveBeenCalledWith('No examples found. Add examples with `ft add` first.');
|
|
257
|
+
});
|
|
258
|
+
it('should fail if project is not initialized', async () => {
|
|
259
|
+
await rm('.ftpipeline.json');
|
|
260
|
+
const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => {
|
|
261
|
+
throw new Error('process.exit called');
|
|
262
|
+
});
|
|
263
|
+
const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
|
|
264
|
+
const program = new Command();
|
|
265
|
+
registerRate(program);
|
|
266
|
+
await expect(program.parseAsync(['node', 'test', 'rate'])).rejects.toThrow('process.exit called');
|
|
267
|
+
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Project not initialized'));
|
|
268
|
+
expect(mockExit).toHaveBeenCalledWith(1);
|
|
269
|
+
});
|
|
270
|
+
it('should validate --min flag', async () => {
|
|
271
|
+
await writeFile('data/examples.jsonl', '');
|
|
272
|
+
const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => {
|
|
273
|
+
throw new Error('process.exit called');
|
|
274
|
+
});
|
|
275
|
+
const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
|
|
276
|
+
const program = new Command();
|
|
277
|
+
registerRate(program);
|
|
278
|
+
await expect(program.parseAsync(['node', 'test', 'rate', '--min', '15'])).rejects.toThrow('process.exit called');
|
|
279
|
+
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('--min must be a number between 0 and 10'));
|
|
280
|
+
expect(mockExit).toHaveBeenCalledWith(1);
|
|
281
|
+
});
|
|
282
|
+
it('should display session summary with statistics', async () => {
|
|
283
|
+
const examples = [
|
|
284
|
+
{
|
|
285
|
+
id: 1,
|
|
286
|
+
messages: [
|
|
287
|
+
{ role: 'user', content: 'Q1' },
|
|
288
|
+
{ role: 'assistant', content: 'A1' },
|
|
289
|
+
],
|
|
290
|
+
rating: null,
|
|
291
|
+
createdAt: new Date().toISOString(),
|
|
292
|
+
version: 1,
|
|
293
|
+
},
|
|
294
|
+
{
|
|
295
|
+
id: 2,
|
|
296
|
+
messages: [
|
|
297
|
+
{ role: 'user', content: 'Q2' },
|
|
298
|
+
{ role: 'assistant', content: 'A2' },
|
|
299
|
+
],
|
|
300
|
+
rating: null,
|
|
301
|
+
createdAt: new Date().toISOString(),
|
|
302
|
+
version: 1,
|
|
303
|
+
},
|
|
304
|
+
];
|
|
305
|
+
await writeFile('data/examples.jsonl', examples.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
|
306
|
+
const consoleLogSpy = vi.spyOn(console, 'log');
|
|
307
|
+
// Mock inquirer: rate both examples
|
|
308
|
+
vi.spyOn(inquirer, 'prompt')
|
|
309
|
+
.mockResolvedValueOnce({ action: 'rate' })
|
|
310
|
+
.mockResolvedValueOnce({ rating: '8' })
|
|
311
|
+
.mockResolvedValueOnce({ action: 'rate' })
|
|
312
|
+
.mockResolvedValueOnce({ rating: '9' });
|
|
313
|
+
const program = new Command();
|
|
314
|
+
registerRate(program);
|
|
315
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
316
|
+
// Verify summary was displayed
|
|
317
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('Rating Session Complete'));
|
|
318
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('Examples shown: 2'));
|
|
319
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('Rated: 2'));
|
|
320
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('Average rating: 8.5/10'));
|
|
321
|
+
});
|
|
322
|
+
it('should handle examples with system prompt', async () => {
|
|
323
|
+
const example = {
|
|
324
|
+
id: 1,
|
|
325
|
+
messages: [
|
|
326
|
+
{ role: 'system', content: 'You are a helpful assistant.' },
|
|
327
|
+
{ role: 'user', content: 'Hello' },
|
|
328
|
+
{ role: 'assistant', content: 'Hi!' },
|
|
329
|
+
],
|
|
330
|
+
rating: null,
|
|
331
|
+
createdAt: new Date().toISOString(),
|
|
332
|
+
version: 1,
|
|
333
|
+
};
|
|
334
|
+
await writeFile('data/examples.jsonl', JSON.stringify(example) + '\n');
|
|
335
|
+
// Mock inquirer to rate
|
|
336
|
+
vi.spyOn(inquirer, 'prompt')
|
|
337
|
+
.mockResolvedValueOnce({ action: 'rate' })
|
|
338
|
+
.mockResolvedValueOnce({ rating: '7' });
|
|
339
|
+
const program = new Command();
|
|
340
|
+
registerRate(program);
|
|
341
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
342
|
+
// Verify the rating was saved and all messages preserved
|
|
343
|
+
const content = await readFile('data/examples.jsonl', 'utf-8');
|
|
344
|
+
const updated = JSON.parse(content.trim());
|
|
345
|
+
expect(updated.rating).toBe(7);
|
|
346
|
+
expect(updated.messages).toHaveLength(3);
|
|
347
|
+
expect(updated.messages[0].role).toBe('system');
|
|
348
|
+
});
|
|
349
|
+
it('should sort examples by ID when filtering', async () => {
|
|
350
|
+
const examples = [
|
|
351
|
+
{
|
|
352
|
+
id: 3,
|
|
353
|
+
messages: [
|
|
354
|
+
{ role: 'user', content: 'Q3' },
|
|
355
|
+
{ role: 'assistant', content: 'A3' },
|
|
356
|
+
],
|
|
357
|
+
rating: null,
|
|
358
|
+
createdAt: new Date().toISOString(),
|
|
359
|
+
version: 1,
|
|
360
|
+
},
|
|
361
|
+
{
|
|
362
|
+
id: 1,
|
|
363
|
+
messages: [
|
|
364
|
+
{ role: 'user', content: 'Q1' },
|
|
365
|
+
{ role: 'assistant', content: 'A1' },
|
|
366
|
+
],
|
|
367
|
+
rating: null,
|
|
368
|
+
createdAt: new Date().toISOString(),
|
|
369
|
+
version: 1,
|
|
370
|
+
},
|
|
371
|
+
{
|
|
372
|
+
id: 2,
|
|
373
|
+
messages: [
|
|
374
|
+
{ role: 'user', content: 'Q2' },
|
|
375
|
+
{ role: 'assistant', content: 'A2' },
|
|
376
|
+
],
|
|
377
|
+
rating: null,
|
|
378
|
+
createdAt: new Date().toISOString(),
|
|
379
|
+
version: 1,
|
|
380
|
+
},
|
|
381
|
+
];
|
|
382
|
+
await writeFile('data/examples.jsonl', examples.map((e) => JSON.stringify(e)).join('\n') + '\n');
|
|
383
|
+
const consoleLogSpy = vi.spyOn(console, 'log');
|
|
384
|
+
// Mock inquirer: just check the first example shown
|
|
385
|
+
vi.spyOn(inquirer, 'prompt').mockResolvedValueOnce({ action: 'quit' });
|
|
386
|
+
const program = new Command();
|
|
387
|
+
registerRate(program);
|
|
388
|
+
await program.parseAsync(['node', 'test', 'rate']);
|
|
389
|
+
// Verify Example #1 was shown first (sorted by ID)
|
|
390
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('Example #1'));
|
|
391
|
+
});
|
|
392
|
+
});
|
|
393
|
+
//# sourceMappingURL=rate.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rate.test.js","sourceRoot":"","sources":["../../src/commands/rate.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAIzC,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACvB,IAAI,OAAe,CAAC;IACpB,IAAI,WAAmB,CAAC;IAExB,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,6CAA6C;QAC7C,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QACzD,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEvB,iCAAiC;QACjC,MAAM,MAAM,GAAkB;YAC5B,IAAI,EAAE,cAAc;YACpB,QAAQ,EAAE,UAAU;YACpB,KAAK,EAAE,mCAAmC;YAC1C,gBAAgB,EAAE,CAAC;YACnB,IAAI,EAAE,EAAE;SACT,CAAC;QACF,MAAM,SAAS,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,4BAA4B;QAC5B,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,CAAC;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE;gBAC3C,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,iBAAiB,EAAE;aAClD;YACD,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,CAAC;SACX,CAAC;QACF,MAAM,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,8CAA8C;QAC9C,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;aACzC,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,8BAA8B;QAC9B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,CAAC;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE;gBACxC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,kBAAkB,EAAE;aACnD;YACD,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,CAAC;SACX,CAAC;QACF,MAAM,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,qDAAqD;QACrD,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;aAC5C,qBAAqB,CAAC;YACrB,MAAM,EAAE,6DAA6D;SACtE,CAAC;aACD,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,gCAAgC;QAChC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CACtC,6DAA6D,CAC9D,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,CAAC;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE;gBACxC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,kBAAkB,EAAE;aACnD;YACD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,kBAAkB;YAClC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,CAAC;SACX,CAAC;QACF,MAAM,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,+BAA+B;QAC/B,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;aAC5C,qBAAqB,CAAC,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;aACjE,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,qCAAqC;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACxD,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,CAAC;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE;gBAClC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE;aAC5C;YACD,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,CAAC;SACX,CAAC;QACF,MAAM,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,sBAAsB;QACtB,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAEvE,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,oBAAoB;QACpB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAClC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,QAAQ,GAAc;YAC1B;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE;oBACvC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE;iBAC3C;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;YACD;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE;oBACvC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE;iBAC3C;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;SACF,CAAC;QACF,MAAM,SAAS,CACb,qBAAqB,EACrB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACzD,CAAC;QAEF,qDAAqD;QACrD,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;aACzC,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;aACtC,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAE7C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,yCAAyC;QACzC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,QAAQ,GAAc;YAC1B;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,CAAC;gBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;YACD;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,CAAC;gBACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;YACD;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;SACF,CAAC;QACF,MAAM,SAAS,CACb,qBAAqB,EACrB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACzD,CAAC;QAEF,4DAA4D;QAC5D,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;aACzC,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;aACtC,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAE7C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAEjE,0DAA0D;QAC1D,kDAAkD;QAClD,sCAAsC;QACtC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAEtD,kCAAkC;QAClC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,qCAAqC;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,8DAA8D;QAC9D,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,CAAC;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;gBACjC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE;aAC3C;YACD,MAAM,EAAE,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,CAAC;SACX,CAAC;QACF,MAAM,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CACxC,0DAA0D,CAC3D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,MAAM,SAAS,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAE3C,MAAM,aAAa,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CACxC,sDAAsD,CACvD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,EAAE,CAAC,kBAAkB,CAAC,CAAC;QAE7B,MAAM,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QAEtB,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACxE,qBAAqB,CACtB,CAAC;QAEF,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAC1C,MAAM,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CACnD,CAAC;QACF,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;QAC1C,MAAM,SAAS,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAE3C,MAAM,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QAEtB,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACvF,qBAAqB,CACtB,CAAC;QAEF,MAAM,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAC1C,MAAM,CAAC,gBAAgB,CAAC,yCAAyC,CAAC,CACnE,CAAC;QACF,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,QAAQ,GAAc;YAC1B;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;YACD;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;SACF,CAAC;QACF,MAAM,SAAS,CACb,qBAAqB,EACrB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACzD,CAAC;QAEF,MAAM,aAAa,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE/C,oCAAoC;QACpC,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;aACzC,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;aACtC,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;aACzC,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,+BAA+B;QAC/B,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,CAAC;QAC/F,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACzF,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAChG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,OAAO,GAAY;YACvB,EAAE,EAAE,CAAC;YACL,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,8BAA8B,EAAE;gBAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE;gBAClC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE;aACtC;YACD,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,CAAC;SACX,CAAC;QACF,MAAM,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvE,wBAAwB;QACxB,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;aACzB,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;aACzC,qBAAqB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,yDAAyD;QACzD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,QAAQ,GAAc;YAC1B;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;YACD;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;YACD;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE;iBACrC;gBACD,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,CAAC;aACX;SACF,CAAC;QACF,MAAM,SAAS,CACb,qBAAqB,EACrB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACzD,CAAC;QAEF,MAAM,aAAa,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE/C,oDAAoD;QACpD,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,qBAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAEvE,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC9B,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,mDAAmD;QACnD,MAAM,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../src/commands/split.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAsBzC,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAkBpD"}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import inquirer from 'inquirer';
|
|
2
|
+
import { access } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { readExamples, writeExamples } from '../storage/dataset.js';
|
|
5
|
+
import { loadConfig } from '../storage/config.js';
|
|
6
|
+
const CONFIG_FILE = '.ftpipeline.json';
|
|
7
|
+
const EXAMPLES_FILE = 'data/examples.jsonl';
|
|
8
|
+
export function registerSplit(program) {
|
|
9
|
+
program
|
|
10
|
+
.command('split')
|
|
11
|
+
.description('Assign train/validation splits to examples')
|
|
12
|
+
.option('--ratio <ratio>', 'Train ratio for the split (e.g., 0.8 for 80/20, 0.9 for 90/10)', '0.8')
|
|
13
|
+
.option('--reshuffle', 'Force re-split of all examples (requires confirmation)')
|
|
14
|
+
.action(async (options) => {
|
|
15
|
+
try {
|
|
16
|
+
await splitCommand(options);
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
20
|
+
process.exit(1);
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
async function splitCommand(options) {
|
|
25
|
+
const cwd = process.cwd();
|
|
26
|
+
// Check if project is initialized
|
|
27
|
+
try {
|
|
28
|
+
await access(join(cwd, CONFIG_FILE));
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
throw new Error('Project not initialized. Run `ft init` first to create .ftpipeline.json');
|
|
32
|
+
}
|
|
33
|
+
// Parse and validate --ratio flag
|
|
34
|
+
const trainRatio = parseFloat(options.ratio);
|
|
35
|
+
if (isNaN(trainRatio) || trainRatio <= 0 || trainRatio >= 1) {
|
|
36
|
+
throw new Error('--ratio must be a number between 0 and 1 (exclusive)');
|
|
37
|
+
}
|
|
38
|
+
// Load config to get quality threshold
|
|
39
|
+
const config = await loadConfig(cwd);
|
|
40
|
+
// Load all examples
|
|
41
|
+
const examplesPath = join(cwd, EXAMPLES_FILE);
|
|
42
|
+
let examples;
|
|
43
|
+
try {
|
|
44
|
+
examples = await readExamples(examplesPath);
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error.code === 'ENOENT') {
|
|
48
|
+
console.log('No examples found. Add examples with `ft add` first.');
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
// Check if file has any examples
|
|
54
|
+
if (examples.length === 0) {
|
|
55
|
+
console.log('No examples found. Add examples with `ft add` first.');
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
// Check for rated examples meeting threshold
|
|
59
|
+
const qualified = examples.filter((e) => e.rating !== null && e.rating >= config.qualityThreshold);
|
|
60
|
+
if (qualified.length === 0) {
|
|
61
|
+
console.log(`No examples meet quality threshold (${config.qualityThreshold}/10). Rate examples with \`ft rate\` first.`);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
// Check for existing validation assignments
|
|
65
|
+
const existingVal = examples.filter((e) => e.split === 'val');
|
|
66
|
+
const hasExistingSplit = existingVal.length > 0;
|
|
67
|
+
// Handle --reshuffle flag
|
|
68
|
+
if (options.reshuffle) {
|
|
69
|
+
if (!hasExistingSplit) {
|
|
70
|
+
console.log('No existing split found. Proceeding with initial split.');
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
const confirmed = await confirmReshuffle(existingVal.length);
|
|
74
|
+
if (!confirmed) {
|
|
75
|
+
console.log('Split cancelled.');
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
// Clear all split assignments
|
|
79
|
+
for (const example of examples) {
|
|
80
|
+
delete example.split;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
else if (hasExistingSplit) {
|
|
85
|
+
// Lock validation set unless --reshuffle
|
|
86
|
+
console.log(`Found ${existingVal.length} locked validation examples. These will not be reassigned.`);
|
|
87
|
+
console.log('Use --reshuffle to force re-split with confirmation.\n');
|
|
88
|
+
}
|
|
89
|
+
// Perform the split
|
|
90
|
+
const stats = performSplit(qualified, trainRatio, !options.reshuffle && hasExistingSplit);
|
|
91
|
+
// Write updated examples back
|
|
92
|
+
await writeExamples(examplesPath, examples);
|
|
93
|
+
// Display summary
|
|
94
|
+
displaySummary(stats, config.qualityThreshold, trainRatio);
|
|
95
|
+
}
|
|
96
|
+
async function confirmReshuffle(valCount) {
|
|
97
|
+
const answer = await inquirer.prompt([
|
|
98
|
+
{
|
|
99
|
+
type: 'confirm',
|
|
100
|
+
name: 'confirm',
|
|
101
|
+
message: `This will reshuffle all ${valCount} validation examples. Are you sure?`,
|
|
102
|
+
default: false,
|
|
103
|
+
},
|
|
104
|
+
]);
|
|
105
|
+
return answer.confirm;
|
|
106
|
+
}
|
|
107
|
+
function performSplit(qualified, trainRatio, lockValidation) {
|
|
108
|
+
const stats = {
|
|
109
|
+
totalExamples: 0,
|
|
110
|
+
qualifiedExamples: qualified.length,
|
|
111
|
+
belowThreshold: 0,
|
|
112
|
+
newTrain: 0,
|
|
113
|
+
newVal: 0,
|
|
114
|
+
lockedVal: 0,
|
|
115
|
+
totalTrain: 0,
|
|
116
|
+
totalVal: 0,
|
|
117
|
+
};
|
|
118
|
+
// Separate locked validation examples from those to split
|
|
119
|
+
const lockedVal = lockValidation ? qualified.filter((e) => e.split === 'val') : [];
|
|
120
|
+
const toSplit = lockValidation ? qualified.filter((e) => e.split !== 'val') : qualified;
|
|
121
|
+
stats.lockedVal = lockedVal.length;
|
|
122
|
+
if (toSplit.length === 0) {
|
|
123
|
+
// All qualified examples are already locked as validation
|
|
124
|
+
stats.totalVal = lockedVal.length;
|
|
125
|
+
return stats;
|
|
126
|
+
}
|
|
127
|
+
// Calculate target validation count from available examples
|
|
128
|
+
const targetValCount = Math.round(toSplit.length * (1 - trainRatio));
|
|
129
|
+
const actualValCount = Math.max(1, targetValCount); // At least 1 validation example
|
|
130
|
+
// Stratified sampling by rating when possible
|
|
131
|
+
const shuffled = stratifiedShuffle(toSplit);
|
|
132
|
+
// Assign new validation examples
|
|
133
|
+
const newVal = shuffled.slice(0, actualValCount);
|
|
134
|
+
const newTrain = shuffled.slice(actualValCount);
|
|
135
|
+
for (const example of newVal) {
|
|
136
|
+
example.split = 'val';
|
|
137
|
+
stats.newVal++;
|
|
138
|
+
}
|
|
139
|
+
for (const example of newTrain) {
|
|
140
|
+
example.split = 'train';
|
|
141
|
+
stats.newTrain++;
|
|
142
|
+
}
|
|
143
|
+
stats.totalTrain = stats.newTrain;
|
|
144
|
+
stats.totalVal = stats.newVal + stats.lockedVal;
|
|
145
|
+
return stats;
|
|
146
|
+
}
|
|
147
|
+
function stratifiedShuffle(examples) {
|
|
148
|
+
// Group examples by rating for stratified sampling
|
|
149
|
+
const byRating = new Map();
|
|
150
|
+
for (const example of examples) {
|
|
151
|
+
if (example.rating === null)
|
|
152
|
+
continue; // Should not happen since we filtered
|
|
153
|
+
const rating = example.rating;
|
|
154
|
+
if (!byRating.has(rating)) {
|
|
155
|
+
byRating.set(rating, []);
|
|
156
|
+
}
|
|
157
|
+
byRating.get(rating).push(example);
|
|
158
|
+
}
|
|
159
|
+
// Shuffle within each rating group
|
|
160
|
+
for (const group of byRating.values()) {
|
|
161
|
+
shuffleArray(group);
|
|
162
|
+
}
|
|
163
|
+
// Interleave groups to maintain stratification
|
|
164
|
+
const result = [];
|
|
165
|
+
const ratings = Array.from(byRating.keys()).sort((a, b) => b - a); // High to low
|
|
166
|
+
let hasMore = true;
|
|
167
|
+
let index = 0;
|
|
168
|
+
while (hasMore) {
|
|
169
|
+
hasMore = false;
|
|
170
|
+
for (const rating of ratings) {
|
|
171
|
+
const group = byRating.get(rating);
|
|
172
|
+
if (index < group.length) {
|
|
173
|
+
result.push(group[index]);
|
|
174
|
+
hasMore = true;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
index++;
|
|
178
|
+
}
|
|
179
|
+
return result;
|
|
180
|
+
}
|
|
181
|
+
function shuffleArray(array) {
|
|
182
|
+
// Fisher-Yates shuffle
|
|
183
|
+
for (let i = array.length - 1; i > 0; i--) {
|
|
184
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
185
|
+
[array[i], array[j]] = [array[j], array[i]];
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
function displaySummary(stats, threshold, ratio) {
|
|
189
|
+
console.log('\n' + '═'.repeat(70));
|
|
190
|
+
console.log('Train/Val Split Complete');
|
|
191
|
+
console.log('═'.repeat(70));
|
|
192
|
+
console.log(`\nQuality threshold: ${threshold}/10`);
|
|
193
|
+
console.log(`Split ratio: ${(ratio * 100).toFixed(0)}% train / ${((1 - ratio) * 100).toFixed(0)}% val`);
|
|
194
|
+
console.log('\nSplit Summary:');
|
|
195
|
+
console.log('━'.repeat(70));
|
|
196
|
+
console.log(`Total train: ${stats.totalTrain} examples`);
|
|
197
|
+
console.log(`Total val: ${stats.totalVal} examples`);
|
|
198
|
+
if (stats.lockedVal > 0) {
|
|
199
|
+
console.log(` ├─ Locked (existing): ${stats.lockedVal}`);
|
|
200
|
+
console.log(` └─ New: ${stats.newVal}`);
|
|
201
|
+
}
|
|
202
|
+
const totalAssigned = stats.totalTrain + stats.totalVal;
|
|
203
|
+
console.log(`\nTotal assigned: ${totalAssigned} examples`);
|
|
204
|
+
console.log('\nNext Steps:');
|
|
205
|
+
console.log('━'.repeat(70));
|
|
206
|
+
console.log('1. Run `ft format` to generate train.jsonl and val.jsonl');
|
|
207
|
+
console.log('2. Run `ft train` to start fine-tuning');
|
|
208
|
+
console.log('═'.repeat(70) + '\n');
|
|
209
|
+
}
|
|
210
|
+
//# sourceMappingURL=split.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/commands/split.ts"],"names":[],"mappings":"AACA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,MAAM,WAAW,GAAG,kBAAkB,CAAC;AACvC,MAAM,aAAa,GAAG,qBAAqB,CAAC;AAa5C,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,OAAO;SACJ,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,4CAA4C,CAAC;SACzD,MAAM,CACL,iBAAiB,EACjB,gEAAgE,EAChE,KAAK,CACN;SACA,MAAM,CAAC,aAAa,EAAE,wDAAwD,CAAC;SAC/E,MAAM,CAAC,KAAK,EAAE,OAA+C,EAAE,EAAE;QAChE,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,OAA+C;IACzE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE1B,kCAAkC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IAED,kCAAkC;IAClC,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IAErC,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAC9C,IAAI,QAAmB,CAAC;IACxB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;YACpE,OAAO;QACT,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,iCAAiC;IACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO;IACT,CAAC;IAED,6CAA6C;IAC7C,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,gBAAgB,CAChE,CAAC;IAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CACT,uCAAuC,MAAM,CAAC,gBAAgB,6CAA6C,CAC5G,CAAC;QACF,OAAO;IACT,CAAC;IAED,4CAA4C;IAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;IAC9D,MAAM,gBAAgB,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAEhD,0BAA0B;IAC1B,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC7D,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBAChC,OAAO;YACT,CAAC;YACD,8BAA8B;YAC9B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,OAAO,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,gBAAgB,EAAE,CAAC;QAC5B,yCAAyC;QACzC,OAAO,CAAC,GAAG,CACT,SAAS,WAAW,CAAC,MAAM,4DAA4D,CACxF,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACxE,CAAC;IAED,oBAAoB;IACpB,MAAM,KAAK,GAAG,YAAY,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,OAAO,CAAC,SAAS,IAAI,gBAAgB,CAAC,CAAC;IAE1F,8BAA8B;IAC9B,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAE5C,kBAAkB;IAClB,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;AAC7D,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAuB;QACzD;YACE,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,QAAQ,qCAAqC;YACjF,OAAO,EAAE,KAAK;SACf;KACF,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,SAAS,YAAY,CACnB,SAAoB,EACpB,UAAkB,EAClB,cAAuB;IAEvB,MAAM,KAAK,GAAe;QACxB,aAAa,EAAE,CAAC;QAChB,iBAAiB,EAAE,SAAS,CAAC,MAAM;QACnC,cAAc,EAAE,CAAC;QACjB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;KACZ,CAAC;IAEF,0DAA0D;IAC1D,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnF,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAExF,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC;IAEnC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,0DAA0D;QAC1D,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAClC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,4DAA4D;IAC5D,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,gCAAgC;IAEpF,8CAA8C;IAC9C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE5C,iCAAiC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAEhD,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QACtB,KAAK,CAAC,MAAM,EAAE,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC;QACxB,KAAK,CAAC,QAAQ,EAAE,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC;IAClC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;IAEhD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAmB;IAC5C,mDAAmD;IACnD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;IAE9C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI;YAAE,SAAS,CAAC,sCAAsC;QAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,mCAAmC;IACnC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,+CAA+C;IAC/C,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc;IAEjF,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,OAAO,EAAE,CAAC;QACf,OAAO,GAAG,KAAK,CAAC;QAChB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;YACpC,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1B,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;QACH,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAI,KAAU;IACjC,uBAAuB;IACvB,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAiB,EAAE,SAAiB,EAAE,KAAa;IACzE,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,wBAAwB,SAAS,KAAK,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CACT,gBAAgB,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAC3F,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,UAAU,WAAW,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,QAAQ,WAAW,CAAC,CAAC;IAErD,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,2BAA2B,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,aAAa,GAAG,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,aAAa,WAAW,CAAC,CAAC;IAE3D,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IAEtD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"split.test.d.ts","sourceRoot":"","sources":["../../src/commands/split.test.ts"],"names":[],"mappings":""}
|