ai-functions 2.0.2 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -5
- package/CHANGELOG.md +38 -0
- package/LICENSE +21 -0
- package/README.md +361 -159
- package/dist/ai-promise.d.ts +47 -0
- package/dist/ai-promise.d.ts.map +1 -1
- package/dist/ai-promise.js +291 -3
- package/dist/ai-promise.js.map +1 -1
- package/dist/ai.d.ts +17 -18
- package/dist/ai.d.ts.map +1 -1
- package/dist/ai.js +93 -39
- package/dist/ai.js.map +1 -1
- package/dist/batch-map.d.ts +46 -4
- package/dist/batch-map.d.ts.map +1 -1
- package/dist/batch-map.js +35 -2
- package/dist/batch-map.js.map +1 -1
- package/dist/batch-queue.d.ts +116 -12
- package/dist/batch-queue.d.ts.map +1 -1
- package/dist/batch-queue.js +47 -2
- package/dist/batch-queue.js.map +1 -1
- package/dist/budget.d.ts +272 -0
- package/dist/budget.d.ts.map +1 -0
- package/dist/budget.js +500 -0
- package/dist/budget.js.map +1 -0
- package/dist/cache.d.ts +272 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +412 -0
- package/dist/cache.js.map +1 -0
- package/dist/context.d.ts +32 -1
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +16 -1
- package/dist/context.js.map +1 -1
- package/dist/eval/runner.d.ts +2 -1
- package/dist/eval/runner.d.ts.map +1 -1
- package/dist/eval/runner.js.map +1 -1
- package/dist/generate.d.ts.map +1 -1
- package/dist/generate.js +6 -10
- package/dist/generate.js.map +1 -1
- package/dist/index.d.ts +27 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +72 -42
- package/dist/index.js.map +1 -1
- package/dist/primitives.d.ts +17 -0
- package/dist/primitives.d.ts.map +1 -1
- package/dist/primitives.js +19 -1
- package/dist/primitives.js.map +1 -1
- package/dist/retry.d.ts +303 -0
- package/dist/retry.d.ts.map +1 -0
- package/dist/retry.js +539 -0
- package/dist/retry.js.map +1 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +1 -9
- package/dist/schema.js.map +1 -1
- package/dist/tool-orchestration.d.ts +391 -0
- package/dist/tool-orchestration.d.ts.map +1 -0
- package/dist/tool-orchestration.js +663 -0
- package/dist/tool-orchestration.js.map +1 -0
- package/dist/types.d.ts +50 -33
- package/dist/types.d.ts.map +1 -1
- package/evalite.config.js +14 -0
- package/evals/classification.eval.js +97 -0
- package/evals/marketing.eval.js +289 -0
- package/evals/math.eval.js +83 -0
- package/evals/run-evals.js +151 -0
- package/evals/structured-output.eval.js +131 -0
- package/evals/writing.eval.js +105 -0
- package/examples/batch-blog-posts.js +128 -0
- package/package.json +26 -26
- package/src/ai-promise.ts +359 -3
- package/src/ai.ts +155 -110
- package/src/batch/anthropic.js +256 -0
- package/src/batch/bedrock.js +584 -0
- package/src/batch/cloudflare.js +287 -0
- package/src/batch/google.js +359 -0
- package/src/batch/index.js +30 -0
- package/src/batch/memory.js +187 -0
- package/src/batch/openai.js +402 -0
- package/src/batch-map.ts +46 -4
- package/src/batch-queue.ts +116 -12
- package/src/budget.ts +727 -0
- package/src/cache.ts +653 -0
- package/src/context.ts +33 -1
- package/src/eval/index.js +7 -0
- package/src/eval/models.js +119 -0
- package/src/eval/runner.js +147 -0
- package/src/eval/runner.ts +3 -2
- package/src/generate.ts +7 -12
- package/src/index.ts +231 -53
- package/src/primitives.ts +19 -1
- package/src/retry.ts +776 -0
- package/src/schema.ts +1 -10
- package/src/tool-orchestration.ts +1008 -0
- package/src/types.ts +59 -41
- package/test/ai-proxy.test.js +157 -0
- package/test/async-iterators.test.js +261 -0
- package/test/backward-compat.test.ts +147 -0
- package/test/batch-autosubmit-errors.test.ts +598 -0
- package/test/batch-background.test.js +352 -0
- package/test/batch-blog-posts.test.js +293 -0
- package/test/blog-generation.test.js +390 -0
- package/test/browse-read.test.js +480 -0
- package/test/budget-tracking.test.ts +800 -0
- package/test/cache.test.ts +712 -0
- package/test/context-isolation.test.ts +687 -0
- package/test/core-functions.test.js +490 -0
- package/test/decide.test.js +260 -0
- package/test/define.test.js +232 -0
- package/test/e2e-bedrock-manual.js +136 -0
- package/test/e2e-bedrock.test.js +164 -0
- package/test/e2e-flex-gateway.js +131 -0
- package/test/e2e-flex-manual.js +156 -0
- package/test/e2e-flex.test.js +174 -0
- package/test/e2e-google-manual.js +150 -0
- package/test/e2e-google.test.js +181 -0
- package/test/embeddings.test.js +220 -0
- package/test/evals/define-function.eval.test.js +309 -0
- package/test/evals/deterministic.eval.test.ts +376 -0
- package/test/evals/primitives.eval.test.js +360 -0
- package/test/function-types.test.js +407 -0
- package/test/generate-core.test.js +213 -0
- package/test/generate.test.js +143 -0
- package/test/generic-order.test.ts +342 -0
- package/test/implicit-batch.test.js +326 -0
- package/test/json-parse-error-handling.test.ts +463 -0
- package/test/retry.test.ts +1016 -0
- package/test/schema.test.js +96 -0
- package/test/streaming.test.ts +316 -0
- package/test/tagged-templates.test.js +240 -0
- package/test/tool-orchestration.test.ts +770 -0
- package/vitest.config.js +39 -0
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for batch and background processing modes
|
|
3
|
+
*
|
|
4
|
+
* Batch mode: fn.batch([inputs]) - Process many inputs at ~50% discount
|
|
5
|
+
* Background mode: fn(..., { mode: 'background' }) - Returns job ID immediately
|
|
6
|
+
*/
|
|
7
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
8
|
+
// ============================================================================
|
|
9
|
+
// Mock implementations
|
|
10
|
+
// ============================================================================
|
|
11
|
+
const mockBatchProcess = vi.fn();
|
|
12
|
+
const mockBackgroundProcess = vi.fn();
|
|
13
|
+
/**
|
|
14
|
+
* Create a mock function with batch support
|
|
15
|
+
*/
|
|
16
|
+
function createMockFunctionWithBatch(defaultHandler) {
|
|
17
|
+
const fn = async (prompt, options) => {
|
|
18
|
+
if (options?.mode === 'background') {
|
|
19
|
+
return mockBackgroundProcess(prompt, options);
|
|
20
|
+
}
|
|
21
|
+
return defaultHandler(prompt);
|
|
22
|
+
};
|
|
23
|
+
// Add batch method
|
|
24
|
+
fn.batch = async (inputs) => {
|
|
25
|
+
return mockBatchProcess(inputs);
|
|
26
|
+
};
|
|
27
|
+
return fn;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Create a tagged template function with batch support
|
|
31
|
+
*/
|
|
32
|
+
function createMockTemplateFunctionWithBatch(defaultHandler) {
|
|
33
|
+
function fn(promptOrStrings, ...args) {
|
|
34
|
+
let prompt;
|
|
35
|
+
if (Array.isArray(promptOrStrings) && 'raw' in promptOrStrings) {
|
|
36
|
+
prompt = promptOrStrings.reduce((acc, str, i) => {
|
|
37
|
+
return acc + str + (args[i] ?? '');
|
|
38
|
+
}, '');
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
prompt = promptOrStrings;
|
|
42
|
+
}
|
|
43
|
+
return defaultHandler(prompt);
|
|
44
|
+
}
|
|
45
|
+
// Add batch method
|
|
46
|
+
fn.batch = async (inputs) => {
|
|
47
|
+
return mockBatchProcess(inputs);
|
|
48
|
+
};
|
|
49
|
+
return fn;
|
|
50
|
+
}
|
|
51
|
+
// ============================================================================
|
|
52
|
+
// Batch mode tests
|
|
53
|
+
// ============================================================================
|
|
54
|
+
describe('batch mode', () => {
|
|
55
|
+
beforeEach(() => {
|
|
56
|
+
mockBatchProcess.mockReset();
|
|
57
|
+
});
|
|
58
|
+
describe('write.batch()', () => {
|
|
59
|
+
it('processes multiple prompts in batch', async () => {
|
|
60
|
+
const write = createMockFunctionWithBatch(async () => 'Generated content');
|
|
61
|
+
const prompts = [
|
|
62
|
+
'blog post about TypeScript',
|
|
63
|
+
'blog post about React',
|
|
64
|
+
'blog post about Next.js',
|
|
65
|
+
];
|
|
66
|
+
mockBatchProcess.mockResolvedValue([
|
|
67
|
+
'TypeScript content...',
|
|
68
|
+
'React content...',
|
|
69
|
+
'Next.js content...',
|
|
70
|
+
]);
|
|
71
|
+
const results = await write.batch(prompts);
|
|
72
|
+
expect(mockBatchProcess).toHaveBeenCalledWith(prompts);
|
|
73
|
+
expect(results).toHaveLength(3);
|
|
74
|
+
});
|
|
75
|
+
it('processes object inputs with context', async () => {
|
|
76
|
+
const write = createMockFunctionWithBatch(async () => 'Generated content');
|
|
77
|
+
const brand = { voice: 'professional', audience: 'developers' };
|
|
78
|
+
const titles = ['Getting Started', 'Advanced Patterns', 'Best Practices'];
|
|
79
|
+
const inputs = titles.map(title => ({
|
|
80
|
+
title,
|
|
81
|
+
brand,
|
|
82
|
+
tone: 'technical',
|
|
83
|
+
}));
|
|
84
|
+
mockBatchProcess.mockResolvedValue([
|
|
85
|
+
'Getting Started content...',
|
|
86
|
+
'Advanced Patterns content...',
|
|
87
|
+
'Best Practices content...',
|
|
88
|
+
]);
|
|
89
|
+
const results = await write.batch(inputs);
|
|
90
|
+
expect(mockBatchProcess).toHaveBeenCalledWith(inputs);
|
|
91
|
+
expect(results).toHaveLength(3);
|
|
92
|
+
});
|
|
93
|
+
it('returns results in same order as inputs', async () => {
|
|
94
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
95
|
+
const inputs = ['first', 'second', 'third'];
|
|
96
|
+
mockBatchProcess.mockResolvedValue([
|
|
97
|
+
'Result for first',
|
|
98
|
+
'Result for second',
|
|
99
|
+
'Result for third',
|
|
100
|
+
]);
|
|
101
|
+
const results = await write.batch(inputs);
|
|
102
|
+
expect(results[0]).toContain('first');
|
|
103
|
+
expect(results[1]).toContain('second');
|
|
104
|
+
expect(results[2]).toContain('third');
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
describe('list.batch()', () => {
|
|
108
|
+
it('generates multiple lists in batch', async () => {
|
|
109
|
+
const list = createMockFunctionWithBatch(async () => ['item']);
|
|
110
|
+
mockBatchProcess.mockResolvedValue([
|
|
111
|
+
['TypeScript tip 1', 'TypeScript tip 2'],
|
|
112
|
+
['React tip 1', 'React tip 2'],
|
|
113
|
+
['Next.js tip 1', 'Next.js tip 2'],
|
|
114
|
+
]);
|
|
115
|
+
const results = await list.batch([
|
|
116
|
+
'3 TypeScript tips',
|
|
117
|
+
'3 React tips',
|
|
118
|
+
'3 Next.js tips',
|
|
119
|
+
]);
|
|
120
|
+
expect(results).toHaveLength(3);
|
|
121
|
+
expect(results[0]).toEqual(['TypeScript tip 1', 'TypeScript tip 2']);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
describe('code.batch()', () => {
|
|
125
|
+
it('generates multiple code snippets in batch', async () => {
|
|
126
|
+
const code = createMockFunctionWithBatch(async () => 'code');
|
|
127
|
+
mockBatchProcess.mockResolvedValue([
|
|
128
|
+
'function validateEmail(email) { ... }',
|
|
129
|
+
'function validatePhone(phone) { ... }',
|
|
130
|
+
'function validateUrl(url) { ... }',
|
|
131
|
+
]);
|
|
132
|
+
const results = await code.batch([
|
|
133
|
+
{ description: 'email validator', language: 'typescript' },
|
|
134
|
+
{ description: 'phone validator', language: 'typescript' },
|
|
135
|
+
{ description: 'url validator', language: 'typescript' },
|
|
136
|
+
]);
|
|
137
|
+
expect(results).toHaveLength(3);
|
|
138
|
+
expect(results[0]).toContain('validateEmail');
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
describe('batch with options', () => {
|
|
142
|
+
it('accepts batch-level options', async () => {
|
|
143
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
144
|
+
// Simulating batch with options
|
|
145
|
+
const mockBatchWithOptions = vi.fn().mockResolvedValue(['r1', 'r2']);
|
|
146
|
+
const inputs = ['prompt1', 'prompt2'];
|
|
147
|
+
const options = { model: 'claude-opus-4-5' };
|
|
148
|
+
await mockBatchWithOptions(inputs, options);
|
|
149
|
+
expect(mockBatchWithOptions).toHaveBeenCalledWith(inputs, options);
|
|
150
|
+
});
|
|
151
|
+
it('supports priority option for urgent batches', async () => {
|
|
152
|
+
const mockBatchWithPriority = vi.fn().mockResolvedValue(['result']);
|
|
153
|
+
await mockBatchWithPriority(['prompt'], { priority: 'high' });
|
|
154
|
+
expect(mockBatchWithPriority).toHaveBeenCalledWith(['prompt'], expect.objectContaining({ priority: 'high' }));
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
// ============================================================================
|
|
159
|
+
// Background mode tests
|
|
160
|
+
// ============================================================================
|
|
161
|
+
describe('background mode', () => {
|
|
162
|
+
beforeEach(() => {
|
|
163
|
+
mockBackgroundProcess.mockReset();
|
|
164
|
+
});
|
|
165
|
+
it('returns job ID immediately', async () => {
|
|
166
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
167
|
+
mockBackgroundProcess.mockResolvedValue({
|
|
168
|
+
jobId: 'job_abc123',
|
|
169
|
+
status: 'pending',
|
|
170
|
+
});
|
|
171
|
+
const job = await write('long form article', { mode: 'background' });
|
|
172
|
+
expect(mockBackgroundProcess).toHaveBeenCalledWith('long form article', expect.objectContaining({ mode: 'background' }));
|
|
173
|
+
expect(job).toHaveProperty('jobId');
|
|
174
|
+
expect(job.status).toBe('pending');
|
|
175
|
+
});
|
|
176
|
+
it('can check job status', async () => {
|
|
177
|
+
const mockGetJobStatus = vi.fn();
|
|
178
|
+
// Simulating job status check
|
|
179
|
+
mockGetJobStatus.mockResolvedValueOnce({ status: 'processing' });
|
|
180
|
+
mockGetJobStatus.mockResolvedValueOnce({ status: 'completed', result: 'Generated content' });
|
|
181
|
+
const status1 = await mockGetJobStatus('job_abc123');
|
|
182
|
+
expect(status1.status).toBe('processing');
|
|
183
|
+
const status2 = await mockGetJobStatus('job_abc123');
|
|
184
|
+
expect(status2.status).toBe('completed');
|
|
185
|
+
expect(status2.result).toBe('Generated content');
|
|
186
|
+
});
|
|
187
|
+
it('supports webhook callback', async () => {
|
|
188
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
189
|
+
mockBackgroundProcess.mockResolvedValue({
|
|
190
|
+
jobId: 'job_xyz789',
|
|
191
|
+
status: 'pending',
|
|
192
|
+
});
|
|
193
|
+
await write('content', {
|
|
194
|
+
mode: 'background',
|
|
195
|
+
webhook: 'https://myapp.com/webhooks/ai-complete',
|
|
196
|
+
});
|
|
197
|
+
expect(mockBackgroundProcess).toHaveBeenCalledWith('content', expect.objectContaining({
|
|
198
|
+
mode: 'background',
|
|
199
|
+
webhook: 'https://myapp.com/webhooks/ai-complete',
|
|
200
|
+
}));
|
|
201
|
+
});
|
|
202
|
+
it('supports polling for result', async () => {
|
|
203
|
+
// Simulating a poll function
|
|
204
|
+
const mockPollForResult = vi.fn();
|
|
205
|
+
mockPollForResult.mockImplementation(async (jobId) => {
|
|
206
|
+
// Simulate polling - would normally check periodically
|
|
207
|
+
return { status: 'completed', result: 'Final result' };
|
|
208
|
+
});
|
|
209
|
+
const result = await mockPollForResult('job_abc123');
|
|
210
|
+
expect(result.status).toBe('completed');
|
|
211
|
+
expect(result.result).toBe('Final result');
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
// ============================================================================
|
|
215
|
+
// Combined batch and background
|
|
216
|
+
// ============================================================================
|
|
217
|
+
describe('batch + background mode', () => {
|
|
218
|
+
it('can run batch in background', async () => {
|
|
219
|
+
const mockBatchBackground = vi.fn();
|
|
220
|
+
mockBatchBackground.mockResolvedValue({
|
|
221
|
+
jobId: 'batch_job_123',
|
|
222
|
+
status: 'pending',
|
|
223
|
+
inputCount: 100,
|
|
224
|
+
});
|
|
225
|
+
// Large batch job in background
|
|
226
|
+
const job = await mockBatchBackground(Array(100).fill('Generate content'), { mode: 'background' });
|
|
227
|
+
expect(job.jobId).toBe('batch_job_123');
|
|
228
|
+
expect(job.inputCount).toBe(100);
|
|
229
|
+
});
|
|
230
|
+
it('tracks progress of background batch', async () => {
|
|
231
|
+
const mockBatchProgress = vi.fn();
|
|
232
|
+
mockBatchProgress
|
|
233
|
+
.mockResolvedValueOnce({ status: 'processing', completed: 10, total: 100 })
|
|
234
|
+
.mockResolvedValueOnce({ status: 'processing', completed: 50, total: 100 })
|
|
235
|
+
.mockResolvedValueOnce({ status: 'completed', completed: 100, total: 100 });
|
|
236
|
+
const p1 = await mockBatchProgress('batch_job_123');
|
|
237
|
+
expect(p1.completed).toBe(10);
|
|
238
|
+
const p2 = await mockBatchProgress('batch_job_123');
|
|
239
|
+
expect(p2.completed).toBe(50);
|
|
240
|
+
const p3 = await mockBatchProgress('batch_job_123');
|
|
241
|
+
expect(p3.status).toBe('completed');
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
// ============================================================================
|
|
245
|
+
// Batch pricing and limits
|
|
246
|
+
// ============================================================================
|
|
247
|
+
describe('batch characteristics', () => {
|
|
248
|
+
it('batch provides cost savings (documentation test)', () => {
|
|
249
|
+
// This documents expected behavior
|
|
250
|
+
const batchInfo = {
|
|
251
|
+
discount: '50%',
|
|
252
|
+
turnaround: '24 hours max',
|
|
253
|
+
minBatchSize: 1,
|
|
254
|
+
maxBatchSize: 10000,
|
|
255
|
+
};
|
|
256
|
+
expect(batchInfo.discount).toBe('50%');
|
|
257
|
+
expect(batchInfo.turnaround).toBe('24 hours max');
|
|
258
|
+
});
|
|
259
|
+
it('batch handles errors gracefully', async () => {
|
|
260
|
+
const mockBatchWithErrors = vi.fn();
|
|
261
|
+
// Some items succeed, some fail
|
|
262
|
+
mockBatchWithErrors.mockResolvedValue({
|
|
263
|
+
results: [
|
|
264
|
+
{ index: 0, status: 'success', result: 'Content 1' },
|
|
265
|
+
{ index: 1, status: 'error', error: 'Content policy violation' },
|
|
266
|
+
{ index: 2, status: 'success', result: 'Content 3' },
|
|
267
|
+
],
|
|
268
|
+
summary: { succeeded: 2, failed: 1 },
|
|
269
|
+
});
|
|
270
|
+
const response = await mockBatchWithErrors(['p1', 'p2', 'p3']);
|
|
271
|
+
expect(response.summary.succeeded).toBe(2);
|
|
272
|
+
expect(response.summary.failed).toBe(1);
|
|
273
|
+
expect(response.results[1].status).toBe('error');
|
|
274
|
+
});
|
|
275
|
+
});
|
|
276
|
+
// ============================================================================
|
|
277
|
+
// Use cases from README
|
|
278
|
+
// ============================================================================
|
|
279
|
+
describe('batch use cases', () => {
|
|
280
|
+
beforeEach(() => {
|
|
281
|
+
mockBatchProcess.mockReset();
|
|
282
|
+
});
|
|
283
|
+
it('content generation at scale', async () => {
|
|
284
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
285
|
+
// Generate blog posts for 100 topics
|
|
286
|
+
const topics = Array(100)
|
|
287
|
+
.fill(null)
|
|
288
|
+
.map((_, i) => `Topic ${i + 1}`);
|
|
289
|
+
const inputs = topics.map(topic => ({
|
|
290
|
+
topic,
|
|
291
|
+
length: 'medium',
|
|
292
|
+
style: 'informative',
|
|
293
|
+
}));
|
|
294
|
+
mockBatchProcess.mockResolvedValue(topics.map(t => `Blog post about ${t}...`));
|
|
295
|
+
const posts = await write.batch(inputs);
|
|
296
|
+
expect(posts).toHaveLength(100);
|
|
297
|
+
});
|
|
298
|
+
it('product description generation', async () => {
|
|
299
|
+
const write = createMockFunctionWithBatch(async () => 'description');
|
|
300
|
+
const products = [
|
|
301
|
+
{ name: 'Widget Pro', category: 'tools', features: ['durable', 'lightweight'] },
|
|
302
|
+
{ name: 'Gadget Plus', category: 'electronics', features: ['wireless', 'rechargeable'] },
|
|
303
|
+
];
|
|
304
|
+
mockBatchProcess.mockResolvedValue([
|
|
305
|
+
'Widget Pro is a durable, lightweight tool...',
|
|
306
|
+
'Gadget Plus is a wireless, rechargeable electronic...',
|
|
307
|
+
]);
|
|
308
|
+
const descriptions = await write.batch(products.map(p => ({
|
|
309
|
+
prompt: `product description for ${p.name}`,
|
|
310
|
+
product: p,
|
|
311
|
+
})));
|
|
312
|
+
expect(descriptions).toHaveLength(2);
|
|
313
|
+
expect(descriptions[0]).toContain('Widget Pro');
|
|
314
|
+
});
|
|
315
|
+
it('code generation for multiple functions', async () => {
|
|
316
|
+
const code = createMockFunctionWithBatch(async () => 'code');
|
|
317
|
+
const functions = [
|
|
318
|
+
{ name: 'validateEmail', description: 'Validate email format' },
|
|
319
|
+
{ name: 'validatePhone', description: 'Validate phone number' },
|
|
320
|
+
{ name: 'validateUrl', description: 'Validate URL format' },
|
|
321
|
+
];
|
|
322
|
+
mockBatchProcess.mockResolvedValue(functions.map(f => `function ${f.name}(value) { ... }`));
|
|
323
|
+
const implementations = await code.batch(functions.map(f => ({
|
|
324
|
+
description: f.description,
|
|
325
|
+
functionName: f.name,
|
|
326
|
+
language: 'typescript',
|
|
327
|
+
})));
|
|
328
|
+
expect(implementations).toHaveLength(3);
|
|
329
|
+
functions.forEach((f, i) => {
|
|
330
|
+
expect(implementations[i]).toContain(f.name);
|
|
331
|
+
});
|
|
332
|
+
});
|
|
333
|
+
});
|
|
334
|
+
// ============================================================================
|
|
335
|
+
// Error handling
|
|
336
|
+
// ============================================================================
|
|
337
|
+
describe('batch error handling', () => {
|
|
338
|
+
beforeEach(() => {
|
|
339
|
+
mockBatchProcess.mockReset();
|
|
340
|
+
});
|
|
341
|
+
it('handles empty input array', async () => {
|
|
342
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
343
|
+
mockBatchProcess.mockResolvedValue([]);
|
|
344
|
+
const results = await write.batch([]);
|
|
345
|
+
expect(results).toEqual([]);
|
|
346
|
+
});
|
|
347
|
+
it('propagates batch-level errors', async () => {
|
|
348
|
+
const write = createMockFunctionWithBatch(async () => 'content');
|
|
349
|
+
mockBatchProcess.mockRejectedValue(new Error('Batch quota exceeded'));
|
|
350
|
+
await expect(write.batch(['prompt'])).rejects.toThrow('Batch quota exceeded');
|
|
351
|
+
});
|
|
352
|
+
});
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch Blog Post Generation Test
|
|
3
|
+
*
|
|
4
|
+
* Tests the batch processing workflow where:
|
|
5
|
+
* 1. list`10 blog post titles` executes immediately
|
|
6
|
+
* 2. The mapped write operations are deferred to a batch
|
|
7
|
+
* 3. The batch is submitted to the provider (OpenAI/Anthropic)
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```ts
|
|
11
|
+
* const titles = await list`10 blog post titles about building startups in 2026`
|
|
12
|
+
* const posts = titles.map(title => batch.add(write`blog post about ${title}`))
|
|
13
|
+
* const job = await batch.submit()
|
|
14
|
+
* const results = await batch.wait()
|
|
15
|
+
* ```
|
|
16
|
+
*/
|
|
17
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
18
|
+
import { createBatch, withBatch, } from '../src/batch-queue.js';
|
|
19
|
+
// Import memory adapter to register it
|
|
20
|
+
import '../src/batch/memory.js';
|
|
21
|
+
import { configureMemoryAdapter, clearBatches } from '../src/batch/memory.js';
|
|
22
|
+
// ============================================================================
|
|
23
|
+
// Mock Setup
|
|
24
|
+
// ============================================================================
|
|
25
|
+
// Mock the generate functions
|
|
26
|
+
vi.mock('../src/generate.js', () => ({
|
|
27
|
+
generateObject: vi.fn().mockImplementation(async ({ prompt, schema }) => {
|
|
28
|
+
// Simulate list generation
|
|
29
|
+
if (schema?.items) {
|
|
30
|
+
return {
|
|
31
|
+
object: {
|
|
32
|
+
items: [
|
|
33
|
+
'How AI is Revolutionizing Startup Fundraising in 2026',
|
|
34
|
+
'The Rise of Solo Founders: Building $10M ARR Companies Alone',
|
|
35
|
+
'Why Remote-First is Non-Negotiable for 2026 Startups',
|
|
36
|
+
'Sustainable Growth vs Hypergrowth: The 2026 Paradigm Shift',
|
|
37
|
+
'Building in Public: How Transparency Became a Competitive Advantage',
|
|
38
|
+
'The API-First Startup: Lessons from 2026 Unicorns',
|
|
39
|
+
'From Side Project to Series A: The 2026 Playbook',
|
|
40
|
+
'Climate Tech Startups: The Hottest Sector of 2026',
|
|
41
|
+
'The Death of Traditional MVPs: Ship Faster, Learn Faster',
|
|
42
|
+
'Community-Led Growth: The New GTM Strategy for 2026',
|
|
43
|
+
],
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
// Simulate blog post generation
|
|
48
|
+
if (prompt.includes('blog post about')) {
|
|
49
|
+
const titleMatch = prompt.match(/blog post about (.+)/);
|
|
50
|
+
const title = titleMatch?.[1] || 'Unknown Topic';
|
|
51
|
+
return {
|
|
52
|
+
object: {
|
|
53
|
+
text: `# ${title}\n\nThis is a comprehensive blog post about ${title}.\n\n## Introduction\n\nIn 2026, the startup landscape continues to evolve...\n\n## Key Takeaways\n\n1. Innovation is key\n2. Focus on customer value\n3. Build sustainable businesses\n\n## Conclusion\n\nThe future of startups is bright for those who adapt.`,
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return { object: { result: 'Generated content' } };
|
|
58
|
+
}),
|
|
59
|
+
generateText: vi.fn().mockImplementation(async ({ prompt }) => {
|
|
60
|
+
// Simulate blog post text generation
|
|
61
|
+
if (prompt.includes('blog post about')) {
|
|
62
|
+
const titleMatch = prompt.match(/blog post about (.+)/);
|
|
63
|
+
const title = titleMatch?.[1] || 'Unknown Topic';
|
|
64
|
+
return {
|
|
65
|
+
text: `# ${title}\n\nThis is a comprehensive blog post about ${title}.\n\n## Introduction\n\nIn 2026, the startup landscape continues to evolve rapidly. Entrepreneurs are finding new ways to build, scale, and succeed.\n\n## The State of Startups in 2026\n\nThe ecosystem has matured significantly. AI tools have become indispensable, funding patterns have shifted, and remote work is now the default.\n\n## Key Strategies for Success\n\n1. **Leverage AI Wisely** - Use AI as a multiplier, not a replacement\n2. **Build Community First** - Your early adopters are your growth engine\n3. **Focus on Unit Economics** - Hypergrowth without sustainability is dead\n4. **Embrace Transparency** - Building in public creates trust and accountability\n\n## Practical Steps\n\n- Start with a problem you deeply understand\n- Validate with paying customers, not surveys\n- Build the smallest thing that delivers value\n- Iterate based on real usage data\n\n## Conclusion\n\nBuilding a startup in 2026 requires a blend of traditional business fundamentals and modern tools. The founders who succeed will be those who can navigate this balance effectively.`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return { text: 'Generated text content' };
|
|
69
|
+
}),
|
|
70
|
+
}));
|
|
71
|
+
// ============================================================================
|
|
72
|
+
// Test Helpers
|
|
73
|
+
// ============================================================================
|
|
74
|
+
/**
|
|
75
|
+
* Simulate the list template function
|
|
76
|
+
*/
|
|
77
|
+
async function mockList(prompt) {
|
|
78
|
+
const { generateObject } = await import('../src/generate.js');
|
|
79
|
+
const result = await generateObject({
|
|
80
|
+
model: 'sonnet',
|
|
81
|
+
schema: { items: ['List items'] },
|
|
82
|
+
prompt,
|
|
83
|
+
});
|
|
84
|
+
return result.object.items;
|
|
85
|
+
}
|
|
86
|
+
// ============================================================================
|
|
87
|
+
// Tests
|
|
88
|
+
// ============================================================================
|
|
89
|
+
describe('Batch Blog Post Generation', () => {
|
|
90
|
+
beforeEach(() => {
|
|
91
|
+
vi.clearAllMocks();
|
|
92
|
+
clearBatches();
|
|
93
|
+
// Use default handler that calls the mock
|
|
94
|
+
configureMemoryAdapter({});
|
|
95
|
+
});
|
|
96
|
+
afterEach(() => {
|
|
97
|
+
clearBatches();
|
|
98
|
+
});
|
|
99
|
+
describe('list` immediate execution', () => {
|
|
100
|
+
it('list` executes immediately and returns titles', async () => {
|
|
101
|
+
const titles = await mockList('10 blog post titles about building startups in 2026');
|
|
102
|
+
expect(titles).toHaveLength(10);
|
|
103
|
+
expect(titles[0]).toBe('How AI is Revolutionizing Startup Fundraising in 2026');
|
|
104
|
+
expect(titles[9]).toBe('Community-Led Growth: The New GTM Strategy for 2026');
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
describe('batch processing workflow', () => {
|
|
108
|
+
it('creates batch queue and adds items', async () => {
|
|
109
|
+
const batch = createBatch({ provider: 'openai', model: 'gpt-4o' });
|
|
110
|
+
const titles = await mockList('10 blog post titles about building startups in 2026');
|
|
111
|
+
// Add each title to the batch
|
|
112
|
+
const items = titles.map((title) => batch.add(`Write a comprehensive blog post about: ${title}`, {
|
|
113
|
+
customId: title.slice(0, 50).replace(/\s+/g, '-').toLowerCase(),
|
|
114
|
+
}));
|
|
115
|
+
expect(batch.size).toBe(10);
|
|
116
|
+
expect(items).toHaveLength(10);
|
|
117
|
+
expect(items[0].status).toBe('pending');
|
|
118
|
+
});
|
|
119
|
+
it('submits batch and returns job info', async () => {
|
|
120
|
+
const batch = createBatch({ provider: 'openai', model: 'gpt-4o' });
|
|
121
|
+
const titles = await mockList('10 blog post titles about building startups in 2026');
|
|
122
|
+
titles.forEach((title) => batch.add(`Write a comprehensive blog post about: ${title}`));
|
|
123
|
+
const { job, completion } = await batch.submit();
|
|
124
|
+
expect(job.id).toMatch(/^batch_memory_/);
|
|
125
|
+
expect(job.provider).toBe('openai');
|
|
126
|
+
expect(job.totalItems).toBe(10);
|
|
127
|
+
expect(job.status).toBe('pending');
|
|
128
|
+
// Wait for completion
|
|
129
|
+
const results = await completion;
|
|
130
|
+
expect(results).toHaveLength(10);
|
|
131
|
+
});
|
|
132
|
+
it('waits for batch completion and returns results', async () => {
|
|
133
|
+
const batch = createBatch({ provider: 'openai', model: 'gpt-4o' });
|
|
134
|
+
const titles = await mockList('10 blog post titles about building startups in 2026');
|
|
135
|
+
titles.forEach((title) => batch.add(`Write a comprehensive blog post about: ${title}`));
|
|
136
|
+
await batch.submit();
|
|
137
|
+
const results = await batch.wait();
|
|
138
|
+
expect(results).toHaveLength(10);
|
|
139
|
+
expect(results.every((r) => r.status === 'completed')).toBe(true);
|
|
140
|
+
expect(results[0].result).toBeDefined();
|
|
141
|
+
});
|
|
142
|
+
it('processes items in order', async () => {
|
|
143
|
+
const batch = createBatch({ provider: 'openai' });
|
|
144
|
+
const titles = ['First', 'Second', 'Third'];
|
|
145
|
+
const items = titles.map((title, i) => batch.add(`Write about: ${title}`, { customId: `item_${i}` }));
|
|
146
|
+
await batch.submit();
|
|
147
|
+
const results = await batch.wait();
|
|
148
|
+
expect(results[0].id).toBe('item_0');
|
|
149
|
+
expect(results[1].id).toBe('item_1');
|
|
150
|
+
expect(results[2].id).toBe('item_2');
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
describe('withBatchQueue helper', () => {
|
|
154
|
+
it('provides convenient batch execution', async () => {
|
|
155
|
+
const titles = await mockList('10 blog post titles about building startups in 2026');
|
|
156
|
+
const results = await withBatch((batch) => titles.map((title) => batch.add(`Write a blog post about: ${title}`)), { provider: 'openai', model: 'gpt-4o' });
|
|
157
|
+
expect(results).toHaveLength(10);
|
|
158
|
+
expect(results.every((r) => r.status === 'completed')).toBe(true);
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
describe('batch status tracking', () => {
|
|
162
|
+
it('tracks completion progress', async () => {
|
|
163
|
+
const batch = createBatch({ provider: 'openai' });
|
|
164
|
+
batch.add('Write post 1');
|
|
165
|
+
batch.add('Write post 2');
|
|
166
|
+
batch.add('Write post 3');
|
|
167
|
+
const { job } = await batch.submit();
|
|
168
|
+
expect(job.completedItems).toBe(0);
|
|
169
|
+
// Wait for completion
|
|
170
|
+
await batch.wait();
|
|
171
|
+
const finalStatus = await batch.getStatus();
|
|
172
|
+
expect(finalStatus.status).toBe('completed');
|
|
173
|
+
expect(finalStatus.completedItems).toBe(3);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
describe('error handling', () => {
|
|
177
|
+
it('handles partial failures', async () => {
|
|
178
|
+
// Configure adapter to fail 30% of requests
|
|
179
|
+
configureMemoryAdapter({ failureRate: 0.3 });
|
|
180
|
+
const batch = createBatch({ provider: 'openai' });
|
|
181
|
+
for (let i = 0; i < 10; i++) {
|
|
182
|
+
batch.add(`Write post ${i}`);
|
|
183
|
+
}
|
|
184
|
+
await batch.submit();
|
|
185
|
+
const results = await batch.wait();
|
|
186
|
+
// Some should fail, some should succeed
|
|
187
|
+
const succeeded = results.filter((r) => r.status === 'completed').length;
|
|
188
|
+
const failed = results.filter((r) => r.status === 'failed').length;
|
|
189
|
+
expect(succeeded + failed).toBe(10);
|
|
190
|
+
// With 30% failure rate, expect roughly 3 failures (with some variance)
|
|
191
|
+
expect(failed).toBeGreaterThanOrEqual(0);
|
|
192
|
+
expect(failed).toBeLessThanOrEqual(10);
|
|
193
|
+
});
|
|
194
|
+
it('prevents adding items after submission', async () => {
|
|
195
|
+
const batch = createBatch({ provider: 'openai' });
|
|
196
|
+
batch.add('Write post 1');
|
|
197
|
+
await batch.submit();
|
|
198
|
+
expect(() => batch.add('Write post 2')).toThrow('Cannot add items to a submitted batch');
|
|
199
|
+
});
|
|
200
|
+
it('prevents double submission', async () => {
|
|
201
|
+
const batch = createBatch({ provider: 'openai' });
|
|
202
|
+
batch.add('Write post 1');
|
|
203
|
+
await batch.submit();
|
|
204
|
+
await expect(batch.submit()).rejects.toThrow('Batch has already been submitted');
|
|
205
|
+
});
|
|
206
|
+
it('prevents empty batch submission', async () => {
|
|
207
|
+
const batch = createBatch({ provider: 'openai' });
|
|
208
|
+
await expect(batch.submit()).rejects.toThrow('Cannot submit empty batch');
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
describe('batch with custom handler', () => {
|
|
212
|
+
it('uses custom handler for processing', async () => {
|
|
213
|
+
const customHandler = vi.fn().mockImplementation(async (item) => {
|
|
214
|
+
return `Custom result for: ${item.prompt}`;
|
|
215
|
+
});
|
|
216
|
+
configureMemoryAdapter({ handler: customHandler });
|
|
217
|
+
const batch = createBatch({ provider: 'openai' });
|
|
218
|
+
batch.add('Topic 1');
|
|
219
|
+
batch.add('Topic 2');
|
|
220
|
+
await batch.submit();
|
|
221
|
+
const results = await batch.wait();
|
|
222
|
+
expect(customHandler).toHaveBeenCalledTimes(2);
|
|
223
|
+
expect(results[0].result).toBe('Custom result for: Topic 1');
|
|
224
|
+
expect(results[1].result).toBe('Custom result for: Topic 2');
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
describe('full workflow: list → map → batch', () => {
|
|
228
|
+
it('executes the complete blog post generation workflow', async () => {
|
|
229
|
+
// Step 1: Get titles (executes immediately)
|
|
230
|
+
const titles = await mockList('10 blog post titles about building startups in 2026');
|
|
231
|
+
expect(titles).toHaveLength(10);
|
|
232
|
+
// Step 2: Create batch for blog posts (deferred)
|
|
233
|
+
const batch = createBatch({
|
|
234
|
+
provider: 'openai',
|
|
235
|
+
model: 'gpt-4o',
|
|
236
|
+
metadata: { task: 'blog-generation', topic: 'startups-2026' },
|
|
237
|
+
});
|
|
238
|
+
// Step 3: Map titles to batch items
|
|
239
|
+
const blogItems = titles.map((title, index) => batch.add(`Write a comprehensive blog post about: ${title}`, {
|
|
240
|
+
customId: `blog-${index}`,
|
|
241
|
+
metadata: { title },
|
|
242
|
+
}));
|
|
243
|
+
expect(batch.size).toBe(10);
|
|
244
|
+
expect(blogItems.every((item) => item.status === 'pending')).toBe(true);
|
|
245
|
+
// Step 4: Submit the batch
|
|
246
|
+
const { job, completion } = await batch.submit();
|
|
247
|
+
expect(job.id).toBeDefined();
|
|
248
|
+
expect(job.totalItems).toBe(10);
|
|
249
|
+
expect(batch.isSubmitted).toBe(true);
|
|
250
|
+
// Step 5: Wait for results
|
|
251
|
+
const results = await completion;
|
|
252
|
+
expect(results).toHaveLength(10);
|
|
253
|
+
expect(results.every((r) => r.status === 'completed')).toBe(true);
|
|
254
|
+
// Verify results have blog post content
|
|
255
|
+
for (const result of results) {
|
|
256
|
+
expect(result.result).toBeDefined();
|
|
257
|
+
expect(typeof result.result).toBe('string');
|
|
258
|
+
// Blog posts should have some content
|
|
259
|
+
expect(result.result.length).toBeGreaterThan(100);
|
|
260
|
+
}
|
|
261
|
+
// Verify items are updated after completion
|
|
262
|
+
expect(blogItems.every((item) => item.status === 'completed')).toBe(true);
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
describe('Provider-specific batch behavior', () => {
|
|
267
|
+
beforeEach(() => {
|
|
268
|
+
clearBatches();
|
|
269
|
+
configureMemoryAdapter({});
|
|
270
|
+
});
|
|
271
|
+
it('uses specified provider', async () => {
|
|
272
|
+
const openAIBatch = createBatch({ provider: 'openai' });
|
|
273
|
+
const anthropicBatch = createBatch({ provider: 'anthropic' });
|
|
274
|
+
openAIBatch.add('Test prompt');
|
|
275
|
+
anthropicBatch.add('Test prompt');
|
|
276
|
+
const { job: oaiJob } = await openAIBatch.submit();
|
|
277
|
+
const { job: antJob } = await anthropicBatch.submit();
|
|
278
|
+
// Memory adapter simulates OpenAI for all providers
|
|
279
|
+
expect(oaiJob.provider).toBe('openai');
|
|
280
|
+
expect(antJob.provider).toBe('openai');
|
|
281
|
+
});
|
|
282
|
+
it('respects model configuration', async () => {
|
|
283
|
+
const customHandler = vi.fn().mockResolvedValue('Result');
|
|
284
|
+
configureMemoryAdapter({ handler: customHandler });
|
|
285
|
+
const batch = createBatch({ provider: 'openai', model: 'gpt-4o-mini' });
|
|
286
|
+
batch.add('Test prompt');
|
|
287
|
+
await batch.submit();
|
|
288
|
+
await batch.wait();
|
|
289
|
+
// The model should be passed to the handler via batch options
|
|
290
|
+
// (memory adapter doesn't use it, but real adapters would)
|
|
291
|
+
expect(customHandler).toHaveBeenCalled();
|
|
292
|
+
});
|
|
293
|
+
});
|