mcp-rubber-duck 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/pricing-updater.md +111 -0
- package/.claude/commands/update-pricing.md +22 -0
- package/CHANGELOG.md +7 -0
- package/dist/config/types.d.ts +72 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/types.js +8 -0
- package/dist/config/types.js.map +1 -1
- package/dist/data/default-pricing.d.ts +18 -0
- package/dist/data/default-pricing.d.ts.map +1 -0
- package/dist/data/default-pricing.js +307 -0
- package/dist/data/default-pricing.js.map +1 -0
- package/dist/providers/enhanced-manager.d.ts +2 -1
- package/dist/providers/enhanced-manager.d.ts.map +1 -1
- package/dist/providers/enhanced-manager.js +20 -2
- package/dist/providers/enhanced-manager.js.map +1 -1
- package/dist/providers/manager.d.ts +3 -1
- package/dist/providers/manager.d.ts.map +1 -1
- package/dist/providers/manager.js +12 -1
- package/dist/providers/manager.js.map +1 -1
- package/dist/server.d.ts +2 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +35 -4
- package/dist/server.js.map +1 -1
- package/dist/services/pricing.d.ts +56 -0
- package/dist/services/pricing.d.ts.map +1 -0
- package/dist/services/pricing.js +124 -0
- package/dist/services/pricing.js.map +1 -0
- package/dist/services/usage.d.ts +48 -0
- package/dist/services/usage.d.ts.map +1 -0
- package/dist/services/usage.js +243 -0
- package/dist/services/usage.js.map +1 -0
- package/dist/tools/get-usage-stats.d.ts +8 -0
- package/dist/tools/get-usage-stats.d.ts.map +1 -0
- package/dist/tools/get-usage-stats.js +92 -0
- package/dist/tools/get-usage-stats.js.map +1 -0
- package/package.json +1 -1
- package/src/config/types.ts +51 -0
- package/src/data/default-pricing.ts +368 -0
- package/src/providers/enhanced-manager.ts +41 -4
- package/src/providers/manager.ts +22 -1
- package/src/server.ts +42 -4
- package/src/services/pricing.ts +155 -0
- package/src/services/usage.ts +293 -0
- package/src/tools/get-usage-stats.ts +109 -0
- package/tests/pricing.test.ts +335 -0
- package/tests/tools/get-usage-stats.test.ts +236 -0
- package/tests/usage.test.ts +661 -0
|
@@ -0,0 +1,661 @@
|
|
|
1
|
+
import { describe, it, expect, jest, beforeEach, afterEach } from '@jest/globals';
|
|
2
|
+
import { UsageService } from '../src/services/usage.js';
|
|
3
|
+
import { PricingService } from '../src/services/pricing.js';
|
|
4
|
+
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'fs';
|
|
5
|
+
import { join } from 'path';
|
|
6
|
+
import { tmpdir } from 'os';
|
|
7
|
+
|
|
8
|
+
// Mock logger to avoid console noise during tests
|
|
9
|
+
jest.mock('../src/utils/logger');
|
|
10
|
+
|
|
11
|
+
describe('UsageService', () => {
|
|
12
|
+
let tempDir: string;
|
|
13
|
+
let pricingService: PricingService;
|
|
14
|
+
let usageService: UsageService;
|
|
15
|
+
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
// Create a temporary directory for each test
|
|
18
|
+
tempDir = mkdtempSync(join(tmpdir(), 'usage-test-'));
|
|
19
|
+
|
|
20
|
+
// Create pricing service with test pricing
|
|
21
|
+
pricingService = new PricingService({
|
|
22
|
+
testprovider: {
|
|
23
|
+
'test-model': { inputPricePerMillion: 5, outputPricePerMillion: 15 },
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
// Create usage service with temp directory and no debounce for testing
|
|
28
|
+
usageService = new UsageService(pricingService, {
|
|
29
|
+
dataDir: tempDir,
|
|
30
|
+
debounceMs: 0, // Immediate writes for testing
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
afterEach(() => {
|
|
35
|
+
// Clean up
|
|
36
|
+
usageService.shutdown();
|
|
37
|
+
if (existsSync(tempDir)) {
|
|
38
|
+
rmSync(tempDir, { recursive: true });
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
describe('recordUsage', () => {
|
|
43
|
+
it('should create nested structure on first record', () => {
|
|
44
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
45
|
+
|
|
46
|
+
const stats = usageService.getStats('today');
|
|
47
|
+
expect(stats.usage['openai']).toBeDefined();
|
|
48
|
+
expect(stats.usage['openai']['gpt-4o']).toBeDefined();
|
|
49
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(1);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('should increment stats on subsequent records', () => {
|
|
53
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
54
|
+
usageService.recordUsage('openai', 'gpt-4o', 200, 100, false, false);
|
|
55
|
+
|
|
56
|
+
const stats = usageService.getStats('today');
|
|
57
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(2);
|
|
58
|
+
expect(stats.usage['openai']['gpt-4o'].promptTokens).toBe(300);
|
|
59
|
+
expect(stats.usage['openai']['gpt-4o'].completionTokens).toBe(150);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('should track cache hits', () => {
|
|
63
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, true, false);
|
|
64
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
65
|
+
|
|
66
|
+
const stats = usageService.getStats('today');
|
|
67
|
+
expect(stats.usage['openai']['gpt-4o'].cacheHits).toBe(1);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it('should track errors', () => {
|
|
71
|
+
usageService.recordUsage('openai', 'gpt-4o', 0, 0, false, true);
|
|
72
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
73
|
+
|
|
74
|
+
const stats = usageService.getStats('today');
|
|
75
|
+
expect(stats.usage['openai']['gpt-4o'].errors).toBe(1);
|
|
76
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(2);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('should track multiple providers separately', () => {
|
|
80
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
81
|
+
usageService.recordUsage('anthropic', 'claude-3', 200, 100, false, false);
|
|
82
|
+
|
|
83
|
+
const stats = usageService.getStats('today');
|
|
84
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(1);
|
|
85
|
+
expect(stats.usage['anthropic']['claude-3'].requests).toBe(1);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('should track multiple models separately', () => {
|
|
89
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
90
|
+
usageService.recordUsage('openai', 'gpt-4o-mini', 200, 100, false, false);
|
|
91
|
+
|
|
92
|
+
const stats = usageService.getStats('today');
|
|
93
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(1);
|
|
94
|
+
expect(stats.usage['openai']['gpt-4o-mini'].requests).toBe(1);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
describe('getStats', () => {
|
|
99
|
+
it('should return zero totals for empty usage', () => {
|
|
100
|
+
const stats = usageService.getStats('today');
|
|
101
|
+
|
|
102
|
+
expect(stats.totals.requests).toBe(0);
|
|
103
|
+
expect(stats.totals.promptTokens).toBe(0);
|
|
104
|
+
expect(stats.totals.completionTokens).toBe(0);
|
|
105
|
+
expect(stats.totals.cacheHits).toBe(0);
|
|
106
|
+
expect(stats.totals.errors).toBe(0);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('should aggregate totals correctly', () => {
|
|
110
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
111
|
+
usageService.recordUsage('anthropic', 'claude-3', 200, 100, true, false);
|
|
112
|
+
usageService.recordUsage('groq', 'llama', 50, 25, false, true);
|
|
113
|
+
|
|
114
|
+
const stats = usageService.getStats('today');
|
|
115
|
+
|
|
116
|
+
expect(stats.totals.requests).toBe(3);
|
|
117
|
+
expect(stats.totals.promptTokens).toBe(350);
|
|
118
|
+
expect(stats.totals.completionTokens).toBe(175);
|
|
119
|
+
expect(stats.totals.cacheHits).toBe(1);
|
|
120
|
+
expect(stats.totals.errors).toBe(1);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('should return correct period label', () => {
|
|
124
|
+
expect(usageService.getStats('today').period).toBe('today');
|
|
125
|
+
expect(usageService.getStats('7d').period).toBe('7d');
|
|
126
|
+
expect(usageService.getStats('30d').period).toBe('30d');
|
|
127
|
+
expect(usageService.getStats('all').period).toBe('all');
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it('should return correct date range for today', () => {
|
|
131
|
+
const stats = usageService.getStats('today');
|
|
132
|
+
|
|
133
|
+
// Both startDate and endDate should be the same for 'today'
|
|
134
|
+
expect(stats.startDate).toBe(stats.endDate);
|
|
135
|
+
// Should be a valid date format
|
|
136
|
+
expect(stats.startDate).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('should aggregate data across multiple days for 7d period', (done) => {
|
|
140
|
+
// Record some usage for today
|
|
141
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
142
|
+
|
|
143
|
+
// Wait for write, then manually add data for previous days
|
|
144
|
+
setTimeout(() => {
|
|
145
|
+
usageService.shutdown();
|
|
146
|
+
|
|
147
|
+
// Read and modify the data file to add historical data
|
|
148
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
149
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
150
|
+
|
|
151
|
+
// Add data for 3 days ago
|
|
152
|
+
const threeDaysAgo = new Date();
|
|
153
|
+
threeDaysAgo.setDate(threeDaysAgo.getDate() - 3);
|
|
154
|
+
const threeDaysAgoKey = threeDaysAgo.toISOString().split('T')[0];
|
|
155
|
+
|
|
156
|
+
data.daily[threeDaysAgoKey] = {
|
|
157
|
+
anthropic: {
|
|
158
|
+
'claude-3': { requests: 5, promptTokens: 500, completionTokens: 250, cacheHits: 1, errors: 0 },
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
writeFileSync(usageFile, JSON.stringify(data, null, 2));
|
|
163
|
+
|
|
164
|
+
// Create new service and check 7d aggregation
|
|
165
|
+
const newService = new UsageService(pricingService, {
|
|
166
|
+
dataDir: tempDir,
|
|
167
|
+
debounceMs: 0,
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
const stats = newService.getStats('7d');
|
|
171
|
+
|
|
172
|
+
// Should have both today's and 3-days-ago data
|
|
173
|
+
expect(stats.totals.requests).toBe(6); // 1 + 5
|
|
174
|
+
expect(stats.totals.promptTokens).toBe(600); // 100 + 500
|
|
175
|
+
expect(stats.usage['openai']).toBeDefined();
|
|
176
|
+
expect(stats.usage['anthropic']).toBeDefined();
|
|
177
|
+
|
|
178
|
+
newService.shutdown();
|
|
179
|
+
done();
|
|
180
|
+
}, 50);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it('should exclude data outside the requested period', (done) => {
|
|
184
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
185
|
+
|
|
186
|
+
setTimeout(() => {
|
|
187
|
+
usageService.shutdown();
|
|
188
|
+
|
|
189
|
+
// Add data for 10 days ago (outside 7d window)
|
|
190
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
191
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
192
|
+
|
|
193
|
+
const tenDaysAgo = new Date();
|
|
194
|
+
tenDaysAgo.setDate(tenDaysAgo.getDate() - 10);
|
|
195
|
+
const tenDaysAgoKey = tenDaysAgo.toISOString().split('T')[0];
|
|
196
|
+
|
|
197
|
+
data.daily[tenDaysAgoKey] = {
|
|
198
|
+
anthropic: {
|
|
199
|
+
'claude-3': { requests: 99, promptTokens: 9999, completionTokens: 9999, cacheHits: 0, errors: 0 },
|
|
200
|
+
},
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
writeFileSync(usageFile, JSON.stringify(data, null, 2));
|
|
204
|
+
|
|
205
|
+
const newService = new UsageService(pricingService, {
|
|
206
|
+
dataDir: tempDir,
|
|
207
|
+
debounceMs: 0,
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
// 7d should NOT include 10-day-old data
|
|
211
|
+
const stats7d = newService.getStats('7d');
|
|
212
|
+
expect(stats7d.totals.requests).toBe(1); // Only today's data
|
|
213
|
+
expect(stats7d.usage['anthropic']).toBeUndefined();
|
|
214
|
+
|
|
215
|
+
// But 30d SHOULD include it
|
|
216
|
+
const stats30d = newService.getStats('30d');
|
|
217
|
+
expect(stats30d.totals.requests).toBe(100); // 1 + 99
|
|
218
|
+
expect(stats30d.usage['anthropic']).toBeDefined();
|
|
219
|
+
|
|
220
|
+
newService.shutdown();
|
|
221
|
+
done();
|
|
222
|
+
}, 50);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it('should exclude future dates from stats', (done) => {
|
|
226
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
227
|
+
|
|
228
|
+
setTimeout(() => {
|
|
229
|
+
usageService.shutdown();
|
|
230
|
+
|
|
231
|
+
// Add data for a future date (should be excluded)
|
|
232
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
233
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
234
|
+
|
|
235
|
+
const tomorrow = new Date();
|
|
236
|
+
tomorrow.setDate(tomorrow.getDate() + 1);
|
|
237
|
+
const tomorrowKey = tomorrow.toISOString().split('T')[0];
|
|
238
|
+
|
|
239
|
+
data.daily[tomorrowKey] = {
|
|
240
|
+
future: {
|
|
241
|
+
'model': { requests: 999, promptTokens: 9999, completionTokens: 9999, cacheHits: 0, errors: 0 },
|
|
242
|
+
},
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
writeFileSync(usageFile, JSON.stringify(data, null, 2));
|
|
246
|
+
|
|
247
|
+
const newService = new UsageService(pricingService, {
|
|
248
|
+
dataDir: tempDir,
|
|
249
|
+
debounceMs: 0,
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
// Future data should be excluded from all periods
|
|
253
|
+
const statsAll = newService.getStats('all');
|
|
254
|
+
expect(statsAll.totals.requests).toBe(1); // Only today's data
|
|
255
|
+
expect(statsAll.usage['future']).toBeUndefined();
|
|
256
|
+
|
|
257
|
+
newService.shutdown();
|
|
258
|
+
done();
|
|
259
|
+
}, 50);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it('should include cost data when pricing available', () => {
|
|
263
|
+
usageService.recordUsage('testprovider', 'test-model', 1000, 500, false, false);
|
|
264
|
+
|
|
265
|
+
const stats = usageService.getStats('today');
|
|
266
|
+
|
|
267
|
+
// 1000 tokens at $5/M = $0.005 input
|
|
268
|
+
// 500 tokens at $15/M = $0.0075 output
|
|
269
|
+
// Total = $0.0125
|
|
270
|
+
expect(stats.totals.estimatedCostUSD).toBeCloseTo(0.0125, 6);
|
|
271
|
+
expect(stats.costByProvider?.['testprovider']).toBeCloseTo(0.0125, 6);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it('should omit cost data when pricing unavailable', () => {
|
|
275
|
+
usageService.recordUsage('unknown-provider', 'unknown-model', 1000, 500, false, false);
|
|
276
|
+
|
|
277
|
+
const stats = usageService.getStats('today');
|
|
278
|
+
|
|
279
|
+
expect(stats.totals.estimatedCostUSD).toBeUndefined();
|
|
280
|
+
expect(stats.costByProvider).toBeUndefined();
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
it('should handle mixed pricing availability', () => {
|
|
284
|
+
// Provider with pricing
|
|
285
|
+
usageService.recordUsage('testprovider', 'test-model', 1000, 500, false, false);
|
|
286
|
+
// Provider without pricing
|
|
287
|
+
usageService.recordUsage('unknown-provider', 'unknown-model', 2000, 1000, false, false);
|
|
288
|
+
|
|
289
|
+
const stats = usageService.getStats('today');
|
|
290
|
+
|
|
291
|
+
// Should still have cost data (only from priced provider)
|
|
292
|
+
expect(stats.totals.estimatedCostUSD).toBeCloseTo(0.0125, 6);
|
|
293
|
+
// Both providers should be in usage
|
|
294
|
+
expect(stats.usage['testprovider']).toBeDefined();
|
|
295
|
+
expect(stats.usage['unknown-provider']).toBeDefined();
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
it('should include cost data for free models (zero cost)', () => {
|
|
299
|
+
// Create service with free pricing
|
|
300
|
+
const freePricingService = new PricingService({
|
|
301
|
+
freeprovider: {
|
|
302
|
+
'free-model': { inputPricePerMillion: 0, outputPricePerMillion: 0 },
|
|
303
|
+
},
|
|
304
|
+
});
|
|
305
|
+
const freeUsageService = new UsageService(freePricingService, {
|
|
306
|
+
dataDir: tempDir,
|
|
307
|
+
debounceMs: 0,
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
freeUsageService.recordUsage('freeprovider', 'free-model', 1000, 500, false, false);
|
|
311
|
+
|
|
312
|
+
const stats = freeUsageService.getStats('today');
|
|
313
|
+
|
|
314
|
+
// Cost should be $0, but still present
|
|
315
|
+
expect(stats.totals.estimatedCostUSD).toBe(0);
|
|
316
|
+
expect(stats.costByProvider?.['freeprovider']).toBe(0);
|
|
317
|
+
|
|
318
|
+
freeUsageService.shutdown();
|
|
319
|
+
});
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
describe('persistence', () => {
|
|
323
|
+
it('should create usage file after recording', (done) => {
|
|
324
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
325
|
+
|
|
326
|
+
// Wait for debounced write
|
|
327
|
+
setTimeout(() => {
|
|
328
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
329
|
+
expect(existsSync(usageFile)).toBe(true);
|
|
330
|
+
done();
|
|
331
|
+
}, 50);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
it('should persist data that survives restart', (done) => {
|
|
335
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
336
|
+
|
|
337
|
+
// Wait for write, then create new service
|
|
338
|
+
setTimeout(() => {
|
|
339
|
+
usageService.shutdown();
|
|
340
|
+
|
|
341
|
+
const newService = new UsageService(pricingService, {
|
|
342
|
+
dataDir: tempDir,
|
|
343
|
+
debounceMs: 0,
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
const stats = newService.getStats('today');
|
|
347
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(1);
|
|
348
|
+
expect(stats.usage['openai']['gpt-4o'].promptTokens).toBe(100);
|
|
349
|
+
|
|
350
|
+
newService.shutdown();
|
|
351
|
+
done();
|
|
352
|
+
}, 50);
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
it('should flush pending writes on shutdown', () => {
|
|
356
|
+
// Use a longer debounce to ensure we test shutdown flushing
|
|
357
|
+
const serviceWithDebounce = new UsageService(pricingService, {
|
|
358
|
+
dataDir: tempDir,
|
|
359
|
+
debounceMs: 10000, // 10 second debounce
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
serviceWithDebounce.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
363
|
+
|
|
364
|
+
// File shouldn't exist yet (debounce pending)
|
|
365
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
366
|
+
|
|
367
|
+
// Shutdown should flush immediately
|
|
368
|
+
serviceWithDebounce.shutdown();
|
|
369
|
+
|
|
370
|
+
// Now file should exist
|
|
371
|
+
expect(existsSync(usageFile)).toBe(true);
|
|
372
|
+
|
|
373
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
374
|
+
expect(data.daily).toBeDefined();
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
it('should create data directory if it does not exist', () => {
|
|
378
|
+
const newDataDir = join(tempDir, 'nested', 'data', 'dir');
|
|
379
|
+
|
|
380
|
+
// Directory should not exist
|
|
381
|
+
expect(existsSync(newDataDir)).toBe(false);
|
|
382
|
+
|
|
383
|
+
// Create service - should create directory
|
|
384
|
+
const newService = new UsageService(pricingService, {
|
|
385
|
+
dataDir: newDataDir,
|
|
386
|
+
debounceMs: 0,
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
// Directory should now exist
|
|
390
|
+
expect(existsSync(newDataDir)).toBe(true);
|
|
391
|
+
|
|
392
|
+
newService.shutdown();
|
|
393
|
+
});
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
describe('clearData', () => {
|
|
397
|
+
it('should clear all usage data', (done) => {
|
|
398
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
399
|
+
|
|
400
|
+
let stats = usageService.getStats('today');
|
|
401
|
+
expect(stats.totals.requests).toBe(1);
|
|
402
|
+
|
|
403
|
+
usageService.clearData();
|
|
404
|
+
|
|
405
|
+
stats = usageService.getStats('today');
|
|
406
|
+
expect(stats.totals.requests).toBe(0);
|
|
407
|
+
|
|
408
|
+
done();
|
|
409
|
+
});
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
describe('getRawData', () => {
|
|
413
|
+
it('should return raw usage data', () => {
|
|
414
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
415
|
+
|
|
416
|
+
const rawData = usageService.getRawData();
|
|
417
|
+
|
|
418
|
+
expect(rawData.version).toBe(1);
|
|
419
|
+
expect(rawData.daily).toBeDefined();
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
it('should return a copy', () => {
|
|
423
|
+
const data1 = usageService.getRawData();
|
|
424
|
+
const data2 = usageService.getRawData();
|
|
425
|
+
|
|
426
|
+
expect(data1).not.toBe(data2);
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
it('should return a deep copy (mutations do not affect original)', () => {
|
|
430
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
431
|
+
|
|
432
|
+
const rawData = usageService.getRawData();
|
|
433
|
+
const today = Object.keys(rawData.daily)[0];
|
|
434
|
+
|
|
435
|
+
// Mutate the copy
|
|
436
|
+
rawData.daily[today]['openai']['gpt-4o'].requests = 999;
|
|
437
|
+
rawData.daily[today]['openai']['gpt-4o'].promptTokens = 999;
|
|
438
|
+
|
|
439
|
+
// Original should be unchanged
|
|
440
|
+
const stats = usageService.getStats('today');
|
|
441
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(1);
|
|
442
|
+
expect(stats.usage['openai']['gpt-4o'].promptTokens).toBe(100);
|
|
443
|
+
});
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
describe('edge cases', () => {
|
|
447
|
+
it('should handle zero token counts', () => {
|
|
448
|
+
usageService.recordUsage('openai', 'gpt-4o', 0, 0, false, false);
|
|
449
|
+
|
|
450
|
+
const stats = usageService.getStats('today');
|
|
451
|
+
expect(stats.usage['openai']['gpt-4o'].requests).toBe(1);
|
|
452
|
+
expect(stats.usage['openai']['gpt-4o'].promptTokens).toBe(0);
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it('should handle very large token counts', () => {
|
|
456
|
+
usageService.recordUsage('openai', 'gpt-4o', 10_000_000, 5_000_000, false, false);
|
|
457
|
+
|
|
458
|
+
const stats = usageService.getStats('today');
|
|
459
|
+
expect(stats.usage['openai']['gpt-4o'].promptTokens).toBe(10_000_000);
|
|
460
|
+
expect(stats.usage['openai']['gpt-4o'].completionTokens).toBe(5_000_000);
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
it('should handle special characters in provider/model names', () => {
|
|
464
|
+
usageService.recordUsage('my-provider', 'model/v2:latest', 100, 50, false, false);
|
|
465
|
+
|
|
466
|
+
const stats = usageService.getStats('today');
|
|
467
|
+
expect(stats.usage['my-provider']['model/v2:latest'].requests).toBe(1);
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
it('should handle empty strings for provider/model names', () => {
|
|
471
|
+
usageService.recordUsage('', '', 100, 50, false, false);
|
|
472
|
+
|
|
473
|
+
const stats = usageService.getStats('today');
|
|
474
|
+
expect(stats.usage['']['']).toBeDefined();
|
|
475
|
+
expect(stats.usage[''][''].requests).toBe(1);
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
it('should handle corrupted usage file gracefully', (done) => {
|
|
479
|
+
// Write corrupted data to file
|
|
480
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
481
|
+
writeFileSync(usageFile, 'not valid json');
|
|
482
|
+
|
|
483
|
+
// Create new service - should start fresh
|
|
484
|
+
const newService = new UsageService(pricingService, {
|
|
485
|
+
dataDir: tempDir,
|
|
486
|
+
debounceMs: 0,
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
const stats = newService.getStats('today');
|
|
490
|
+
expect(stats.totals.requests).toBe(0);
|
|
491
|
+
|
|
492
|
+
newService.shutdown();
|
|
493
|
+
done();
|
|
494
|
+
});
|
|
495
|
+
|
|
496
|
+
it('should handle invalid structure in usage file gracefully', (done) => {
|
|
497
|
+
// Write data with missing daily field
|
|
498
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
499
|
+
writeFileSync(usageFile, JSON.stringify({ version: 1 }));
|
|
500
|
+
|
|
501
|
+
// Create new service - should start fresh
|
|
502
|
+
const newService = new UsageService(pricingService, {
|
|
503
|
+
dataDir: tempDir,
|
|
504
|
+
debounceMs: 0,
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
const stats = newService.getStats('today');
|
|
508
|
+
expect(stats.totals.requests).toBe(0);
|
|
509
|
+
|
|
510
|
+
newService.shutdown();
|
|
511
|
+
done();
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
it('should handle malformed date keys in data file', (done) => {
|
|
515
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
516
|
+
|
|
517
|
+
setTimeout(() => {
|
|
518
|
+
usageService.shutdown();
|
|
519
|
+
|
|
520
|
+
// Add malformed date key to the data file
|
|
521
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
522
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
523
|
+
|
|
524
|
+
// Add entry with invalid date key
|
|
525
|
+
data.daily['not-a-date'] = {
|
|
526
|
+
badprovider: {
|
|
527
|
+
'bad-model': { requests: 999, promptTokens: 9999, completionTokens: 9999, cacheHits: 0, errors: 0 },
|
|
528
|
+
},
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
writeFileSync(usageFile, JSON.stringify(data, null, 2));
|
|
532
|
+
|
|
533
|
+
const newService = new UsageService(pricingService, {
|
|
534
|
+
dataDir: tempDir,
|
|
535
|
+
debounceMs: 0,
|
|
536
|
+
});
|
|
537
|
+
|
|
538
|
+
// Malformed date should be skipped, only valid data included
|
|
539
|
+
const stats = newService.getStats('all');
|
|
540
|
+
expect(stats.totals.requests).toBe(1); // Only today's valid data
|
|
541
|
+
expect(stats.usage['badprovider']).toBeUndefined();
|
|
542
|
+
|
|
543
|
+
newService.shutdown();
|
|
544
|
+
done();
|
|
545
|
+
}, 50);
|
|
546
|
+
});
|
|
547
|
+
|
|
548
|
+
it('should include data exactly at period boundary (6 days ago for 7d)', (done) => {
|
|
549
|
+
usageService.recordUsage('openai', 'gpt-4o', 100, 50, false, false);
|
|
550
|
+
|
|
551
|
+
setTimeout(() => {
|
|
552
|
+
usageService.shutdown();
|
|
553
|
+
|
|
554
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
555
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
556
|
+
|
|
557
|
+
// Add data for exactly 6 days ago (should be included in 7d)
|
|
558
|
+
const sixDaysAgo = new Date();
|
|
559
|
+
sixDaysAgo.setDate(sixDaysAgo.getDate() - 6);
|
|
560
|
+
const sixDaysAgoKey = `${sixDaysAgo.getFullYear()}-${String(sixDaysAgo.getMonth() + 1).padStart(2, '0')}-${String(sixDaysAgo.getDate()).padStart(2, '0')}`;
|
|
561
|
+
|
|
562
|
+
data.daily[sixDaysAgoKey] = {
|
|
563
|
+
boundary: {
|
|
564
|
+
'model': { requests: 10, promptTokens: 1000, completionTokens: 500, cacheHits: 0, errors: 0 },
|
|
565
|
+
},
|
|
566
|
+
};
|
|
567
|
+
|
|
568
|
+
// Add data for exactly 7 days ago (should NOT be included in 7d)
|
|
569
|
+
const sevenDaysAgo = new Date();
|
|
570
|
+
sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
|
|
571
|
+
const sevenDaysAgoKey = `${sevenDaysAgo.getFullYear()}-${String(sevenDaysAgo.getMonth() + 1).padStart(2, '0')}-${String(sevenDaysAgo.getDate()).padStart(2, '0')}`;
|
|
572
|
+
|
|
573
|
+
data.daily[sevenDaysAgoKey] = {
|
|
574
|
+
outside: {
|
|
575
|
+
'model': { requests: 99, promptTokens: 9999, completionTokens: 9999, cacheHits: 0, errors: 0 },
|
|
576
|
+
},
|
|
577
|
+
};
|
|
578
|
+
|
|
579
|
+
writeFileSync(usageFile, JSON.stringify(data, null, 2));
|
|
580
|
+
|
|
581
|
+
const newService = new UsageService(pricingService, {
|
|
582
|
+
dataDir: tempDir,
|
|
583
|
+
debounceMs: 0,
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
const stats = newService.getStats('7d');
|
|
587
|
+
|
|
588
|
+
// 6 days ago should be included
|
|
589
|
+
expect(stats.usage['boundary']).toBeDefined();
|
|
590
|
+
// 7 days ago should NOT be included (7d means last 7 days including today)
|
|
591
|
+
expect(stats.usage['outside']).toBeUndefined();
|
|
592
|
+
// Total: today (1) + 6 days ago (10) = 11
|
|
593
|
+
expect(stats.totals.requests).toBe(11);
|
|
594
|
+
|
|
595
|
+
newService.shutdown();
|
|
596
|
+
done();
|
|
597
|
+
}, 50);
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
it('should accumulate costs correctly across multiple days', (done) => {
|
|
601
|
+
// Use the test pricing service which has testprovider configured
|
|
602
|
+
usageService.recordUsage('testprovider', 'test-model', 1000, 500, false, false);
|
|
603
|
+
|
|
604
|
+
setTimeout(() => {
|
|
605
|
+
usageService.shutdown();
|
|
606
|
+
|
|
607
|
+
const usageFile = join(tempDir, 'usage.json');
|
|
608
|
+
const data = JSON.parse(readFileSync(usageFile, 'utf-8'));
|
|
609
|
+
|
|
610
|
+
// Add same provider/model for yesterday
|
|
611
|
+
const yesterday = new Date();
|
|
612
|
+
yesterday.setDate(yesterday.getDate() - 1);
|
|
613
|
+
const yesterdayKey = `${yesterday.getFullYear()}-${String(yesterday.getMonth() + 1).padStart(2, '0')}-${String(yesterday.getDate()).padStart(2, '0')}`;
|
|
614
|
+
|
|
615
|
+
data.daily[yesterdayKey] = {
|
|
616
|
+
testprovider: {
|
|
617
|
+
'test-model': { requests: 2, promptTokens: 2000, completionTokens: 1000, cacheHits: 0, errors: 0 },
|
|
618
|
+
},
|
|
619
|
+
};
|
|
620
|
+
|
|
621
|
+
writeFileSync(usageFile, JSON.stringify(data, null, 2));
|
|
622
|
+
|
|
623
|
+
const newService = new UsageService(pricingService, {
|
|
624
|
+
dataDir: tempDir,
|
|
625
|
+
debounceMs: 0,
|
|
626
|
+
});
|
|
627
|
+
|
|
628
|
+
const stats = newService.getStats('7d');
|
|
629
|
+
|
|
630
|
+
// Total tokens: today (1000+500) + yesterday (2000+1000) = 4500
|
|
631
|
+
expect(stats.totals.promptTokens).toBe(3000);
|
|
632
|
+
expect(stats.totals.completionTokens).toBe(1500);
|
|
633
|
+
|
|
634
|
+
// Cost calculation:
|
|
635
|
+
// Today: 1000 * $5/M + 500 * $15/M = $0.005 + $0.0075 = $0.0125
|
|
636
|
+
// Yesterday: 2000 * $5/M + 1000 * $15/M = $0.01 + $0.015 = $0.025
|
|
637
|
+
// Total: $0.0375
|
|
638
|
+
expect(stats.totals.estimatedCostUSD).toBeCloseTo(0.0375, 6);
|
|
639
|
+
expect(stats.costByProvider?.['testprovider']).toBeCloseTo(0.0375, 6);
|
|
640
|
+
|
|
641
|
+
newService.shutdown();
|
|
642
|
+
done();
|
|
643
|
+
}, 50);
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
it('should handle rapid successive recordUsage calls', () => {
|
|
647
|
+
// Record many usage entries in quick succession
|
|
648
|
+
for (let i = 0; i < 100; i++) {
|
|
649
|
+
usageService.recordUsage('openai', 'gpt-4o', 10, 5, i % 10 === 0, i % 20 === 0);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
const stats = usageService.getStats('today');
|
|
653
|
+
|
|
654
|
+
expect(stats.totals.requests).toBe(100);
|
|
655
|
+
expect(stats.totals.promptTokens).toBe(1000); // 100 * 10
|
|
656
|
+
expect(stats.totals.completionTokens).toBe(500); // 100 * 5
|
|
657
|
+
expect(stats.totals.cacheHits).toBe(10); // every 10th
|
|
658
|
+
expect(stats.totals.errors).toBe(5); // every 20th
|
|
659
|
+
});
|
|
660
|
+
});
|
|
661
|
+
});
|