cachegate 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +127 -112
- package/README.md +31 -13
- package/cache.js +72 -51
- package/embeddings.js +42 -32
- package/metrics.js +609 -556
- package/package.json +15 -1
- package/redisClient.js +55 -45
- package/router.js +254 -218
- package/semanticCache.js +159 -154
- package/server.js +282 -113
- package/.dockerignore +0 -11
- package/.gitattributes +0 -12
- package/.github/ISSUE_TEMPLATE/bug_report.md +0 -33
- package/.github/ISSUE_TEMPLATE/config.yml +0 -5
- package/.github/ISSUE_TEMPLATE/feature_request.md +0 -29
- package/.github/PULL_REQUEST_TEMPLATE.md +0 -25
- package/.github/workflows/test.yml +0 -63
- package/CODE_OF_CONDUCT.md +0 -66
- package/CONTRIBUTING.md +0 -94
- package/Dockerfile +0 -24
- package/OPEN_SOURCE_ROADMAP.md +0 -855
- package/ROADMAP.md +0 -281
- package/SECURITY.md +0 -39
- package/sync-oss-release.sh +0 -160
- package/test/auth-config.test.js +0 -27
- package/test/cache.test.js +0 -33
- package/test/embeddings.test.js +0 -24
- package/test/env-path.test.js +0 -41
- package/test/failover.test.js +0 -99
- package/test/metrics-postgres.test.js +0 -183
- package/test/metrics.test.js +0 -282
- package/test/router.test.js +0 -195
- package/test/semanticCache.test.js +0 -167
- package/test/server.test.js +0 -357
- package/test/streaming.test.js +0 -248
package/test/failover.test.js
DELETED
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
const { test } = require('node:test');
|
|
2
|
-
const assert = require('node:assert/strict');
|
|
3
|
-
const { isRetryableError, dispatchWithFailover } = require('../failover');
|
|
4
|
-
|
|
5
|
-
function httpError(message, status) {
|
|
6
|
-
return Object.assign(new Error(message), { status });
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
test('isRetryableError treats a bad request (400) as not retryable', () => {
|
|
10
|
-
assert.equal(isRetryableError(httpError('bad request', 400)), false);
|
|
11
|
-
});
|
|
12
|
-
|
|
13
|
-
test('isRetryableError treats an unknown model (404) as not retryable', () => {
|
|
14
|
-
assert.equal(isRetryableError(httpError('not found', 404)), false);
|
|
15
|
-
});
|
|
16
|
-
|
|
17
|
-
test('isRetryableError treats rate limits, server errors, and auth failures as retryable', () => {
|
|
18
|
-
assert.equal(isRetryableError(httpError('rate limited', 429)), true);
|
|
19
|
-
assert.equal(isRetryableError(httpError('server error', 500)), true);
|
|
20
|
-
assert.equal(isRetryableError(httpError('bad gateway', 502)), true);
|
|
21
|
-
assert.equal(isRetryableError(httpError('unauthorized', 401)), true);
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
test('isRetryableError treats a network failure with no status at all as retryable', () => {
|
|
25
|
-
assert.equal(isRetryableError(new Error('ECONNRESET')), true);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
test('dispatchWithFailover resolves on the first candidate when it succeeds, attempts=1', async () => {
|
|
29
|
-
const candidates = [{ provider: 'openai', model: 'gpt-4o-mini' }, { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }];
|
|
30
|
-
const dispatch = async (c) => ({ ok: true, provider: c.provider });
|
|
31
|
-
|
|
32
|
-
const outcome = await dispatchWithFailover(candidates, dispatch);
|
|
33
|
-
assert.equal(outcome.attempts, 1);
|
|
34
|
-
assert.equal(outcome.candidate.provider, 'openai');
|
|
35
|
-
assert.deepEqual(outcome.result, { ok: true, provider: 'openai' });
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
test('dispatchWithFailover moves to the next candidate on a retryable failure', async () => {
|
|
39
|
-
const candidates = [{ provider: 'openai', model: 'gpt-4o-mini' }, { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }];
|
|
40
|
-
const failedAttempts = [];
|
|
41
|
-
const dispatch = async (c) => {
|
|
42
|
-
if (c.provider === 'openai') throw httpError('rate limited', 429);
|
|
43
|
-
return { ok: true, provider: c.provider };
|
|
44
|
-
};
|
|
45
|
-
|
|
46
|
-
const outcome = await dispatchWithFailover(candidates, dispatch, (candidate, err) => {
|
|
47
|
-
failedAttempts.push({ candidate, message: err.message });
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
assert.equal(outcome.attempts, 2);
|
|
51
|
-
assert.equal(outcome.candidate.provider, 'anthropic');
|
|
52
|
-
assert.equal(failedAttempts.length, 1);
|
|
53
|
-
assert.equal(failedAttempts[0].candidate.provider, 'openai');
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
test('dispatchWithFailover does not retry a non-retryable (400) error, even with candidates left', async () => {
|
|
57
|
-
const candidates = [{ provider: 'openai', model: 'gpt-4o-mini' }, { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }];
|
|
58
|
-
let anthropicCalled = false;
|
|
59
|
-
const dispatch = async (c) => {
|
|
60
|
-
if (c.provider === 'openai') throw httpError('bad request', 400);
|
|
61
|
-
anthropicCalled = true;
|
|
62
|
-
return { ok: true };
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
await assert.rejects(
|
|
66
|
-
() => dispatchWithFailover(candidates, dispatch),
|
|
67
|
-
/bad request/
|
|
68
|
-
);
|
|
69
|
-
assert.equal(anthropicCalled, false);
|
|
70
|
-
});
|
|
71
|
-
|
|
72
|
-
test('dispatchWithFailover rethrows the last error when every candidate fails', async () => {
|
|
73
|
-
const candidates = [{ provider: 'openai', model: 'gpt-4o-mini' }, { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }];
|
|
74
|
-
const dispatch = async (c) => {
|
|
75
|
-
throw httpError(`${c.provider} down`, 503);
|
|
76
|
-
};
|
|
77
|
-
|
|
78
|
-
await assert.rejects(
|
|
79
|
-
() => dispatchWithFailover(candidates, dispatch),
|
|
80
|
-
/anthropic down/ // the LAST attempt's error, not the first
|
|
81
|
-
);
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
test('dispatchWithFailover calls onAttemptFailed for every failed candidate, including the last', async () => {
|
|
85
|
-
const candidates = [{ provider: 'openai', model: 'gpt-4o-mini' }, { provider: 'anthropic', model: 'claude-haiku-4-5-20251001' }];
|
|
86
|
-
const seen = [];
|
|
87
|
-
const dispatch = async (c) => {
|
|
88
|
-
throw httpError(`${c.provider} down`, 503);
|
|
89
|
-
};
|
|
90
|
-
|
|
91
|
-
await assert.rejects(() => dispatchWithFailover(candidates, dispatch, (candidate, err, isLastCandidate) => {
|
|
92
|
-
seen.push({ provider: candidate.provider, isLastCandidate });
|
|
93
|
-
}));
|
|
94
|
-
|
|
95
|
-
assert.deepEqual(seen, [
|
|
96
|
-
{ provider: 'openai', isLastCandidate: false },
|
|
97
|
-
{ provider: 'anthropic', isLastCandidate: true }
|
|
98
|
-
]);
|
|
99
|
-
});
|
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
// Postgres-backed persistence is OPT-IN (see metrics.js's own comment) -
|
|
2
|
-
// these tests exercise that path specifically, against a real local
|
|
3
|
-
// Postgres. Needs a real local server to actually run against; the
|
|
4
|
-
// `before()` hook below probes connectivity ONCE with a clear warning
|
|
5
|
-
// (`⚠️ Skipping metrics-postgres.test.js...`) rather than letting 20
|
|
6
|
-
// individual tests each fail with their own raw ECONNREFUSED - every
|
|
7
|
-
// test still reports as passing (each returns early via `pgAvailable`)
|
|
8
|
-
// so a machine without Postgres available doesn't see a red suite, it
|
|
9
|
-
// sees a skipped one. **Corrected 2026-08-29**: this comment previously
|
|
10
|
-
// claimed the opposite ("fails loudly... rather than silently
|
|
11
|
-
// skipping") - checked directly by actually stopping Postgres and
|
|
12
|
-
// running the suite, and that was never what the code below does. A CI
|
|
13
|
-
// environment without a real Postgres service configured will report
|
|
14
|
-
// green here while silently not exercising this file at all - the
|
|
15
|
-
// workflow's own Postgres service container is what makes these tests
|
|
16
|
-
// actually run, not just avoid failing.
|
|
17
|
-
//
|
|
18
|
-
// MEMOCODE_ROUTER_DATABASE_URL (not DATABASE_URL) is used here on
|
|
19
|
-
// purpose - keeps this suite from ever accidentally pointing at a real
|
|
20
|
-
// production database if that variable happened to be set in whatever
|
|
21
|
-
// environment runs the tests.
|
|
22
|
-
const { test, before, after, beforeEach } = require('node:test');
|
|
23
|
-
const assert = require('node:assert/strict');
|
|
24
|
-
|
|
25
|
-
const TEST_DATABASE_URL =
|
|
26
|
-
process.env.MEMOCODE_ROUTER_TEST_DATABASE_URL || 'postgres://postgres:dryrun@localhost:5432/router_metrics_dryrun';
|
|
27
|
-
|
|
28
|
-
function freshPgMetrics() {
|
|
29
|
-
process.env.MEMOCODE_ROUTER_DATABASE_URL = TEST_DATABASE_URL;
|
|
30
|
-
delete process.env.DATABASE_URL; // make sure only the test var is what's triggering Postgres mode
|
|
31
|
-
delete require.cache[require.resolve('../metrics')];
|
|
32
|
-
return require('../metrics');
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
async function flush() {
|
|
36
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
let pgAvailable = true;
|
|
40
|
-
let metrics;
|
|
41
|
-
|
|
42
|
-
before(async () => {
|
|
43
|
-
metrics = freshPgMetrics();
|
|
44
|
-
try {
|
|
45
|
-
// Prove connectivity up front with a clear failure message, rather
|
|
46
|
-
// than 20 individual tests each failing with their own ECONNREFUSED.
|
|
47
|
-
await metrics.readRecent(1);
|
|
48
|
-
} catch (err) {
|
|
49
|
-
pgAvailable = false;
|
|
50
|
-
console.warn(`⚠️ Skipping metrics-postgres.test.js - no local Postgres reachable at ${TEST_DATABASE_URL} (${err.message})`);
|
|
51
|
-
}
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
beforeEach(async () => {
|
|
55
|
-
if (!pgAvailable) return;
|
|
56
|
-
// Reset between tests so each one starts from a known-empty table,
|
|
57
|
-
// same isolation guarantee freshMetrics() gives the file-backed suite
|
|
58
|
-
// via a fresh temp directory per test.
|
|
59
|
-
const { Pool } = require('pg');
|
|
60
|
-
const pool = new Pool({ connectionString: TEST_DATABASE_URL });
|
|
61
|
-
await pool.query('DROP TABLE IF EXISTS router_metrics');
|
|
62
|
-
await pool.end();
|
|
63
|
-
metrics = freshPgMetrics();
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
after(async () => {
|
|
67
|
-
if (metrics) await metrics.closePostgresPoolForTests();
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
test('usingPostgres() is true once MEMOCODE_ROUTER_DATABASE_URL is set', () => {
|
|
71
|
-
if (!pgAvailable) return;
|
|
72
|
-
assert.equal(metrics.usingPostgres(), true);
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
test('record() + readRecent(): a real INSERT round-trips back out with the same fields', async () => {
|
|
76
|
-
if (!pgAvailable) return;
|
|
77
|
-
metrics.record({ provider: 'openai', model: 'gpt-4o-mini', latency_ms: 120, cost_usd: 0.001, cache_hit: false });
|
|
78
|
-
await flush();
|
|
79
|
-
|
|
80
|
-
const rows = await metrics.readRecent();
|
|
81
|
-
assert.equal(rows.length, 1);
|
|
82
|
-
assert.equal(rows[0].provider, 'openai');
|
|
83
|
-
assert.equal(rows[0].model, 'gpt-4o-mini');
|
|
84
|
-
assert.equal(rows[0].latency_ms, 120);
|
|
85
|
-
assert.equal(rows[0].cost_usd, 0.001);
|
|
86
|
-
assert.ok(rows[0].timestamp, 'expected a real timestamp column value back');
|
|
87
|
-
});
|
|
88
|
-
|
|
89
|
-
test('record() persists error + error_type together, exactly like the file backend', async () => {
|
|
90
|
-
if (!pgAvailable) return;
|
|
91
|
-
metrics.record({
|
|
92
|
-
provider: 'anthropic',
|
|
93
|
-
error: '401 {"type":"error","error":{"type":"authentication_error"}}',
|
|
94
|
-
error_type: 'authentication_error'
|
|
95
|
-
});
|
|
96
|
-
await flush();
|
|
97
|
-
|
|
98
|
-
const rows = await metrics.readRecent();
|
|
99
|
-
assert.equal(rows[0].error_type, 'authentication_error');
|
|
100
|
-
assert.ok(rows[0].error.includes('authentication_error'));
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
test('providerStats(): error rate, avg latency, and lastErrorType all compute correctly from real rows', async () => {
|
|
104
|
-
if (!pgAvailable) return;
|
|
105
|
-
metrics.record({ provider: 'openai', latency_ms: 100 });
|
|
106
|
-
metrics.record({ provider: 'openai', latency_ms: 200 });
|
|
107
|
-
metrics.record({ provider: 'openai', error: '429 {"error":{"type":"rate_limit_error"}}', error_type: 'rate_limit_error' });
|
|
108
|
-
await flush();
|
|
109
|
-
|
|
110
|
-
const stats = await metrics.providerStats();
|
|
111
|
-
assert.equal(stats.openai.sampleSize, 3);
|
|
112
|
-
assert.equal(stats.openai.errorRate, 1 / 3);
|
|
113
|
-
assert.equal(stats.openai.avgLatencyMs, 150);
|
|
114
|
-
assert.equal(stats.openai.lastErrorType, 'rate_limit_error');
|
|
115
|
-
});
|
|
116
|
-
|
|
117
|
-
test('rangeSummary(): cost, cache-hit breakdown, and by_provider all agree with what was actually inserted', async () => {
|
|
118
|
-
if (!pgAvailable) return;
|
|
119
|
-
metrics.record({ provider: 'openai', cost_usd: 0.001, cache_hit: false });
|
|
120
|
-
metrics.record({ provider: 'openai', cost_usd: 0, cache_hit: true, cache_type: 'exact' });
|
|
121
|
-
metrics.record({ provider: 'anthropic', error: '500 boom', error_type: 'unknown' });
|
|
122
|
-
await flush();
|
|
123
|
-
|
|
124
|
-
const summary = await metrics.rangeSummary(14);
|
|
125
|
-
assert.equal(summary.sample_size, 3);
|
|
126
|
-
assert.equal(summary.total_cost_usd, 0.001);
|
|
127
|
-
assert.equal(summary.cache_hit_rate.exact, 1 / 3);
|
|
128
|
-
assert.equal(summary.error_rate, 1 / 3);
|
|
129
|
-
assert.equal(summary.by_provider.openai.requests, 2);
|
|
130
|
-
assert.equal(summary.by_provider.anthropic.requests, 1);
|
|
131
|
-
assert.equal(summary.by_provider.anthropic.errorRate, 1);
|
|
132
|
-
});
|
|
133
|
-
|
|
134
|
-
test('rangeSummary(): a row older than the requested window is excluded', async () => {
|
|
135
|
-
if (!pgAvailable) return;
|
|
136
|
-
const { Pool } = require('pg');
|
|
137
|
-
const pool = new Pool({ connectionString: TEST_DATABASE_URL });
|
|
138
|
-
await pool.query(
|
|
139
|
-
`CREATE TABLE IF NOT EXISTS router_metrics (id BIGSERIAL PRIMARY KEY, ts TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
140
|
-
provider TEXT, model TEXT, requested_model TEXT, cache_hit BOOLEAN, cache_type TEXT,
|
|
141
|
-
latency_ms INTEGER, cost_usd DOUBLE PRECISION, error TEXT, error_type TEXT)`
|
|
142
|
-
);
|
|
143
|
-
await pool.query(`INSERT INTO router_metrics (ts, provider, cost_usd) VALUES (now() - interval '30 days', 'openai', 0.05)`);
|
|
144
|
-
await pool.end();
|
|
145
|
-
|
|
146
|
-
metrics.record({ provider: 'openai', cost_usd: 0.001 }); // today, should count
|
|
147
|
-
await flush();
|
|
148
|
-
|
|
149
|
-
const summary = await metrics.rangeSummary(14); // 30-day-old row is outside this window
|
|
150
|
-
assert.equal(summary.sample_size, 1);
|
|
151
|
-
assert.equal(summary.total_cost_usd, 0.001);
|
|
152
|
-
});
|
|
153
|
-
|
|
154
|
-
test('pruneOlderThan(): deletes only rows past the cutoff, returns their ids, leaves fresh rows alone', async () => {
|
|
155
|
-
if (!pgAvailable) return;
|
|
156
|
-
const { Pool } = require('pg');
|
|
157
|
-
const pool = new Pool({ connectionString: TEST_DATABASE_URL });
|
|
158
|
-
await pool.query(
|
|
159
|
-
`CREATE TABLE IF NOT EXISTS router_metrics (id BIGSERIAL PRIMARY KEY, ts TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
160
|
-
provider TEXT, model TEXT, requested_model TEXT, cache_hit BOOLEAN, cache_type TEXT,
|
|
161
|
-
latency_ms INTEGER, cost_usd DOUBLE PRECISION, error TEXT, error_type TEXT)`
|
|
162
|
-
);
|
|
163
|
-
await pool.query(`INSERT INTO router_metrics (ts, provider) VALUES (now() - interval '100 days', 'openai')`);
|
|
164
|
-
await pool.end();
|
|
165
|
-
|
|
166
|
-
metrics.record({ provider: 'anthropic' }); // fresh, should survive
|
|
167
|
-
await flush();
|
|
168
|
-
|
|
169
|
-
const deleted = await metrics.pruneOlderThan(30);
|
|
170
|
-
assert.equal(deleted.length, 1);
|
|
171
|
-
|
|
172
|
-
const remaining = await metrics.readRecent();
|
|
173
|
-
assert.equal(remaining.length, 1);
|
|
174
|
-
assert.equal(remaining[0].provider, 'anthropic');
|
|
175
|
-
});
|
|
176
|
-
|
|
177
|
-
test('classifyErrorType() is the exact same function regardless of storage backend (not duplicated/reimplemented)', () => {
|
|
178
|
-
if (!pgAvailable) return;
|
|
179
|
-
assert.equal(
|
|
180
|
-
metrics.classifyErrorType('401 {"error":{"type":"authentication_error"}}'),
|
|
181
|
-
'authentication_error'
|
|
182
|
-
);
|
|
183
|
-
});
|
package/test/metrics.test.js
DELETED
|
@@ -1,282 +0,0 @@
|
|
|
1
|
-
const { test } = require('node:test');
|
|
2
|
-
const assert = require('node:assert/strict');
|
|
3
|
-
const fs = require('fs');
|
|
4
|
-
const os = require('os');
|
|
5
|
-
const path = require('path');
|
|
6
|
-
|
|
7
|
-
function freshMetrics() {
|
|
8
|
-
const logPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'metrics-test-')), 'metrics.jsonl');
|
|
9
|
-
process.env.METRICS_LOG_PATH = logPath;
|
|
10
|
-
delete require.cache[require.resolve('../metrics')];
|
|
11
|
-
return require('../metrics');
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
async function flush() {
|
|
15
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
test('readRecent returns [] when no log file exists yet', async () => {
|
|
19
|
-
const metrics = freshMetrics();
|
|
20
|
-
const rows = await metrics.readRecent();
|
|
21
|
-
assert.deepEqual(rows, []);
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
test('record() writes a line that readRecent() can read back, with a timestamp added', async () => {
|
|
25
|
-
const metrics = freshMetrics();
|
|
26
|
-
metrics.record({ provider: 'openai', latency_ms: 120, cost_usd: 0.001 });
|
|
27
|
-
await flush();
|
|
28
|
-
|
|
29
|
-
const rows = await metrics.readRecent();
|
|
30
|
-
assert.equal(rows.length, 1);
|
|
31
|
-
assert.equal(rows[0].provider, 'openai');
|
|
32
|
-
assert.ok(rows[0].timestamp, 'expected record() to stamp a timestamp');
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
test('a malformed line in the log is skipped, not fatal', async () => {
|
|
36
|
-
const metrics = freshMetrics();
|
|
37
|
-
metrics.record({ provider: 'openai', latency_ms: 10 });
|
|
38
|
-
await flush();
|
|
39
|
-
fs.appendFileSync(metrics.currentLogPath(), 'not valid json\n');
|
|
40
|
-
metrics.record({ provider: 'anthropic', latency_ms: 20 });
|
|
41
|
-
await flush();
|
|
42
|
-
|
|
43
|
-
const rows = await metrics.readRecent();
|
|
44
|
-
assert.equal(rows.length, 2);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
test('currentLogPath() names a per-UTC-day file, and record() actually writes there', async () => {
|
|
48
|
-
const metrics = freshMetrics();
|
|
49
|
-
const today = new Date().toISOString().slice(0, 10);
|
|
50
|
-
assert.match(metrics.currentLogPath(), new RegExp(`metrics-${today}\\.jsonl$`));
|
|
51
|
-
|
|
52
|
-
metrics.record({ provider: 'openai' });
|
|
53
|
-
await flush();
|
|
54
|
-
assert.ok(fs.existsSync(metrics.currentLogPath()), "record() should write to today's rotated file");
|
|
55
|
-
});
|
|
56
|
-
|
|
57
|
-
test('listLogFiles() finds only files matching the rotation naming pattern, sorted ascending by date', async () => {
|
|
58
|
-
const metrics = freshMetrics();
|
|
59
|
-
fs.writeFileSync(path.join(metrics.DATA_DIR, 'metrics-2026-08-10.jsonl'), '');
|
|
60
|
-
fs.writeFileSync(path.join(metrics.DATA_DIR, 'metrics-2026-08-01.jsonl'), '');
|
|
61
|
-
fs.writeFileSync(path.join(metrics.DATA_DIR, 'not-a-metrics-file.txt'), '');
|
|
62
|
-
|
|
63
|
-
const files = await metrics.listLogFiles();
|
|
64
|
-
assert.deepEqual(files.map((f) => f.date), ['2026-08-01', '2026-08-10']);
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
test('readRecent() aggregates across multiple real day-files, newest data included first', async () => {
|
|
68
|
-
const metrics = freshMetrics();
|
|
69
|
-
fs.writeFileSync(
|
|
70
|
-
path.join(metrics.DATA_DIR, 'metrics-2026-08-01.jsonl'),
|
|
71
|
-
JSON.stringify({ timestamp: '2026-08-01T00:00:00.000Z', provider: 'anthropic', tag: 'old' }) + '\n'
|
|
72
|
-
);
|
|
73
|
-
fs.writeFileSync(
|
|
74
|
-
path.join(metrics.DATA_DIR, 'metrics-2026-08-02.jsonl'),
|
|
75
|
-
JSON.stringify({ timestamp: '2026-08-02T00:00:00.000Z', provider: 'openai', tag: 'newer' }) + '\n'
|
|
76
|
-
);
|
|
77
|
-
|
|
78
|
-
const rows = await metrics.readRecent(10);
|
|
79
|
-
assert.equal(rows.length, 2);
|
|
80
|
-
assert.deepEqual(rows.map((r) => r.tag), ['old', 'newer']); // chronological order preserved
|
|
81
|
-
});
|
|
82
|
-
|
|
83
|
-
test('readRecent(limit) stops opening older files once enough rows are collected', async () => {
|
|
84
|
-
const metrics = freshMetrics();
|
|
85
|
-
fs.writeFileSync(
|
|
86
|
-
path.join(metrics.DATA_DIR, 'metrics-2026-08-01.jsonl'),
|
|
87
|
-
JSON.stringify({ timestamp: '2026-08-01T00:00:00.000Z', tag: 'should-not-be-needed' }) + '\n'
|
|
88
|
-
);
|
|
89
|
-
fs.writeFileSync(
|
|
90
|
-
path.join(metrics.DATA_DIR, 'metrics-2026-08-02.jsonl'),
|
|
91
|
-
Array.from({ length: 5 }, (_, i) => JSON.stringify({ timestamp: '2026-08-02T00:00:00.000Z', tag: `row-${i}` })).join('\n') + '\n'
|
|
92
|
-
);
|
|
93
|
-
|
|
94
|
-
const rows = await metrics.readRecent(5);
|
|
95
|
-
assert.equal(rows.length, 5);
|
|
96
|
-
assert.ok(rows.every((r) => r.tag.startsWith('row-')), 'the older file should not have been needed to satisfy limit:5');
|
|
97
|
-
});
|
|
98
|
-
|
|
99
|
-
test('pruneOlderThan() deletes only day-files strictly older than the cutoff, and is never called by anything else in this module', async () => {
|
|
100
|
-
const metrics = freshMetrics();
|
|
101
|
-
const oldPath = path.join(metrics.DATA_DIR, 'metrics-2026-01-01.jsonl');
|
|
102
|
-
const recentPath = metrics.currentLogPath();
|
|
103
|
-
fs.writeFileSync(oldPath, '{}\n');
|
|
104
|
-
metrics.record({ provider: 'openai' });
|
|
105
|
-
await flush();
|
|
106
|
-
|
|
107
|
-
const deleted = await metrics.pruneOlderThan(30);
|
|
108
|
-
assert.deepEqual(deleted, [oldPath]);
|
|
109
|
-
assert.equal(fs.existsSync(oldPath), false);
|
|
110
|
-
assert.equal(fs.existsSync(recentPath), true, "pruneOlderThan should never touch today's file");
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
test('providerStats() computes error rate and average latency per provider', async () => {
|
|
114
|
-
const metrics = freshMetrics();
|
|
115
|
-
metrics.record({ provider: 'openai', latency_ms: 100 });
|
|
116
|
-
metrics.record({ provider: 'openai', latency_ms: 200 });
|
|
117
|
-
metrics.record({ provider: 'openai', error: 'boom' });
|
|
118
|
-
await flush();
|
|
119
|
-
|
|
120
|
-
const stats = await metrics.providerStats();
|
|
121
|
-
assert.equal(stats.openai.sampleSize, 3);
|
|
122
|
-
assert.equal(stats.openai.errorRate, 1 / 3);
|
|
123
|
-
assert.equal(stats.openai.avgLatencyMs, 150); // average of the two non-error latencies
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
test('providerStats() only considers the most recent windowSize entries', async () => {
|
|
127
|
-
const metrics = freshMetrics();
|
|
128
|
-
for (let i = 0; i < 5; i++) metrics.record({ provider: 'openai', error: 'boom' });
|
|
129
|
-
for (let i = 0; i < 5; i++) metrics.record({ provider: 'openai', latency_ms: 50 });
|
|
130
|
-
await flush();
|
|
131
|
-
|
|
132
|
-
const stats = await metrics.providerStats(5); // window covers only the second batch
|
|
133
|
-
assert.equal(stats.openai.errorRate, 0);
|
|
134
|
-
assert.equal(stats.openai.avgLatencyMs, 50);
|
|
135
|
-
});
|
|
136
|
-
|
|
137
|
-
test('classifyErrorType() reads Anthropic\'s own {error:{type}} shape (SDK message = "<status> <json>")', () => {
|
|
138
|
-
const metrics = freshMetrics();
|
|
139
|
-
const msg = '401 {"type":"error","error":{"type":"authentication_error","message":"API key is invalid."},"request_id":null}';
|
|
140
|
-
assert.equal(metrics.classifyErrorType(msg), 'authentication_error');
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
test('classifyErrorType() reads OpenAI\'s own {error:{type,code}} shape', () => {
|
|
144
|
-
const metrics = freshMetrics();
|
|
145
|
-
const msg = '429 {"error":{"message":"You exceeded your current quota.","type":"insufficient_quota","param":null,"code":"insufficient_quota"}}';
|
|
146
|
-
assert.equal(metrics.classifyErrorType(msg), 'insufficient_quota');
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
test('classifyErrorType() reads an invalid-key error identified only by code, not type', () => {
|
|
150
|
-
const metrics = freshMetrics();
|
|
151
|
-
const msg = '401 {"error":{"message":"Incorrect API key provided.","type":"invalid_request_error","param":null,"code":"invalid_api_key"}}';
|
|
152
|
-
assert.equal(metrics.classifyErrorType(msg), 'authentication_error');
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
test('classifyErrorType() falls back to keyword matching when the message has no parseable JSON body', () => {
|
|
156
|
-
const metrics = freshMetrics();
|
|
157
|
-
assert.equal(metrics.classifyErrorType('connect ECONNREFUSED - the key looks invalid'), 'authentication_error');
|
|
158
|
-
assert.equal(metrics.classifyErrorType('insufficient funds on this account'), 'insufficient_quota');
|
|
159
|
-
assert.equal(metrics.classifyErrorType('429 rate limit exceeded, slow down'), 'rate_limit_error');
|
|
160
|
-
assert.equal(metrics.classifyErrorType('connect ECONNREFUSED 127.0.0.1:4000'), 'unknown');
|
|
161
|
-
});
|
|
162
|
-
|
|
163
|
-
test('classifyErrorType() never throws on an empty/undefined message', () => {
|
|
164
|
-
const metrics = freshMetrics();
|
|
165
|
-
assert.equal(metrics.classifyErrorType(''), 'unknown');
|
|
166
|
-
assert.equal(metrics.classifyErrorType(undefined), 'unknown');
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
test('providerStats() reports the MOST RECENT error\'s classified type, not an earlier one still in the window', async () => {
|
|
170
|
-
const metrics = freshMetrics();
|
|
171
|
-
metrics.record({
|
|
172
|
-
provider: 'anthropic',
|
|
173
|
-
error: '429 {"type":"error","error":{"type":"rate_limit_error"}}',
|
|
174
|
-
error_type: 'rate_limit_error'
|
|
175
|
-
});
|
|
176
|
-
metrics.record({ provider: 'anthropic', latency_ms: 500 }); // recovered in between
|
|
177
|
-
metrics.record({
|
|
178
|
-
provider: 'anthropic',
|
|
179
|
-
error: '401 {"type":"error","error":{"type":"authentication_error"}}',
|
|
180
|
-
error_type: 'authentication_error'
|
|
181
|
-
});
|
|
182
|
-
await flush();
|
|
183
|
-
|
|
184
|
-
const stats = await metrics.providerStats();
|
|
185
|
-
assert.equal(stats.anthropic.lastErrorType, 'authentication_error');
|
|
186
|
-
assert.ok(stats.anthropic.lastErrorAt, 'expected a timestamp on the last error');
|
|
187
|
-
});
|
|
188
|
-
|
|
189
|
-
test('providerStats() classifies on the fly for an older record written before error_type existed', async () => {
|
|
190
|
-
const metrics = freshMetrics();
|
|
191
|
-
// No error_type field at all - simulates a log line from before this
|
|
192
|
-
// feature existed, still stored on disk after an upgrade.
|
|
193
|
-
metrics.record({ provider: 'openai', error: '401 {"error":{"type":"authentication_error"}}' });
|
|
194
|
-
await flush();
|
|
195
|
-
|
|
196
|
-
const stats = await metrics.providerStats();
|
|
197
|
-
assert.equal(stats.openai.lastErrorType, 'authentication_error');
|
|
198
|
-
});
|
|
199
|
-
|
|
200
|
-
test('providerStats() reports lastErrorType as null for a provider with no errors at all', async () => {
|
|
201
|
-
const metrics = freshMetrics();
|
|
202
|
-
metrics.record({ provider: 'openai', latency_ms: 100 });
|
|
203
|
-
await flush();
|
|
204
|
-
|
|
205
|
-
const stats = await metrics.providerStats();
|
|
206
|
-
assert.equal(stats.openai.lastErrorType, null);
|
|
207
|
-
assert.equal(stats.openai.lastErrorAt, null);
|
|
208
|
-
});
|
|
209
|
-
|
|
210
|
-
function isoDaysAgo(days) {
|
|
211
|
-
return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
function appendRaw(metrics, entry) {
|
|
215
|
-
// Deliberately appended to TODAY's physical file regardless of the
|
|
216
|
-
// entry's own (possibly historical) timestamp - this test is about
|
|
217
|
-
// rangeSummary()'s PER-ROW timestamp filtering, not file rotation
|
|
218
|
-
// (that has its own dedicated tests above). Today's file always
|
|
219
|
-
// passes rangeSummary()'s file-level date filter for any days >= 0,
|
|
220
|
-
// so the row-level filter is what's actually being exercised here.
|
|
221
|
-
fs.appendFileSync(metrics.currentLogPath(), JSON.stringify(entry) + '\n');
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
test('rangeSummary() buckets by calendar day and excludes rows outside the window', async () => {
|
|
225
|
-
const metrics = freshMetrics();
|
|
226
|
-
appendRaw(metrics, { timestamp: isoDaysAgo(0), provider: 'openai', cost_usd: 0.01, cache_hit: false });
|
|
227
|
-
appendRaw(metrics, { timestamp: isoDaysAgo(0), provider: 'openai', cost_usd: 0.02, cache_hit: true, cache_type: 'exact' });
|
|
228
|
-
appendRaw(metrics, { timestamp: isoDaysAgo(1), provider: 'anthropic', cost_usd: 0.05, cache_hit: true, cache_type: 'semantic' });
|
|
229
|
-
appendRaw(metrics, { timestamp: isoDaysAgo(30), provider: 'anthropic', cost_usd: 99, cache_hit: false }); // outside a 14-day window
|
|
230
|
-
|
|
231
|
-
const summary = await metrics.rangeSummary(14);
|
|
232
|
-
assert.equal(summary.sample_size, 3);
|
|
233
|
-
assert.equal(summary.total_cost_usd, 0.08);
|
|
234
|
-
assert.equal(summary.daily.length, 2); // two distinct calendar days, the 30-day-old row excluded
|
|
235
|
-
const todayBucket = summary.daily[summary.daily.length - 1];
|
|
236
|
-
assert.equal(todayBucket.requests, 2);
|
|
237
|
-
assert.equal(todayBucket.exact_hits, 1);
|
|
238
|
-
assert.equal(todayBucket.misses, 1);
|
|
239
|
-
});
|
|
240
|
-
|
|
241
|
-
test('rangeSummary() reports exact/semantic/combined hit rate and error rate separately', async () => {
|
|
242
|
-
const metrics = freshMetrics();
|
|
243
|
-
metrics.record({ provider: 'openai', cost_usd: 0.01, cache_hit: false });
|
|
244
|
-
metrics.record({ provider: 'openai', cost_usd: 0, cache_hit: true, cache_type: 'exact' });
|
|
245
|
-
metrics.record({ provider: 'openai', cost_usd: 0, cache_hit: true, cache_type: 'semantic' });
|
|
246
|
-
metrics.record({ provider: 'openai', error: 'boom' });
|
|
247
|
-
await flush();
|
|
248
|
-
|
|
249
|
-
const summary = await metrics.rangeSummary(14);
|
|
250
|
-
assert.equal(summary.sample_size, 4);
|
|
251
|
-
assert.equal(summary.cache_hit_rate.exact, 0.25);
|
|
252
|
-
assert.equal(summary.cache_hit_rate.semantic, 0.25);
|
|
253
|
-
assert.equal(summary.cache_hit_rate.combined, 0.5);
|
|
254
|
-
assert.equal(summary.error_rate, 0.25);
|
|
255
|
-
});
|
|
256
|
-
|
|
257
|
-
test('rangeSummary() computes per-provider cost, requests, error rate and avg latency', async () => {
|
|
258
|
-
const metrics = freshMetrics();
|
|
259
|
-
metrics.record({ provider: 'openai', cost_usd: 0.01, latency_ms: 100 });
|
|
260
|
-
metrics.record({ provider: 'openai', cost_usd: 0.02, latency_ms: 300 });
|
|
261
|
-
metrics.record({ provider: 'openai', error: 'boom' });
|
|
262
|
-
metrics.record({ provider: 'anthropic', cost_usd: 0.5, latency_ms: 200 });
|
|
263
|
-
await flush();
|
|
264
|
-
|
|
265
|
-
const summary = await metrics.rangeSummary(14);
|
|
266
|
-
assert.equal(summary.by_provider.openai.requests, 3);
|
|
267
|
-
assert.equal(summary.by_provider.openai.cost_usd, 0.03);
|
|
268
|
-
assert.equal(summary.by_provider.openai.errorRate, 1 / 3);
|
|
269
|
-
assert.equal(summary.by_provider.openai.avgLatencyMs, 200); // average of the two non-error latencies
|
|
270
|
-
assert.equal(summary.by_provider.anthropic.requests, 1);
|
|
271
|
-
assert.equal(summary.by_provider.anthropic.cost_usd, 0.5);
|
|
272
|
-
});
|
|
273
|
-
|
|
274
|
-
test('rangeSummary() on an empty log returns zeroed rates, not NaN or a crash', async () => {
|
|
275
|
-
const metrics = freshMetrics();
|
|
276
|
-
const summary = await metrics.rangeSummary(14);
|
|
277
|
-
assert.equal(summary.sample_size, 0);
|
|
278
|
-
assert.equal(summary.cache_hit_rate.combined, 0);
|
|
279
|
-
assert.equal(summary.error_rate, 0);
|
|
280
|
-
assert.deepEqual(summary.daily, []);
|
|
281
|
-
assert.deepEqual(summary.by_provider, {});
|
|
282
|
-
});
|