@drsamonline/telemetry 0.0.0-stage → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +26 -4
- package/src/cache.js +216 -0
- package/src/drift.js +174 -0
- package/src/embedding.js +66 -0
- package/src/history.js +177 -0
- package/src/index.js +5 -0
- package/src/ledger.js +404 -0
- package/README.md +0 -3
package/package.json
CHANGED
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drsamonline/telemetry",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.12.2",
|
|
4
|
+
"description": "Ledger, cache, aggregates, CSV.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "./src/index.js",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": "./src/index.js",
|
|
10
|
+
"./src/*": "./src/*"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"src",
|
|
14
|
+
"web",
|
|
15
|
+
"bin",
|
|
16
|
+
"LICENSE"
|
|
17
|
+
],
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git://github.com/drsamonline/sui.git",
|
|
21
|
+
"directory": "packages/telemetry"
|
|
22
|
+
},
|
|
23
|
+
"homepage": "https://github.com/drsamonline/sui#readme",
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public",
|
|
26
|
+
"registry": "https://registry.npmjs.org/"
|
|
27
|
+
}
|
|
28
|
+
}
|
package/src/cache.js
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response cache — two tiers.
|
|
3
|
+
*
|
|
4
|
+
* exact a hash of the whole request (combo, messages, sampling params)
|
|
5
|
+
* semantic tf-idf similarity between prompts, via the vendored ml-kit
|
|
6
|
+
*
|
|
7
|
+
* The semantic tier is **off by default** and that is a privacy decision, not a
|
|
8
|
+
* performance one: to compare prompts it has to keep recent prompt text in
|
|
9
|
+
* memory. Nothing is written to disk, and turning it on is an explicit choice
|
|
10
|
+
* (`server.cache.semantic.enabled`).
|
|
11
|
+
*
|
|
12
|
+
* A cache hit consumes no provider quota, which is the whole point: it is the
|
|
13
|
+
* cheapest possible route.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { createHash } from 'node:crypto';
|
|
17
|
+
import { loadKit } from '@drsamonline/kits';
|
|
18
|
+
import { cosine, embed } from './embedding.js';
|
|
19
|
+
|
|
20
|
+
export class ResponseCache {
|
|
21
|
+
/**
|
|
22
|
+
* @param {{
|
|
23
|
+
* ttlMs?: number, maxEntries?: number, enabled?: boolean, now?: () => number,
|
|
24
|
+
* semantic?: {enabled?: boolean, threshold?: number}
|
|
25
|
+
* }} [opts]
|
|
26
|
+
*/
|
|
27
|
+
constructor(opts = {}) {
|
|
28
|
+
this.ttlMs = opts.ttlMs ?? 900000;
|
|
29
|
+
this.maxEntries = opts.maxEntries ?? 500;
|
|
30
|
+
this.enabled = opts.enabled ?? true;
|
|
31
|
+
this.now = opts.now ?? Date.now;
|
|
32
|
+
this.semanticEnabled = Boolean(opts.semantic?.enabled);
|
|
33
|
+
// read from the nested block or the flat option, so both call styles behave
|
|
34
|
+
this.semanticThreshold = Number(opts.semantic?.threshold ?? opts.semanticThreshold ?? 0.98);
|
|
35
|
+
// B4.7: 'lexical' (tf-idf from ml-kit) or 'embedding' (hashed vectors, no model)
|
|
36
|
+
this.semanticTier = (opts.semantic?.tier ?? opts.semanticTier) === 'embedding' ? 'embedding' : 'lexical';
|
|
37
|
+
|
|
38
|
+
/** @type {Map<string, {value: any, expiresAt: number, promptText: string|null}>} */
|
|
39
|
+
this.map = new Map();
|
|
40
|
+
this.hits = 0;
|
|
41
|
+
this.semanticHits = 0;
|
|
42
|
+
this.misses = 0;
|
|
43
|
+
|
|
44
|
+
/** @type {{index: any, keys: string[]}|null} */
|
|
45
|
+
this._index = null;
|
|
46
|
+
this._indexDirty = true;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** @param {any} request */
|
|
50
|
+
static key(request) {
|
|
51
|
+
const basis = JSON.stringify({
|
|
52
|
+
combo: request.combo ?? null,
|
|
53
|
+
messages: request.messages,
|
|
54
|
+
temperature: request.temperature ?? null,
|
|
55
|
+
max_tokens: request.max_tokens ?? null,
|
|
56
|
+
top_p: request.top_p ?? null,
|
|
57
|
+
stop: request.stop ?? null,
|
|
58
|
+
model: request.model ?? null,
|
|
59
|
+
});
|
|
60
|
+
return createHash('sha256').update(basis).digest('hex').slice(0, 32);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* @param {string} key
|
|
65
|
+
* @param {string} [promptText] the query, for the semantic tier
|
|
66
|
+
* @returns {{value: any, kind: 'exact'|'semantic'}|null}
|
|
67
|
+
*/
|
|
68
|
+
get(key, promptText) {
|
|
69
|
+
if (!this.enabled) return null;
|
|
70
|
+
|
|
71
|
+
const hit = this.map.get(key);
|
|
72
|
+
if (hit) {
|
|
73
|
+
if (hit.expiresAt <= this.now()) {
|
|
74
|
+
this.map.delete(key);
|
|
75
|
+
this._indexDirty = true;
|
|
76
|
+
} else {
|
|
77
|
+
this.map.delete(key);
|
|
78
|
+
this.map.set(key, hit); // refresh LRU position
|
|
79
|
+
this.hits += 1;
|
|
80
|
+
return { value: hit.value, kind: 'exact' };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const semantic = this.#semanticLookup(promptText);
|
|
85
|
+
if (semantic) {
|
|
86
|
+
this.semanticHits += 1;
|
|
87
|
+
this.hits += 1;
|
|
88
|
+
return semantic;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
this.misses += 1;
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @param {string} key
|
|
97
|
+
* @param {any} value
|
|
98
|
+
* @param {{promptText?: string}} [meta]
|
|
99
|
+
*/
|
|
100
|
+
set(key, value, meta = {}) {
|
|
101
|
+
if (!this.enabled) return;
|
|
102
|
+
if (this.map.size >= this.maxEntries) {
|
|
103
|
+
const oldest = this.map.keys().next();
|
|
104
|
+
if (!oldest.done) {
|
|
105
|
+
this.map.delete(oldest.value);
|
|
106
|
+
this._indexDirty = true;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
this.map.set(key, {
|
|
110
|
+
value,
|
|
111
|
+
expiresAt: this.now() + this.ttlMs,
|
|
112
|
+
promptText: this.semanticEnabled ? meta.promptText ?? null : null,
|
|
113
|
+
vector: this.semanticEnabled && this.semanticTier === 'embedding' && meta.promptText ? embed(meta.promptText) : null,
|
|
114
|
+
});
|
|
115
|
+
this._indexDirty = true;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* @param {string|undefined} query
|
|
120
|
+
* @returns {{value: any, kind: 'semantic'}|null}
|
|
121
|
+
*/
|
|
122
|
+
#semanticLookup(query) {
|
|
123
|
+
if (!this.semanticEnabled || !query) return null;
|
|
124
|
+
|
|
125
|
+
// B4.7: two tiers behind the same seam. Lexical is tf-idf from ml-kit;
|
|
126
|
+
// embedding is hashed vectors, which cost no model and no quota.
|
|
127
|
+
if (this.semanticTier === 'embedding') {
|
|
128
|
+
const queryVector = embed(query);
|
|
129
|
+
let best = null;
|
|
130
|
+
let bestScore = 0;
|
|
131
|
+
for (const [key, entry] of this.map) {
|
|
132
|
+
if (entry.expiresAt <= this.now() || !entry.promptText) continue;
|
|
133
|
+
const score = cosine(queryVector, entry.vector ?? (entry.vector = embed(entry.promptText)));
|
|
134
|
+
if (score > bestScore) {
|
|
135
|
+
bestScore = score;
|
|
136
|
+
best = { key, entry };
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (best && bestScore >= this.semanticThreshold) {
|
|
140
|
+
return { value: best.entry.value, kind: 'embedding', score: Math.round(bestScore * 1000) / 1000 };
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const index = this.#ensureIndex();
|
|
146
|
+
if (!index || !index.keys.length) return null;
|
|
147
|
+
|
|
148
|
+
const hits = index.search(query, 1);
|
|
149
|
+
if (!hits.length) return null;
|
|
150
|
+
const top = hits[0];
|
|
151
|
+
if (top.score < this.semanticThreshold) return null;
|
|
152
|
+
|
|
153
|
+
const key = index.keys[top.i ?? top.index];
|
|
154
|
+
const entry = key ? this.map.get(key) : null;
|
|
155
|
+
if (!entry || entry.expiresAt <= this.now()) return null;
|
|
156
|
+
return { value: entry.value, kind: 'semantic', score: Math.round(top.score * 1000) / 1000 };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Build (or reuse) the tf-idf index over the cached prompts. */
|
|
160
|
+
#ensureIndex() {
|
|
161
|
+
if (!this._indexDirty && this._index) return this._index;
|
|
162
|
+
const keys = [];
|
|
163
|
+
const docs = [];
|
|
164
|
+
for (const [key, entry] of this.map) {
|
|
165
|
+
if (!entry.promptText) continue;
|
|
166
|
+
keys.push(key);
|
|
167
|
+
docs.push(entry.promptText);
|
|
168
|
+
}
|
|
169
|
+
if (!docs.length) {
|
|
170
|
+
this._index = { index: null, keys: [] };
|
|
171
|
+
this._indexDirty = false;
|
|
172
|
+
return this._index;
|
|
173
|
+
}
|
|
174
|
+
const MLKit = loadKit('ml-kit');
|
|
175
|
+
this._index = { index: MLKit.tfidf(docs), keys, search: (q, k) => MLKit.search(this._index.index, q, k) };
|
|
176
|
+
this._indexDirty = false;
|
|
177
|
+
return this._index;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
stats() {
|
|
181
|
+
const total = this.hits + this.misses;
|
|
182
|
+
return {
|
|
183
|
+
enabled: this.enabled,
|
|
184
|
+
entries: this.map.size,
|
|
185
|
+
hits: this.hits,
|
|
186
|
+
semanticHits: this.semanticHits,
|
|
187
|
+
misses: this.misses,
|
|
188
|
+
hitRate: total ? Math.round((this.hits / total) * 100) / 100 : 0,
|
|
189
|
+
semantic: { enabled: this.semanticEnabled, threshold: this.semanticThreshold },
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
clear() {
|
|
194
|
+
this.map.clear();
|
|
195
|
+
this._index = null;
|
|
196
|
+
this._indexDirty = true;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Drop expired entries so a long-running gateway's memory is bounded by the
|
|
201
|
+
* TTL, not by the number of prompts it has ever seen. Called by the
|
|
202
|
+
* cache_sweep worker.
|
|
203
|
+
*/
|
|
204
|
+
sweep() {
|
|
205
|
+
const now = this.now();
|
|
206
|
+
let removed = 0;
|
|
207
|
+
for (const [key, entry] of this.map.entries()) {
|
|
208
|
+
if (entry.expiresAt <= now) {
|
|
209
|
+
this.map.delete(key);
|
|
210
|
+
removed += 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (removed) this._indexDirty = true;
|
|
214
|
+
return { removed, remaining: this.map.size };
|
|
215
|
+
}
|
|
216
|
+
}
|
package/src/drift.js
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @drsamonline/telemetry/drift — B3.7, detecting that a target changed behaviour.
|
|
3
|
+
*
|
|
4
|
+
* Two standard detectors, because they answer different questions:
|
|
5
|
+
*
|
|
6
|
+
* EWMA "where is it now?" — a smoothed level, compared to a baseline.
|
|
7
|
+
* Fast to react, but a single bad minute can trigger it.
|
|
8
|
+
* CUSUM "how long has it been off?" — accumulates small deviations in both
|
|
9
|
+
* directions, so a slow 5% slide is caught even though no single
|
|
10
|
+
* sample looks wrong. This is the one a breaker cannot see.
|
|
11
|
+
*
|
|
12
|
+
* Both are deterministic and cheap: a few multiplications per sample. Neither
|
|
13
|
+
* needs a model, a window buffer, or a second request.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** A one-sided CUSUM pair plus an EWMA, per series. */
|
|
17
|
+
export class DriftMonitor {
|
|
18
|
+
/**
|
|
19
|
+
* @param {{alpha?: number, slack?: number, threshold?: number, baselineSamples?: number}} [opts]
|
|
20
|
+
*/
|
|
21
|
+
constructor(opts = {}) {
|
|
22
|
+
this.alpha = Number(opts.alpha ?? 0.2);
|
|
23
|
+
// slack is in standard deviations of the baseline: noise inside this band is
|
|
24
|
+
// not drift, it is variance, and treating it as drift is how alerting dies
|
|
25
|
+
this.slack = Number(opts.slack ?? 0.5);
|
|
26
|
+
this.threshold = Number(opts.threshold ?? 5);
|
|
27
|
+
this.baselineSamples = Number(opts.baselineSamples ?? 10);
|
|
28
|
+
// A perfect baseline has a standard deviation of zero, which makes every
|
|
29
|
+
// wobble infinitely significant. Floor it at a small fraction of the level.
|
|
30
|
+
this.minSdRatio = Number(opts.minSdRatio ?? 0.02);
|
|
31
|
+
|
|
32
|
+
/** @type {Map<string, any>} */
|
|
33
|
+
this.series = new Map();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
#state(key) {
|
|
37
|
+
let state = this.series.get(key);
|
|
38
|
+
if (!state) {
|
|
39
|
+
state = {
|
|
40
|
+
ewma: null,
|
|
41
|
+
sum: 0,
|
|
42
|
+
sumSq: 0,
|
|
43
|
+
count: 0,
|
|
44
|
+
baselineMean: null,
|
|
45
|
+
baselineSd: null,
|
|
46
|
+
cusumUp: 0,
|
|
47
|
+
cusumDown: 0,
|
|
48
|
+
alarm: null,
|
|
49
|
+
latched: false,
|
|
50
|
+
samples: 0,
|
|
51
|
+
};
|
|
52
|
+
this.series.set(key, state);
|
|
53
|
+
}
|
|
54
|
+
return state;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @param {string} key
|
|
59
|
+
* @param {number} value
|
|
60
|
+
*/
|
|
61
|
+
record(key, value) {
|
|
62
|
+
const state = this.#state(key);
|
|
63
|
+
const x = Number(value);
|
|
64
|
+
if (!Number.isFinite(x)) return this.snapshot(key);
|
|
65
|
+
|
|
66
|
+
state.samples += 1;
|
|
67
|
+
state.ewma = state.ewma === null ? x : this.alpha * x + (1 - this.alpha) * state.ewma;
|
|
68
|
+
|
|
69
|
+
// Build the baseline from the first N samples, then freeze it. A baseline that
|
|
70
|
+
// keeps moving with the drift can never detect the drift.
|
|
71
|
+
if (state.count < this.baselineSamples) {
|
|
72
|
+
state.count += 1;
|
|
73
|
+
state.sum += x;
|
|
74
|
+
state.sumSq += x * x;
|
|
75
|
+
if (state.count === this.baselineSamples) {
|
|
76
|
+
state.baselineMean = state.sum / state.count;
|
|
77
|
+
const variance = Math.max(0, state.sumSq / state.count - state.baselineMean ** 2);
|
|
78
|
+
const sd = Math.sqrt(variance);
|
|
79
|
+
state.baselineSd = Math.max(sd, Math.abs(state.baselineMean) * this.minSdRatio, 1e-6);
|
|
80
|
+
}
|
|
81
|
+
return this.snapshot(key);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const slack = this.slack * state.baselineSd;
|
|
85
|
+
const deviation = x - state.baselineMean;
|
|
86
|
+
state.cusumUp = Math.max(0, state.cusumUp + deviation - slack);
|
|
87
|
+
state.cusumDown = Math.max(0, state.cusumDown - deviation - slack);
|
|
88
|
+
|
|
89
|
+
if (state.cusumUp > this.threshold || state.cusumDown > this.threshold) {
|
|
90
|
+
const direction = state.cusumUp > this.threshold ? 'up' : 'down';
|
|
91
|
+
const cusum = direction === 'up' ? state.cusumUp : state.cusumDown;
|
|
92
|
+
state.alarm = { direction, cusum: Math.round(cusum * 100) / 100, at: state.samples };
|
|
93
|
+
// A signalled CUSUM restarts, or it grows without bound and stops meaning
|
|
94
|
+
// anything. The alarm stays latched until the level itself comes back.
|
|
95
|
+
state.cusumUp = 0;
|
|
96
|
+
state.cusumDown = 0;
|
|
97
|
+
state.latched = true;
|
|
98
|
+
} else if (state.latched) {
|
|
99
|
+
// Recovered when the smoothed level is back within two standard deviations
|
|
100
|
+
// of the baseline, not merely when one sample looks normal.
|
|
101
|
+
const back = Math.abs(state.ewma - state.baselineMean) <= 2 * state.baselineSd;
|
|
102
|
+
if (back) {
|
|
103
|
+
state.latched = false;
|
|
104
|
+
state.alarm = null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return this.snapshot(key);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** @param {string} key */
|
|
112
|
+
snapshot(key) {
|
|
113
|
+
const state = this.series.get(key);
|
|
114
|
+
if (!state) return null;
|
|
115
|
+
return {
|
|
116
|
+
key,
|
|
117
|
+
samples: state.samples,
|
|
118
|
+
ewma: state.ewma === null ? null : Math.round(state.ewma * 1000) / 1000,
|
|
119
|
+
baseline: state.baselineMean === null ? null : Math.round(state.baselineMean * 1000) / 1000,
|
|
120
|
+
sd: state.baselineSd === null ? null : Math.round(state.baselineSd * 1000) / 1000,
|
|
121
|
+
cusumUp: Math.round(state.cusumUp * 100) / 100,
|
|
122
|
+
cusumDown: Math.round(state.cusumDown * 100) / 100,
|
|
123
|
+
alarm: state.alarm,
|
|
124
|
+
latched: state.latched,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Every series that is currently alarming, worst first. */
|
|
129
|
+
alarms() {
|
|
130
|
+
return [...this.series.keys()]
|
|
131
|
+
.map((key) => this.snapshot(key))
|
|
132
|
+
.filter((s) => s?.alarm)
|
|
133
|
+
.sort((a, b) => (b?.alarm?.cusum ?? 0) - (a?.alarm?.cusum ?? 0));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Forget a series — used when a target is edited or the baseline is wrong. */
|
|
137
|
+
reset(key) {
|
|
138
|
+
if (key) this.series.delete(key);
|
|
139
|
+
else this.series.clear();
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Read a ledger and report drift per model on latency and failure rate.
|
|
145
|
+
* Called by the worker, never on the request path.
|
|
146
|
+
*
|
|
147
|
+
* @param {{records: Array<any>, monitor?: DriftMonitor, since?: number, minSamples?: number}} input
|
|
148
|
+
*/
|
|
149
|
+
export function driftFromLedger(input) {
|
|
150
|
+
const monitor = input.monitor ?? new DriftMonitor();
|
|
151
|
+
const since = input.since ?? 0;
|
|
152
|
+
const rows = input.records.filter((r) => (r.ts ?? 0) >= since);
|
|
153
|
+
|
|
154
|
+
/** @type {Map<string, {latency: number[], failures: number[]}>} */
|
|
155
|
+
const byModel = new Map();
|
|
156
|
+
for (const row of rows) {
|
|
157
|
+
const bucket = byModel.get(row.modelId) ?? { latency: [], failures: [] };
|
|
158
|
+
if (row.ok && Number.isFinite(row.latencyMs)) bucket.latency.push(row.latencyMs);
|
|
159
|
+
bucket.failures.push(row.ok ? 0 : 1);
|
|
160
|
+
byModel.set(row.modelId, bucket);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const minSamples = Number(input.minSamples ?? 10);
|
|
164
|
+
for (const [modelId, bucket] of byModel) {
|
|
165
|
+
if (bucket.latency.length >= minSamples) {
|
|
166
|
+
for (const value of bucket.latency) monitor.record(`${modelId}:latency`, value);
|
|
167
|
+
}
|
|
168
|
+
if (bucket.failures.length >= minSamples) {
|
|
169
|
+
for (const value of bucket.failures) monitor.record(`${modelId}:failures`, value);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return { monitor, alarms: monitor.alarms() };
|
|
174
|
+
}
|
package/src/embedding.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* B4.7 — the embedding tier, without a model.
|
|
3
|
+
*
|
|
4
|
+
* This is **feature hashing**, not a learned embedding: each token and each
|
|
5
|
+
* adjacent token pair is hashed into a fixed number of dimensions with a signed
|
|
6
|
+
* contribution, then the vector is normalised. Two prompts that share most of
|
|
7
|
+
* their wording land close together; two paraphrases that share no words do not.
|
|
8
|
+
*
|
|
9
|
+
* That limitation is the point of saying it out loud. A real embedding model
|
|
10
|
+
* needs a provider and quota; this needs neither, costs microseconds, and beats
|
|
11
|
+
* exact-hash matching for the case that actually happens — the same question
|
|
12
|
+
* asked again with a word changed.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const DIMS = 256;
|
|
16
|
+
|
|
17
|
+
/** FNV-1a, 32-bit: fast, deterministic, good enough for bucketing. */
|
|
18
|
+
function hash32(str) {
|
|
19
|
+
let h = 0x811c9dc5;
|
|
20
|
+
for (let i = 0; i < str.length; i++) {
|
|
21
|
+
h ^= str.charCodeAt(i);
|
|
22
|
+
h = Math.imul(h, 0x01000193);
|
|
23
|
+
}
|
|
24
|
+
return h >>> 0;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function tokens(text) {
|
|
28
|
+
return String(text ?? '')
|
|
29
|
+
.toLowerCase()
|
|
30
|
+
.split(/[^a-z0-9']+/)
|
|
31
|
+
.filter((t) => t.length > 0);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {string} text
|
|
36
|
+
* @param {number} [dims]
|
|
37
|
+
* @returns {Float64Array}
|
|
38
|
+
*/
|
|
39
|
+
export function embed(text, dims = DIMS) {
|
|
40
|
+
const vector = new Float64Array(dims);
|
|
41
|
+
const words = tokens(text);
|
|
42
|
+
|
|
43
|
+
const add = (term, weight) => {
|
|
44
|
+
const h = hash32(term);
|
|
45
|
+
const index = h % dims;
|
|
46
|
+
const sign = (h >>> 31) === 0 ? 1 : -1; // signed hashing keeps collisions from only adding
|
|
47
|
+
vector[index] += sign * weight;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
for (const word of words) add(word, 1);
|
|
51
|
+
for (let i = 0; i + 1 < words.length; i++) add(`${words[i]} ${words[i + 1]}`, 0.5);
|
|
52
|
+
|
|
53
|
+
let norm = 0;
|
|
54
|
+
for (let i = 0; i < dims; i++) norm += vector[i] * vector[i];
|
|
55
|
+
norm = Math.sqrt(norm);
|
|
56
|
+
if (norm > 0) for (let i = 0; i < dims; i++) vector[i] /= norm;
|
|
57
|
+
return vector;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** @param {Float64Array} a @param {Float64Array} b */
|
|
61
|
+
export function cosine(a, b) {
|
|
62
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
63
|
+
let dot = 0;
|
|
64
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
65
|
+
return dot; // both vectors are already unit length
|
|
66
|
+
}
|
package/src/history.js
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @drsamonline/telemetry/history — B2.7, health over time.
|
|
3
|
+
*
|
|
4
|
+
* The prober already keeps 24 observations per target in memory, which is enough
|
|
5
|
+
* for a sparkline and useless the moment the process restarts. This keeps a
|
|
6
|
+
* rolling history on disk so "has it been flaky all morning?" is answerable.
|
|
7
|
+
*
|
|
8
|
+
* Three constraints, because this is the one place that writes over time:
|
|
9
|
+
*
|
|
10
|
+
* - **Prompt-free, like everything else** (I11). A sample is a timestamp, a
|
|
11
|
+
* model id, a duration and a boolean. There is no field a prompt could go in,
|
|
12
|
+
* and `test/history.test.js` asserts the stored rows have exactly that shape.
|
|
13
|
+
* - **Bounded.** A cap on rows and on age, pruned on write, so a gateway left
|
|
14
|
+
* running for a month does not fill a disk.
|
|
15
|
+
* - **Cheap.** Appends are buffered and flushed in batches; a probe pass must
|
|
16
|
+
* not turn into a hundred file writes.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
20
|
+
import { dirname } from 'node:path';
|
|
21
|
+
|
|
22
|
+
/** The only fields a sample may have. */
|
|
23
|
+
export const SAMPLE_FIELDS = ['ts', 'modelId', 'ok', 'latencyMs', 'state'];
|
|
24
|
+
|
|
25
|
+
export class HistoryStore {
|
|
26
|
+
/**
|
|
27
|
+
* @param {{file?: string|null, maxRows?: number, maxAgeMs?: number, now?: () => number}} [opts]
|
|
28
|
+
*/
|
|
29
|
+
constructor(opts = {}) {
|
|
30
|
+
this.file = opts.file ?? null;
|
|
31
|
+
this.maxRows = Number(opts.maxRows ?? 20_000);
|
|
32
|
+
this.maxAgeMs = Number(opts.maxAgeMs ?? 7 * 24 * 3600_000);
|
|
33
|
+
this.now = opts.now ?? Date.now;
|
|
34
|
+
|
|
35
|
+
/** @type {Array<{ts: number, modelId: string, ok: boolean, latencyMs: number|null, state: string}>} */
|
|
36
|
+
this.rows = [];
|
|
37
|
+
this.dirty = 0;
|
|
38
|
+
|
|
39
|
+
if (this.file && existsSync(this.file)) this.#load();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
#load() {
|
|
43
|
+
try {
|
|
44
|
+
const text = readFileSync(this.file, 'utf8');
|
|
45
|
+
const rows = [];
|
|
46
|
+
for (const line of text.split('\n')) {
|
|
47
|
+
if (!line.trim()) continue;
|
|
48
|
+
try {
|
|
49
|
+
const row = JSON.parse(line);
|
|
50
|
+
// a row with an unexpected field is dropped rather than trusted: this
|
|
51
|
+
// file is the one place a prompt could hide
|
|
52
|
+
const extra = Object.keys(row).filter((k) => !SAMPLE_FIELDS.includes(k));
|
|
53
|
+
if (extra.length) continue;
|
|
54
|
+
rows.push(row);
|
|
55
|
+
} catch {
|
|
56
|
+
/* a torn last line from a kill is not a reason to refuse the history */
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
this.rows = rows;
|
|
60
|
+
this.prune();
|
|
61
|
+
} catch {
|
|
62
|
+
this.rows = [];
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param {{modelId: string, ok: boolean, latencyMs?: number|null, state?: string, ts?: number}} sample
|
|
68
|
+
*/
|
|
69
|
+
record(sample) {
|
|
70
|
+
const row = {
|
|
71
|
+
ts: sample.ts ?? this.now(),
|
|
72
|
+
modelId: String(sample.modelId),
|
|
73
|
+
ok: Boolean(sample.ok),
|
|
74
|
+
latencyMs: Number.isFinite(sample.latencyMs) ? Math.round(Number(sample.latencyMs)) : null,
|
|
75
|
+
state: String(sample.state ?? (sample.ok ? 'closed' : 'open')),
|
|
76
|
+
};
|
|
77
|
+
this.rows.push(row);
|
|
78
|
+
this.dirty += 1;
|
|
79
|
+
|
|
80
|
+
if (this.file) {
|
|
81
|
+
try {
|
|
82
|
+
mkdirSync(dirname(this.file), { recursive: true });
|
|
83
|
+
appendFileSync(this.file, `${JSON.stringify(row)}\n`);
|
|
84
|
+
} catch {
|
|
85
|
+
/* a history that cannot be written must not take the gateway down */
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// prune on a threshold rather than every write: cheap, and bounded
|
|
90
|
+
if (this.rows.length > this.maxRows * 1.2) this.prune();
|
|
91
|
+
return row;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
prune() {
|
|
95
|
+
const cutoff = this.now() - this.maxAgeMs;
|
|
96
|
+
const before = this.rows.length;
|
|
97
|
+
this.rows = this.rows.filter((r) => r.ts >= cutoff);
|
|
98
|
+
if (this.rows.length > this.maxRows) this.rows = this.rows.slice(-this.maxRows);
|
|
99
|
+
|
|
100
|
+
if (this.file && this.rows.length !== before) this.#rewrite();
|
|
101
|
+
return before - this.rows.length;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
#rewrite() {
|
|
105
|
+
try {
|
|
106
|
+
const tmp = `${this.file}.tmp`;
|
|
107
|
+
writeFileSync(tmp, this.rows.map((r) => `${JSON.stringify(r)}\n`).join(''));
|
|
108
|
+
renameSync(tmp, this.file); // atomic: a reader never sees half a file
|
|
109
|
+
this.dirty = 0;
|
|
110
|
+
} catch {
|
|
111
|
+
/* keep the old file rather than a truncated one */
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Which models have samples. */
|
|
116
|
+
models() {
|
|
117
|
+
return [...new Set(this.rows.map((r) => r.modelId))].sort();
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* A bucketed series for one model: enough for a sparkline, small enough to send.
|
|
122
|
+
*
|
|
123
|
+
* @param {string} modelId
|
|
124
|
+
* @param {{hours?: number, buckets?: number}} [opts]
|
|
125
|
+
*/
|
|
126
|
+
series(modelId, opts = {}) {
|
|
127
|
+
const hours = Number(opts.hours ?? 24);
|
|
128
|
+
const buckets = Number(opts.buckets ?? 24);
|
|
129
|
+
const now = this.now();
|
|
130
|
+
const from = now - hours * 3600_000;
|
|
131
|
+
const width = (hours * 3600_000) / buckets;
|
|
132
|
+
|
|
133
|
+
const mine = this.rows.filter((r) => r.modelId === modelId && r.ts >= from);
|
|
134
|
+
/** @type {Array<any>} */
|
|
135
|
+
const points = [];
|
|
136
|
+
for (let i = 0; i < buckets; i++) {
|
|
137
|
+
const start = from + i * width;
|
|
138
|
+
const inBucket = mine.filter((r) => r.ts >= start && r.ts < start + width);
|
|
139
|
+
if (!inBucket.length) {
|
|
140
|
+
points.push({ t: Math.round(start), samples: 0, ok: null, p50: null, errorRate: null, state: null });
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
const latencies = inBucket.map((r) => r.latencyMs).filter((v) => v !== null).sort((a, b) => a - b);
|
|
144
|
+
const failures = inBucket.filter((r) => !r.ok).length;
|
|
145
|
+
points.push({
|
|
146
|
+
t: Math.round(start),
|
|
147
|
+
samples: inBucket.length,
|
|
148
|
+
ok: failures === 0,
|
|
149
|
+
p50: latencies.length ? latencies[Math.floor(latencies.length / 2)] : null,
|
|
150
|
+
errorRate: Math.round((failures / inBucket.length) * 1000) / 1000,
|
|
151
|
+
state: inBucket[inBucket.length - 1].state,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const observed = points.filter((p) => p.samples > 0);
|
|
156
|
+
const uptime = observed.length ? Math.round((observed.filter((p) => p.ok).length / observed.length) * 1000) / 1000 : null;
|
|
157
|
+
return { modelId, hours, buckets, points, uptime, samples: mine.length };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Every model's series in one call, for the Weather Map. */
|
|
161
|
+
all(opts = {}) {
|
|
162
|
+
return this.models().map((modelId) => this.series(modelId, opts));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Compact stats for `/api/status`: no point arrays. */
|
|
166
|
+
summary() {
|
|
167
|
+
const cutoff = this.now() - 24 * 3600_000;
|
|
168
|
+
const rows = this.rows.filter((r) => r.ts >= cutoff);
|
|
169
|
+
return {
|
|
170
|
+
rows: this.rows.length,
|
|
171
|
+
last24h: rows.length,
|
|
172
|
+
models: this.models().length,
|
|
173
|
+
oldest: this.rows.length ? this.rows[0].ts : null,
|
|
174
|
+
maxRows: this.maxRows,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** @drsamonline/telemetry — the ledger, the cache, and the aggregates built from them. */
|
|
2
|
+
export * from './ledger.js';
|
|
3
|
+
export * from './cache.js';
|
|
4
|
+
export { DriftMonitor, driftFromLedger } from './drift.js';
|
|
5
|
+
export { HistoryStore, SAMPLE_FIELDS } from './history.js';
|
package/src/ledger.js
ADDED
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ledger: one append-only record per completed turn.
|
|
3
|
+
*
|
|
4
|
+
* It is what makes "the same task at lower cost" a measurable claim rather than
|
|
5
|
+
* a slogan — every record carries provider, model, combo, arm (for A/B), tokens,
|
|
6
|
+
* cost, latency, time-to-first-token, cache and verification status.
|
|
7
|
+
*
|
|
8
|
+
* Prompt text is never stored unless guardrails.logPrompts is on. What is stored
|
|
9
|
+
* is a salted hash, so the same prompt can be recognised across turns (prompt-free
|
|
10
|
+
* usage view) without the prompt being recoverable.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
14
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { dirname } from 'node:path';
|
|
16
|
+
|
|
17
|
+
/** @typedef {{
|
|
18
|
+
* ts: number, provider: string, model: string, modelId: string, combo: string, arm: string|null,
|
|
19
|
+
* ok: boolean, error: string|null, promptTokens: number, completionTokens: number,
|
|
20
|
+
* costUsd: number, latencyMs: number, ttftMs: number|null, cached: boolean,
|
|
21
|
+
* verified: boolean, promptHash: string, promptChars: number, streamed: boolean
|
|
22
|
+
* }} LedgerRecord */
|
|
23
|
+
|
|
24
|
+
export class Ledger {
|
|
25
|
+
/**
|
|
26
|
+
* @param {{file?: string|null, maxRecords?: number, salt?: string, hashSecretPath?: string}} [opts]
|
|
27
|
+
*/
|
|
28
|
+
constructor(opts = {}) {
|
|
29
|
+
this.file = opts.file ?? null;
|
|
30
|
+
this.maxRecords = opts.maxRecords ?? 5000;
|
|
31
|
+
this.salt = opts.salt ?? 'sui-local-salt';
|
|
32
|
+
/**
|
|
33
|
+
* Per-day totals, kept outside the record window.
|
|
34
|
+
*
|
|
35
|
+
* The window is capped at maxRecords for memory, which is right — but a
|
|
36
|
+
* budget computed from the window silently stops counting once a day passes
|
|
37
|
+
* 5,000 turns, and a budget that stops counting is worse than no budget. This
|
|
38
|
+
* map is O(1) to update and does not depend on the window at all.
|
|
39
|
+
*
|
|
40
|
+
* @type {Map<string, {costUsd: number, turns: number, failed: number, cached: number, byProvider: Map<string, number>}>}
|
|
41
|
+
*/
|
|
42
|
+
this.daily = new Map();
|
|
43
|
+
this.maxFileBytes = Number(opts.maxFileBytes ?? 64 * 1024 * 1024);
|
|
44
|
+
this.fileBytes = 0;
|
|
45
|
+
/** @type {LedgerRecord[]} */
|
|
46
|
+
this.records = [];
|
|
47
|
+
if (this.file) {
|
|
48
|
+
mkdirSync(dirname(this.file), { recursive: true });
|
|
49
|
+
if (existsSync(this.file)) this.#load();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
#load() {
|
|
54
|
+
try {
|
|
55
|
+
const raw = readFileSync(this.file, 'utf8');
|
|
56
|
+
this.fileBytes = Buffer.byteLength(raw);
|
|
57
|
+
for (const line of raw.split('\n')) {
|
|
58
|
+
if (!line.trim()) continue;
|
|
59
|
+
let row;
|
|
60
|
+
try {
|
|
61
|
+
row = JSON.parse(line);
|
|
62
|
+
} catch {
|
|
63
|
+
continue; // a torn last line from a hard kill is not worth failing over
|
|
64
|
+
}
|
|
65
|
+
// every row counts towards its day, including rows that fall outside the
|
|
66
|
+
// window: the totals must not depend on how much history is held
|
|
67
|
+
this.#bumpDaily(row);
|
|
68
|
+
this.records.push(row);
|
|
69
|
+
}
|
|
70
|
+
if (this.records.length > this.maxRecords) {
|
|
71
|
+
this.records = this.records.slice(-this.maxRecords);
|
|
72
|
+
this.#rewrite();
|
|
73
|
+
}
|
|
74
|
+
} catch {
|
|
75
|
+
this.records = [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** @param {any} row */
|
|
80
|
+
#dayKey(row) {
|
|
81
|
+
const d = new Date(row?.ts ?? Date.now());
|
|
82
|
+
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** @param {any} row */
|
|
86
|
+
#bumpDaily(row) {
|
|
87
|
+
const key = this.#dayKey(row);
|
|
88
|
+
let day = this.daily.get(key);
|
|
89
|
+
if (!day) {
|
|
90
|
+
day = { costUsd: 0, turns: 0, failed: 0, cached: 0, byProvider: new Map() };
|
|
91
|
+
this.daily.set(key, day);
|
|
92
|
+
// keep at most three days: yesterday is useful, last week is not
|
|
93
|
+
if (this.daily.size > 3) {
|
|
94
|
+
const oldest = [...this.daily.keys()].sort()[0];
|
|
95
|
+
this.daily.delete(oldest);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
day.costUsd += Number(row.costUsd ?? 0);
|
|
99
|
+
if (row.ok) day.turns += 1;
|
|
100
|
+
else day.failed += 1;
|
|
101
|
+
if (row.cached) day.cached += 1;
|
|
102
|
+
const provider = row.provider ?? 'unknown';
|
|
103
|
+
day.byProvider.set(provider, (day.byProvider.get(provider) ?? 0) + Number(row.costUsd ?? 0));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The bucket for a given day, or an empty one. */
|
|
107
|
+
#day(now = new Date()) {
|
|
108
|
+
const key = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`;
|
|
109
|
+
return this.daily.get(key) ?? { costUsd: 0, turns: 0, failed: 0, cached: 0, byProvider: new Map() };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Rotate the file when it grows past the cap.
|
|
114
|
+
*
|
|
115
|
+
* Without this a gateway left running appends forever: at 20 rps a three-day
|
|
116
|
+
* soak writes millions of lines, and nothing prunes a file, only an array.
|
|
117
|
+
*/
|
|
118
|
+
#rotateIfNeeded(nextBytes) {
|
|
119
|
+
if (!this.file) return;
|
|
120
|
+
this.fileBytes += nextBytes;
|
|
121
|
+
if (this.fileBytes <= this.maxFileBytes) return;
|
|
122
|
+
try {
|
|
123
|
+
renameSync(this.file, `${this.file}.1`);
|
|
124
|
+
// leave an empty file behind: a reader that stats the live path should not
|
|
125
|
+
// find it missing just because a rotation happened between two writes
|
|
126
|
+
writeFileSync(this.file, '');
|
|
127
|
+
this.fileBytes = 0;
|
|
128
|
+
} catch {
|
|
129
|
+
this.fileBytes = 0; // if the rename fails, keep appending rather than stop recording
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
#rewrite() {
|
|
134
|
+
if (!this.file) return;
|
|
135
|
+
writeFileSync(this.file, this.records.map((r) => JSON.stringify(r)).join('\n') + '\n');
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Stable, non-reversible prompt fingerprint. */
|
|
139
|
+
hashPrompt(text) {
|
|
140
|
+
return createHash('sha256').update(this.salt).update(text).digest('hex').slice(0, 16);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** @param {Partial<LedgerRecord> & {promptText?: string}} entry */
|
|
144
|
+
record(entry) {
|
|
145
|
+
/** @type {LedgerRecord} */
|
|
146
|
+
const rec = {
|
|
147
|
+
ts: entry.ts ?? Date.now(),
|
|
148
|
+
provider: entry.provider ?? 'unknown',
|
|
149
|
+
model: entry.model ?? 'unknown',
|
|
150
|
+
modelId: entry.modelId ?? 'unknown',
|
|
151
|
+
combo: entry.combo ?? 'unknown',
|
|
152
|
+
arm: entry.arm ?? null,
|
|
153
|
+
ok: entry.ok ?? true,
|
|
154
|
+
error: entry.error ?? null,
|
|
155
|
+
promptTokens: entry.promptTokens ?? 0,
|
|
156
|
+
completionTokens: entry.completionTokens ?? 0,
|
|
157
|
+
costUsd: entry.costUsd ?? 0,
|
|
158
|
+
latencyMs: entry.latencyMs ?? 0,
|
|
159
|
+
ttftMs: entry.ttftMs ?? null,
|
|
160
|
+
cached: entry.cached ?? false,
|
|
161
|
+
verified: entry.verified ?? false,
|
|
162
|
+
promptHash: entry.promptHash ?? (entry.promptText ? this.hashPrompt(entry.promptText) : ''),
|
|
163
|
+
promptChars: entry.promptChars ?? (entry.promptText ? entry.promptText.length : 0),
|
|
164
|
+
streamed: entry.streamed ?? false,
|
|
165
|
+
// B7.2: what compression saved on this turn, valued at the model's price.
|
|
166
|
+
compressionPreset: entry.compressionPreset ?? null,
|
|
167
|
+
compressionSavedTokens: entry.compressionSavedTokens ?? 0,
|
|
168
|
+
compressionSavedUsd: entry.compressionSavedUsd ?? 0,
|
|
169
|
+
// B6.6: which tenant this turn belongs to. A name, never a key.
|
|
170
|
+
apiKey: entry.apiKey ?? null,
|
|
171
|
+
// B7.1: who asked, and with what. Both are labels the caller supplies;
|
|
172
|
+
// the gateway never infers an identity it was not given.
|
|
173
|
+
user: entry.user ?? null,
|
|
174
|
+
tool: entry.tool ?? null,
|
|
175
|
+
promptCacheWarm: entry.promptCacheWarm ?? false,
|
|
176
|
+
};
|
|
177
|
+
this.records.push(rec);
|
|
178
|
+
this.#bumpDaily(rec);
|
|
179
|
+
if (this.records.length > this.maxRecords) this.records.shift();
|
|
180
|
+
if (this.file) {
|
|
181
|
+
try {
|
|
182
|
+
const line = `${JSON.stringify(rec)}\n`;
|
|
183
|
+
appendFileSync(this.file, line);
|
|
184
|
+
this.#rotateIfNeeded(Buffer.byteLength(line));
|
|
185
|
+
} catch {
|
|
186
|
+
/* ledger is observability, never the request path's failure */
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return rec;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* @param {{since?: number, combo?: string}} [opts]
|
|
194
|
+
* @returns {LedgerRecord[]}
|
|
195
|
+
*/
|
|
196
|
+
entries(opts = {}) {
|
|
197
|
+
const since = opts.since ?? 0;
|
|
198
|
+
return this.records.filter((r) => r.ts >= since && (!opts.combo || r.combo === opts.combo));
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Spend since local midnight, in USD. */
|
|
202
|
+
todaySpendUsd(now = new Date()) {
|
|
203
|
+
// from the daily bucket, not from the record window: a budget that stops
|
|
204
|
+
// counting after 5,000 turns is not a budget
|
|
205
|
+
return round6(this.#day(now).costUsd);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
providerSpendToday(provider, now = new Date()) {
|
|
209
|
+
// same rule as todaySpendUsd: from the day bucket, not the window
|
|
210
|
+
return round6(this.#day(now).byProvider.get(provider) ?? 0);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** @deprecated kept for callers that still read the window; use providerSpendToday */
|
|
214
|
+
#providerSpendFromWindow(provider, now = new Date()) {
|
|
215
|
+
const start = new Date(now);
|
|
216
|
+
start.setHours(0, 0, 0, 0);
|
|
217
|
+
return round6(
|
|
218
|
+
this.records
|
|
219
|
+
.filter((r) => r.ts >= start.getTime() && r.provider === provider)
|
|
220
|
+
.reduce((a, r) => a + r.costUsd, 0),
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* @param {{since?: number}} [opts]
|
|
226
|
+
*/
|
|
227
|
+
aggregate(opts = {}) {
|
|
228
|
+
const rows = this.entries(opts);
|
|
229
|
+
// B7.1: attribution uses the same grouping shape as provider/model/combo, so
|
|
230
|
+
// one `bump` does all of it and nothing can drift.
|
|
231
|
+
/** @type {Record<string, any>} */
|
|
232
|
+
const byUser = {};
|
|
233
|
+
/** @type {Record<string, any>} */
|
|
234
|
+
const byTool = {};
|
|
235
|
+
for (const row of rows) {
|
|
236
|
+
bump(byUser, row.user ?? 'unattributed', row);
|
|
237
|
+
bump(byTool, row.tool ?? 'unattributed', row);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const totals = {
|
|
241
|
+
// A turn is a request that produced an answer. A failed attempt is
|
|
242
|
+
// recorded too (it is how failover is auditable), but it is not a turn.
|
|
243
|
+
turns: rows.filter((r) => r.ok).length,
|
|
244
|
+
// B7.2 Savings Ledger: dollars not spent, and the tokens that bought them.
|
|
245
|
+
savings: { cacheUsd: 0, compressionUsd: 0, totalUsd: 0, tokensSaved: 0, cacheHits: 0, compressedTurns: 0 },
|
|
246
|
+
attempts: rows.length,
|
|
247
|
+
ok: rows.filter((r) => r.ok).length,
|
|
248
|
+
failed: rows.filter((r) => !r.ok).length,
|
|
249
|
+
cached: rows.filter((r) => r.cached).length,
|
|
250
|
+
costUsd: 0,
|
|
251
|
+
savedUsd: 0,
|
|
252
|
+
promptTokens: 0,
|
|
253
|
+
completionTokens: 0,
|
|
254
|
+
avgLatencyMs: 0,
|
|
255
|
+
p50LatencyMs: 0,
|
|
256
|
+
p95LatencyMs: 0,
|
|
257
|
+
avgTtftMs: 0,
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
/** @type {Record<string, any>} */
|
|
261
|
+
const byProvider = {};
|
|
262
|
+
/** @type {Record<string, any>} */
|
|
263
|
+
const byModel = {};
|
|
264
|
+
/** @type {Record<string, any>} */
|
|
265
|
+
const byCombo = {};
|
|
266
|
+
/** @type {Record<string, any>} */
|
|
267
|
+
const byArm = {};
|
|
268
|
+
|
|
269
|
+
for (const r of rows) {
|
|
270
|
+
totals.costUsd += r.costUsd;
|
|
271
|
+
totals.promptTokens += r.promptTokens;
|
|
272
|
+
totals.completionTokens += r.completionTokens;
|
|
273
|
+
if (r.cached) {
|
|
274
|
+
totals.savedUsd += r.costUsd;
|
|
275
|
+
totals.savings.cacheUsd += r.costUsd;
|
|
276
|
+
totals.savings.cacheHits += 1;
|
|
277
|
+
}
|
|
278
|
+
if (r.compressionSavedUsd) {
|
|
279
|
+
totals.savings.compressionUsd += r.compressionSavedUsd;
|
|
280
|
+
totals.savings.tokensSaved += r.compressionSavedTokens ?? 0;
|
|
281
|
+
totals.savings.compressedTurns += 1;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
bump(byProvider, r.provider, r);
|
|
285
|
+
bump(byModel, r.modelId, r);
|
|
286
|
+
bump(byCombo, r.combo, r);
|
|
287
|
+
if (r.arm) bump(byArm, `${r.combo}/${r.arm}`, r);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// A cache hit answers in microseconds and would drag p50 to zero; it is not
|
|
291
|
+
// a model latency, so it is excluded from latency statistics on purpose.
|
|
292
|
+
const latencies = rows
|
|
293
|
+
.filter((r) => r.ok && !r.cached)
|
|
294
|
+
.map((r) => r.latencyMs)
|
|
295
|
+
.sort((a, b) => a - b);
|
|
296
|
+
const ttfts = rows.filter((r) => r.ok && r.ttftMs != null).map((r) => /** @type {number} */ (r.ttftMs));
|
|
297
|
+
totals.costUsd = round6(totals.costUsd);
|
|
298
|
+
totals.savings.cacheUsd = round6(totals.savings.cacheUsd);
|
|
299
|
+
totals.savings.compressionUsd = round6(totals.savings.compressionUsd);
|
|
300
|
+
totals.savings.totalUsd = round6(totals.savings.cacheUsd + totals.savings.compressionUsd);
|
|
301
|
+
// `savedUsd` predates the savings block and is kept for older readers.
|
|
302
|
+
totals.savedUsd = round6(totals.savings.totalUsd);
|
|
303
|
+
totals.avgLatencyMs = latencies.length ? Math.round(mean(latencies)) : 0;
|
|
304
|
+
totals.p50LatencyMs = quantile(latencies, 0.5);
|
|
305
|
+
totals.p95LatencyMs = quantile(latencies, 0.95);
|
|
306
|
+
totals.avgTtftMs = ttfts.length ? Math.round(mean(ttfts)) : 0;
|
|
307
|
+
|
|
308
|
+
for (const group of [byProvider, byModel, byCombo, byArm]) {
|
|
309
|
+
for (const v of Object.values(group)) {
|
|
310
|
+
v.costUsd = round6(v.costUsd);
|
|
311
|
+
v.avgLatencyMs = v.latencySamples ? Math.round(v.latencySum / v.latencySamples) : 0;
|
|
312
|
+
delete v.latencySum;
|
|
313
|
+
delete v.latencySamples;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
return {
|
|
318
|
+
since: opts.since ?? 0,
|
|
319
|
+
totals,
|
|
320
|
+
byProvider,
|
|
321
|
+
byModel,
|
|
322
|
+
byCombo,
|
|
323
|
+
byArm,
|
|
324
|
+
// B7.1: who asked and with what, so cost can be charged back
|
|
325
|
+
byUser,
|
|
326
|
+
byTool,
|
|
327
|
+
recent: rows.slice(-50).reverse(),
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** Prompt-free usage view: which prompt fingerprints recurred, and what they cost. */
|
|
332
|
+
promptUsage(limit = 50) {
|
|
333
|
+
/** @type {Map<string, {promptHash: string, turns: number, chars: number, costUsd: number, combos: Set<string>, models: Set<string>}>} */
|
|
334
|
+
const map = new Map();
|
|
335
|
+
for (const r of this.records) {
|
|
336
|
+
if (!r.promptHash) continue;
|
|
337
|
+
let e = map.get(r.promptHash);
|
|
338
|
+
if (!e) {
|
|
339
|
+
e = { promptHash: r.promptHash, turns: 0, chars: r.promptChars, costUsd: 0, combos: new Set(), models: new Set() };
|
|
340
|
+
map.set(r.promptHash, e);
|
|
341
|
+
}
|
|
342
|
+
e.turns += 1;
|
|
343
|
+
e.costUsd = round6(e.costUsd + r.costUsd);
|
|
344
|
+
e.combos.add(r.combo);
|
|
345
|
+
e.models.add(r.modelId);
|
|
346
|
+
}
|
|
347
|
+
return [...map.values()]
|
|
348
|
+
.map((e) => ({ ...e, combos: [...e.combos], models: [...e.models] }))
|
|
349
|
+
.sort((a, b) => b.costUsd - a.costUsd)
|
|
350
|
+
.slice(0, limit);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
csv() {
|
|
354
|
+
const header = [
|
|
355
|
+
'timestamp_iso', 'provider', 'model', 'model_id', 'combo', 'arm', 'ok', 'error',
|
|
356
|
+
'prompt_tokens', 'completion_tokens', 'cost_usd', 'latency_ms', 'ttft_ms',
|
|
357
|
+
'cached', 'verified', 'streamed', 'prompt_hash', 'prompt_chars',
|
|
358
|
+
'compression_preset', 'compression_saved_tokens', 'compression_saved_usd',
|
|
359
|
+
'api_key', 'prompt_cache_warm', 'user', 'tool',
|
|
360
|
+
].join(',');
|
|
361
|
+
const lines = this.records.map((r) =>
|
|
362
|
+
[
|
|
363
|
+
new Date(r.ts).toISOString(), r.provider, r.model, r.modelId, r.combo, r.arm ?? '',
|
|
364
|
+
r.ok, r.error ?? '', r.promptTokens, r.completionTokens, r.costUsd, r.latencyMs,
|
|
365
|
+
r.ttftMs ?? '', r.cached, r.verified, r.streamed, r.promptHash, r.promptChars,
|
|
366
|
+
r.compressionPreset ?? '', r.compressionSavedTokens ?? 0, r.compressionSavedUsd ?? 0,
|
|
367
|
+
r.apiKey ?? '', r.promptCacheWarm ?? false, r.user ?? '', r.tool ?? '',
|
|
368
|
+
]
|
|
369
|
+
.map((v) => (typeof v === 'string' && /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v))
|
|
370
|
+
.join(','),
|
|
371
|
+
);
|
|
372
|
+
return [header, ...lines].join('\n');
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function bump(group, key, r) {
|
|
377
|
+
const e = (group[key] ??= { turns: 0, ok: 0, failed: 0, cached: 0, costUsd: 0, tokens: 0, latencySum: 0 });
|
|
378
|
+
e.turns += 1;
|
|
379
|
+
if (r.ok) e.ok += 1;
|
|
380
|
+
else e.failed += 1;
|
|
381
|
+
if (r.cached) e.cached += 1;
|
|
382
|
+
e.costUsd += r.costUsd;
|
|
383
|
+
e.tokens += r.promptTokens + r.completionTokens;
|
|
384
|
+
if (r.ok && !r.cached) {
|
|
385
|
+
e.latencySum += r.latencyMs;
|
|
386
|
+
e.latencySamples = (e.latencySamples ?? 0) + 1;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function mean(a) {
|
|
391
|
+
return a.reduce((x, y) => x + y, 0) / a.length;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function quantile(sorted, q) {
|
|
395
|
+
if (!sorted.length) return 0;
|
|
396
|
+
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1));
|
|
397
|
+
return sorted[idx];
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function round6(n) {
|
|
401
|
+
return Math.round(n * 1e6) / 1e6;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
export { randomBytes };
|
package/README.md
DELETED