@drsamonline/telemetry 0.0.0-stage → 0.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,28 @@
1
1
  {
2
2
  "name": "@drsamonline/telemetry",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.12.2",
4
+ "description": "Ledger, cache, aggregates, CSV.",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "main": "./src/index.js",
8
+ "exports": {
9
+ ".": "./src/index.js",
10
+ "./src/*": "./src/*"
11
+ },
12
+ "files": [
13
+ "src",
14
+ "web",
15
+ "bin",
16
+ "LICENSE"
17
+ ],
18
+ "repository": {
19
+ "type": "git",
20
+ "url": "git://github.com/drsamonline/sui.git",
21
+ "directory": "packages/telemetry"
22
+ },
23
+ "homepage": "https://github.com/drsamonline/sui#readme",
24
+ "publishConfig": {
25
+ "access": "public",
26
+ "registry": "https://registry.npmjs.org/"
27
+ }
28
+ }
package/src/cache.js ADDED
@@ -0,0 +1,216 @@
1
+ /**
2
+ * Response cache — two tiers.
3
+ *
4
+ * exact a hash of the whole request (combo, messages, sampling params)
5
+ * semantic tf-idf similarity between prompts, via the vendored ml-kit
6
+ *
7
+ * The semantic tier is **off by default** and that is a privacy decision, not a
8
+ * performance one: to compare prompts it has to keep recent prompt text in
9
+ * memory. Nothing is written to disk, and turning it on is an explicit choice
10
+ * (`server.cache.semantic.enabled`).
11
+ *
12
+ * A cache hit consumes no provider quota, which is the whole point: it is the
13
+ * cheapest possible route.
14
+ */
15
+
16
+ import { createHash } from 'node:crypto';
17
+ import { loadKit } from '@drsamonline/kits';
18
+ import { cosine, embed } from './embedding.js';
19
+
20
+ export class ResponseCache {
21
+ /**
22
+ * @param {{
23
+ * ttlMs?: number, maxEntries?: number, enabled?: boolean, now?: () => number,
24
+ * semantic?: {enabled?: boolean, threshold?: number}
25
+ * }} [opts]
26
+ */
27
+ constructor(opts = {}) {
28
+ this.ttlMs = opts.ttlMs ?? 900000;
29
+ this.maxEntries = opts.maxEntries ?? 500;
30
+ this.enabled = opts.enabled ?? true;
31
+ this.now = opts.now ?? Date.now;
32
+ this.semanticEnabled = Boolean(opts.semantic?.enabled);
33
+ // read from the nested block or the flat option, so both call styles behave
34
+ this.semanticThreshold = Number(opts.semantic?.threshold ?? opts.semanticThreshold ?? 0.98);
35
+ // B4.7: 'lexical' (tf-idf from ml-kit) or 'embedding' (hashed vectors, no model)
36
+ this.semanticTier = (opts.semantic?.tier ?? opts.semanticTier) === 'embedding' ? 'embedding' : 'lexical';
37
+
38
+ /** @type {Map<string, {value: any, expiresAt: number, promptText: string|null}>} */
39
+ this.map = new Map();
40
+ this.hits = 0;
41
+ this.semanticHits = 0;
42
+ this.misses = 0;
43
+
44
+ /** @type {{index: any, keys: string[]}|null} */
45
+ this._index = null;
46
+ this._indexDirty = true;
47
+ }
48
+
49
+ /** @param {any} request */
50
+ static key(request) {
51
+ const basis = JSON.stringify({
52
+ combo: request.combo ?? null,
53
+ messages: request.messages,
54
+ temperature: request.temperature ?? null,
55
+ max_tokens: request.max_tokens ?? null,
56
+ top_p: request.top_p ?? null,
57
+ stop: request.stop ?? null,
58
+ model: request.model ?? null,
59
+ });
60
+ return createHash('sha256').update(basis).digest('hex').slice(0, 32);
61
+ }
62
+
63
+ /**
64
+ * @param {string} key
65
+ * @param {string} [promptText] the query, for the semantic tier
66
+ * @returns {{value: any, kind: 'exact'|'semantic'}|null}
67
+ */
68
+ get(key, promptText) {
69
+ if (!this.enabled) return null;
70
+
71
+ const hit = this.map.get(key);
72
+ if (hit) {
73
+ if (hit.expiresAt <= this.now()) {
74
+ this.map.delete(key);
75
+ this._indexDirty = true;
76
+ } else {
77
+ this.map.delete(key);
78
+ this.map.set(key, hit); // refresh LRU position
79
+ this.hits += 1;
80
+ return { value: hit.value, kind: 'exact' };
81
+ }
82
+ }
83
+
84
+ const semantic = this.#semanticLookup(promptText);
85
+ if (semantic) {
86
+ this.semanticHits += 1;
87
+ this.hits += 1;
88
+ return semantic;
89
+ }
90
+
91
+ this.misses += 1;
92
+ return null;
93
+ }
94
+
95
+ /**
96
+ * @param {string} key
97
+ * @param {any} value
98
+ * @param {{promptText?: string}} [meta]
99
+ */
100
+ set(key, value, meta = {}) {
101
+ if (!this.enabled) return;
102
+ if (this.map.size >= this.maxEntries) {
103
+ const oldest = this.map.keys().next();
104
+ if (!oldest.done) {
105
+ this.map.delete(oldest.value);
106
+ this._indexDirty = true;
107
+ }
108
+ }
109
+ this.map.set(key, {
110
+ value,
111
+ expiresAt: this.now() + this.ttlMs,
112
+ promptText: this.semanticEnabled ? meta.promptText ?? null : null,
113
+ vector: this.semanticEnabled && this.semanticTier === 'embedding' && meta.promptText ? embed(meta.promptText) : null,
114
+ });
115
+ this._indexDirty = true;
116
+ }
117
+
118
+ /**
119
+ * @param {string|undefined} query
120
+ * @returns {{value: any, kind: 'semantic'}|null}
121
+ */
122
+ #semanticLookup(query) {
123
+ if (!this.semanticEnabled || !query) return null;
124
+
125
+ // B4.7: two tiers behind the same seam. Lexical is tf-idf from ml-kit;
126
+ // embedding is hashed vectors, which cost no model and no quota.
127
+ if (this.semanticTier === 'embedding') {
128
+ const queryVector = embed(query);
129
+ let best = null;
130
+ let bestScore = 0;
131
+ for (const [key, entry] of this.map) {
132
+ if (entry.expiresAt <= this.now() || !entry.promptText) continue;
133
+ const score = cosine(queryVector, entry.vector ?? (entry.vector = embed(entry.promptText)));
134
+ if (score > bestScore) {
135
+ bestScore = score;
136
+ best = { key, entry };
137
+ }
138
+ }
139
+ if (best && bestScore >= this.semanticThreshold) {
140
+ return { value: best.entry.value, kind: 'embedding', score: Math.round(bestScore * 1000) / 1000 };
141
+ }
142
+ return null;
143
+ }
144
+
145
+ const index = this.#ensureIndex();
146
+ if (!index || !index.keys.length) return null;
147
+
148
+ const hits = index.search(query, 1);
149
+ if (!hits.length) return null;
150
+ const top = hits[0];
151
+ if (top.score < this.semanticThreshold) return null;
152
+
153
+ const key = index.keys[top.i ?? top.index];
154
+ const entry = key ? this.map.get(key) : null;
155
+ if (!entry || entry.expiresAt <= this.now()) return null;
156
+ return { value: entry.value, kind: 'semantic', score: Math.round(top.score * 1000) / 1000 };
157
+ }
158
+
159
+ /** Build (or reuse) the tf-idf index over the cached prompts. */
160
+ #ensureIndex() {
161
+ if (!this._indexDirty && this._index) return this._index;
162
+ const keys = [];
163
+ const docs = [];
164
+ for (const [key, entry] of this.map) {
165
+ if (!entry.promptText) continue;
166
+ keys.push(key);
167
+ docs.push(entry.promptText);
168
+ }
169
+ if (!docs.length) {
170
+ this._index = { index: null, keys: [] };
171
+ this._indexDirty = false;
172
+ return this._index;
173
+ }
174
+ const MLKit = loadKit('ml-kit');
175
+ this._index = { index: MLKit.tfidf(docs), keys, search: (q, k) => MLKit.search(this._index.index, q, k) };
176
+ this._indexDirty = false;
177
+ return this._index;
178
+ }
179
+
180
+ stats() {
181
+ const total = this.hits + this.misses;
182
+ return {
183
+ enabled: this.enabled,
184
+ entries: this.map.size,
185
+ hits: this.hits,
186
+ semanticHits: this.semanticHits,
187
+ misses: this.misses,
188
+ hitRate: total ? Math.round((this.hits / total) * 100) / 100 : 0,
189
+ semantic: { enabled: this.semanticEnabled, threshold: this.semanticThreshold },
190
+ };
191
+ }
192
+
193
+ clear() {
194
+ this.map.clear();
195
+ this._index = null;
196
+ this._indexDirty = true;
197
+ }
198
+
199
+ /**
200
+ * Drop expired entries so a long-running gateway's memory is bounded by the
201
+ * TTL, not by the number of prompts it has ever seen. Called by the
202
+ * cache_sweep worker.
203
+ */
204
+ sweep() {
205
+ const now = this.now();
206
+ let removed = 0;
207
+ for (const [key, entry] of this.map.entries()) {
208
+ if (entry.expiresAt <= now) {
209
+ this.map.delete(key);
210
+ removed += 1;
211
+ }
212
+ }
213
+ if (removed) this._indexDirty = true;
214
+ return { removed, remaining: this.map.size };
215
+ }
216
+ }
package/src/drift.js ADDED
@@ -0,0 +1,174 @@
1
+ /**
2
+ * @drsamonline/telemetry/drift — B3.7, detecting that a target changed behaviour.
3
+ *
4
+ * Two standard detectors, because they answer different questions:
5
+ *
6
+ * EWMA "where is it now?" — a smoothed level, compared to a baseline.
7
+ * Fast to react, but a single bad minute can trigger it.
8
+ * CUSUM "how long has it been off?" — accumulates small deviations in both
9
+ * directions, so a slow 5% slide is caught even though no single
10
+ * sample looks wrong. This is the one a breaker cannot see.
11
+ *
12
+ * Both are deterministic and cheap: a few multiplications per sample. Neither
13
+ * needs a model, a window buffer, or a second request.
14
+ */
15
+
16
+ /** A one-sided CUSUM pair plus an EWMA, per series. */
17
+ export class DriftMonitor {
18
+ /**
19
+ * @param {{alpha?: number, slack?: number, threshold?: number, baselineSamples?: number}} [opts]
20
+ */
21
+ constructor(opts = {}) {
22
+ this.alpha = Number(opts.alpha ?? 0.2);
23
+ // slack is in standard deviations of the baseline: noise inside this band is
24
+ // not drift, it is variance, and treating it as drift is how alerting dies
25
+ this.slack = Number(opts.slack ?? 0.5);
26
+ this.threshold = Number(opts.threshold ?? 5);
27
+ this.baselineSamples = Number(opts.baselineSamples ?? 10);
28
+ // A perfect baseline has a standard deviation of zero, which makes every
29
+ // wobble infinitely significant. Floor it at a small fraction of the level.
30
+ this.minSdRatio = Number(opts.minSdRatio ?? 0.02);
31
+
32
+ /** @type {Map<string, any>} */
33
+ this.series = new Map();
34
+ }
35
+
36
+ #state(key) {
37
+ let state = this.series.get(key);
38
+ if (!state) {
39
+ state = {
40
+ ewma: null,
41
+ sum: 0,
42
+ sumSq: 0,
43
+ count: 0,
44
+ baselineMean: null,
45
+ baselineSd: null,
46
+ cusumUp: 0,
47
+ cusumDown: 0,
48
+ alarm: null,
49
+ latched: false,
50
+ samples: 0,
51
+ };
52
+ this.series.set(key, state);
53
+ }
54
+ return state;
55
+ }
56
+
57
+ /**
58
+ * @param {string} key
59
+ * @param {number} value
60
+ */
61
+ record(key, value) {
62
+ const state = this.#state(key);
63
+ const x = Number(value);
64
+ if (!Number.isFinite(x)) return this.snapshot(key);
65
+
66
+ state.samples += 1;
67
+ state.ewma = state.ewma === null ? x : this.alpha * x + (1 - this.alpha) * state.ewma;
68
+
69
+ // Build the baseline from the first N samples, then freeze it. A baseline that
70
+ // keeps moving with the drift can never detect the drift.
71
+ if (state.count < this.baselineSamples) {
72
+ state.count += 1;
73
+ state.sum += x;
74
+ state.sumSq += x * x;
75
+ if (state.count === this.baselineSamples) {
76
+ state.baselineMean = state.sum / state.count;
77
+ const variance = Math.max(0, state.sumSq / state.count - state.baselineMean ** 2);
78
+ const sd = Math.sqrt(variance);
79
+ state.baselineSd = Math.max(sd, Math.abs(state.baselineMean) * this.minSdRatio, 1e-6);
80
+ }
81
+ return this.snapshot(key);
82
+ }
83
+
84
+ const slack = this.slack * state.baselineSd;
85
+ const deviation = x - state.baselineMean;
86
+ state.cusumUp = Math.max(0, state.cusumUp + deviation - slack);
87
+ state.cusumDown = Math.max(0, state.cusumDown - deviation - slack);
88
+
89
+ if (state.cusumUp > this.threshold || state.cusumDown > this.threshold) {
90
+ const direction = state.cusumUp > this.threshold ? 'up' : 'down';
91
+ const cusum = direction === 'up' ? state.cusumUp : state.cusumDown;
92
+ state.alarm = { direction, cusum: Math.round(cusum * 100) / 100, at: state.samples };
93
+ // A signalled CUSUM restarts, or it grows without bound and stops meaning
94
+ // anything. The alarm stays latched until the level itself comes back.
95
+ state.cusumUp = 0;
96
+ state.cusumDown = 0;
97
+ state.latched = true;
98
+ } else if (state.latched) {
99
+ // Recovered when the smoothed level is back within two standard deviations
100
+ // of the baseline, not merely when one sample looks normal.
101
+ const back = Math.abs(state.ewma - state.baselineMean) <= 2 * state.baselineSd;
102
+ if (back) {
103
+ state.latched = false;
104
+ state.alarm = null;
105
+ }
106
+ }
107
+
108
+ return this.snapshot(key);
109
+ }
110
+
111
+ /** @param {string} key */
112
+ snapshot(key) {
113
+ const state = this.series.get(key);
114
+ if (!state) return null;
115
+ return {
116
+ key,
117
+ samples: state.samples,
118
+ ewma: state.ewma === null ? null : Math.round(state.ewma * 1000) / 1000,
119
+ baseline: state.baselineMean === null ? null : Math.round(state.baselineMean * 1000) / 1000,
120
+ sd: state.baselineSd === null ? null : Math.round(state.baselineSd * 1000) / 1000,
121
+ cusumUp: Math.round(state.cusumUp * 100) / 100,
122
+ cusumDown: Math.round(state.cusumDown * 100) / 100,
123
+ alarm: state.alarm,
124
+ latched: state.latched,
125
+ };
126
+ }
127
+
128
+ /** Every series that is currently alarming, worst first. */
129
+ alarms() {
130
+ return [...this.series.keys()]
131
+ .map((key) => this.snapshot(key))
132
+ .filter((s) => s?.alarm)
133
+ .sort((a, b) => (b?.alarm?.cusum ?? 0) - (a?.alarm?.cusum ?? 0));
134
+ }
135
+
136
+ /** Forget a series — used when a target is edited or the baseline is wrong. */
137
+ reset(key) {
138
+ if (key) this.series.delete(key);
139
+ else this.series.clear();
140
+ }
141
+ }
142
+
143
+ /**
144
+ * Read a ledger and report drift per model on latency and failure rate.
145
+ * Called by the worker, never on the request path.
146
+ *
147
+ * @param {{records: Array<any>, monitor?: DriftMonitor, since?: number, minSamples?: number}} input
148
+ */
149
+ export function driftFromLedger(input) {
150
+ const monitor = input.monitor ?? new DriftMonitor();
151
+ const since = input.since ?? 0;
152
+ const rows = input.records.filter((r) => (r.ts ?? 0) >= since);
153
+
154
+ /** @type {Map<string, {latency: number[], failures: number[]}>} */
155
+ const byModel = new Map();
156
+ for (const row of rows) {
157
+ const bucket = byModel.get(row.modelId) ?? { latency: [], failures: [] };
158
+ if (row.ok && Number.isFinite(row.latencyMs)) bucket.latency.push(row.latencyMs);
159
+ bucket.failures.push(row.ok ? 0 : 1);
160
+ byModel.set(row.modelId, bucket);
161
+ }
162
+
163
+ const minSamples = Number(input.minSamples ?? 10);
164
+ for (const [modelId, bucket] of byModel) {
165
+ if (bucket.latency.length >= minSamples) {
166
+ for (const value of bucket.latency) monitor.record(`${modelId}:latency`, value);
167
+ }
168
+ if (bucket.failures.length >= minSamples) {
169
+ for (const value of bucket.failures) monitor.record(`${modelId}:failures`, value);
170
+ }
171
+ }
172
+
173
+ return { monitor, alarms: monitor.alarms() };
174
+ }
@@ -0,0 +1,66 @@
1
+ /**
2
+ * B4.7 — the embedding tier, without a model.
3
+ *
4
+ * This is **feature hashing**, not a learned embedding: each token and each
5
+ * adjacent token pair is hashed into a fixed number of dimensions with a signed
6
+ * contribution, then the vector is normalised. Two prompts that share most of
7
+ * their wording land close together; two paraphrases that share no words do not.
8
+ *
9
+ * That limitation is the point of saying it out loud. A real embedding model
10
+ * needs a provider and quota; this needs neither, costs microseconds, and beats
11
+ * exact-hash matching for the case that actually happens — the same question
12
+ * asked again with a word changed.
13
+ */
14
+
15
+ const DIMS = 256;
16
+
17
+ /** FNV-1a, 32-bit: fast, deterministic, good enough for bucketing. */
18
+ function hash32(str) {
19
+ let h = 0x811c9dc5;
20
+ for (let i = 0; i < str.length; i++) {
21
+ h ^= str.charCodeAt(i);
22
+ h = Math.imul(h, 0x01000193);
23
+ }
24
+ return h >>> 0;
25
+ }
26
+
27
+ function tokens(text) {
28
+ return String(text ?? '')
29
+ .toLowerCase()
30
+ .split(/[^a-z0-9']+/)
31
+ .filter((t) => t.length > 0);
32
+ }
33
+
34
+ /**
35
+ * @param {string} text
36
+ * @param {number} [dims]
37
+ * @returns {Float64Array}
38
+ */
39
+ export function embed(text, dims = DIMS) {
40
+ const vector = new Float64Array(dims);
41
+ const words = tokens(text);
42
+
43
+ const add = (term, weight) => {
44
+ const h = hash32(term);
45
+ const index = h % dims;
46
+ const sign = (h >>> 31) === 0 ? 1 : -1; // signed hashing keeps collisions from only adding
47
+ vector[index] += sign * weight;
48
+ };
49
+
50
+ for (const word of words) add(word, 1);
51
+ for (let i = 0; i + 1 < words.length; i++) add(`${words[i]} ${words[i + 1]}`, 0.5);
52
+
53
+ let norm = 0;
54
+ for (let i = 0; i < dims; i++) norm += vector[i] * vector[i];
55
+ norm = Math.sqrt(norm);
56
+ if (norm > 0) for (let i = 0; i < dims; i++) vector[i] /= norm;
57
+ return vector;
58
+ }
59
+
60
+ /** @param {Float64Array} a @param {Float64Array} b */
61
+ export function cosine(a, b) {
62
+ if (!a || !b || a.length !== b.length) return 0;
63
+ let dot = 0;
64
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
65
+ return dot; // both vectors are already unit length
66
+ }
package/src/history.js ADDED
@@ -0,0 +1,177 @@
1
+ /**
2
+ * @drsamonline/telemetry/history — B2.7, health over time.
3
+ *
4
+ * The prober already keeps 24 observations per target in memory, which is enough
5
+ * for a sparkline and useless the moment the process restarts. This keeps a
6
+ * rolling history on disk so "has it been flaky all morning?" is answerable.
7
+ *
8
+ * Three constraints, because this is the one place that writes over time:
9
+ *
10
+ * - **Prompt-free, like everything else** (I11). A sample is a timestamp, a
11
+ * model id, a duration and a boolean. There is no field a prompt could go in,
12
+ * and `test/history.test.js` asserts the stored rows have exactly that shape.
13
+ * - **Bounded.** A cap on rows and on age, pruned on write, so a gateway left
14
+ * running for a month does not fill a disk.
15
+ * - **Cheap.** Appends are buffered and flushed in batches; a probe pass must
16
+ * not turn into a hundred file writes.
17
+ */
18
+
19
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
20
+ import { dirname } from 'node:path';
21
+
22
+ /** The only fields a sample may have. */
23
+ export const SAMPLE_FIELDS = ['ts', 'modelId', 'ok', 'latencyMs', 'state'];
24
+
25
+ export class HistoryStore {
26
+ /**
27
+ * @param {{file?: string|null, maxRows?: number, maxAgeMs?: number, now?: () => number}} [opts]
28
+ */
29
+ constructor(opts = {}) {
30
+ this.file = opts.file ?? null;
31
+ this.maxRows = Number(opts.maxRows ?? 20_000);
32
+ this.maxAgeMs = Number(opts.maxAgeMs ?? 7 * 24 * 3600_000);
33
+ this.now = opts.now ?? Date.now;
34
+
35
+ /** @type {Array<{ts: number, modelId: string, ok: boolean, latencyMs: number|null, state: string}>} */
36
+ this.rows = [];
37
+ this.dirty = 0;
38
+
39
+ if (this.file && existsSync(this.file)) this.#load();
40
+ }
41
+
42
+ #load() {
43
+ try {
44
+ const text = readFileSync(this.file, 'utf8');
45
+ const rows = [];
46
+ for (const line of text.split('\n')) {
47
+ if (!line.trim()) continue;
48
+ try {
49
+ const row = JSON.parse(line);
50
+ // a row with an unexpected field is dropped rather than trusted: this
51
+ // file is the one place a prompt could hide
52
+ const extra = Object.keys(row).filter((k) => !SAMPLE_FIELDS.includes(k));
53
+ if (extra.length) continue;
54
+ rows.push(row);
55
+ } catch {
56
+ /* a torn last line from a kill is not a reason to refuse the history */
57
+ }
58
+ }
59
+ this.rows = rows;
60
+ this.prune();
61
+ } catch {
62
+ this.rows = [];
63
+ }
64
+ }
65
+
66
+ /**
67
+ * @param {{modelId: string, ok: boolean, latencyMs?: number|null, state?: string, ts?: number}} sample
68
+ */
69
+ record(sample) {
70
+ const row = {
71
+ ts: sample.ts ?? this.now(),
72
+ modelId: String(sample.modelId),
73
+ ok: Boolean(sample.ok),
74
+ latencyMs: Number.isFinite(sample.latencyMs) ? Math.round(Number(sample.latencyMs)) : null,
75
+ state: String(sample.state ?? (sample.ok ? 'closed' : 'open')),
76
+ };
77
+ this.rows.push(row);
78
+ this.dirty += 1;
79
+
80
+ if (this.file) {
81
+ try {
82
+ mkdirSync(dirname(this.file), { recursive: true });
83
+ appendFileSync(this.file, `${JSON.stringify(row)}\n`);
84
+ } catch {
85
+ /* a history that cannot be written must not take the gateway down */
86
+ }
87
+ }
88
+
89
+ // prune on a threshold rather than every write: cheap, and bounded
90
+ if (this.rows.length > this.maxRows * 1.2) this.prune();
91
+ return row;
92
+ }
93
+
94
+ prune() {
95
+ const cutoff = this.now() - this.maxAgeMs;
96
+ const before = this.rows.length;
97
+ this.rows = this.rows.filter((r) => r.ts >= cutoff);
98
+ if (this.rows.length > this.maxRows) this.rows = this.rows.slice(-this.maxRows);
99
+
100
+ if (this.file && this.rows.length !== before) this.#rewrite();
101
+ return before - this.rows.length;
102
+ }
103
+
104
+ #rewrite() {
105
+ try {
106
+ const tmp = `${this.file}.tmp`;
107
+ writeFileSync(tmp, this.rows.map((r) => `${JSON.stringify(r)}\n`).join(''));
108
+ renameSync(tmp, this.file); // atomic: a reader never sees half a file
109
+ this.dirty = 0;
110
+ } catch {
111
+ /* keep the old file rather than a truncated one */
112
+ }
113
+ }
114
+
115
+ /** Which models have samples. */
116
+ models() {
117
+ return [...new Set(this.rows.map((r) => r.modelId))].sort();
118
+ }
119
+
120
+ /**
121
+ * A bucketed series for one model: enough for a sparkline, small enough to send.
122
+ *
123
+ * @param {string} modelId
124
+ * @param {{hours?: number, buckets?: number}} [opts]
125
+ */
126
+ series(modelId, opts = {}) {
127
+ const hours = Number(opts.hours ?? 24);
128
+ const buckets = Number(opts.buckets ?? 24);
129
+ const now = this.now();
130
+ const from = now - hours * 3600_000;
131
+ const width = (hours * 3600_000) / buckets;
132
+
133
+ const mine = this.rows.filter((r) => r.modelId === modelId && r.ts >= from);
134
+ /** @type {Array<any>} */
135
+ const points = [];
136
+ for (let i = 0; i < buckets; i++) {
137
+ const start = from + i * width;
138
+ const inBucket = mine.filter((r) => r.ts >= start && r.ts < start + width);
139
+ if (!inBucket.length) {
140
+ points.push({ t: Math.round(start), samples: 0, ok: null, p50: null, errorRate: null, state: null });
141
+ continue;
142
+ }
143
+ const latencies = inBucket.map((r) => r.latencyMs).filter((v) => v !== null).sort((a, b) => a - b);
144
+ const failures = inBucket.filter((r) => !r.ok).length;
145
+ points.push({
146
+ t: Math.round(start),
147
+ samples: inBucket.length,
148
+ ok: failures === 0,
149
+ p50: latencies.length ? latencies[Math.floor(latencies.length / 2)] : null,
150
+ errorRate: Math.round((failures / inBucket.length) * 1000) / 1000,
151
+ state: inBucket[inBucket.length - 1].state,
152
+ });
153
+ }
154
+
155
+ const observed = points.filter((p) => p.samples > 0);
156
+ const uptime = observed.length ? Math.round((observed.filter((p) => p.ok).length / observed.length) * 1000) / 1000 : null;
157
+ return { modelId, hours, buckets, points, uptime, samples: mine.length };
158
+ }
159
+
160
+ /** Every model's series in one call, for the Weather Map. */
161
+ all(opts = {}) {
162
+ return this.models().map((modelId) => this.series(modelId, opts));
163
+ }
164
+
165
+ /** Compact stats for `/api/status`: no point arrays. */
166
+ summary() {
167
+ const cutoff = this.now() - 24 * 3600_000;
168
+ const rows = this.rows.filter((r) => r.ts >= cutoff);
169
+ return {
170
+ rows: this.rows.length,
171
+ last24h: rows.length,
172
+ models: this.models().length,
173
+ oldest: this.rows.length ? this.rows[0].ts : null,
174
+ maxRows: this.maxRows,
175
+ };
176
+ }
177
+ }
package/src/index.js ADDED
@@ -0,0 +1,5 @@
1
+ /** @drsamonline/telemetry — the ledger, the cache, and the aggregates built from them. */
2
+ export * from './ledger.js';
3
+ export * from './cache.js';
4
+ export { DriftMonitor, driftFromLedger } from './drift.js';
5
+ export { HistoryStore, SAMPLE_FIELDS } from './history.js';
package/src/ledger.js ADDED
@@ -0,0 +1,404 @@
1
+ /**
2
+ * The ledger: one append-only record per completed turn.
3
+ *
4
+ * It is what makes "the same task at lower cost" a measurable claim rather than
5
+ * a slogan — every record carries provider, model, combo, arm (for A/B), tokens,
6
+ * cost, latency, time-to-first-token, cache and verification status.
7
+ *
8
+ * Prompt text is never stored unless guardrails.logPrompts is on. What is stored
9
+ * is a salted hash, so the same prompt can be recognised across turns (prompt-free
10
+ * usage view) without the prompt being recoverable.
11
+ */
12
+
13
+ import { createHash, randomBytes } from 'node:crypto';
14
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
15
+ import { dirname } from 'node:path';
16
+
17
+ /** @typedef {{
18
+ * ts: number, provider: string, model: string, modelId: string, combo: string, arm: string|null,
19
+ * ok: boolean, error: string|null, promptTokens: number, completionTokens: number,
20
+ * costUsd: number, latencyMs: number, ttftMs: number|null, cached: boolean,
21
+ * verified: boolean, promptHash: string, promptChars: number, streamed: boolean
22
+ * }} LedgerRecord */
23
+
24
+ export class Ledger {
25
+ /**
26
+ * @param {{file?: string|null, maxRecords?: number, salt?: string, hashSecretPath?: string}} [opts]
27
+ */
28
+ constructor(opts = {}) {
29
+ this.file = opts.file ?? null;
30
+ this.maxRecords = opts.maxRecords ?? 5000;
31
+ this.salt = opts.salt ?? 'sui-local-salt';
32
+ /**
33
+ * Per-day totals, kept outside the record window.
34
+ *
35
+ * The window is capped at maxRecords for memory, which is right — but a
36
+ * budget computed from the window silently stops counting once a day passes
37
+ * 5,000 turns, and a budget that stops counting is worse than no budget. This
38
+ * map is O(1) to update and does not depend on the window at all.
39
+ *
40
+ * @type {Map<string, {costUsd: number, turns: number, failed: number, cached: number, byProvider: Map<string, number>}>}
41
+ */
42
+ this.daily = new Map();
43
+ this.maxFileBytes = Number(opts.maxFileBytes ?? 64 * 1024 * 1024);
44
+ this.fileBytes = 0;
45
+ /** @type {LedgerRecord[]} */
46
+ this.records = [];
47
+ if (this.file) {
48
+ mkdirSync(dirname(this.file), { recursive: true });
49
+ if (existsSync(this.file)) this.#load();
50
+ }
51
+ }
52
+
53
+ #load() {
54
+ try {
55
+ const raw = readFileSync(this.file, 'utf8');
56
+ this.fileBytes = Buffer.byteLength(raw);
57
+ for (const line of raw.split('\n')) {
58
+ if (!line.trim()) continue;
59
+ let row;
60
+ try {
61
+ row = JSON.parse(line);
62
+ } catch {
63
+ continue; // a torn last line from a hard kill is not worth failing over
64
+ }
65
+ // every row counts towards its day, including rows that fall outside the
66
+ // window: the totals must not depend on how much history is held
67
+ this.#bumpDaily(row);
68
+ this.records.push(row);
69
+ }
70
+ if (this.records.length > this.maxRecords) {
71
+ this.records = this.records.slice(-this.maxRecords);
72
+ this.#rewrite();
73
+ }
74
+ } catch {
75
+ this.records = [];
76
+ }
77
+ }
78
+
79
+ /** @param {any} row */
80
+ #dayKey(row) {
81
+ const d = new Date(row?.ts ?? Date.now());
82
+ return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
83
+ }
84
+
85
+ /** @param {any} row */
86
+ #bumpDaily(row) {
87
+ const key = this.#dayKey(row);
88
+ let day = this.daily.get(key);
89
+ if (!day) {
90
+ day = { costUsd: 0, turns: 0, failed: 0, cached: 0, byProvider: new Map() };
91
+ this.daily.set(key, day);
92
+ // keep at most three days: yesterday is useful, last week is not
93
+ if (this.daily.size > 3) {
94
+ const oldest = [...this.daily.keys()].sort()[0];
95
+ this.daily.delete(oldest);
96
+ }
97
+ }
98
+ day.costUsd += Number(row.costUsd ?? 0);
99
+ if (row.ok) day.turns += 1;
100
+ else day.failed += 1;
101
+ if (row.cached) day.cached += 1;
102
+ const provider = row.provider ?? 'unknown';
103
+ day.byProvider.set(provider, (day.byProvider.get(provider) ?? 0) + Number(row.costUsd ?? 0));
104
+ }
105
+
106
+ /** The bucket for a given day, or an empty one. */
107
+ #day(now = new Date()) {
108
+ const key = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`;
109
+ return this.daily.get(key) ?? { costUsd: 0, turns: 0, failed: 0, cached: 0, byProvider: new Map() };
110
+ }
111
+
112
+ /**
113
+ * Rotate the file when it grows past the cap.
114
+ *
115
+ * Without this a gateway left running appends forever: at 20 rps a three-day
116
+ * soak writes millions of lines, and nothing prunes a file, only an array.
117
+ */
118
+ #rotateIfNeeded(nextBytes) {
119
+ if (!this.file) return;
120
+ this.fileBytes += nextBytes;
121
+ if (this.fileBytes <= this.maxFileBytes) return;
122
+ try {
123
+ renameSync(this.file, `${this.file}.1`);
124
+ // leave an empty file behind: a reader that stats the live path should not
125
+ // find it missing just because a rotation happened between two writes
126
+ writeFileSync(this.file, '');
127
+ this.fileBytes = 0;
128
+ } catch {
129
+ this.fileBytes = 0; // if the rename fails, keep appending rather than stop recording
130
+ }
131
+ }
132
+
133
+ #rewrite() {
134
+ if (!this.file) return;
135
+ writeFileSync(this.file, this.records.map((r) => JSON.stringify(r)).join('\n') + '\n');
136
+ }
137
+
138
+ /** Stable, non-reversible prompt fingerprint. */
139
+ hashPrompt(text) {
140
+ return createHash('sha256').update(this.salt).update(text).digest('hex').slice(0, 16);
141
+ }
142
+
143
+ /** @param {Partial<LedgerRecord> & {promptText?: string}} entry */
144
+ record(entry) {
145
+ /** @type {LedgerRecord} */
146
+ const rec = {
147
+ ts: entry.ts ?? Date.now(),
148
+ provider: entry.provider ?? 'unknown',
149
+ model: entry.model ?? 'unknown',
150
+ modelId: entry.modelId ?? 'unknown',
151
+ combo: entry.combo ?? 'unknown',
152
+ arm: entry.arm ?? null,
153
+ ok: entry.ok ?? true,
154
+ error: entry.error ?? null,
155
+ promptTokens: entry.promptTokens ?? 0,
156
+ completionTokens: entry.completionTokens ?? 0,
157
+ costUsd: entry.costUsd ?? 0,
158
+ latencyMs: entry.latencyMs ?? 0,
159
+ ttftMs: entry.ttftMs ?? null,
160
+ cached: entry.cached ?? false,
161
+ verified: entry.verified ?? false,
162
+ promptHash: entry.promptHash ?? (entry.promptText ? this.hashPrompt(entry.promptText) : ''),
163
+ promptChars: entry.promptChars ?? (entry.promptText ? entry.promptText.length : 0),
164
+ streamed: entry.streamed ?? false,
165
+ // B7.2: what compression saved on this turn, valued at the model's price.
166
+ compressionPreset: entry.compressionPreset ?? null,
167
+ compressionSavedTokens: entry.compressionSavedTokens ?? 0,
168
+ compressionSavedUsd: entry.compressionSavedUsd ?? 0,
169
+ // B6.6: which tenant this turn belongs to. A name, never a key.
170
+ apiKey: entry.apiKey ?? null,
171
+ // B7.1: who asked, and with what. Both are labels the caller supplies;
172
+ // the gateway never infers an identity it was not given.
173
+ user: entry.user ?? null,
174
+ tool: entry.tool ?? null,
175
+ promptCacheWarm: entry.promptCacheWarm ?? false,
176
+ };
177
+ this.records.push(rec);
178
+ this.#bumpDaily(rec);
179
+ if (this.records.length > this.maxRecords) this.records.shift();
180
+ if (this.file) {
181
+ try {
182
+ const line = `${JSON.stringify(rec)}\n`;
183
+ appendFileSync(this.file, line);
184
+ this.#rotateIfNeeded(Buffer.byteLength(line));
185
+ } catch {
186
+ /* ledger is observability, never the request path's failure */
187
+ }
188
+ }
189
+ return rec;
190
+ }
191
+
192
+ /**
193
+ * @param {{since?: number, combo?: string}} [opts]
194
+ * @returns {LedgerRecord[]}
195
+ */
196
+ entries(opts = {}) {
197
+ const since = opts.since ?? 0;
198
+ return this.records.filter((r) => r.ts >= since && (!opts.combo || r.combo === opts.combo));
199
+ }
200
+
201
+ /** Spend since local midnight, in USD. */
202
+ todaySpendUsd(now = new Date()) {
203
+ // from the daily bucket, not from the record window: a budget that stops
204
+ // counting after 5,000 turns is not a budget
205
+ return round6(this.#day(now).costUsd);
206
+ }
207
+
208
+ providerSpendToday(provider, now = new Date()) {
209
+ // same rule as todaySpendUsd: from the day bucket, not the window
210
+ return round6(this.#day(now).byProvider.get(provider) ?? 0);
211
+ }
212
+
213
+ /** @deprecated kept for callers that still read the window; use providerSpendToday */
214
+ #providerSpendFromWindow(provider, now = new Date()) {
215
+ const start = new Date(now);
216
+ start.setHours(0, 0, 0, 0);
217
+ return round6(
218
+ this.records
219
+ .filter((r) => r.ts >= start.getTime() && r.provider === provider)
220
+ .reduce((a, r) => a + r.costUsd, 0),
221
+ );
222
+ }
223
+
224
+ /**
225
+ * @param {{since?: number}} [opts]
226
+ */
227
+ aggregate(opts = {}) {
228
+ const rows = this.entries(opts);
229
+ // B7.1: attribution uses the same grouping shape as provider/model/combo, so
230
+ // one `bump` does all of it and nothing can drift.
231
+ /** @type {Record<string, any>} */
232
+ const byUser = {};
233
+ /** @type {Record<string, any>} */
234
+ const byTool = {};
235
+ for (const row of rows) {
236
+ bump(byUser, row.user ?? 'unattributed', row);
237
+ bump(byTool, row.tool ?? 'unattributed', row);
238
+ }
239
+
240
+ const totals = {
241
+ // A turn is a request that produced an answer. A failed attempt is
242
+ // recorded too (it is how failover is auditable), but it is not a turn.
243
+ turns: rows.filter((r) => r.ok).length,
244
+ // B7.2 Savings Ledger: dollars not spent, and the tokens that bought them.
245
+ savings: { cacheUsd: 0, compressionUsd: 0, totalUsd: 0, tokensSaved: 0, cacheHits: 0, compressedTurns: 0 },
246
+ attempts: rows.length,
247
+ ok: rows.filter((r) => r.ok).length,
248
+ failed: rows.filter((r) => !r.ok).length,
249
+ cached: rows.filter((r) => r.cached).length,
250
+ costUsd: 0,
251
+ savedUsd: 0,
252
+ promptTokens: 0,
253
+ completionTokens: 0,
254
+ avgLatencyMs: 0,
255
+ p50LatencyMs: 0,
256
+ p95LatencyMs: 0,
257
+ avgTtftMs: 0,
258
+ };
259
+
260
+ /** @type {Record<string, any>} */
261
+ const byProvider = {};
262
+ /** @type {Record<string, any>} */
263
+ const byModel = {};
264
+ /** @type {Record<string, any>} */
265
+ const byCombo = {};
266
+ /** @type {Record<string, any>} */
267
+ const byArm = {};
268
+
269
+ for (const r of rows) {
270
+ totals.costUsd += r.costUsd;
271
+ totals.promptTokens += r.promptTokens;
272
+ totals.completionTokens += r.completionTokens;
273
+ if (r.cached) {
274
+ totals.savedUsd += r.costUsd;
275
+ totals.savings.cacheUsd += r.costUsd;
276
+ totals.savings.cacheHits += 1;
277
+ }
278
+ if (r.compressionSavedUsd) {
279
+ totals.savings.compressionUsd += r.compressionSavedUsd;
280
+ totals.savings.tokensSaved += r.compressionSavedTokens ?? 0;
281
+ totals.savings.compressedTurns += 1;
282
+ }
283
+
284
+ bump(byProvider, r.provider, r);
285
+ bump(byModel, r.modelId, r);
286
+ bump(byCombo, r.combo, r);
287
+ if (r.arm) bump(byArm, `${r.combo}/${r.arm}`, r);
288
+ }
289
+
290
+ // A cache hit answers in microseconds and would drag p50 to zero; it is not
291
+ // a model latency, so it is excluded from latency statistics on purpose.
292
+ const latencies = rows
293
+ .filter((r) => r.ok && !r.cached)
294
+ .map((r) => r.latencyMs)
295
+ .sort((a, b) => a - b);
296
+ const ttfts = rows.filter((r) => r.ok && r.ttftMs != null).map((r) => /** @type {number} */ (r.ttftMs));
297
+ totals.costUsd = round6(totals.costUsd);
298
+ totals.savings.cacheUsd = round6(totals.savings.cacheUsd);
299
+ totals.savings.compressionUsd = round6(totals.savings.compressionUsd);
300
+ totals.savings.totalUsd = round6(totals.savings.cacheUsd + totals.savings.compressionUsd);
301
+ // `savedUsd` predates the savings block and is kept for older readers.
302
+ totals.savedUsd = round6(totals.savings.totalUsd);
303
+ totals.avgLatencyMs = latencies.length ? Math.round(mean(latencies)) : 0;
304
+ totals.p50LatencyMs = quantile(latencies, 0.5);
305
+ totals.p95LatencyMs = quantile(latencies, 0.95);
306
+ totals.avgTtftMs = ttfts.length ? Math.round(mean(ttfts)) : 0;
307
+
308
+ for (const group of [byProvider, byModel, byCombo, byArm]) {
309
+ for (const v of Object.values(group)) {
310
+ v.costUsd = round6(v.costUsd);
311
+ v.avgLatencyMs = v.latencySamples ? Math.round(v.latencySum / v.latencySamples) : 0;
312
+ delete v.latencySum;
313
+ delete v.latencySamples;
314
+ }
315
+ }
316
+
317
+ return {
318
+ since: opts.since ?? 0,
319
+ totals,
320
+ byProvider,
321
+ byModel,
322
+ byCombo,
323
+ byArm,
324
+ // B7.1: who asked and with what, so cost can be charged back
325
+ byUser,
326
+ byTool,
327
+ recent: rows.slice(-50).reverse(),
328
+ };
329
+ }
330
+
331
+ /** Prompt-free usage view: which prompt fingerprints recurred, and what they cost. */
332
+ promptUsage(limit = 50) {
333
+ /** @type {Map<string, {promptHash: string, turns: number, chars: number, costUsd: number, combos: Set<string>, models: Set<string>}>} */
334
+ const map = new Map();
335
+ for (const r of this.records) {
336
+ if (!r.promptHash) continue;
337
+ let e = map.get(r.promptHash);
338
+ if (!e) {
339
+ e = { promptHash: r.promptHash, turns: 0, chars: r.promptChars, costUsd: 0, combos: new Set(), models: new Set() };
340
+ map.set(r.promptHash, e);
341
+ }
342
+ e.turns += 1;
343
+ e.costUsd = round6(e.costUsd + r.costUsd);
344
+ e.combos.add(r.combo);
345
+ e.models.add(r.modelId);
346
+ }
347
+ return [...map.values()]
348
+ .map((e) => ({ ...e, combos: [...e.combos], models: [...e.models] }))
349
+ .sort((a, b) => b.costUsd - a.costUsd)
350
+ .slice(0, limit);
351
+ }
352
+
353
+ csv() {
354
+ const header = [
355
+ 'timestamp_iso', 'provider', 'model', 'model_id', 'combo', 'arm', 'ok', 'error',
356
+ 'prompt_tokens', 'completion_tokens', 'cost_usd', 'latency_ms', 'ttft_ms',
357
+ 'cached', 'verified', 'streamed', 'prompt_hash', 'prompt_chars',
358
+ 'compression_preset', 'compression_saved_tokens', 'compression_saved_usd',
359
+ 'api_key', 'prompt_cache_warm', 'user', 'tool',
360
+ ].join(',');
361
+ const lines = this.records.map((r) =>
362
+ [
363
+ new Date(r.ts).toISOString(), r.provider, r.model, r.modelId, r.combo, r.arm ?? '',
364
+ r.ok, r.error ?? '', r.promptTokens, r.completionTokens, r.costUsd, r.latencyMs,
365
+ r.ttftMs ?? '', r.cached, r.verified, r.streamed, r.promptHash, r.promptChars,
366
+ r.compressionPreset ?? '', r.compressionSavedTokens ?? 0, r.compressionSavedUsd ?? 0,
367
+ r.apiKey ?? '', r.promptCacheWarm ?? false, r.user ?? '', r.tool ?? '',
368
+ ]
369
+ .map((v) => (typeof v === 'string' && /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v))
370
+ .join(','),
371
+ );
372
+ return [header, ...lines].join('\n');
373
+ }
374
+ }
375
+
376
+ function bump(group, key, r) {
377
+ const e = (group[key] ??= { turns: 0, ok: 0, failed: 0, cached: 0, costUsd: 0, tokens: 0, latencySum: 0 });
378
+ e.turns += 1;
379
+ if (r.ok) e.ok += 1;
380
+ else e.failed += 1;
381
+ if (r.cached) e.cached += 1;
382
+ e.costUsd += r.costUsd;
383
+ e.tokens += r.promptTokens + r.completionTokens;
384
+ if (r.ok && !r.cached) {
385
+ e.latencySum += r.latencyMs;
386
+ e.latencySamples = (e.latencySamples ?? 0) + 1;
387
+ }
388
+ }
389
+
390
+ function mean(a) {
391
+ return a.reduce((x, y) => x + y, 0) / a.length;
392
+ }
393
+
394
+ function quantile(sorted, q) {
395
+ if (!sorted.length) return 0;
396
+ const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1));
397
+ return sorted[idx];
398
+ }
399
+
400
+ function round6(n) {
401
+ return Math.round(n * 1e6) / 1e6;
402
+ }
403
+
404
+ export { randomBytes };
package/README.md DELETED
@@ -1,3 +0,0 @@
1
- # Temporary Holding Version
2
-
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.