@bytesbrains/cruise 0.0.0-stage → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +59 -2
- package/dist/budget-FIRNSN67.mjs +132 -0
- package/dist/chunk-2FB2T3UI.mjs +122 -0
- package/dist/chunk-2NVDAK5J.mjs +76 -0
- package/dist/chunk-42ERHAJD.mjs +79 -0
- package/dist/chunk-4447IUTF.mjs +7 -0
- package/dist/chunk-5TVD3ZAI.mjs +69 -0
- package/dist/chunk-7NQHBXVZ.mjs +104 -0
- package/dist/chunk-BNP63KB2.mjs +42 -0
- package/dist/chunk-FGLMW5IY.mjs +575 -0
- package/dist/chunk-GUZNI3NS.mjs +267 -0
- package/dist/chunk-HQDQTXRX.mjs +57 -0
- package/dist/chunk-ICJVJSTC.mjs +42 -0
- package/dist/chunk-JZPUTXM7.mjs +236 -0
- package/dist/chunk-LALZUKEC.mjs +137 -0
- package/dist/chunk-N3YKWKOA.mjs +9 -0
- package/dist/chunk-O7TNRKSA.mjs +2 -0
- package/dist/chunk-RT4R2W6H.mjs +21 -0
- package/dist/chunk-SYTYOTO4.mjs +18 -0
- package/dist/chunk-T3ICD43U.mjs +15 -0
- package/dist/chunk-THEXUHQC.mjs +98 -0
- package/dist/chunk-UNXM6FSM.mjs +83 -0
- package/dist/chunk-WS6NFRIU.mjs +66 -0
- package/dist/chunk-XOH3XKM3.mjs +126 -0
- package/dist/compress-3DWG35BY.mjs +49 -0
- package/dist/conformance-NPFDYCQL.mjs +922 -0
- package/dist/cruise.mjs +693 -0
- package/dist/fallback-P5KK55M4.mjs +139 -0
- package/dist/keys-FWPQPMTW.mjs +127 -0
- package/dist/ledger-XBGWXY6C.mjs +97 -0
- package/dist/member-ZBV4YG2M.mjs +74 -0
- package/dist/report-VRYJ3W5L.mjs +414 -0
- package/dist/service-key-N2RZ7FMZ.mjs +59 -0
- package/dist/smoke-ZNYX6RO5.mjs +105 -0
- package/dist/sweep-SZX7OIDK.mjs +33 -0
- package/dist/tenant-4YDGI23Q.mjs +122 -0
- package/dist/tier-4L3SPLFP.mjs +116 -0
- package/package.json +38 -4
- package/templates/docker-compose.yml +66 -0
- package/templates/env.example +44 -0
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Built by packages/cli/build.ts.
|
|
3
|
+
import {
|
|
4
|
+
compressionSavings,
|
|
5
|
+
savedShare
|
|
6
|
+
} from "./chunk-HQDQTXRX.mjs";
|
|
7
|
+
import {
|
|
8
|
+
laneSavings
|
|
9
|
+
} from "./chunk-42ERHAJD.mjs";
|
|
10
|
+
import {
|
|
11
|
+
rescuesBy
|
|
12
|
+
} from "./chunk-BNP63KB2.mjs";
|
|
13
|
+
import "./chunk-SYTYOTO4.mjs";
|
|
14
|
+
import "./chunk-O7TNRKSA.mjs";
|
|
15
|
+
import "./chunk-T3ICD43U.mjs";
|
|
16
|
+
import {
|
|
17
|
+
formatUsd
|
|
18
|
+
} from "./chunk-THEXUHQC.mjs";
|
|
19
|
+
import "./chunk-5TVD3ZAI.mjs";
|
|
20
|
+
import {
|
|
21
|
+
connect,
|
|
22
|
+
environmentOf,
|
|
23
|
+
parseArgs,
|
|
24
|
+
run
|
|
25
|
+
} from "./chunk-7NQHBXVZ.mjs";
|
|
26
|
+
import "./chunk-JZPUTXM7.mjs";
|
|
27
|
+
|
|
28
|
+
// src/report-cache.ts
|
|
29
|
+
async function byLaneCache(db, month, tenantId) {
|
|
30
|
+
const { results } = await db.prepare(
|
|
31
|
+
`SELECT l.lane AS lane,
|
|
32
|
+
l.traffic_class AS trafficClass,
|
|
33
|
+
count(*) AS requests,
|
|
34
|
+
sum(CASE WHEN l.cache_status = 'HIT' AND l.cache_scope = 'model' THEN 1 ELSE 0 END) AS modelHits,
|
|
35
|
+
sum(CASE WHEN l.cache_status = 'HIT' AND l.cache_scope = 'lane' THEN 1 ELSE 0 END) AS laneHits
|
|
36
|
+
FROM ledger l
|
|
37
|
+
WHERE (?1 IS NULL OR strftime('%Y-%m', l.at) = ?1)
|
|
38
|
+
AND (?2 IS NULL OR l.tenant_id = ?2)
|
|
39
|
+
AND l.requested_model = l.lane
|
|
40
|
+
AND EXISTS (SELECT 1 FROM lanes ln WHERE ln.id = l.lane)
|
|
41
|
+
GROUP BY l.lane, l.traffic_class
|
|
42
|
+
ORDER BY l.lane, l.traffic_class`
|
|
43
|
+
).bind(month, tenantId ?? null).all();
|
|
44
|
+
return results.map((row) => ({
|
|
45
|
+
...row,
|
|
46
|
+
modelHitRate: row.requests > 0 ? row.modelHits / row.requests : null,
|
|
47
|
+
laneHitRate: row.requests > 0 ? row.laneHits / row.requests : null
|
|
48
|
+
}));
|
|
49
|
+
}
|
|
50
|
+
var VIOLATION_WINDOW_MINUTES = 10;
|
|
51
|
+
async function tierViolations(db, month, tenantId, windowMinutes = VIOLATION_WINDOW_MINUTES) {
|
|
52
|
+
const { results } = await db.prepare(
|
|
53
|
+
`WITH hits AS (
|
|
54
|
+
SELECT id, rowid AS seq, tenant_id, lane, key_prefix, cache_digest, at
|
|
55
|
+
FROM ledger
|
|
56
|
+
WHERE cache_status = 'HIT' AND cache_scope = 'lane' AND cache_digest IS NOT NULL
|
|
57
|
+
AND (?1 IS NULL OR strftime('%Y-%m', at) = ?1)
|
|
58
|
+
AND (?2 IS NULL OR tenant_id = ?2)
|
|
59
|
+
)
|
|
60
|
+
SELECT h.lane AS lane,
|
|
61
|
+
count(*) AS laneHits,
|
|
62
|
+
sum(EXISTS (
|
|
63
|
+
SELECT 1 FROM ledger r
|
|
64
|
+
WHERE r.tenant_id = h.tenant_id AND r.key_prefix = h.key_prefix AND r.cache_digest = h.cache_digest
|
|
65
|
+
AND (r.at > h.at OR (r.at = h.at AND r.rowid > h.seq))
|
|
66
|
+
AND (julianday(r.at) - julianday(h.at)) * 1440.0 <= ?3
|
|
67
|
+
AND r.cache_requested IN ('off', 'model')
|
|
68
|
+
)) AS suspected
|
|
69
|
+
FROM hits h
|
|
70
|
+
GROUP BY h.lane
|
|
71
|
+
ORDER BY h.lane`
|
|
72
|
+
).bind(month, tenantId ?? null, windowMinutes).all();
|
|
73
|
+
return results;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// src/report.ts
|
|
77
|
+
async function observationWindow(db) {
|
|
78
|
+
const row = await db.prepare(
|
|
79
|
+
`SELECT min(at) AS firstAt, max(at) AS lastAt, count(*) AS requests,
|
|
80
|
+
CAST((julianday(max(at)) - julianday(min(at))) AS REAL) AS days
|
|
81
|
+
FROM ledger`
|
|
82
|
+
).first();
|
|
83
|
+
return {
|
|
84
|
+
firstAt: row?.firstAt ?? null,
|
|
85
|
+
lastAt: row?.lastAt ?? null,
|
|
86
|
+
days: row?.days ?? 0,
|
|
87
|
+
requests: row?.requests ?? 0
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
async function byTrafficClass(db) {
|
|
91
|
+
const { results } = await db.prepare(
|
|
92
|
+
`SELECT traffic_class AS trafficClass,
|
|
93
|
+
count(*) AS requests,
|
|
94
|
+
sum(CASE WHEN cache_status = 'HIT' THEN 1 ELSE 0 END) AS hits,
|
|
95
|
+
sum(CASE WHEN streamed = 1 THEN 1 ELSE 0 END) AS streamedCount
|
|
96
|
+
FROM ledger
|
|
97
|
+
GROUP BY traffic_class
|
|
98
|
+
ORDER BY requests DESC`
|
|
99
|
+
).all();
|
|
100
|
+
const rows = [];
|
|
101
|
+
for (const row of results) {
|
|
102
|
+
const timing = await db.prepare(
|
|
103
|
+
`SELECT
|
|
104
|
+
(SELECT CAST(avg(upstream_ms) AS INTEGER) FROM ledger
|
|
105
|
+
WHERE traffic_class = ?1 AND cache_status != 'HIT' AND upstream_ms IS NOT NULL) AS miss,
|
|
106
|
+
(SELECT CAST(avg(upstream_ms) AS INTEGER) FROM ledger
|
|
107
|
+
WHERE traffic_class = ?1 AND cache_status = 'HIT' AND upstream_ms IS NOT NULL) AS hit`
|
|
108
|
+
).bind(row.trafficClass).first();
|
|
109
|
+
rows.push({
|
|
110
|
+
trafficClass: row.trafficClass,
|
|
111
|
+
requests: row.requests,
|
|
112
|
+
hits: row.hits,
|
|
113
|
+
hitRate: row.requests > 0 ? row.hits / row.requests : null,
|
|
114
|
+
streamedShare: row.requests > 0 ? row.streamedCount / row.requests : null,
|
|
115
|
+
medianMissMs: timing?.miss ?? null,
|
|
116
|
+
medianHitMs: timing?.hit ?? null
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return rows;
|
|
120
|
+
}
|
|
121
|
+
async function byCacheScope(db) {
|
|
122
|
+
const { results } = await db.prepare(
|
|
123
|
+
`SELECT traffic_class AS trafficClass,
|
|
124
|
+
cache_scope AS scope,
|
|
125
|
+
count(*) AS hits,
|
|
126
|
+
COALESCE(sum(cost_micros), 0) AS micros,
|
|
127
|
+
COALESCE(sum(list_micros), 0) AS listMicros
|
|
128
|
+
FROM ledger
|
|
129
|
+
WHERE cache_scope IS NOT NULL
|
|
130
|
+
GROUP BY traffic_class, cache_scope
|
|
131
|
+
ORDER BY hits DESC`
|
|
132
|
+
).all();
|
|
133
|
+
return results;
|
|
134
|
+
}
|
|
135
|
+
async function reasoningExhaustedSpend(db) {
|
|
136
|
+
const { results } = await db.prepare(
|
|
137
|
+
`SELECT model,
|
|
138
|
+
sum(CASE WHEN completion_tokens > 0 AND reasoning_tokens = completion_tokens THEN 1 ELSE 0 END) AS requests,
|
|
139
|
+
COALESCE(sum(CASE WHEN completion_tokens > 0 AND reasoning_tokens = completion_tokens THEN cost_micros ELSE 0 END), 0) AS micros,
|
|
140
|
+
count(*) AS empties
|
|
141
|
+
FROM ledger
|
|
142
|
+
WHERE outcome = 'empty'
|
|
143
|
+
GROUP BY model
|
|
144
|
+
HAVING requests > 0
|
|
145
|
+
ORDER BY micros DESC`
|
|
146
|
+
).all();
|
|
147
|
+
return results;
|
|
148
|
+
}
|
|
149
|
+
async function fallbacks(db) {
|
|
150
|
+
const { results } = await db.prepare(
|
|
151
|
+
`SELECT requested_model AS requestedModel, model AS servedModel, count(*) AS times
|
|
152
|
+
FROM ledger
|
|
153
|
+
WHERE requested_model IS NOT NULL AND requested_model != model
|
|
154
|
+
-- A lane request always differs from what served; that is allocation,
|
|
155
|
+
-- not a chain firing, and has its own section (0027).
|
|
156
|
+
AND requested_model NOT LIKE 'bb/%'
|
|
157
|
+
GROUP BY requested_model, model
|
|
158
|
+
ORDER BY times DESC`
|
|
159
|
+
).all();
|
|
160
|
+
return results;
|
|
161
|
+
}
|
|
162
|
+
async function priceProblems(db) {
|
|
163
|
+
const { results } = await db.prepare(
|
|
164
|
+
`SELECT m.id AS model,
|
|
165
|
+
CASE WHEN EXISTS (SELECT 1 FROM model_prices p WHERE p.model = m.id)
|
|
166
|
+
THEN 'future' ELSE 'missing' END AS kind,
|
|
167
|
+
(SELECT min(p.effective_from) FROM model_prices p
|
|
168
|
+
WHERE p.model = m.id AND p.effective_from > date('now')) AS startsOn
|
|
169
|
+
FROM models m
|
|
170
|
+
JOIN upstreams u ON u.id = m.upstream_id
|
|
171
|
+
WHERE m.enabled = 1 AND u.enabled = 1
|
|
172
|
+
AND NOT EXISTS (
|
|
173
|
+
SELECT 1 FROM model_prices p
|
|
174
|
+
WHERE p.model = m.id AND p.effective_from <= date('now'))
|
|
175
|
+
-- A speech row is priced by its character line, not a token row
|
|
176
|
+
-- (0059, src/billing-block.ts); read the log that holds it.
|
|
177
|
+
AND NOT (m.modality = 'speech' AND EXISTS (
|
|
178
|
+
SELECT 1 FROM model_components c
|
|
179
|
+
WHERE c.model = m.id AND c.component = 'character'
|
|
180
|
+
AND c.effective_from <= date('now')))
|
|
181
|
+
ORDER BY m.id`
|
|
182
|
+
).all();
|
|
183
|
+
return results;
|
|
184
|
+
}
|
|
185
|
+
async function byProvider(db) {
|
|
186
|
+
const { results } = await db.prepare(
|
|
187
|
+
`SELECT upstream_id AS upstreamId,
|
|
188
|
+
count(*) AS requests,
|
|
189
|
+
COALESCE(sum(cost_micros), 0) AS micros,
|
|
190
|
+
sum(CASE WHEN http_status >= 500 THEN 1 ELSE 0 END) AS errors,
|
|
191
|
+
sum(CASE WHEN http_status = 429 THEN 1 ELSE 0 END) AS rateLimited
|
|
192
|
+
FROM ledger
|
|
193
|
+
GROUP BY upstream_id
|
|
194
|
+
ORDER BY micros DESC`
|
|
195
|
+
).all();
|
|
196
|
+
return results;
|
|
197
|
+
}
|
|
198
|
+
var MIN_DAYS = 14;
|
|
199
|
+
var MIN_REQUESTS = 1e3;
|
|
200
|
+
function verdict(window) {
|
|
201
|
+
if (window.requests < MIN_REQUESTS) {
|
|
202
|
+
return {
|
|
203
|
+
sufficient: false,
|
|
204
|
+
why: `${window.requests} requests is below the ${MIN_REQUESTS} needed for any rate here to mean much.`
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
if (window.days < MIN_DAYS) {
|
|
208
|
+
return {
|
|
209
|
+
sufficient: false,
|
|
210
|
+
why: `${window.days.toFixed(1)} days is below the ${MIN_DAYS} needed to survive a weekly cycle.`
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
return { sufficient: true, why: `${window.requests} requests over ${window.days.toFixed(1)} days.` };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// src/selection.ts
|
|
217
|
+
async function costPerCompleted(db, tenantId, since) {
|
|
218
|
+
const { results } = await db.prepare(
|
|
219
|
+
`SELECT model,
|
|
220
|
+
count(*) AS requests,
|
|
221
|
+
sum(CASE WHEN outcome = 'served' THEN 1 ELSE 0 END) AS completed,
|
|
222
|
+
COALESCE(sum(cost_micros), 0) AS micros
|
|
223
|
+
FROM ledger
|
|
224
|
+
WHERE tenant_id = ?1 AND at >= ?2
|
|
225
|
+
GROUP BY model
|
|
226
|
+
ORDER BY micros DESC`
|
|
227
|
+
).bind(tenantId, since).all();
|
|
228
|
+
return results.map((row) => ({
|
|
229
|
+
model: row.model,
|
|
230
|
+
requests: row.requests,
|
|
231
|
+
completed: row.completed,
|
|
232
|
+
micros: row.micros,
|
|
233
|
+
microsPerCompleted: row.completed > 0 ? Math.round(row.micros / row.completed) : null,
|
|
234
|
+
completionRate: row.requests > 0 ? row.completed / row.requests : 0
|
|
235
|
+
}));
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// scripts/report.ts
|
|
239
|
+
var pct = (value) => value === null ? "-" : `${(value * 100).toFixed(1)}%`;
|
|
240
|
+
var ms = (value) => value === null ? "-" : `${value}ms`;
|
|
241
|
+
run(async () => {
|
|
242
|
+
const args = parseArgs(process.argv.slice(2));
|
|
243
|
+
const db = connect(environmentOf(args));
|
|
244
|
+
const window = await observationWindow(db);
|
|
245
|
+
const status = verdict(window);
|
|
246
|
+
console.log("Cruise \u2014 phase 1 measurement report\n");
|
|
247
|
+
console.log(`observation window ${window.firstAt ?? "-"} \u2192 ${window.lastAt ?? "-"}`);
|
|
248
|
+
console.log(` ${window.requests} requests over ${window.days.toFixed(1)} days`);
|
|
249
|
+
if (!status.sufficient) {
|
|
250
|
+
console.log(`
|
|
251
|
+
\u26A0 PROVISIONAL \u2014 ${status.why}`);
|
|
252
|
+
console.log(" No gate in v0.md \xA76 should be decided on this. The traffic that makes these");
|
|
253
|
+
console.log(" numbers real is the traffic this build releases.");
|
|
254
|
+
} else {
|
|
255
|
+
console.log(`
|
|
256
|
+
sufficient \u2014 ${status.why}`);
|
|
257
|
+
}
|
|
258
|
+
const classes = await byTrafficClass(db);
|
|
259
|
+
console.log("\n\ncache behaviour, by traffic class \u2014 never blended");
|
|
260
|
+
console.log("class reqs hit rate streamed miss TTFB hit TTFB");
|
|
261
|
+
if (classes.length === 0) console.log(" (no traffic)");
|
|
262
|
+
for (const row of classes) {
|
|
263
|
+
console.log(
|
|
264
|
+
`${row.trafficClass.padEnd(14)} ${String(row.requests).padStart(4)} ${pct(row.hitRate).padStart(8)} ${pct(row.streamedShare).padStart(8)} ${ms(row.medianMissMs).padStart(9)} ${ms(row.medianHitMs).padStart(8)}`
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
console.log("\n Scheduled pipelines re-run near-identical prompts by construction, so only");
|
|
268
|
+
console.log(" the interactive and agentic rows generalise to anyone else. A blended figure");
|
|
269
|
+
console.log(" would flatter us badly, which is why one is not printed.");
|
|
270
|
+
console.log(" One hit rate, not two: the platform caches streams (measured 2026-08-13), so");
|
|
271
|
+
console.log(" the served/shadow split v0.md \xA77 once required would now describe nothing.");
|
|
272
|
+
const scopes = await byCacheScope(db);
|
|
273
|
+
console.log("\n\ncache hits by scope and class \u2014 never blended");
|
|
274
|
+
console.log("class scope hits charged at list");
|
|
275
|
+
if (scopes.length === 0) console.log(" (nothing has been served from the cache)");
|
|
276
|
+
for (const row of scopes) {
|
|
277
|
+
console.log(
|
|
278
|
+
`${row.trafficClass.padEnd(14)} ${row.scope.padEnd(9)} ${String(row.hits).padStart(4)} ${formatUsd(row.micros).padStart(10)} ${formatUsd(row.listMicros).padStart(12)}`
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
const providers = await byProvider(db);
|
|
282
|
+
console.log("\n\nspend and reliability, by provider");
|
|
283
|
+
console.log("provider reqs spend 5xx 429");
|
|
284
|
+
if (providers.length === 0) console.log(" (no traffic)");
|
|
285
|
+
for (const row of providers) {
|
|
286
|
+
console.log(
|
|
287
|
+
`${row.upstreamId.padEnd(15)} ${String(row.requests).padStart(4)} ${formatUsd(row.micros, 4).padStart(11)} ${String(row.errors).padStart(5)} ${String(row.rateLimited).padStart(5)}`
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
const completed = await costPerCompleted(db, "bytesbrains", "2000-01-01");
|
|
291
|
+
console.log("\n\ncost per completed request \u2014 what an answer actually costs");
|
|
292
|
+
console.log("model reqs done spend per done");
|
|
293
|
+
if (completed.length === 0) console.log(" (no traffic)");
|
|
294
|
+
for (const row of completed) {
|
|
295
|
+
console.log(
|
|
296
|
+
`${row.model.slice(0, 44).padEnd(45)} ${String(row.requests).padStart(4)} ${pct(row.completionRate).padStart(5)} ${formatUsd(row.micros, 4).padStart(9)} ${(row.microsPerCompleted === null ? "never" : formatUsd(row.microsPerCompleted, 4)).padStart(9)}`
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
console.log(
|
|
300
|
+
"\n `never` is a model that has completed nothing: unrankable rather than\n merely expensive, and the spend beside it is real."
|
|
301
|
+
);
|
|
302
|
+
const exhausted = await reasoningExhaustedSpend(db);
|
|
303
|
+
console.log("\n\ncompletions that were entirely reasoning \u2014 billed, and nothing to show (#142)");
|
|
304
|
+
console.log("model reqs of empties spend");
|
|
305
|
+
if (exhausted.length === 0) console.log(" none");
|
|
306
|
+
for (const row of exhausted) {
|
|
307
|
+
console.log(
|
|
308
|
+
`${row.model.slice(0, 44).padEnd(45)} ${String(row.requests).padStart(4)} ${String(row.empties).padStart(10)} ${formatUsd(row.micros, 4).padStart(9)}`
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
if (exhausted.length > 0) {
|
|
312
|
+
console.log(" An output budget too small for a reasoning model, or a lane allocating one to");
|
|
313
|
+
console.log(" requests it cannot finish. A unary response says so: x-cruise-warning: reasoning_exhausted.");
|
|
314
|
+
}
|
|
315
|
+
const priced = await priceProblems(db);
|
|
316
|
+
console.log("\n\nmodels whose spend is not being priced");
|
|
317
|
+
if (priced.length === 0) {
|
|
318
|
+
console.log(" none \u2014 every routable model has a price in force today");
|
|
319
|
+
}
|
|
320
|
+
for (const row of priced) {
|
|
321
|
+
console.log(
|
|
322
|
+
row.kind === "future" ? ` ${row.model} \u2014 a price row exists but does not start until ${row.startsOn} (UTC)` : ` ${row.model} \u2014 no price row at all; charged its full reservation`
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
if (priced.length > 0) {
|
|
326
|
+
console.log(" Until then these are billed at the reservation, which over-counts. A");
|
|
327
|
+
console.log(" future-dated row is the worse of the two: it looks like the work was done.");
|
|
328
|
+
}
|
|
329
|
+
const chains = await fallbacks(db);
|
|
330
|
+
console.log("\n\nfallback chains that fired");
|
|
331
|
+
if (chains.length === 0) {
|
|
332
|
+
console.log(" none \u2014 every request was served by the model it asked for");
|
|
333
|
+
}
|
|
334
|
+
for (const row of chains) {
|
|
335
|
+
console.log(` ${row.requestedModel} \u2192 ${row.servedModel} ${row.times}\xD7`);
|
|
336
|
+
}
|
|
337
|
+
const month = (/* @__PURE__ */ new Date()).toISOString().slice(0, 7);
|
|
338
|
+
const rescues = await rescuesBy(db, { month });
|
|
339
|
+
console.log(`
|
|
340
|
+
|
|
341
|
+
rescues \u2014 requests a failed model would have lost, ${month} (#647)`);
|
|
342
|
+
console.log(` ${rescues.rescued} of ${rescues.requests} request(s) rescued, ${pct(rescues.rescued_share)}`);
|
|
343
|
+
for (const row of rescues.rows) {
|
|
344
|
+
console.log(` ${row.lane.padEnd(26)} ${row.failed_model} (${row.failed_status ?? "no answer"}) \u2192 ${row.served_model} ${row.rescued}\xD7`);
|
|
345
|
+
}
|
|
346
|
+
const compression = await compressionSavings(db);
|
|
347
|
+
console.log("\n\nrequest compression \u2014 bytes removed from tool outputs (#593)");
|
|
348
|
+
console.log("month project reqs applied before after saved billed in");
|
|
349
|
+
if (compression.length === 0) console.log(" (compression has not run \u2014 every project starts off, and COMPRESS_MODE gates it)");
|
|
350
|
+
for (const row of compression) {
|
|
351
|
+
console.log(
|
|
352
|
+
`${row.month} ${row.projectId.slice(0, 26).padEnd(26)}${String(row.requests).padStart(6)} ${String(row.applied).padStart(7)} ${String(row.bytesBefore).padStart(10)} ${String(row.bytesAfter).padStart(10)} ${pct(savedShare(row)).padStart(6)} ${String(row.promptTokens).padStart(10)}`
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
if (compression.length > 0) {
|
|
356
|
+
console.log(" Bytes of the body sent to the model that answered, built with and without compression.");
|
|
357
|
+
console.log(" 'billed in' is the provider's prompt tokens, which are already the after figure.");
|
|
358
|
+
}
|
|
359
|
+
const lanes = await laneSavings(db, null);
|
|
360
|
+
console.log("\n\nlanes \u2014 what allocating by price saved (0027)");
|
|
361
|
+
console.log("lane reqs members spent vs dearest vs first-listed");
|
|
362
|
+
if (lanes.length === 0) console.log(" (no lane traffic)");
|
|
363
|
+
for (const row of lanes) {
|
|
364
|
+
const vsDearest = row.dearestMicros - row.spentMicros;
|
|
365
|
+
const vsRef = row.referenceMicros - row.spentMicros;
|
|
366
|
+
const dearestPct = row.dearestMicros > 0 ? vsDearest / row.dearestMicros : null;
|
|
367
|
+
const refPct = row.referenceMicros > 0 ? vsRef / row.referenceMicros : null;
|
|
368
|
+
console.log(
|
|
369
|
+
`${row.lane.padEnd(22)}${String(row.requests).padStart(4)} ${String(row.membersUsed).padStart(7)} ${formatUsd(row.spentMicros, 4).padStart(9)} ${formatUsd(vsDearest, 4).padStart(8)} ${pct(dearestPct).padStart(6)} ${formatUsd(vsRef, 4).padStart(8)} ${pct(refPct).padStart(6)} (${row.referenceRequests} reqs)`
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
if (lanes.length > 0) {
|
|
373
|
+
console.log(" Each alternative is priced at its own rate, at that hour, for the tokens the");
|
|
374
|
+
console.log(" served member actually used \u2014 the same work at the other member's price, not a");
|
|
375
|
+
console.log(" claim about what the other member would have written. 'vs dearest' is the upper");
|
|
376
|
+
console.log(" bound; 'vs first-listed' is against the member a human would have pinned, over");
|
|
377
|
+
console.log(" only the requests where it passed the gates, so its request count differs.");
|
|
378
|
+
console.log(" A negative number means the first-listed member was the cheapest that passed.");
|
|
379
|
+
console.log("\nlane cache hits cache saved selection saved");
|
|
380
|
+
for (const row of lanes) {
|
|
381
|
+
console.log(
|
|
382
|
+
`${row.lane.padEnd(22)}${String(row.cacheHits).padStart(10)} ${formatUsd(row.cacheMicros, 4).padStart(11)} ${formatUsd(row.dearestMicros - row.spentMicros, 4).padStart(15)}`
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
console.log(" 'cache saved' is what the lane's hits saved against list; 'selection saved' is");
|
|
386
|
+
console.log(" 'vs dearest' above. Never added: one chose a cheaper model, the other replayed");
|
|
387
|
+
console.log(" an answer, and they carry different risks.");
|
|
388
|
+
}
|
|
389
|
+
const laneCache = await byLaneCache(db, null);
|
|
390
|
+
console.log("\n\ncache hit rate by lane, scope and class \u2014 never blended");
|
|
391
|
+
console.log("lane class reqs model scope lane scope");
|
|
392
|
+
if (laneCache.length === 0) console.log(" (no lane traffic)");
|
|
393
|
+
for (const row of laneCache) {
|
|
394
|
+
console.log(
|
|
395
|
+
`${row.lane.padEnd(22)}${row.trafficClass.padEnd(14)} ${String(row.requests).padStart(4)} ${pct(row.modelHitRate).padStart(6)} (${String(row.modelHits).padStart(4)}) ${pct(row.laneHitRate).padStart(6)} (${String(row.laneHits).padStart(4)})`
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
console.log(" The switch-on rule reads a lane's interactive row and nothing");
|
|
399
|
+
console.log(" else: a scheduled pipeline re-runs the same prompts by construction.");
|
|
400
|
+
const violations = await tierViolations(db, null);
|
|
401
|
+
console.log("\n\nsuspected tier-rule violations \u2014 a lane hit the caller then disowned (#21)");
|
|
402
|
+
if (violations.length === 0) console.log(" none \u2014 no lane-scope hit has been served");
|
|
403
|
+
for (const row of violations) {
|
|
404
|
+
console.log(
|
|
405
|
+
` ${row.lane.padEnd(22)} ${String(row.suspected).padStart(3)} of ${row.laneHits} lane hits re-sent by the same key within ${VIOLATION_WINDOW_MINUTES} minutes with scope=off or scope=model`
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
console.log(" Target zero. Above it, a rank in that lane's quality_ranks is wrong, or lane");
|
|
409
|
+
console.log(" scope should be withdrawn from it. Suspected, not proven: a retry loop looks the same.");
|
|
410
|
+
console.log("\n\nnot yet answerable, and why");
|
|
411
|
+
console.log(" \u2022 OpenRouter spend \u2014 no models routed there yet (#15)");
|
|
412
|
+
console.log(" \u2022 is phase 1a worth it \u2014 needs the hit TTFB gap above, over real traffic");
|
|
413
|
+
console.log(" \u2022 is lane scope worth it \u2014 the interactive rows above, thirty days after model scope");
|
|
414
|
+
});
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Built by packages/cli/build.ts.
|
|
3
|
+
import {
|
|
4
|
+
generateServiceKey
|
|
5
|
+
} from "./chunk-5TVD3ZAI.mjs";
|
|
6
|
+
import {
|
|
7
|
+
connect,
|
|
8
|
+
environmentOf,
|
|
9
|
+
parseArgs,
|
|
10
|
+
requireString,
|
|
11
|
+
run
|
|
12
|
+
} from "./chunk-7NQHBXVZ.mjs";
|
|
13
|
+
import "./chunk-JZPUTXM7.mjs";
|
|
14
|
+
|
|
15
|
+
// scripts/service-key.ts
|
|
16
|
+
run(async () => {
|
|
17
|
+
const [command, ...rest] = process.argv.slice(2);
|
|
18
|
+
const args = parseArgs(rest);
|
|
19
|
+
const db = connect(environmentOf(args));
|
|
20
|
+
switch (command) {
|
|
21
|
+
case "issue": {
|
|
22
|
+
const actor = requireString(args, "actor");
|
|
23
|
+
const purpose = typeof args.purpose === "string" ? args.purpose : "credits";
|
|
24
|
+
if (purpose !== "credits" && purpose !== "balance" && purpose !== "provision" && purpose !== "tier" && purpose !== "economics" && purpose !== "promo") {
|
|
25
|
+
throw new Error("--purpose must be credits, balance, provision, tier, economics or promo");
|
|
26
|
+
}
|
|
27
|
+
const generated = await generateServiceKey();
|
|
28
|
+
await db.prepare(
|
|
29
|
+
`INSERT INTO service_keys (id, prefix, key_hash, purpose, created_by)
|
|
30
|
+
VALUES (?, ?, ?, ?, ?)`
|
|
31
|
+
).bind(crypto.randomUUID(), generated.prefix, generated.hash, purpose, actor).run();
|
|
32
|
+
console.log(`Issued for '${purpose}'. Shown once, stored hashed:
|
|
33
|
+
|
|
34
|
+
${generated.key}
|
|
35
|
+
`);
|
|
36
|
+
console.log(`Prefix (safe in logs): ${generated.prefix}`);
|
|
37
|
+
break;
|
|
38
|
+
}
|
|
39
|
+
case "revoke": {
|
|
40
|
+
const prefix = requireString(args, "prefix");
|
|
41
|
+
const reason = requireString(args, "reason");
|
|
42
|
+
await db.prepare(`UPDATE service_keys SET revoked_at = datetime('now') WHERE prefix = ? AND revoked_at IS NULL`).bind(prefix).run();
|
|
43
|
+
await db.prepare(
|
|
44
|
+
`INSERT INTO audit_events (id, actor, action, subject, detail) VALUES (?, 'cli', 'service_key.revoke', ?, ?)`
|
|
45
|
+
).bind(crypto.randomUUID(), prefix, JSON.stringify({ reason })).run();
|
|
46
|
+
console.log(`Revoked ${prefix}.`);
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
case "list": {
|
|
50
|
+
const { results } = await db.prepare(
|
|
51
|
+
`SELECT prefix, purpose, created_by, created_at, revoked_at, last_used_at FROM service_keys ORDER BY created_at`
|
|
52
|
+
).all();
|
|
53
|
+
console.table(results);
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
default:
|
|
57
|
+
throw new Error("Usage: service-key.ts <issue|revoke|list>");
|
|
58
|
+
}
|
|
59
|
+
});
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Built by packages/cli/build.ts.
|
|
3
|
+
import {
|
|
4
|
+
parseArgs,
|
|
5
|
+
requireString,
|
|
6
|
+
run
|
|
7
|
+
} from "./chunk-7NQHBXVZ.mjs";
|
|
8
|
+
import "./chunk-JZPUTXM7.mjs";
|
|
9
|
+
|
|
10
|
+
// src/smoke.ts
|
|
11
|
+
var NOT_CHECKED = "not checked \u2014 the catalogue did not list";
|
|
12
|
+
function catalogueChecks(status, listed) {
|
|
13
|
+
const entries = listed?.data ?? [];
|
|
14
|
+
const listedOk = status === 200 && entries.length > 0;
|
|
15
|
+
const checks = [
|
|
16
|
+
{
|
|
17
|
+
name: "models listed",
|
|
18
|
+
ok: listedOk,
|
|
19
|
+
detail: status === 200 ? `${entries.length} model(s)` : `HTTP ${status}, so nothing was listed`
|
|
20
|
+
}
|
|
21
|
+
];
|
|
22
|
+
const unpriced = entries.filter((model) => model["x-cruise"]?.pricing == null).map((model) => model.id);
|
|
23
|
+
checks.push({
|
|
24
|
+
name: "every listed model is priced",
|
|
25
|
+
ok: listedOk && unpriced.length === 0,
|
|
26
|
+
detail: listedOk ? unpriced.join(", ") || "all priced" : NOT_CHECKED
|
|
27
|
+
});
|
|
28
|
+
const lanes = entries.filter((model) => model["x-cruise"]?.lane === true);
|
|
29
|
+
const concrete = entries.filter((model) => model["x-cruise"]?.lane !== true);
|
|
30
|
+
const unmeasured = concrete.filter((model) => model["x-cruise"]?.measured_at == null).map((model) => model.id);
|
|
31
|
+
checks.push({
|
|
32
|
+
name: "every listed model is measured",
|
|
33
|
+
ok: listedOk && unmeasured.length === 0,
|
|
34
|
+
detail: listedOk ? unmeasured.join(", ") || "all measured" : NOT_CHECKED
|
|
35
|
+
});
|
|
36
|
+
const ids = new Set(entries.map((model) => model.id));
|
|
37
|
+
const badLanes = lanes.filter((lane) => {
|
|
38
|
+
const members = lane["x-cruise"]?.members ?? [];
|
|
39
|
+
return members.length === 0 || members.some((m) => !ids.has(m));
|
|
40
|
+
}).map((lane) => lane.id);
|
|
41
|
+
checks.push({
|
|
42
|
+
name: "every lane lists members from this catalog",
|
|
43
|
+
ok: listedOk && lanes.length > 0 && badLanes.length === 0,
|
|
44
|
+
detail: !listedOk ? NOT_CHECKED : lanes.length === 0 ? "no lanes listed" : badLanes.join(", ") || `${lanes.length} lane(s), all members listed`
|
|
45
|
+
});
|
|
46
|
+
return checks;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// scripts/smoke.ts
|
|
50
|
+
var results = [];
|
|
51
|
+
function record(name, ok, detail) {
|
|
52
|
+
results.push({ name, ok, detail });
|
|
53
|
+
console.log(` ${ok ? "ok " : "FAIL"} ${name.padEnd(38)} ${detail}`);
|
|
54
|
+
}
|
|
55
|
+
async function main() {
|
|
56
|
+
const args = parseArgs(process.argv.slice(2));
|
|
57
|
+
const base = requireString(args, "base").replace(/\/$/, "");
|
|
58
|
+
const key = typeof args["key"] === "string" ? args["key"] : null;
|
|
59
|
+
const url = new URL(base);
|
|
60
|
+
if (url.protocol !== "https:") throw new Error(`--base must be https, got ${base}`);
|
|
61
|
+
console.log(`
|
|
62
|
+
smoke: ${base}
|
|
63
|
+
`);
|
|
64
|
+
const health = await fetch(`${base}/health`);
|
|
65
|
+
const body = await health.json().catch(() => null);
|
|
66
|
+
record("health responds", health.status === 200 && body?.ok === true, `${health.status} ${body?.environment ?? ""}`);
|
|
67
|
+
const hsts = health.headers.get("strict-transport-security");
|
|
68
|
+
record("HSTS present", hsts !== null && hsts.includes("max-age="), hsts ?? "absent");
|
|
69
|
+
record(
|
|
70
|
+
"nosniff present",
|
|
71
|
+
health.headers.get("x-content-type-options") === "nosniff",
|
|
72
|
+
health.headers.get("x-content-type-options") ?? "absent"
|
|
73
|
+
);
|
|
74
|
+
const cleartext = await fetch(`${base.replace(/^https:/, "http:")}/health`, { redirect: "manual" });
|
|
75
|
+
record(
|
|
76
|
+
"plain http refused, not redirected",
|
|
77
|
+
cleartext.status === 400,
|
|
78
|
+
`${cleartext.status}${cleartext.status >= 300 && cleartext.status < 400 ? " \u2014 REDIRECTED, the key still leaked" : ""}`
|
|
79
|
+
);
|
|
80
|
+
const unauthenticated = await fetch(`${base}/v1/chat/completions`, {
|
|
81
|
+
method: "POST",
|
|
82
|
+
headers: { "content-type": "application/json" },
|
|
83
|
+
body: JSON.stringify({ model: "x", messages: [] })
|
|
84
|
+
});
|
|
85
|
+
record("completions require a key", unauthenticated.status === 401, String(unauthenticated.status));
|
|
86
|
+
if (key === null) {
|
|
87
|
+
console.log("\n (no --key, so the model catalog was not checked)");
|
|
88
|
+
} else {
|
|
89
|
+
const models = await fetch(`${base}/v1/models`, { headers: { authorization: `Bearer ${key}` } });
|
|
90
|
+
const listed = await models.json().catch(() => null);
|
|
91
|
+
for (const check of catalogueChecks(models.status, listed)) {
|
|
92
|
+
record(check.name, check.ok, check.detail);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const failed = results.filter((check) => !check.ok);
|
|
96
|
+
console.log(
|
|
97
|
+
failed.length === 0 ? `
|
|
98
|
+
${results.length} checks passed
|
|
99
|
+
` : `
|
|
100
|
+
${failed.length} of ${results.length} checks FAILED
|
|
101
|
+
`
|
|
102
|
+
);
|
|
103
|
+
if (failed.length > 0) process.exitCode = 1;
|
|
104
|
+
}
|
|
105
|
+
run(main);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Built by packages/cli/build.ts.
|
|
3
|
+
import {
|
|
4
|
+
SESSION_IDLE_DAYS,
|
|
5
|
+
pruneIdleSessions,
|
|
6
|
+
sweepAbandonedHolds,
|
|
7
|
+
sweepRateBuckets
|
|
8
|
+
} from "./chunk-UNXM6FSM.mjs";
|
|
9
|
+
import "./chunk-SYTYOTO4.mjs";
|
|
10
|
+
import "./chunk-O7TNRKSA.mjs";
|
|
11
|
+
import "./chunk-T3ICD43U.mjs";
|
|
12
|
+
import "./chunk-THEXUHQC.mjs";
|
|
13
|
+
import "./chunk-5TVD3ZAI.mjs";
|
|
14
|
+
import {
|
|
15
|
+
connect,
|
|
16
|
+
environmentOf,
|
|
17
|
+
parseArgs,
|
|
18
|
+
run
|
|
19
|
+
} from "./chunk-7NQHBXVZ.mjs";
|
|
20
|
+
import "./chunk-JZPUTXM7.mjs";
|
|
21
|
+
|
|
22
|
+
// scripts/sweep.ts
|
|
23
|
+
run(async () => {
|
|
24
|
+
const args = parseArgs(process.argv.slice(2));
|
|
25
|
+
const environment = environmentOf(args);
|
|
26
|
+
const db = connect(environment);
|
|
27
|
+
const removed = await sweepRateBuckets(db, /* @__PURE__ */ new Date());
|
|
28
|
+
console.log(`${environment}: swept ${removed} stale rate-limit bucket(s)`);
|
|
29
|
+
const abandoned = await sweepAbandonedHolds(db, /* @__PURE__ */ new Date());
|
|
30
|
+
console.log(`${environment}: closed ${abandoned} abandoned reservation(s) into the ledger`);
|
|
31
|
+
const idle = await pruneIdleSessions(db);
|
|
32
|
+
console.log(`${environment}: removed ${idle} run counter(s) idle over ${SESSION_IDLE_DAYS} days`);
|
|
33
|
+
});
|