@harrym1707/sdk 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +96 -0
- package/README.md +48 -2
- package/bin/mesh-demo.mjs +105 -0
- package/bin/mesh-proxy.mjs +39 -0
- package/bin/mesh-quote.mjs +45 -0
- package/bin/mesh-report.mjs +47 -0
- package/index.cjs +4923 -0
- package/index.js +4850 -0
- package/package.json +39 -4
- package/types/blind-features.d.ts +28 -0
- package/types/cache-ttl-advisor.d.ts +31 -0
- package/types/cache-volatility-guard.d.ts +17 -0
- package/types/core/cache-simulator.d.ts +271 -0
- package/types/core/confidence.d.ts +36 -0
- package/types/core/cost.d.ts +16 -0
- package/types/core/evaluator.d.ts +6 -0
- package/types/core/gateway.d.ts +75 -0
- package/types/core/geo-visibility.d.ts +25 -0
- package/types/core/helicone.d.ts +6 -0
- package/types/core/index.d.ts +26 -0
- package/types/core/langfuse.d.ts +7 -0
- package/types/core/litellm.d.ts +15 -0
- package/types/core/local-tier.d.ts +82 -0
- package/types/core/model-prices.d.ts +25 -0
- package/types/core/output-diagnostics.d.ts +39 -0
- package/types/core/price-catalogue.d.ts +168 -0
- package/types/core/provider-usage.d.ts +84 -0
- package/types/core/quote-report.d.ts +54 -0
- package/types/core/replay-runner.d.ts +11 -0
- package/types/core/replay.d.ts +2 -0
- package/types/core/router.d.ts +9 -0
- package/types/core/routing-simulation.d.ts +24 -0
- package/types/core/savings-ledger.d.ts +349 -0
- package/types/core/savings-model.d.ts +132 -0
- package/types/core/stripe-billing.d.ts +67 -0
- package/types/core/stripe-sink.d.ts +77 -0
- package/types/core/theo-coverage.d.ts +49 -0
- package/types/core/token-reduction.d.ts +208 -0
- package/types/core/tool-schema.d.ts +64 -0
- package/types/core/traffic.d.ts +182 -0
- package/types/core/triage.d.ts +70 -0
- package/types/core/types.d.ts +138 -0
- package/types/evaluation-capture.d.ts +11 -0
- package/types/exact-cache.d.ts +20 -0
- package/types/graders.d.ts +53 -0
- package/types/index.d.ts +26 -0
- package/types/openai.d.ts +23 -0
- package/types/proxy.d.ts +63 -0
- package/types/reduce-request.d.ts +247 -0
- package/types/replay-gate.d.ts +120 -0
- package/types/report.d.ts +33 -0
- package/types/routing.d.ts +11 -0
- package/types/savings-ledger.d.ts +12 -0
- package/types/semantic-compaction.d.ts +84 -0
- package/types/telemetry.d.ts +6 -0
- package/types/theo-policy.d.ts +119 -0
- package/types/types.d.ts +263 -0
- package/types/wrap.d.ts +52 -0
package/index.js
ADDED
|
@@ -0,0 +1,4850 @@
|
|
|
1
|
+
// packages/sdk-js/src/openai.ts
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
|
|
4
|
+
// packages/core/dist/src/cost.js
|
|
5
|
+
function estimateTokenCost(inputTokens, outputTokens, price, cacheUsage = {}) {
|
|
6
|
+
const cached = cacheUsage.cachedInputTokens ?? 0;
|
|
7
|
+
const cacheWrite = cacheUsage.cacheWriteInputTokens ?? 0;
|
|
8
|
+
const cacheWrite1h = cacheUsage.cacheWrite1hInputTokens ?? 0;
|
|
9
|
+
if (!Number.isFinite(cached) || cached < 0 || !Number.isFinite(cacheWrite) || cacheWrite < 0 || !Number.isFinite(cacheWrite1h) || cacheWrite1h < 0 || cached + cacheWrite > inputTokens || cacheWrite1h > cacheWrite) {
|
|
10
|
+
throw new Error("Cached input token counts must be non-negative and no greater than total input tokens.");
|
|
11
|
+
}
|
|
12
|
+
const cacheWrite5m = cacheWrite - cacheWrite1h;
|
|
13
|
+
if (cached > 0 && price.cachedInputPerMillion === void 0 || cacheWrite5m > 0 && price.cacheWriteInputPerMillion === void 0 || cacheWrite1h > 0 && price.cacheWrite1hInputPerMillion === void 0) {
|
|
14
|
+
throw new Error("Cached input pricing is required to calculate actual cost for cached tokens.");
|
|
15
|
+
}
|
|
16
|
+
const uncached = inputTokens - cached - cacheWrite;
|
|
17
|
+
return uncached / 1e6 * price.inputPerMillion + cached / 1e6 * (price.cachedInputPerMillion ?? 0) + cacheWrite5m / 1e6 * (price.cacheWriteInputPerMillion ?? 0) + cacheWrite1h / 1e6 * (price.cacheWrite1hInputPerMillion ?? 0) + outputTokens / 1e6 * price.outputPerMillion;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// packages/core/dist/src/confidence.js
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
function mean(values) {
|
|
23
|
+
if (values.length === 0)
|
|
24
|
+
throw new Error("Cannot take the mean of an empty set.");
|
|
25
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
26
|
+
}
|
|
27
|
+
function seededRandom(seed) {
|
|
28
|
+
let state = Number.parseInt(createHash("sha256").update(seed).digest("hex").slice(0, 8), 16) >>> 0;
|
|
29
|
+
return () => {
|
|
30
|
+
state = state + 1831565813 >>> 0;
|
|
31
|
+
let value = state;
|
|
32
|
+
value = Math.imul(value ^ value >>> 15, value | 1);
|
|
33
|
+
value ^= value + Math.imul(value ^ value >>> 7, value | 61);
|
|
34
|
+
return ((value ^ value >>> 14) >>> 0) / 4294967296;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function simultaneousGroupBootstrapLowerBounds(deltasByConfiguration, groups, samples, seed, alpha = 0.05) {
|
|
38
|
+
const configurations = deltasByConfiguration.length;
|
|
39
|
+
if (configurations === 0)
|
|
40
|
+
return [];
|
|
41
|
+
if (deltasByConfiguration.some((deltas) => deltas.length !== groups.length)) {
|
|
42
|
+
throw new Error("Every configuration needs one delta per validation record.");
|
|
43
|
+
}
|
|
44
|
+
const indicesByGroup = /* @__PURE__ */ new Map();
|
|
45
|
+
groups.forEach((group, index) => {
|
|
46
|
+
const list = indicesByGroup.get(group) ?? [];
|
|
47
|
+
list.push(index);
|
|
48
|
+
indicesByGroup.set(group, list);
|
|
49
|
+
});
|
|
50
|
+
const groupIndices = [...indicesByGroup.values()];
|
|
51
|
+
const pointEstimates = deltasByConfiguration.map((deltas) => mean(deltas));
|
|
52
|
+
const random = seededRandom(seed);
|
|
53
|
+
const resampleMeans = new Array(samples);
|
|
54
|
+
for (let sample = 0; sample < samples; sample += 1) {
|
|
55
|
+
const totals = new Array(configurations).fill(0);
|
|
56
|
+
let count = 0;
|
|
57
|
+
for (let draw = 0; draw < groupIndices.length; draw += 1) {
|
|
58
|
+
const selected = groupIndices[Math.floor(random() * groupIndices.length)];
|
|
59
|
+
for (const index of selected) {
|
|
60
|
+
for (let configuration = 0; configuration < configurations; configuration += 1) {
|
|
61
|
+
totals[configuration] += deltasByConfiguration[configuration][index];
|
|
62
|
+
}
|
|
63
|
+
count += 1;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
for (let configuration = 0; configuration < configurations; configuration += 1) {
|
|
67
|
+
totals[configuration] /= count;
|
|
68
|
+
}
|
|
69
|
+
resampleMeans[sample] = totals;
|
|
70
|
+
}
|
|
71
|
+
const standardErrors = new Array(configurations);
|
|
72
|
+
for (let configuration = 0; configuration < configurations; configuration += 1) {
|
|
73
|
+
let sumSquares = 0;
|
|
74
|
+
for (let sample = 0; sample < samples; sample += 1) {
|
|
75
|
+
const deviation = resampleMeans[sample][configuration] - pointEstimates[configuration];
|
|
76
|
+
sumSquares += deviation * deviation;
|
|
77
|
+
}
|
|
78
|
+
standardErrors[configuration] = Math.sqrt(sumSquares / Math.max(1, samples - 1)) || Number.EPSILON;
|
|
79
|
+
}
|
|
80
|
+
const errors = new Array(samples);
|
|
81
|
+
for (let sample = 0; sample < samples; sample += 1) {
|
|
82
|
+
let worst = Number.NEGATIVE_INFINITY;
|
|
83
|
+
for (let configuration = 0; configuration < configurations; configuration += 1) {
|
|
84
|
+
const studentised = (pointEstimates[configuration] - resampleMeans[sample][configuration]) / standardErrors[configuration];
|
|
85
|
+
if (studentised > worst)
|
|
86
|
+
worst = studentised;
|
|
87
|
+
}
|
|
88
|
+
errors[sample] = worst;
|
|
89
|
+
}
|
|
90
|
+
errors.sort((left, right) => left - right);
|
|
91
|
+
const critical = errors[Math.min(samples - 1, Math.ceil((samples - 1) * (1 - alpha)))];
|
|
92
|
+
return pointEstimates.map((estimate, configuration) => estimate - critical * standardErrors[configuration]);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// packages/core/dist/src/token-reduction.js
|
|
96
|
+
var approximateTokenCount = (text) => {
|
|
97
|
+
if (text === "")
|
|
98
|
+
return 0;
|
|
99
|
+
let total = 0;
|
|
100
|
+
for (const match of text.matchAll(/[\p{L}\p{N}']+|[^\p{L}\p{N}'\s]+|\s+/gu)) {
|
|
101
|
+
const chunk = match[0];
|
|
102
|
+
if (/^\s+$/.test(chunk)) {
|
|
103
|
+
total += chunk.match(/\n/g)?.length ?? 0;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (/^[\p{L}\p{N}']+$/u.test(chunk)) {
|
|
107
|
+
total += Math.max(1, Math.ceil(chunk.length / 4.5));
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
total += Math.max(1, Math.ceil(chunk.length / 2));
|
|
111
|
+
}
|
|
112
|
+
return total;
|
|
113
|
+
};
|
|
114
|
+
function dedupeRepeatedBlocks(minimumBlockTokens = 24, countTokens = approximateTokenCount) {
|
|
115
|
+
return {
|
|
116
|
+
id: "dedupe-repeated-blocks",
|
|
117
|
+
version: "2",
|
|
118
|
+
risk: "lossless",
|
|
119
|
+
reduce: (text) => {
|
|
120
|
+
const seen = /* @__PURE__ */ new Set();
|
|
121
|
+
const notes = [];
|
|
122
|
+
let dropped = false;
|
|
123
|
+
const describe = (unit, tokens) => `Dropped a repeated ${tokens}-token block beginning "${unit.trim().replace(/\s+/g, " ").slice(0, 60)}".`;
|
|
124
|
+
const dedupeLines = (block) => {
|
|
125
|
+
if (block.includes("```") || !block.includes("\n"))
|
|
126
|
+
return block;
|
|
127
|
+
const lines = block.split(/(\n)/);
|
|
128
|
+
let changed = false;
|
|
129
|
+
const keptLines = [];
|
|
130
|
+
for (let index = 0; index < lines.length; index += 2) {
|
|
131
|
+
const line = lines[index];
|
|
132
|
+
const separator = lines[index + 1] ?? "";
|
|
133
|
+
const tokens = countTokens(line);
|
|
134
|
+
if (line.trim() !== "" && tokens >= minimumBlockTokens && seen.has(line)) {
|
|
135
|
+
dropped = true;
|
|
136
|
+
changed = true;
|
|
137
|
+
notes.push(describe(line, tokens));
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
seen.add(line);
|
|
141
|
+
keptLines.push(line, separator);
|
|
142
|
+
}
|
|
143
|
+
return changed ? keptLines.join("") : block;
|
|
144
|
+
};
|
|
145
|
+
const parts = text.split(/(\n{2,})/);
|
|
146
|
+
const kept = [];
|
|
147
|
+
for (let index = 0; index < parts.length; index += 2) {
|
|
148
|
+
const block = parts[index];
|
|
149
|
+
const separator = parts[index + 1] ?? "";
|
|
150
|
+
const tokens = countTokens(block);
|
|
151
|
+
if (block.trim() !== "" && !block.includes("```") && tokens >= minimumBlockTokens && seen.has(block)) {
|
|
152
|
+
dropped = true;
|
|
153
|
+
notes.push(describe(block, tokens));
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
seen.add(block);
|
|
157
|
+
kept.push(dedupeLines(block), separator);
|
|
158
|
+
}
|
|
159
|
+
if (!dropped)
|
|
160
|
+
return { text, notes: [] };
|
|
161
|
+
return { text: kept.join(""), notes };
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
var BREAKER_PATTERNS = [
|
|
166
|
+
{ kind: "timestamp", pattern: /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?Z?/g },
|
|
167
|
+
{ kind: "uuid", pattern: /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi },
|
|
168
|
+
{ kind: "date", pattern: /\d{4}-\d{2}-\d{2}/g },
|
|
169
|
+
{ kind: "long-number", pattern: /\b\d{6,}\b/g }
|
|
170
|
+
];
|
|
171
|
+
function commonPrefixLength(left, right) {
|
|
172
|
+
const limit = Math.min(left.length, right.length);
|
|
173
|
+
let index = 0;
|
|
174
|
+
while (index < limit && left[index] === right[index])
|
|
175
|
+
index += 1;
|
|
176
|
+
return index;
|
|
177
|
+
}
|
|
178
|
+
function dedupeRepeatedLeadingBlock(texts, minimumBlockTokens = 24, countTokens = approximateTokenCount) {
|
|
179
|
+
if (texts.length === 0)
|
|
180
|
+
return { texts: [], notes: [] };
|
|
181
|
+
const canonical = texts[0];
|
|
182
|
+
const notes = [];
|
|
183
|
+
const nextTexts = texts.map((text, index) => {
|
|
184
|
+
if (index === 0)
|
|
185
|
+
return text;
|
|
186
|
+
const raw = commonPrefixLength(canonical, text);
|
|
187
|
+
if (raw === 0)
|
|
188
|
+
return text;
|
|
189
|
+
const fullContainment = raw === canonical.length || raw === text.length;
|
|
190
|
+
let safe = raw;
|
|
191
|
+
if (!fullContainment) {
|
|
192
|
+
const lastNewline = text.lastIndexOf("\n", raw - 1);
|
|
193
|
+
safe = lastNewline === -1 ? 0 : lastNewline + 1;
|
|
194
|
+
}
|
|
195
|
+
if (safe === 0)
|
|
196
|
+
return text;
|
|
197
|
+
const removed = text.slice(0, safe);
|
|
198
|
+
if (removed.trim() === "" || removed.includes("```"))
|
|
199
|
+
return text;
|
|
200
|
+
const tokens = countTokens(removed);
|
|
201
|
+
if (tokens < minimumBlockTokens)
|
|
202
|
+
return text;
|
|
203
|
+
notes.push(`Turn ${index + 1}: replaced a ${tokens}-token leading block identical to turn 1's opening with a reference marker.`);
|
|
204
|
+
return `[mesh: ${tokens} tokens of this turn's leading context, identical to turn 1, omitted]${text.slice(safe)}`;
|
|
205
|
+
});
|
|
206
|
+
return { texts: nextTexts, notes };
|
|
207
|
+
}
|
|
208
|
+
function maskBreakers(text) {
|
|
209
|
+
let masked = text;
|
|
210
|
+
const found = /* @__PURE__ */ new Map();
|
|
211
|
+
for (const { kind, pattern } of BREAKER_PATTERNS) {
|
|
212
|
+
masked = masked.replace(pattern, (match) => {
|
|
213
|
+
const existing = found.get(match);
|
|
214
|
+
if (existing)
|
|
215
|
+
existing.occurrences += 1;
|
|
216
|
+
else
|
|
217
|
+
found.set(match, { kind, example: match, occurrences: 1 });
|
|
218
|
+
return `[[${kind}]]`;
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
return { masked, found };
|
|
222
|
+
}
|
|
223
|
+
function analysePromptCaching(requests, options = {}) {
|
|
224
|
+
if (requests.length < 2) {
|
|
225
|
+
throw new Error("Prefix-cache analysis needs at least two requests to find what they share.");
|
|
226
|
+
}
|
|
227
|
+
const countTokens = options.countTokens ?? approximateTokenCount;
|
|
228
|
+
const minimumCacheable = options.minimumCacheableTokens ?? 1024;
|
|
229
|
+
const rawPrefix = requests.reduce((prefix, request) => prefix.slice(0, commonPrefixLength(prefix, request)), requests[0]);
|
|
230
|
+
const masked = requests.map(maskBreakers);
|
|
231
|
+
const maskedPrefix = masked.reduce((prefix, { masked: text }) => prefix.slice(0, commonPrefixLength(prefix, text)), masked[0].masked);
|
|
232
|
+
const breakers = /* @__PURE__ */ new Map();
|
|
233
|
+
for (const { found } of masked) {
|
|
234
|
+
for (const [value, breaker] of found) {
|
|
235
|
+
const existing = breakers.get(value);
|
|
236
|
+
if (existing)
|
|
237
|
+
existing.occurrences += breaker.occurrences;
|
|
238
|
+
else
|
|
239
|
+
breakers.set(value, { ...breaker });
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const sharedPrefixTokens = countTokens(rawPrefix);
|
|
243
|
+
const recoverablePrefixTokens = countTokens(maskedPrefix);
|
|
244
|
+
const meanInputTokens = requests.reduce((sum, request) => sum + countTokens(request), 0) / requests.length;
|
|
245
|
+
const effectivePrefix = sharedPrefixTokens >= minimumCacheable ? sharedPrefixTokens : 0;
|
|
246
|
+
const effectiveRecoverable = recoverablePrefixTokens >= minimumCacheable ? recoverablePrefixTokens : 0;
|
|
247
|
+
let projection;
|
|
248
|
+
if (!options.price) {
|
|
249
|
+
projection = { status: "unpriced", reason: "No price was supplied, so only token counts are reported." };
|
|
250
|
+
} else if (options.price.cachedInputPerMillion === void 0) {
|
|
251
|
+
projection = {
|
|
252
|
+
status: "unpriced",
|
|
253
|
+
reason: "The supplied price has no cached-input rate, so a caching saving cannot be computed."
|
|
254
|
+
};
|
|
255
|
+
} else {
|
|
256
|
+
const price = options.price;
|
|
257
|
+
const cost = (prefixTokens) => {
|
|
258
|
+
const body = Math.max(0, meanInputTokens - prefixTokens);
|
|
259
|
+
const write = price.cacheWriteInputPerMillion === void 0 ? estimateTokenCost(prefixTokens, 0, price) : estimateTokenCost(prefixTokens, 0, price, { cacheWriteInputTokens: prefixTokens });
|
|
260
|
+
const read = estimateTokenCost(prefixTokens, 0, price, { cachedInputTokens: prefixTokens });
|
|
261
|
+
const bodyCost = estimateTokenCost(body, 0, price);
|
|
262
|
+
return write + (requests.length - 1) * read + requests.length * bodyCost;
|
|
263
|
+
};
|
|
264
|
+
const baselineCostUsd = estimateTokenCost(meanInputTokens, 0, price) * requests.length;
|
|
265
|
+
const cachedCostUsd = effectivePrefix === 0 ? baselineCostUsd : cost(effectivePrefix);
|
|
266
|
+
const recoverableCostUsd = effectiveRecoverable === 0 ? baselineCostUsd : cost(effectiveRecoverable);
|
|
267
|
+
projection = {
|
|
268
|
+
status: "priced",
|
|
269
|
+
baselineCostUsd,
|
|
270
|
+
cachedCostUsd,
|
|
271
|
+
savingUsd: baselineCostUsd - cachedCostUsd,
|
|
272
|
+
savingPercent: baselineCostUsd === 0 ? 0 : (baselineCostUsd - cachedCostUsd) / baselineCostUsd * 100,
|
|
273
|
+
recoverableSavingUsd: Math.max(0, cachedCostUsd - recoverableCostUsd)
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
return {
|
|
277
|
+
requests: requests.length,
|
|
278
|
+
meanInputTokens,
|
|
279
|
+
sharedPrefixTokens,
|
|
280
|
+
cacheablePrefixShare: meanInputTokens === 0 ? 0 : effectivePrefix / meanInputTokens,
|
|
281
|
+
recoverablePrefixTokens,
|
|
282
|
+
prefixBreakers: [...breakers.values()].sort((left, right) => right.occurrences - left.occurrences).slice(0, 20),
|
|
283
|
+
projection
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
function recommendOutputCap(observedOutputTokens, options = {}) {
|
|
287
|
+
if (observedOutputTokens.length === 0) {
|
|
288
|
+
throw new Error("An output cap needs observed response lengths to be derived from.");
|
|
289
|
+
}
|
|
290
|
+
if (observedOutputTokens.some((value) => !Number.isFinite(value) || value < 0)) {
|
|
291
|
+
throw new Error("Observed output token counts must be non-negative finite numbers.");
|
|
292
|
+
}
|
|
293
|
+
const quantile = options.quantile ?? 0.99;
|
|
294
|
+
if (quantile <= 0 || quantile > 1)
|
|
295
|
+
throw new Error("quantile must be above 0 and at most 1.");
|
|
296
|
+
const headroom = options.headroom ?? 1.1;
|
|
297
|
+
if (headroom < 1)
|
|
298
|
+
throw new Error("headroom must be at least 1.");
|
|
299
|
+
const sorted = [...observedOutputTokens].sort((left, right) => left - right);
|
|
300
|
+
const index = Math.min(sorted.length - 1, Math.ceil(quantile * sorted.length) - 1);
|
|
301
|
+
const recommendedCap = Math.ceil(sorted[Math.max(0, index)] * headroom);
|
|
302
|
+
const truncated = sorted.filter((value) => value > recommendedCap);
|
|
303
|
+
const projectedTokensSaved = truncated.reduce((sum, value) => sum + (value - recommendedCap), 0);
|
|
304
|
+
const meanOutputTokens = sorted.reduce((sum, value) => sum + value, 0) / sorted.length;
|
|
305
|
+
return {
|
|
306
|
+
observations: sorted.length,
|
|
307
|
+
maxObserved: sorted[sorted.length - 1],
|
|
308
|
+
recommendedCap,
|
|
309
|
+
truncationRate: truncated.length / sorted.length,
|
|
310
|
+
meanOutputTokens,
|
|
311
|
+
projectedTokensSaved,
|
|
312
|
+
...options.price ? { projectedSavingUsd: estimateTokenCost(0, projectedTokensSaved, options.price) } : {}
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// packages/core/dist/src/cache-simulator.js
|
|
317
|
+
var ANTHROPIC_NO_CACHE = {
|
|
318
|
+
name: "anthropic-uncached",
|
|
319
|
+
minimumCacheableTokens: Number.MAX_SAFE_INTEGER,
|
|
320
|
+
incrementTokens: 1,
|
|
321
|
+
ttlSeconds: 0,
|
|
322
|
+
writeMultiplier: 1,
|
|
323
|
+
readMultiplier: 1
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
// packages/core/dist/src/price-catalogue.js
|
|
327
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
328
|
+
var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
329
|
+
var OPEN_ENDED = "9999-12-31";
|
|
330
|
+
function assertIsoDate(value, label) {
|
|
331
|
+
if (!ISO_DATE.test(value) || Number.isNaN(Date.parse(value))) {
|
|
332
|
+
throw new Error(`${label} must be an ISO calendar date (YYYY-MM-DD); received ${JSON.stringify(value)}.`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
function assertRate(value, label) {
|
|
336
|
+
if (value === void 0)
|
|
337
|
+
return;
|
|
338
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
339
|
+
throw new Error(`${label} must be a non-negative finite number of USD per million tokens.`);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
function assertTiers(tiers, label) {
|
|
343
|
+
if (tiers.length === 0)
|
|
344
|
+
throw new Error(`${label} must declare at least one price tier.`);
|
|
345
|
+
const unbounded = tiers.filter((tier) => tier.maxInputTokens === void 0);
|
|
346
|
+
if (unbounded.length !== 1) {
|
|
347
|
+
throw new Error(`${label} must declare exactly one tier without maxInputTokens; found ${unbounded.length}.`);
|
|
348
|
+
}
|
|
349
|
+
const bounds = tiers.map((tier) => tier.maxInputTokens).filter((bound) => bound !== void 0);
|
|
350
|
+
if (bounds.some((bound) => !Number.isInteger(bound) || bound <= 0)) {
|
|
351
|
+
throw new Error(`${label} tier bounds must be positive integers.`);
|
|
352
|
+
}
|
|
353
|
+
if (new Set(bounds).size !== bounds.length) {
|
|
354
|
+
throw new Error(`${label} declares duplicate tier bounds.`);
|
|
355
|
+
}
|
|
356
|
+
for (const tier of tiers) {
|
|
357
|
+
assertRate(tier.inputPerMillion, `${label} inputPerMillion`);
|
|
358
|
+
assertRate(tier.outputPerMillion, `${label} outputPerMillion`);
|
|
359
|
+
assertRate(tier.cachedInputPerMillion, `${label} cachedInputPerMillion`);
|
|
360
|
+
assertRate(tier.cacheWriteInputPerMillion, `${label} cacheWriteInputPerMillion`);
|
|
361
|
+
assertRate(tier.cacheWrite1hInputPerMillion, `${label} cacheWrite1hInputPerMillion`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
function assertProvenance(provenance, label) {
|
|
365
|
+
if (!provenance || typeof provenance.source !== "string" || provenance.source.trim() === "") {
|
|
366
|
+
throw new Error(`${label} needs a provenance.source so the rate can be traced to a published price.`);
|
|
367
|
+
}
|
|
368
|
+
assertIsoDate(provenance.capturedOn, `${label} provenance.capturedOn`);
|
|
369
|
+
}
|
|
370
|
+
function rangeKey(item) {
|
|
371
|
+
return `${item.provider} ${item.model} ${item.mode ?? "standard"}`;
|
|
372
|
+
}
|
|
373
|
+
function overlaps(left, right) {
|
|
374
|
+
return left.effectiveFrom < (right.effectiveTo ?? OPEN_ENDED) && right.effectiveFrom < (left.effectiveTo ?? OPEN_ENDED);
|
|
375
|
+
}
|
|
376
|
+
function validatePriceCatalogue(catalogue) {
|
|
377
|
+
const seen = /* @__PURE__ */ new Map();
|
|
378
|
+
for (const entry2 of catalogue.entries) {
|
|
379
|
+
const label = `Price entry ${entry2.provider}/${entry2.model} (${entry2.mode}) from ${entry2.effectiveFrom}`;
|
|
380
|
+
if (!entry2.provider.trim() || !entry2.model.trim()) {
|
|
381
|
+
throw new Error("Price entries need a non-empty provider and model.");
|
|
382
|
+
}
|
|
383
|
+
assertIsoDate(entry2.effectiveFrom, `${label} effectiveFrom`);
|
|
384
|
+
if (entry2.effectiveTo !== void 0) {
|
|
385
|
+
assertIsoDate(entry2.effectiveTo, `${label} effectiveTo`);
|
|
386
|
+
if (entry2.effectiveTo <= entry2.effectiveFrom) {
|
|
387
|
+
throw new Error(`${label} ends on or before it starts.`);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
assertTiers(entry2.tiers, label);
|
|
391
|
+
assertProvenance(entry2.provenance, label);
|
|
392
|
+
const key = rangeKey(entry2);
|
|
393
|
+
const ranges = seen.get(key) ?? [];
|
|
394
|
+
const clash = ranges.find((range) => overlaps(range, entry2));
|
|
395
|
+
if (clash) {
|
|
396
|
+
throw new Error(`${label} overlaps another rate for the same model and mode starting ${clash.effectiveFrom}; close the earlier rate with effectiveTo so each billing date resolves to exactly one price.`);
|
|
397
|
+
}
|
|
398
|
+
ranges.push(entry2);
|
|
399
|
+
seen.set(key, ranges);
|
|
400
|
+
}
|
|
401
|
+
const negotiatedSeen = /* @__PURE__ */ new Map();
|
|
402
|
+
for (const rate of catalogue.negotiated) {
|
|
403
|
+
const label = `Negotiated rate ${rate.provider}/${rate.model} from ${rate.effectiveFrom}`;
|
|
404
|
+
const hasDiscount = rate.discountPercent !== void 0;
|
|
405
|
+
const hasTiers = rate.tiers !== void 0;
|
|
406
|
+
if (hasDiscount === hasTiers) {
|
|
407
|
+
throw new Error(`${label} must set exactly one of discountPercent or tiers.`);
|
|
408
|
+
}
|
|
409
|
+
assertIsoDate(rate.effectiveFrom, `${label} effectiveFrom`);
|
|
410
|
+
if (rate.effectiveTo !== void 0) {
|
|
411
|
+
assertIsoDate(rate.effectiveTo, `${label} effectiveTo`);
|
|
412
|
+
if (rate.effectiveTo <= rate.effectiveFrom)
|
|
413
|
+
throw new Error(`${label} ends on or before it starts.`);
|
|
414
|
+
}
|
|
415
|
+
if (hasDiscount && (!Number.isFinite(rate.discountPercent) || rate.discountPercent < 0 || rate.discountPercent >= 100)) {
|
|
416
|
+
throw new Error(`${label} discountPercent must be at least 0 and below 100.`);
|
|
417
|
+
}
|
|
418
|
+
if (hasTiers)
|
|
419
|
+
assertTiers(rate.tiers, label);
|
|
420
|
+
assertProvenance(rate.provenance, label);
|
|
421
|
+
const key = rangeKey(rate);
|
|
422
|
+
const ranges = negotiatedSeen.get(key) ?? [];
|
|
423
|
+
if (ranges.some((range) => overlaps(range, rate))) {
|
|
424
|
+
throw new Error(`${label} overlaps another negotiated rate for the same model and mode.`);
|
|
425
|
+
}
|
|
426
|
+
ranges.push(rate);
|
|
427
|
+
negotiatedSeen.set(key, ranges);
|
|
428
|
+
}
|
|
429
|
+
return catalogue;
|
|
430
|
+
}
|
|
431
|
+
function applicable(items, at) {
|
|
432
|
+
return items.find((item) => item.effectiveFrom <= at && (item.effectiveTo === void 0 || at < item.effectiveTo));
|
|
433
|
+
}
|
|
434
|
+
function tierFor(tiers, inputTokens) {
|
|
435
|
+
const bounded = tiers.filter((tier) => tier.maxInputTokens !== void 0).sort((left, right) => left.maxInputTokens - right.maxInputTokens);
|
|
436
|
+
const match = bounded.find((tier) => inputTokens <= tier.maxInputTokens);
|
|
437
|
+
return match ?? tiers.find((tier) => tier.maxInputTokens === void 0);
|
|
438
|
+
}
|
|
439
|
+
function withoutBound(tier) {
|
|
440
|
+
const { maxInputTokens: _bound, ...price } = tier;
|
|
441
|
+
return price;
|
|
442
|
+
}
|
|
443
|
+
function discounted(tier, discountPercent) {
|
|
444
|
+
const factor = 1 - discountPercent / 100;
|
|
445
|
+
return {
|
|
446
|
+
inputPerMillion: tier.inputPerMillion * factor,
|
|
447
|
+
outputPerMillion: tier.outputPerMillion * factor,
|
|
448
|
+
...tier.cachedInputPerMillion === void 0 ? {} : { cachedInputPerMillion: tier.cachedInputPerMillion * factor },
|
|
449
|
+
...tier.cacheWriteInputPerMillion === void 0 ? {} : { cacheWriteInputPerMillion: tier.cacheWriteInputPerMillion * factor },
|
|
450
|
+
...tier.cacheWrite1hInputPerMillion === void 0 ? {} : { cacheWrite1hInputPerMillion: tier.cacheWrite1hInputPerMillion * factor }
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
function lookupPrice(catalogue, query, inputTokens = 0) {
|
|
454
|
+
assertIsoDate(query.at, "Price query `at`");
|
|
455
|
+
const mode = query.mode ?? "standard";
|
|
456
|
+
const candidates = catalogue.entries.filter((entry3) => entry3.provider === query.provider && entry3.model === query.model && entry3.mode === mode);
|
|
457
|
+
if (candidates.length === 0) {
|
|
458
|
+
return { status: "unpriced", reason: `No ${mode} rate is catalogued for ${query.provider}/${query.model}.` };
|
|
459
|
+
}
|
|
460
|
+
const entry2 = applicable(candidates, query.at);
|
|
461
|
+
if (!entry2) {
|
|
462
|
+
const ranges = candidates.map((candidate) => `${candidate.effectiveFrom} to ${candidate.effectiveTo ?? "current"}`).join("; ");
|
|
463
|
+
return {
|
|
464
|
+
status: "unpriced",
|
|
465
|
+
reason: `No ${mode} rate for ${query.provider}/${query.model} covers ${query.at}; catalogued ranges are ${ranges}.`
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
const tier = tierFor(entry2.tiers, inputTokens);
|
|
469
|
+
const override = applicable(catalogue.negotiated.filter((rate) => rate.provider === query.provider && rate.model === query.model && (rate.mode ?? "standard") === mode), query.at);
|
|
470
|
+
if (!override)
|
|
471
|
+
return { status: "priced", price: withoutBound(tier), entry: entry2 };
|
|
472
|
+
const price = override.tiers ? withoutBound(tierFor(override.tiers, inputTokens)) : discounted(tier, override.discountPercent);
|
|
473
|
+
return { status: "priced", price, entry: entry2, negotiated: override };
|
|
474
|
+
}
|
|
475
|
+
function priceUsage(catalogue, query, usage) {
|
|
476
|
+
for (const [name, value] of Object.entries(usage)) {
|
|
477
|
+
if (value === void 0)
|
|
478
|
+
continue;
|
|
479
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
480
|
+
throw new Error(`Token usage ${name} must be a non-negative finite number.`);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
const lookup = lookupPrice(catalogue, query, usage.inputTokens);
|
|
484
|
+
if (lookup.status === "unpriced")
|
|
485
|
+
return lookup;
|
|
486
|
+
const cached = usage.cachedInputTokens ?? 0;
|
|
487
|
+
const cacheWrite = usage.cacheWriteInputTokens ?? 0;
|
|
488
|
+
const cacheWrite1h = usage.cacheWrite1hInputTokens ?? 0;
|
|
489
|
+
if (cacheWrite1h > cacheWrite) {
|
|
490
|
+
return {
|
|
491
|
+
status: "unpriced",
|
|
492
|
+
reason: `${query.provider}/${query.model} reports more 1h cache writes (${cacheWrite1h}) than total cache writes (${cacheWrite}).`
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
if (cached > 0 && lookup.price.cachedInputPerMillion === void 0 || cacheWrite - cacheWrite1h > 0 && lookup.price.cacheWriteInputPerMillion === void 0 || cacheWrite1h > 0 && lookup.price.cacheWrite1hInputPerMillion === void 0) {
|
|
496
|
+
return {
|
|
497
|
+
status: "unpriced",
|
|
498
|
+
reason: `${query.provider}/${query.model} billed cached input tokens on ${query.at}, but the catalogue has no cached-input rate for it.`
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
const costUsd = estimateTokenCost(usage.inputTokens, usage.outputTokens, lookup.price, {
|
|
502
|
+
cachedInputTokens: usage.cachedInputTokens,
|
|
503
|
+
cacheWriteInputTokens: usage.cacheWriteInputTokens,
|
|
504
|
+
cacheWrite1hInputTokens: usage.cacheWrite1hInputTokens
|
|
505
|
+
});
|
|
506
|
+
return { status: "priced", costUsd, price: lookup.price, entry: lookup.entry, negotiated: lookup.negotiated };
|
|
507
|
+
}
|
|
508
|
+
function canonicalJson(value) {
|
|
509
|
+
if (value === null || typeof value !== "object")
|
|
510
|
+
return JSON.stringify(value) ?? "null";
|
|
511
|
+
if (Array.isArray(value)) {
|
|
512
|
+
return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
|
|
513
|
+
}
|
|
514
|
+
const entries = Object.entries(value).filter(([, member]) => member !== void 0).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, member]) => `${JSON.stringify(key)}:${canonicalJson(member)}`);
|
|
515
|
+
return `{${entries.join(",")}}`;
|
|
516
|
+
}
|
|
517
|
+
function catalogueVersion(catalogue) {
|
|
518
|
+
return createHash2("sha256").update(canonicalJson(catalogue)).digest("hex").slice(0, 16);
|
|
519
|
+
}
|
|
520
|
+
function canonicalSegments(value) {
|
|
521
|
+
return value.toLowerCase().replace(/[-_]latest$/, "").replace(/[-_]20\d{2}[-_]\d{2}[-_]\d{2}$/, "").replace(/[-_]20\d{6}$/, "").split(/[^a-z0-9]+/).filter((segment) => segment !== "");
|
|
522
|
+
}
|
|
523
|
+
function containsSegments(id, key) {
|
|
524
|
+
if (key.length === 0 || key.length > id.length)
|
|
525
|
+
return false;
|
|
526
|
+
for (let offset = 0; offset + key.length <= id.length; offset += 1) {
|
|
527
|
+
if (key.every((segment, index) => id[offset + index] === segment))
|
|
528
|
+
return true;
|
|
529
|
+
}
|
|
530
|
+
return false;
|
|
531
|
+
}
|
|
532
|
+
function resolveModelId(catalogue, provider, rawModelId) {
|
|
533
|
+
const known = [...new Set(catalogue.entries.filter((entry2) => entry2.provider === provider).map((entry2) => entry2.model))];
|
|
534
|
+
if (known.length === 0) {
|
|
535
|
+
return { status: "unresolved", reason: `No models are catalogued for provider ${provider}.`, candidates: [] };
|
|
536
|
+
}
|
|
537
|
+
if (known.includes(rawModelId))
|
|
538
|
+
return { status: "resolved", model: rawModelId, via: "exact" };
|
|
539
|
+
for (const alias of catalogue.aliases ?? []) {
|
|
540
|
+
if (alias.provider !== provider)
|
|
541
|
+
continue;
|
|
542
|
+
const matches = alias.id.endsWith("*") ? rawModelId.startsWith(alias.id.slice(0, -1)) : alias.id === rawModelId;
|
|
543
|
+
if (!matches)
|
|
544
|
+
continue;
|
|
545
|
+
if (!known.includes(alias.model)) {
|
|
546
|
+
return {
|
|
547
|
+
status: "unresolved",
|
|
548
|
+
reason: `Alias ${alias.id} points at ${alias.model}, which is not catalogued for ${provider}.`,
|
|
549
|
+
candidates: known
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
return { status: "resolved", model: alias.model, via: "alias" };
|
|
553
|
+
}
|
|
554
|
+
const target = canonicalSegments(rawModelId);
|
|
555
|
+
const hits = known.map((model) => ({ model, key: canonicalSegments(model) })).filter(({ key }) => containsSegments(target, key)).sort((left, right) => right.key.length - left.key.length || right.key.join("").length - left.key.join("").length);
|
|
556
|
+
if (hits.length === 0) {
|
|
557
|
+
return {
|
|
558
|
+
status: "unresolved",
|
|
559
|
+
reason: `No catalogue key for ${provider} matches ${rawModelId}; add an alias.`,
|
|
560
|
+
candidates: known
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
const longest = hits[0].key.length;
|
|
564
|
+
const tied = hits.filter(({ key }) => key.length === longest);
|
|
565
|
+
if (tied.length > 1) {
|
|
566
|
+
return {
|
|
567
|
+
status: "unresolved",
|
|
568
|
+
reason: `${rawModelId} matches ${tied.length} catalogue keys equally well; add an explicit alias.`,
|
|
569
|
+
candidates: tied.map(({ model }) => model)
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
return { status: "resolved", model: hits[0].model, via: "normalised" };
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// packages/core/dist/src/tool-schema.js
|
|
576
|
+
var PLUMBING_KEYS = /* @__PURE__ */ new Set(["$schema", "$id", "$comment", "$defs"]);
|
|
577
|
+
function toolNameOf(tool, index) {
|
|
578
|
+
if (typeof tool === "object" && tool !== null) {
|
|
579
|
+
const row = tool;
|
|
580
|
+
const direct = typeof row.name === "string" ? row.name : void 0;
|
|
581
|
+
const nested = typeof row.function === "object" && row.function !== null ? row.function.name : void 0;
|
|
582
|
+
return direct ?? (typeof nested === "string" ? nested : `tool-${index}`);
|
|
583
|
+
}
|
|
584
|
+
return `tool-${index}`;
|
|
585
|
+
}
|
|
586
|
+
function truncateToTokens(text, limit, countTokens) {
|
|
587
|
+
if (countTokens(text) <= limit)
|
|
588
|
+
return text;
|
|
589
|
+
const sentences = text.split(/(?<=[.!?])\s+/);
|
|
590
|
+
let kept = "";
|
|
591
|
+
for (const sentence of sentences) {
|
|
592
|
+
const candidate = kept === "" ? sentence : `${kept} ${sentence}`;
|
|
593
|
+
if (countTokens(candidate) > limit)
|
|
594
|
+
break;
|
|
595
|
+
kept = candidate;
|
|
596
|
+
}
|
|
597
|
+
if (kept !== "")
|
|
598
|
+
return kept;
|
|
599
|
+
const words2 = text.split(/\s+/);
|
|
600
|
+
let fallback = "";
|
|
601
|
+
for (const word of words2) {
|
|
602
|
+
const candidate = fallback === "" ? word : `${fallback} ${word}`;
|
|
603
|
+
if (countTokens(candidate) > limit)
|
|
604
|
+
break;
|
|
605
|
+
fallback = candidate;
|
|
606
|
+
}
|
|
607
|
+
return fallback;
|
|
608
|
+
}
|
|
609
|
+
function minifyToolSchemas(tools, options = {}) {
|
|
610
|
+
if (!Array.isArray(tools))
|
|
611
|
+
throw new Error("A tool catalogue must be an array.");
|
|
612
|
+
const countTokens = options.countTokens ?? approximateTokenCount;
|
|
613
|
+
const descriptions = options.descriptions ?? "keep";
|
|
614
|
+
const maximumDescriptionTokens = options.maximumDescriptionTokens ?? 24;
|
|
615
|
+
if (!Number.isInteger(maximumDescriptionTokens) || maximumDescriptionTokens < 1) {
|
|
616
|
+
throw new Error("maximumDescriptionTokens must be a positive integer.");
|
|
617
|
+
}
|
|
618
|
+
const changes = [];
|
|
619
|
+
const tokensBefore = countTokens(JSON.stringify(tools));
|
|
620
|
+
const keep = options.keepTools === void 0 ? void 0 : new Set(options.keepTools);
|
|
621
|
+
const retained = [];
|
|
622
|
+
tools.forEach((tool, index) => {
|
|
623
|
+
const name = toolNameOf(tool, index);
|
|
624
|
+
if (keep !== void 0 && !keep.has(name)) {
|
|
625
|
+
changes.push({
|
|
626
|
+
kind: "dropped-tool",
|
|
627
|
+
lossless: false,
|
|
628
|
+
at: name,
|
|
629
|
+
tokensSaved: countTokens(JSON.stringify(tool)),
|
|
630
|
+
detail: "Tool removed from the catalogue. The model cannot call what it cannot see, so this is safe only where the tool is genuinely unreachable for the workload."
|
|
631
|
+
});
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
retained.push(prune(tool, name));
|
|
635
|
+
});
|
|
636
|
+
function prune(node, path) {
|
|
637
|
+
if (Array.isArray(node))
|
|
638
|
+
return node.map((item, index) => prune(item, `${path}[${index}]`));
|
|
639
|
+
if (typeof node !== "object" || node === null)
|
|
640
|
+
return node;
|
|
641
|
+
const row = node;
|
|
642
|
+
const output = {};
|
|
643
|
+
for (const [key, value] of Object.entries(row)) {
|
|
644
|
+
if (PLUMBING_KEYS.has(key)) {
|
|
645
|
+
changes.push({
|
|
646
|
+
kind: "plumbing",
|
|
647
|
+
lossless: true,
|
|
648
|
+
at: `${path}.${key}`,
|
|
649
|
+
tokensSaved: countTokens(JSON.stringify({ [key]: value })),
|
|
650
|
+
detail: `Removed ${key}, which the model never reads.`
|
|
651
|
+
});
|
|
652
|
+
continue;
|
|
653
|
+
}
|
|
654
|
+
if (key === "title" && typeof value === "string") {
|
|
655
|
+
const leaf = path.split(".").pop() ?? "";
|
|
656
|
+
if (value.toLowerCase().replace(/[^a-z0-9]/g, "") === leaf.toLowerCase().replace(/[^a-z0-9]/g, "")) {
|
|
657
|
+
changes.push({
|
|
658
|
+
kind: "redundant-title",
|
|
659
|
+
lossless: true,
|
|
660
|
+
at: `${path}.title`,
|
|
661
|
+
tokensSaved: countTokens(JSON.stringify({ title: value })),
|
|
662
|
+
detail: `Removed title "${value}", which restates the property name.`
|
|
663
|
+
});
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
if (key === "examples" && options.dropExamples) {
|
|
668
|
+
changes.push({
|
|
669
|
+
kind: "examples",
|
|
670
|
+
lossless: false,
|
|
671
|
+
at: `${path}.examples`,
|
|
672
|
+
tokensSaved: countTokens(JSON.stringify({ examples: value })),
|
|
673
|
+
detail: "Removed examples. The model does read these when choosing arguments."
|
|
674
|
+
});
|
|
675
|
+
continue;
|
|
676
|
+
}
|
|
677
|
+
if (key === "description" && typeof value === "string" && descriptions !== "keep") {
|
|
678
|
+
if (descriptions === "drop") {
|
|
679
|
+
changes.push({
|
|
680
|
+
kind: "description",
|
|
681
|
+
lossless: false,
|
|
682
|
+
at: `${path}.description`,
|
|
683
|
+
tokensSaved: countTokens(JSON.stringify({ description: value })),
|
|
684
|
+
detail: "Removed description. Descriptions are how the model chooses between tools."
|
|
685
|
+
});
|
|
686
|
+
continue;
|
|
687
|
+
}
|
|
688
|
+
const trimmed = truncateToTokens(value, maximumDescriptionTokens, countTokens);
|
|
689
|
+
if (trimmed !== value) {
|
|
690
|
+
changes.push({
|
|
691
|
+
kind: "description",
|
|
692
|
+
lossless: false,
|
|
693
|
+
at: `${path}.description`,
|
|
694
|
+
tokensSaved: countTokens(value) - countTokens(trimmed),
|
|
695
|
+
detail: `Truncated description from ${countTokens(value)} to ${countTokens(trimmed)} tokens.`
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
output[key] = trimmed;
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
output[key] = prune(value, `${path}.${key}`);
|
|
702
|
+
}
|
|
703
|
+
return output;
|
|
704
|
+
}
|
|
705
|
+
const compact = JSON.stringify(retained);
|
|
706
|
+
const tokensAfter = countTokens(compact);
|
|
707
|
+
return {
|
|
708
|
+
tools: retained,
|
|
709
|
+
tokensBefore,
|
|
710
|
+
tokensAfter,
|
|
711
|
+
losslessTokensSaved: changes.filter((change) => change.lossless).reduce((sum, change) => sum + change.tokensSaved, 0),
|
|
712
|
+
lossyTokensSaved: changes.filter((change) => !change.lossless).reduce((sum, change) => sum + change.tokensSaved, 0),
|
|
713
|
+
changes
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
function auditToolCatalogue(tools, options = {}) {
|
|
717
|
+
const countTokens = options.countTokens ?? approximateTokenCount;
|
|
718
|
+
const lossless = minifyToolSchemas(tools, { countTokens, descriptions: "keep" });
|
|
719
|
+
const trimmed = minifyToolSchemas(tools, {
|
|
720
|
+
countTokens,
|
|
721
|
+
descriptions: "truncate",
|
|
722
|
+
maximumDescriptionTokens: options.maximumDescriptionTokens ?? 24
|
|
723
|
+
});
|
|
724
|
+
const largest = tools.map((tool, index) => {
|
|
725
|
+
const serialised = JSON.stringify(tool);
|
|
726
|
+
const descriptions = [...serialised.matchAll(/"description"\s*:\s*"((?:[^"\\]|\\.)*)"/g)].reduce((sum, match) => sum + countTokens(match[1] ?? ""), 0);
|
|
727
|
+
return { name: toolNameOf(tool, index), tokens: countTokens(serialised), descriptionTokens: descriptions };
|
|
728
|
+
}).sort((left, right) => right.tokens - left.tokens);
|
|
729
|
+
const tokens = countTokens(JSON.stringify(tools));
|
|
730
|
+
return {
|
|
731
|
+
tools: tools.length,
|
|
732
|
+
tokens,
|
|
733
|
+
meanTokensPerTool: tools.length === 0 ? 0 : tokens / tools.length,
|
|
734
|
+
largest: largest.slice(0, 10),
|
|
735
|
+
losslessTokensSaved: Math.max(0, lossless.tokensBefore - lossless.tokensAfter),
|
|
736
|
+
descriptionTokensSaved: trimmed.lossyTokensSaved
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// packages/core/dist/src/savings-ledger.js
|
|
741
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
742
|
+
var LEDGER_GENESIS_HASH = "0".repeat(64);
|
|
743
|
+
function ledgerEventHash(seq, prevHash, event) {
|
|
744
|
+
return createHash3("sha256").update(`${seq}
|
|
745
|
+
${prevHash}
|
|
746
|
+
${canonicalJson(event)}`).digest("hex");
|
|
747
|
+
}
|
|
748
|
+
function chainLedgerRecord(event, previous) {
|
|
749
|
+
const seq = previous ? previous.seq + 1 : 0;
|
|
750
|
+
const prevHash = previous ? previous.eventHash : LEDGER_GENESIS_HASH;
|
|
751
|
+
return { seq, prevHash, eventHash: ledgerEventHash(seq, prevHash, event), event };
|
|
752
|
+
}
|
|
753
|
+
function verifyLedgerChain(records) {
|
|
754
|
+
let prevHash = LEDGER_GENESIS_HASH;
|
|
755
|
+
for (let index = 0; index < records.length; index += 1) {
|
|
756
|
+
const record = records[index];
|
|
757
|
+
if (record.seq !== index) {
|
|
758
|
+
return { ok: false, seq: record.seq, reason: `Expected seq ${index}, found ${record.seq}.` };
|
|
759
|
+
}
|
|
760
|
+
if (record.prevHash !== prevHash) {
|
|
761
|
+
return { ok: false, seq: record.seq, reason: "prevHash does not match the preceding record." };
|
|
762
|
+
}
|
|
763
|
+
const expected = ledgerEventHash(record.seq, record.prevHash, record.event);
|
|
764
|
+
if (record.eventHash !== expected) {
|
|
765
|
+
return { ok: false, seq: record.seq, reason: "eventHash does not match the event content." };
|
|
766
|
+
}
|
|
767
|
+
prevHash = record.eventHash;
|
|
768
|
+
}
|
|
769
|
+
return { ok: true, length: records.length };
|
|
770
|
+
}
|
|
771
|
+
function priceLedgerUsage(catalogue, input) {
|
|
772
|
+
const resolution = resolveModelId(catalogue, input.provider, input.model);
|
|
773
|
+
if (resolution.status === "unresolved") {
|
|
774
|
+
return { pricingStatus: "unpriced", reason: resolution.reason };
|
|
775
|
+
}
|
|
776
|
+
const split5m = input.usage.cacheWrite5mInputTokens;
|
|
777
|
+
const split1h = input.usage.cacheWrite1hInputTokens;
|
|
778
|
+
const splitReported = split5m !== void 0 || split1h !== void 0;
|
|
779
|
+
if (splitReported && (split5m ?? 0) + (split1h ?? 0) !== input.usage.cacheWriteInputTokens) {
|
|
780
|
+
return {
|
|
781
|
+
pricingStatus: "unpriced",
|
|
782
|
+
reason: `Cache-write TTL split (5m ${split5m ?? 0} + 1h ${split1h ?? 0}) does not sum to cacheWriteInputTokens ${input.usage.cacheWriteInputTokens}.`
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
const write1hForPricing = splitReported ? split1h ?? 0 : input.meshPlacedCacheControl && input.meshPlacedTtl !== "5m" ? input.usage.cacheWriteInputTokens : 0;
|
|
786
|
+
const pricingUsage = {
|
|
787
|
+
inputTokens: input.usage.inputTokens,
|
|
788
|
+
outputTokens: input.usage.outputTokens,
|
|
789
|
+
cachedInputTokens: input.usage.cachedInputTokens,
|
|
790
|
+
cacheWriteInputTokens: input.usage.cacheWriteInputTokens,
|
|
791
|
+
cacheWrite1hInputTokens: write1hForPricing
|
|
792
|
+
};
|
|
793
|
+
const query = { provider: input.provider, model: resolution.model, at: input.at };
|
|
794
|
+
const actual = priceUsage(catalogue, { ...query, mode: input.executionMode }, pricingUsage);
|
|
795
|
+
if (actual.status === "unpriced")
|
|
796
|
+
return { pricingStatus: "unpriced", reason: actual.reason };
|
|
797
|
+
const actualInputOnly = priceUsage(catalogue, { ...query, mode: input.executionMode }, {
|
|
798
|
+
...pricingUsage,
|
|
799
|
+
outputTokens: 0
|
|
800
|
+
});
|
|
801
|
+
if (actualInputOnly.status === "unpriced") {
|
|
802
|
+
return { pricingStatus: "unpriced", reason: actualInputOnly.reason };
|
|
803
|
+
}
|
|
804
|
+
const standardWithCache = input.executionMode === "standard" ? actual : priceUsage(catalogue, { ...query, mode: "standard" }, pricingUsage);
|
|
805
|
+
if (standardWithCache.status === "unpriced") {
|
|
806
|
+
return { pricingStatus: "unpriced", reason: standardWithCache.reason };
|
|
807
|
+
}
|
|
808
|
+
const fullRateStandard = priceUsage(catalogue, { ...query, mode: "standard" }, {
|
|
809
|
+
inputTokens: input.usage.inputTokens,
|
|
810
|
+
outputTokens: input.usage.outputTokens
|
|
811
|
+
});
|
|
812
|
+
if (fullRateStandard.status === "unpriced") {
|
|
813
|
+
return { pricingStatus: "unpriced", reason: fullRateStandard.reason };
|
|
814
|
+
}
|
|
815
|
+
const cacheDeltaUsd = fullRateStandard.costUsd - standardWithCache.costUsd;
|
|
816
|
+
const hasCacheTraffic = input.usage.cachedInputTokens > 0 || input.usage.cacheWriteInputTokens > 0;
|
|
817
|
+
return {
|
|
818
|
+
pricingStatus: "priced",
|
|
819
|
+
actualUsd: actual.costUsd,
|
|
820
|
+
inputUsd: actualInputOnly.costUsd,
|
|
821
|
+
...input.executionMode === "standard" ? {} : { tierSavedUsd: standardWithCache.costUsd - actual.costUsd },
|
|
822
|
+
...hasCacheTraffic ? input.meshPlacedCacheControl ? { cacheAttributedUsd: cacheDeltaUsd } : { providerCacheAlreadyOwnedUsd: cacheDeltaUsd } : {}
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
function auditEventPricing(event, catalogue) {
|
|
826
|
+
const version = catalogueVersion(catalogue);
|
|
827
|
+
if (event.catalogueVersion !== void 0 && event.catalogueVersion !== version) {
|
|
828
|
+
return { status: "catalogue_version_mismatch", eventVersion: event.catalogueVersion, catalogueVersion: version };
|
|
829
|
+
}
|
|
830
|
+
if (!event.usage) {
|
|
831
|
+
return event.actualUsd === void 0 ? { status: "verified" } : { status: "mismatch", field: "actualUsd", storedUsd: event.actualUsd, recomputedUsd: Number.NaN };
|
|
832
|
+
}
|
|
833
|
+
const cacheLever = event.levers.find((lever) => lever.kind === "provider_cache");
|
|
834
|
+
const meshPlacedCacheControl = cacheLever !== void 0;
|
|
835
|
+
const meshPlacedTtl = cacheLever?.leverId === "cache-ttl-5m" ? "5m" : cacheLever?.leverId === "cache-ttl-1h" ? "1h" : void 0;
|
|
836
|
+
const priced = priceLedgerUsage(catalogue, {
|
|
837
|
+
provider: event.provider,
|
|
838
|
+
model: event.model,
|
|
839
|
+
at: event.timestamp.slice(0, 10),
|
|
840
|
+
executionMode: event.executionMode,
|
|
841
|
+
usage: event.usage,
|
|
842
|
+
meshPlacedCacheControl,
|
|
843
|
+
...meshPlacedTtl === void 0 ? {} : { meshPlacedTtl }
|
|
844
|
+
});
|
|
845
|
+
if (priced.pricingStatus === "unpriced") {
|
|
846
|
+
return event.pricingStatus === "unpriced" ? { status: "verified" } : { status: "unpriced", reason: priced.reason };
|
|
847
|
+
}
|
|
848
|
+
if (priced.actualUsd !== event.actualUsd) {
|
|
849
|
+
return { status: "mismatch", field: "actualUsd", storedUsd: event.actualUsd, recomputedUsd: priced.actualUsd };
|
|
850
|
+
}
|
|
851
|
+
return { status: "verified" };
|
|
852
|
+
}
|
|
853
|
+
function compareEvents(left, right) {
|
|
854
|
+
if (left.timestamp !== right.timestamp)
|
|
855
|
+
return left.timestamp < right.timestamp ? -1 : 1;
|
|
856
|
+
if (left.requestId !== right.requestId)
|
|
857
|
+
return left.requestId < right.requestId ? -1 : 1;
|
|
858
|
+
return 0;
|
|
859
|
+
}
|
|
860
|
+
function leverUnbilledReason(lever, gateVerdictOverrides) {
|
|
861
|
+
const verdict = gateVerdictOverrides?.[lever.leverId] ?? lever.gateVerdict?.verdict;
|
|
862
|
+
if (lever.savedUsd === void 0 || !Number.isFinite(lever.savedUsd)) {
|
|
863
|
+
return "no per-request verified measurement";
|
|
864
|
+
}
|
|
865
|
+
if (lever.kind === "lossy" && verdict !== "approved") {
|
|
866
|
+
return verdict === "rejected" ? "replay gate verdict rejected" : "no approved replay gate verdict";
|
|
867
|
+
}
|
|
868
|
+
return void 0;
|
|
869
|
+
}
|
|
870
|
+
function selectBillableEvents(events, options = {}) {
|
|
871
|
+
for (const event of events) {
|
|
872
|
+
if (event.schemaVersion !== 1) {
|
|
873
|
+
throw new Error(`Unsupported billable event schemaVersion ${JSON.stringify(event.schemaVersion)}.`);
|
|
874
|
+
}
|
|
875
|
+
if (typeof event.requestId !== "string" || event.requestId === "") {
|
|
876
|
+
throw new Error("Billable events need a non-empty requestId idempotency key.");
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
const sorted = [...events].sort(compareEvents);
|
|
880
|
+
const seen = /* @__PURE__ */ new Set();
|
|
881
|
+
let duplicatesIgnored = 0;
|
|
882
|
+
const deduped = [];
|
|
883
|
+
for (const event of sorted) {
|
|
884
|
+
if (seen.has(event.requestId)) {
|
|
885
|
+
duplicatesIgnored += 1;
|
|
886
|
+
continue;
|
|
887
|
+
}
|
|
888
|
+
seen.add(event.requestId);
|
|
889
|
+
deduped.push(event);
|
|
890
|
+
}
|
|
891
|
+
const consideredEvents = options.period === void 0 ? deduped : deduped.filter((event) => event.invoicePeriod === options.period);
|
|
892
|
+
const failedTaskIds = new Set(options.failedTaskIds ?? []);
|
|
893
|
+
for (const event of consideredEvents) {
|
|
894
|
+
if (event.status === "failed" && event.taskId !== void 0)
|
|
895
|
+
failedTaskIds.add(event.taskId);
|
|
896
|
+
}
|
|
897
|
+
let eventsExcludedForFailedTasks = 0;
|
|
898
|
+
const billableEvents = [];
|
|
899
|
+
for (const event of consideredEvents) {
|
|
900
|
+
const taskFailed = event.taskId !== void 0 && failedTaskIds.has(event.taskId);
|
|
901
|
+
if (taskFailed || event.status === "failed") {
|
|
902
|
+
eventsExcludedForFailedTasks += 1;
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
billableEvents.push(event);
|
|
906
|
+
}
|
|
907
|
+
return { consideredEvents, billableEvents, failedTaskIds, eventsExcludedForFailedTasks, duplicatesIgnored };
|
|
908
|
+
}
|
|
909
|
+
function recomputeInvoice(events, options = {}) {
|
|
910
|
+
return recomputeInvoiceWithAudit(events, options).invoice;
|
|
911
|
+
}
|
|
912
|
+
function recomputeInvoiceWithAudit(events, options = {}) {
|
|
913
|
+
if (options.feePercent !== void 0 && (!Number.isFinite(options.feePercent) || options.feePercent < 0 || options.feePercent > 100)) {
|
|
914
|
+
throw new Error("feePercent must be between 0 and 100.");
|
|
915
|
+
}
|
|
916
|
+
const { consideredEvents: inPeriod, billableEvents, failedTaskIds, eventsExcludedForFailedTasks, duplicatesIgnored } = selectBillableEvents(events, options);
|
|
917
|
+
const lineMap = /* @__PURE__ */ new Map();
|
|
918
|
+
const unbilledMap = /* @__PURE__ */ new Map();
|
|
919
|
+
const catalogueVersions = /* @__PURE__ */ new Set();
|
|
920
|
+
let unpricedEvents = 0;
|
|
921
|
+
let providerCacheOnAppendedHistoryUsd = 0;
|
|
922
|
+
let gatewayCacheUsd = 0;
|
|
923
|
+
let eventsBilled = 0;
|
|
924
|
+
for (const event of billableEvents) {
|
|
925
|
+
if (event.catalogueVersion !== void 0)
|
|
926
|
+
catalogueVersions.add(event.catalogueVersion);
|
|
927
|
+
if (event.pricingStatus === "unpriced")
|
|
928
|
+
unpricedEvents += 1;
|
|
929
|
+
providerCacheOnAppendedHistoryUsd += event.excluded?.providerCacheAlreadyOwnedUsd ?? 0;
|
|
930
|
+
gatewayCacheUsd += event.excluded?.gatewayCacheUsd ?? 0;
|
|
931
|
+
let billedAny = false;
|
|
932
|
+
for (const lever of event.levers) {
|
|
933
|
+
const unbilledReason = leverUnbilledReason(lever, options.gateVerdictOverrides);
|
|
934
|
+
if (unbilledReason !== void 0) {
|
|
935
|
+
const key2 = `${lever.kind}
|
|
936
|
+
${lever.leverId}
|
|
937
|
+
${unbilledReason}`;
|
|
938
|
+
const existing = unbilledMap.get(key2);
|
|
939
|
+
if (existing)
|
|
940
|
+
existing.events += 1;
|
|
941
|
+
else
|
|
942
|
+
unbilledMap.set(key2, { leverId: lever.leverId, kind: lever.kind, events: 1, reason: unbilledReason });
|
|
943
|
+
continue;
|
|
944
|
+
}
|
|
945
|
+
const key = `${lever.kind}
|
|
946
|
+
${lever.leverId}`;
|
|
947
|
+
const line = lineMap.get(key) ?? { leverId: lever.leverId, kind: lever.kind, events: 0, savedUsd: 0, estimated: false, refs: /* @__PURE__ */ new Set() };
|
|
948
|
+
line.events += 1;
|
|
949
|
+
line.savedUsd += lever.savedUsd;
|
|
950
|
+
if (lever.basis === "estimated-baseline")
|
|
951
|
+
line.estimated = true;
|
|
952
|
+
if (lever.gateVerdict?.ref)
|
|
953
|
+
line.refs.add(lever.gateVerdict.ref);
|
|
954
|
+
lineMap.set(key, line);
|
|
955
|
+
billedAny = true;
|
|
956
|
+
}
|
|
957
|
+
if (billedAny)
|
|
958
|
+
eventsBilled += 1;
|
|
959
|
+
}
|
|
960
|
+
const lineItems = [...lineMap.values()].sort((left, right) => left.leverId < right.leverId ? -1 : left.leverId > right.leverId ? 1 : 0).map((line) => ({
|
|
961
|
+
leverId: line.leverId,
|
|
962
|
+
kind: line.kind,
|
|
963
|
+
events: line.events,
|
|
964
|
+
savedUsd: line.savedUsd,
|
|
965
|
+
savedCents: Math.round(line.savedUsd * 100),
|
|
966
|
+
...line.estimated ? { basis: "estimated-baseline" } : {},
|
|
967
|
+
gateVerdictRefs: [...line.refs].sort()
|
|
968
|
+
}));
|
|
969
|
+
const recordedUnbilled = [...unbilledMap.values()].sort((left, right) => left.leverId < right.leverId ? -1 : left.leverId > right.leverId ? 1 : 0);
|
|
970
|
+
const netVerifiedSavingsUsd = lineItems.reduce((sum, line) => sum + line.savedUsd, 0);
|
|
971
|
+
const netVerifiedSavingsCents = Math.round(netVerifiedSavingsUsd * 100);
|
|
972
|
+
const carryIn = options.carryForwardDeficitUsd ?? 0;
|
|
973
|
+
if (carryIn < 0 || !Number.isFinite(carryIn)) {
|
|
974
|
+
throw new Error("carryForwardDeficitUsd must be a non-negative, finite USD amount.");
|
|
975
|
+
}
|
|
976
|
+
const netAfterCarryUsd = netVerifiedSavingsUsd - carryIn;
|
|
977
|
+
const carryOutUsd = Math.max(0, -netAfterCarryUsd);
|
|
978
|
+
const billableCents = Math.round(netAfterCarryUsd * 100);
|
|
979
|
+
const verdict = billableCents > 0 ? "charge" : "do_not_charge";
|
|
980
|
+
const amountDueUsd = options.feePercent === void 0 ? void 0 : verdict === "charge" ? netAfterCarryUsd * options.feePercent / 100 : 0;
|
|
981
|
+
const invoice = {
|
|
982
|
+
schemaVersion: 1,
|
|
983
|
+
period: options.period ?? null,
|
|
984
|
+
catalogueVersions: [...catalogueVersions].sort(),
|
|
985
|
+
eventsConsidered: inPeriod.length,
|
|
986
|
+
eventsBilled,
|
|
987
|
+
failedTaskIds: [...failedTaskIds].sort(),
|
|
988
|
+
eventsExcludedForFailedTasks,
|
|
989
|
+
unpricedEvents,
|
|
990
|
+
lineItems,
|
|
991
|
+
recordedUnbilled,
|
|
992
|
+
excludedAlreadyOwned: {
|
|
993
|
+
providerCacheOnAppendedHistoryUsd,
|
|
994
|
+
gatewayCacheUsd
|
|
995
|
+
},
|
|
996
|
+
netVerifiedSavingsUsd,
|
|
997
|
+
netVerifiedSavingsCents,
|
|
998
|
+
...options.carryForwardDeficitUsd === void 0 ? {} : { carryForwardDeficitInUsd: carryIn },
|
|
999
|
+
...carryOutUsd > 0 ? { carryForwardDeficitOutUsd: carryOutUsd } : {},
|
|
1000
|
+
verdict,
|
|
1001
|
+
...options.feePercent === void 0 ? {} : {
|
|
1002
|
+
feePercent: options.feePercent,
|
|
1003
|
+
amountDueUsd,
|
|
1004
|
+
amountDueCents: Math.round((amountDueUsd ?? 0) * 100)
|
|
1005
|
+
}
|
|
1006
|
+
};
|
|
1007
|
+
return { invoice, audit: { duplicatesIgnored } };
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
// packages/core/dist/src/gateway.js
|
|
1011
|
+
function reconcileGatewaySpend(parsed, catalogue, options = {}) {
|
|
1012
|
+
const tolerance = options.tolerancePercent ?? 10;
|
|
1013
|
+
let comparedRows = 0;
|
|
1014
|
+
let reportedSpendUsd = 0;
|
|
1015
|
+
let catalogueCostUsd = 0;
|
|
1016
|
+
let unpricedRows = 0;
|
|
1017
|
+
const flagged = [];
|
|
1018
|
+
const priceRow = (entry2) => {
|
|
1019
|
+
const resolved = resolveModelId(catalogue, entry2.record.provider, entry2.record.model);
|
|
1020
|
+
if (resolved.status !== "resolved")
|
|
1021
|
+
return void 0;
|
|
1022
|
+
const priced = priceUsage(catalogue, { provider: entry2.record.provider, model: resolved.model, at: entry2.record.at }, {
|
|
1023
|
+
inputTokens: entry2.record.inputTokens,
|
|
1024
|
+
outputTokens: entry2.record.outputTokens,
|
|
1025
|
+
cachedInputTokens: entry2.record.cachedInputTokens,
|
|
1026
|
+
cacheWriteInputTokens: entry2.record.cacheWriteInputTokens
|
|
1027
|
+
});
|
|
1028
|
+
return priced.status === "priced" ? priced.costUsd : void 0;
|
|
1029
|
+
};
|
|
1030
|
+
for (const entry2 of parsed.accepted) {
|
|
1031
|
+
const catalogued = priceRow(entry2);
|
|
1032
|
+
if (catalogued === void 0) {
|
|
1033
|
+
unpricedRows += 1;
|
|
1034
|
+
continue;
|
|
1035
|
+
}
|
|
1036
|
+
if (entry2.reportedSpendUsd === void 0)
|
|
1037
|
+
continue;
|
|
1038
|
+
comparedRows += 1;
|
|
1039
|
+
reportedSpendUsd += entry2.reportedSpendUsd;
|
|
1040
|
+
catalogueCostUsd += catalogued;
|
|
1041
|
+
const drift = catalogued === 0 ? 0 : (entry2.reportedSpendUsd - catalogued) / catalogued * 100;
|
|
1042
|
+
if (Math.abs(drift) > tolerance && Math.abs(entry2.reportedSpendUsd - catalogued) > 1e-4) {
|
|
1043
|
+
flagged.push({
|
|
1044
|
+
id: entry2.record.id,
|
|
1045
|
+
model: entry2.record.model,
|
|
1046
|
+
reportedUsd: entry2.reportedSpendUsd,
|
|
1047
|
+
catalogueUsd: catalogued,
|
|
1048
|
+
driftPercent: drift
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
let cacheHitSavedUsd = 0;
|
|
1053
|
+
for (const entry2 of parsed.cacheHits) {
|
|
1054
|
+
cacheHitSavedUsd += priceRow(entry2) ?? 0;
|
|
1055
|
+
}
|
|
1056
|
+
return {
|
|
1057
|
+
comparedRows,
|
|
1058
|
+
reportedSpendUsd,
|
|
1059
|
+
catalogueCostUsd,
|
|
1060
|
+
driftPercent: catalogueCostUsd === 0 ? 0 : (reportedSpendUsd - catalogueCostUsd) / catalogueCostUsd * 100,
|
|
1061
|
+
flagged,
|
|
1062
|
+
unpricedRows,
|
|
1063
|
+
cacheHitRows: parsed.cacheHits.length,
|
|
1064
|
+
cacheHitSavedUsd
|
|
1065
|
+
};
|
|
1066
|
+
}
|
|
1067
|
+
function signalsFromGateway(stream, cacheHits, measurement) {
|
|
1068
|
+
const endUsers = new Set(stream.map((entry2) => entry2.endUser).filter((user) => !!user));
|
|
1069
|
+
const withEndUser = stream.filter((entry2) => entry2.endUser !== void 0).length;
|
|
1070
|
+
let weightedRoundTrips = 0;
|
|
1071
|
+
let identifiedRequests = 0;
|
|
1072
|
+
let inputTokens = 0;
|
|
1073
|
+
let toolTokens = 0;
|
|
1074
|
+
let breakers = false;
|
|
1075
|
+
for (const workload of measurement.workloads) {
|
|
1076
|
+
if (workload.episodes.identified) {
|
|
1077
|
+
weightedRoundTrips += workload.episodes.meanRoundTrips * workload.requests;
|
|
1078
|
+
identifiedRequests += workload.requests;
|
|
1079
|
+
}
|
|
1080
|
+
inputTokens += workload.inputTokens;
|
|
1081
|
+
toolTokens += workload.meanToolSchemaTokens * workload.requests;
|
|
1082
|
+
if ((workload.caching?.prefixBreakers.length ?? 0) > 0)
|
|
1083
|
+
breakers = true;
|
|
1084
|
+
}
|
|
1085
|
+
return {
|
|
1086
|
+
requests: stream.length,
|
|
1087
|
+
distinctEndUsers: endUsers.size,
|
|
1088
|
+
requestsPerEndUser: endUsers.size === 0 ? stream.length : withEndUser / endUsers.size,
|
|
1089
|
+
episodesIdentified: measurement.episodesIdentified,
|
|
1090
|
+
meanRoundTrips: identifiedRequests === 0 ? 1 : weightedRoundTrips / identifiedRequests,
|
|
1091
|
+
toolSchemaShare: inputTokens === 0 ? 0 : toolTokens / inputTokens,
|
|
1092
|
+
alreadyCachedShare: measurement.alreadyCachedShare,
|
|
1093
|
+
gatewayCacheHitShare: stream.length + cacheHits.length === 0 ? 0 : cacheHits.length / (stream.length + cacheHits.length),
|
|
1094
|
+
promptTextAvailable: measurement.promptTextAvailable,
|
|
1095
|
+
prefixBreakersFound: breakers,
|
|
1096
|
+
meanInputTokens: stream.length === 0 ? 0 : inputTokens / stream.length
|
|
1097
|
+
};
|
|
1098
|
+
}
|
|
1099
|
+
function groupByStream(entries) {
|
|
1100
|
+
const streams = /* @__PURE__ */ new Map();
|
|
1101
|
+
for (const entry2 of entries) {
|
|
1102
|
+
const list = streams.get(entry2.record.workload) ?? [];
|
|
1103
|
+
list.push(entry2);
|
|
1104
|
+
streams.set(entry2.record.workload, list);
|
|
1105
|
+
}
|
|
1106
|
+
return streams;
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
// packages/core/dist/src/helicone.js
|
|
1110
|
+
function asString(value) {
|
|
1111
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
1112
|
+
}
|
|
1113
|
+
function asNumber(value) {
|
|
1114
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
1115
|
+
}
|
|
1116
|
+
function isPlainObject(value) {
|
|
1117
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1118
|
+
}
|
|
1119
|
+
function providerFor(model, explicit) {
|
|
1120
|
+
if (explicit)
|
|
1121
|
+
return explicit.toLowerCase();
|
|
1122
|
+
if (/^(?:claude|opus|sonnet|haiku|fable)/i.test(model))
|
|
1123
|
+
return "anthropic";
|
|
1124
|
+
if (/^(?:gpt|o\d|chatgpt|text-)/i.test(model))
|
|
1125
|
+
return "openai";
|
|
1126
|
+
if (/^gemini/i.test(model))
|
|
1127
|
+
return "google";
|
|
1128
|
+
return "unknown";
|
|
1129
|
+
}
|
|
1130
|
+
function property(properties, name) {
|
|
1131
|
+
for (const [key, value] of Object.entries(properties)) {
|
|
1132
|
+
if (key.toLowerCase() === name.toLowerCase())
|
|
1133
|
+
return asString(value);
|
|
1134
|
+
}
|
|
1135
|
+
return void 0;
|
|
1136
|
+
}
|
|
1137
|
+
function flattenMessages(value) {
|
|
1138
|
+
if (!Array.isArray(value) || value.length === 0)
|
|
1139
|
+
return void 0;
|
|
1140
|
+
const parts = [];
|
|
1141
|
+
for (const message of value) {
|
|
1142
|
+
if (!isPlainObject(message))
|
|
1143
|
+
continue;
|
|
1144
|
+
const content = message.content;
|
|
1145
|
+
if (typeof content === "string" && content.trim() !== "")
|
|
1146
|
+
parts.push(content);
|
|
1147
|
+
else if (Array.isArray(content)) {
|
|
1148
|
+
for (const piece of content) {
|
|
1149
|
+
if (isPlainObject(piece) && typeof piece.text === "string")
|
|
1150
|
+
parts.push(piece.text);
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
return parts.length === 0 ? void 0 : parts.join("\n\n");
|
|
1155
|
+
}
|
|
1156
|
+
function parseHeliconeRequests(rows, fallbackDate) {
|
|
1157
|
+
const accepted = [];
|
|
1158
|
+
const cacheHits = [];
|
|
1159
|
+
const issues = [];
|
|
1160
|
+
rows.forEach((raw, index) => {
|
|
1161
|
+
if (!isPlainObject(raw)) {
|
|
1162
|
+
issues.push({ line: index + 1, reason: "Row is not a JSON object." });
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
const properties = isPlainObject(raw.request_properties) ? raw.request_properties : isPlainObject(raw.properties) ? raw.properties : {};
|
|
1166
|
+
const model = asString(raw.model) ?? asString(raw.request_model) ?? asString(raw.response_model);
|
|
1167
|
+
if (model === void 0) {
|
|
1168
|
+
issues.push({ line: index + 1, reason: "No model field." });
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
const inputTokens = asNumber(raw.prompt_tokens);
|
|
1172
|
+
const outputTokens = asNumber(raw.completion_tokens);
|
|
1173
|
+
if (inputTokens === void 0 || outputTokens === void 0) {
|
|
1174
|
+
issues.push({ line: index + 1, reason: "Missing prompt_tokens or completion_tokens." });
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
const created = asString(raw.request_created_at) ?? asString(raw.response_created_at) ?? asString(raw.created_at);
|
|
1178
|
+
const at = created !== void 0 && /^\d{4}-\d{2}-\d{2}/.test(created) ? created.slice(0, 10) : fallbackDate;
|
|
1179
|
+
const cacheRead = asNumber(raw.prompt_cache_read_tokens) ?? 0;
|
|
1180
|
+
const cacheWrite = asNumber(raw.prompt_cache_write_tokens) ?? 0;
|
|
1181
|
+
const requestBody = isPlainObject(raw.request_body) ? raw.request_body : void 0;
|
|
1182
|
+
const promptText = requestBody === void 0 ? void 0 : flattenMessages(requestBody.messages) ?? asString(requestBody.prompt);
|
|
1183
|
+
const toolSchemaText = requestBody !== void 0 && Array.isArray(requestBody.tools) && requestBody.tools.length > 0 ? JSON.stringify(requestBody.tools) : void 0;
|
|
1184
|
+
const sessionId = property(properties, "Helicone-Session-Id") ?? asString(raw.session_id) ?? asString(raw.trace_id);
|
|
1185
|
+
const workload = property(properties, "Helicone-Session-Path")?.split("/").filter(Boolean)[0] ?? property(properties, "Helicone-Property-Workload") ?? property(properties, "workload") ?? property(properties, "app") ?? asString(raw.request_path) ?? model;
|
|
1186
|
+
const spend = asNumber(raw.cost) ?? asNumber(raw.costUSD) ?? asNumber(raw.provider_total_cost);
|
|
1187
|
+
const endUser = asString(raw.request_user_id) ?? property(properties, "Helicone-User-Id");
|
|
1188
|
+
const entry2 = {
|
|
1189
|
+
record: {
|
|
1190
|
+
id: asString(raw.request_id) ?? asString(raw.response_id) ?? `helicone-${index}`,
|
|
1191
|
+
model,
|
|
1192
|
+
provider: providerFor(model, asString(raw.provider)),
|
|
1193
|
+
at,
|
|
1194
|
+
inputTokens: inputTokens + cacheRead + cacheWrite,
|
|
1195
|
+
outputTokens,
|
|
1196
|
+
...cacheRead > 0 ? { cachedInputTokens: cacheRead } : {},
|
|
1197
|
+
...cacheWrite > 0 ? { cacheWriteInputTokens: cacheWrite } : {},
|
|
1198
|
+
...promptText === void 0 ? {} : { promptText },
|
|
1199
|
+
...toolSchemaText === void 0 ? {} : { toolSchemaText },
|
|
1200
|
+
workload,
|
|
1201
|
+
...sessionId === void 0 ? {} : { episodeId: sessionId }
|
|
1202
|
+
},
|
|
1203
|
+
...spend === void 0 ? {} : { reportedSpendUsd: spend },
|
|
1204
|
+
...endUser === void 0 ? {} : { endUser }
|
|
1205
|
+
};
|
|
1206
|
+
const servedFromHeliconeCache = raw.cache_hit === true || raw.cached === true || property(properties, "Helicone-Cache")?.toUpperCase() === "HIT";
|
|
1207
|
+
if (servedFromHeliconeCache)
|
|
1208
|
+
cacheHits.push(entry2);
|
|
1209
|
+
else
|
|
1210
|
+
accepted.push(entry2);
|
|
1211
|
+
});
|
|
1212
|
+
return { accepted, cacheHits, issues };
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
// packages/core/dist/src/langfuse.js
|
|
1216
|
+
function asString2(value) {
|
|
1217
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
1218
|
+
}
|
|
1219
|
+
function asNumber2(value) {
|
|
1220
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
1221
|
+
}
|
|
1222
|
+
function isPlainObject2(value) {
|
|
1223
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1224
|
+
}
|
|
1225
|
+
function providerFor2(model) {
|
|
1226
|
+
if (/^(?:claude|opus|sonnet|haiku|fable)/i.test(model))
|
|
1227
|
+
return "anthropic";
|
|
1228
|
+
if (/^(?:gpt|o\d|chatgpt|text-)/i.test(model))
|
|
1229
|
+
return "openai";
|
|
1230
|
+
if (/^gemini/i.test(model))
|
|
1231
|
+
return "google";
|
|
1232
|
+
return "unknown";
|
|
1233
|
+
}
|
|
1234
|
+
function usageOf(raw) {
|
|
1235
|
+
const details = isPlainObject2(raw.usageDetails) ? raw.usageDetails : void 0;
|
|
1236
|
+
const usage = isPlainObject2(raw.usage) ? raw.usage : void 0;
|
|
1237
|
+
const cachedInput = asNumber2(details?.cache_read_input_tokens) ?? asNumber2(details?.input_cached_tokens) ?? asNumber2(details?.cached_tokens) ?? 0;
|
|
1238
|
+
const cacheWriteInput = asNumber2(details?.cache_creation_input_tokens) ?? 0;
|
|
1239
|
+
const input = asNumber2(details?.input) ?? asNumber2(usage?.input) ?? asNumber2(usage?.promptTokens) ?? asNumber2(raw.promptTokens) ?? asNumber2(raw.inputUsage);
|
|
1240
|
+
const output = asNumber2(details?.output) ?? asNumber2(usage?.output) ?? asNumber2(usage?.completionTokens) ?? asNumber2(raw.completionTokens) ?? asNumber2(raw.outputUsage);
|
|
1241
|
+
return {
|
|
1242
|
+
...input === void 0 ? {} : { input },
|
|
1243
|
+
...output === void 0 ? {} : { output },
|
|
1244
|
+
cachedInput,
|
|
1245
|
+
cacheWriteInput
|
|
1246
|
+
};
|
|
1247
|
+
}
|
|
1248
|
+
function flattenInput(value) {
|
|
1249
|
+
if (typeof value === "string")
|
|
1250
|
+
return value.trim() === "" ? void 0 : value;
|
|
1251
|
+
if (Array.isArray(value)) {
|
|
1252
|
+
const parts = [];
|
|
1253
|
+
for (const message of value) {
|
|
1254
|
+
if (!isPlainObject2(message))
|
|
1255
|
+
continue;
|
|
1256
|
+
const content = message.content;
|
|
1257
|
+
if (typeof content === "string" && content.trim() !== "")
|
|
1258
|
+
parts.push(content);
|
|
1259
|
+
else if (Array.isArray(content)) {
|
|
1260
|
+
for (const piece of content) {
|
|
1261
|
+
if (isPlainObject2(piece) && typeof piece.text === "string")
|
|
1262
|
+
parts.push(piece.text);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
return parts.length === 0 ? void 0 : parts.join("\n\n");
|
|
1267
|
+
}
|
|
1268
|
+
if (isPlainObject2(value)) {
|
|
1269
|
+
return flattenInput(value.messages) ?? flattenInput(value.input) ?? asString2(value.prompt);
|
|
1270
|
+
}
|
|
1271
|
+
return void 0;
|
|
1272
|
+
}
|
|
1273
|
+
function parseLangfuseGenerations(rows, fallbackDate) {
|
|
1274
|
+
const accepted = [];
|
|
1275
|
+
const issues = [];
|
|
1276
|
+
rows.forEach((raw, index) => {
|
|
1277
|
+
if (!isPlainObject2(raw)) {
|
|
1278
|
+
issues.push({ line: index + 1, reason: "Row is not a JSON object." });
|
|
1279
|
+
return;
|
|
1280
|
+
}
|
|
1281
|
+
const type = asString2(raw.type);
|
|
1282
|
+
if (type !== void 0 && type.toUpperCase() !== "GENERATION")
|
|
1283
|
+
return;
|
|
1284
|
+
const model = asString2(raw.model) ?? asString2(raw.modelId) ?? asString2(raw.internalModelId);
|
|
1285
|
+
if (model === void 0) {
|
|
1286
|
+
issues.push({ line: index + 1, reason: "Generation has no model field." });
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
const usage = usageOf(raw);
|
|
1290
|
+
if (usage.input === void 0 || usage.output === void 0) {
|
|
1291
|
+
issues.push({ line: index + 1, reason: "Generation has no readable token usage." });
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
const started = asString2(raw.startTime) ?? asString2(raw.timestamp);
|
|
1295
|
+
const at = started !== void 0 && /^\d{4}-\d{2}-\d{2}/.test(started) ? started.slice(0, 10) : fallbackDate;
|
|
1296
|
+
const metadata = isPlainObject2(raw.metadata) ? raw.metadata : {};
|
|
1297
|
+
const tags = Array.isArray(raw.tags) ? raw.tags.filter((tag) => typeof tag === "string") : [];
|
|
1298
|
+
const workload = asString2(raw.traceName) ?? asString2(raw.name) ?? tags[0] ?? asString2(metadata.workload) ?? asString2(raw.environment) ?? model;
|
|
1299
|
+
const spend = asNumber2(raw.calculatedTotalCost) ?? asNumber2(raw.totalCost) ?? asNumber2(raw.totalPrice) ?? (isPlainObject2(raw.costDetails) ? asNumber2(raw.costDetails.total) : void 0);
|
|
1300
|
+
const promptText = flattenInput(raw.input);
|
|
1301
|
+
const episodeId = asString2(raw.sessionId) ?? asString2(raw.traceId);
|
|
1302
|
+
const endUser = asString2(raw.userId) ?? asString2(metadata.userId);
|
|
1303
|
+
accepted.push({
|
|
1304
|
+
record: {
|
|
1305
|
+
id: asString2(raw.id) ?? `langfuse-${index}`,
|
|
1306
|
+
model,
|
|
1307
|
+
provider: providerFor2(model),
|
|
1308
|
+
at,
|
|
1309
|
+
inputTokens: usage.input,
|
|
1310
|
+
outputTokens: usage.output,
|
|
1311
|
+
...usage.cachedInput > 0 ? { cachedInputTokens: usage.cachedInput } : {},
|
|
1312
|
+
...usage.cacheWriteInput > 0 ? { cacheWriteInputTokens: usage.cacheWriteInput } : {},
|
|
1313
|
+
...promptText === void 0 ? {} : { promptText },
|
|
1314
|
+
workload,
|
|
1315
|
+
...episodeId === void 0 ? {} : { episodeId }
|
|
1316
|
+
},
|
|
1317
|
+
...spend === void 0 ? {} : { reportedSpendUsd: spend },
|
|
1318
|
+
...endUser === void 0 ? {} : { endUser }
|
|
1319
|
+
});
|
|
1320
|
+
});
|
|
1321
|
+
return { accepted, cacheHits: [], issues };
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// packages/core/dist/src/litellm.js
|
|
1325
|
+
function asString3(value) {
|
|
1326
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
1327
|
+
}
|
|
1328
|
+
function asNumber3(value) {
|
|
1329
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
1330
|
+
}
|
|
1331
|
+
function isPlainObject3(value) {
|
|
1332
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1333
|
+
}
|
|
1334
|
+
function isCacheHit(value) {
|
|
1335
|
+
return value === true || value === "True" || value === "true";
|
|
1336
|
+
}
|
|
1337
|
+
function providerFor3(model, explicit) {
|
|
1338
|
+
if (explicit)
|
|
1339
|
+
return explicit;
|
|
1340
|
+
if (/^(?:claude|opus|sonnet|haiku|fable)/i.test(model))
|
|
1341
|
+
return "anthropic";
|
|
1342
|
+
if (/^(?:gpt|o\d|chatgpt|text-)/i.test(model))
|
|
1343
|
+
return "openai";
|
|
1344
|
+
if (/^gemini/i.test(model))
|
|
1345
|
+
return "google";
|
|
1346
|
+
return "unknown";
|
|
1347
|
+
}
|
|
1348
|
+
function flattenMessages2(value) {
|
|
1349
|
+
if (!Array.isArray(value) || value.length === 0)
|
|
1350
|
+
return void 0;
|
|
1351
|
+
const parts = [];
|
|
1352
|
+
for (const message of value) {
|
|
1353
|
+
if (!isPlainObject3(message))
|
|
1354
|
+
continue;
|
|
1355
|
+
const content = message.content;
|
|
1356
|
+
if (typeof content === "string" && content.trim() !== "")
|
|
1357
|
+
parts.push(content);
|
|
1358
|
+
else if (Array.isArray(content)) {
|
|
1359
|
+
for (const piece of content) {
|
|
1360
|
+
if (isPlainObject3(piece) && typeof piece.text === "string")
|
|
1361
|
+
parts.push(piece.text);
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
return parts.length === 0 ? void 0 : parts.join("\n\n");
|
|
1366
|
+
}
|
|
1367
|
+
function parseLiteLLMSpendLogs(rows, fallbackDate) {
|
|
1368
|
+
const accepted = [];
|
|
1369
|
+
const cacheHits = [];
|
|
1370
|
+
const issues = [];
|
|
1371
|
+
rows.forEach((raw, index) => {
|
|
1372
|
+
if (!isPlainObject3(raw)) {
|
|
1373
|
+
issues.push({ line: index + 1, reason: "Row is not a JSON object." });
|
|
1374
|
+
return;
|
|
1375
|
+
}
|
|
1376
|
+
const metadata = isPlainObject3(raw.metadata) ? raw.metadata : {};
|
|
1377
|
+
const model = asString3(raw.model) ?? asString3(raw.model_group);
|
|
1378
|
+
if (model === void 0) {
|
|
1379
|
+
issues.push({ line: index + 1, reason: "No model or model_group field." });
|
|
1380
|
+
return;
|
|
1381
|
+
}
|
|
1382
|
+
const inputTokens = asNumber3(raw.prompt_tokens);
|
|
1383
|
+
const outputTokens = asNumber3(raw.completion_tokens);
|
|
1384
|
+
if (inputTokens === void 0 || outputTokens === void 0) {
|
|
1385
|
+
issues.push({ line: index + 1, reason: "Missing prompt_tokens or completion_tokens." });
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
const started = asString3(raw.startTime) ?? asString3(raw.start_time) ?? asString3(raw.endTime);
|
|
1389
|
+
const at = started !== void 0 && /^\d{4}-\d{2}-\d{2}/.test(started) ? started.slice(0, 10) : fallbackDate;
|
|
1390
|
+
const episodeId = asString3(raw.session_id) ?? asString3(raw.litellm_session_id) ?? asString3(metadata.session_id) ?? asString3(metadata.litellm_session_id) ?? asString3(raw.trace_id) ?? asString3(metadata.trace_id);
|
|
1391
|
+
const tags = Array.isArray(raw.request_tags) ? raw.request_tags.filter((tag) => typeof tag === "string") : [];
|
|
1392
|
+
const workload = tags[0] ?? asString3(raw.team_id) ?? asString3(raw.model_group) ?? model;
|
|
1393
|
+
const promptText = flattenMessages2(raw.messages) ?? flattenMessages2(isPlainObject3(raw.proxy_server_request) ? raw.proxy_server_request.messages ?? (isPlainObject3(raw.proxy_server_request.body) ? raw.proxy_server_request.body.messages : void 0) : void 0);
|
|
1394
|
+
const entry2 = {
|
|
1395
|
+
record: {
|
|
1396
|
+
id: asString3(raw.request_id) ?? asString3(raw.id) ?? `litellm-${index}`,
|
|
1397
|
+
model,
|
|
1398
|
+
provider: providerFor3(model, asString3(raw.custom_llm_provider)),
|
|
1399
|
+
at,
|
|
1400
|
+
inputTokens,
|
|
1401
|
+
outputTokens,
|
|
1402
|
+
...promptText === void 0 ? {} : { promptText },
|
|
1403
|
+
workload,
|
|
1404
|
+
...episodeId === void 0 ? {} : { episodeId }
|
|
1405
|
+
},
|
|
1406
|
+
...(asNumber3(raw.spend) ?? asNumber3(raw.response_cost)) === void 0 ? {} : { reportedSpendUsd: asNumber3(raw.spend) ?? asNumber3(raw.response_cost) },
|
|
1407
|
+
...(asString3(raw.end_user) ?? asString3(metadata.user_api_key_end_user_id)) === void 0 ? {} : { endUser: asString3(raw.end_user) ?? asString3(metadata.user_api_key_end_user_id) },
|
|
1408
|
+
...asString3(raw.team_id) === void 0 ? {} : { teamId: asString3(raw.team_id) }
|
|
1409
|
+
};
|
|
1410
|
+
if (isCacheHit(raw.cache_hit) || isCacheHit(metadata.cache_hit))
|
|
1411
|
+
cacheHits.push(entry2);
|
|
1412
|
+
else
|
|
1413
|
+
accepted.push(entry2);
|
|
1414
|
+
});
|
|
1415
|
+
return { accepted, cacheHits, issues };
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
// packages/core/dist/src/traffic.js
|
|
1419
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
1420
|
+
function groupKey(record) {
|
|
1421
|
+
return `${record.workload}\0${record.provider}\0${record.model}`;
|
|
1422
|
+
}
|
|
1423
|
+
function measureTraffic(records, catalogue, options = {}) {
|
|
1424
|
+
if (records.length === 0)
|
|
1425
|
+
throw new Error("Measuring traffic needs at least one record.");
|
|
1426
|
+
const countTokens = options.countTokens ?? approximateTokenCount;
|
|
1427
|
+
const maximumPromptSample = options.maximumPromptSample ?? 400;
|
|
1428
|
+
const notes = [];
|
|
1429
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1430
|
+
for (const record of records) {
|
|
1431
|
+
const key = groupKey(record);
|
|
1432
|
+
const list = groups.get(key) ?? [];
|
|
1433
|
+
list.push(record);
|
|
1434
|
+
groups.set(key, list);
|
|
1435
|
+
}
|
|
1436
|
+
const workloads = [];
|
|
1437
|
+
const unpricedModels = /* @__PURE__ */ new Set();
|
|
1438
|
+
let totalBaselineCostUsd = 0;
|
|
1439
|
+
let totalInputTokens = 0;
|
|
1440
|
+
let totalAlreadyCached = 0;
|
|
1441
|
+
let anyPromptText = false;
|
|
1442
|
+
for (const group of groups.values()) {
|
|
1443
|
+
const first = group[0];
|
|
1444
|
+
const inputTokens = group.reduce((sum, record) => sum + record.inputTokens, 0);
|
|
1445
|
+
const outputTokens = group.reduce((sum, record) => sum + record.outputTokens, 0);
|
|
1446
|
+
const alreadyCachedInputTokens = group.reduce((sum, record) => sum + (record.cachedInputTokens ?? 0), 0);
|
|
1447
|
+
totalInputTokens += inputTokens;
|
|
1448
|
+
totalAlreadyCached += alreadyCachedInputTokens;
|
|
1449
|
+
const resolution = resolveModelId(catalogue, first.provider, first.model);
|
|
1450
|
+
const resolvedModel = resolution.status === "resolved" ? resolution.model : void 0;
|
|
1451
|
+
let baselineCostUsd = 0;
|
|
1452
|
+
let unpricedReason;
|
|
1453
|
+
if (resolvedModel === void 0) {
|
|
1454
|
+
baselineCostUsd = void 0;
|
|
1455
|
+
unpricedReason = resolution.status === "unresolved" ? resolution.reason : "Model did not resolve.";
|
|
1456
|
+
unpricedModels.add(`${first.provider}/${first.model}`);
|
|
1457
|
+
}
|
|
1458
|
+
if (resolvedModel !== void 0) {
|
|
1459
|
+
let running = 0;
|
|
1460
|
+
for (const record of group) {
|
|
1461
|
+
const priced = priceUsage(catalogue, { provider: record.provider, model: resolvedModel, at: record.at }, {
|
|
1462
|
+
inputTokens: record.inputTokens,
|
|
1463
|
+
outputTokens: record.outputTokens,
|
|
1464
|
+
cachedInputTokens: record.cachedInputTokens,
|
|
1465
|
+
cacheWriteInputTokens: record.cacheWriteInputTokens
|
|
1466
|
+
});
|
|
1467
|
+
if (priced.status === "unpriced") {
|
|
1468
|
+
unpricedReason = priced.reason;
|
|
1469
|
+
baselineCostUsd = void 0;
|
|
1470
|
+
unpricedModels.add(`${record.provider}/${record.model}`);
|
|
1471
|
+
break;
|
|
1472
|
+
}
|
|
1473
|
+
running += priced.costUsd;
|
|
1474
|
+
}
|
|
1475
|
+
if (baselineCostUsd !== void 0)
|
|
1476
|
+
baselineCostUsd = running;
|
|
1477
|
+
}
|
|
1478
|
+
if (baselineCostUsd !== void 0)
|
|
1479
|
+
totalBaselineCostUsd += baselineCostUsd;
|
|
1480
|
+
const withPrompts = group.filter((record) => record.promptText !== void 0);
|
|
1481
|
+
if (withPrompts.length > 0)
|
|
1482
|
+
anyPromptText = true;
|
|
1483
|
+
const toolSchemaTokens = group.filter((record) => record.toolSchemaText !== void 0).map((record) => countTokens(record.toolSchemaText));
|
|
1484
|
+
const meanToolSchemaTokens = toolSchemaTokens.length === 0 ? 0 : toolSchemaTokens.reduce((sum, value) => sum + value, 0) / toolSchemaTokens.length;
|
|
1485
|
+
let toolAudit;
|
|
1486
|
+
const withTools = group.find((record) => record.toolSchemaText !== void 0);
|
|
1487
|
+
if (withTools !== void 0) {
|
|
1488
|
+
try {
|
|
1489
|
+
const parsed = JSON.parse(withTools.toolSchemaText);
|
|
1490
|
+
if (Array.isArray(parsed))
|
|
1491
|
+
toolAudit = auditToolCatalogue(parsed, { countTokens });
|
|
1492
|
+
} catch {
|
|
1493
|
+
notes.push(`${first.workload}: tool definitions could not be parsed, so no catalogue audit was run.`);
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
let caching;
|
|
1497
|
+
if (withPrompts.length >= 2) {
|
|
1498
|
+
const lookup = resolvedModel === void 0 ? { status: "unpriced" } : lookupPrice(catalogue, { provider: first.provider, model: resolvedModel, at: first.at });
|
|
1499
|
+
const sample = withPrompts.slice(0, maximumPromptSample).map((record) => record.promptText);
|
|
1500
|
+
if (sample.length < withPrompts.length) {
|
|
1501
|
+
notes.push(`${first.workload}: prefix analysis used the first ${sample.length} of ${withPrompts.length} prompts.`);
|
|
1502
|
+
}
|
|
1503
|
+
caching = analysePromptCaching(sample, {
|
|
1504
|
+
countTokens,
|
|
1505
|
+
...lookup.status === "priced" ? { price: lookup.price } : {}
|
|
1506
|
+
});
|
|
1507
|
+
}
|
|
1508
|
+
const duplicateContextTokens = withPrompts.reduce((sum, record) => {
|
|
1509
|
+
const reduced = dedupeRepeatedBlocks(24, countTokens).reduce(record.promptText);
|
|
1510
|
+
return sum + (countTokens(record.promptText) - countTokens(reduced.text));
|
|
1511
|
+
}, 0);
|
|
1512
|
+
const byEpisode = /* @__PURE__ */ new Map();
|
|
1513
|
+
for (const record of group) {
|
|
1514
|
+
const key = record.episodeId ?? `__single__${record.id}`;
|
|
1515
|
+
const list = byEpisode.get(key) ?? [];
|
|
1516
|
+
list.push(record);
|
|
1517
|
+
byEpisode.set(key, list);
|
|
1518
|
+
}
|
|
1519
|
+
const episodeGroups = [...byEpisode.values()];
|
|
1520
|
+
const identified = group.some((record) => record.episodeId !== void 0);
|
|
1521
|
+
const roundTrips = episodeGroups.map((episode) => episode.length);
|
|
1522
|
+
const episodes = {
|
|
1523
|
+
episodes: episodeGroups.length,
|
|
1524
|
+
meanRoundTrips: roundTrips.reduce((sum, value) => sum + value, 0) / episodeGroups.length,
|
|
1525
|
+
maxRoundTrips: Math.max(...roundTrips),
|
|
1526
|
+
meanInputTokens: inputTokens / episodeGroups.length,
|
|
1527
|
+
meanOutputTokens: outputTokens / episodeGroups.length,
|
|
1528
|
+
...baselineCostUsd === void 0 ? {} : { meanCostUsd: baselineCostUsd / episodeGroups.length },
|
|
1529
|
+
identified
|
|
1530
|
+
};
|
|
1531
|
+
workloads.push({
|
|
1532
|
+
workload: first.workload,
|
|
1533
|
+
model: first.model,
|
|
1534
|
+
...resolvedModel === void 0 ? {} : { resolvedModel },
|
|
1535
|
+
provider: first.provider,
|
|
1536
|
+
requests: group.length,
|
|
1537
|
+
inputTokens,
|
|
1538
|
+
outputTokens,
|
|
1539
|
+
alreadyCachedInputTokens,
|
|
1540
|
+
meanToolSchemaTokens,
|
|
1541
|
+
...toolAudit ? { toolAudit } : {},
|
|
1542
|
+
...caching ? { caching } : {},
|
|
1543
|
+
...group.length >= 2 ? { outputCap: recommendOutputCap(group.map((record) => record.outputTokens), { quantile: 0.99 }) } : {},
|
|
1544
|
+
duplicateContextTokens,
|
|
1545
|
+
episodes,
|
|
1546
|
+
...baselineCostUsd === void 0 ? {} : { baselineCostUsd },
|
|
1547
|
+
...unpricedReason === void 0 ? {} : { unpricedReason }
|
|
1548
|
+
});
|
|
1549
|
+
}
|
|
1550
|
+
const episodesIdentified = records.some((record) => record.episodeId !== void 0);
|
|
1551
|
+
if (!episodesIdentified) {
|
|
1552
|
+
notes.push("No task identifier was found, so every request counts as its own episode and round trips read as 1.0. Per-task figures are the unit that matters for agentic traffic: add a conversation, session or trace id to the log before trusting any technique that changes how many calls a task takes.");
|
|
1553
|
+
}
|
|
1554
|
+
if (!anyPromptText) {
|
|
1555
|
+
notes.push("No prompt text was present, so this is a usage-only measurement: baseline spend and output caps only. Prefix-cache headroom, the largest risk-free lever, cannot be measured without prompts.");
|
|
1556
|
+
}
|
|
1557
|
+
if (unpricedModels.size > 0) {
|
|
1558
|
+
notes.push(`${unpricedModels.size} model identifier(s) did not resolve to a catalogued rate. Their spend is excluded from the baseline rather than counted as zero; add a dated entry or an alias to include them.`);
|
|
1559
|
+
}
|
|
1560
|
+
workloads.sort((left, right) => (right.baselineCostUsd ?? 0) - (left.baselineCostUsd ?? 0));
|
|
1561
|
+
return {
|
|
1562
|
+
requests: records.length,
|
|
1563
|
+
workloads,
|
|
1564
|
+
totalBaselineCostUsd,
|
|
1565
|
+
unpricedModels: [...unpricedModels].sort(),
|
|
1566
|
+
alreadyCachedShare: totalInputTokens === 0 ? 0 : totalAlreadyCached / totalInputTokens,
|
|
1567
|
+
promptTextAvailable: anyPromptText,
|
|
1568
|
+
episodesIdentified,
|
|
1569
|
+
notes
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
function trafficFingerprint(records) {
|
|
1573
|
+
const hash = createHash4("sha256");
|
|
1574
|
+
for (const record of [...records].sort((left, right) => left.id.localeCompare(right.id))) {
|
|
1575
|
+
hash.update(`${record.id}|${record.model}|${record.inputTokens}|${record.outputTokens}|${record.at}`);
|
|
1576
|
+
}
|
|
1577
|
+
return hash.digest("hex").slice(0, 16);
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
// packages/core/dist/src/triage.js
|
|
1581
|
+
function classifySegment(signals) {
|
|
1582
|
+
const evidence = [];
|
|
1583
|
+
if (signals.alreadyCachedShare > 0.5 || signals.gatewayCacheHitShare > 0.3) {
|
|
1584
|
+
if (signals.alreadyCachedShare > 0.5) {
|
|
1585
|
+
evidence.push(`${(signals.alreadyCachedShare * 100).toFixed(0)}% of input already arrives provider-cached.`);
|
|
1586
|
+
}
|
|
1587
|
+
if (signals.gatewayCacheHitShare > 0.3) {
|
|
1588
|
+
evidence.push(`${(signals.gatewayCacheHitShare * 100).toFixed(0)}% of requests are answered by the gateway's own cache.`);
|
|
1589
|
+
}
|
|
1590
|
+
return { segment: "already-optimised", evidence };
|
|
1591
|
+
}
|
|
1592
|
+
if (signals.distinctEndUsers >= 20 && signals.requestsPerEndUser <= 5) {
|
|
1593
|
+
evidence.push(`${signals.distinctEndUsers} distinct end users at ${signals.requestsPerEndUser.toFixed(1)} requests each: traffic fans out across many downstream customers.`);
|
|
1594
|
+
return { segment: "consumer-app", evidence };
|
|
1595
|
+
}
|
|
1596
|
+
if (signals.episodesIdentified && signals.meanRoundTrips >= 3) {
|
|
1597
|
+
evidence.push(`Sessions identified at ${signals.meanRoundTrips.toFixed(1)} calls per task` + (signals.toolSchemaShare > 0.1 ? `, with tool schemas ${(signals.toolSchemaShare * 100).toFixed(0)}% of input` : "") + ": multi-turn agent traffic.");
|
|
1598
|
+
return { segment: "in-house-agent", evidence };
|
|
1599
|
+
}
|
|
1600
|
+
evidence.push("Server-side traffic without a wide end-user fan-out or long sessions: application-layer calls built from a template.");
|
|
1601
|
+
return { segment: "application-layer", evidence };
|
|
1602
|
+
}
|
|
1603
|
+
function quoteForSegment(classification, signals) {
|
|
1604
|
+
const { segment } = classification;
|
|
1605
|
+
if (segment === "already-optimised") {
|
|
1606
|
+
return {
|
|
1607
|
+
segment,
|
|
1608
|
+
bandPercent: [0, 5],
|
|
1609
|
+
topLever: "none worth charging for",
|
|
1610
|
+
confidence: "do-not-charge",
|
|
1611
|
+
doNotCharge: true,
|
|
1612
|
+
reasoning: [
|
|
1613
|
+
...classification.evidence,
|
|
1614
|
+
"The support-assistant fixture with 77% of input already cached measured 2.4% addressable.",
|
|
1615
|
+
"A percent-of-savings fee on savings the customer already has is a refund waiting to happen."
|
|
1616
|
+
]
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
1619
|
+
if (segment === "in-house-agent") {
|
|
1620
|
+
return {
|
|
1621
|
+
segment,
|
|
1622
|
+
bandPercent: [25, 35],
|
|
1623
|
+
topLever: "output budgeting (replay-gated)",
|
|
1624
|
+
confidence: "measured-analogue",
|
|
1625
|
+
firmUpBy: "a replay-gate run on ~50 recorded tasks to pick the cap this workload supports",
|
|
1626
|
+
reasoning: [
|
|
1627
|
+
...classification.evidence,
|
|
1628
|
+
"A/B on 40 five-turn tasks: output cap saved 32.2% at 89% answer completeness; schema work added 1-3%.",
|
|
1629
|
+
"Caching is NOT quoted: appended history already hit the provider cache 92% of the time with no MESH, so that saving is the provider's gift, not ours to bill."
|
|
1630
|
+
]
|
|
1631
|
+
};
|
|
1632
|
+
}
|
|
1633
|
+
if (segment === "consumer-app") {
|
|
1634
|
+
const reasoning = [
|
|
1635
|
+
...classification.evidence,
|
|
1636
|
+
"Per-user traffic is sparse, and provider caches expire from last use (~5 minutes), so per-user history caching decays between visits: at 600s spacing the simulated hit rate fell from 92% to 78% and prefix repair's benefit vanished.",
|
|
1637
|
+
"The durable lever is the shared system template across ALL users, plus output budgeting."
|
|
1638
|
+
];
|
|
1639
|
+
if (!signals.promptTextAvailable) {
|
|
1640
|
+
return {
|
|
1641
|
+
segment,
|
|
1642
|
+
bandPercent: [5, 40],
|
|
1643
|
+
topLever: "shared-template cache repair + output budgeting",
|
|
1644
|
+
confidence: "needs-prompt-text",
|
|
1645
|
+
firmUpBy: "a sample of ~50 prompts, to measure the shared template and what breaks its prefix",
|
|
1646
|
+
reasoning: [
|
|
1647
|
+
...reasoning,
|
|
1648
|
+
"Without prompt text the shared-template share is unknown, so the band is wide by design."
|
|
1649
|
+
]
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1652
|
+
return {
|
|
1653
|
+
segment,
|
|
1654
|
+
bandPercent: signals.prefixBreakersFound ? [20, 45] : [10, 30],
|
|
1655
|
+
topLever: signals.prefixBreakersFound ? "cache-prefix repair on the shared template" : "output budgeting (replay-gated)",
|
|
1656
|
+
confidence: "measured-analogue",
|
|
1657
|
+
firmUpBy: "a replay-gate run to confirm the output cap",
|
|
1658
|
+
reasoning
|
|
1659
|
+
};
|
|
1660
|
+
}
|
|
1661
|
+
if (!signals.promptTextAvailable) {
|
|
1662
|
+
return {
|
|
1663
|
+
segment,
|
|
1664
|
+
bandPercent: [10, 55],
|
|
1665
|
+
topLever: "cache-prefix repair (unconfirmed) + output budgeting",
|
|
1666
|
+
confidence: "needs-prompt-text",
|
|
1667
|
+
firmUpBy: "a sample of ~50 prompts, to confirm the template and find what breaks its prefix",
|
|
1668
|
+
reasoning: [
|
|
1669
|
+
...classification.evidence,
|
|
1670
|
+
"One-shot templated traffic measured 39.7% from prefix repair alone when a per-request value was blocking the cache, and 0% when nothing was. Which of those this stream is cannot be known from token counts, hence the wide band."
|
|
1671
|
+
]
|
|
1672
|
+
};
|
|
1673
|
+
}
|
|
1674
|
+
if (signals.prefixBreakersFound) {
|
|
1675
|
+
return {
|
|
1676
|
+
segment,
|
|
1677
|
+
bandPercent: [35, 55],
|
|
1678
|
+
topLever: "cache-prefix repair",
|
|
1679
|
+
confidence: "measured-analogue",
|
|
1680
|
+
firmUpBy: "enabling repair on a canary slice and reading the provider's cached-token counts",
|
|
1681
|
+
reasoning: [
|
|
1682
|
+
...classification.evidence,
|
|
1683
|
+
"Per-request values were found inside the shared prefix. The one-shot simulation measured the hit rate going from 0% to 78% and cost falling 39.7% when such values were moved out of the prefix."
|
|
1684
|
+
]
|
|
1685
|
+
};
|
|
1686
|
+
}
|
|
1687
|
+
return {
|
|
1688
|
+
segment,
|
|
1689
|
+
bandPercent: [5, 25],
|
|
1690
|
+
topLever: "output budgeting (replay-gated)",
|
|
1691
|
+
confidence: "measured-analogue",
|
|
1692
|
+
firmUpBy: "a replay-gate run on ~50 recorded requests",
|
|
1693
|
+
reasoning: [
|
|
1694
|
+
...classification.evidence,
|
|
1695
|
+
"The template's prefix is already clean, so caching needs no repair; the remaining levers are output budgeting and lossless schema compaction."
|
|
1696
|
+
]
|
|
1697
|
+
};
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
// packages/core/dist/src/quote-report.js
|
|
1701
|
+
function parseGatewayExport(rows, format, fallbackDate) {
|
|
1702
|
+
switch (format) {
|
|
1703
|
+
case "litellm":
|
|
1704
|
+
return parseLiteLLMSpendLogs(rows, fallbackDate);
|
|
1705
|
+
case "helicone":
|
|
1706
|
+
return parseHeliconeRequests(rows, fallbackDate);
|
|
1707
|
+
case "langfuse":
|
|
1708
|
+
return parseLangfuseGenerations(rows, fallbackDate);
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
function buildQuoteReport(rows, options) {
|
|
1712
|
+
const parsed = parseGatewayExport(rows, options.format, options.at);
|
|
1713
|
+
const reconciliation = reconcileGatewaySpend(parsed, options.catalogue);
|
|
1714
|
+
const streams = groupByStream(parsed.accepted);
|
|
1715
|
+
const cacheHitsByStream = groupByStream(parsed.cacheHits);
|
|
1716
|
+
const reports = [];
|
|
1717
|
+
for (const [stream, entries] of streams) {
|
|
1718
|
+
const measurement = measureTraffic(entries.map((entry2) => entry2.record), options.catalogue);
|
|
1719
|
+
const signals = signalsFromGateway(entries, cacheHitsByStream.get(stream) ?? [], measurement);
|
|
1720
|
+
const classification = classifySegment(signals);
|
|
1721
|
+
reports.push({
|
|
1722
|
+
stream,
|
|
1723
|
+
requests: entries.length,
|
|
1724
|
+
baselineUsd: measurement.totalBaselineCostUsd,
|
|
1725
|
+
quote: quoteForSegment(classification, signals)
|
|
1726
|
+
});
|
|
1727
|
+
}
|
|
1728
|
+
reports.sort((left, right) => right.baselineUsd - left.baselineUsd || (left.stream < right.stream ? -1 : left.stream > right.stream ? 1 : 0));
|
|
1729
|
+
return {
|
|
1730
|
+
schemaVersion: 1,
|
|
1731
|
+
format: options.format,
|
|
1732
|
+
at: options.at,
|
|
1733
|
+
catalogueVersion: catalogueVersion(options.catalogue),
|
|
1734
|
+
trafficFingerprint: trafficFingerprint(parsed.accepted.map((entry2) => entry2.record)),
|
|
1735
|
+
rows: {
|
|
1736
|
+
accepted: parsed.accepted.length,
|
|
1737
|
+
gatewayCacheHits: parsed.cacheHits.length,
|
|
1738
|
+
unreadable: parsed.issues.length
|
|
1739
|
+
},
|
|
1740
|
+
streams: reports,
|
|
1741
|
+
reconciliation,
|
|
1742
|
+
doNotPayUs: reports.length === 0 || reports.every((report) => report.quote.doNotCharge === true)
|
|
1743
|
+
};
|
|
1744
|
+
}
|
|
1745
|
+
function renderQuoteReport(report) {
|
|
1746
|
+
const lines = [];
|
|
1747
|
+
lines.push(`=== MESH quote from your ${report.format} export ===`);
|
|
1748
|
+
lines.push(`${report.rows.accepted} provider calls, ${report.rows.gatewayCacheHits} gateway cache hits, ${report.rows.unreadable} unreadable rows (reported, never dropped silently)`);
|
|
1749
|
+
lines.push(`rates: catalogue ${report.catalogueVersion} on ${report.at}; traffic fingerprint ${report.trafficFingerprint}`);
|
|
1750
|
+
lines.push("");
|
|
1751
|
+
for (const stream of report.streams) {
|
|
1752
|
+
const quote = stream.quote;
|
|
1753
|
+
lines.push(`${stream.stream} - ${quote.segment}`);
|
|
1754
|
+
lines.push(` requests ${stream.requests}, baseline $${stream.baselineUsd.toFixed(2)}`);
|
|
1755
|
+
lines.push(` quote: ${quote.bandPercent[0]}-${quote.bandPercent[1]}% via ${quote.topLever} [${quote.confidence}]${quote.doNotCharge ? " - DO NOT CHARGE" : ""}`);
|
|
1756
|
+
if (quote.firmUpBy)
|
|
1757
|
+
lines.push(` firm up by: ${quote.firmUpBy}`);
|
|
1758
|
+
for (const reason of quote.reasoning)
|
|
1759
|
+
lines.push(` - ${reason}`);
|
|
1760
|
+
lines.push("");
|
|
1761
|
+
}
|
|
1762
|
+
const rec = report.reconciliation;
|
|
1763
|
+
lines.push("=== spend reconciliation against published rates ===");
|
|
1764
|
+
lines.push(`compared ${rec.comparedRows} rows: your tool reports $${rec.reportedSpendUsd.toFixed(2)}, catalogue says $${rec.catalogueCostUsd.toFixed(2)} (drift ${rec.driftPercent.toFixed(1)}%)`);
|
|
1765
|
+
lines.push(`${rec.flagged.length} rows flagged beyond tolerance; ${rec.unpricedRows} unpriced rows excluded, never zeroed`);
|
|
1766
|
+
lines.push(`your gateway's cache already saves you $${rec.cacheHitSavedUsd.toFixed(2)} across ${rec.cacheHitRows} hits - credited to you, excluded from every quote above`);
|
|
1767
|
+
if (report.doNotPayUs) {
|
|
1768
|
+
lines.push("");
|
|
1769
|
+
lines.push("VERDICT: DO NOT PAY US. Nothing in this traffic is worth charging for; a percent-of-savings fee on savings you already have would be a refund waiting to happen.");
|
|
1770
|
+
}
|
|
1771
|
+
return lines.join("\n");
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
// packages/core/dist/src/model-prices.js
|
|
1775
|
+
var ANTHROPIC = {
|
|
1776
|
+
source: "https://claude.com/pricing",
|
|
1777
|
+
capturedOn: "2026-10-08",
|
|
1778
|
+
note: "Published display names; cache read/write rates as listed. 5m cache writes bill 1.25x base input, 1h cache writes 2x base input, per the published prompt-caching pricing."
|
|
1779
|
+
};
|
|
1780
|
+
var OPENAI = {
|
|
1781
|
+
source: "https://developers.openai.com/api/docs/pricing",
|
|
1782
|
+
capturedOn: "2026-10-08",
|
|
1783
|
+
note: "Standard tier unless the entry says otherwise."
|
|
1784
|
+
};
|
|
1785
|
+
var ANTHROPIC_BATCH = {
|
|
1786
|
+
source: "https://claude.com/pricing",
|
|
1787
|
+
capturedOn: "2026-10-27",
|
|
1788
|
+
note: `Batch processing = 50% of standard, per the page's definitive "Save 50% with batch processing". Derived by halving the dated standard rates; cache-write premiums under batch are not published and are omitted (priced unpriced, never guessed).`
|
|
1789
|
+
};
|
|
1790
|
+
var DEEPSEEK = {
|
|
1791
|
+
source: "https://api-docs.deepseek.com/quick_start/pricing",
|
|
1792
|
+
capturedOn: "2026-11-03",
|
|
1793
|
+
note: "Off-peak list rates (peak is 2x in the 01-10 UTC weekday windows); catalogued at off-peak so a saving is never over-valued. Per 1M tokens."
|
|
1794
|
+
};
|
|
1795
|
+
var OBSERVED_FROM = "2026-10-08";
|
|
1796
|
+
function entry(provider, model, provenance, bands, mode = "standard", effectiveFrom = OBSERVED_FROM) {
|
|
1797
|
+
return {
|
|
1798
|
+
provider,
|
|
1799
|
+
model,
|
|
1800
|
+
mode,
|
|
1801
|
+
effectiveFrom,
|
|
1802
|
+
provenance,
|
|
1803
|
+
tiers: bands.map((band) => ({
|
|
1804
|
+
...band.maxInputTokens === void 0 ? {} : { maxInputTokens: band.maxInputTokens },
|
|
1805
|
+
inputPerMillion: band.input,
|
|
1806
|
+
outputPerMillion: band.output,
|
|
1807
|
+
...band.cachedInput === void 0 ? {} : { cachedInputPerMillion: band.cachedInput },
|
|
1808
|
+
...band.cacheWriteInput === void 0 ? {} : { cacheWriteInputPerMillion: band.cacheWriteInput },
|
|
1809
|
+
...band.cacheWrite1hInput === void 0 ? {} : { cacheWrite1hInputPerMillion: band.cacheWrite1hInput }
|
|
1810
|
+
}))
|
|
1811
|
+
};
|
|
1812
|
+
}
|
|
1813
|
+
var anthropic = [
|
|
1814
|
+
entry("anthropic", "Fable 5.1", ANTHROPIC, [
|
|
1815
|
+
{ input: 10, output: 50, cachedInput: 0.25, cacheWriteInput: 12.5, cacheWrite1hInput: 20 }
|
|
1816
|
+
]),
|
|
1817
|
+
entry("anthropic", "Opus 5.5", ANTHROPIC, [
|
|
1818
|
+
{ input: 4, output: 20, cachedInput: 0.2, cacheWriteInput: 5, cacheWrite1hInput: 8 }
|
|
1819
|
+
]),
|
|
1820
|
+
entry("anthropic", "Sonnet 5.5", ANTHROPIC, [
|
|
1821
|
+
{ input: 2, output: 10, cachedInput: 0.1, cacheWriteInput: 2.5, cacheWrite1hInput: 4 }
|
|
1822
|
+
]),
|
|
1823
|
+
// Haiku 5.5 is tiered by prompt size: the higher band applies above 100K input tokens.
|
|
1824
|
+
entry("anthropic", "Haiku 5.5", ANTHROPIC, [
|
|
1825
|
+
{ maxInputTokens: 1e5, input: 0.1, output: 0.5, cachedInput: 0.01, cacheWriteInput: 0.125, cacheWrite1hInput: 0.2 },
|
|
1826
|
+
{ input: 0.5, output: 2.5, cachedInput: 0.05, cacheWriteInput: 0.625, cacheWrite1hInput: 1 }
|
|
1827
|
+
]),
|
|
1828
|
+
// Batch mode: 50% of standard, per the definitive "Save 50% with batch processing"
|
|
1829
|
+
// (ANTHROPIC_BATCH provenance). Each rate is exactly half its standard entry above.
|
|
1830
|
+
entry("anthropic", "Fable 5.1", ANTHROPIC_BATCH, [
|
|
1831
|
+
{ input: 5, output: 25, cachedInput: 0.125 }
|
|
1832
|
+
], "batch", "2026-10-27"),
|
|
1833
|
+
entry("anthropic", "Opus 5.5", ANTHROPIC_BATCH, [
|
|
1834
|
+
{ input: 2, output: 10, cachedInput: 0.1 }
|
|
1835
|
+
], "batch", "2026-10-27"),
|
|
1836
|
+
entry("anthropic", "Sonnet 5.5", ANTHROPIC_BATCH, [
|
|
1837
|
+
{ input: 1, output: 5, cachedInput: 0.05 }
|
|
1838
|
+
], "batch", "2026-10-27"),
|
|
1839
|
+
entry("anthropic", "Haiku 5.5", ANTHROPIC_BATCH, [
|
|
1840
|
+
{ maxInputTokens: 1e5, input: 0.05, output: 0.25, cachedInput: 5e-3 },
|
|
1841
|
+
{ input: 0.25, output: 1.25, cachedInput: 0.025 }
|
|
1842
|
+
], "batch", "2026-10-27")
|
|
1843
|
+
];
|
|
1844
|
+
var openai = [
|
|
1845
|
+
entry("openai", "gpt-6-astra", OPENAI, [{ input: 10, output: 50, cachedInput: 1 }]),
|
|
1846
|
+
entry("openai", "gpt-6.1-sol", OPENAI, [{ input: 2, output: 10, cachedInput: 0.1 }]),
|
|
1847
|
+
entry("openai", "gpt-6-luna", OPENAI, [{ input: 0.1, output: 0.5, cachedInput: 0.01 }]),
|
|
1848
|
+
entry("openai", "gpt-5.5", OPENAI, [{ input: 5, output: 30, cachedInput: 0.5 }]),
|
|
1849
|
+
entry("openai", "gpt-5.4", OPENAI, [{ input: 2.5, output: 15, cachedInput: 0.25 }]),
|
|
1850
|
+
entry("openai", "gpt-5.2", OPENAI, [{ input: 1.75, output: 14, cachedInput: 0.175 }]),
|
|
1851
|
+
entry("openai", "gpt-5", OPENAI, [{ input: 1.25, output: 10, cachedInput: 0.125 }]),
|
|
1852
|
+
entry("openai", "gpt-4o", OPENAI, [{ input: 2.5, output: 10, cachedInput: 1.25 }]),
|
|
1853
|
+
entry("openai", "gpt-4o-mini", OPENAI, [{ input: 0.15, output: 0.6, cachedInput: 0.075 }]),
|
|
1854
|
+
entry("openai", "o1", OPENAI, [{ input: 15, output: 60, cachedInput: 7.5 }]),
|
|
1855
|
+
entry("openai", "o3-mini", OPENAI, [{ input: 1.1, output: 4.4, cachedInput: 0.55 }]),
|
|
1856
|
+
// Only the batch rate published explicitly is catalogued. The page describes batch
|
|
1857
|
+
// and flex as "typically" half of standard; that is not a rate, so it is not seeded.
|
|
1858
|
+
entry("openai", "gpt-6-astra", OPENAI, [{ input: 5, output: 25, cachedInput: 0.5 }], "batch"),
|
|
1859
|
+
// Flex processing is billed at Batch API rates = 50% of standard, on a
|
|
1860
|
+
// synchronous API, per OpenAI's flex-processing guide. Cached-input discount
|
|
1861
|
+
// stacks on top. Catalogued so the ledger can price flex-tier events rather
|
|
1862
|
+
// than leaving async savings unpriced.
|
|
1863
|
+
entry("openai", "gpt-5", OPENAI, [{ input: 0.625, output: 5, cachedInput: 0.0625 }], "flex")
|
|
1864
|
+
];
|
|
1865
|
+
var deepseek = [
|
|
1866
|
+
entry("deepseek", "deepseek-flash", DEEPSEEK, [{ input: 0.15, output: 0.6, cachedInput: 3e-3 }], "standard", "2026-11-03"),
|
|
1867
|
+
entry("deepseek", "deepseek-v4-pro", DEEPSEEK, [{ input: 0.66, output: 1.98, cachedInput: 0.022 }], "standard", "2026-11-03")
|
|
1868
|
+
];
|
|
1869
|
+
var aliases = [
|
|
1870
|
+
// Anthropic ids embed the published name, so normalisation covers dated
|
|
1871
|
+
// builds such as claude-sonnet-5-5-20260514. These cover the pointers.
|
|
1872
|
+
{ provider: "anthropic", id: "claude-sonnet-5-5-latest", model: "Sonnet 5.5" },
|
|
1873
|
+
{ provider: "anthropic", id: "claude-opus-5-5-latest", model: "Opus 5.5" },
|
|
1874
|
+
{ provider: "anthropic", id: "claude-haiku-5-5-latest", model: "Haiku 5.5" },
|
|
1875
|
+
{ provider: "anthropic", id: "claude-fable-5-1-latest", model: "Fable 5.1" },
|
|
1876
|
+
// OpenAI ids match their published names directly; these are the pointers
|
|
1877
|
+
// and the dated snapshots that do not normalise onto a catalogued key.
|
|
1878
|
+
{ provider: "openai", id: "chatgpt-4o-latest", model: "gpt-4o" },
|
|
1879
|
+
{ provider: "openai", id: "gpt-4o-2024-*", model: "gpt-4o" },
|
|
1880
|
+
{ provider: "openai", id: "gpt-4o-mini-2024-*", model: "gpt-4o-mini" },
|
|
1881
|
+
{ provider: "openai", id: "o3-mini-2025-*", model: "o3-mini" },
|
|
1882
|
+
// DeepSeek: the current ids match their keys (normalisation covers them); only
|
|
1883
|
+
// the legacy pointer needs an explicit map. deepseek-v4-flash bills at Flash,
|
|
1884
|
+
// per DeepSeek's pricing page.
|
|
1885
|
+
{ provider: "deepseek", id: "deepseek-v4-flash", model: "deepseek-flash" }
|
|
1886
|
+
];
|
|
1887
|
+
var publishedModelPrices = validatePriceCatalogue({
|
|
1888
|
+
entries: [...anthropic, ...openai, ...deepseek],
|
|
1889
|
+
negotiated: [],
|
|
1890
|
+
aliases
|
|
1891
|
+
});
|
|
1892
|
+
function anthropicCacheableMinimumTokens(modelId) {
|
|
1893
|
+
return /haiku/i.test(modelId) ? 2048 : 1024;
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
// packages/core/dist/src/provider-usage.js
|
|
1897
|
+
function isPlainObject4(value) {
|
|
1898
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1899
|
+
}
|
|
1900
|
+
function asCount(value) {
|
|
1901
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
1902
|
+
}
|
|
1903
|
+
function normalizeProviderUsage(response) {
|
|
1904
|
+
if (!isPlainObject4(response))
|
|
1905
|
+
return { status: "missing" };
|
|
1906
|
+
const usage = isPlainObject4(response.usage) ? response.usage : void 0;
|
|
1907
|
+
if (!usage)
|
|
1908
|
+
return { status: "missing" };
|
|
1909
|
+
const output = asCount(usage.output_tokens) ?? asCount(usage.completion_tokens);
|
|
1910
|
+
if (output === void 0) {
|
|
1911
|
+
return { status: "invalid", reason: "Usage carries no readable output_tokens/completion_tokens." };
|
|
1912
|
+
}
|
|
1913
|
+
const anthropicRead = asCount(usage.cache_read_input_tokens);
|
|
1914
|
+
const anthropicWriteFlat = asCount(usage.cache_creation_input_tokens);
|
|
1915
|
+
const creation = isPlainObject4(usage.cache_creation) ? usage.cache_creation : void 0;
|
|
1916
|
+
if (anthropicRead !== void 0 || anthropicWriteFlat !== void 0 || creation !== void 0) {
|
|
1917
|
+
const fresh = asCount(usage.input_tokens) ?? 0;
|
|
1918
|
+
const cached2 = anthropicRead ?? 0;
|
|
1919
|
+
let written = anthropicWriteFlat ?? 0;
|
|
1920
|
+
let write5m;
|
|
1921
|
+
let write1h;
|
|
1922
|
+
if (creation !== void 0) {
|
|
1923
|
+
write5m = asCount(creation.ephemeral_5m_input_tokens) ?? 0;
|
|
1924
|
+
write1h = asCount(creation.ephemeral_1h_input_tokens) ?? 0;
|
|
1925
|
+
const splitTotal = write5m + write1h;
|
|
1926
|
+
if (anthropicWriteFlat !== void 0 && splitTotal !== anthropicWriteFlat) {
|
|
1927
|
+
return {
|
|
1928
|
+
status: "invalid",
|
|
1929
|
+
reason: `cache_creation split (5m ${write5m} + 1h ${write1h} = ${splitTotal}) does not sum to cache_creation_input_tokens ${anthropicWriteFlat}.`
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
written = anthropicWriteFlat ?? splitTotal;
|
|
1933
|
+
}
|
|
1934
|
+
return {
|
|
1935
|
+
status: "ok",
|
|
1936
|
+
usage: {
|
|
1937
|
+
inputTokens: fresh + cached2 + written,
|
|
1938
|
+
uncachedInputTokens: fresh,
|
|
1939
|
+
cachedInputTokens: cached2,
|
|
1940
|
+
cacheWriteInputTokens: written,
|
|
1941
|
+
...write5m === void 0 ? {} : { cacheWrite5mInputTokens: write5m },
|
|
1942
|
+
...write1h === void 0 ? {} : { cacheWrite1hInputTokens: write1h },
|
|
1943
|
+
outputTokens: output
|
|
1944
|
+
}
|
|
1945
|
+
};
|
|
1946
|
+
}
|
|
1947
|
+
const input = asCount(usage.input_tokens) ?? asCount(usage.prompt_tokens);
|
|
1948
|
+
if (input === void 0) {
|
|
1949
|
+
return { status: "invalid", reason: "Usage carries no readable input_tokens/prompt_tokens." };
|
|
1950
|
+
}
|
|
1951
|
+
const details = isPlainObject4(usage.input_tokens_details) ? usage.input_tokens_details : void 0;
|
|
1952
|
+
const promptDetails = isPlainObject4(usage.prompt_tokens_details) ? usage.prompt_tokens_details : void 0;
|
|
1953
|
+
const cached = asCount(details?.cached_tokens) ?? asCount(promptDetails?.cached_tokens) ?? 0;
|
|
1954
|
+
const boundedCached = Math.min(cached, input);
|
|
1955
|
+
return {
|
|
1956
|
+
status: "ok",
|
|
1957
|
+
usage: {
|
|
1958
|
+
inputTokens: input,
|
|
1959
|
+
uncachedInputTokens: input - boundedCached,
|
|
1960
|
+
cachedInputTokens: boundedCached,
|
|
1961
|
+
cacheWriteInputTokens: 0,
|
|
1962
|
+
outputTokens: output
|
|
1963
|
+
}
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
function responseServiceTier(response) {
|
|
1967
|
+
if (!isPlainObject4(response))
|
|
1968
|
+
return void 0;
|
|
1969
|
+
if (typeof response.service_tier === "string")
|
|
1970
|
+
return response.service_tier;
|
|
1971
|
+
const usage = isPlainObject4(response.usage) ? response.usage : void 0;
|
|
1972
|
+
return typeof usage?.service_tier === "string" ? usage.service_tier : void 0;
|
|
1973
|
+
}
|
|
1974
|
+
|
|
1975
|
+
// packages/core/dist/src/output-diagnostics.js
|
|
1976
|
+
var LINE_NUMBER_PREFIX = /^\s*\d+[:.|]\s?/;
|
|
1977
|
+
function normalizeLine(line) {
|
|
1978
|
+
return line.replace(LINE_NUMBER_PREFIX, "").trim().replace(/\s+/g, " ");
|
|
1979
|
+
}
|
|
1980
|
+
function looksLikeCode(line) {
|
|
1981
|
+
if (/[{};=()[\]<>]/.test(line) && (line.match(/[{};=()[\]<>.:]/g) ?? []).length >= 3)
|
|
1982
|
+
return true;
|
|
1983
|
+
return /^(import |from |def |class |function |const |let |var |return |if |for |while |#include|public |private )/.test(line.trimStart());
|
|
1984
|
+
}
|
|
1985
|
+
function collectRequestText(value, depth = 0) {
|
|
1986
|
+
if (depth > 12)
|
|
1987
|
+
return "";
|
|
1988
|
+
if (typeof value === "string")
|
|
1989
|
+
return value;
|
|
1990
|
+
if (Array.isArray(value))
|
|
1991
|
+
return value.map((entry2) => collectRequestText(entry2, depth + 1)).filter(Boolean).join("\n");
|
|
1992
|
+
if (typeof value === "object" && value !== null) {
|
|
1993
|
+
return Object.values(value).map((entry2) => collectRequestText(entry2, depth + 1)).filter(Boolean).join("\n");
|
|
1994
|
+
}
|
|
1995
|
+
return "";
|
|
1996
|
+
}
|
|
1997
|
+
function measureOutputReEmission(requestText, outputText, countTokens = approximateTokenCount) {
|
|
1998
|
+
const contextLines = /* @__PURE__ */ new Set();
|
|
1999
|
+
for (const line of requestText.split("\n")) {
|
|
2000
|
+
const normalized = normalizeLine(line);
|
|
2001
|
+
if (normalized.length >= 8)
|
|
2002
|
+
contextLines.add(normalized);
|
|
2003
|
+
}
|
|
2004
|
+
let inspected = 0;
|
|
2005
|
+
let reEmitted = 0;
|
|
2006
|
+
let reEmittedCode = 0;
|
|
2007
|
+
for (const line of outputText.split("\n")) {
|
|
2008
|
+
const tokens = countTokens(line);
|
|
2009
|
+
inspected += tokens;
|
|
2010
|
+
const normalized = normalizeLine(line);
|
|
2011
|
+
if (normalized.length >= 8 && contextLines.has(normalized)) {
|
|
2012
|
+
reEmitted += tokens;
|
|
2013
|
+
if (looksLikeCode(normalized))
|
|
2014
|
+
reEmittedCode += tokens;
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
return {
|
|
2018
|
+
inspectedOutputTokens: inspected,
|
|
2019
|
+
reEmittedOutputTokens: reEmitted,
|
|
2020
|
+
reEmittedCodeTokens: reEmittedCode
|
|
2021
|
+
};
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
// packages/sdk-js/src/telemetry.ts
|
|
2025
|
+
var TelemetryClient = class {
|
|
2026
|
+
constructor(config = {}) {
|
|
2027
|
+
this.config = config;
|
|
2028
|
+
}
|
|
2029
|
+
config;
|
|
2030
|
+
async emit(event) {
|
|
2031
|
+
if (!this.config.endpoint) return;
|
|
2032
|
+
const metadataEvent = {
|
|
2033
|
+
...event,
|
|
2034
|
+
// The local telemetry endpoint needs failure status, not provider error text.
|
|
2035
|
+
error: event.error ? "provider_error" : void 0
|
|
2036
|
+
};
|
|
2037
|
+
void fetch(this.config.endpoint, {
|
|
2038
|
+
method: "POST",
|
|
2039
|
+
headers: { "content-type": "application/json" },
|
|
2040
|
+
body: JSON.stringify(metadataEvent),
|
|
2041
|
+
signal: AbortSignal.timeout(1e4)
|
|
2042
|
+
}).catch(() => {
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
};
|
|
2046
|
+
|
|
2047
|
+
// packages/sdk-js/src/routing.ts
|
|
2048
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
2049
|
+
function bucket(key) {
|
|
2050
|
+
const firstWord = createHash5("sha256").update(key).digest().readUInt32BE(0);
|
|
2051
|
+
return firstWord / 4294967296;
|
|
2052
|
+
}
|
|
2053
|
+
function validPolicy(policy) {
|
|
2054
|
+
if (!policy.id || !policy.taskClass || !policy.baselineModel) return false;
|
|
2055
|
+
if (policy.workflow !== void 0 && typeof policy.workflow !== "string") return false;
|
|
2056
|
+
if (!Number.isInteger(policy.minimumSamples) || policy.minimumSamples < 1) return false;
|
|
2057
|
+
if (!Number.isFinite(policy.minimumCandidateAccuracyLower95) || policy.minimumCandidateAccuracyLower95 < 0 || policy.minimumCandidateAccuracyLower95 > 1) return false;
|
|
2058
|
+
if (!Number.isFinite(policy.maximumAccuracyDrop) || policy.maximumAccuracyDrop < 0 || policy.maximumAccuracyDrop > 1) return false;
|
|
2059
|
+
if (policy.difficulty !== void 0 && !["low", "medium", "high"].includes(policy.difficulty)) return false;
|
|
2060
|
+
if (policy.riskLevel !== void 0 && !["low", "medium", "high"].includes(policy.riskLevel)) return false;
|
|
2061
|
+
if (!Array.isArray(policy.candidates) || policy.candidates.some(
|
|
2062
|
+
(candidate) => typeof candidate !== "object" || candidate === null
|
|
2063
|
+
)) return false;
|
|
2064
|
+
const shares = policy.candidates.reduce((total, candidate) => total + candidate.trafficShare, 0);
|
|
2065
|
+
if (shares > 1 || policy.candidates.some(
|
|
2066
|
+
(candidate) => !candidate.model || candidate.model === policy.baselineModel && candidate.maxOutputTokens === void 0 && !candidate.contextProfileId || !Number.isFinite(candidate.trafficShare) || candidate.trafficShare < 0 || candidate.trafficShare > 1 || candidate.maxOutputTokens !== void 0 && (!Number.isInteger(candidate.maxOutputTokens) || candidate.maxOutputTokens < 1) || candidate.contextProfileId !== void 0 && (typeof candidate.contextProfileId !== "string" || candidate.contextProfileId.trim() === "")
|
|
2067
|
+
)) return false;
|
|
2068
|
+
return true;
|
|
2069
|
+
}
|
|
2070
|
+
function chooseRoute(config, request, taskClass, context = {}) {
|
|
2071
|
+
const requestedModel = typeof request.model === "string" ? request.model : "unknown";
|
|
2072
|
+
const policy = Array.isArray(config?.policies) ? config.policies.find((candidate) => candidate && candidate.taskClass === taskClass && candidate.baselineModel === requestedModel && (candidate.workflow === void 0 || candidate.workflow === context.workflow) && (candidate.difficulty === void 0 || candidate.difficulty === context.difficulty) && (candidate.riskLevel === void 0 || candidate.riskLevel === context.riskLevel)) : void 0;
|
|
2073
|
+
const baseline = (reason, policyId) => ({
|
|
2074
|
+
model: requestedModel,
|
|
2075
|
+
requestedModel,
|
|
2076
|
+
policyId,
|
|
2077
|
+
reason
|
|
2078
|
+
});
|
|
2079
|
+
if (!policy || !validPolicy(policy)) return baseline("no_matching_policy");
|
|
2080
|
+
let requestKey;
|
|
2081
|
+
try {
|
|
2082
|
+
requestKey = config?.getRequestKey(request);
|
|
2083
|
+
} catch {
|
|
2084
|
+
requestKey = void 0;
|
|
2085
|
+
}
|
|
2086
|
+
if (typeof requestKey !== "string" || requestKey.trim() === "") return baseline("missing_stable_key", policy.id);
|
|
2087
|
+
const position = bucket(`${policy.id}:${requestKey}`);
|
|
2088
|
+
const candidatesByCost = [...policy.candidates].sort((left, right) => {
|
|
2089
|
+
const leftCost = (left.evidence?.candidateCostUsd ?? Number.POSITIVE_INFINITY) / Math.max(left.evidence?.sampleSize ?? 1, 1);
|
|
2090
|
+
const rightCost = (right.evidence?.candidateCostUsd ?? Number.POSITIVE_INFINITY) / Math.max(right.evidence?.sampleSize ?? 1, 1);
|
|
2091
|
+
return leftCost - rightCost;
|
|
2092
|
+
});
|
|
2093
|
+
let intervalStart = 0;
|
|
2094
|
+
for (const candidate of candidatesByCost) {
|
|
2095
|
+
const intervalEnd = intervalStart + candidate.trafficShare;
|
|
2096
|
+
const selected = position >= intervalStart && position < intervalEnd;
|
|
2097
|
+
intervalStart = intervalEnd;
|
|
2098
|
+
if (!selected) continue;
|
|
2099
|
+
const evidence = candidate.evidence;
|
|
2100
|
+
const evidenceMatches = evidence?.baselineModel === policy.baselineModel && evidence.candidateModel === candidate.model && evidence.eligibleForReview === true && Number.isInteger(evidence.sampleSize) && evidence.sampleSize >= policy.minimumSamples && Number.isFinite(evidence.candidateAccuracyLower95) && evidence.candidateAccuracyLower95 >= policy.minimumCandidateAccuracyLower95 && Number.isFinite(evidence.accuracyDrop) && evidence.accuracyDrop <= policy.maximumAccuracyDrop && Number.isFinite(evidence.baselineCostUsd) && evidence.baselineCostUsd > 0 && Number.isFinite(evidence.candidateCostUsd) && evidence.candidateCostUsd >= 0 && evidence.candidateCostUsd < evidence.baselineCostUsd;
|
|
2101
|
+
if (candidate.approved !== true || !evidenceMatches) return baseline("baseline", policy.id);
|
|
2102
|
+
return {
|
|
2103
|
+
model: candidate.model,
|
|
2104
|
+
requestedModel,
|
|
2105
|
+
policyId: policy.id,
|
|
2106
|
+
reason: "candidate_quality_and_cost_gates_passed",
|
|
2107
|
+
estimatedCostSavingsPercent: (1 - evidence.candidateCostUsd / evidence.baselineCostUsd) * 100,
|
|
2108
|
+
maxOutputTokens: candidate.maxOutputTokens,
|
|
2109
|
+
contextProfileId: candidate.contextProfileId
|
|
2110
|
+
};
|
|
2111
|
+
}
|
|
2112
|
+
return baseline("baseline", policy.id);
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
// packages/sdk-js/src/exact-cache.ts
|
|
2116
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
2117
|
+
function canonicalJson2(value, seen = /* @__PURE__ */ new Set()) {
|
|
2118
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
|
|
2119
|
+
if (typeof value === "number") return Number.isFinite(value) ? JSON.stringify(value) : void 0;
|
|
2120
|
+
if (Array.isArray(value)) {
|
|
2121
|
+
if (seen.has(value)) return void 0;
|
|
2122
|
+
seen.add(value);
|
|
2123
|
+
const values = value.map((item) => canonicalJson2(item, seen));
|
|
2124
|
+
seen.delete(value);
|
|
2125
|
+
return values.some((item) => item === void 0) ? void 0 : `[${values.join(",")}]`;
|
|
2126
|
+
}
|
|
2127
|
+
if (typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) {
|
|
2128
|
+
return void 0;
|
|
2129
|
+
}
|
|
2130
|
+
if (seen.has(value)) return void 0;
|
|
2131
|
+
seen.add(value);
|
|
2132
|
+
const record = value;
|
|
2133
|
+
const entries = Object.keys(record).sort().map((key) => {
|
|
2134
|
+
const child = canonicalJson2(record[key], seen);
|
|
2135
|
+
return child === void 0 ? void 0 : `${JSON.stringify(key)}:${child}`;
|
|
2136
|
+
});
|
|
2137
|
+
seen.delete(value);
|
|
2138
|
+
return entries.some((entry2) => entry2 === void 0) ? void 0 : `{${entries.join(",")}}`;
|
|
2139
|
+
}
|
|
2140
|
+
var ExactResponseCache = class {
|
|
2141
|
+
constructor(config) {
|
|
2142
|
+
this.config = config;
|
|
2143
|
+
if (!Number.isInteger(config.maxEntries) || config.maxEntries < 1) {
|
|
2144
|
+
throw new Error("MESH exact cache maxEntries must be a positive integer.");
|
|
2145
|
+
}
|
|
2146
|
+
if (!Number.isFinite(config.ttlMs) || config.ttlMs < 1) {
|
|
2147
|
+
throw new Error("MESH exact cache ttlMs must be a positive number.");
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
2150
|
+
config;
|
|
2151
|
+
entries = /* @__PURE__ */ new Map();
|
|
2152
|
+
keyFor(request, scopeKey) {
|
|
2153
|
+
if (typeof scopeKey !== "string" || scopeKey.trim() === "" || !request || typeof request !== "object") return void 0;
|
|
2154
|
+
const body = request;
|
|
2155
|
+
if (body.stream === true || body.tools !== void 0 || body.tool_choice !== void 0 || body.previous_response_id !== void 0 || body.conversation !== void 0) return void 0;
|
|
2156
|
+
const serialized = canonicalJson2(request);
|
|
2157
|
+
if (serialized === void 0) return void 0;
|
|
2158
|
+
return createHash6("sha256").update(`${scopeKey}
|
|
2159
|
+
${serialized}`).digest("hex");
|
|
2160
|
+
}
|
|
2161
|
+
get(key) {
|
|
2162
|
+
const entry2 = this.entries.get(key);
|
|
2163
|
+
if (!entry2) return void 0;
|
|
2164
|
+
if (entry2.expiresAt <= Date.now()) {
|
|
2165
|
+
this.entries.delete(key);
|
|
2166
|
+
return void 0;
|
|
2167
|
+
}
|
|
2168
|
+
try {
|
|
2169
|
+
const response = structuredClone(entry2.response);
|
|
2170
|
+
this.entries.delete(key);
|
|
2171
|
+
this.entries.set(key, entry2);
|
|
2172
|
+
return { ...entry2, response };
|
|
2173
|
+
} catch {
|
|
2174
|
+
this.entries.delete(key);
|
|
2175
|
+
return void 0;
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
set(key, entry2) {
|
|
2179
|
+
try {
|
|
2180
|
+
const response = structuredClone(entry2.response);
|
|
2181
|
+
this.entries.delete(key);
|
|
2182
|
+
this.entries.set(key, { ...entry2, response, expiresAt: Date.now() + this.config.ttlMs });
|
|
2183
|
+
while (this.entries.size > this.config.maxEntries) {
|
|
2184
|
+
const oldestKey = this.entries.keys().next().value;
|
|
2185
|
+
if (oldestKey === void 0) break;
|
|
2186
|
+
this.entries.delete(oldestKey);
|
|
2187
|
+
}
|
|
2188
|
+
} catch {
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
};
|
|
2192
|
+
|
|
2193
|
+
// packages/sdk-js/src/cache-volatility-guard.ts
|
|
2194
|
+
import { createHash as createHash7 } from "node:crypto";
|
|
2195
|
+
var isPlainObject5 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2196
|
+
var digest = (value) => createHash7("sha256").update(JSON.stringify(value ?? null) ?? "null").digest("hex").slice(0, 32);
|
|
2197
|
+
var CachePrefixVolatilityGuard = class {
|
|
2198
|
+
constructor(maxConversations = 1e3) {
|
|
2199
|
+
this.maxConversations = maxConversations;
|
|
2200
|
+
if (!Number.isInteger(maxConversations) || maxConversations < 1) {
|
|
2201
|
+
throw new Error("maxConversations must be a positive integer.");
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
maxConversations;
|
|
2205
|
+
/** Insertion-ordered; re-inserted on touch, so the first key is the LRU. */
|
|
2206
|
+
conversations = /* @__PURE__ */ new Map();
|
|
2207
|
+
/**
|
|
2208
|
+
* Record this request and decide whether its conversation has earned a
|
|
2209
|
+
* cache stamp. Call once per request, with the raw (unreduced) request.
|
|
2210
|
+
*/
|
|
2211
|
+
observe(request) {
|
|
2212
|
+
if (!isPlainObject5(request) || !Array.isArray(request.messages) || request.messages.length === 0) {
|
|
2213
|
+
return { stamp: true };
|
|
2214
|
+
}
|
|
2215
|
+
const key = digest([request.model ?? null, request.messages[0]]);
|
|
2216
|
+
const prefixHash = digest([request.tools ?? null, request.system ?? null]);
|
|
2217
|
+
const messageHashes = request.messages.map(digest);
|
|
2218
|
+
const prior = this.conversations.get(key);
|
|
2219
|
+
if (prior !== void 0) this.conversations.delete(key);
|
|
2220
|
+
if (prior === void 0) {
|
|
2221
|
+
this.conversations.set(key, { prefixHash, messageHashes, volatile: false, stableStreak: 0 });
|
|
2222
|
+
this.evict();
|
|
2223
|
+
return { stamp: true };
|
|
2224
|
+
}
|
|
2225
|
+
const prefixChanged = prior.prefixHash !== prefixHash;
|
|
2226
|
+
const historyExtends = messageHashes.length >= prior.messageHashes.length && prior.messageHashes.every((hash, index) => messageHashes[index] === hash);
|
|
2227
|
+
const entry2 = { prefixHash, messageHashes, volatile: prior.volatile, stableStreak: prior.stableStreak };
|
|
2228
|
+
let decision = { stamp: true };
|
|
2229
|
+
if (prefixChanged || !historyExtends) {
|
|
2230
|
+
entry2.volatile = true;
|
|
2231
|
+
entry2.stableStreak = 0;
|
|
2232
|
+
decision = {
|
|
2233
|
+
stamp: false,
|
|
2234
|
+
reason: prefixChanged ? "tools/system changed between turns of this conversation, so a cached prefix could never be read back" : "the conversation history was rewritten between turns, so a cached prefix could never be read back"
|
|
2235
|
+
};
|
|
2236
|
+
} else if (entry2.volatile) {
|
|
2237
|
+
entry2.stableStreak += 1;
|
|
2238
|
+
if (entry2.stableStreak >= 2) {
|
|
2239
|
+
entry2.volatile = false;
|
|
2240
|
+
entry2.stableStreak = 0;
|
|
2241
|
+
} else {
|
|
2242
|
+
decision = {
|
|
2243
|
+
stamp: false,
|
|
2244
|
+
reason: "this conversation's prefix changed on an earlier turn; waiting for it to prove stable before stamping again"
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
this.conversations.set(key, entry2);
|
|
2249
|
+
return decision;
|
|
2250
|
+
}
|
|
2251
|
+
evict() {
|
|
2252
|
+
while (this.conversations.size > this.maxConversations) {
|
|
2253
|
+
const oldest = this.conversations.keys().next().value;
|
|
2254
|
+
if (oldest === void 0) return;
|
|
2255
|
+
this.conversations.delete(oldest);
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
};
|
|
2259
|
+
|
|
2260
|
+
// packages/sdk-js/src/reduce-request.ts
|
|
2261
|
+
var PREFIX_BREAKERS = [
|
|
2262
|
+
/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?Z?/g,
|
|
2263
|
+
/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
|
2264
|
+
];
|
|
2265
|
+
function isPlainObject6(value) {
|
|
2266
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2267
|
+
}
|
|
2268
|
+
function anthropicEffortSupport(model) {
|
|
2269
|
+
const m = model.toLowerCase();
|
|
2270
|
+
if (!m.includes("claude")) return /* @__PURE__ */ new Set();
|
|
2271
|
+
if (m.includes("opus-5") || m.includes("opus-4-8") || m.includes("opus-4-7") || m.includes("sonnet-5") || m.includes("fable-5") || m.includes("mythos-5")) {
|
|
2272
|
+
return /* @__PURE__ */ new Set(["low", "medium", "high", "xhigh", "max"]);
|
|
2273
|
+
}
|
|
2274
|
+
if (m.includes("opus-4-6") || m.includes("sonnet-4-6")) {
|
|
2275
|
+
return /* @__PURE__ */ new Set(["low", "medium", "high", "max"]);
|
|
2276
|
+
}
|
|
2277
|
+
if (m.includes("opus-4-5")) return /* @__PURE__ */ new Set(["low", "medium", "high"]);
|
|
2278
|
+
return /* @__PURE__ */ new Set();
|
|
2279
|
+
}
|
|
2280
|
+
function repairPrefix(instructions) {
|
|
2281
|
+
const lines = instructions.split("\n");
|
|
2282
|
+
const moved = [];
|
|
2283
|
+
const kept = [];
|
|
2284
|
+
const window = Math.min(lines.length, 8);
|
|
2285
|
+
for (const [index, line] of lines.entries()) {
|
|
2286
|
+
const breaks = index < window && PREFIX_BREAKERS.some((pattern) => {
|
|
2287
|
+
pattern.lastIndex = 0;
|
|
2288
|
+
return pattern.test(line);
|
|
2289
|
+
});
|
|
2290
|
+
if (breaks) moved.push(line);
|
|
2291
|
+
else kept.push(line);
|
|
2292
|
+
}
|
|
2293
|
+
if (moved.length === 0) return { text: instructions, moved };
|
|
2294
|
+
return { text: [...kept, ...moved].join("\n"), moved };
|
|
2295
|
+
}
|
|
2296
|
+
var STALE_BODY_MUST_KEEP = /\b(error|errors|errored|fail|fails|failed|failure|exception|traceback|stack trace|denied|timeout|timed out|warning|fatal|panic|refused|rejected|not found|invalid)\b/i;
|
|
2297
|
+
var STALE_BODY_FILE_PATH = /(?:^|[\s"'`(=[])((?:\/|\.{1,2}\/|~\/|[A-Za-z]:\\)[\w.\\/-]+|[\w.-]+\/[\w./-]+|[\w-]+\.(?:ts|tsx|js|jsx|mjs|cjs|py|json|jsonl|md|yml|yaml|toml|txt|csv|tsv|html|css|go|rs|java|rb|sh|sql|log|xml|lock))\b/;
|
|
2298
|
+
var MESH_MARKER = /\[mesh: \d+ chars of [\w -]+\]/;
|
|
2299
|
+
function digestStaleBody(text) {
|
|
2300
|
+
const segments = text.split(/(?<=[.!?])\s+|\n+/).filter((segment) => segment.length > 0);
|
|
2301
|
+
if (segments.length <= 1) return text;
|
|
2302
|
+
const kept = [];
|
|
2303
|
+
let droppedChars = 0;
|
|
2304
|
+
for (const [index, segment] of segments.entries()) {
|
|
2305
|
+
const keep = index === 0 || STALE_BODY_MUST_KEEP.test(segment) || STALE_BODY_FILE_PATH.test(segment) || MESH_MARKER.test(segment);
|
|
2306
|
+
if (keep) kept.push(segment);
|
|
2307
|
+
else droppedChars += segment.length;
|
|
2308
|
+
}
|
|
2309
|
+
if (droppedChars === 0) return text;
|
|
2310
|
+
const digest2 = `${kept.join(" ")} [mesh: ${droppedChars} chars of stale detail compacted]`;
|
|
2311
|
+
return digest2.length < text.length ? digest2 : text;
|
|
2312
|
+
}
|
|
2313
|
+
function capToolResultBody(text, maximumTokens, countTokens) {
|
|
2314
|
+
if (!Number.isInteger(maximumTokens) || maximumTokens < 64) {
|
|
2315
|
+
throw new Error("freshToolResultCap maximumTokens must be an integer of at least 64.");
|
|
2316
|
+
}
|
|
2317
|
+
if (countTokens(text) <= maximumTokens) return text;
|
|
2318
|
+
const segments = text.split(/(?<=[.!?])\s+|\n+/).filter((segment) => segment.length > 0);
|
|
2319
|
+
if (segments.length <= 1) return text;
|
|
2320
|
+
const kept = [];
|
|
2321
|
+
let droppedChars = 0;
|
|
2322
|
+
for (const [index, segment] of segments.entries()) {
|
|
2323
|
+
const keep = index === 0 || STALE_BODY_MUST_KEEP.test(segment) || STALE_BODY_FILE_PATH.test(segment) || MESH_MARKER.test(segment);
|
|
2324
|
+
if (keep) kept.push(segment);
|
|
2325
|
+
else droppedChars += segment.length;
|
|
2326
|
+
}
|
|
2327
|
+
if (droppedChars === 0) return text;
|
|
2328
|
+
const capped = `${kept.join(" ")} [mesh: ${droppedChars} chars of fresh tool-result detail capped]`;
|
|
2329
|
+
return capped.length < text.length ? capped : text;
|
|
2330
|
+
}
|
|
2331
|
+
function transformBodyValue(value, transform, state) {
|
|
2332
|
+
if (typeof value === "string") {
|
|
2333
|
+
const transformed = transform(value);
|
|
2334
|
+
if (transformed !== value) state.bodies += 1;
|
|
2335
|
+
return transformed;
|
|
2336
|
+
}
|
|
2337
|
+
if (Array.isArray(value)) {
|
|
2338
|
+
return value.map((part) => {
|
|
2339
|
+
if (!isPlainObject6(part)) return part;
|
|
2340
|
+
if (typeof part.text === "string") {
|
|
2341
|
+
const transformed = transform(part.text);
|
|
2342
|
+
if (transformed !== part.text) state.bodies += 1;
|
|
2343
|
+
return transformed === part.text ? part : { ...part, text: transformed };
|
|
2344
|
+
}
|
|
2345
|
+
if ("content" in part) {
|
|
2346
|
+
const transformed = transformBodyValue(part.content, transform, state);
|
|
2347
|
+
return transformed === part.content ? part : { ...part, content: transformed };
|
|
2348
|
+
}
|
|
2349
|
+
return part;
|
|
2350
|
+
});
|
|
2351
|
+
}
|
|
2352
|
+
return value;
|
|
2353
|
+
}
|
|
2354
|
+
function digestBodyValue(value, state) {
|
|
2355
|
+
return transformBodyValue(value, digestStaleBody, state);
|
|
2356
|
+
}
|
|
2357
|
+
function isToolResultCarrier(message) {
|
|
2358
|
+
return isPlainObject6(message) && message.role === "user" && Array.isArray(message.content) && message.content.length > 0 && message.content.every((block) => isPlainObject6(block) && block.type === "tool_result");
|
|
2359
|
+
}
|
|
2360
|
+
function transformUserToolResultBlocks(message, transform, countTokens, state, addSavedTokens) {
|
|
2361
|
+
if (!Array.isArray(message.content)) return message;
|
|
2362
|
+
const sizeOf = (value) => countTokens(typeof value === "string" ? value : JSON.stringify(value) ?? "");
|
|
2363
|
+
let changed = false;
|
|
2364
|
+
const blocks = message.content.map((block) => {
|
|
2365
|
+
if (!isPlainObject6(block) || block.type !== "tool_result" || !("content" in block)) return block;
|
|
2366
|
+
const before = sizeOf(block.content);
|
|
2367
|
+
const bodiesBefore = state.bodies;
|
|
2368
|
+
const transformed = transformBodyValue(block.content, transform, state);
|
|
2369
|
+
if (state.bodies === bodiesBefore) return block;
|
|
2370
|
+
changed = true;
|
|
2371
|
+
addSavedTokens(Math.max(0, before - sizeOf(transformed)));
|
|
2372
|
+
return { ...block, content: transformed };
|
|
2373
|
+
});
|
|
2374
|
+
return changed ? { ...message, content: blocks } : message;
|
|
2375
|
+
}
|
|
2376
|
+
function capFreshToolResults(input, maximumTokens, countTokens, changes) {
|
|
2377
|
+
const cap = (text) => capToolResultBody(text, maximumTokens, countTokens);
|
|
2378
|
+
const state = { bodies: 0 };
|
|
2379
|
+
let savedTokens = 0;
|
|
2380
|
+
const next = input.map((message) => {
|
|
2381
|
+
if (!isPlainObject6(message)) return message;
|
|
2382
|
+
const role = typeof message.role === "string" ? message.role : void 0;
|
|
2383
|
+
if (role === "system" || role === "developer") return message;
|
|
2384
|
+
if (role === "user") {
|
|
2385
|
+
return transformUserToolResultBlocks(message, cap, countTokens, state, (saved) => {
|
|
2386
|
+
savedTokens += saved;
|
|
2387
|
+
});
|
|
2388
|
+
}
|
|
2389
|
+
if (message.type === "function_call_output" && typeof message.output === "string") {
|
|
2390
|
+
const before = countTokens(message.output);
|
|
2391
|
+
const capped = transformBodyValue(message.output, cap, state);
|
|
2392
|
+
if (capped === message.output) return message;
|
|
2393
|
+
savedTokens += Math.max(0, before - countTokens(capped));
|
|
2394
|
+
return { ...message, output: capped };
|
|
2395
|
+
}
|
|
2396
|
+
if ((role === "tool" || role === "assistant") && "content" in message) {
|
|
2397
|
+
const before = countTokens(
|
|
2398
|
+
typeof message.content === "string" ? message.content : JSON.stringify(message.content) ?? ""
|
|
2399
|
+
);
|
|
2400
|
+
const capped = transformBodyValue(message.content, cap, state);
|
|
2401
|
+
if (capped === message.content) return message;
|
|
2402
|
+
savedTokens += Math.max(0, before - countTokens(
|
|
2403
|
+
typeof capped === "string" ? capped : JSON.stringify(capped) ?? ""
|
|
2404
|
+
));
|
|
2405
|
+
return { ...message, content: capped };
|
|
2406
|
+
}
|
|
2407
|
+
return message;
|
|
2408
|
+
});
|
|
2409
|
+
if (state.bodies === 0) return input;
|
|
2410
|
+
changes.push({
|
|
2411
|
+
lever: "fresh-tool-result-cap",
|
|
2412
|
+
lossless: false,
|
|
2413
|
+
estimatedTokensSaved: savedTokens,
|
|
2414
|
+
detail: `Capped ${state.bodies} oversized tool-result/assistant body(ies) above ${maximumTokens} tokens at injection, before first send (headlines, errors, file paths and MESH markers kept verbatim). These tokens would have been billed once at the FULL input rate; history compaction only reaches their cached rebills.`
|
|
2415
|
+
});
|
|
2416
|
+
return next;
|
|
2417
|
+
}
|
|
2418
|
+
function compactStaleHistory(input, options, countTokens, changes) {
|
|
2419
|
+
const minimumTurns = options.minimumTurns ?? 3;
|
|
2420
|
+
const keepRecentTurns = options.keepRecentTurns ?? 2;
|
|
2421
|
+
const epochTurns = options.epochTurns ?? 2;
|
|
2422
|
+
if (minimumTurns < 1 || keepRecentTurns < 1 || epochTurns < 1) {
|
|
2423
|
+
throw new Error("History compaction turn counts must be positive integers.");
|
|
2424
|
+
}
|
|
2425
|
+
const isRealUserTurn2 = (message) => isPlainObject6(message) && message.role === "user" && !isToolResultCarrier(message);
|
|
2426
|
+
const totalTurns = input.filter(isRealUserTurn2).length;
|
|
2427
|
+
if (totalTurns < minimumTurns) return input;
|
|
2428
|
+
const compactedTurns = Math.floor((totalTurns - keepRecentTurns) / epochTurns) * epochTurns;
|
|
2429
|
+
if (compactedTurns <= 0) return input;
|
|
2430
|
+
const state = { bodies: 0 };
|
|
2431
|
+
let savedTokens = 0;
|
|
2432
|
+
let turn = -1;
|
|
2433
|
+
const next = input.map((message) => {
|
|
2434
|
+
if (isRealUserTurn2(message)) turn += 1;
|
|
2435
|
+
if (turn < 0 || turn >= compactedTurns) return message;
|
|
2436
|
+
if (!isPlainObject6(message)) return message;
|
|
2437
|
+
const role = typeof message.role === "string" ? message.role : void 0;
|
|
2438
|
+
if (role === "system" || role === "developer") return message;
|
|
2439
|
+
if (role === "user") {
|
|
2440
|
+
return transformUserToolResultBlocks(message, digestStaleBody, countTokens, state, (saved) => {
|
|
2441
|
+
savedTokens += saved;
|
|
2442
|
+
});
|
|
2443
|
+
}
|
|
2444
|
+
if (message.type === "function_call_output" && typeof message.output === "string") {
|
|
2445
|
+
const before = countTokens(message.output);
|
|
2446
|
+
const digested = digestBodyValue(message.output, state);
|
|
2447
|
+
if (digested === message.output) return message;
|
|
2448
|
+
savedTokens += Math.max(0, before - countTokens(digested));
|
|
2449
|
+
return { ...message, output: digested };
|
|
2450
|
+
}
|
|
2451
|
+
if ((role === "tool" || role === "assistant") && "content" in message) {
|
|
2452
|
+
const before = countTokens(
|
|
2453
|
+
typeof message.content === "string" ? message.content : JSON.stringify(message.content) ?? ""
|
|
2454
|
+
);
|
|
2455
|
+
const digested = digestBodyValue(message.content, state);
|
|
2456
|
+
if (digested === message.content) return message;
|
|
2457
|
+
savedTokens += Math.max(0, before - countTokens(
|
|
2458
|
+
typeof digested === "string" ? digested : JSON.stringify(digested) ?? ""
|
|
2459
|
+
));
|
|
2460
|
+
return { ...message, content: digested };
|
|
2461
|
+
}
|
|
2462
|
+
return message;
|
|
2463
|
+
});
|
|
2464
|
+
if (state.bodies === 0) return input;
|
|
2465
|
+
changes.push({
|
|
2466
|
+
lever: "stale-history-compaction",
|
|
2467
|
+
lossless: false,
|
|
2468
|
+
estimatedTokensSaved: savedTokens,
|
|
2469
|
+
detail: `Compacted ${state.bodies} stale tool/assistant body(ies) from the first ${compactedTurns} of ${totalTurns} turns into deterministic digests (errors, file paths and headlines kept verbatim). The boundary is frozen to ${epochTurns}-turn epochs so the compacted prefix stays byte-identical across consecutive requests.`
|
|
2470
|
+
});
|
|
2471
|
+
return next;
|
|
2472
|
+
}
|
|
2473
|
+
function leadingText(message) {
|
|
2474
|
+
if (typeof message.content === "string") return message.content;
|
|
2475
|
+
if (Array.isArray(message.content) && message.content.length > 0) {
|
|
2476
|
+
const first = message.content[0];
|
|
2477
|
+
if (isPlainObject6(first) && first.type === "text" && typeof first.text === "string") return first.text;
|
|
2478
|
+
}
|
|
2479
|
+
return void 0;
|
|
2480
|
+
}
|
|
2481
|
+
function withLeadingText(message, text) {
|
|
2482
|
+
if (typeof message.content === "string") return { ...message, content: text };
|
|
2483
|
+
const content = message.content;
|
|
2484
|
+
return { ...message, content: [{ ...content[0], text }, ...content.slice(1)] };
|
|
2485
|
+
}
|
|
2486
|
+
function dedupeMemoryAcrossTurns(input, options, countTokens, changes) {
|
|
2487
|
+
const minimumBlockTokens = options.minimumBlockTokens ?? 64;
|
|
2488
|
+
if (!Number.isInteger(minimumBlockTokens) || minimumBlockTokens < 1) {
|
|
2489
|
+
throw new Error("crossTurnMemoryDedup minimumBlockTokens must be a positive integer.");
|
|
2490
|
+
}
|
|
2491
|
+
const indices = [];
|
|
2492
|
+
for (const [index, message] of input.entries()) {
|
|
2493
|
+
if (!isPlainObject6(message) || message.role !== "user") continue;
|
|
2494
|
+
if (leadingText(message) === void 0) continue;
|
|
2495
|
+
indices.push(index);
|
|
2496
|
+
}
|
|
2497
|
+
if (indices.length < 2) return input;
|
|
2498
|
+
const texts = indices.map((index) => leadingText(input[index]));
|
|
2499
|
+
const before = texts.reduce((sum, text) => sum + countTokens(text), 0);
|
|
2500
|
+
const result = dedupeRepeatedLeadingBlock(texts, minimumBlockTokens, countTokens);
|
|
2501
|
+
if (result.notes.length === 0) return input;
|
|
2502
|
+
const after = result.texts.reduce((sum, text) => sum + countTokens(text), 0);
|
|
2503
|
+
const next = [...input];
|
|
2504
|
+
indices.forEach((index, position) => {
|
|
2505
|
+
if (result.texts[position] === texts[position]) return;
|
|
2506
|
+
next[index] = withLeadingText(next[index], result.texts[position]);
|
|
2507
|
+
});
|
|
2508
|
+
changes.push({
|
|
2509
|
+
lever: "cross-turn-memory-dedup",
|
|
2510
|
+
lossless: false,
|
|
2511
|
+
estimatedTokensSaved: Math.max(0, before - after),
|
|
2512
|
+
detail: `Replaced ${result.notes.length} repeated leading block(s) across user turns with reference markers (the first turn's block, and every turn's own differing remainder, kept verbatim). This is a behaviour change, not a cache-rate change, so it needs a replay-gate pass on the customer's own traffic before the saving may be billed.`
|
|
2513
|
+
});
|
|
2514
|
+
return next;
|
|
2515
|
+
}
|
|
2516
|
+
function reduceText(text, config, countTokens, changes, label) {
|
|
2517
|
+
let current = text;
|
|
2518
|
+
if (config.duplicateContext !== false) {
|
|
2519
|
+
const before = countTokens(current);
|
|
2520
|
+
const reduced = dedupeRepeatedBlocks(24, countTokens).reduce(current);
|
|
2521
|
+
const saved = before - countTokens(reduced.text);
|
|
2522
|
+
if (saved > 0) {
|
|
2523
|
+
current = reduced.text;
|
|
2524
|
+
changes.push({
|
|
2525
|
+
lever: "duplicate-context-removal",
|
|
2526
|
+
lossless: true,
|
|
2527
|
+
estimatedTokensSaved: saved,
|
|
2528
|
+
detail: `Removed ${reduced.notes.length} duplicate block(s) from ${label}.`
|
|
2529
|
+
});
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
return current;
|
|
2533
|
+
}
|
|
2534
|
+
function reduceMessageContent(content, config, countTokens, changes) {
|
|
2535
|
+
if (typeof content === "string") return reduceText(content, config, countTokens, changes, "a message");
|
|
2536
|
+
if (Array.isArray(content)) {
|
|
2537
|
+
return content.map((part) => {
|
|
2538
|
+
if (!isPlainObject6(part) || typeof part.text !== "string") return part;
|
|
2539
|
+
return { ...part, text: reduceText(part.text, config, countTokens, changes, "a message part") };
|
|
2540
|
+
});
|
|
2541
|
+
}
|
|
2542
|
+
return content;
|
|
2543
|
+
}
|
|
2544
|
+
function stampAnthropicCacheTtl(next, ttl, systemVolatileTail, countTokens) {
|
|
2545
|
+
const model = typeof next.model === "string" ? next.model.toLowerCase() : "";
|
|
2546
|
+
if (!model.includes("claude")) return { stamped: false };
|
|
2547
|
+
if (!Array.isArray(next.messages) || next.messages.length === 0) return { stamped: false };
|
|
2548
|
+
if (next.cache_control !== void 0) return { stamped: false };
|
|
2549
|
+
const carriesCacheControl = (value) => (JSON.stringify(value ?? null) ?? "").includes('"cache_control"');
|
|
2550
|
+
if (carriesCacheControl(next.messages) || carriesCacheControl(next.system) || carriesCacheControl(next.tools)) {
|
|
2551
|
+
return { stamped: false };
|
|
2552
|
+
}
|
|
2553
|
+
const APPROX_OVERCOUNT_MARGIN = 1.3;
|
|
2554
|
+
const minimumTokens = Math.ceil(anthropicCacheableMinimumTokens(model) * APPROX_OVERCOUNT_MARGIN);
|
|
2555
|
+
const measureTokens = (value) => {
|
|
2556
|
+
try {
|
|
2557
|
+
return typeof value === "string" ? countTokens(value) : countTokens(JSON.stringify(value) ?? "");
|
|
2558
|
+
} catch {
|
|
2559
|
+
return 0;
|
|
2560
|
+
}
|
|
2561
|
+
};
|
|
2562
|
+
const toolsTokens = Array.isArray(next.tools) && next.tools.length > 0 ? measureTokens(next.tools) : 0;
|
|
2563
|
+
const fullSystemTokens = measureTokens(next.system);
|
|
2564
|
+
const fullMessagesTokens = measureTokens(next.messages);
|
|
2565
|
+
let belowMinimum;
|
|
2566
|
+
const recordBelowMinimum = (measuredTokens) => {
|
|
2567
|
+
if (belowMinimum === void 0) belowMinimum = { measuredTokens, minimumTokens };
|
|
2568
|
+
};
|
|
2569
|
+
const cacheControl = { type: "ephemeral", ttl };
|
|
2570
|
+
let stamped = false;
|
|
2571
|
+
let volatileTailPresent = false;
|
|
2572
|
+
if (typeof next.system === "string" && next.system.trim() !== "") {
|
|
2573
|
+
const tailText = systemVolatileTail.join("\n");
|
|
2574
|
+
if (systemVolatileTail.length > 0 && tailText.trim() !== "" && (next.system === tailText || next.system.endsWith(`
|
|
2575
|
+
${tailText}`))) {
|
|
2576
|
+
volatileTailPresent = true;
|
|
2577
|
+
const stable = next.system === tailText ? "" : next.system.slice(0, -tailText.length);
|
|
2578
|
+
if (stable.trim() !== "") {
|
|
2579
|
+
const stableTokens = toolsTokens + measureTokens(stable);
|
|
2580
|
+
if (stableTokens < minimumTokens) {
|
|
2581
|
+
recordBelowMinimum(stableTokens);
|
|
2582
|
+
} else {
|
|
2583
|
+
next.system = [
|
|
2584
|
+
{ type: "text", text: stable, cache_control: cacheControl },
|
|
2585
|
+
{ type: "text", text: tailText }
|
|
2586
|
+
];
|
|
2587
|
+
stamped = true;
|
|
2588
|
+
}
|
|
2589
|
+
}
|
|
2590
|
+
} else {
|
|
2591
|
+
const stableTokens = toolsTokens + fullSystemTokens;
|
|
2592
|
+
if (stableTokens < minimumTokens) {
|
|
2593
|
+
recordBelowMinimum(stableTokens);
|
|
2594
|
+
} else {
|
|
2595
|
+
next.system = [{ type: "text", text: next.system, cache_control: cacheControl }];
|
|
2596
|
+
stamped = true;
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
} else if (Array.isArray(next.system) && next.system.length > 0) {
|
|
2600
|
+
const blocks = [...next.system];
|
|
2601
|
+
const lastBlock = blocks[blocks.length - 1];
|
|
2602
|
+
if (isPlainObject6(lastBlock) && !(typeof lastBlock.text === "string" && lastBlock.text.trim() === "")) {
|
|
2603
|
+
const stableTokens = toolsTokens + fullSystemTokens;
|
|
2604
|
+
if (stableTokens < minimumTokens) {
|
|
2605
|
+
recordBelowMinimum(stableTokens);
|
|
2606
|
+
} else {
|
|
2607
|
+
blocks[blocks.length - 1] = { ...lastBlock, cache_control: cacheControl };
|
|
2608
|
+
next.system = blocks;
|
|
2609
|
+
stamped = true;
|
|
2610
|
+
}
|
|
2611
|
+
}
|
|
2612
|
+
}
|
|
2613
|
+
if (!volatileTailPresent && next.messages.length >= 2) {
|
|
2614
|
+
const messages = [...next.messages];
|
|
2615
|
+
const lastIndex = messages.length - 1;
|
|
2616
|
+
const last = messages[lastIndex];
|
|
2617
|
+
if (isPlainObject6(last)) {
|
|
2618
|
+
const content = last.content;
|
|
2619
|
+
const prefixTokens = toolsTokens + fullSystemTokens + fullMessagesTokens;
|
|
2620
|
+
if (typeof content === "string") {
|
|
2621
|
+
if (content.trim() !== "") {
|
|
2622
|
+
if (prefixTokens < minimumTokens) {
|
|
2623
|
+
recordBelowMinimum(prefixTokens);
|
|
2624
|
+
} else {
|
|
2625
|
+
messages[lastIndex] = {
|
|
2626
|
+
...last,
|
|
2627
|
+
content: [{ type: "text", text: content, cache_control: cacheControl }]
|
|
2628
|
+
};
|
|
2629
|
+
next.messages = messages;
|
|
2630
|
+
stamped = true;
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
} else if (Array.isArray(content) && content.length > 0) {
|
|
2634
|
+
const blocks = [...content];
|
|
2635
|
+
const lastBlock = blocks[blocks.length - 1];
|
|
2636
|
+
if (isPlainObject6(lastBlock) && !(typeof lastBlock.text === "string" && lastBlock.text.trim() === "")) {
|
|
2637
|
+
if (prefixTokens < minimumTokens) {
|
|
2638
|
+
recordBelowMinimum(prefixTokens);
|
|
2639
|
+
} else {
|
|
2640
|
+
blocks[blocks.length - 1] = { ...lastBlock, cache_control: cacheControl };
|
|
2641
|
+
messages[lastIndex] = { ...last, content: blocks };
|
|
2642
|
+
next.messages = messages;
|
|
2643
|
+
stamped = true;
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
return stamped ? { stamped: true } : { stamped: false, belowMinimum };
|
|
2650
|
+
}
|
|
2651
|
+
function reduceRequest(request, config = {}) {
|
|
2652
|
+
const countTokens = config.countTokens ?? approximateTokenCount;
|
|
2653
|
+
const changes = [];
|
|
2654
|
+
const unchanged = (reason) => ({
|
|
2655
|
+
request,
|
|
2656
|
+
summary: {
|
|
2657
|
+
estimatedTokensBefore: 0,
|
|
2658
|
+
estimatedTokensAfter: 0,
|
|
2659
|
+
estimatedTokensSaved: 0,
|
|
2660
|
+
estimatedLosslessTokensSaved: 0,
|
|
2661
|
+
changes: [],
|
|
2662
|
+
skipped: true,
|
|
2663
|
+
skippedReason: reason
|
|
2664
|
+
}
|
|
2665
|
+
});
|
|
2666
|
+
if (!isPlainObject6(request)) return unchanged("Request is not an object.");
|
|
2667
|
+
try {
|
|
2668
|
+
const next = { ...request };
|
|
2669
|
+
const measure = (value) => {
|
|
2670
|
+
try {
|
|
2671
|
+
return typeof value === "string" ? countTokens(value) : countTokens(JSON.stringify(value) ?? "");
|
|
2672
|
+
} catch {
|
|
2673
|
+
return 0;
|
|
2674
|
+
}
|
|
2675
|
+
};
|
|
2676
|
+
const before = measure(next.instructions) + measure(next.system) + measure(next.input) + measure(next.messages) + measure(next.tools);
|
|
2677
|
+
if (Array.isArray(next.tools) && next.tools.length > 0 && config.toolSchemas !== false) {
|
|
2678
|
+
const toolOptions = isPlainObject6(config.toolSchemas) ? config.toolSchemas : {};
|
|
2679
|
+
const minified = minifyToolSchemas(next.tools, {
|
|
2680
|
+
countTokens,
|
|
2681
|
+
descriptions: toolOptions.descriptions ?? "keep",
|
|
2682
|
+
...typeof toolOptions.maximumDescriptionTokens === "number" ? { maximumDescriptionTokens: toolOptions.maximumDescriptionTokens } : {}
|
|
2683
|
+
});
|
|
2684
|
+
const saved = minified.tokensBefore - minified.tokensAfter;
|
|
2685
|
+
if (saved > 0) {
|
|
2686
|
+
next.tools = minified.tools;
|
|
2687
|
+
if (minified.losslessTokensSaved > 0) {
|
|
2688
|
+
changes.push({
|
|
2689
|
+
lever: "tool-schema-compaction",
|
|
2690
|
+
lossless: true,
|
|
2691
|
+
estimatedTokensSaved: minified.losslessTokensSaved,
|
|
2692
|
+
detail: `Compacted ${minified.tools.length} tool definition(s); the decoded schema is unchanged.`
|
|
2693
|
+
});
|
|
2694
|
+
}
|
|
2695
|
+
if (minified.lossyTokensSaved > 0) {
|
|
2696
|
+
changes.push({
|
|
2697
|
+
lever: "tool-schema-description-trim",
|
|
2698
|
+
lossless: false,
|
|
2699
|
+
estimatedTokensSaved: minified.lossyTokensSaved,
|
|
2700
|
+
detail: "Trimmed tool descriptions, which is how the model chooses between tools."
|
|
2701
|
+
});
|
|
2702
|
+
}
|
|
2703
|
+
}
|
|
2704
|
+
}
|
|
2705
|
+
let systemVolatileTail = [];
|
|
2706
|
+
const reduceSystemText = (text, label) => {
|
|
2707
|
+
let current = reduceText(text, config, countTokens, changes, label);
|
|
2708
|
+
if (config.repairCachePrefix) {
|
|
2709
|
+
const repaired = repairPrefix(current);
|
|
2710
|
+
if (repaired.moved.length > 0) {
|
|
2711
|
+
if (label === "system") systemVolatileTail = repaired.moved;
|
|
2712
|
+
current = repaired.text;
|
|
2713
|
+
changes.push({
|
|
2714
|
+
lever: "cache-prefix-repair",
|
|
2715
|
+
lossless: true,
|
|
2716
|
+
// The saving is a cache-rate change, not tokens removed.
|
|
2717
|
+
estimatedTokensSaved: 0,
|
|
2718
|
+
detail: `Moved ${repaired.moved.length} per-request value(s) out of the ${label} prefix, making the stable remainder cacheable. The text is reordered, not removed.`
|
|
2719
|
+
});
|
|
2720
|
+
}
|
|
2721
|
+
}
|
|
2722
|
+
return current;
|
|
2723
|
+
};
|
|
2724
|
+
if (typeof next.instructions === "string") {
|
|
2725
|
+
next.instructions = reduceSystemText(next.instructions, "instructions");
|
|
2726
|
+
}
|
|
2727
|
+
if (typeof next.system === "string") {
|
|
2728
|
+
next.system = reduceSystemText(next.system, "system");
|
|
2729
|
+
}
|
|
2730
|
+
if (config.freshToolResultCap) {
|
|
2731
|
+
const capOptions = isPlainObject6(config.freshToolResultCap) ? config.freshToolResultCap : {};
|
|
2732
|
+
const threshold = capOptions.maximumTokens === void 0 ? 600 : capOptions.maximumTokens;
|
|
2733
|
+
if (typeof threshold !== "number" || !Number.isInteger(threshold) || threshold < 64) {
|
|
2734
|
+
throw new Error("freshToolResultCap maximumTokens must be an integer of at least 64.");
|
|
2735
|
+
}
|
|
2736
|
+
if (Array.isArray(next.input)) {
|
|
2737
|
+
next.input = capFreshToolResults(next.input, threshold, countTokens, changes);
|
|
2738
|
+
}
|
|
2739
|
+
if (Array.isArray(next.messages)) {
|
|
2740
|
+
next.messages = capFreshToolResults(next.messages, threshold, countTokens, changes);
|
|
2741
|
+
}
|
|
2742
|
+
}
|
|
2743
|
+
if (config.historyCompaction) {
|
|
2744
|
+
const options = isPlainObject6(config.historyCompaction) ? config.historyCompaction : {};
|
|
2745
|
+
if (Array.isArray(next.input)) {
|
|
2746
|
+
next.input = compactStaleHistory(next.input, options, countTokens, changes);
|
|
2747
|
+
}
|
|
2748
|
+
if (Array.isArray(next.messages)) {
|
|
2749
|
+
next.messages = compactStaleHistory(next.messages, options, countTokens, changes);
|
|
2750
|
+
}
|
|
2751
|
+
}
|
|
2752
|
+
if (config.crossTurnMemoryDedup) {
|
|
2753
|
+
const memoryOptions = isPlainObject6(config.crossTurnMemoryDedup) ? config.crossTurnMemoryDedup : {};
|
|
2754
|
+
if (Array.isArray(next.input)) {
|
|
2755
|
+
next.input = dedupeMemoryAcrossTurns(next.input, memoryOptions, countTokens, changes);
|
|
2756
|
+
}
|
|
2757
|
+
if (Array.isArray(next.messages)) {
|
|
2758
|
+
next.messages = dedupeMemoryAcrossTurns(next.messages, memoryOptions, countTokens, changes);
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
if (Array.isArray(next.input)) {
|
|
2762
|
+
next.input = next.input.map((message) => {
|
|
2763
|
+
if (!isPlainObject6(message)) return message;
|
|
2764
|
+
if (!("content" in message)) return message;
|
|
2765
|
+
return { ...message, content: reduceMessageContent(message.content, config, countTokens, changes) };
|
|
2766
|
+
});
|
|
2767
|
+
} else if (typeof next.input === "string") {
|
|
2768
|
+
next.input = reduceText(next.input, config, countTokens, changes, "input");
|
|
2769
|
+
}
|
|
2770
|
+
if (Array.isArray(next.messages)) {
|
|
2771
|
+
next.messages = next.messages.map((message) => {
|
|
2772
|
+
if (!isPlainObject6(message)) return message;
|
|
2773
|
+
if (!("content" in message)) return message;
|
|
2774
|
+
return { ...message, content: reduceMessageContent(message.content, config, countTokens, changes) };
|
|
2775
|
+
});
|
|
2776
|
+
}
|
|
2777
|
+
if (config.executionTier === "flex" && next.service_tier === void 0) {
|
|
2778
|
+
const model = typeof next.model === "string" ? next.model : "";
|
|
2779
|
+
const anthropicShaped = model.toLowerCase().includes("claude");
|
|
2780
|
+
const catalogued = !anthropicShaped && resolveModelId(publishedModelPrices, "openai", model).status === "resolved";
|
|
2781
|
+
if (catalogued) {
|
|
2782
|
+
next.service_tier = "flex";
|
|
2783
|
+
changes.push({
|
|
2784
|
+
lever: "execution-tier-flex",
|
|
2785
|
+
lossless: true,
|
|
2786
|
+
// The saving is a rate change on the whole request, not tokens removed.
|
|
2787
|
+
estimatedTokensSaved: 0,
|
|
2788
|
+
detail: "Set service_tier to flex: batch-level pricing (50% off) on a synchronous API. Same model, same answer; latency may rise and 429s fall back to standard, so this applies only to workloads the caller flagged latency-tolerant."
|
|
2789
|
+
});
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2792
|
+
if (config.cacheTtl === "5m" || config.cacheTtl === "1h") {
|
|
2793
|
+
const ttl = config.cacheTtl;
|
|
2794
|
+
const decision = config.cacheVolatilityGuard?.observe(request) ?? { stamp: true };
|
|
2795
|
+
const result = decision.stamp ? stampAnthropicCacheTtl(next, ttl, systemVolatileTail, countTokens) : { stamped: false };
|
|
2796
|
+
if (!decision.stamp) {
|
|
2797
|
+
changes.push({
|
|
2798
|
+
lever: "cache-ttl-suppressed",
|
|
2799
|
+
lossless: true,
|
|
2800
|
+
// Nothing was changed; this records WHY no stamp was placed.
|
|
2801
|
+
estimatedTokensSaved: 0,
|
|
2802
|
+
detail: `Cache stamp withheld: ${decision.reason ?? "the conversation's prefix is not stable"}. A breakpoint over a churning prefix buys write premiums and never reads \u2014 a measured loss, not a saving.`
|
|
2803
|
+
});
|
|
2804
|
+
} else if (!result.stamped && result.belowMinimum) {
|
|
2805
|
+
changes.push({
|
|
2806
|
+
lever: "cache-ttl-suppressed",
|
|
2807
|
+
lossless: true,
|
|
2808
|
+
// Nothing was changed; this records WHY no stamp was placed.
|
|
2809
|
+
estimatedTokensSaved: 0,
|
|
2810
|
+
detail: `Cache stamp withheld: the stable prefix measures ${result.belowMinimum.measuredTokens} token(s) by MESH's approximate counter, under the ${result.belowMinimum.minimumTokens}-token threshold MESH requires \u2014 the model's documented ${anthropicCacheableMinimumTokens(typeof next.model === "string" ? next.model : "")}-token cacheable minimum with a 1.30x margin for the counter's calibrated overcount on prose. Anthropic processes a prefix under its minimum without caching at all \u2014 a provider-side no-op \u2014 so a breakpoint here would buy a write premium and never read, and would record a saving that can never happen.`
|
|
2811
|
+
});
|
|
2812
|
+
}
|
|
2813
|
+
if (result.stamped) {
|
|
2814
|
+
changes.push({
|
|
2815
|
+
lever: `cache-ttl-${ttl}`,
|
|
2816
|
+
lossless: true,
|
|
2817
|
+
// The saving is a rate change (cache reads at 0.1x), not tokens removed.
|
|
2818
|
+
estimatedTokensSaved: 0,
|
|
2819
|
+
detail: `Stamped cache_control {type: 'ephemeral', ttl: '${ttl}'} on the stable prefix (system prompt; plus the last message on multi-turn requests, whose history repeats next turn). ` + (ttl === "5m" ? "Writes bill at 1.25x base input, reads at 0.1x: the multi-turn agent setting, since Anthropic's cache is opt-in and an unstamped agent re-pays full rate for its history every turn. The caller must flag the workload - it is never set by default." : "Writes bill at 2x base input, reads at 0.1x: this pays only when calls arrive more than ~5 minutes apart, and the caller must flag that workload - it is never set by default.")
|
|
2820
|
+
});
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
if (config.maxOutputTokens !== void 0) {
|
|
2824
|
+
const assertValidCap = () => {
|
|
2825
|
+
if (!Number.isInteger(config.maxOutputTokens) || config.maxOutputTokens < 1) {
|
|
2826
|
+
throw new Error("maxOutputTokens must be a positive integer.");
|
|
2827
|
+
}
|
|
2828
|
+
};
|
|
2829
|
+
const claudeModel = typeof next.model === "string" && next.model.toLowerCase().includes("claude");
|
|
2830
|
+
const messagesShaped = Array.isArray(next.messages);
|
|
2831
|
+
if (claudeModel && messagesShaped && typeof next.max_tokens === "number" && Number.isFinite(next.max_tokens)) {
|
|
2832
|
+
assertValidCap();
|
|
2833
|
+
if (next.max_tokens > config.maxOutputTokens) {
|
|
2834
|
+
next.max_tokens = config.maxOutputTokens;
|
|
2835
|
+
changes.push({
|
|
2836
|
+
lever: "output-cap",
|
|
2837
|
+
lossless: false,
|
|
2838
|
+
estimatedTokensSaved: 0,
|
|
2839
|
+
detail: `Clamped max_tokens down to ${config.maxOutputTokens} (downward only; a caller cap at or below the budget is never touched). Responses that would have run longer are truncated, so this needs checking against real output lengths.`
|
|
2840
|
+
});
|
|
2841
|
+
}
|
|
2842
|
+
} else if (!claudeModel && messagesShaped) {
|
|
2843
|
+
if (next.max_completion_tokens === void 0 && next.max_tokens === void 0 && next.max_output_tokens === void 0) {
|
|
2844
|
+
assertValidCap();
|
|
2845
|
+
next.max_completion_tokens = config.maxOutputTokens;
|
|
2846
|
+
changes.push({
|
|
2847
|
+
lever: "output-cap",
|
|
2848
|
+
lossless: false,
|
|
2849
|
+
estimatedTokensSaved: 0,
|
|
2850
|
+
detail: `Set max_completion_tokens to ${config.maxOutputTokens}. Responses that would have run longer are truncated, so this needs checking against real output lengths.`
|
|
2851
|
+
});
|
|
2852
|
+
}
|
|
2853
|
+
} else if (next.max_output_tokens === void 0) {
|
|
2854
|
+
assertValidCap();
|
|
2855
|
+
next.max_output_tokens = config.maxOutputTokens;
|
|
2856
|
+
changes.push({
|
|
2857
|
+
lever: "output-cap",
|
|
2858
|
+
lossless: false,
|
|
2859
|
+
estimatedTokensSaved: 0,
|
|
2860
|
+
detail: `Set max_output_tokens to ${config.maxOutputTokens}. Responses that would have run longer are truncated, so this needs checking against real output lengths.`
|
|
2861
|
+
});
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
if (config.outputEffort !== void 0) {
|
|
2865
|
+
const level = config.outputEffort;
|
|
2866
|
+
const model = typeof next.model === "string" ? next.model : "";
|
|
2867
|
+
const claudeMessagesShaped = model.toLowerCase().includes("claude") && Array.isArray(next.messages);
|
|
2868
|
+
const existingConfig = isPlainObject6(next.output_config) ? next.output_config : void 0;
|
|
2869
|
+
const callerSetEffort = existingConfig !== void 0 && "effort" in existingConfig;
|
|
2870
|
+
if (claudeMessagesShaped && !callerSetEffort) {
|
|
2871
|
+
if (anthropicEffortSupport(model).has(level)) {
|
|
2872
|
+
next.output_config = { ...existingConfig ?? {}, effort: level };
|
|
2873
|
+
changes.push({
|
|
2874
|
+
lever: `output-effort-${level}`,
|
|
2875
|
+
lossless: false,
|
|
2876
|
+
// The saving is fewer output tokens the model chooses to emit, read
|
|
2877
|
+
// from provider usage by the gate, not an input-token count here.
|
|
2878
|
+
estimatedTokensSaved: 0,
|
|
2879
|
+
detail: `Set output_config.effort to '${level}'. The model thinks less and answers terser, cutting output tokens (the dearer half of the bill). A top-level field outside the cached prefix, so a stable value never invalidates caching. Lossy \u2014 it changes the answer's depth, so it needs a task-completion replay-gate pass on the caller's own traffic before the saving may be billed.`
|
|
2880
|
+
});
|
|
2881
|
+
} else {
|
|
2882
|
+
changes.push({
|
|
2883
|
+
lever: "output-effort-suppressed",
|
|
2884
|
+
lossless: true,
|
|
2885
|
+
// Nothing was changed; this records WHY no effort was set.
|
|
2886
|
+
estimatedTokensSaved: 0,
|
|
2887
|
+
detail: `Effort '${level}' withheld: model '${model || "(none)"}' is not in MESH's dated allow-list of Claude models known to accept that level. An unsupported effort value is a 400, and the reducer never risks breaking a request, so it is left as the caller sent it.`
|
|
2888
|
+
});
|
|
2889
|
+
}
|
|
2890
|
+
}
|
|
2891
|
+
}
|
|
2892
|
+
const after = measure(next.instructions) + measure(next.system) + measure(next.input) + measure(next.messages) + measure(next.tools);
|
|
2893
|
+
const summary = {
|
|
2894
|
+
estimatedTokensBefore: before,
|
|
2895
|
+
estimatedTokensAfter: after,
|
|
2896
|
+
estimatedTokensSaved: Math.max(0, before - after),
|
|
2897
|
+
estimatedLosslessTokensSaved: changes.filter((change) => change.lossless).reduce((sum, change) => sum + change.estimatedTokensSaved, 0),
|
|
2898
|
+
changes,
|
|
2899
|
+
skipped: false
|
|
2900
|
+
};
|
|
2901
|
+
config.onReduction?.(summary);
|
|
2902
|
+
return { request: changes.length === 0 ? request : next, summary };
|
|
2903
|
+
} catch (cause) {
|
|
2904
|
+
return unchanged(cause instanceof Error ? cause.message : String(cause));
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
|
|
2908
|
+
// packages/sdk-js/src/savings-ledger.ts
|
|
2909
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2910
|
+
import { dirname } from "node:path";
|
|
2911
|
+
var EXECUTION_TIER_LEVERS = /* @__PURE__ */ new Set(["execution-tier-flex"]);
|
|
2912
|
+
var CACHE_TTL_LEVERS = /* @__PURE__ */ new Map([["cache-ttl-5m", "5m"], ["cache-ttl-1h", "1h"]]);
|
|
2913
|
+
var MEASURED_COMPOSITE_LEVER = "token-reduction-measured";
|
|
2914
|
+
var EXACT_CACHE_LEVER = "exact-response-cache";
|
|
2915
|
+
function executionModeOf(responseServiceTier2) {
|
|
2916
|
+
if (responseServiceTier2 === "flex") return "flex";
|
|
2917
|
+
if (responseServiceTier2 === "batch") return "batch";
|
|
2918
|
+
return "standard";
|
|
2919
|
+
}
|
|
2920
|
+
function usageOf2(event) {
|
|
2921
|
+
const { inputTokens, outputTokens, cachedInputTokens, cacheWriteInputTokens } = event;
|
|
2922
|
+
if ([inputTokens, outputTokens, cachedInputTokens, cacheWriteInputTokens].some((count) => typeof count !== "number" || !Number.isFinite(count) || count < 0)) {
|
|
2923
|
+
return void 0;
|
|
2924
|
+
}
|
|
2925
|
+
const validCount = (count) => typeof count === "number" && Number.isFinite(count) && count >= 0;
|
|
2926
|
+
const splitReported = event.cacheWrite5mInputTokens !== void 0 || event.cacheWrite1hInputTokens !== void 0;
|
|
2927
|
+
const splitValid = splitReported && validCount(event.cacheWrite5mInputTokens ?? 0) && validCount(event.cacheWrite1hInputTokens ?? 0);
|
|
2928
|
+
if (splitReported && !splitValid) return void 0;
|
|
2929
|
+
return {
|
|
2930
|
+
inputTokens,
|
|
2931
|
+
outputTokens,
|
|
2932
|
+
cachedInputTokens,
|
|
2933
|
+
cacheWriteInputTokens,
|
|
2934
|
+
...splitValid ? {
|
|
2935
|
+
cacheWrite5mInputTokens: event.cacheWrite5mInputTokens ?? 0,
|
|
2936
|
+
cacheWrite1hInputTokens: event.cacheWrite1hInputTokens ?? 0
|
|
2937
|
+
} : {}
|
|
2938
|
+
};
|
|
2939
|
+
}
|
|
2940
|
+
function billableEventFromRequest(event, config, catalogue, pinnedCatalogueVersion) {
|
|
2941
|
+
const provider = config.provider ?? "openai";
|
|
2942
|
+
const changes = event.reduction && !event.reduction.skipped ? event.reduction.changes : [];
|
|
2943
|
+
const ttlChange = changes.find((change) => CACHE_TTL_LEVERS.has(change.lever));
|
|
2944
|
+
const meshPlacedCacheControl = ttlChange !== void 0;
|
|
2945
|
+
const meshPlacedTtl = ttlChange === void 0 ? void 0 : CACHE_TTL_LEVERS.get(ttlChange.lever);
|
|
2946
|
+
const executionMode = executionModeOf(event.responseServiceTier);
|
|
2947
|
+
const usage = usageOf2(event);
|
|
2948
|
+
const pricing = usage === void 0 ? { pricingStatus: "unpriced", reason: "The event carries no complete, valid provider-verbatim usage." } : priceLedgerUsage(catalogue, {
|
|
2949
|
+
provider,
|
|
2950
|
+
model: event.model,
|
|
2951
|
+
at: event.timestamp.slice(0, 10),
|
|
2952
|
+
executionMode,
|
|
2953
|
+
usage,
|
|
2954
|
+
meshPlacedCacheControl,
|
|
2955
|
+
...meshPlacedTtl === void 0 ? {} : { meshPlacedTtl }
|
|
2956
|
+
});
|
|
2957
|
+
const priced = pricing.pricingStatus === "priced" ? pricing : void 0;
|
|
2958
|
+
const levers = [];
|
|
2959
|
+
const excluded = {};
|
|
2960
|
+
if (event.cacheHit) {
|
|
2961
|
+
levers.push({
|
|
2962
|
+
leverId: EXACT_CACHE_LEVER,
|
|
2963
|
+
kind: "exact_cache",
|
|
2964
|
+
...event.costAvoidedUsd === void 0 ? {} : { savedUsd: event.costAvoidedUsd }
|
|
2965
|
+
});
|
|
2966
|
+
}
|
|
2967
|
+
if (priced?.tierSavedUsd !== void 0) {
|
|
2968
|
+
levers.push({
|
|
2969
|
+
leverId: `execution-tier-${executionMode}`,
|
|
2970
|
+
kind: "tier",
|
|
2971
|
+
savedUsd: priced.tierSavedUsd
|
|
2972
|
+
});
|
|
2973
|
+
}
|
|
2974
|
+
if (priced?.cacheAttributedUsd !== void 0 && ttlChange !== void 0) {
|
|
2975
|
+
levers.push({
|
|
2976
|
+
leverId: ttlChange.lever,
|
|
2977
|
+
kind: "provider_cache",
|
|
2978
|
+
savedUsd: priced.cacheAttributedUsd
|
|
2979
|
+
});
|
|
2980
|
+
}
|
|
2981
|
+
if (priced?.providerCacheAlreadyOwnedUsd !== void 0) {
|
|
2982
|
+
excluded.providerCacheAlreadyOwnedUsd = priced.providerCacheAlreadyOwnedUsd;
|
|
2983
|
+
}
|
|
2984
|
+
let baselineUsd;
|
|
2985
|
+
try {
|
|
2986
|
+
const supplied = config.getBaselineUsd?.(event);
|
|
2987
|
+
if (typeof supplied === "number" && Number.isFinite(supplied) && supplied >= 0) baselineUsd = supplied;
|
|
2988
|
+
} catch {
|
|
2989
|
+
baselineUsd = void 0;
|
|
2990
|
+
}
|
|
2991
|
+
const tokenChanges = changes.filter((change) => !EXECUTION_TIER_LEVERS.has(change.lever) && !(CACHE_TTL_LEVERS.has(change.lever) && priced?.cacheAttributedUsd !== void 0));
|
|
2992
|
+
const summary = event.reduction && !event.reduction.skipped ? event.reduction : void 0;
|
|
2993
|
+
const estimatedBefore = summary?.estimatedTokensBefore ?? 0;
|
|
2994
|
+
const estimatedRemoved = tokenChanges.reduce((sum, change) => sum + (Number.isFinite(change.estimatedTokensSaved) && change.estimatedTokensSaved > 0 ? change.estimatedTokensSaved : 0), 0);
|
|
2995
|
+
const removedFraction = estimatedBefore > 0 ? estimatedRemoved / estimatedBefore : 0;
|
|
2996
|
+
const estimateUsable = baselineUsd === void 0 && priced !== void 0 && usage !== void 0 && usage.inputTokens > 0 && estimatedRemoved > 0 && removedFraction > 0 && removedFraction < 0.95;
|
|
2997
|
+
const estimatedSavedUsdTotal = estimateUsable ? (usage.inputTokens / (1 - removedFraction) - usage.inputTokens) * (priced.inputUsd / usage.inputTokens) : 0;
|
|
2998
|
+
for (const change of tokenChanges) {
|
|
2999
|
+
const share = estimateUsable && change.estimatedTokensSaved > 0 ? estimatedSavedUsdTotal * (change.estimatedTokensSaved / estimatedRemoved) : void 0;
|
|
3000
|
+
levers.push({
|
|
3001
|
+
leverId: change.lever,
|
|
3002
|
+
kind: change.lossless ? "lossless" : "lossy",
|
|
3003
|
+
...share === void 0 ? {} : { savedUsd: share, basis: "estimated-baseline" },
|
|
3004
|
+
...config.gateVerdicts?.[change.lever] === void 0 ? {} : { gateVerdict: config.gateVerdicts[change.lever] }
|
|
3005
|
+
});
|
|
3006
|
+
}
|
|
3007
|
+
for (const change of changes) {
|
|
3008
|
+
if (EXECUTION_TIER_LEVERS.has(change.lever) && priced?.tierSavedUsd === void 0) {
|
|
3009
|
+
levers.push({ leverId: change.lever, kind: "lossless" });
|
|
3010
|
+
}
|
|
3011
|
+
}
|
|
3012
|
+
const measuredTokenChanges = changes.filter((change) => !EXECUTION_TIER_LEVERS.has(change.lever) && !CACHE_TTL_LEVERS.has(change.lever));
|
|
3013
|
+
if (baselineUsd !== void 0 && priced !== void 0 && measuredTokenChanges.length > 0) {
|
|
3014
|
+
const lossy = measuredTokenChanges.filter((change) => !change.lossless);
|
|
3015
|
+
const verdicts = lossy.map((change) => config.gateVerdicts?.[change.lever]);
|
|
3016
|
+
const allApproved = verdicts.every((verdict) => verdict?.verdict === "approved");
|
|
3017
|
+
const savedUsd = baselineUsd - priced.actualUsd - (priced.tierSavedUsd ?? 0) - (priced.cacheAttributedUsd ?? 0);
|
|
3018
|
+
levers.push({
|
|
3019
|
+
leverId: MEASURED_COMPOSITE_LEVER,
|
|
3020
|
+
kind: lossy.length > 0 ? "lossy" : "lossless",
|
|
3021
|
+
savedUsd,
|
|
3022
|
+
...lossy.length > 0 && allApproved ? {
|
|
3023
|
+
gateVerdict: {
|
|
3024
|
+
ref: verdicts.map((verdict) => verdict.ref).sort().join("+"),
|
|
3025
|
+
verdict: "approved"
|
|
3026
|
+
}
|
|
3027
|
+
} : {},
|
|
3028
|
+
detail: "baselineUsd minus actual catalogue cost, net of tier and cache components"
|
|
3029
|
+
});
|
|
3030
|
+
}
|
|
3031
|
+
let correlation;
|
|
3032
|
+
try {
|
|
3033
|
+
correlation = config.getCorrelation?.(event);
|
|
3034
|
+
} catch {
|
|
3035
|
+
correlation = void 0;
|
|
3036
|
+
}
|
|
3037
|
+
return {
|
|
3038
|
+
schemaVersion: 1,
|
|
3039
|
+
requestId: event.requestId,
|
|
3040
|
+
applicationId: event.applicationId,
|
|
3041
|
+
...correlation?.taskId === void 0 ? {} : { taskId: correlation.taskId },
|
|
3042
|
+
...correlation?.turnIndex === void 0 ? {} : { turnIndex: correlation.turnIndex },
|
|
3043
|
+
invoicePeriod: event.timestamp.slice(0, 7),
|
|
3044
|
+
timestamp: event.timestamp,
|
|
3045
|
+
provider,
|
|
3046
|
+
model: event.model,
|
|
3047
|
+
executionMode,
|
|
3048
|
+
pricingStatus: pricing.pricingStatus,
|
|
3049
|
+
...pricing.pricingStatus === "unpriced" ? { pricingReason: pricing.reason } : {},
|
|
3050
|
+
catalogueVersion: pinnedCatalogueVersion,
|
|
3051
|
+
...usage === void 0 ? {} : { usage },
|
|
3052
|
+
...priced === void 0 ? {} : { actualUsd: priced.actualUsd },
|
|
3053
|
+
...baselineUsd === void 0 ? {} : { baselineUsd },
|
|
3054
|
+
...event.costUsd === void 0 ? {} : { reportedCostUsd: event.costUsd },
|
|
3055
|
+
levers,
|
|
3056
|
+
...estimateUsable ? {
|
|
3057
|
+
estimatedReduction: {
|
|
3058
|
+
estimatedInputTokensBefore: estimatedBefore,
|
|
3059
|
+
estimatedInputTokensRemoved: estimatedRemoved
|
|
3060
|
+
}
|
|
3061
|
+
} : {},
|
|
3062
|
+
...Object.keys(excluded).length === 0 ? {} : { excluded },
|
|
3063
|
+
...event.outputDiagnostics === void 0 ? {} : { outputDiagnostics: event.outputDiagnostics },
|
|
3064
|
+
...event.duplicateOfRequestId === void 0 ? {} : { duplicateOfRequestId: event.duplicateOfRequestId },
|
|
3065
|
+
status: event.status
|
|
3066
|
+
};
|
|
3067
|
+
}
|
|
3068
|
+
function resumeChainTail(path) {
|
|
3069
|
+
try {
|
|
3070
|
+
if (!existsSync(path)) return { disabled: false };
|
|
3071
|
+
const lines = readFileSync(path, "utf8").split("\n").filter((line) => line.trim() !== "");
|
|
3072
|
+
if (lines.length === 0) return { disabled: false };
|
|
3073
|
+
const tail = JSON.parse(lines[lines.length - 1]);
|
|
3074
|
+
if (typeof tail.seq !== "number" || typeof tail.eventHash !== "string") return { disabled: true };
|
|
3075
|
+
return { previous: tail, disabled: false };
|
|
3076
|
+
} catch {
|
|
3077
|
+
return { disabled: true };
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
function createSavingsLedger(config) {
|
|
3081
|
+
if (typeof config.path !== "string" || config.path.trim() === "") {
|
|
3082
|
+
throw new Error("The savings ledger needs a file path for its append-only JSONL spool.");
|
|
3083
|
+
}
|
|
3084
|
+
const catalogue = config.catalogue ?? publishedModelPrices;
|
|
3085
|
+
const pinnedCatalogueVersion = catalogueVersion(catalogue);
|
|
3086
|
+
const tail = resumeChainTail(config.path);
|
|
3087
|
+
let previous = tail.previous;
|
|
3088
|
+
let disabled = tail.disabled;
|
|
3089
|
+
return {
|
|
3090
|
+
path: config.path,
|
|
3091
|
+
emit(event) {
|
|
3092
|
+
if (disabled) return;
|
|
3093
|
+
try {
|
|
3094
|
+
const billable = billableEventFromRequest(event, config, catalogue, pinnedCatalogueVersion);
|
|
3095
|
+
const record = chainLedgerRecord(billable, previous);
|
|
3096
|
+
mkdirSync(dirname(config.path), { recursive: true, mode: 448 });
|
|
3097
|
+
appendFileSync(config.path, `${JSON.stringify(record)}
|
|
3098
|
+
`, { encoding: "utf8", mode: 384 });
|
|
3099
|
+
previous = record;
|
|
3100
|
+
} catch {
|
|
3101
|
+
disabled = true;
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
|
|
3107
|
+
// packages/sdk-js/src/openai.ts
|
|
3108
|
+
function routeCapField(surface) {
|
|
3109
|
+
if (surface === "chat_completions") return "max_completion_tokens";
|
|
3110
|
+
if (surface === "anthropic_messages") return "max_tokens";
|
|
3111
|
+
return "max_output_tokens";
|
|
3112
|
+
}
|
|
3113
|
+
function validRate(value) {
|
|
3114
|
+
return value === void 0 || typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
3115
|
+
}
|
|
3116
|
+
function tokenUsage(response) {
|
|
3117
|
+
const result = normalizeProviderUsage(response);
|
|
3118
|
+
return result.status === "ok" ? result.usage : void 0;
|
|
3119
|
+
}
|
|
3120
|
+
function serviceTierOf(response) {
|
|
3121
|
+
return responseServiceTier(response);
|
|
3122
|
+
}
|
|
3123
|
+
function usageCost(usage, pricing) {
|
|
3124
|
+
if (!usage || !pricing || usage.cachedInputTokens > 0 && pricing.cachedInputPerMillion === void 0 || usage.cacheWriteInputTokens > 0 && pricing.cacheWriteInputPerMillion === void 0) return void 0;
|
|
3125
|
+
const write1h = usage.cacheWrite1hInputTokens ?? 0;
|
|
3126
|
+
if (write1h > 0 && pricing.cacheWrite1hInputPerMillion === void 0) return void 0;
|
|
3127
|
+
const write5m = usage.cacheWriteInputTokens - write1h;
|
|
3128
|
+
return usage.uncachedInputTokens / 1e6 * pricing.inputPerMillion + usage.cachedInputTokens / 1e6 * (pricing.cachedInputPerMillion ?? 0) + write5m / 1e6 * (pricing.cacheWriteInputPerMillion ?? 0) + write1h / 1e6 * (pricing.cacheWrite1hInputPerMillion ?? 0) + usage.outputTokens / 1e6 * pricing.outputPerMillion;
|
|
3129
|
+
}
|
|
3130
|
+
function matchesContextProfile(profile, context) {
|
|
3131
|
+
return Object.entries(profile.match ?? {}).every(
|
|
3132
|
+
([key, value]) => context[key] === value
|
|
3133
|
+
);
|
|
3134
|
+
}
|
|
3135
|
+
function withMesh(client, options) {
|
|
3136
|
+
const profileIds = /* @__PURE__ */ new Set();
|
|
3137
|
+
for (const profile of options.contextProfiles ?? []) {
|
|
3138
|
+
if (!profile.id || !profile.version || typeof profile.transform !== "function" || profileIds.has(profile.id)) {
|
|
3139
|
+
throw new Error("MESH context profiles need unique IDs, versions, and transform functions.");
|
|
3140
|
+
}
|
|
3141
|
+
profileIds.add(profile.id);
|
|
3142
|
+
}
|
|
3143
|
+
if (options.pricing && (!Number.isFinite(options.pricing.inputPerMillion) || options.pricing.inputPerMillion < 0 || !Number.isFinite(options.pricing.outputPerMillion) || options.pricing.outputPerMillion < 0 || !validRate(options.pricing.cachedInputPerMillion) || !validRate(options.pricing.cacheWriteInputPerMillion) || !validRate(options.pricing.cacheWrite1hInputPerMillion))) {
|
|
3144
|
+
throw new Error("MESH pricing rates must be finite, non-negative USD per million tokens.");
|
|
3145
|
+
}
|
|
3146
|
+
for (const [model, pricing] of Object.entries(options.pricingByModel ?? {})) {
|
|
3147
|
+
if (!model || !Number.isFinite(pricing.inputPerMillion) || pricing.inputPerMillion < 0 || !Number.isFinite(pricing.outputPerMillion) || pricing.outputPerMillion < 0 || !validRate(pricing.cachedInputPerMillion) || !validRate(pricing.cacheWriteInputPerMillion) || !validRate(pricing.cacheWrite1hInputPerMillion)) {
|
|
3148
|
+
throw new Error("MESH per-model pricing rates must be finite, non-negative USD per million tokens.");
|
|
3149
|
+
}
|
|
3150
|
+
}
|
|
3151
|
+
const originalResponses = client.responses;
|
|
3152
|
+
const originalChat = client.chat;
|
|
3153
|
+
const originalChatCompletions = originalChat && typeof originalChat === "object" ? originalChat.completions : void 0;
|
|
3154
|
+
const originalMessages = client.messages;
|
|
3155
|
+
const hasResponses = typeof originalResponses?.create === "function";
|
|
3156
|
+
const hasChat = typeof originalChatCompletions?.create === "function";
|
|
3157
|
+
const hasMessages = typeof originalMessages?.create === "function";
|
|
3158
|
+
if (!hasResponses && !hasChat && !hasMessages) {
|
|
3159
|
+
throw new Error(
|
|
3160
|
+
"MESH supports OpenAI-compatible clients with responses.create() or chat.completions.create(), and Anthropic-SDK-shaped clients with messages.create()."
|
|
3161
|
+
);
|
|
3162
|
+
}
|
|
3163
|
+
const telemetry = new TelemetryClient(options.telemetry);
|
|
3164
|
+
const exactCache = options.exactCache ? new ExactResponseCache(options.exactCache) : void 0;
|
|
3165
|
+
const ledger = options.ledger ? createSavingsLedger(options.ledger) : void 0;
|
|
3166
|
+
const tokenReduction = options.tokenReduction?.cacheTtl !== void 0 && options.tokenReduction.cacheVolatilityGuard === void 0 ? { ...options.tokenReduction, cacheVolatilityGuard: new CachePrefixVolatilityGuard() } : options.tokenReduction;
|
|
3167
|
+
const surfaceCreate = (surface, target) => async (request, ...rest) => {
|
|
3168
|
+
const requestId = randomUUID();
|
|
3169
|
+
const started = performance.now();
|
|
3170
|
+
let context = {};
|
|
3171
|
+
try {
|
|
3172
|
+
context = options.getContext?.(request) ?? {};
|
|
3173
|
+
} catch {
|
|
3174
|
+
}
|
|
3175
|
+
let taskClass = context.taskClass;
|
|
3176
|
+
if (!taskClass) {
|
|
3177
|
+
try {
|
|
3178
|
+
taskClass = options.classify?.(request);
|
|
3179
|
+
} catch {
|
|
3180
|
+
taskClass = void 0;
|
|
3181
|
+
}
|
|
3182
|
+
}
|
|
3183
|
+
context = { ...context, taskClass };
|
|
3184
|
+
const model = typeof request?.model === "string" ? request.model : "unknown";
|
|
3185
|
+
const reduction = tokenReduction ? reduceRequest(request, tokenReduction) : void 0;
|
|
3186
|
+
const effectiveRequest = reduction?.request ?? request;
|
|
3187
|
+
const route = chooseRoute(options.routing, effectiveRequest, taskClass, context);
|
|
3188
|
+
const candidateSelected = route.reason === "candidate_quality_and_cost_gates_passed";
|
|
3189
|
+
let fallbackReason;
|
|
3190
|
+
let routedRequest = candidateSelected ? { ...effectiveRequest, model: route.model } : effectiveRequest;
|
|
3191
|
+
const contextProfile = candidateSelected && route.contextProfileId ? options.contextProfiles?.find((profile) => profile.id === route.contextProfileId) : void 0;
|
|
3192
|
+
if (candidateSelected && route.contextProfileId) {
|
|
3193
|
+
const transform = contextProfile?.transform ?? options.contextTransforms?.[route.contextProfileId];
|
|
3194
|
+
if (contextProfile && !matchesContextProfile(contextProfile, context)) {
|
|
3195
|
+
fallbackReason = "context_profile_mismatch";
|
|
3196
|
+
} else if (!transform) {
|
|
3197
|
+
fallbackReason = "context_transform_unavailable";
|
|
3198
|
+
} else {
|
|
3199
|
+
try {
|
|
3200
|
+
const transformed = transform(effectiveRequest, context);
|
|
3201
|
+
if (!transformed || typeof transformed !== "object" || Array.isArray(transformed)) {
|
|
3202
|
+
throw new Error("Context transform must return a request object.");
|
|
3203
|
+
}
|
|
3204
|
+
routedRequest = { ...transformed, model: route.model };
|
|
3205
|
+
} catch {
|
|
3206
|
+
fallbackReason = "context_transform_error";
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
if (candidateSelected && route.maxOutputTokens !== void 0) {
|
|
3211
|
+
const capField = routeCapField(surface);
|
|
3212
|
+
const requestedCap = request[capField];
|
|
3213
|
+
routedRequest = {
|
|
3214
|
+
...routedRequest,
|
|
3215
|
+
[capField]: typeof requestedCap === "number" && Number.isFinite(requestedCap) ? Math.min(requestedCap, route.maxOutputTokens) : route.maxOutputTokens
|
|
3216
|
+
};
|
|
3217
|
+
}
|
|
3218
|
+
let cacheKey;
|
|
3219
|
+
if (exactCache && rest.length === 0 && fallbackReason === void 0) {
|
|
3220
|
+
try {
|
|
3221
|
+
const scopeKey = options.exactCache?.getScopeKey(request);
|
|
3222
|
+
cacheKey = exactCache.keyFor(routedRequest, scopeKey);
|
|
3223
|
+
} catch {
|
|
3224
|
+
cacheKey = void 0;
|
|
3225
|
+
}
|
|
3226
|
+
}
|
|
3227
|
+
const cached = cacheKey ? exactCache?.get(cacheKey) : void 0;
|
|
3228
|
+
const cacheHit = cached !== void 0;
|
|
3229
|
+
let status = "succeeded";
|
|
3230
|
+
let error;
|
|
3231
|
+
let response;
|
|
3232
|
+
let finalModel = cached?.model ?? route.model;
|
|
3233
|
+
let qualityCheckPassed;
|
|
3234
|
+
const attempts = [];
|
|
3235
|
+
const create = Reflect.get(target, "create", target);
|
|
3236
|
+
try {
|
|
3237
|
+
if (cacheHit) {
|
|
3238
|
+
response = cached.response;
|
|
3239
|
+
} else if (fallbackReason === void 0) {
|
|
3240
|
+
attempts.push({
|
|
3241
|
+
model: route.model,
|
|
3242
|
+
request: routedRequest,
|
|
3243
|
+
contextProfileId: candidateSelected ? route.contextProfileId : void 0,
|
|
3244
|
+
maxOutputTokens: candidateSelected ? route.maxOutputTokens : void 0
|
|
3245
|
+
});
|
|
3246
|
+
try {
|
|
3247
|
+
response = await create.call(target, routedRequest, ...rest);
|
|
3248
|
+
attempts[attempts.length - 1].response = response;
|
|
3249
|
+
} catch (cause) {
|
|
3250
|
+
attempts[attempts.length - 1].error = cause instanceof Error ? cause.message : String(cause);
|
|
3251
|
+
if (!candidateSelected) throw cause;
|
|
3252
|
+
fallbackReason = "candidate_error";
|
|
3253
|
+
}
|
|
3254
|
+
}
|
|
3255
|
+
if (candidateSelected && fallbackReason === void 0 && options.qualityCheck) {
|
|
3256
|
+
try {
|
|
3257
|
+
qualityCheckPassed = await options.qualityCheck({
|
|
3258
|
+
request: effectiveRequest,
|
|
3259
|
+
response,
|
|
3260
|
+
taskClass,
|
|
3261
|
+
context,
|
|
3262
|
+
candidateModel: route.model
|
|
3263
|
+
});
|
|
3264
|
+
if (qualityCheckPassed !== true) fallbackReason = "quality_rejected";
|
|
3265
|
+
} catch {
|
|
3266
|
+
qualityCheckPassed = false;
|
|
3267
|
+
fallbackReason = "quality_check_error";
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
if (candidateSelected && fallbackReason !== void 0) {
|
|
3271
|
+
finalModel = model;
|
|
3272
|
+
attempts.push({ model, request: effectiveRequest });
|
|
3273
|
+
response = await create.call(target, effectiveRequest, ...rest);
|
|
3274
|
+
attempts[attempts.length - 1].response = response;
|
|
3275
|
+
}
|
|
3276
|
+
if (!cacheHit && fallbackReason === void 0 && cacheKey && response !== void 0) {
|
|
3277
|
+
const usage = tokenUsage(response);
|
|
3278
|
+
const pricing = options.pricingByModel?.[route.model] ?? (route.model === model ? options.pricing : void 0);
|
|
3279
|
+
const cost = usageCost(usage, pricing);
|
|
3280
|
+
exactCache?.set(cacheKey, {
|
|
3281
|
+
response,
|
|
3282
|
+
model: route.model,
|
|
3283
|
+
inputTokens: usage?.inputTokens,
|
|
3284
|
+
uncachedInputTokens: usage?.uncachedInputTokens,
|
|
3285
|
+
cachedInputTokens: usage?.cachedInputTokens,
|
|
3286
|
+
cacheWriteInputTokens: usage?.cacheWriteInputTokens,
|
|
3287
|
+
outputTokens: usage?.outputTokens,
|
|
3288
|
+
costUsd: cost
|
|
3289
|
+
});
|
|
3290
|
+
}
|
|
3291
|
+
return response;
|
|
3292
|
+
} catch (cause) {
|
|
3293
|
+
status = "failed";
|
|
3294
|
+
error = cause instanceof Error ? cause.message : String(cause);
|
|
3295
|
+
throw cause;
|
|
3296
|
+
} finally {
|
|
3297
|
+
const usages = attempts.map(({ response: attemptResponse }) => tokenUsage(attemptResponse));
|
|
3298
|
+
const hasAllUsages = attempts.length > 0 && usages.every((usage) => usage !== void 0);
|
|
3299
|
+
const splitKnownForAll = hasAllUsages && usages.some((usage) => usage.cacheWrite5mInputTokens !== void 0 || usage.cacheWrite1hInputTokens !== void 0) && usages.every((usage) => usage.cacheWriteInputTokens === 0 || (usage.cacheWrite5mInputTokens !== void 0 || usage.cacheWrite1hInputTokens !== void 0));
|
|
3300
|
+
const usageTotals = hasAllUsages ? usages.reduce((total, usage) => ({
|
|
3301
|
+
inputTokens: total.inputTokens + usage.inputTokens,
|
|
3302
|
+
uncachedInputTokens: total.uncachedInputTokens + usage.uncachedInputTokens,
|
|
3303
|
+
cachedInputTokens: total.cachedInputTokens + usage.cachedInputTokens,
|
|
3304
|
+
cacheWriteInputTokens: total.cacheWriteInputTokens + usage.cacheWriteInputTokens,
|
|
3305
|
+
...splitKnownForAll ? {
|
|
3306
|
+
cacheWrite5mInputTokens: (total.cacheWrite5mInputTokens ?? 0) + (usage.cacheWrite5mInputTokens ?? 0),
|
|
3307
|
+
cacheWrite1hInputTokens: (total.cacheWrite1hInputTokens ?? 0) + (usage.cacheWrite1hInputTokens ?? 0)
|
|
3308
|
+
} : {},
|
|
3309
|
+
outputTokens: total.outputTokens + usage.outputTokens
|
|
3310
|
+
}), { inputTokens: 0, uncachedInputTokens: 0, cachedInputTokens: 0, cacheWriteInputTokens: 0, outputTokens: 0 }) : void 0;
|
|
3311
|
+
const emptyCacheUsage = { inputTokens: 0, uncachedInputTokens: 0, cachedInputTokens: 0, cacheWriteInputTokens: 0, outputTokens: 0 };
|
|
3312
|
+
const actualUsage = cacheHit && attempts.length === 0 ? emptyCacheUsage : usageTotals;
|
|
3313
|
+
const inputTokens = actualUsage?.inputTokens;
|
|
3314
|
+
const outputTokens = actualUsage?.outputTokens;
|
|
3315
|
+
const attemptCosts = attempts.map(({ model: attemptedModel, response: attemptResponse }) => {
|
|
3316
|
+
const pricing = options.pricingByModel?.[attemptedModel] ?? (attemptedModel === model ? options.pricing : void 0);
|
|
3317
|
+
return usageCost(tokenUsage(attemptResponse), pricing);
|
|
3318
|
+
});
|
|
3319
|
+
const costUsd = cacheHit && attempts.length === 0 ? 0 : attemptCosts.length > 0 && attemptCosts.every((cost) => cost !== void 0) ? attemptCosts.reduce((sum, cost) => sum + cost, 0) : void 0;
|
|
3320
|
+
const attemptServiceTiers = attempts.map(({ response: attemptResponse }) => serviceTierOf(attemptResponse) ?? null);
|
|
3321
|
+
const responseServiceTier2 = attempts.length > 0 ? serviceTierOf(attempts[attempts.length - 1].response) : void 0;
|
|
3322
|
+
const event = {
|
|
3323
|
+
requestId,
|
|
3324
|
+
applicationId: options.applicationId,
|
|
3325
|
+
...reduction === void 0 ? {} : { reduction: reduction.summary },
|
|
3326
|
+
provider: surface === "anthropic_messages" ? "anthropic-compatible" : "openai-compatible",
|
|
3327
|
+
model: finalModel,
|
|
3328
|
+
requestedModel: candidateSelected ? model : void 0,
|
|
3329
|
+
taskClass,
|
|
3330
|
+
workflow: context.workflow,
|
|
3331
|
+
difficulty: context.difficulty,
|
|
3332
|
+
riskLevel: context.riskLevel,
|
|
3333
|
+
contextProfileId: candidateSelected ? route.contextProfileId : void 0,
|
|
3334
|
+
contextProfileVersion: contextProfile?.version,
|
|
3335
|
+
routePolicyId: route.policyId,
|
|
3336
|
+
routeReason: fallbackReason ? "fallback_to_baseline" : cacheHit ? "exact_cache_hit" : route.reason,
|
|
3337
|
+
estimatedCostSavingsPercent: fallbackReason ? void 0 : route.estimatedCostSavingsPercent,
|
|
3338
|
+
modelsAttempted: attempts.map(({ model: attemptedModel }) => attemptedModel),
|
|
3339
|
+
fallbackUsed: fallbackReason !== void 0,
|
|
3340
|
+
fallbackReason,
|
|
3341
|
+
qualityCheckPassed,
|
|
3342
|
+
cacheHit,
|
|
3343
|
+
tokensAvoided: cacheHit ? (cached.inputTokens ?? 0) + (cached.outputTokens ?? 0) : void 0,
|
|
3344
|
+
costAvoidedUsd: cacheHit ? cached.costUsd : void 0,
|
|
3345
|
+
inputTokens,
|
|
3346
|
+
outputTokens,
|
|
3347
|
+
uncachedInputTokens: actualUsage?.uncachedInputTokens,
|
|
3348
|
+
cachedInputTokens: actualUsage?.cachedInputTokens,
|
|
3349
|
+
cacheWriteInputTokens: actualUsage?.cacheWriteInputTokens,
|
|
3350
|
+
// Provider-reported TTL split of the cache writes; absent when the
|
|
3351
|
+
// provider did not report one, so billing can fall back dispute-safe.
|
|
3352
|
+
...actualUsage !== void 0 && (actualUsage.cacheWrite5mInputTokens !== void 0 || actualUsage.cacheWrite1hInputTokens !== void 0) ? {
|
|
3353
|
+
cacheWrite5mInputTokens: actualUsage.cacheWrite5mInputTokens ?? 0,
|
|
3354
|
+
cacheWrite1hInputTokens: actualUsage.cacheWrite1hInputTokens ?? 0
|
|
3355
|
+
} : {},
|
|
3356
|
+
costUsd,
|
|
3357
|
+
responseServiceTier: responseServiceTier2,
|
|
3358
|
+
...attempts.length > 0 ? { attemptServiceTiers } : {},
|
|
3359
|
+
latencyMs: performance.now() - started,
|
|
3360
|
+
status,
|
|
3361
|
+
error,
|
|
3362
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3363
|
+
};
|
|
3364
|
+
try {
|
|
3365
|
+
ledger?.emit(event);
|
|
3366
|
+
} catch {
|
|
3367
|
+
}
|
|
3368
|
+
void Promise.resolve().then(() => options.onEvent?.(event)).catch(() => {
|
|
3369
|
+
});
|
|
3370
|
+
void telemetry.emit(event);
|
|
3371
|
+
const capture = options.evaluationCapture;
|
|
3372
|
+
if (capture) {
|
|
3373
|
+
void Promise.resolve().then(() => {
|
|
3374
|
+
let caseId = requestId;
|
|
3375
|
+
let groupId;
|
|
3376
|
+
try {
|
|
3377
|
+
const supplied = capture.getCaseId?.(request);
|
|
3378
|
+
if (typeof supplied === "string" && supplied.trim() !== "") caseId = supplied;
|
|
3379
|
+
} catch {
|
|
3380
|
+
}
|
|
3381
|
+
try {
|
|
3382
|
+
const supplied = capture.getGroupId?.(request);
|
|
3383
|
+
if (typeof supplied === "string" && supplied.trim() !== "") groupId = supplied;
|
|
3384
|
+
} catch {
|
|
3385
|
+
}
|
|
3386
|
+
let expected;
|
|
3387
|
+
try {
|
|
3388
|
+
expected = capture.getExpected?.(request);
|
|
3389
|
+
} catch {
|
|
3390
|
+
expected = void 0;
|
|
3391
|
+
}
|
|
3392
|
+
const captureAttempts = (attempts.length > 0 ? attempts : cacheHit ? [{ model: finalModel, request: routedRequest, response, contextProfileId: candidateSelected ? route.contextProfileId : void 0, maxOutputTokens: candidateSelected ? route.maxOutputTokens : void 0 }] : []).map((attempt) => {
|
|
3393
|
+
const usage = tokenUsage(attempt.response);
|
|
3394
|
+
const pricing = options.pricingByModel?.[attempt.model] ?? (attempt.model === model ? options.pricing : void 0);
|
|
3395
|
+
const attemptCapture = {
|
|
3396
|
+
model: attempt.model,
|
|
3397
|
+
contextProfileId: attempt.contextProfileId,
|
|
3398
|
+
maxOutputTokens: attempt.maxOutputTokens,
|
|
3399
|
+
request: attempt.request,
|
|
3400
|
+
response: attempt.response,
|
|
3401
|
+
error: attempt.error,
|
|
3402
|
+
inputTokens: cacheHit && attempts.length === 0 ? 0 : usage?.inputTokens,
|
|
3403
|
+
outputTokens: cacheHit && attempts.length === 0 ? 0 : usage?.outputTokens,
|
|
3404
|
+
costUsd: cacheHit && attempts.length === 0 ? 0 : usageCost(usage, pricing)
|
|
3405
|
+
};
|
|
3406
|
+
if (expected !== void 0 && attempt.response !== void 0 && capture.score) {
|
|
3407
|
+
try {
|
|
3408
|
+
const score = capture.score(expected, attempt.response, {
|
|
3409
|
+
model: attempt.model,
|
|
3410
|
+
workflow: context.workflow,
|
|
3411
|
+
taskClass
|
|
3412
|
+
});
|
|
3413
|
+
if (Number.isFinite(score) && score >= 0 && score <= 1) attemptCapture.score = score;
|
|
3414
|
+
} catch {
|
|
3415
|
+
}
|
|
3416
|
+
}
|
|
3417
|
+
return attemptCapture;
|
|
3418
|
+
});
|
|
3419
|
+
const rawRecord = {
|
|
3420
|
+
schemaVersion: 1,
|
|
3421
|
+
id: requestId,
|
|
3422
|
+
caseId,
|
|
3423
|
+
...groupId ? { groupId } : {},
|
|
3424
|
+
applicationId: options.applicationId,
|
|
3425
|
+
requestedModel: model,
|
|
3426
|
+
finalModel,
|
|
3427
|
+
taskClass,
|
|
3428
|
+
workflow: context.workflow,
|
|
3429
|
+
difficulty: context.difficulty,
|
|
3430
|
+
riskLevel: context.riskLevel,
|
|
3431
|
+
request,
|
|
3432
|
+
...expected !== void 0 ? { expected } : {},
|
|
3433
|
+
attempts: captureAttempts,
|
|
3434
|
+
cacheHit,
|
|
3435
|
+
routeReason: fallbackReason ? "fallback_to_baseline" : cacheHit ? "exact_cache_hit" : route.reason,
|
|
3436
|
+
status,
|
|
3437
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
3438
|
+
};
|
|
3439
|
+
const redacted = capture.redact(rawRecord);
|
|
3440
|
+
return redacted ? capture.write(redacted) : void 0;
|
|
3441
|
+
}).catch(() => {
|
|
3442
|
+
});
|
|
3443
|
+
}
|
|
3444
|
+
}
|
|
3445
|
+
};
|
|
3446
|
+
const wrapCreate = (surface, surfaceTarget) => new Proxy(surfaceTarget, {
|
|
3447
|
+
get(target, property2, receiver) {
|
|
3448
|
+
if (property2 !== "create") return Reflect.get(target, property2, receiver);
|
|
3449
|
+
return surfaceCreate(surface, target);
|
|
3450
|
+
}
|
|
3451
|
+
});
|
|
3452
|
+
const wrappedResponses = hasResponses ? wrapCreate("responses", originalResponses) : void 0;
|
|
3453
|
+
const wrappedChatCompletions = hasChat ? wrapCreate("chat_completions", originalChatCompletions) : void 0;
|
|
3454
|
+
const wrappedChat = hasChat ? new Proxy(originalChat, {
|
|
3455
|
+
get(target, property2, receiver) {
|
|
3456
|
+
if (property2 === "completions") return wrappedChatCompletions;
|
|
3457
|
+
return Reflect.get(target, property2, receiver);
|
|
3458
|
+
}
|
|
3459
|
+
}) : void 0;
|
|
3460
|
+
const wrappedMessages = hasMessages ? wrapCreate("anthropic_messages", originalMessages) : void 0;
|
|
3461
|
+
return new Proxy(client, {
|
|
3462
|
+
get(target, property2, receiver) {
|
|
3463
|
+
if (property2 === "responses" && wrappedResponses !== void 0) return wrappedResponses;
|
|
3464
|
+
if (property2 === "chat" && wrappedChat !== void 0) return wrappedChat;
|
|
3465
|
+
if (property2 === "messages" && wrappedMessages !== void 0) return wrappedMessages;
|
|
3466
|
+
return Reflect.get(target, property2, receiver);
|
|
3467
|
+
}
|
|
3468
|
+
});
|
|
3469
|
+
}
|
|
3470
|
+
|
|
3471
|
+
// packages/sdk-js/src/evaluation-capture.ts
|
|
3472
|
+
import { appendFile, mkdir } from "node:fs/promises";
|
|
3473
|
+
import { dirname as dirname2, resolve } from "node:path";
|
|
3474
|
+
function createLocalJsonlCapture(path, redact, hooks = {}) {
|
|
3475
|
+
if (!path.trim() || typeof redact !== "function") {
|
|
3476
|
+
throw new Error("Local evaluation capture requires a path and a redaction function.");
|
|
3477
|
+
}
|
|
3478
|
+
const outputPath = resolve(path);
|
|
3479
|
+
let queue = Promise.resolve();
|
|
3480
|
+
const write = (record) => {
|
|
3481
|
+
queue = queue.catch(() => void 0).then(async () => {
|
|
3482
|
+
await mkdir(dirname2(outputPath), { recursive: true, mode: 448 });
|
|
3483
|
+
await appendFile(outputPath, `${JSON.stringify(record)}
|
|
3484
|
+
`, { encoding: "utf8", mode: 384 });
|
|
3485
|
+
});
|
|
3486
|
+
return queue;
|
|
3487
|
+
};
|
|
3488
|
+
return { ...hooks, redact, write, flush: () => queue };
|
|
3489
|
+
}
|
|
3490
|
+
|
|
3491
|
+
// packages/sdk-js/src/proxy.ts
|
|
3492
|
+
import { createServer } from "node:http";
|
|
3493
|
+
import { createHash as createHash8, randomUUID as randomUUID2 } from "node:crypto";
|
|
3494
|
+
import { appendFileSync as appendFileSync2, mkdirSync as mkdirSync2 } from "node:fs";
|
|
3495
|
+
import { join } from "node:path";
|
|
3496
|
+
|
|
3497
|
+
// packages/sdk-js/src/replay-gate.ts
|
|
3498
|
+
function isPlainObject7(value) {
|
|
3499
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3500
|
+
}
|
|
3501
|
+
function replayUsage(response) {
|
|
3502
|
+
const result = normalizeProviderUsage(response);
|
|
3503
|
+
if (result.status !== "ok") return void 0;
|
|
3504
|
+
const { inputTokens, outputTokens, cachedInputTokens, cacheWriteInputTokens } = result.usage;
|
|
3505
|
+
return { inputTokens, outputTokens, cachedInputTokens, cacheWriteInputTokens };
|
|
3506
|
+
}
|
|
3507
|
+
function responseText(response) {
|
|
3508
|
+
if (!isPlainObject7(response)) return "";
|
|
3509
|
+
if (typeof response.output_text === "string") return response.output_text;
|
|
3510
|
+
const parts = [];
|
|
3511
|
+
const walk = (value) => {
|
|
3512
|
+
if (Array.isArray(value)) {
|
|
3513
|
+
for (const item of value) walk(item);
|
|
3514
|
+
return;
|
|
3515
|
+
}
|
|
3516
|
+
if (!isPlainObject7(value)) return;
|
|
3517
|
+
if (typeof value.text === "string") parts.push(value.text);
|
|
3518
|
+
if (Array.isArray(value.content)) walk(value.content);
|
|
3519
|
+
if (isPlainObject7(value.message)) walk(value.message);
|
|
3520
|
+
};
|
|
3521
|
+
walk(response.output ?? response.choices ?? response.content ?? []);
|
|
3522
|
+
return parts.join("\n");
|
|
3523
|
+
}
|
|
3524
|
+
function canonicalJson3(value) {
|
|
3525
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson3).join(",")}]`;
|
|
3526
|
+
if (value !== null && typeof value === "object") {
|
|
3527
|
+
const entries = Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, nested]) => `${JSON.stringify(key)}:${canonicalJson3(nested)}`);
|
|
3528
|
+
return `{${entries.join(",")}}`;
|
|
3529
|
+
}
|
|
3530
|
+
return JSON.stringify(value) ?? "null";
|
|
3531
|
+
}
|
|
3532
|
+
function responseToolCalls(response) {
|
|
3533
|
+
const calls = [];
|
|
3534
|
+
const walk = (value) => {
|
|
3535
|
+
if (Array.isArray(value)) {
|
|
3536
|
+
for (const item of value) walk(item);
|
|
3537
|
+
return;
|
|
3538
|
+
}
|
|
3539
|
+
if (!isPlainObject7(value)) return;
|
|
3540
|
+
const name = typeof value.name === "string" ? value.name : void 0;
|
|
3541
|
+
const isCall = value.type === "function_call" || value.type === "tool_call" || isPlainObject7(value.function);
|
|
3542
|
+
if (isCall) {
|
|
3543
|
+
const fn = isPlainObject7(value.function) ? value.function : void 0;
|
|
3544
|
+
const callName = name ?? (fn && typeof fn.name === "string" ? fn.name : void 0);
|
|
3545
|
+
const rawArguments = value.arguments ?? (fn ? fn.arguments : void 0);
|
|
3546
|
+
if (callName !== void 0) {
|
|
3547
|
+
let argumentsJson = "";
|
|
3548
|
+
try {
|
|
3549
|
+
argumentsJson = typeof rawArguments === "string" ? canonicalJson3(JSON.parse(rawArguments)) : canonicalJson3(rawArguments ?? null);
|
|
3550
|
+
} catch {
|
|
3551
|
+
argumentsJson = typeof rawArguments === "string" ? rawArguments : "";
|
|
3552
|
+
}
|
|
3553
|
+
calls.push({ name: callName, argumentsJson });
|
|
3554
|
+
}
|
|
3555
|
+
}
|
|
3556
|
+
for (const nested of Object.values(value)) {
|
|
3557
|
+
if (Array.isArray(nested) || isPlainObject7(nested)) walk(nested);
|
|
3558
|
+
}
|
|
3559
|
+
};
|
|
3560
|
+
walk(response);
|
|
3561
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3562
|
+
return calls.filter((call) => {
|
|
3563
|
+
const key = `${call.name}|${call.argumentsJson}`;
|
|
3564
|
+
if (seen.has(key)) return false;
|
|
3565
|
+
seen.add(key);
|
|
3566
|
+
return true;
|
|
3567
|
+
});
|
|
3568
|
+
}
|
|
3569
|
+
var exactTextGrader = ({ baselineResponse, candidateResponse }) => responseText(baselineResponse).trim() === responseText(candidateResponse).trim() ? 1 : 0;
|
|
3570
|
+
var toolSelectionGrader = ({ baselineResponse, candidateResponse }) => {
|
|
3571
|
+
const baseline = responseToolCalls(baselineResponse);
|
|
3572
|
+
const candidate = responseToolCalls(candidateResponse);
|
|
3573
|
+
if (baseline.length === 0 && candidate.length === 0) return 1;
|
|
3574
|
+
const key = (call) => `${call.name}|${call.argumentsJson}`;
|
|
3575
|
+
const baselineKeys = new Set(baseline.map(key));
|
|
3576
|
+
const candidateKeys = new Set(candidate.map(key));
|
|
3577
|
+
const matched = [...baselineKeys].filter((value) => candidateKeys.has(value)).length;
|
|
3578
|
+
const union = (/* @__PURE__ */ new Set([...baselineKeys, ...candidateKeys])).size;
|
|
3579
|
+
return union === 0 ? 1 : matched / union;
|
|
3580
|
+
};
|
|
3581
|
+
var toolNameGrader = ({ baselineResponse, candidateResponse }) => {
|
|
3582
|
+
const names = (response) => new Set(responseToolCalls(response).map((call) => call.name));
|
|
3583
|
+
const baseline = names(baselineResponse);
|
|
3584
|
+
const candidate = names(candidateResponse);
|
|
3585
|
+
if (baseline.size === 0 && candidate.size === 0) return 1;
|
|
3586
|
+
const matched = [...baseline].filter((name) => candidate.has(name)).length;
|
|
3587
|
+
const union = (/* @__PURE__ */ new Set([...baseline, ...candidate])).size;
|
|
3588
|
+
return union === 0 ? 1 : matched / union;
|
|
3589
|
+
};
|
|
3590
|
+
async function replayGate(options) {
|
|
3591
|
+
const {
|
|
3592
|
+
requests,
|
|
3593
|
+
call,
|
|
3594
|
+
candidates,
|
|
3595
|
+
grade,
|
|
3596
|
+
maximumQualityDrop = 0.02,
|
|
3597
|
+
bootstrapSamples = 2e3,
|
|
3598
|
+
seed = "mesh-replay-gate-v1",
|
|
3599
|
+
price
|
|
3600
|
+
} = options;
|
|
3601
|
+
if (requests.length === 0) throw new Error("A replay gate needs at least one request.");
|
|
3602
|
+
if (candidates.length === 0) throw new Error("A replay gate needs at least one candidate configuration.");
|
|
3603
|
+
if (!Number.isFinite(maximumQualityDrop) || maximumQualityDrop < 0 || maximumQualityDrop > 1) {
|
|
3604
|
+
throw new Error("maximumQualityDrop must be between 0 and 1.");
|
|
3605
|
+
}
|
|
3606
|
+
if (options.baselineResponses !== void 0 && options.baselineResponses.length !== requests.length) {
|
|
3607
|
+
throw new Error("baselineResponses must line up one-to-one with requests.");
|
|
3608
|
+
}
|
|
3609
|
+
const notes = [];
|
|
3610
|
+
let calls = 0;
|
|
3611
|
+
const groups = requests.map((request, index) => options.groupOf?.(request, index) ?? `request-${index}`);
|
|
3612
|
+
if (!options.baselineResponses) await options.beforeRun?.("baseline");
|
|
3613
|
+
const baselineResponses = [];
|
|
3614
|
+
for (const [index, request] of requests.entries()) {
|
|
3615
|
+
if (options.baselineResponses) {
|
|
3616
|
+
baselineResponses.push(options.baselineResponses[index]);
|
|
3617
|
+
continue;
|
|
3618
|
+
}
|
|
3619
|
+
try {
|
|
3620
|
+
baselineResponses.push(await call(request));
|
|
3621
|
+
calls += 1;
|
|
3622
|
+
} catch {
|
|
3623
|
+
baselineResponses.push(void 0);
|
|
3624
|
+
}
|
|
3625
|
+
}
|
|
3626
|
+
if (options.baselineResponses) {
|
|
3627
|
+
notes.push("Baseline responses were supplied, so only the candidate runs were paid for.");
|
|
3628
|
+
}
|
|
3629
|
+
const perCandidate = [];
|
|
3630
|
+
for (const candidate of candidates) {
|
|
3631
|
+
await options.beforeRun?.(candidate.label);
|
|
3632
|
+
const deltas = new Array(requests.length).fill(0);
|
|
3633
|
+
let graded = 0;
|
|
3634
|
+
let failed = 0;
|
|
3635
|
+
let reductionSkipped = 0;
|
|
3636
|
+
const baselineUsage = {
|
|
3637
|
+
inputTokens: 0,
|
|
3638
|
+
outputTokens: 0,
|
|
3639
|
+
cachedInputTokens: 0,
|
|
3640
|
+
cacheWriteInputTokens: 0
|
|
3641
|
+
};
|
|
3642
|
+
const candidateUsage = {
|
|
3643
|
+
inputTokens: 0,
|
|
3644
|
+
outputTokens: 0,
|
|
3645
|
+
cachedInputTokens: 0,
|
|
3646
|
+
cacheWriteInputTokens: 0
|
|
3647
|
+
};
|
|
3648
|
+
for (const [index, request] of requests.entries()) {
|
|
3649
|
+
const baselineResponse = baselineResponses[index];
|
|
3650
|
+
const reduced = reduceRequest(request, candidate.config);
|
|
3651
|
+
if (reduced.summary.skipped) reductionSkipped += 1;
|
|
3652
|
+
let candidateResponse;
|
|
3653
|
+
try {
|
|
3654
|
+
candidateResponse = await call(reduced.request);
|
|
3655
|
+
calls += 1;
|
|
3656
|
+
} catch {
|
|
3657
|
+
failed += 1;
|
|
3658
|
+
continue;
|
|
3659
|
+
}
|
|
3660
|
+
if (baselineResponse === void 0) {
|
|
3661
|
+
failed += 1;
|
|
3662
|
+
continue;
|
|
3663
|
+
}
|
|
3664
|
+
const baseline = replayUsage(baselineResponse);
|
|
3665
|
+
const candidateUsageRow = replayUsage(candidateResponse);
|
|
3666
|
+
if (baseline && candidateUsageRow) {
|
|
3667
|
+
baselineUsage.inputTokens += baseline.inputTokens;
|
|
3668
|
+
baselineUsage.outputTokens += baseline.outputTokens;
|
|
3669
|
+
baselineUsage.cachedInputTokens += baseline.cachedInputTokens;
|
|
3670
|
+
baselineUsage.cacheWriteInputTokens += baseline.cacheWriteInputTokens;
|
|
3671
|
+
candidateUsage.inputTokens += candidateUsageRow.inputTokens;
|
|
3672
|
+
candidateUsage.outputTokens += candidateUsageRow.outputTokens;
|
|
3673
|
+
candidateUsage.cachedInputTokens += candidateUsageRow.cachedInputTokens;
|
|
3674
|
+
candidateUsage.cacheWriteInputTokens += candidateUsageRow.cacheWriteInputTokens;
|
|
3675
|
+
}
|
|
3676
|
+
try {
|
|
3677
|
+
const score = await grade({ request, baselineResponse, candidateResponse });
|
|
3678
|
+
if (!Number.isFinite(score)) throw new Error("Grader returned a non-finite score.");
|
|
3679
|
+
deltas[index] = Math.min(1, Math.max(0, score)) - 1;
|
|
3680
|
+
graded += 1;
|
|
3681
|
+
} catch {
|
|
3682
|
+
failed += 1;
|
|
3683
|
+
}
|
|
3684
|
+
}
|
|
3685
|
+
if (reductionSkipped > 0) {
|
|
3686
|
+
notes.push(`${candidate.label}: reduction was skipped on ${reductionSkipped} request(s); those were sent unchanged, so its saving is understated.`);
|
|
3687
|
+
}
|
|
3688
|
+
perCandidate.push({
|
|
3689
|
+
candidate,
|
|
3690
|
+
deltas,
|
|
3691
|
+
graded,
|
|
3692
|
+
failed,
|
|
3693
|
+
baselineUsage,
|
|
3694
|
+
candidateUsage,
|
|
3695
|
+
reductionSkipped
|
|
3696
|
+
});
|
|
3697
|
+
}
|
|
3698
|
+
const usable = perCandidate.filter((row) => row.graded > 0);
|
|
3699
|
+
if (usable.length === 0) {
|
|
3700
|
+
throw new Error("No candidate produced a single gradeable comparison; check the grader and the client.");
|
|
3701
|
+
}
|
|
3702
|
+
const bounds = simultaneousGroupBootstrapLowerBounds(
|
|
3703
|
+
usable.map((row) => row.deltas),
|
|
3704
|
+
groups,
|
|
3705
|
+
bootstrapSamples,
|
|
3706
|
+
seed
|
|
3707
|
+
);
|
|
3708
|
+
const results = perCandidate.map((row) => {
|
|
3709
|
+
const boundIndex = usable.indexOf(row);
|
|
3710
|
+
const lower = boundIndex === -1 ? Number.NEGATIVE_INFINITY : bounds[boundIndex];
|
|
3711
|
+
const meanDelta = row.deltas.reduce((sum, value) => sum + value, 0) / row.deltas.length;
|
|
3712
|
+
const inputSaved = row.baselineUsage.inputTokens - row.candidateUsage.inputTokens;
|
|
3713
|
+
const outputSaved = row.baselineUsage.outputTokens - row.candidateUsage.outputTokens;
|
|
3714
|
+
const baselineTokens = row.baselineUsage.inputTokens + row.baselineUsage.outputTokens;
|
|
3715
|
+
const priceUsageRow = (usage) => {
|
|
3716
|
+
if (!price) return void 0;
|
|
3717
|
+
const cached = price.cachedInputPerMillion === void 0 ? 0 : usage.cachedInputTokens;
|
|
3718
|
+
const written = price.cacheWriteInputPerMillion === void 0 ? 0 : usage.cacheWriteInputTokens;
|
|
3719
|
+
return estimateTokenCost(usage.inputTokens, usage.outputTokens, price, {
|
|
3720
|
+
cachedInputTokens: cached,
|
|
3721
|
+
cacheWriteInputTokens: written
|
|
3722
|
+
});
|
|
3723
|
+
};
|
|
3724
|
+
const costBaseline = priceUsageRow(row.baselineUsage);
|
|
3725
|
+
const costCandidate = priceUsageRow(row.candidateUsage);
|
|
3726
|
+
const costFell = costBaseline !== void 0 && costCandidate !== void 0 && costCandidate < costBaseline;
|
|
3727
|
+
let approved2 = lower >= -maximumQualityDrop;
|
|
3728
|
+
let rejectionReason;
|
|
3729
|
+
if (row.graded === 0) {
|
|
3730
|
+
approved2 = false;
|
|
3731
|
+
rejectionReason = "No gradeable comparison was produced.";
|
|
3732
|
+
} else if (!approved2) {
|
|
3733
|
+
rejectionReason = `Quality lower bound ${(lower * 100).toFixed(2)} pp breaches the ${(maximumQualityDrop * 100).toFixed(2)} pp limit.`;
|
|
3734
|
+
} else if (inputSaved + outputSaved <= 0 && !costFell) {
|
|
3735
|
+
approved2 = false;
|
|
3736
|
+
rejectionReason = "Quality held but neither tokens nor cost fell, so there is nothing to enable.";
|
|
3737
|
+
}
|
|
3738
|
+
return {
|
|
3739
|
+
label: row.candidate.label,
|
|
3740
|
+
config: row.candidate.config,
|
|
3741
|
+
graded: row.graded,
|
|
3742
|
+
failed: row.failed,
|
|
3743
|
+
meanQualityDelta: meanDelta,
|
|
3744
|
+
lower95QualityDelta: lower,
|
|
3745
|
+
approved: approved2,
|
|
3746
|
+
...rejectionReason === void 0 ? {} : { rejectionReason },
|
|
3747
|
+
baselineUsage: row.baselineUsage,
|
|
3748
|
+
candidateUsage: row.candidateUsage,
|
|
3749
|
+
inputTokensSaved: inputSaved,
|
|
3750
|
+
outputTokensSaved: outputSaved,
|
|
3751
|
+
tokenSavingPercent: baselineTokens === 0 ? 0 : (inputSaved + outputSaved) / baselineTokens * 100,
|
|
3752
|
+
...costBaseline !== void 0 && costCandidate !== void 0 ? {
|
|
3753
|
+
costSavedUsd: costBaseline - costCandidate,
|
|
3754
|
+
costSavingPercent: costBaseline === 0 ? 0 : (costBaseline - costCandidate) / costBaseline * 100
|
|
3755
|
+
} : {}
|
|
3756
|
+
};
|
|
3757
|
+
});
|
|
3758
|
+
const approved = results.filter((result) => result.approved).sort((left, right) => (right.costSavingPercent ?? right.tokenSavingPercent) - (left.costSavingPercent ?? left.tokenSavingPercent))[0];
|
|
3759
|
+
if (approved === void 0) {
|
|
3760
|
+
notes.push("No candidate held quality. The lossless levers remain available and need no gate.");
|
|
3761
|
+
}
|
|
3762
|
+
return {
|
|
3763
|
+
candidates: results,
|
|
3764
|
+
...approved === void 0 ? {} : { approved },
|
|
3765
|
+
maximumQualityDrop,
|
|
3766
|
+
calls,
|
|
3767
|
+
notes
|
|
3768
|
+
};
|
|
3769
|
+
}
|
|
3770
|
+
|
|
3771
|
+
// packages/sdk-js/src/blind-features.ts
|
|
3772
|
+
import { createHmac } from "node:crypto";
|
|
3773
|
+
function isPlainObject8(value) {
|
|
3774
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3775
|
+
}
|
|
3776
|
+
function textOf(value) {
|
|
3777
|
+
if (typeof value === "string") return value;
|
|
3778
|
+
if (Array.isArray(value)) {
|
|
3779
|
+
return value.map((part) => isPlainObject8(part) && typeof part.text === "string" ? part.text : "").join("\n");
|
|
3780
|
+
}
|
|
3781
|
+
if (isPlainObject8(value) && typeof value.text === "string") return value.text;
|
|
3782
|
+
return "";
|
|
3783
|
+
}
|
|
3784
|
+
function extractBlindFeatures(request, options) {
|
|
3785
|
+
if (!options.hmacKey) throw new Error("extractBlindFeatures requires a per-deployment hmacKey.");
|
|
3786
|
+
const hash = (text) => createHmac("sha256", options.hmacKey).update(text).digest("hex").slice(0, 32);
|
|
3787
|
+
const req = isPlainObject8(request) ? request : {};
|
|
3788
|
+
const model = typeof req.model === "string" ? req.model : "unknown";
|
|
3789
|
+
const messages = Array.isArray(req.messages) ? req.messages : Array.isArray(req.input) ? req.input : [];
|
|
3790
|
+
const tools = Array.isArray(req.tools) ? req.tools : [];
|
|
3791
|
+
const anthropicShaped = model.toLowerCase().includes("claude") || typeof req.system !== "undefined";
|
|
3792
|
+
const provider = model.toLowerCase().includes("claude") ? "anthropic" : model === "unknown" ? "unknown" : "openai";
|
|
3793
|
+
const roleTokensApprox = {};
|
|
3794
|
+
const blockHashes = [];
|
|
3795
|
+
let inputTokensApprox = 0;
|
|
3796
|
+
if (req.system !== void 0) {
|
|
3797
|
+
const text = textOf(req.system);
|
|
3798
|
+
const tokens = approximateTokenCount(text);
|
|
3799
|
+
roleTokensApprox.system = (roleTokensApprox.system ?? 0) + tokens;
|
|
3800
|
+
inputTokensApprox += tokens;
|
|
3801
|
+
if (text.trim() !== "") blockHashes.push(hash(text));
|
|
3802
|
+
}
|
|
3803
|
+
for (const message of messages) {
|
|
3804
|
+
if (!isPlainObject8(message)) continue;
|
|
3805
|
+
const role = typeof message.role === "string" ? message.role : message.type === "function_call_output" ? "tool" : "unknown";
|
|
3806
|
+
const text = textOf("content" in message ? message.content : message.output);
|
|
3807
|
+
const tokens = approximateTokenCount(text);
|
|
3808
|
+
roleTokensApprox[role] = (roleTokensApprox[role] ?? 0) + tokens;
|
|
3809
|
+
inputTokensApprox += tokens;
|
|
3810
|
+
if (text.trim() !== "") blockHashes.push(hash(text));
|
|
3811
|
+
}
|
|
3812
|
+
const toolSchemaText = tools.length > 0 ? JSON.stringify(tools) : "";
|
|
3813
|
+
const toolSchemaTokensApprox = approximateTokenCount(toolSchemaText);
|
|
3814
|
+
inputTokensApprox += toolSchemaTokensApprox;
|
|
3815
|
+
const uniqueBlocks = new Set(blockHashes);
|
|
3816
|
+
const duplicateBlockFraction = blockHashes.length === 0 ? 0 : (blockHashes.length - uniqueBlocks.size) / blockHashes.length;
|
|
3817
|
+
const prefixText = `${toolSchemaText}\0${anthropicShaped ? textOf(req.system) : ""}`;
|
|
3818
|
+
const stablePrefixHash = hash(prefixText);
|
|
3819
|
+
return {
|
|
3820
|
+
schemaVersion: 1,
|
|
3821
|
+
provider,
|
|
3822
|
+
model,
|
|
3823
|
+
messageCount: messages.length,
|
|
3824
|
+
toolCount: tools.length,
|
|
3825
|
+
roleTokensApprox,
|
|
3826
|
+
inputTokensApprox,
|
|
3827
|
+
toolSchemaTokensApprox,
|
|
3828
|
+
blockHashes,
|
|
3829
|
+
duplicateBlockFraction,
|
|
3830
|
+
stablePrefixHash
|
|
3831
|
+
};
|
|
3832
|
+
}
|
|
3833
|
+
|
|
3834
|
+
// packages/sdk-js/src/proxy.ts
|
|
3835
|
+
var LOSSLESS_DEFAULT = {
|
|
3836
|
+
toolSchemas: true,
|
|
3837
|
+
duplicateContext: true,
|
|
3838
|
+
repairCachePrefix: true
|
|
3839
|
+
};
|
|
3840
|
+
var HOP_HEADERS = /* @__PURE__ */ new Set([
|
|
3841
|
+
"host",
|
|
3842
|
+
"content-length",
|
|
3843
|
+
"connection",
|
|
3844
|
+
"accept-encoding",
|
|
3845
|
+
"transfer-encoding",
|
|
3846
|
+
"upgrade",
|
|
3847
|
+
"te",
|
|
3848
|
+
"trailer",
|
|
3849
|
+
"keep-alive",
|
|
3850
|
+
"proxy-authorization"
|
|
3851
|
+
]);
|
|
3852
|
+
function forwardHeaders(req, provider) {
|
|
3853
|
+
const out = {};
|
|
3854
|
+
for (const [name, value] of Object.entries(req.headers)) {
|
|
3855
|
+
if (HOP_HEADERS.has(name.toLowerCase()) || value === void 0) continue;
|
|
3856
|
+
out[name.toLowerCase()] = Array.isArray(value) ? value.join(", ") : value;
|
|
3857
|
+
}
|
|
3858
|
+
delete out["x-mesh-application"];
|
|
3859
|
+
if (provider === "openai") {
|
|
3860
|
+
delete out["x-api-key"];
|
|
3861
|
+
} else if (out["x-api-key"] !== void 0) {
|
|
3862
|
+
delete out.authorization;
|
|
3863
|
+
}
|
|
3864
|
+
out["accept-encoding"] = "identity";
|
|
3865
|
+
return out;
|
|
3866
|
+
}
|
|
3867
|
+
var BODY_CAP = 64 * 1024 * 1024;
|
|
3868
|
+
var BodyTooLargeError = class extends Error {
|
|
3869
|
+
constructor() {
|
|
3870
|
+
super(`Request body exceeds the ${BODY_CAP / (1024 * 1024)} MB cap.`);
|
|
3871
|
+
}
|
|
3872
|
+
};
|
|
3873
|
+
function readBody(req) {
|
|
3874
|
+
return new Promise((resolve2, reject) => {
|
|
3875
|
+
const chunks = [];
|
|
3876
|
+
let received = 0;
|
|
3877
|
+
req.on("data", (chunk) => {
|
|
3878
|
+
received += chunk.byteLength;
|
|
3879
|
+
if (received > BODY_CAP) {
|
|
3880
|
+
reject(new BodyTooLargeError());
|
|
3881
|
+
req.destroy();
|
|
3882
|
+
return;
|
|
3883
|
+
}
|
|
3884
|
+
chunks.push(chunk);
|
|
3885
|
+
});
|
|
3886
|
+
req.on("end", () => resolve2(Buffer.concat(chunks)));
|
|
3887
|
+
req.on("error", reject);
|
|
3888
|
+
});
|
|
3889
|
+
}
|
|
3890
|
+
function hostAllowed(req) {
|
|
3891
|
+
const host = (req.headers.host ?? "").toLowerCase().replace(/:\d+$/, "");
|
|
3892
|
+
return host === "127.0.0.1" || host === "localhost" || host === "[::1]" || host === "";
|
|
3893
|
+
}
|
|
3894
|
+
function surfaceOf(path) {
|
|
3895
|
+
if (path === "/v1/messages") return { provider: "anthropic", upstreamPath: "/v1/messages" };
|
|
3896
|
+
if (path === "/v1/responses") return { provider: "openai", upstreamPath: "/v1/responses" };
|
|
3897
|
+
if (path === "/v1/chat/completions") return { provider: "openai", upstreamPath: "/v1/chat/completions" };
|
|
3898
|
+
return void 0;
|
|
3899
|
+
}
|
|
3900
|
+
function usageFromSse(provider, sseText) {
|
|
3901
|
+
const events = [];
|
|
3902
|
+
for (const line of sseText.split(/\r?\n/)) {
|
|
3903
|
+
if (!line.startsWith("data:")) continue;
|
|
3904
|
+
const payload = line.slice(5).trim();
|
|
3905
|
+
if (payload === "" || payload === "[DONE]") continue;
|
|
3906
|
+
try {
|
|
3907
|
+
const parsed = JSON.parse(payload);
|
|
3908
|
+
if (typeof parsed === "object" && parsed !== null) events.push(parsed);
|
|
3909
|
+
} catch {
|
|
3910
|
+
}
|
|
3911
|
+
}
|
|
3912
|
+
if (events.length === 0) return void 0;
|
|
3913
|
+
if (provider === "anthropic") {
|
|
3914
|
+
const start = events.find((e) => e.type === "message_start");
|
|
3915
|
+
const startUsage = start?.message?.usage;
|
|
3916
|
+
if (startUsage === void 0) return void 0;
|
|
3917
|
+
let outputTokens = startUsage.output_tokens;
|
|
3918
|
+
let tier = start?.message?.service_tier ?? startUsage.service_tier;
|
|
3919
|
+
for (const e of events) {
|
|
3920
|
+
if (e.type !== "message_delta") continue;
|
|
3921
|
+
const deltaUsage = e.usage;
|
|
3922
|
+
if (deltaUsage?.output_tokens !== void 0) outputTokens = deltaUsage.output_tokens;
|
|
3923
|
+
}
|
|
3924
|
+
return {
|
|
3925
|
+
usage: { ...startUsage, output_tokens: outputTokens ?? 0 },
|
|
3926
|
+
...typeof tier === "string" ? { service_tier: tier } : {}
|
|
3927
|
+
};
|
|
3928
|
+
}
|
|
3929
|
+
const completed = [...events].reverse().find((e) => e.type === "response.completed");
|
|
3930
|
+
const completedResponse = completed?.response;
|
|
3931
|
+
if (completedResponse?.usage !== void 0) return completedResponse;
|
|
3932
|
+
const withUsage = [...events].reverse().find((e) => e.usage !== void 0 && e.usage !== null);
|
|
3933
|
+
return withUsage;
|
|
3934
|
+
}
|
|
3935
|
+
function outputTextFromSse(provider, sseText) {
|
|
3936
|
+
const parts = [];
|
|
3937
|
+
for (const line of sseText.split(/\r?\n/)) {
|
|
3938
|
+
if (!line.startsWith("data:")) continue;
|
|
3939
|
+
const payload = line.slice(5).trim();
|
|
3940
|
+
if (payload === "" || payload === "[DONE]") continue;
|
|
3941
|
+
try {
|
|
3942
|
+
const parsed = JSON.parse(payload);
|
|
3943
|
+
if (provider === "anthropic") {
|
|
3944
|
+
const delta = parsed.delta;
|
|
3945
|
+
if (parsed.type === "content_block_delta" && typeof delta?.text === "string") parts.push(delta.text);
|
|
3946
|
+
} else {
|
|
3947
|
+
if (parsed.type === "response.output_text.delta" && typeof parsed.delta === "string") parts.push(parsed.delta);
|
|
3948
|
+
const choices = parsed.choices;
|
|
3949
|
+
const chatDelta = choices?.[0]?.delta;
|
|
3950
|
+
if (typeof chatDelta?.content === "string") parts.push(chatDelta.content);
|
|
3951
|
+
}
|
|
3952
|
+
} catch {
|
|
3953
|
+
}
|
|
3954
|
+
}
|
|
3955
|
+
return parts.join("");
|
|
3956
|
+
}
|
|
3957
|
+
function startMeshProxy(options) {
|
|
3958
|
+
const baseReduction = options.tokenReduction ?? LOSSLESS_DEFAULT;
|
|
3959
|
+
const reduction = baseReduction.cacheTtl !== void 0 && baseReduction.cacheVolatilityGuard === void 0 ? { ...baseReduction, cacheVolatilityGuard: new CachePrefixVolatilityGuard() } : baseReduction;
|
|
3960
|
+
const upstreams = {
|
|
3961
|
+
anthropic: (options.anthropicUpstream ?? "https://api.anthropic.com").replace(/\/$/, ""),
|
|
3962
|
+
openai: (options.openaiUpstream ?? "https://api.openai.com").replace(/\/$/, "")
|
|
3963
|
+
};
|
|
3964
|
+
const ledgers = /* @__PURE__ */ new Map();
|
|
3965
|
+
const LEDGER_CAP = 200;
|
|
3966
|
+
const recentRequestHashes = /* @__PURE__ */ new Map();
|
|
3967
|
+
const RECENT_HASH_CAP = 500;
|
|
3968
|
+
const ledgerFor = (applicationId, provider) => {
|
|
3969
|
+
const safeApp = applicationId.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 64) || "proxy";
|
|
3970
|
+
const key = `${safeApp}
|
|
3971
|
+
${provider}`;
|
|
3972
|
+
const existing = ledgers.get(key);
|
|
3973
|
+
if (existing !== void 0) {
|
|
3974
|
+
ledgers.delete(key);
|
|
3975
|
+
ledgers.set(key, existing);
|
|
3976
|
+
return existing;
|
|
3977
|
+
}
|
|
3978
|
+
while (ledgers.size >= LEDGER_CAP) {
|
|
3979
|
+
const oldest = ledgers.keys().next().value;
|
|
3980
|
+
if (oldest === void 0) break;
|
|
3981
|
+
ledgers.delete(oldest);
|
|
3982
|
+
}
|
|
3983
|
+
const ledger = createSavingsLedger({
|
|
3984
|
+
path: join(options.ledgerDirectory, `${safeApp}-${provider}.jsonl`),
|
|
3985
|
+
provider
|
|
3986
|
+
});
|
|
3987
|
+
ledgers.set(key, ledger);
|
|
3988
|
+
return ledger;
|
|
3989
|
+
};
|
|
3990
|
+
const blindFeaturesKey = options.blindFeaturesKey;
|
|
3991
|
+
const writeBlindFeatures = (applicationId, provider, record) => {
|
|
3992
|
+
const safeApp = applicationId.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 64) || "proxy";
|
|
3993
|
+
mkdirSync2(options.ledgerDirectory, { recursive: true, mode: 448 });
|
|
3994
|
+
appendFileSync2(join(options.ledgerDirectory, `${safeApp}-${provider}.features.jsonl`), `${JSON.stringify(record)}
|
|
3995
|
+
`, { mode: 384 });
|
|
3996
|
+
};
|
|
3997
|
+
const handle = async (req, res) => {
|
|
3998
|
+
const path = (req.url ?? "").split("?")[0];
|
|
3999
|
+
if (!hostAllowed(req)) {
|
|
4000
|
+
res.writeHead(403, { "content-type": "application/json" });
|
|
4001
|
+
res.end(JSON.stringify({ error: "MESH proxy serves loopback clients only." }));
|
|
4002
|
+
return;
|
|
4003
|
+
}
|
|
4004
|
+
const fetchSite = req.headers["sec-fetch-site"];
|
|
4005
|
+
if (fetchSite === "cross-site" || fetchSite === "same-site") {
|
|
4006
|
+
res.writeHead(403, { "content-type": "application/json" });
|
|
4007
|
+
res.end(JSON.stringify({ error: "MESH proxy does not serve cross-origin browser requests." }));
|
|
4008
|
+
return;
|
|
4009
|
+
}
|
|
4010
|
+
if (req.method === "GET" && path.startsWith("/v1/")) {
|
|
4011
|
+
const provider = req.headers["x-api-key"] !== void 0 ? "anthropic" : "openai";
|
|
4012
|
+
let upstreamResponse2;
|
|
4013
|
+
try {
|
|
4014
|
+
upstreamResponse2 = await fetch(`${upstreams[provider]}${req.url}`, {
|
|
4015
|
+
headers: forwardHeaders(req, provider),
|
|
4016
|
+
signal: AbortSignal.timeout(6e4)
|
|
4017
|
+
});
|
|
4018
|
+
} catch (cause) {
|
|
4019
|
+
res.writeHead(502, { "content-type": "application/json" });
|
|
4020
|
+
res.end(JSON.stringify({ error: `Upstream unreachable: ${cause.message}` }));
|
|
4021
|
+
return;
|
|
4022
|
+
}
|
|
4023
|
+
const passHeaders = {};
|
|
4024
|
+
upstreamResponse2.headers.forEach((value, name) => {
|
|
4025
|
+
if (!HOP_HEADERS.has(name.toLowerCase())) passHeaders[name] = value;
|
|
4026
|
+
});
|
|
4027
|
+
res.writeHead(upstreamResponse2.status, passHeaders);
|
|
4028
|
+
res.end(Buffer.from(await upstreamResponse2.arrayBuffer()));
|
|
4029
|
+
return;
|
|
4030
|
+
}
|
|
4031
|
+
const surface = req.method === "POST" ? surfaceOf(path) : void 0;
|
|
4032
|
+
if (surface === void 0) {
|
|
4033
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
4034
|
+
res.end(JSON.stringify({
|
|
4035
|
+
error: "MESH proxy forwards POST /v1/messages (Anthropic), /v1/responses and /v1/chat/completions (OpenAI-compatible). Point your client's base_url at this server."
|
|
4036
|
+
}));
|
|
4037
|
+
return;
|
|
4038
|
+
}
|
|
4039
|
+
let rawBody;
|
|
4040
|
+
try {
|
|
4041
|
+
rawBody = await readBody(req);
|
|
4042
|
+
} catch (cause) {
|
|
4043
|
+
if (cause instanceof BodyTooLargeError && !res.headersSent) {
|
|
4044
|
+
res.writeHead(413, { "content-type": "application/json" });
|
|
4045
|
+
res.end(JSON.stringify({ error: cause.message }));
|
|
4046
|
+
}
|
|
4047
|
+
return;
|
|
4048
|
+
}
|
|
4049
|
+
let body;
|
|
4050
|
+
try {
|
|
4051
|
+
body = JSON.parse(rawBody.toString("utf8"));
|
|
4052
|
+
} catch {
|
|
4053
|
+
body = void 0;
|
|
4054
|
+
}
|
|
4055
|
+
if (body === void 0 || typeof body !== "object") {
|
|
4056
|
+
res.writeHead(400, { "content-type": "application/json" });
|
|
4057
|
+
res.end(JSON.stringify({ error: "The request body is not a JSON object." }));
|
|
4058
|
+
return;
|
|
4059
|
+
}
|
|
4060
|
+
const { request: reducedRequest, summary } = reduceRequest(body, reduction);
|
|
4061
|
+
const sendBody = JSON.stringify(reducedRequest);
|
|
4062
|
+
const applicationIdHeader = req.headers["x-mesh-application"];
|
|
4063
|
+
const applicationId = ((Array.isArray(applicationIdHeader) ? applicationIdHeader[0] : applicationIdHeader) ?? "proxy").slice(0, 128);
|
|
4064
|
+
const headers = forwardHeaders(req, surface.provider);
|
|
4065
|
+
headers["content-type"] = "application/json";
|
|
4066
|
+
const upstreamControl = new AbortController();
|
|
4067
|
+
const abortUpstream = (reason) => upstreamControl.abort(new Error(reason));
|
|
4068
|
+
const onClientClose = () => {
|
|
4069
|
+
if (!res.writableEnded) abortUpstream("client disconnected");
|
|
4070
|
+
};
|
|
4071
|
+
res.on("close", onClientClose);
|
|
4072
|
+
const HEADER_TIMEOUT_MS = 12e4;
|
|
4073
|
+
const IDLE_TIMEOUT_MS = 12e4;
|
|
4074
|
+
let phaseTimer = setTimeout(() => abortUpstream("upstream produced no response headers within 120s"), HEADER_TIMEOUT_MS);
|
|
4075
|
+
const requestId = randomUUID2();
|
|
4076
|
+
const bodyHash = createHash8("sha256").update(applicationId).update("\n").update(rawBody).digest("hex");
|
|
4077
|
+
const duplicateOfRequestId = recentRequestHashes.get(bodyHash);
|
|
4078
|
+
if (duplicateOfRequestId === void 0) {
|
|
4079
|
+
while (recentRequestHashes.size >= RECENT_HASH_CAP) {
|
|
4080
|
+
const oldest = recentRequestHashes.keys().next().value;
|
|
4081
|
+
if (oldest === void 0) break;
|
|
4082
|
+
recentRequestHashes.delete(oldest);
|
|
4083
|
+
}
|
|
4084
|
+
recentRequestHashes.set(bodyHash, requestId);
|
|
4085
|
+
}
|
|
4086
|
+
const startedAt = Date.now();
|
|
4087
|
+
let upstreamResponse;
|
|
4088
|
+
try {
|
|
4089
|
+
upstreamResponse = await fetch(`${upstreams[surface.provider]}${surface.upstreamPath}`, {
|
|
4090
|
+
method: "POST",
|
|
4091
|
+
headers,
|
|
4092
|
+
body: sendBody,
|
|
4093
|
+
signal: upstreamControl.signal
|
|
4094
|
+
});
|
|
4095
|
+
} catch (cause) {
|
|
4096
|
+
clearTimeout(phaseTimer);
|
|
4097
|
+
if (!res.headersSent && !res.destroyed) {
|
|
4098
|
+
res.writeHead(502, { "content-type": "application/json" });
|
|
4099
|
+
res.end(JSON.stringify({ error: `Upstream unreachable: ${cause.message}` }));
|
|
4100
|
+
}
|
|
4101
|
+
return;
|
|
4102
|
+
}
|
|
4103
|
+
clearTimeout(phaseTimer);
|
|
4104
|
+
const responseHeaders = {};
|
|
4105
|
+
upstreamResponse.headers.forEach((value, name) => {
|
|
4106
|
+
if (!HOP_HEADERS.has(name.toLowerCase())) responseHeaders[name] = value;
|
|
4107
|
+
});
|
|
4108
|
+
if (res.destroyed) {
|
|
4109
|
+
abortUpstream("client disconnected");
|
|
4110
|
+
return;
|
|
4111
|
+
}
|
|
4112
|
+
res.writeHead(upstreamResponse.status, responseHeaders);
|
|
4113
|
+
const chunks = [];
|
|
4114
|
+
let accumulated = 0;
|
|
4115
|
+
const ACCUMULATION_CAP = 32 * 1024 * 1024;
|
|
4116
|
+
let clientLost = false;
|
|
4117
|
+
if (upstreamResponse.body !== null) {
|
|
4118
|
+
const reader = upstreamResponse.body.getReader();
|
|
4119
|
+
try {
|
|
4120
|
+
for (; ; ) {
|
|
4121
|
+
phaseTimer = setTimeout(() => abortUpstream("upstream stream idle for 120s"), IDLE_TIMEOUT_MS);
|
|
4122
|
+
let done;
|
|
4123
|
+
let value;
|
|
4124
|
+
try {
|
|
4125
|
+
({ done, value } = await reader.read());
|
|
4126
|
+
} finally {
|
|
4127
|
+
clearTimeout(phaseTimer);
|
|
4128
|
+
}
|
|
4129
|
+
if (done) break;
|
|
4130
|
+
if (res.destroyed) {
|
|
4131
|
+
clientLost = true;
|
|
4132
|
+
abortUpstream("client disconnected");
|
|
4133
|
+
break;
|
|
4134
|
+
}
|
|
4135
|
+
const chunk = Buffer.from(value);
|
|
4136
|
+
const flushed = res.write(chunk);
|
|
4137
|
+
if (accumulated < ACCUMULATION_CAP) {
|
|
4138
|
+
chunks.push(chunk);
|
|
4139
|
+
accumulated += chunk.byteLength;
|
|
4140
|
+
}
|
|
4141
|
+
if (!flushed) {
|
|
4142
|
+
await new Promise((proceed) => {
|
|
4143
|
+
const onDrain = () => {
|
|
4144
|
+
res.off("close", onGone);
|
|
4145
|
+
proceed();
|
|
4146
|
+
};
|
|
4147
|
+
const onGone = () => {
|
|
4148
|
+
res.off("drain", onDrain);
|
|
4149
|
+
proceed();
|
|
4150
|
+
};
|
|
4151
|
+
res.once("drain", onDrain);
|
|
4152
|
+
res.once("close", onGone);
|
|
4153
|
+
});
|
|
4154
|
+
}
|
|
4155
|
+
}
|
|
4156
|
+
} catch {
|
|
4157
|
+
clientLost = res.destroyed;
|
|
4158
|
+
}
|
|
4159
|
+
}
|
|
4160
|
+
res.off("close", onClientClose);
|
|
4161
|
+
if (!clientLost && !res.destroyed) res.end();
|
|
4162
|
+
try {
|
|
4163
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
4164
|
+
const contentType = upstreamResponse.headers.get("content-type") ?? "";
|
|
4165
|
+
if (process.env.MESH_PROXY_DEBUG === "1" && !upstreamResponse.ok) {
|
|
4166
|
+
console.warn(
|
|
4167
|
+
`[mesh-debug] upstream ${upstreamResponse.status} model=${JSON.stringify(body.model)} path=${path} anthropic-beta=${JSON.stringify(req.headers["anthropic-beta"] ?? null)} errbody=${text.slice(0, 500)}`
|
|
4168
|
+
);
|
|
4169
|
+
}
|
|
4170
|
+
let responseShaped;
|
|
4171
|
+
if (contentType.includes("text/event-stream")) {
|
|
4172
|
+
responseShaped = usageFromSse(surface.provider, text);
|
|
4173
|
+
} else {
|
|
4174
|
+
try {
|
|
4175
|
+
responseShaped = JSON.parse(text);
|
|
4176
|
+
} catch {
|
|
4177
|
+
responseShaped = void 0;
|
|
4178
|
+
}
|
|
4179
|
+
}
|
|
4180
|
+
const usageResult = responseShaped === void 0 ? void 0 : normalizeProviderUsage(responseShaped);
|
|
4181
|
+
const usage = usageResult?.status === "ok" ? usageResult.usage : void 0;
|
|
4182
|
+
const tier = responseShaped === void 0 ? void 0 : responseServiceTier(responseShaped);
|
|
4183
|
+
let outputDiagnostics;
|
|
4184
|
+
if (usage !== void 0 && usage.outputTokens >= 200 && responseShaped !== void 0) {
|
|
4185
|
+
const outputText = contentType.includes("text/event-stream") ? outputTextFromSse(surface.provider, text) : responseText(responseShaped);
|
|
4186
|
+
if (outputText.length > 0) {
|
|
4187
|
+
outputDiagnostics = measureOutputReEmission(collectRequestText(body), outputText);
|
|
4188
|
+
}
|
|
4189
|
+
}
|
|
4190
|
+
const event = {
|
|
4191
|
+
requestId,
|
|
4192
|
+
applicationId,
|
|
4193
|
+
provider: `${surface.provider}-compatible`,
|
|
4194
|
+
model: typeof body.model === "string" ? body.model : "unknown",
|
|
4195
|
+
latencyMs: Date.now() - startedAt,
|
|
4196
|
+
status: upstreamResponse.ok ? "succeeded" : "failed",
|
|
4197
|
+
timestamp: new Date(startedAt).toISOString(),
|
|
4198
|
+
...summary.skipped ? {} : { reduction: summary },
|
|
4199
|
+
...usage === void 0 ? {} : {
|
|
4200
|
+
inputTokens: usage.inputTokens,
|
|
4201
|
+
outputTokens: usage.outputTokens,
|
|
4202
|
+
uncachedInputTokens: usage.uncachedInputTokens,
|
|
4203
|
+
cachedInputTokens: usage.cachedInputTokens,
|
|
4204
|
+
cacheWriteInputTokens: usage.cacheWriteInputTokens,
|
|
4205
|
+
...usage.cacheWrite5mInputTokens === void 0 ? {} : {
|
|
4206
|
+
cacheWrite5mInputTokens: usage.cacheWrite5mInputTokens,
|
|
4207
|
+
cacheWrite1hInputTokens: usage.cacheWrite1hInputTokens ?? 0
|
|
4208
|
+
}
|
|
4209
|
+
},
|
|
4210
|
+
...tier === void 0 ? {} : { responseServiceTier: tier },
|
|
4211
|
+
...outputDiagnostics === void 0 ? {} : { outputDiagnostics },
|
|
4212
|
+
...duplicateOfRequestId === void 0 ? {} : { duplicateOfRequestId }
|
|
4213
|
+
};
|
|
4214
|
+
ledgerFor(applicationId, surface.provider).emit(event);
|
|
4215
|
+
if (blindFeaturesKey !== void 0) {
|
|
4216
|
+
const features = extractBlindFeatures(body, { hmacKey: blindFeaturesKey });
|
|
4217
|
+
writeBlindFeatures(applicationId, surface.provider, {
|
|
4218
|
+
requestId,
|
|
4219
|
+
applicationId,
|
|
4220
|
+
timestamp: event.timestamp,
|
|
4221
|
+
features
|
|
4222
|
+
});
|
|
4223
|
+
}
|
|
4224
|
+
} catch {
|
|
4225
|
+
}
|
|
4226
|
+
};
|
|
4227
|
+
const server = createServer((req, res) => {
|
|
4228
|
+
handle(req, res).catch(() => {
|
|
4229
|
+
if (!res.headersSent) {
|
|
4230
|
+
res.writeHead(500, { "content-type": "application/json" });
|
|
4231
|
+
res.end(JSON.stringify({ error: "MESH proxy internal error." }));
|
|
4232
|
+
} else {
|
|
4233
|
+
res.end();
|
|
4234
|
+
}
|
|
4235
|
+
});
|
|
4236
|
+
});
|
|
4237
|
+
server.maxConnections = 256;
|
|
4238
|
+
return new Promise((resolve2, reject) => {
|
|
4239
|
+
server.once("error", reject);
|
|
4240
|
+
server.listen(options.port ?? 0, "127.0.0.1", () => {
|
|
4241
|
+
const address = server.address();
|
|
4242
|
+
const port = typeof address === "object" && address !== null ? address.port : options.port ?? 0;
|
|
4243
|
+
resolve2({
|
|
4244
|
+
port,
|
|
4245
|
+
url: `http://127.0.0.1:${port}`,
|
|
4246
|
+
close: () => new Promise((done, fail) => server.close((err) => err ? fail(err) : done()))
|
|
4247
|
+
});
|
|
4248
|
+
});
|
|
4249
|
+
});
|
|
4250
|
+
}
|
|
4251
|
+
|
|
4252
|
+
// packages/sdk-js/src/wrap.ts
|
|
4253
|
+
import { readFileSync as readFileSync2 } from "node:fs";
|
|
4254
|
+
import { join as join2 } from "node:path";
|
|
4255
|
+
var LOSSLESS_DEFAULT2 = {
|
|
4256
|
+
toolSchemas: true,
|
|
4257
|
+
duplicateContext: true,
|
|
4258
|
+
repairCachePrefix: true
|
|
4259
|
+
};
|
|
4260
|
+
function looksAnthropic(client) {
|
|
4261
|
+
const c = client;
|
|
4262
|
+
const hasMessages = typeof c?.messages?.create === "function";
|
|
4263
|
+
const hasChat = typeof c?.chat?.completions?.create === "function";
|
|
4264
|
+
const hasResponses = typeof c?.responses?.create === "function";
|
|
4265
|
+
return hasMessages && !hasChat && !hasResponses;
|
|
4266
|
+
}
|
|
4267
|
+
function wrapMesh(client, options = {}) {
|
|
4268
|
+
const app = options.app ?? "app";
|
|
4269
|
+
const ledgerPath = options.ledgerPath ?? join2(process.cwd(), ".mesh", `${app}.jsonl`);
|
|
4270
|
+
const provider = options.provider ?? (looksAnthropic(client) ? "anthropic" : "openai");
|
|
4271
|
+
const tokenReduction = {
|
|
4272
|
+
...options.tokenReduction ?? LOSSLESS_DEFAULT2,
|
|
4273
|
+
...options.cacheTtl ? { cacheTtl: options.cacheTtl } : {}
|
|
4274
|
+
};
|
|
4275
|
+
const wrapped = withMesh(client, {
|
|
4276
|
+
applicationId: app,
|
|
4277
|
+
tokenReduction,
|
|
4278
|
+
ledger: { path: ledgerPath, provider }
|
|
4279
|
+
});
|
|
4280
|
+
const mesh = {
|
|
4281
|
+
ledgerPath,
|
|
4282
|
+
savings() {
|
|
4283
|
+
const empty = {
|
|
4284
|
+
requests: 0,
|
|
4285
|
+
savedUsd: 0,
|
|
4286
|
+
feeUsd: 0,
|
|
4287
|
+
verdict: "do_not_charge",
|
|
4288
|
+
ledgerPath
|
|
4289
|
+
};
|
|
4290
|
+
let records;
|
|
4291
|
+
try {
|
|
4292
|
+
const lines = readFileSync2(ledgerPath, "utf8").trim().split(/\r?\n/).filter(Boolean);
|
|
4293
|
+
if (lines.length === 0) return empty;
|
|
4294
|
+
records = lines.map((line) => JSON.parse(line));
|
|
4295
|
+
} catch {
|
|
4296
|
+
return empty;
|
|
4297
|
+
}
|
|
4298
|
+
if (!verifyLedgerChain(records).ok) return empty;
|
|
4299
|
+
const invoice = recomputeInvoice(records.map((r) => r.event), { feePercent: 1 });
|
|
4300
|
+
return {
|
|
4301
|
+
requests: invoice.eventsConsidered,
|
|
4302
|
+
savedUsd: invoice.netVerifiedSavingsUsd,
|
|
4303
|
+
feeUsd: invoice.amountDueUsd ?? 0,
|
|
4304
|
+
verdict: invoice.verdict,
|
|
4305
|
+
ledgerPath
|
|
4306
|
+
};
|
|
4307
|
+
}
|
|
4308
|
+
};
|
|
4309
|
+
return new Proxy(wrapped, {
|
|
4310
|
+
get(target, property2, receiver) {
|
|
4311
|
+
if (property2 === "mesh") return mesh;
|
|
4312
|
+
return Reflect.get(target, property2, receiver);
|
|
4313
|
+
}
|
|
4314
|
+
});
|
|
4315
|
+
}
|
|
4316
|
+
|
|
4317
|
+
// packages/sdk-js/src/theo-policy.ts
|
|
4318
|
+
function proofClearsCeiling(lever, ceilingPercent) {
|
|
4319
|
+
if (lever.gate === void 0) return false;
|
|
4320
|
+
const y = lever.gate.qualityDeltaPercent;
|
|
4321
|
+
return Number.isFinite(y) && Math.abs(y) <= ceilingPercent;
|
|
4322
|
+
}
|
|
4323
|
+
function applyLever(config, lever) {
|
|
4324
|
+
const p = lever.params ?? {};
|
|
4325
|
+
switch (lever.lever) {
|
|
4326
|
+
case "tool-schema-compaction":
|
|
4327
|
+
if (config.toolSchemas === void 0) config.toolSchemas = true;
|
|
4328
|
+
return;
|
|
4329
|
+
case "tool-schema-description-trim":
|
|
4330
|
+
config.toolSchemas = {
|
|
4331
|
+
descriptions: "truncate",
|
|
4332
|
+
...typeof p.maximumDescriptionTokens === "number" ? { maximumDescriptionTokens: p.maximumDescriptionTokens } : {}
|
|
4333
|
+
};
|
|
4334
|
+
return;
|
|
4335
|
+
case "duplicate-context-removal":
|
|
4336
|
+
config.duplicateContext = true;
|
|
4337
|
+
return;
|
|
4338
|
+
case "cache-prefix-repair":
|
|
4339
|
+
config.repairCachePrefix = true;
|
|
4340
|
+
return;
|
|
4341
|
+
case "stale-history-compaction":
|
|
4342
|
+
config.historyCompaction = true;
|
|
4343
|
+
return;
|
|
4344
|
+
case "cross-turn-memory-dedup":
|
|
4345
|
+
config.crossTurnMemoryDedup = true;
|
|
4346
|
+
return;
|
|
4347
|
+
case "fresh-tool-result-cap":
|
|
4348
|
+
config.freshToolResultCap = typeof p.maximumTokens === "number" ? { maximumTokens: p.maximumTokens } : true;
|
|
4349
|
+
return;
|
|
4350
|
+
case "output-cap":
|
|
4351
|
+
if (typeof p.maxOutputTokens === "number") config.maxOutputTokens = p.maxOutputTokens;
|
|
4352
|
+
return;
|
|
4353
|
+
case "execution-tier-flex":
|
|
4354
|
+
config.executionTier = "flex";
|
|
4355
|
+
return;
|
|
4356
|
+
case "cache-ttl-5m":
|
|
4357
|
+
config.cacheTtl = "5m";
|
|
4358
|
+
return;
|
|
4359
|
+
case "cache-ttl-1h":
|
|
4360
|
+
config.cacheTtl = "1h";
|
|
4361
|
+
return;
|
|
4362
|
+
case "output-effort-low":
|
|
4363
|
+
case "output-effort-medium":
|
|
4364
|
+
case "output-effort-high":
|
|
4365
|
+
case "output-effort-xhigh":
|
|
4366
|
+
case "output-effort-max":
|
|
4367
|
+
config.outputEffort = lever.lever.replace("output-effort-", "");
|
|
4368
|
+
return;
|
|
4369
|
+
default:
|
|
4370
|
+
return;
|
|
4371
|
+
}
|
|
4372
|
+
}
|
|
4373
|
+
function policyToReductionConfig(policy) {
|
|
4374
|
+
const config = {};
|
|
4375
|
+
for (const lever of policy.levers) {
|
|
4376
|
+
if (lever.class === "lossy" && !proofClearsCeiling(lever, policy.qualityCeilingPercent)) continue;
|
|
4377
|
+
applyLever(config, lever);
|
|
4378
|
+
}
|
|
4379
|
+
return config;
|
|
4380
|
+
}
|
|
4381
|
+
function withheldLevers(policy) {
|
|
4382
|
+
const out = [];
|
|
4383
|
+
for (const lever of policy.levers) {
|
|
4384
|
+
if (lever.class !== "lossy") continue;
|
|
4385
|
+
if (lever.gate === void 0) {
|
|
4386
|
+
out.push({ lever: lever.lever, reason: "no gate proof yet \u2014 queued for the live gate" });
|
|
4387
|
+
} else if (!proofClearsCeiling(lever, policy.qualityCeilingPercent)) {
|
|
4388
|
+
out.push({
|
|
4389
|
+
lever: lever.lever,
|
|
4390
|
+
reason: `measured quality drop ${Math.abs(lever.gate.qualityDeltaPercent).toFixed(2)}% exceeds the ${policy.qualityCeilingPercent}% ceiling for this route`
|
|
4391
|
+
});
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
return out;
|
|
4395
|
+
}
|
|
4396
|
+
function publishLever(policy, lever, at) {
|
|
4397
|
+
if (lever.class === "lossy" && lever.gate === void 0) {
|
|
4398
|
+
throw new Error(`Refusing to publish lossy lever '${lever.lever}' without a gate proof.`);
|
|
4399
|
+
}
|
|
4400
|
+
const levers = policy.levers.filter((existing) => existing.lever !== lever.lever);
|
|
4401
|
+
levers.push(lever);
|
|
4402
|
+
return { ...policy, levers, version: policy.version + 1, updatedAt: at };
|
|
4403
|
+
}
|
|
4404
|
+
function defaultPolicy(user, at) {
|
|
4405
|
+
return {
|
|
4406
|
+
schemaVersion: 1,
|
|
4407
|
+
user,
|
|
4408
|
+
version: 1,
|
|
4409
|
+
qualityCeilingPercent: 2,
|
|
4410
|
+
// strict by default; loosened only by the user's revealed preference
|
|
4411
|
+
levers: [
|
|
4412
|
+
{ lever: "tool-schema-compaction", class: "lossless" },
|
|
4413
|
+
{ lever: "duplicate-context-removal", class: "lossless" },
|
|
4414
|
+
{ lever: "cache-prefix-repair", class: "lossless" }
|
|
4415
|
+
],
|
|
4416
|
+
updatedAt: at
|
|
4417
|
+
};
|
|
4418
|
+
}
|
|
4419
|
+
function knownLeverIds() {
|
|
4420
|
+
return [
|
|
4421
|
+
"tool-schema-compaction",
|
|
4422
|
+
"tool-schema-description-trim",
|
|
4423
|
+
"duplicate-context-removal",
|
|
4424
|
+
"cache-prefix-repair",
|
|
4425
|
+
"stale-history-compaction",
|
|
4426
|
+
"cross-turn-memory-dedup",
|
|
4427
|
+
"fresh-tool-result-cap",
|
|
4428
|
+
"output-cap",
|
|
4429
|
+
"execution-tier-flex",
|
|
4430
|
+
"cache-ttl-5m",
|
|
4431
|
+
"cache-ttl-1h",
|
|
4432
|
+
"output-effort-low",
|
|
4433
|
+
"output-effort-medium",
|
|
4434
|
+
"output-effort-high",
|
|
4435
|
+
"output-effort-xhigh",
|
|
4436
|
+
"output-effort-max"
|
|
4437
|
+
];
|
|
4438
|
+
}
|
|
4439
|
+
|
|
4440
|
+
// packages/sdk-js/src/semantic-compaction.ts
|
|
4441
|
+
function isPlainObject9(value) {
|
|
4442
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4443
|
+
}
|
|
4444
|
+
function isToolResultCarrier2(message) {
|
|
4445
|
+
return isPlainObject9(message) && message.role === "user" && Array.isArray(message.content) && message.content.length > 0 && message.content.every((block) => isPlainObject9(block) && block.type === "tool_result");
|
|
4446
|
+
}
|
|
4447
|
+
function isRealUserTurn(message) {
|
|
4448
|
+
return isPlainObject9(message) && message.role === "user" && !isToolResultCarrier2(message);
|
|
4449
|
+
}
|
|
4450
|
+
function collectStaleToolResults(messages, staleBoundary, maxTokens, countTokens) {
|
|
4451
|
+
const targets = [];
|
|
4452
|
+
let turn = -1;
|
|
4453
|
+
const oversized = (text) => countTokens(text) > maxTokens;
|
|
4454
|
+
messages.forEach((message, messageIndex) => {
|
|
4455
|
+
if (isRealUserTurn(message)) turn += 1;
|
|
4456
|
+
if (turn < 0 || turn >= staleBoundary) return;
|
|
4457
|
+
if (!isPlainObject9(message)) return;
|
|
4458
|
+
const role = typeof message.role === "string" ? message.role : void 0;
|
|
4459
|
+
if (role === "tool" && typeof message.content === "string") {
|
|
4460
|
+
if (oversized(message.content)) {
|
|
4461
|
+
targets.push({ kind: "chat-tool", messageIndex, text: message.content });
|
|
4462
|
+
}
|
|
4463
|
+
return;
|
|
4464
|
+
}
|
|
4465
|
+
if (message.type === "function_call_output" && typeof message.output === "string") {
|
|
4466
|
+
if (oversized(message.output)) {
|
|
4467
|
+
targets.push({ kind: "responses-output", messageIndex, text: message.output });
|
|
4468
|
+
}
|
|
4469
|
+
return;
|
|
4470
|
+
}
|
|
4471
|
+
if (role === "user" && Array.isArray(message.content)) {
|
|
4472
|
+
message.content.forEach((block, blockIndex) => {
|
|
4473
|
+
if (!isPlainObject9(block) || block.type !== "tool_result") return;
|
|
4474
|
+
if (typeof block.content !== "string") return;
|
|
4475
|
+
if (oversized(block.content)) {
|
|
4476
|
+
targets.push({ kind: "anthropic-block", messageIndex, blockIndex, text: block.content });
|
|
4477
|
+
}
|
|
4478
|
+
});
|
|
4479
|
+
}
|
|
4480
|
+
});
|
|
4481
|
+
return targets;
|
|
4482
|
+
}
|
|
4483
|
+
function applyCompaction(messages, target, compacted) {
|
|
4484
|
+
const next = [...messages];
|
|
4485
|
+
if (target.kind === "chat-tool") {
|
|
4486
|
+
next[target.messageIndex] = { ...next[target.messageIndex], content: compacted };
|
|
4487
|
+
return next;
|
|
4488
|
+
}
|
|
4489
|
+
if (target.kind === "responses-output") {
|
|
4490
|
+
next[target.messageIndex] = { ...next[target.messageIndex], output: compacted };
|
|
4491
|
+
return next;
|
|
4492
|
+
}
|
|
4493
|
+
const message = next[target.messageIndex];
|
|
4494
|
+
const content = [...message.content];
|
|
4495
|
+
content[target.blockIndex] = { ...content[target.blockIndex], content: compacted };
|
|
4496
|
+
next[target.messageIndex] = { ...message, content };
|
|
4497
|
+
return next;
|
|
4498
|
+
}
|
|
4499
|
+
async function compactStaleContext(request, config = {}) {
|
|
4500
|
+
const countTokens = config.countTokens ?? approximateTokenCount;
|
|
4501
|
+
const noop = (reason) => ({ request, tokensBefore: 0, tokensAfter: 0, skippedReason: reason });
|
|
4502
|
+
if (!isPlainObject9(request)) return noop("Request is not an object.");
|
|
4503
|
+
if (!config.compactor) return noop("Semantic compaction is off: no compactor supplied.");
|
|
4504
|
+
if (config.gateVerdict?.verdict !== "approved") {
|
|
4505
|
+
return noop(
|
|
4506
|
+
"Semantic compaction withheld: no approved replay-gate verdict for the 'semantic-compaction' lever. This lossy, model-driven lever may not touch content until a gate has proven quality held on the customer's own recorded requests."
|
|
4507
|
+
);
|
|
4508
|
+
}
|
|
4509
|
+
const maxTokens = config.maxTokens ?? 500;
|
|
4510
|
+
const minimumTurns = config.minimumTurns ?? 3;
|
|
4511
|
+
const keepRecentTurns = config.keepRecentTurns ?? 2;
|
|
4512
|
+
if (!Number.isInteger(maxTokens) || maxTokens < 1) throw new Error("semanticCompaction maxTokens must be a positive integer.");
|
|
4513
|
+
if (!Number.isInteger(minimumTurns) || minimumTurns < 1) throw new Error("semanticCompaction minimumTurns must be a positive integer.");
|
|
4514
|
+
if (!Number.isInteger(keepRecentTurns) || keepRecentTurns < 1) throw new Error("semanticCompaction keepRecentTurns must be a positive integer.");
|
|
4515
|
+
const field = Array.isArray(request.messages) ? "messages" : Array.isArray(request.input) ? "input" : void 0;
|
|
4516
|
+
if (field === void 0) return noop("No message array to compact (string or absent input).");
|
|
4517
|
+
const messages = request[field];
|
|
4518
|
+
const totalTurns = messages.filter(isRealUserTurn).length;
|
|
4519
|
+
if (totalTurns < minimumTurns) return noop(`Only ${totalTurns} user turn(s); below the ${minimumTurns}-turn minimum.`);
|
|
4520
|
+
const staleBoundary = totalTurns - keepRecentTurns;
|
|
4521
|
+
if (staleBoundary <= 0) return noop("No stale turns: the whole conversation is inside the recent-turn window.");
|
|
4522
|
+
const targets = collectStaleToolResults(messages, staleBoundary, maxTokens, countTokens);
|
|
4523
|
+
if (targets.length === 0) return noop("No oversized stale tool-result bodies to compact.");
|
|
4524
|
+
const tokensBefore = targets.reduce((sum, target) => sum + countTokens(target.text), 0);
|
|
4525
|
+
let compactedMessages = messages;
|
|
4526
|
+
let bodiesChanged = 0;
|
|
4527
|
+
let tokensAfterTargets = 0;
|
|
4528
|
+
for (const target of targets) {
|
|
4529
|
+
const result = await config.compactor.compact({ text: target.text, maxTokens, kind: "tool-result" });
|
|
4530
|
+
const compacted = typeof result === "string" ? result : target.text;
|
|
4531
|
+
if (compacted.length < target.text.length) {
|
|
4532
|
+
compactedMessages = applyCompaction(compactedMessages, target, compacted);
|
|
4533
|
+
bodiesChanged += 1;
|
|
4534
|
+
tokensAfterTargets += countTokens(compacted);
|
|
4535
|
+
} else {
|
|
4536
|
+
tokensAfterTargets += countTokens(target.text);
|
|
4537
|
+
}
|
|
4538
|
+
}
|
|
4539
|
+
if (bodiesChanged === 0) {
|
|
4540
|
+
return { request, tokensBefore, tokensAfter: tokensBefore, skippedReason: "Compactor returned nothing shorter." };
|
|
4541
|
+
}
|
|
4542
|
+
const nextRequest = { ...request, [field]: compactedMessages };
|
|
4543
|
+
const estimatedTokensSaved = Math.max(0, tokensBefore - tokensAfterTargets);
|
|
4544
|
+
const change = {
|
|
4545
|
+
lever: "semantic-compaction",
|
|
4546
|
+
lossless: false,
|
|
4547
|
+
estimatedTokensSaved,
|
|
4548
|
+
detail: `Semantically compacted ${bodiesChanged} oversized stale tool-result body(ies) with the customer's LOCAL compactor (tokens-before ${tokensBefore}, tokens-after ${tokensAfterTargets}, cap ${maxTokens}; stale boundary at turn ${staleBoundary} of ${totalTurns}, keeping the last ${keepRecentTurns} turn(s) verbatim). LOSSY and model-driven: the prompt never left the machine, and the saving may only be billed after an approved replay-gate verdict (ref ${config.gateVerdict.ref}). Model output and user turns were never touched.`
|
|
4549
|
+
};
|
|
4550
|
+
return { request: nextRequest, change, tokensBefore, tokensAfter: tokensAfterTargets };
|
|
4551
|
+
}
|
|
4552
|
+
async function reduceRequestAsync(request, config = {}, semantic = {}) {
|
|
4553
|
+
const countTokens = config.countTokens ?? semantic.countTokens ?? approximateTokenCount;
|
|
4554
|
+
const measure = (value) => {
|
|
4555
|
+
try {
|
|
4556
|
+
return typeof value === "string" ? countTokens(value) : countTokens(JSON.stringify(value) ?? "");
|
|
4557
|
+
} catch {
|
|
4558
|
+
return 0;
|
|
4559
|
+
}
|
|
4560
|
+
};
|
|
4561
|
+
const originalBefore = isPlainObject9(request) ? measure(request.instructions) + measure(request.system) + measure(request.input) + measure(request.messages) + measure(request.tools) : 0;
|
|
4562
|
+
const pre = await compactStaleContext(request, { ...semantic, countTokens });
|
|
4563
|
+
const reduced = reduceRequest(pre.request, config);
|
|
4564
|
+
if (pre.change === void 0) return reduced;
|
|
4565
|
+
const afterAll = isPlainObject9(reduced.request) ? measure(reduced.request.instructions) + measure(reduced.request.system) + measure(reduced.request.input) + measure(reduced.request.messages) + measure(reduced.request.tools) : originalBefore;
|
|
4566
|
+
const changes = [pre.change, ...reduced.summary.changes];
|
|
4567
|
+
const summary = {
|
|
4568
|
+
estimatedTokensBefore: originalBefore,
|
|
4569
|
+
estimatedTokensAfter: afterAll,
|
|
4570
|
+
estimatedTokensSaved: Math.max(0, originalBefore - afterAll),
|
|
4571
|
+
estimatedLosslessTokensSaved: changes.filter((change) => change.lossless).reduce((sum, change) => sum + change.estimatedTokensSaved, 0),
|
|
4572
|
+
changes,
|
|
4573
|
+
skipped: false
|
|
4574
|
+
};
|
|
4575
|
+
config.onReduction?.(summary);
|
|
4576
|
+
return { request: reduced.request, summary };
|
|
4577
|
+
}
|
|
4578
|
+
|
|
4579
|
+
// packages/sdk-js/src/report.ts
|
|
4580
|
+
var escapeHtml = (text) => text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
4581
|
+
var usd = (value) => {
|
|
4582
|
+
const sign = value < 0 ? "\u2212" : "";
|
|
4583
|
+
return `${sign}$${Math.abs(value).toFixed(4)}`;
|
|
4584
|
+
};
|
|
4585
|
+
function sectionOf(spool, options) {
|
|
4586
|
+
const chain = verifyLedgerChain(spool.records);
|
|
4587
|
+
const events = spool.records.map((record) => record.event);
|
|
4588
|
+
const inPeriod = options.period === void 0 ? events : events.filter((event) => event.invoicePeriod === options.period);
|
|
4589
|
+
const invoice = recomputeInvoice(events, {
|
|
4590
|
+
...options.period === void 0 ? {} : { period: options.period },
|
|
4591
|
+
feePercent: options.feePercent ?? 1
|
|
4592
|
+
});
|
|
4593
|
+
let spendUsd = 0;
|
|
4594
|
+
let pricedEvents = 0;
|
|
4595
|
+
const waste = { inspected: 0, reEmitted: 0, reEmittedCode: 0, duplicates: 0 };
|
|
4596
|
+
for (const event of inPeriod) {
|
|
4597
|
+
if (typeof event.actualUsd === "number") {
|
|
4598
|
+
spendUsd += event.actualUsd;
|
|
4599
|
+
pricedEvents += 1;
|
|
4600
|
+
}
|
|
4601
|
+
if (event.outputDiagnostics !== void 0) {
|
|
4602
|
+
waste.inspected += event.outputDiagnostics.inspectedOutputTokens;
|
|
4603
|
+
waste.reEmitted += event.outputDiagnostics.reEmittedOutputTokens;
|
|
4604
|
+
waste.reEmittedCode += event.outputDiagnostics.reEmittedCodeTokens;
|
|
4605
|
+
}
|
|
4606
|
+
if (event.duplicateOfRequestId !== void 0) waste.duplicates += 1;
|
|
4607
|
+
}
|
|
4608
|
+
return {
|
|
4609
|
+
name: spool.name,
|
|
4610
|
+
chainOk: chain.ok,
|
|
4611
|
+
chainDetail: chain.ok ? `${chain.length} record(s), every hash links` : `broken at seq ${chain.seq}: ${chain.reason}`,
|
|
4612
|
+
invoice,
|
|
4613
|
+
spendUsd,
|
|
4614
|
+
pricedEvents,
|
|
4615
|
+
waste
|
|
4616
|
+
};
|
|
4617
|
+
}
|
|
4618
|
+
function renderLedgerReport(spools, options = {}) {
|
|
4619
|
+
const feePercent = options.feePercent ?? 1;
|
|
4620
|
+
const sections = spools.map((spool) => sectionOf(spool, options));
|
|
4621
|
+
const allChainsOk = sections.every((section) => section.chainOk);
|
|
4622
|
+
const totalSpend = sections.reduce((sum, section) => sum + section.spendUsd, 0);
|
|
4623
|
+
const totalSavings = sections.reduce((sum, section) => sum + section.invoice.netVerifiedSavingsUsd, 0);
|
|
4624
|
+
const carryIn = options.carryForwardDeficitUsd ?? 0;
|
|
4625
|
+
const netAfterCarry = totalSavings - carryIn;
|
|
4626
|
+
const carryOut = Math.max(0, -netAfterCarry);
|
|
4627
|
+
const totalDue = allChainsOk && Math.round(netAfterCarry * 100) > 0 ? netAfterCarry * feePercent / 100 : 0;
|
|
4628
|
+
const periodLabel = options.period ?? "all recorded periods";
|
|
4629
|
+
const generatedAt = options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
4630
|
+
const sectionHtml = sections.map((section) => {
|
|
4631
|
+
const inv = section.invoice;
|
|
4632
|
+
const lines = inv.lineItems.map((line) => `
|
|
4633
|
+
<tr>
|
|
4634
|
+
<td>${escapeHtml(line.leverId)}${line.basis === "estimated-baseline" ? ' <span class="tag">estimated-baseline</span>' : ""}${line.kind === "lossy" ? ' <span class="tag gate">gate-approved</span>' : ""}</td>
|
|
4635
|
+
<td class="num">${line.events}</td>
|
|
4636
|
+
<td class="num ${line.savedUsd < 0 ? "neg" : ""}">${usd(line.savedUsd)}</td>
|
|
4637
|
+
</tr>`).join("");
|
|
4638
|
+
const unbilled = inv.recordedUnbilled.map((item) => `
|
|
4639
|
+
<tr class="muted"><td>${escapeHtml(item.leverId)} \u2014 ${escapeHtml(item.reason)}</td><td class="num">${item.events}</td><td class="num">not billed</td></tr>`).join("");
|
|
4640
|
+
const excluded = inv.excludedAlreadyOwned;
|
|
4641
|
+
const excludedRows = [
|
|
4642
|
+
["your own provider-cache savings", excluded.providerCacheOnAppendedHistoryUsd],
|
|
4643
|
+
["your own gateway-cache savings", excluded.gatewayCacheUsd]
|
|
4644
|
+
].filter(([, value]) => value !== 0).map(([label, value]) => `
|
|
4645
|
+
<tr class="muted"><td>${label} \u2014 excluded, never billed</td><td class="num"></td><td class="num">${usd(value)}</td></tr>`).join("");
|
|
4646
|
+
return `
|
|
4647
|
+
<section>
|
|
4648
|
+
<h2>${escapeHtml(section.name)}</h2>
|
|
4649
|
+
<p class="chain ${section.chainOk ? "ok" : "bad"}">hash chain ${section.chainOk ? "VERIFIED" : "BROKEN"} \u2014 ${escapeHtml(section.chainDetail)}</p>
|
|
4650
|
+
<p class="meta">${inv.eventsConsidered} event(s) in period \xB7 ${section.pricedEvents} priced \xB7 ${inv.unpricedEvents} unpriced (never guessed)${inv.eventsExcludedForFailedTasks > 0 ? ` \xB7 ${inv.eventsExcludedForFailedTasks} excluded for failed tasks` : ""}</p>
|
|
4651
|
+
<table>
|
|
4652
|
+
<thead><tr><th>verified savings, by lever</th><th class="num">events</th><th class="num">USD</th></tr></thead>
|
|
4653
|
+
<tbody>${lines || '<tr class="muted"><td colspan="3">no billable savings in this period</td></tr>'}${unbilled}${excludedRows}</tbody>
|
|
4654
|
+
<tfoot><tr><td>net verified savings \u2014 spend ${usd(section.spendUsd)}</td><td></td><td class="num ${inv.netVerifiedSavingsUsd < 0 ? "neg" : ""}">${usd(inv.netVerifiedSavingsUsd)}</td></tr></tfoot>
|
|
4655
|
+
</table>
|
|
4656
|
+
${section.waste.inspected > 0 || section.waste.duplicates > 0 ? `<p class="meta">output waste observed (measured, not billed): ${section.waste.reEmitted.toLocaleString("en-US")} of ${section.waste.inspected.toLocaleString("en-US")} inspected output tokens (${section.waste.inspected > 0 ? Math.round(section.waste.reEmitted / section.waste.inspected * 100) : 0}%) re-emit text already in the request${section.waste.reEmittedCode > 0 ? `, ${Math.round(section.waste.reEmittedCode / Math.max(1, section.waste.reEmitted) * 100)}% of it code` : ""}${section.waste.duplicates > 0 ? ` \xB7 ${section.waste.duplicates} byte-identical retried request(s)` : ""} \u2014 a diff-shaped integration would reclaim part of this.</p>` : ""}
|
|
4657
|
+
</section>`;
|
|
4658
|
+
}).join("\n");
|
|
4659
|
+
const verdictLine = !allChainsOk ? "DO NOT PAY \u2014 a ledger chain failed verification. A statement over an unverifiable ledger is not an invoice." : Math.round(netAfterCarry * 100) > 0 ? `Amount due: ${usd(totalDue)} (${feePercent}% of ${usd(netAfterCarry)} net verified savings${carryIn > 0 ? ` after ${usd(carryIn)} deficit carried in` : ""})` : carryOut > 0 ? `Amount due: $0.0000 \u2014 do_not_charge. ${usd(carryOut)} deficit carries forward: future fees apply only after it is recovered.` : "Amount due: $0.0000 \u2014 do_not_charge. A period that saved nothing bills nothing.";
|
|
4660
|
+
return `<!doctype html>
|
|
4661
|
+
<html lang="en">
|
|
4662
|
+
<head>
|
|
4663
|
+
<meta charset="utf-8">
|
|
4664
|
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
4665
|
+
<title>MESH statement \u2014 ${escapeHtml(periodLabel)}</title>
|
|
4666
|
+
<style>
|
|
4667
|
+
:root{--bg:#F1F7FB;--ink:#102A43;--muted:#486581;--ocean:#2864A5;--line:#d9e6f0;--paper:#ffffff;--neg:#B3261E}
|
|
4668
|
+
*{box-sizing:border-box}
|
|
4669
|
+
body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.6 Manrope,-apple-system,"Segoe UI",sans-serif;font-variant-numeric:tabular-nums}
|
|
4670
|
+
main{max-width:720px;margin:0 auto;padding:40px 20px 64px}
|
|
4671
|
+
header{border-bottom:3px solid var(--ocean);padding-bottom:16px;margin-bottom:8px}
|
|
4672
|
+
h1{font-size:26px;margin:0;letter-spacing:-.02em}
|
|
4673
|
+
h2{font-size:15px;margin:28px 0 4px;color:var(--ocean)}
|
|
4674
|
+
.meta{color:var(--muted);font-size:13px;margin:2px 0 8px}
|
|
4675
|
+
.chain{font-size:13px;font-weight:700;margin:2px 0}
|
|
4676
|
+
.chain.ok{color:#1b7f4b}.chain.bad{color:var(--neg)}
|
|
4677
|
+
table{width:100%;border-collapse:collapse;background:var(--paper);border:1px solid var(--line);border-radius:8px;overflow:hidden}
|
|
4678
|
+
th,td{text-align:left;padding:7px 12px;border-bottom:1px solid var(--line);font-size:13.5px}
|
|
4679
|
+
th{background:#e4f4fd;font-weight:700}
|
|
4680
|
+
tfoot td{font-weight:800;border-bottom:none;background:#f7fbfe}
|
|
4681
|
+
.num{text-align:right;white-space:nowrap}
|
|
4682
|
+
.neg{color:var(--neg)}
|
|
4683
|
+
.muted td{color:var(--muted)}
|
|
4684
|
+
.tag{font-size:10.5px;background:#e4f4fd;color:var(--ocean);border-radius:4px;padding:1px 6px;font-weight:700}
|
|
4685
|
+
.tag.gate{background:#e8f6ee;color:#1b7f4b}
|
|
4686
|
+
.due{margin-top:28px;background:var(--ink);color:#F1F7FB;border-radius:10px;padding:18px 20px;font-size:17px;font-weight:800}
|
|
4687
|
+
.due.zero{background:var(--ocean)}
|
|
4688
|
+
.due.bad{background:var(--neg)}
|
|
4689
|
+
footer{margin-top:28px;color:var(--muted);font-size:12.5px;border-top:1px solid var(--line);padding-top:14px}
|
|
4690
|
+
code{background:#e8f1f8;padding:1px 5px;border-radius:4px;font-size:12px}
|
|
4691
|
+
</style>
|
|
4692
|
+
</head>
|
|
4693
|
+
<body>
|
|
4694
|
+
<main>
|
|
4695
|
+
<header>
|
|
4696
|
+
<h1>MESH statement</h1>
|
|
4697
|
+
<p class="meta">period: ${escapeHtml(periodLabel)} \xB7 generated ${escapeHtml(generatedAt)} \xB7 fee ${feePercent}% of verified savings \u2014 nothing else</p>
|
|
4698
|
+
</header>
|
|
4699
|
+
${sectionHtml}
|
|
4700
|
+
<div class="due ${!allChainsOk ? "bad" : Math.round(totalSavings * 100) > 0 ? "" : "zero"}">${escapeHtml(verdictLine)}</div>
|
|
4701
|
+
<p class="meta">total spend across spools ${usd(totalSpend)} \xB7 total net verified savings ${usd(totalSavings)}</p>
|
|
4702
|
+
<footer>
|
|
4703
|
+
Every figure on this statement is recomputed from ledger files on YOUR machine \u2014
|
|
4704
|
+
usage numbers from the provider's own responses, never prompt content. Verify it
|
|
4705
|
+
independently, without MESH and without accepting any terms:
|
|
4706
|
+
<code>npx -y -p @harrym1707/verify mesh-verify <spool.jsonl></code> \u2014 if your
|
|
4707
|
+
recomputation disagrees with this statement, yours wins. Negative levers are
|
|
4708
|
+
shown negative and reduce the bill; savings you already had are itemised above
|
|
4709
|
+
and never billed. Methodology: https://www.mesh-technology.com/methodology.html<br>
|
|
4710
|
+
MESH is a product of Mesh AI Technologies Limited, registered in England and Wales, company no. 17543982.
|
|
4711
|
+
</footer>
|
|
4712
|
+
</main>
|
|
4713
|
+
</body>
|
|
4714
|
+
</html>
|
|
4715
|
+
`;
|
|
4716
|
+
}
|
|
4717
|
+
|
|
4718
|
+
// packages/sdk-js/src/graders.ts
|
|
4719
|
+
function words(text) {
|
|
4720
|
+
return text.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? [];
|
|
4721
|
+
}
|
|
4722
|
+
function overlap(left, right) {
|
|
4723
|
+
const counts = /* @__PURE__ */ new Map();
|
|
4724
|
+
for (const word of left) counts.set(word, (counts.get(word) ?? 0) + 1);
|
|
4725
|
+
let shared = 0;
|
|
4726
|
+
for (const word of right) {
|
|
4727
|
+
const remaining = counts.get(word) ?? 0;
|
|
4728
|
+
if (remaining > 0) {
|
|
4729
|
+
shared += 1;
|
|
4730
|
+
counts.set(word, remaining - 1);
|
|
4731
|
+
}
|
|
4732
|
+
}
|
|
4733
|
+
return shared;
|
|
4734
|
+
}
|
|
4735
|
+
var textSimilarityGrader = ({ baselineResponse, candidateResponse }) => {
|
|
4736
|
+
const baseline = words(responseText(baselineResponse));
|
|
4737
|
+
const candidate = words(responseText(candidateResponse));
|
|
4738
|
+
if (baseline.length === 0 && candidate.length === 0) return 1;
|
|
4739
|
+
if (baseline.length === 0 || candidate.length === 0) return 0;
|
|
4740
|
+
const shared = overlap(baseline, candidate);
|
|
4741
|
+
if (shared === 0) return 0;
|
|
4742
|
+
const precision = shared / candidate.length;
|
|
4743
|
+
const recall = shared / baseline.length;
|
|
4744
|
+
return 2 * precision * recall / (precision + recall);
|
|
4745
|
+
};
|
|
4746
|
+
function wasTruncated(response) {
|
|
4747
|
+
if (typeof response !== "object" || response === null) return false;
|
|
4748
|
+
const row = response;
|
|
4749
|
+
if (row.finish_reason === "length" || row.stop_reason === "max_tokens") return true;
|
|
4750
|
+
if (row.incomplete_details !== null && typeof row.incomplete_details === "object") {
|
|
4751
|
+
const reason = row.incomplete_details.reason;
|
|
4752
|
+
if (reason === "max_output_tokens" || reason === "max_tokens") return true;
|
|
4753
|
+
}
|
|
4754
|
+
if (Array.isArray(row.choices)) {
|
|
4755
|
+
return row.choices.some((choice) => typeof choice === "object" && choice !== null && choice.finish_reason === "length");
|
|
4756
|
+
}
|
|
4757
|
+
return false;
|
|
4758
|
+
}
|
|
4759
|
+
function truncationAwareGrader(options = {}) {
|
|
4760
|
+
const floor = options.uselessBelowCoverage ?? 0.5;
|
|
4761
|
+
const penalty = options.truncationPenalty ?? 0.05;
|
|
4762
|
+
if (floor < 0 || floor > 1) throw new Error("uselessBelowCoverage must be between 0 and 1.");
|
|
4763
|
+
if (penalty < 0 || penalty > 1) throw new Error("truncationPenalty must be between 0 and 1.");
|
|
4764
|
+
return (comparison) => {
|
|
4765
|
+
const baseline = words(responseText(comparison.baselineResponse));
|
|
4766
|
+
const candidate = words(responseText(comparison.candidateResponse));
|
|
4767
|
+
const truncated = wasTruncated(comparison.candidateResponse);
|
|
4768
|
+
if (!truncated) return 1;
|
|
4769
|
+
if (baseline.length === 0) return 1;
|
|
4770
|
+
const coverage = Math.min(1, candidate.length / baseline.length);
|
|
4771
|
+
if (coverage < floor) return 0;
|
|
4772
|
+
return Math.max(0, coverage - penalty);
|
|
4773
|
+
};
|
|
4774
|
+
}
|
|
4775
|
+
function compositeGrader(parts) {
|
|
4776
|
+
if (parts.length === 0) throw new Error("A composite grader needs at least one part.");
|
|
4777
|
+
if (parts.some((part) => !Number.isFinite(part.weight) || part.weight <= 0)) {
|
|
4778
|
+
throw new Error("Composite grader weights must be positive finite numbers.");
|
|
4779
|
+
}
|
|
4780
|
+
const total = parts.reduce((sum, part) => sum + part.weight, 0);
|
|
4781
|
+
return async (comparison) => {
|
|
4782
|
+
let score = 0;
|
|
4783
|
+
for (const part of parts) {
|
|
4784
|
+
const value = await part.grader(comparison);
|
|
4785
|
+
score += (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0) * part.weight;
|
|
4786
|
+
}
|
|
4787
|
+
return score / total;
|
|
4788
|
+
};
|
|
4789
|
+
}
|
|
4790
|
+
function agentGrader(options = {}) {
|
|
4791
|
+
const truncation = truncationAwareGrader(options);
|
|
4792
|
+
return async (comparison) => {
|
|
4793
|
+
const baselineCalls = responseToolCalls(comparison.baselineResponse);
|
|
4794
|
+
const candidateCalls = responseToolCalls(comparison.candidateResponse);
|
|
4795
|
+
if (baselineCalls.length > 0 || candidateCalls.length > 0) {
|
|
4796
|
+
const tool = await toolSelectionGrader(comparison);
|
|
4797
|
+
if (tool < 1) return tool;
|
|
4798
|
+
}
|
|
4799
|
+
return truncation(comparison);
|
|
4800
|
+
};
|
|
4801
|
+
}
|
|
4802
|
+
export {
|
|
4803
|
+
CachePrefixVolatilityGuard,
|
|
4804
|
+
ExactResponseCache,
|
|
4805
|
+
LEDGER_GENESIS_HASH,
|
|
4806
|
+
agentGrader,
|
|
4807
|
+
auditEventPricing,
|
|
4808
|
+
billableEventFromRequest,
|
|
4809
|
+
buildQuoteReport,
|
|
4810
|
+
canonicalJson,
|
|
4811
|
+
capToolResultBody,
|
|
4812
|
+
catalogueVersion,
|
|
4813
|
+
chainLedgerRecord,
|
|
4814
|
+
chooseRoute,
|
|
4815
|
+
compactStaleContext,
|
|
4816
|
+
compositeGrader,
|
|
4817
|
+
createLocalJsonlCapture,
|
|
4818
|
+
createSavingsLedger,
|
|
4819
|
+
defaultPolicy,
|
|
4820
|
+
digestStaleBody,
|
|
4821
|
+
exactTextGrader,
|
|
4822
|
+
extractBlindFeatures,
|
|
4823
|
+
knownLeverIds,
|
|
4824
|
+
ledgerEventHash,
|
|
4825
|
+
parseGatewayExport,
|
|
4826
|
+
policyToReductionConfig,
|
|
4827
|
+
priceLedgerUsage,
|
|
4828
|
+
publishLever,
|
|
4829
|
+
publishedModelPrices,
|
|
4830
|
+
recomputeInvoice,
|
|
4831
|
+
reduceRequest,
|
|
4832
|
+
reduceRequestAsync,
|
|
4833
|
+
renderLedgerReport,
|
|
4834
|
+
renderQuoteReport,
|
|
4835
|
+
replayGate,
|
|
4836
|
+
replayUsage,
|
|
4837
|
+
responseText,
|
|
4838
|
+
responseToolCalls,
|
|
4839
|
+
serviceTierOf,
|
|
4840
|
+
startMeshProxy,
|
|
4841
|
+
textSimilarityGrader,
|
|
4842
|
+
toolNameGrader,
|
|
4843
|
+
toolSelectionGrader,
|
|
4844
|
+
truncationAwareGrader,
|
|
4845
|
+
usageFromSse,
|
|
4846
|
+
verifyLedgerChain,
|
|
4847
|
+
withMesh,
|
|
4848
|
+
withheldLevers,
|
|
4849
|
+
wrapMesh
|
|
4850
|
+
};
|