@harrym1707/sdk 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +96 -0
- package/README.md +48 -2
- package/bin/mesh-demo.mjs +105 -0
- package/bin/mesh-proxy.mjs +39 -0
- package/bin/mesh-quote.mjs +45 -0
- package/bin/mesh-report.mjs +47 -0
- package/index.cjs +4923 -0
- package/index.js +4850 -0
- package/package.json +39 -4
- package/types/blind-features.d.ts +28 -0
- package/types/cache-ttl-advisor.d.ts +31 -0
- package/types/cache-volatility-guard.d.ts +17 -0
- package/types/core/cache-simulator.d.ts +271 -0
- package/types/core/confidence.d.ts +36 -0
- package/types/core/cost.d.ts +16 -0
- package/types/core/evaluator.d.ts +6 -0
- package/types/core/gateway.d.ts +75 -0
- package/types/core/geo-visibility.d.ts +25 -0
- package/types/core/helicone.d.ts +6 -0
- package/types/core/index.d.ts +26 -0
- package/types/core/langfuse.d.ts +7 -0
- package/types/core/litellm.d.ts +15 -0
- package/types/core/local-tier.d.ts +82 -0
- package/types/core/model-prices.d.ts +25 -0
- package/types/core/output-diagnostics.d.ts +39 -0
- package/types/core/price-catalogue.d.ts +168 -0
- package/types/core/provider-usage.d.ts +84 -0
- package/types/core/quote-report.d.ts +54 -0
- package/types/core/replay-runner.d.ts +11 -0
- package/types/core/replay.d.ts +2 -0
- package/types/core/router.d.ts +9 -0
- package/types/core/routing-simulation.d.ts +24 -0
- package/types/core/savings-ledger.d.ts +349 -0
- package/types/core/savings-model.d.ts +132 -0
- package/types/core/stripe-billing.d.ts +67 -0
- package/types/core/stripe-sink.d.ts +77 -0
- package/types/core/theo-coverage.d.ts +49 -0
- package/types/core/token-reduction.d.ts +208 -0
- package/types/core/tool-schema.d.ts +64 -0
- package/types/core/traffic.d.ts +182 -0
- package/types/core/triage.d.ts +70 -0
- package/types/core/types.d.ts +138 -0
- package/types/evaluation-capture.d.ts +11 -0
- package/types/exact-cache.d.ts +20 -0
- package/types/graders.d.ts +53 -0
- package/types/index.d.ts +26 -0
- package/types/openai.d.ts +23 -0
- package/types/proxy.d.ts +63 -0
- package/types/reduce-request.d.ts +247 -0
- package/types/replay-gate.d.ts +120 -0
- package/types/report.d.ts +33 -0
- package/types/routing.d.ts +11 -0
- package/types/savings-ledger.d.ts +12 -0
- package/types/semantic-compaction.d.ts +84 -0
- package/types/telemetry.d.ts +6 -0
- package/types/theo-policy.d.ts +119 -0
- package/types/types.d.ts +263 -0
- package/types/wrap.d.ts +52 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { type TokenPrice } from "./cost.js";
|
|
2
|
+
/**
|
|
3
|
+
* Token reduction: spend fewer tokens on the SAME model, rather than moving work
|
|
4
|
+
* to a weaker one. Model substitution trades quality for cost and needs a quality
|
|
5
|
+
* gate to be safe. Most of the levers here do not: removing a duplicated context
|
|
6
|
+
* block, or letting a provider's prefix cache serve a prompt you already paid
|
|
7
|
+
* for, changes the bill and not the answer.
|
|
8
|
+
*
|
|
9
|
+
* The economics that make this the primary lever rather than a secondary one,
|
|
10
|
+
* using rates captured in model-prices.ts:
|
|
11
|
+
* - Output tokens cost 5x input (Claude Sonnet 5.5) to 8x (gpt-5), so trimming
|
|
12
|
+
* an over-long answer is worth several times trimming the prompt.
|
|
13
|
+
* - Cached input costs 1/20th of fresh input (Sonnet 5.5: $2.00 -> $0.10), so a
|
|
14
|
+
* stable prefix served from cache is close to free.
|
|
15
|
+
*
|
|
16
|
+
* Token counting here is an ESTIMATE used to decide what to cut and to project a
|
|
17
|
+
* saving. It is never the basis of a realised-saving claim: those come from the
|
|
18
|
+
* provider's own reported usage, priced through the catalogue. A deployment
|
|
19
|
+
* should pass the tokenizer matching its model via `countTokens`, because a
|
|
20
|
+
* tokenizer pinned inside MESH would drift from the provider's own counting.
|
|
21
|
+
*/
|
|
22
|
+
export type TokenCounter = (text: string) => number;
|
|
23
|
+
/**
|
|
24
|
+
* Tokenizer-free estimate. Byte-pair encoders average roughly 4 characters per
|
|
25
|
+
* token on English prose and nearer 3 on code, punctuation and JSON, so this
|
|
26
|
+
* counts word-like runs and separators separately instead of dividing the whole
|
|
27
|
+
* string by one constant. Expect roughly +/-15% against a real tokenizer; good
|
|
28
|
+
* enough to rank candidate reductions, not good enough to bill on.
|
|
29
|
+
*/
|
|
30
|
+
export declare const approximateTokenCount: TokenCounter;
|
|
31
|
+
export type ReductionRisk = "lossless" | "low" | "moderate";
|
|
32
|
+
export type ReductionOutcome = {
|
|
33
|
+
reducer: string;
|
|
34
|
+
version: string;
|
|
35
|
+
risk: ReductionRisk;
|
|
36
|
+
text: string;
|
|
37
|
+
tokensBefore: number;
|
|
38
|
+
tokensAfter: number;
|
|
39
|
+
tokensSaved: number;
|
|
40
|
+
/** What the reducer removed, for review before anything is enabled. */
|
|
41
|
+
notes: string[];
|
|
42
|
+
};
|
|
43
|
+
export type Reducer = {
|
|
44
|
+
id: string;
|
|
45
|
+
version: string;
|
|
46
|
+
risk: ReductionRisk;
|
|
47
|
+
reduce: (text: string) => {
|
|
48
|
+
text: string;
|
|
49
|
+
notes: string[];
|
|
50
|
+
};
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Removes exact duplicate blocks. Retrieval pipelines routinely send the same
|
|
54
|
+
* passage two or three times (overlapping chunks, a document that matched several
|
|
55
|
+
* queries, a system prompt echoed into the first user turn). Dropping later exact
|
|
56
|
+
* copies is lossless: the model already has the text.
|
|
57
|
+
*
|
|
58
|
+
* Only blocks at or above `minimumBlockTokens` are considered, so ordinary
|
|
59
|
+
* repeated phrasing is left alone.
|
|
60
|
+
*
|
|
61
|
+
* Duplicates are caught at two granularities: whole blocks separated by a
|
|
62
|
+
* blank line (as before), and, inside a block that survives that check,
|
|
63
|
+
* individual lines separated by a single newline. A RAG prompt that joins
|
|
64
|
+
* several retrieved excerpts with `picks.join("\n")` never produces a
|
|
65
|
+
* duplicated top-level block — the whole joined block is unique per request
|
|
66
|
+
* because the question text differs — but an excerpt pulled twice is still an
|
|
67
|
+
* exact duplicate line inside it, and dropping it is exactly as lossless as
|
|
68
|
+
* dropping a duplicate paragraph.
|
|
69
|
+
*/
|
|
70
|
+
export declare function dedupeRepeatedBlocks(minimumBlockTokens?: number, countTokens?: TokenCounter): Reducer;
|
|
71
|
+
/**
|
|
72
|
+
* Collapses redundant whitespace outside fenced and inline code, where layout is
|
|
73
|
+
* meaningful. Individually small; free, and it compounds across every request.
|
|
74
|
+
*/
|
|
75
|
+
export declare function collapseWhitespace(): Reducer;
|
|
76
|
+
/** Run reducers in order, reporting what each one saved. */
|
|
77
|
+
export declare function applyReducers(text: string, reducers: Reducer[], countTokens?: TokenCounter): {
|
|
78
|
+
text: string;
|
|
79
|
+
outcomes: ReductionOutcome[];
|
|
80
|
+
tokensBefore: number;
|
|
81
|
+
tokensAfter: number;
|
|
82
|
+
};
|
|
83
|
+
export type PrefixBreaker = {
|
|
84
|
+
/** What kind of per-request value is sitting inside the would-be stable prefix. */
|
|
85
|
+
kind: "timestamp" | "date" | "uuid" | "long-number" | "unknown";
|
|
86
|
+
example: string;
|
|
87
|
+
occurrences: number;
|
|
88
|
+
};
|
|
89
|
+
export type CachingProjection = {
|
|
90
|
+
status: "priced";
|
|
91
|
+
baselineCostUsd: number;
|
|
92
|
+
cachedCostUsd: number;
|
|
93
|
+
savingUsd: number;
|
|
94
|
+
savingPercent: number;
|
|
95
|
+
recoverableSavingUsd: number;
|
|
96
|
+
} | {
|
|
97
|
+
status: "unpriced";
|
|
98
|
+
reason: string;
|
|
99
|
+
};
|
|
100
|
+
export type CachingAnalysis = {
|
|
101
|
+
requests: number;
|
|
102
|
+
meanInputTokens: number;
|
|
103
|
+
/** Tokens identical across every sampled request, from the start of the prompt. */
|
|
104
|
+
sharedPrefixTokens: number;
|
|
105
|
+
/** Share of a request's input that the provider could serve from cache today. */
|
|
106
|
+
cacheablePrefixShare: number;
|
|
107
|
+
/**
|
|
108
|
+
* Shared prefix that would exist if the per-request values listed in
|
|
109
|
+
* `prefixBreakers` were moved out of the prefix. The gap between this and
|
|
110
|
+
* `sharedPrefixTokens` is the cost of a timestamp in a system prompt.
|
|
111
|
+
*/
|
|
112
|
+
recoverablePrefixTokens: number;
|
|
113
|
+
prefixBreakers: PrefixBreaker[];
|
|
114
|
+
projection: CachingProjection;
|
|
115
|
+
};
|
|
116
|
+
export declare function commonPrefixLength(left: string, right: string): number;
|
|
117
|
+
/**
|
|
118
|
+
* Removes a repeated LEADING block from texts after the first, when that
|
|
119
|
+
* block is an exact match for the start of the first text in the group.
|
|
120
|
+
*
|
|
121
|
+
* This is the shape a continuous-learning / continuous-memory agent
|
|
122
|
+
* produces: the application glues the same memory or persistent-instruction
|
|
123
|
+
* block to the front of every turn, with only a short turn-specific
|
|
124
|
+
* remainder varying underneath it. `dedupeRepeatedBlocks` never catches
|
|
125
|
+
* this, because it only ever sees one text at a time and the memory block
|
|
126
|
+
* is not a standalone paragraph inside any single text - it is the shared
|
|
127
|
+
* PREFIX of several texts that each continue differently.
|
|
128
|
+
*
|
|
129
|
+
* LOSSY, not lossless, unlike `dedupeRepeatedBlocks`: a same-text duplicate
|
|
130
|
+
* sits right next to its twin, so the model reads both in the same breath.
|
|
131
|
+
* The duplicate removed here can be many turns upstream of the current one,
|
|
132
|
+
* which is a real behaviour change - a model that attends less to context
|
|
133
|
+
* far back in a long history may act differently with the block restated
|
|
134
|
+
* fresh every turn versus present once near the start. Callers must run
|
|
135
|
+
* this behind a replay-gate pass on their own traffic before billing it;
|
|
136
|
+
* it is never enabled by default.
|
|
137
|
+
*
|
|
138
|
+
* The cut point is always snapped to the last newline at or before the raw
|
|
139
|
+
* common-prefix length (never cut THROUGH a line), unless the match is a
|
|
140
|
+
* complete containment of one text inside the other, which needs no
|
|
141
|
+
* snapping because there is no partially-matched line to protect. This
|
|
142
|
+
* means a false positive - two texts that happen to start with the same
|
|
143
|
+
* few words before diverging mid-line, e.g. two turns both opening "Turn "
|
|
144
|
+
* - can only ever leave that differing line untouched, never truncate into it.
|
|
145
|
+
*/
|
|
146
|
+
export declare function dedupeRepeatedLeadingBlock(texts: readonly string[], minimumBlockTokens?: number, countTokens?: TokenCounter): {
|
|
147
|
+
texts: string[];
|
|
148
|
+
notes: string[];
|
|
149
|
+
};
|
|
150
|
+
/**
|
|
151
|
+
* How much of a workload's prompt a provider's prefix cache could serve, and what
|
|
152
|
+
* is stopping it.
|
|
153
|
+
*
|
|
154
|
+
* Providers cache on an exact prefix match, so a single value that changes per
|
|
155
|
+
* request invalidates everything after it. A current-date line at the top of a
|
|
156
|
+
* system prompt can therefore cost the entire prompt's cache discount, which is
|
|
157
|
+
* the difference between paying full input rate and one twentieth of it. That
|
|
158
|
+
* failure is invisible in a cost dashboard: spend simply stays high.
|
|
159
|
+
*
|
|
160
|
+
* `requests` should be a representative sample of real prompts for one workload.
|
|
161
|
+
*/
|
|
162
|
+
export declare function analysePromptCaching(requests: string[], options?: {
|
|
163
|
+
price?: TokenPrice;
|
|
164
|
+
countTokens?: TokenCounter;
|
|
165
|
+
/** Minimum prefix a provider will cache at all; OpenAI's floor is 1024 tokens. */
|
|
166
|
+
minimumCacheableTokens?: number;
|
|
167
|
+
}): CachingAnalysis;
|
|
168
|
+
export type OutputCapAdvice = {
|
|
169
|
+
observations: number;
|
|
170
|
+
/** Cap that would have truncated none of the observed responses. */
|
|
171
|
+
maxObserved: number;
|
|
172
|
+
recommendedCap: number;
|
|
173
|
+
/** Share of observed responses the recommended cap would have truncated. */
|
|
174
|
+
truncationRate: number;
|
|
175
|
+
meanOutputTokens: number;
|
|
176
|
+
/**
|
|
177
|
+
* Saving only materialises on responses that would have run past the cap, so
|
|
178
|
+
* this is deliberately conservative: it counts the tokens above the cap and
|
|
179
|
+
* nothing else.
|
|
180
|
+
*/
|
|
181
|
+
projectedTokensSaved: number;
|
|
182
|
+
projectedSavingUsd?: number;
|
|
183
|
+
};
|
|
184
|
+
/**
|
|
185
|
+
* Recommend `max_output_tokens` from observed response lengths.
|
|
186
|
+
*
|
|
187
|
+
* Output is the expensive half of the bill (5x input on Sonnet 5.5, 8x on gpt-5),
|
|
188
|
+
* and an uncapped request lets a model ramble at that rate. A cap set at a high
|
|
189
|
+
* quantile of what a workload actually produces removes the tail without touching
|
|
190
|
+
* the responses that were already the right length.
|
|
191
|
+
*
|
|
192
|
+
* This caps length; it does not make answers shorter at equal quality. Any
|
|
193
|
+
* response that would have exceeded the cap gets truncated, so `truncationRate`
|
|
194
|
+
* is the number to judge it on, and a cap belongs behind a replay check before
|
|
195
|
+
* rollout.
|
|
196
|
+
*/
|
|
197
|
+
export declare function recommendOutputCap(observedOutputTokens: number[], options?: {
|
|
198
|
+
quantile?: number;
|
|
199
|
+
price?: TokenPrice;
|
|
200
|
+
headroom?: number;
|
|
201
|
+
}): OutputCapAdvice;
|
|
202
|
+
export type ReductionSaving = {
|
|
203
|
+
inputTokensSaved: number;
|
|
204
|
+
outputTokensSaved: number;
|
|
205
|
+
savingUsd?: number;
|
|
206
|
+
};
|
|
207
|
+
/** Price a projected reduction. Returns tokens only when no price is supplied. */
|
|
208
|
+
export declare function priceReduction(inputTokensSaved: number, outputTokensSaved: number, price?: TokenPrice): ReductionSaving;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { type TokenCounter } from "./token-reduction.js";
|
|
2
|
+
export type MinifyOptions = {
|
|
3
|
+
/**
|
|
4
|
+
* What to do with description fields.
|
|
5
|
+
* `keep` stays lossless. `truncate` clips to `maximumDescriptionTokens`.
|
|
6
|
+
* `drop` removes them, which is the largest saving and the largest risk.
|
|
7
|
+
*/
|
|
8
|
+
descriptions?: "keep" | "truncate" | "drop";
|
|
9
|
+
maximumDescriptionTokens?: number;
|
|
10
|
+
/** Remove `examples` arrays. The model does read these, so this is lossy. */
|
|
11
|
+
dropExamples?: boolean;
|
|
12
|
+
/** Tool names to keep. Omit to keep all; naming a subset is lossy. */
|
|
13
|
+
keepTools?: readonly string[];
|
|
14
|
+
countTokens?: TokenCounter;
|
|
15
|
+
};
|
|
16
|
+
export type MinifyChange = {
|
|
17
|
+
kind: "plumbing" | "redundant-title" | "formatting" | "description" | "examples" | "dropped-tool";
|
|
18
|
+
lossless: boolean;
|
|
19
|
+
/** Where in the catalogue, e.g. a tool name or a property path. */
|
|
20
|
+
at: string;
|
|
21
|
+
tokensSaved: number;
|
|
22
|
+
detail: string;
|
|
23
|
+
};
|
|
24
|
+
export type MinifyResult = {
|
|
25
|
+
tools: unknown[];
|
|
26
|
+
tokensBefore: number;
|
|
27
|
+
tokensAfter: number;
|
|
28
|
+
/** Tokens removed without any possibility of changing tool selection. */
|
|
29
|
+
losslessTokensSaved: number;
|
|
30
|
+
/** Tokens removed by altering what the model reads. Needs a replay check. */
|
|
31
|
+
lossyTokensSaved: number;
|
|
32
|
+
changes: MinifyChange[];
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Minify a tool catalogue, reporting every change and which tier it belongs to.
|
|
36
|
+
*
|
|
37
|
+
* Defaults are lossless only: descriptions are kept. A caller wanting the large
|
|
38
|
+
* saving opts into it explicitly and gets it reported as lossy.
|
|
39
|
+
*/
|
|
40
|
+
export declare function minifyToolSchemas(tools: unknown[], options?: MinifyOptions): MinifyResult;
|
|
41
|
+
export type ToolCatalogueAudit = {
|
|
42
|
+
tools: number;
|
|
43
|
+
tokens: number;
|
|
44
|
+
meanTokensPerTool: number;
|
|
45
|
+
/** Largest offenders first, so a catalogue can be trimmed by hand. */
|
|
46
|
+
largest: Array<{
|
|
47
|
+
name: string;
|
|
48
|
+
tokens: number;
|
|
49
|
+
descriptionTokens: number;
|
|
50
|
+
}>;
|
|
51
|
+
/** Achievable without any behaviour change. */
|
|
52
|
+
losslessTokensSaved: number;
|
|
53
|
+
/** Additionally achievable by trimming descriptions, behind a replay check. */
|
|
54
|
+
descriptionTokensSaved: number;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Audit a catalogue: how big it is, which tools dominate, and how much each tier
|
|
58
|
+
* of minification would recover. This is what makes the lever measured rather
|
|
59
|
+
* than assumed, so a workload with an already-lean catalogue is credited nothing.
|
|
60
|
+
*/
|
|
61
|
+
export declare function auditToolCatalogue(tools: unknown[], options?: {
|
|
62
|
+
countTokens?: TokenCounter;
|
|
63
|
+
maximumDescriptionTokens?: number;
|
|
64
|
+
}): ToolCatalogueAudit;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { type PriceCatalogue } from "./price-catalogue.js";
|
|
2
|
+
import { type Lever, type WorkloadProfile } from "./savings-model.js";
|
|
3
|
+
import { type ToolCatalogueAudit } from "./tool-schema.js";
|
|
4
|
+
import { type CachingAnalysis, type OutputCapAdvice, type TokenCounter } from "./token-reduction.js";
|
|
5
|
+
/**
|
|
6
|
+
* Ingest real traffic and measure what a reduction would be worth on it.
|
|
7
|
+
*
|
|
8
|
+
* Two tiers of input, because most teams can produce the first immediately and
|
|
9
|
+
* the second only after a conversation about privacy:
|
|
10
|
+
*
|
|
11
|
+
* Tier 1, usage only (model, token counts, date): prices the baseline, finds
|
|
12
|
+
* the model mix, and recommends output caps. No prompt text is needed, so
|
|
13
|
+
* this runs on billing exports.
|
|
14
|
+
* Tier 2, usage plus prompt text: adds prefix-cache analysis, what is breaking
|
|
15
|
+
* the cache, duplicate context, and measured tool-schema overhead.
|
|
16
|
+
*
|
|
17
|
+
* Nothing here performs I/O or leaves the process. Prompt text is used to
|
|
18
|
+
* compute counts and hashes and is never copied into a report.
|
|
19
|
+
*/
|
|
20
|
+
export type TrafficRecord = {
|
|
21
|
+
id: string;
|
|
22
|
+
model: string;
|
|
23
|
+
provider: string;
|
|
24
|
+
/** ISO date the request was billed on, used to resolve the rate that applied. */
|
|
25
|
+
at: string;
|
|
26
|
+
inputTokens: number;
|
|
27
|
+
outputTokens: number;
|
|
28
|
+
/** Input already served from the provider's cache. A saving the customer HAS. */
|
|
29
|
+
cachedInputTokens?: number;
|
|
30
|
+
cacheWriteInputTokens?: number;
|
|
31
|
+
/** Full prompt text, when the log carries it. Enables tier 2 analysis. */
|
|
32
|
+
promptText?: string;
|
|
33
|
+
/** Serialised tool or function definitions sent with the request. */
|
|
34
|
+
toolSchemaText?: string;
|
|
35
|
+
/** Logical grouping. Prompts only share a prefix within one workload. */
|
|
36
|
+
workload: string;
|
|
37
|
+
/**
|
|
38
|
+
* Task this request belongs to, where the log says so. Several requests share
|
|
39
|
+
* one episode id when an agent takes several turns to finish one job.
|
|
40
|
+
*/
|
|
41
|
+
episodeId?: string;
|
|
42
|
+
};
|
|
43
|
+
export type IngestIssue = {
|
|
44
|
+
line: number;
|
|
45
|
+
reason: string;
|
|
46
|
+
};
|
|
47
|
+
export type IngestResult = {
|
|
48
|
+
records: TrafficRecord[];
|
|
49
|
+
issues: IngestIssue[];
|
|
50
|
+
/** Which adapter matched each record, so an odd result can be traced. */
|
|
51
|
+
adapterCounts: Record<string, number>;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Normalise one logged row.
|
|
55
|
+
*
|
|
56
|
+
* Accepts the OpenAI Responses and Chat Completions shapes, the Anthropic
|
|
57
|
+
* Messages shape, and a bare usage row, with the request and response either
|
|
58
|
+
* nested under `request`/`response` or flattened onto one object. Returns the
|
|
59
|
+
* adapter that matched so a surprising report can be traced to its parser.
|
|
60
|
+
*/
|
|
61
|
+
export declare function normaliseTrafficRow(row: unknown, fallbackDate: string, index: number): {
|
|
62
|
+
record: TrafficRecord;
|
|
63
|
+
adapter: string;
|
|
64
|
+
} | {
|
|
65
|
+
issue: string;
|
|
66
|
+
};
|
|
67
|
+
/** Normalise many rows, collecting per-row problems rather than failing the run. */
|
|
68
|
+
export declare function ingestTraffic(rows: unknown[], fallbackDate: string): IngestResult;
|
|
69
|
+
/**
|
|
70
|
+
* Per-task view of a workload.
|
|
71
|
+
*
|
|
72
|
+
* The unit that matters for agentic traffic, and the one the market does not
|
|
73
|
+
* report. A technique that cuts each prompt by 91% while tripling the number of
|
|
74
|
+
* prompts has saved nothing, and per-request figures cannot show that: one
|
|
75
|
+
* published bench found progressive disclosure modelled at ~77% delivering ~13%
|
|
76
|
+
* in production, because a router adds a round trip per novel tool call.
|
|
77
|
+
*
|
|
78
|
+
* `identified` matters. Without a task identifier in the log every request is
|
|
79
|
+
* its own episode, round trips read as 1.0, and nothing here is informative.
|
|
80
|
+
* It is reported rather than silently assumed.
|
|
81
|
+
*/
|
|
82
|
+
export type EpisodeStats = {
|
|
83
|
+
episodes: number;
|
|
84
|
+
/** Mean requests per task. This is the round-trip count a technique can change. */
|
|
85
|
+
meanRoundTrips: number;
|
|
86
|
+
maxRoundTrips: number;
|
|
87
|
+
meanInputTokens: number;
|
|
88
|
+
meanOutputTokens: number;
|
|
89
|
+
meanCostUsd?: number;
|
|
90
|
+
/** False when the log carried no task identifier, making the above meaningless. */
|
|
91
|
+
identified: boolean;
|
|
92
|
+
};
|
|
93
|
+
export type WorkloadMeasurement = {
|
|
94
|
+
workload: string;
|
|
95
|
+
/** Model identifier exactly as the log carried it. */
|
|
96
|
+
model: string;
|
|
97
|
+
/** Catalogue key the identifier resolved to, when it resolved at all. */
|
|
98
|
+
resolvedModel?: string;
|
|
99
|
+
provider: string;
|
|
100
|
+
requests: number;
|
|
101
|
+
inputTokens: number;
|
|
102
|
+
outputTokens: number;
|
|
103
|
+
/** Tokens the provider already served from cache before MESH touched anything. */
|
|
104
|
+
alreadyCachedInputTokens: number;
|
|
105
|
+
meanToolSchemaTokens: number;
|
|
106
|
+
/** Measured from the request's own tool definitions; absent when none were sent. */
|
|
107
|
+
toolAudit?: ToolCatalogueAudit;
|
|
108
|
+
/** Present only when the log carried prompt text. */
|
|
109
|
+
caching?: CachingAnalysis;
|
|
110
|
+
outputCap?: OutputCapAdvice;
|
|
111
|
+
duplicateContextTokens: number;
|
|
112
|
+
episodes: EpisodeStats;
|
|
113
|
+
baselineCostUsd?: number;
|
|
114
|
+
/** Set when the catalogue cannot price this model on these dates. */
|
|
115
|
+
unpricedReason?: string;
|
|
116
|
+
};
|
|
117
|
+
export type TrafficMeasurement = {
|
|
118
|
+
requests: number;
|
|
119
|
+
workloads: WorkloadMeasurement[];
|
|
120
|
+
totalBaselineCostUsd: number;
|
|
121
|
+
/** Models the catalogue could not price; their spend is excluded, never assumed zero. */
|
|
122
|
+
unpricedModels: string[];
|
|
123
|
+
/** Share of input tokens already arriving cached, which MESH must not claim credit for. */
|
|
124
|
+
alreadyCachedShare: number;
|
|
125
|
+
promptTextAvailable: boolean;
|
|
126
|
+
/** True when at least one record carried a task identifier. */
|
|
127
|
+
episodesIdentified: boolean;
|
|
128
|
+
notes: string[];
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Measure a traffic sample.
|
|
132
|
+
*
|
|
133
|
+
* Deliberately conservative in two places. Input already arriving cached is
|
|
134
|
+
* reported separately and never counted as a MESH saving, because the customer
|
|
135
|
+
* already has it. And a model the catalogue cannot price is named and excluded
|
|
136
|
+
* rather than treated as free, the same discipline applied to benchmark sources.
|
|
137
|
+
*/
|
|
138
|
+
export declare function measureTraffic(records: TrafficRecord[], catalogue: PriceCatalogue, options?: {
|
|
139
|
+
countTokens?: TokenCounter;
|
|
140
|
+
maximumPromptSample?: number;
|
|
141
|
+
}): TrafficMeasurement;
|
|
142
|
+
/** Stable fingerprint of a traffic sample, so repeated runs can be compared. */
|
|
143
|
+
export declare function trafficFingerprint(records: TrafficRecord[]): string;
|
|
144
|
+
/**
|
|
145
|
+
* Turn a measured workload into a profile the savings model can project on.
|
|
146
|
+
*
|
|
147
|
+
* Segments come from what was actually observed rather than assumed: the tool
|
|
148
|
+
* catalogue is measured from the request's own tool definitions, the stable
|
|
149
|
+
* prefix from what the sampled prompts genuinely share, and whatever remains is
|
|
150
|
+
* treated as the volatile body. Tokens already arriving cached are removed from
|
|
151
|
+
* the stable segment, because that saving is the customer's already.
|
|
152
|
+
*/
|
|
153
|
+
export declare function profileFromMeasurement(measurement: WorkloadMeasurement, options?: {
|
|
154
|
+
repairPrefix?: boolean;
|
|
155
|
+
}): WorkloadProfile;
|
|
156
|
+
/**
|
|
157
|
+
* Build the lever set a measured workload actually supports.
|
|
158
|
+
*
|
|
159
|
+
* `leversFor` applies published reduction factors to any profile with non-empty
|
|
160
|
+
* segments, which is right for archetype modelling and wrong for a customer
|
|
161
|
+
* report: it credited a 60% output reduction to workloads whose replies were
|
|
162
|
+
* already short and tight, and that was the only lever firing on most samples.
|
|
163
|
+
*
|
|
164
|
+
* Here every factor is either measured from the sample or gated on measured
|
|
165
|
+
* evidence that the lever applies at all:
|
|
166
|
+
*
|
|
167
|
+
* - Output capping uses the observed length distribution, so a workload with
|
|
168
|
+
* no long tail is credited nothing.
|
|
169
|
+
* - Duplicate removal is credited exactly the duplicate tokens found.
|
|
170
|
+
* - Tool levers keep their published factors, which are deterministic
|
|
171
|
+
* transformations of a catalogue we measured directly, but only fire when
|
|
172
|
+
* tool definitions were actually present.
|
|
173
|
+
* - Reasoning-effort tuning is offered only where mean output is long enough
|
|
174
|
+
* for reasoning tokens to plausibly dominate, at well under the published
|
|
175
|
+
* factor, and always as needs-verification.
|
|
176
|
+
*
|
|
177
|
+
* Prompt compression and history compaction are deliberately absent. Their
|
|
178
|
+
* published factors are real but workload-specific, and nothing in a usage log
|
|
179
|
+
* measures whether they transfer. They belong behind a replay check, not in an
|
|
180
|
+
* opening projection.
|
|
181
|
+
*/
|
|
182
|
+
export declare function leversFromMeasurement(measurement: WorkloadMeasurement): Lever[];
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Horizontal customer triage: which kind of traffic is this, and what is MESH
|
|
3
|
+
* honestly worth on it?
|
|
4
|
+
*
|
|
5
|
+
* The A/B results made one thing unambiguous: the saving MESH delivers is a
|
|
6
|
+
* property of the WORKLOAD SHAPE, not of MESH. The same software measured
|
|
7
|
+
* ~2% from caching on a multi-turn agent (whose appended history already hits
|
|
8
|
+
* the provider cache ~90% of the time), ~40% from prefix repair on one-shot
|
|
9
|
+
* templated traffic (whose embedded clock blocked the cache entirely), ~30%
|
|
10
|
+
* from output budgeting where answers had a long tail, and ~3% on a workload
|
|
11
|
+
* that was already well cached.
|
|
12
|
+
*
|
|
13
|
+
* A horizontal product therefore cannot quote one number. It has to classify
|
|
14
|
+
* the stream first and quote the band that stream's shape supports, with the
|
|
15
|
+
* evidence attached — including "you are already optimised, do not pay us",
|
|
16
|
+
* because a percent-of-savings business that charges for savings the customer
|
|
17
|
+
* already has is buying refunds and churn.
|
|
18
|
+
*
|
|
19
|
+
* Deliberately gateway-agnostic: the signals are derivable from any usage log,
|
|
20
|
+
* LiteLLM's spend logs included.
|
|
21
|
+
*/
|
|
22
|
+
export type StreamSignals = {
|
|
23
|
+
requests: number;
|
|
24
|
+
/** Distinct downstream customers (a gateway's end_user), where known. */
|
|
25
|
+
distinctEndUsers: number;
|
|
26
|
+
requestsPerEndUser: number;
|
|
27
|
+
episodesIdentified: boolean;
|
|
28
|
+
meanRoundTrips: number;
|
|
29
|
+
/** Share of input tokens that are tool/MCP schema. */
|
|
30
|
+
toolSchemaShare: number;
|
|
31
|
+
/** Share of input already arriving at the provider cached. */
|
|
32
|
+
alreadyCachedShare: number;
|
|
33
|
+
/** Share of requests a gateway's own response cache already answers. */
|
|
34
|
+
gatewayCacheHitShare: number;
|
|
35
|
+
promptTextAvailable: boolean;
|
|
36
|
+
prefixBreakersFound: boolean;
|
|
37
|
+
meanInputTokens: number;
|
|
38
|
+
};
|
|
39
|
+
export type CustomerSegment = "already-optimised" | "consumer-app" | "in-house-agent" | "application-layer";
|
|
40
|
+
export type SegmentClassification = {
|
|
41
|
+
segment: CustomerSegment;
|
|
42
|
+
evidence: string[];
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* Classify one stream. Rules are ordered so the strongest signal wins:
|
|
46
|
+
* existing optimisation first (nothing honest to sell), then the many-end-user
|
|
47
|
+
* consumer shape, then the session-heavy agent shape, with server-side
|
|
48
|
+
* application traffic as the remainder.
|
|
49
|
+
*/
|
|
50
|
+
export declare function classifySegment(signals: StreamSignals): SegmentClassification;
|
|
51
|
+
export type QuoteConfidence = "measured-analogue" | "needs-prompt-text" | "do-not-charge";
|
|
52
|
+
export type SegmentQuote = {
|
|
53
|
+
segment: CustomerSegment;
|
|
54
|
+
/** Honest range for total cost reduction, as percentages of current spend. */
|
|
55
|
+
bandPercent: [number, number];
|
|
56
|
+
topLever: string;
|
|
57
|
+
confidence: QuoteConfidence;
|
|
58
|
+
/** What would turn the band into a firm figure. */
|
|
59
|
+
firmUpBy?: string;
|
|
60
|
+
reasoning: string[];
|
|
61
|
+
doNotCharge?: boolean;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Quote a band for a classified stream.
|
|
65
|
+
*
|
|
66
|
+
* Every band cites the measurement it is an analogue of; none is a marketing
|
|
67
|
+
* number. Where the deciding evidence (prompt text) is absent, the band is
|
|
68
|
+
* wide and says exactly what would narrow it.
|
|
69
|
+
*/
|
|
70
|
+
export declare function quoteForSegment(classification: SegmentClassification, signals: StreamSignals): SegmentQuote;
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
export type ModelTarget = {
|
|
2
|
+
provider: string;
|
|
3
|
+
model: string;
|
|
4
|
+
};
|
|
5
|
+
export type QualityResult = {
|
|
6
|
+
passed: boolean;
|
|
7
|
+
score?: number;
|
|
8
|
+
checks: Record<string, boolean | number | string>;
|
|
9
|
+
reason?: string;
|
|
10
|
+
};
|
|
11
|
+
export type QualityPolicy = {
|
|
12
|
+
minimumScore?: number;
|
|
13
|
+
requiredChecks?: string[];
|
|
14
|
+
maxErrorRate?: number;
|
|
15
|
+
maxLatencyMs?: number;
|
|
16
|
+
};
|
|
17
|
+
export type RoutingCandidate = {
|
|
18
|
+
target: ModelTarget;
|
|
19
|
+
estimatedCostMultiplier: number;
|
|
20
|
+
rolloutPercentage: number;
|
|
21
|
+
qualityPolicy: QualityPolicy;
|
|
22
|
+
};
|
|
23
|
+
export type RoutingPolicy = {
|
|
24
|
+
id: string;
|
|
25
|
+
version: number;
|
|
26
|
+
taskClass: string;
|
|
27
|
+
baseline: ModelTarget;
|
|
28
|
+
candidates: RoutingCandidate[];
|
|
29
|
+
enabled: boolean;
|
|
30
|
+
};
|
|
31
|
+
export type RoutingDecision = {
|
|
32
|
+
target: ModelTarget;
|
|
33
|
+
reason: string;
|
|
34
|
+
policyId?: string;
|
|
35
|
+
policyVersion?: number;
|
|
36
|
+
isBaseline: boolean;
|
|
37
|
+
};
|
|
38
|
+
export type InferenceEvent = {
|
|
39
|
+
requestId: string;
|
|
40
|
+
applicationId: string;
|
|
41
|
+
taskClass?: string;
|
|
42
|
+
provider: string;
|
|
43
|
+
model: string;
|
|
44
|
+
inputTokens?: number;
|
|
45
|
+
outputTokens?: number;
|
|
46
|
+
costUsd?: number;
|
|
47
|
+
latencyMs?: number;
|
|
48
|
+
policyId?: string;
|
|
49
|
+
policyVersion?: number;
|
|
50
|
+
routeReason?: string;
|
|
51
|
+
timestamp: string;
|
|
52
|
+
};
|
|
53
|
+
export type EvaluationInput = {
|
|
54
|
+
baseline: unknown;
|
|
55
|
+
candidate: unknown;
|
|
56
|
+
metadata?: Record<string, unknown>;
|
|
57
|
+
};
|
|
58
|
+
export type Evaluator = {
|
|
59
|
+
name: string;
|
|
60
|
+
evaluate(input: EvaluationInput): Promise<QualityResult>;
|
|
61
|
+
};
|
|
62
|
+
export type ReplayRecord = {
|
|
63
|
+
id: string;
|
|
64
|
+
expected: unknown;
|
|
65
|
+
baseline: unknown;
|
|
66
|
+
candidate: unknown;
|
|
67
|
+
baselineCostUsd?: number;
|
|
68
|
+
candidateCostUsd?: number;
|
|
69
|
+
baselineError?: boolean;
|
|
70
|
+
candidateError?: boolean;
|
|
71
|
+
};
|
|
72
|
+
export type ReplayInputCase = {
|
|
73
|
+
id: string;
|
|
74
|
+
expected: unknown;
|
|
75
|
+
input: string;
|
|
76
|
+
/** Candidate-side prompt/context after the configured reduction profile. */
|
|
77
|
+
candidateInput?: string;
|
|
78
|
+
mockOutputs?: {
|
|
79
|
+
baseline: unknown;
|
|
80
|
+
candidate: unknown;
|
|
81
|
+
inputTokens: number;
|
|
82
|
+
baselineOutputTokens: number;
|
|
83
|
+
candidateOutputTokens: number;
|
|
84
|
+
};
|
|
85
|
+
};
|
|
86
|
+
export type ReplayTarget = {
|
|
87
|
+
provider: string;
|
|
88
|
+
model: string;
|
|
89
|
+
maxOutputTokens?: number;
|
|
90
|
+
contextProfileId?: string;
|
|
91
|
+
pricing?: {
|
|
92
|
+
inputPerMillion: number;
|
|
93
|
+
cachedInputPerMillion?: number;
|
|
94
|
+
cacheWriteInputPerMillion?: number;
|
|
95
|
+
outputPerMillion: number;
|
|
96
|
+
};
|
|
97
|
+
};
|
|
98
|
+
export type ReplayInvocation = {
|
|
99
|
+
output?: unknown;
|
|
100
|
+
inputTokens?: number;
|
|
101
|
+
cachedInputTokens?: number;
|
|
102
|
+
cacheWriteInputTokens?: number;
|
|
103
|
+
outputTokens?: number;
|
|
104
|
+
error?: string;
|
|
105
|
+
};
|
|
106
|
+
export type ReplayAdapter = {
|
|
107
|
+
invoke(inputCase: ReplayInputCase, target: ReplayTarget): Promise<ReplayInvocation>;
|
|
108
|
+
};
|
|
109
|
+
export type ReplayPolicy = {
|
|
110
|
+
evaluator: "exact_match" | "classification";
|
|
111
|
+
minimumSamples: number;
|
|
112
|
+
minimumCandidateAccuracy: number;
|
|
113
|
+
maximumAccuracyDrop?: number;
|
|
114
|
+
maximumCandidateErrorRate?: number;
|
|
115
|
+
minimumCostSavingsPercent?: number;
|
|
116
|
+
};
|
|
117
|
+
export type ReplayGate = {
|
|
118
|
+
name: string;
|
|
119
|
+
passed: boolean;
|
|
120
|
+
observed: number | null;
|
|
121
|
+
required: number;
|
|
122
|
+
};
|
|
123
|
+
export type ReplayReport = {
|
|
124
|
+
sampleSize: number;
|
|
125
|
+
baselineAccuracy: number;
|
|
126
|
+
candidateAccuracy: number;
|
|
127
|
+
accuracyDrop: number;
|
|
128
|
+
candidateAccuracyLower95: number;
|
|
129
|
+
candidateErrorRate: number;
|
|
130
|
+
baselineCostUsd: number | null;
|
|
131
|
+
candidateCostUsd: number | null;
|
|
132
|
+
costSavingsPercent: number | null;
|
|
133
|
+
candidateFailureIds: string[];
|
|
134
|
+
regressionIds: string[];
|
|
135
|
+
improvementIds: string[];
|
|
136
|
+
gates: ReplayGate[];
|
|
137
|
+
eligibleForReview: boolean;
|
|
138
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { MeshEvaluationCaptureConfig, MeshEvaluationCaptureRecord } from "./types.js";
|
|
2
|
+
export type LocalJsonlCapture = MeshEvaluationCaptureConfig & {
|
|
3
|
+
/** Wait until all queued JSONL writes have completed. */
|
|
4
|
+
flush: () => Promise<void>;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Create an opt-in local JSONL capture sink. A redactor is mandatory because
|
|
8
|
+
* records can contain prompts and model responses. The file is created with
|
|
9
|
+
* owner-only permissions where the host filesystem supports POSIX modes.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createLocalJsonlCapture(path: string, redact: (record: MeshEvaluationCaptureRecord) => MeshEvaluationCaptureRecord | undefined, hooks?: Pick<MeshEvaluationCaptureConfig, "getCaseId" | "getGroupId" | "getExpected" | "score">): LocalJsonlCapture;
|