@harrym1707/sdk 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +98 -0
- package/README.md +48 -2
- package/bin/mesh-demo.mjs +105 -0
- package/bin/mesh-proxy.mjs +39 -0
- package/bin/mesh-quote.mjs +45 -0
- package/bin/mesh-report.mjs +47 -0
- package/index.cjs +6306 -0
- package/index.js +6204 -0
- package/package.json +39 -4
- package/types/blind-features.d.ts +28 -0
- package/types/cache-volatility-guard.d.ts +31 -0
- package/types/core/account-store.d.ts +7 -0
- package/types/core/accounts.d.ts +134 -0
- package/types/core/billing-worker.d.ts +113 -0
- package/types/core/budget.d.ts +216 -0
- package/types/core/cache-simulator.d.ts +271 -0
- package/types/core/confidence.d.ts +36 -0
- package/types/core/cost.d.ts +16 -0
- package/types/core/evaluator.d.ts +16 -0
- package/types/core/gateway.d.ts +75 -0
- package/types/core/geo-visibility.d.ts +25 -0
- package/types/core/github-oauth.d.ts +41 -0
- package/types/core/helicone.d.ts +6 -0
- package/types/core/index.d.ts +36 -0
- package/types/core/langfuse.d.ts +7 -0
- package/types/core/ledger-store.d.ts +62 -0
- package/types/core/litellm.d.ts +15 -0
- package/types/core/local-tier.d.ts +82 -0
- package/types/core/model-prices.d.ts +25 -0
- package/types/core/output-diagnostics.d.ts +39 -0
- package/types/core/price-catalogue.d.ts +168 -0
- package/types/core/provider-usage.d.ts +84 -0
- package/types/core/quote-report.d.ts +54 -0
- package/types/core/replay.d.ts +2 -0
- package/types/core/router.d.ts +9 -0
- package/types/core/routing.d.ts +282 -0
- package/types/core/savings-ledger.d.ts +349 -0
- package/types/core/savings-model.d.ts +132 -0
- package/types/core/stripe-billing.d.ts +67 -0
- package/types/core/stripe-client.d.ts +117 -0
- package/types/core/stripe-sink.d.ts +77 -0
- package/types/core/theo-coverage.d.ts +49 -0
- package/types/core/token-reduction.d.ts +219 -0
- package/types/core/tool-schema.d.ts +64 -0
- package/types/core/traffic.d.ts +182 -0
- package/types/core/triage.d.ts +70 -0
- package/types/core/types.d.ts +149 -0
- package/types/core/workflow-sessions.d.ts +50 -0
- package/types/cta.d.ts +60 -0
- package/types/evaluation-capture.d.ts +11 -0
- package/types/exact-cache.d.ts +20 -0
- package/types/graders.d.ts +53 -0
- package/types/index.d.ts +31 -0
- package/types/ledger-mirror.d.ts +38 -0
- package/types/openai.d.ts +23 -0
- package/types/proxy.d.ts +104 -0
- package/types/reduce-request.d.ts +272 -0
- package/types/replay-gate.d.ts +120 -0
- package/types/report.d.ts +33 -0
- package/types/routing.d.ts +11 -0
- package/types/savings-ledger.d.ts +22 -0
- package/types/savings-snapshot.d.ts +11 -0
- package/types/semantic-compaction.d.ts +84 -0
- package/types/telemetry.d.ts +6 -0
- package/types/theo-agent-store.d.ts +27 -0
- package/types/theo-agent.d.ts +247 -0
- package/types/theo-gate.d.ts +75 -0
- package/types/theo-policy.d.ts +219 -0
- package/types/types.d.ts +287 -0
- package/types/wrap.d.ts +60 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { type TokenCounter } from "./token-reduction.js";
|
|
2
|
+
export type MinifyOptions = {
|
|
3
|
+
/**
|
|
4
|
+
* What to do with description fields.
|
|
5
|
+
* `keep` stays lossless. `truncate` clips to `maximumDescriptionTokens`.
|
|
6
|
+
* `drop` removes them, which is the largest saving and the largest risk.
|
|
7
|
+
*/
|
|
8
|
+
descriptions?: "keep" | "truncate" | "drop";
|
|
9
|
+
maximumDescriptionTokens?: number;
|
|
10
|
+
/** Remove `examples` arrays. The model does read these, so this is lossy. */
|
|
11
|
+
dropExamples?: boolean;
|
|
12
|
+
/** Tool names to keep. Omit to keep all; naming a subset is lossy. */
|
|
13
|
+
keepTools?: readonly string[];
|
|
14
|
+
countTokens?: TokenCounter;
|
|
15
|
+
};
|
|
16
|
+
export type MinifyChange = {
|
|
17
|
+
kind: "plumbing" | "redundant-title" | "formatting" | "description" | "examples" | "dropped-tool";
|
|
18
|
+
lossless: boolean;
|
|
19
|
+
/** Where in the catalogue, e.g. a tool name or a property path. */
|
|
20
|
+
at: string;
|
|
21
|
+
tokensSaved: number;
|
|
22
|
+
detail: string;
|
|
23
|
+
};
|
|
24
|
+
export type MinifyResult = {
|
|
25
|
+
tools: unknown[];
|
|
26
|
+
tokensBefore: number;
|
|
27
|
+
tokensAfter: number;
|
|
28
|
+
/** Tokens removed without any possibility of changing tool selection. */
|
|
29
|
+
losslessTokensSaved: number;
|
|
30
|
+
/** Tokens removed by altering what the model reads. Needs a replay check. */
|
|
31
|
+
lossyTokensSaved: number;
|
|
32
|
+
changes: MinifyChange[];
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Minify a tool catalogue, reporting every change and which tier it belongs to.
|
|
36
|
+
*
|
|
37
|
+
* Defaults are lossless only: descriptions are kept. A caller wanting the large
|
|
38
|
+
* saving opts into it explicitly and gets it reported as lossy.
|
|
39
|
+
*/
|
|
40
|
+
export declare function minifyToolSchemas(tools: unknown[], options?: MinifyOptions): MinifyResult;
|
|
41
|
+
export type ToolCatalogueAudit = {
|
|
42
|
+
tools: number;
|
|
43
|
+
tokens: number;
|
|
44
|
+
meanTokensPerTool: number;
|
|
45
|
+
/** Largest offenders first, so a catalogue can be trimmed by hand. */
|
|
46
|
+
largest: Array<{
|
|
47
|
+
name: string;
|
|
48
|
+
tokens: number;
|
|
49
|
+
descriptionTokens: number;
|
|
50
|
+
}>;
|
|
51
|
+
/** Achievable without any behaviour change. */
|
|
52
|
+
losslessTokensSaved: number;
|
|
53
|
+
/** Additionally achievable by trimming descriptions, behind a replay check. */
|
|
54
|
+
descriptionTokensSaved: number;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Audit a catalogue: how big it is, which tools dominate, and how much each tier
|
|
58
|
+
* of minification would recover. This is what makes the lever measured rather
|
|
59
|
+
* than assumed, so a workload with an already-lean catalogue is credited nothing.
|
|
60
|
+
*/
|
|
61
|
+
export declare function auditToolCatalogue(tools: unknown[], options?: {
|
|
62
|
+
countTokens?: TokenCounter;
|
|
63
|
+
maximumDescriptionTokens?: number;
|
|
64
|
+
}): ToolCatalogueAudit;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { type PriceCatalogue } from "./price-catalogue.js";
|
|
2
|
+
import { type Lever, type WorkloadProfile } from "./savings-model.js";
|
|
3
|
+
import { type ToolCatalogueAudit } from "./tool-schema.js";
|
|
4
|
+
import { type CachingAnalysis, type OutputCapAdvice, type TokenCounter } from "./token-reduction.js";
|
|
5
|
+
/**
|
|
6
|
+
* Ingest real traffic and measure what a reduction would be worth on it.
|
|
7
|
+
*
|
|
8
|
+
* Two tiers of input, because most teams can produce the first immediately and
|
|
9
|
+
* the second only after a conversation about privacy:
|
|
10
|
+
*
|
|
11
|
+
* Tier 1, usage only (model, token counts, date): prices the baseline, finds
|
|
12
|
+
* the model mix, and recommends output caps. No prompt text is needed, so
|
|
13
|
+
* this runs on billing exports.
|
|
14
|
+
* Tier 2, usage plus prompt text: adds prefix-cache analysis, what is breaking
|
|
15
|
+
* the cache, duplicate context, and measured tool-schema overhead.
|
|
16
|
+
*
|
|
17
|
+
* Nothing here performs I/O or leaves the process. Prompt text is used to
|
|
18
|
+
* compute counts and hashes and is never copied into a report.
|
|
19
|
+
*/
|
|
20
|
+
export type TrafficRecord = {
|
|
21
|
+
id: string;
|
|
22
|
+
model: string;
|
|
23
|
+
provider: string;
|
|
24
|
+
/** ISO date the request was billed on, used to resolve the rate that applied. */
|
|
25
|
+
at: string;
|
|
26
|
+
inputTokens: number;
|
|
27
|
+
outputTokens: number;
|
|
28
|
+
/** Input already served from the provider's cache. A saving the customer HAS. */
|
|
29
|
+
cachedInputTokens?: number;
|
|
30
|
+
cacheWriteInputTokens?: number;
|
|
31
|
+
/** Full prompt text, when the log carries it. Enables tier 2 analysis. */
|
|
32
|
+
promptText?: string;
|
|
33
|
+
/** Serialised tool or function definitions sent with the request. */
|
|
34
|
+
toolSchemaText?: string;
|
|
35
|
+
/** Logical grouping. Prompts only share a prefix within one workload. */
|
|
36
|
+
workload: string;
|
|
37
|
+
/**
|
|
38
|
+
* Task this request belongs to, where the log says so. Several requests share
|
|
39
|
+
* one episode id when an agent takes several turns to finish one job.
|
|
40
|
+
*/
|
|
41
|
+
episodeId?: string;
|
|
42
|
+
};
|
|
43
|
+
export type IngestIssue = {
|
|
44
|
+
line: number;
|
|
45
|
+
reason: string;
|
|
46
|
+
};
|
|
47
|
+
export type IngestResult = {
|
|
48
|
+
records: TrafficRecord[];
|
|
49
|
+
issues: IngestIssue[];
|
|
50
|
+
/** Which adapter matched each record, so an odd result can be traced. */
|
|
51
|
+
adapterCounts: Record<string, number>;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Normalise one logged row.
|
|
55
|
+
*
|
|
56
|
+
* Accepts the OpenAI Responses and Chat Completions shapes, the Anthropic
|
|
57
|
+
* Messages shape, and a bare usage row, with the request and response either
|
|
58
|
+
* nested under `request`/`response` or flattened onto one object. Returns the
|
|
59
|
+
* adapter that matched so a surprising report can be traced to its parser.
|
|
60
|
+
*/
|
|
61
|
+
export declare function normaliseTrafficRow(row: unknown, fallbackDate: string, index: number): {
|
|
62
|
+
record: TrafficRecord;
|
|
63
|
+
adapter: string;
|
|
64
|
+
} | {
|
|
65
|
+
issue: string;
|
|
66
|
+
};
|
|
67
|
+
/** Normalise many rows, collecting per-row problems rather than failing the run. */
|
|
68
|
+
export declare function ingestTraffic(rows: unknown[], fallbackDate: string): IngestResult;
|
|
69
|
+
/**
|
|
70
|
+
* Per-task view of a workload.
|
|
71
|
+
*
|
|
72
|
+
* The unit that matters for agentic traffic, and the one the market does not
|
|
73
|
+
* report. A technique that cuts each prompt by 91% while tripling the number of
|
|
74
|
+
* prompts has saved nothing, and per-request figures cannot show that: one
|
|
75
|
+
* published bench found progressive disclosure modelled at ~77% delivering ~13%
|
|
76
|
+
* in production, because a router adds a round trip per novel tool call.
|
|
77
|
+
*
|
|
78
|
+
* `identified` matters. Without a task identifier in the log every request is
|
|
79
|
+
* its own episode, round trips read as 1.0, and nothing here is informative.
|
|
80
|
+
* It is reported rather than silently assumed.
|
|
81
|
+
*/
|
|
82
|
+
export type EpisodeStats = {
|
|
83
|
+
episodes: number;
|
|
84
|
+
/** Mean requests per task. This is the round-trip count a technique can change. */
|
|
85
|
+
meanRoundTrips: number;
|
|
86
|
+
maxRoundTrips: number;
|
|
87
|
+
meanInputTokens: number;
|
|
88
|
+
meanOutputTokens: number;
|
|
89
|
+
meanCostUsd?: number;
|
|
90
|
+
/** False when the log carried no task identifier, making the above meaningless. */
|
|
91
|
+
identified: boolean;
|
|
92
|
+
};
|
|
93
|
+
export type WorkloadMeasurement = {
|
|
94
|
+
workload: string;
|
|
95
|
+
/** Model identifier exactly as the log carried it. */
|
|
96
|
+
model: string;
|
|
97
|
+
/** Catalogue key the identifier resolved to, when it resolved at all. */
|
|
98
|
+
resolvedModel?: string;
|
|
99
|
+
provider: string;
|
|
100
|
+
requests: number;
|
|
101
|
+
inputTokens: number;
|
|
102
|
+
outputTokens: number;
|
|
103
|
+
/** Tokens the provider already served from cache before MESH touched anything. */
|
|
104
|
+
alreadyCachedInputTokens: number;
|
|
105
|
+
meanToolSchemaTokens: number;
|
|
106
|
+
/** Measured from the request's own tool definitions; absent when none were sent. */
|
|
107
|
+
toolAudit?: ToolCatalogueAudit;
|
|
108
|
+
/** Present only when the log carried prompt text. */
|
|
109
|
+
caching?: CachingAnalysis;
|
|
110
|
+
outputCap?: OutputCapAdvice;
|
|
111
|
+
duplicateContextTokens: number;
|
|
112
|
+
episodes: EpisodeStats;
|
|
113
|
+
baselineCostUsd?: number;
|
|
114
|
+
/** Set when the catalogue cannot price this model on these dates. */
|
|
115
|
+
unpricedReason?: string;
|
|
116
|
+
};
|
|
117
|
+
export type TrafficMeasurement = {
|
|
118
|
+
requests: number;
|
|
119
|
+
workloads: WorkloadMeasurement[];
|
|
120
|
+
totalBaselineCostUsd: number;
|
|
121
|
+
/** Models the catalogue could not price; their spend is excluded, never assumed zero. */
|
|
122
|
+
unpricedModels: string[];
|
|
123
|
+
/** Share of input tokens already arriving cached, which MESH must not claim credit for. */
|
|
124
|
+
alreadyCachedShare: number;
|
|
125
|
+
promptTextAvailable: boolean;
|
|
126
|
+
/** True when at least one record carried a task identifier. */
|
|
127
|
+
episodesIdentified: boolean;
|
|
128
|
+
notes: string[];
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Measure a traffic sample.
|
|
132
|
+
*
|
|
133
|
+
* Deliberately conservative in two places. Input already arriving cached is
|
|
134
|
+
* reported separately and never counted as a MESH saving, because the customer
|
|
135
|
+
* already has it. And a model the catalogue cannot price is named and excluded
|
|
136
|
+
* rather than treated as free, the same discipline applied to benchmark sources.
|
|
137
|
+
*/
|
|
138
|
+
export declare function measureTraffic(records: TrafficRecord[], catalogue: PriceCatalogue, options?: {
|
|
139
|
+
countTokens?: TokenCounter;
|
|
140
|
+
maximumPromptSample?: number;
|
|
141
|
+
}): TrafficMeasurement;
|
|
142
|
+
/** Stable fingerprint of a traffic sample, so repeated runs can be compared. */
|
|
143
|
+
export declare function trafficFingerprint(records: TrafficRecord[]): string;
|
|
144
|
+
/**
|
|
145
|
+
* Turn a measured workload into a profile the savings model can project on.
|
|
146
|
+
*
|
|
147
|
+
* Segments come from what was actually observed rather than assumed: the tool
|
|
148
|
+
* catalogue is measured from the request's own tool definitions, the stable
|
|
149
|
+
* prefix from what the sampled prompts genuinely share, and whatever remains is
|
|
150
|
+
* treated as the volatile body. Tokens already arriving cached are removed from
|
|
151
|
+
* the stable segment, because that saving is the customer's already.
|
|
152
|
+
*/
|
|
153
|
+
export declare function profileFromMeasurement(measurement: WorkloadMeasurement, options?: {
|
|
154
|
+
repairPrefix?: boolean;
|
|
155
|
+
}): WorkloadProfile;
|
|
156
|
+
/**
|
|
157
|
+
* Build the lever set a measured workload actually supports.
|
|
158
|
+
*
|
|
159
|
+
* `leversFor` applies published reduction factors to any profile with non-empty
|
|
160
|
+
* segments, which is right for archetype modelling and wrong for a customer
|
|
161
|
+
* report: it credited a 60% output reduction to workloads whose replies were
|
|
162
|
+
* already short and tight, and that was the only lever firing on most samples.
|
|
163
|
+
*
|
|
164
|
+
* Here every factor is either measured from the sample or gated on measured
|
|
165
|
+
* evidence that the lever applies at all:
|
|
166
|
+
*
|
|
167
|
+
* - Output capping uses the observed length distribution, so a workload with
|
|
168
|
+
* no long tail is credited nothing.
|
|
169
|
+
* - Duplicate removal is credited exactly the duplicate tokens found.
|
|
170
|
+
* - Tool levers keep their published factors, which are deterministic
|
|
171
|
+
* transformations of a catalogue we measured directly, but only fire when
|
|
172
|
+
* tool definitions were actually present.
|
|
173
|
+
* - Reasoning-effort tuning is offered only where mean output is long enough
|
|
174
|
+
* for reasoning tokens to plausibly dominate, at well under the published
|
|
175
|
+
* factor, and always as needs-verification.
|
|
176
|
+
*
|
|
177
|
+
* Prompt compression and history compaction are deliberately absent. Their
|
|
178
|
+
* published factors are real but workload-specific, and nothing in a usage log
|
|
179
|
+
* measures whether they transfer. They belong behind a replay check, not in an
|
|
180
|
+
* opening projection.
|
|
181
|
+
*/
|
|
182
|
+
export declare function leversFromMeasurement(measurement: WorkloadMeasurement): Lever[];
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Horizontal customer triage: which kind of traffic is this, and what is MESH
|
|
3
|
+
* honestly worth on it?
|
|
4
|
+
*
|
|
5
|
+
* The A/B results made one thing unambiguous: the saving MESH delivers is a
|
|
6
|
+
* property of the WORKLOAD SHAPE, not of MESH. The same software measured
|
|
7
|
+
* ~2% from caching on a multi-turn agent (whose appended history already hits
|
|
8
|
+
* the provider cache ~90% of the time), ~40% from prefix repair on one-shot
|
|
9
|
+
* templated traffic (whose embedded clock blocked the cache entirely), ~30%
|
|
10
|
+
* from output budgeting where answers had a long tail, and ~3% on a workload
|
|
11
|
+
* that was already well cached.
|
|
12
|
+
*
|
|
13
|
+
* A horizontal product therefore cannot quote one number. It has to classify
|
|
14
|
+
* the stream first and quote the band that stream's shape supports, with the
|
|
15
|
+
* evidence attached — including "you are already optimised, do not pay us",
|
|
16
|
+
* because a percent-of-savings business that charges for savings the customer
|
|
17
|
+
* already has is buying refunds and churn.
|
|
18
|
+
*
|
|
19
|
+
* Deliberately gateway-agnostic: the signals are derivable from any usage log,
|
|
20
|
+
* LiteLLM's spend logs included.
|
|
21
|
+
*/
|
|
22
|
+
export type StreamSignals = {
|
|
23
|
+
requests: number;
|
|
24
|
+
/** Distinct downstream customers (a gateway's end_user), where known. */
|
|
25
|
+
distinctEndUsers: number;
|
|
26
|
+
requestsPerEndUser: number;
|
|
27
|
+
episodesIdentified: boolean;
|
|
28
|
+
meanRoundTrips: number;
|
|
29
|
+
/** Share of input tokens that are tool/MCP schema. */
|
|
30
|
+
toolSchemaShare: number;
|
|
31
|
+
/** Share of input already arriving at the provider cached. */
|
|
32
|
+
alreadyCachedShare: number;
|
|
33
|
+
/** Share of requests a gateway's own response cache already answers. */
|
|
34
|
+
gatewayCacheHitShare: number;
|
|
35
|
+
promptTextAvailable: boolean;
|
|
36
|
+
prefixBreakersFound: boolean;
|
|
37
|
+
meanInputTokens: number;
|
|
38
|
+
};
|
|
39
|
+
export type CustomerSegment = "already-optimised" | "consumer-app" | "in-house-agent" | "application-layer";
|
|
40
|
+
export type SegmentClassification = {
|
|
41
|
+
segment: CustomerSegment;
|
|
42
|
+
evidence: string[];
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* Classify one stream. Rules are ordered so the strongest signal wins:
|
|
46
|
+
* existing optimisation first (nothing honest to sell), then the many-end-user
|
|
47
|
+
* consumer shape, then the session-heavy agent shape, with server-side
|
|
48
|
+
* application traffic as the remainder.
|
|
49
|
+
*/
|
|
50
|
+
export declare function classifySegment(signals: StreamSignals): SegmentClassification;
|
|
51
|
+
export type QuoteConfidence = "measured-analogue" | "needs-prompt-text" | "do-not-charge";
|
|
52
|
+
export type SegmentQuote = {
|
|
53
|
+
segment: CustomerSegment;
|
|
54
|
+
/** Honest range for total cost reduction, as percentages of current spend. */
|
|
55
|
+
bandPercent: [number, number];
|
|
56
|
+
topLever: string;
|
|
57
|
+
confidence: QuoteConfidence;
|
|
58
|
+
/** What would turn the band into a firm figure. */
|
|
59
|
+
firmUpBy?: string;
|
|
60
|
+
reasoning: string[];
|
|
61
|
+
doNotCharge?: boolean;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Quote a band for a classified stream.
|
|
65
|
+
*
|
|
66
|
+
* Every band cites the measurement it is an analogue of; none is a marketing
|
|
67
|
+
* number. Where the deciding evidence (prompt text) is absent, the band is
|
|
68
|
+
* wide and says exactly what would narrow it.
|
|
69
|
+
*/
|
|
70
|
+
export declare function quoteForSegment(classification: SegmentClassification, signals: StreamSignals): SegmentQuote;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
export type ModelTarget = {
|
|
2
|
+
provider: string;
|
|
3
|
+
model: string;
|
|
4
|
+
};
|
|
5
|
+
export type QualityResult = {
|
|
6
|
+
passed: boolean;
|
|
7
|
+
score?: number;
|
|
8
|
+
checks: Record<string, boolean | number | string>;
|
|
9
|
+
reason?: string;
|
|
10
|
+
/** Measured error rate (0..1) behind this result, checked against `maxErrorRate`. */
|
|
11
|
+
errorRate?: number;
|
|
12
|
+
/** Measured latency in ms behind this result, checked against `maxLatencyMs`. */
|
|
13
|
+
latencyMs?: number;
|
|
14
|
+
};
|
|
15
|
+
export type QualityPolicy = {
|
|
16
|
+
minimumScore?: number;
|
|
17
|
+
requiredChecks?: string[];
|
|
18
|
+
maxErrorRate?: number;
|
|
19
|
+
maxLatencyMs?: number;
|
|
20
|
+
/**
|
|
21
|
+
* Minimum number of evaluation samples the quality measurement must rest on
|
|
22
|
+
* before a candidate may take any traffic. A policy is never activated on one
|
|
23
|
+
* or two examples (Technical Architecture §6). Defaults to
|
|
24
|
+
* `DEFAULT_MINIMUM_SAMPLES` when omitted.
|
|
25
|
+
*/
|
|
26
|
+
minimumSamples?: number;
|
|
27
|
+
};
|
|
28
|
+
export type RoutingCandidate = {
|
|
29
|
+
target: ModelTarget;
|
|
30
|
+
estimatedCostMultiplier: number;
|
|
31
|
+
rolloutPercentage: number;
|
|
32
|
+
qualityPolicy: QualityPolicy;
|
|
33
|
+
};
|
|
34
|
+
export type RoutingPolicy = {
|
|
35
|
+
id: string;
|
|
36
|
+
version: number;
|
|
37
|
+
taskClass: string;
|
|
38
|
+
baseline: ModelTarget;
|
|
39
|
+
candidates: RoutingCandidate[];
|
|
40
|
+
enabled: boolean;
|
|
41
|
+
};
|
|
42
|
+
export type RoutingDecision = {
|
|
43
|
+
target: ModelTarget;
|
|
44
|
+
reason: string;
|
|
45
|
+
policyId?: string;
|
|
46
|
+
policyVersion?: number;
|
|
47
|
+
isBaseline: boolean;
|
|
48
|
+
};
|
|
49
|
+
export type InferenceEvent = {
|
|
50
|
+
requestId: string;
|
|
51
|
+
applicationId: string;
|
|
52
|
+
taskClass?: string;
|
|
53
|
+
provider: string;
|
|
54
|
+
model: string;
|
|
55
|
+
inputTokens?: number;
|
|
56
|
+
outputTokens?: number;
|
|
57
|
+
costUsd?: number;
|
|
58
|
+
latencyMs?: number;
|
|
59
|
+
policyId?: string;
|
|
60
|
+
policyVersion?: number;
|
|
61
|
+
routeReason?: string;
|
|
62
|
+
timestamp: string;
|
|
63
|
+
};
|
|
64
|
+
export type EvaluationInput = {
|
|
65
|
+
baseline: unknown;
|
|
66
|
+
candidate: unknown;
|
|
67
|
+
metadata?: Record<string, unknown>;
|
|
68
|
+
};
|
|
69
|
+
export type Evaluator = {
|
|
70
|
+
name: string;
|
|
71
|
+
evaluate(input: EvaluationInput): Promise<QualityResult>;
|
|
72
|
+
};
|
|
73
|
+
export type ReplayRecord = {
|
|
74
|
+
id: string;
|
|
75
|
+
expected: unknown;
|
|
76
|
+
baseline: unknown;
|
|
77
|
+
candidate: unknown;
|
|
78
|
+
baselineCostUsd?: number;
|
|
79
|
+
candidateCostUsd?: number;
|
|
80
|
+
baselineError?: boolean;
|
|
81
|
+
candidateError?: boolean;
|
|
82
|
+
};
|
|
83
|
+
export type ReplayInputCase = {
|
|
84
|
+
id: string;
|
|
85
|
+
expected: unknown;
|
|
86
|
+
input: string;
|
|
87
|
+
/** Candidate-side prompt/context after the configured reduction profile. */
|
|
88
|
+
candidateInput?: string;
|
|
89
|
+
mockOutputs?: {
|
|
90
|
+
baseline: unknown;
|
|
91
|
+
candidate: unknown;
|
|
92
|
+
inputTokens: number;
|
|
93
|
+
baselineOutputTokens: number;
|
|
94
|
+
candidateOutputTokens: number;
|
|
95
|
+
};
|
|
96
|
+
};
|
|
97
|
+
export type ReplayTarget = {
|
|
98
|
+
provider: string;
|
|
99
|
+
model: string;
|
|
100
|
+
maxOutputTokens?: number;
|
|
101
|
+
contextProfileId?: string;
|
|
102
|
+
pricing?: {
|
|
103
|
+
inputPerMillion: number;
|
|
104
|
+
cachedInputPerMillion?: number;
|
|
105
|
+
cacheWriteInputPerMillion?: number;
|
|
106
|
+
outputPerMillion: number;
|
|
107
|
+
};
|
|
108
|
+
};
|
|
109
|
+
export type ReplayInvocation = {
|
|
110
|
+
output?: unknown;
|
|
111
|
+
inputTokens?: number;
|
|
112
|
+
cachedInputTokens?: number;
|
|
113
|
+
cacheWriteInputTokens?: number;
|
|
114
|
+
outputTokens?: number;
|
|
115
|
+
error?: string;
|
|
116
|
+
};
|
|
117
|
+
export type ReplayAdapter = {
|
|
118
|
+
invoke(inputCase: ReplayInputCase, target: ReplayTarget): Promise<ReplayInvocation>;
|
|
119
|
+
};
|
|
120
|
+
export type ReplayPolicy = {
|
|
121
|
+
evaluator: "exact_match" | "classification";
|
|
122
|
+
minimumSamples: number;
|
|
123
|
+
minimumCandidateAccuracy: number;
|
|
124
|
+
maximumAccuracyDrop?: number;
|
|
125
|
+
maximumCandidateErrorRate?: number;
|
|
126
|
+
minimumCostSavingsPercent?: number;
|
|
127
|
+
};
|
|
128
|
+
export type ReplayGate = {
|
|
129
|
+
name: string;
|
|
130
|
+
passed: boolean;
|
|
131
|
+
observed: number | null;
|
|
132
|
+
required: number;
|
|
133
|
+
};
|
|
134
|
+
export type ReplayReport = {
|
|
135
|
+
sampleSize: number;
|
|
136
|
+
baselineAccuracy: number;
|
|
137
|
+
candidateAccuracy: number;
|
|
138
|
+
accuracyDrop: number;
|
|
139
|
+
candidateAccuracyLower95: number;
|
|
140
|
+
candidateErrorRate: number;
|
|
141
|
+
baselineCostUsd: number | null;
|
|
142
|
+
candidateCostUsd: number | null;
|
|
143
|
+
costSavingsPercent: number | null;
|
|
144
|
+
candidateFailureIds: string[];
|
|
145
|
+
regressionIds: string[];
|
|
146
|
+
improvementIds: string[];
|
|
147
|
+
gates: ReplayGate[];
|
|
148
|
+
eligibleForReview: boolean;
|
|
149
|
+
};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type BillableEvent, type LedgerLeverKind } from "./savings-ledger.js";
|
|
2
|
+
/**
|
|
3
|
+
* Group ledger events into working sessions, so the dashboard can show Theo's
|
|
4
|
+
* activity per workflow instead of only in aggregate.
|
|
5
|
+
*
|
|
6
|
+
* MESH's proxy sees a stream of stateless API requests; the LLM APIs carry no
|
|
7
|
+
* task or conversation id, and Claude Code does not send one, so the ledger has
|
|
8
|
+
* no semantic "task" to group by (the `taskId` slot on an event exists for an
|
|
9
|
+
* out-of-band quality signal, and the proxy never fills it). The honest grouping
|
|
10
|
+
* available from the ledger alone is therefore two real dimensions:
|
|
11
|
+
*
|
|
12
|
+
* - `applicationId` — the `x-mesh-application` header, when a client sets it;
|
|
13
|
+
* - activity gaps — a run of requests with no long pause between them is one
|
|
14
|
+
* working session, split when the gap exceeds `gapMinutes`.
|
|
15
|
+
*
|
|
16
|
+
* This is a time-grouped working session, not an inferred semantic task, and the
|
|
17
|
+
* UI labels it as such. Each session's savings come from `recomputeInvoice` over
|
|
18
|
+
* exactly that session's events, so a session total is verified the same way the
|
|
19
|
+
* headline is — never an apportioned estimate.
|
|
20
|
+
*/
|
|
21
|
+
export type WorkflowSessionLever = {
|
|
22
|
+
leverId: string;
|
|
23
|
+
kind: LedgerLeverKind;
|
|
24
|
+
events: number;
|
|
25
|
+
savedUsd: number;
|
|
26
|
+
};
|
|
27
|
+
export type WorkflowSession = {
|
|
28
|
+
/** Stable within a ledger: application id + the session's first timestamp. */
|
|
29
|
+
id: string;
|
|
30
|
+
applicationId: string;
|
|
31
|
+
startedAt: string;
|
|
32
|
+
endedAt: string;
|
|
33
|
+
requests: number;
|
|
34
|
+
/** Distinct models seen, in first-seen order. */
|
|
35
|
+
models: string[];
|
|
36
|
+
netVerifiedSavingsUsd: number;
|
|
37
|
+
verdict: "charge" | "do_not_charge";
|
|
38
|
+
/** The Theo levers that fired this session, with their verified saving. */
|
|
39
|
+
levers: WorkflowSessionLever[];
|
|
40
|
+
succeeded: number;
|
|
41
|
+
failed: number;
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Cluster events into sessions, most recent first. Pure and deterministic: the
|
|
45
|
+
* same events and options always produce the same sessions.
|
|
46
|
+
*/
|
|
47
|
+
export declare function groupWorkflowSessions(events: readonly BillableEvent[], options?: {
|
|
48
|
+
gapMinutes?: number;
|
|
49
|
+
limit?: number;
|
|
50
|
+
}): WorkflowSession[];
|
package/types/cta.d.ts
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The savings CTA — the moment the product asks for the signup.
|
|
3
|
+
*
|
|
4
|
+
* MESH already computes a verified savings number from a hash-chained ledger
|
|
5
|
+
* the customer holds. This module turns that number into (a) a console call to
|
|
6
|
+
* action a runner can print on exit, and (b) a paste-able README badge, so the
|
|
7
|
+
* person running MESH self-identifies and the badge carries MESH into their
|
|
8
|
+
* repo. Everything here is honest: when the ledger says `do_not_charge`, the
|
|
9
|
+
* CTA says so and asks for nothing.
|
|
10
|
+
*
|
|
11
|
+
* This module is intentionally dependency-free (pure string/number logic) so
|
|
12
|
+
* both the SDK and the `mesh` CLI can import it directly without pulling the
|
|
13
|
+
* billing engine into their bundles. The core-dependent readers that build a
|
|
14
|
+
* {@link SavingsSnapshot} from a ledger spool live in `./savings-snapshot.js`.
|
|
15
|
+
*/
|
|
16
|
+
/** Where a developer running MESH goes to claim their verified savings. */
|
|
17
|
+
export declare const MESH_SIGNUP_URL = "https://www.mesh-technology.com";
|
|
18
|
+
export type SavingsSnapshot = {
|
|
19
|
+
/** Requests metered across every spool read. */
|
|
20
|
+
requests: number;
|
|
21
|
+
/** Net verified savings in USD (from provider usage, never an estimate). */
|
|
22
|
+
savedUsd: number;
|
|
23
|
+
/** What the metered traffic actually cost at the provider, in USD. */
|
|
24
|
+
spendUsd: number;
|
|
25
|
+
/** "charge" when MESH saved money; "do_not_charge" when it did not. */
|
|
26
|
+
verdict: "charge" | "do_not_charge";
|
|
27
|
+
/** False when any spool's hash chain failed to verify. */
|
|
28
|
+
chainOk: boolean;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Savings as a percentage of the un-reduced baseline (actual paid + saved),
|
|
32
|
+
* matching the README's "% lower cost" framing. Null when there is no spend to
|
|
33
|
+
* compare against.
|
|
34
|
+
*/
|
|
35
|
+
export declare function savingsPercent(snap: Pick<SavingsSnapshot, "savedUsd" | "spendUsd">): number | null;
|
|
36
|
+
export type SavingsCtaOptions = {
|
|
37
|
+
/** Where to claim savings; defaults to {@link MESH_SIGNUP_URL}. */
|
|
38
|
+
signupUrl?: string;
|
|
39
|
+
/** Branding prefix on each line; defaults to "⬢ MESH". */
|
|
40
|
+
mark?: string;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* The console CTA. Honest by construction: a broken chain reports nothing, an
|
|
44
|
+
* empty spool invites a first request, a `do_not_charge` verdict asks for
|
|
45
|
+
* nothing, and only a real verified saving makes the ask.
|
|
46
|
+
*/
|
|
47
|
+
export declare function formatSavingsCta(snap: SavingsSnapshot, opts?: SavingsCtaOptions): string;
|
|
48
|
+
export type SavingsBadge = {
|
|
49
|
+
/** The shields.io image URL. */
|
|
50
|
+
imageUrl: string;
|
|
51
|
+
/** A ready-to-paste markdown image link to the signup page. */
|
|
52
|
+
markdown: string;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* A paste-able README badge reflecting this spool's verified savings. Every
|
|
56
|
+
* user who adds it carries MESH into their repo — the distribution loop.
|
|
57
|
+
*/
|
|
58
|
+
export declare function savingsBadge(snap: SavingsSnapshot, opts?: {
|
|
59
|
+
signupUrl?: string;
|
|
60
|
+
}): SavingsBadge;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { MeshEvaluationCaptureConfig, MeshEvaluationCaptureRecord } from "./types.js";
|
|
2
|
+
export type LocalJsonlCapture = MeshEvaluationCaptureConfig & {
|
|
3
|
+
/** Wait until all queued JSONL writes have completed. */
|
|
4
|
+
flush: () => Promise<void>;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Create an opt-in local JSONL capture sink. A redactor is mandatory because
|
|
8
|
+
* records can contain prompts and model responses. The file is created with
|
|
9
|
+
* owner-only permissions where the host filesystem supports POSIX modes.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createLocalJsonlCapture(path: string, redact: (record: MeshEvaluationCaptureRecord) => MeshEvaluationCaptureRecord | undefined, hooks?: Pick<MeshEvaluationCaptureConfig, "getCaseId" | "getGroupId" | "getExpected" | "score">): LocalJsonlCapture;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { MeshExactCacheConfig } from "./types.js";
|
|
2
|
+
export type ExactCacheEntry = {
|
|
3
|
+
response: unknown;
|
|
4
|
+
model: string;
|
|
5
|
+
inputTokens?: number;
|
|
6
|
+
uncachedInputTokens?: number;
|
|
7
|
+
cachedInputTokens?: number;
|
|
8
|
+
cacheWriteInputTokens?: number;
|
|
9
|
+
outputTokens?: number;
|
|
10
|
+
costUsd?: number;
|
|
11
|
+
expiresAt: number;
|
|
12
|
+
};
|
|
13
|
+
export declare class ExactResponseCache {
|
|
14
|
+
private readonly config;
|
|
15
|
+
private readonly entries;
|
|
16
|
+
constructor(config: MeshExactCacheConfig);
|
|
17
|
+
keyFor(request: unknown, scopeKey: string | undefined): string | undefined;
|
|
18
|
+
get(key: string): ExactCacheEntry | undefined;
|
|
19
|
+
set(key: string, entry: Omit<ExactCacheEntry, "expiresAt">): void;
|
|
20
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type ReplayGrader } from "./replay-gate.js";
|
|
2
|
+
/**
|
|
3
|
+
* Harmonic mean of precision and recall over word overlap.
|
|
4
|
+
*
|
|
5
|
+
* Standard summarisation-style scoring, and it degrades gracefully in exactly
|
|
6
|
+
* the way truncation does: an answer carrying 90% of the baseline's content
|
|
7
|
+
* scores near 0.95 rather than 0, because its precision is intact and only its
|
|
8
|
+
* recall fell.
|
|
9
|
+
*/
|
|
10
|
+
export declare const textSimilarityGrader: ReplayGrader;
|
|
11
|
+
export type TruncationOptions = {
|
|
12
|
+
/**
|
|
13
|
+
* Share of the baseline's content below which a truncated answer counts as
|
|
14
|
+
* useless rather than merely shorter. Default 0.5.
|
|
15
|
+
*/
|
|
16
|
+
uselessBelowCoverage?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Penalty applied to an answer the provider marked as cut off, on top of its
|
|
19
|
+
* coverage. Default 0.05, so being visibly truncated costs a little even when
|
|
20
|
+
* almost everything survived.
|
|
21
|
+
*/
|
|
22
|
+
truncationPenalty?: number;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Scores a possibly-truncated answer by how much of the baseline's content it
|
|
26
|
+
* still carries, rather than failing it outright.
|
|
27
|
+
*
|
|
28
|
+
* Below `uselessBelowCoverage` the answer is treated as a failure, because an
|
|
29
|
+
* answer cut off in its first third usually is one. Above it, the score is the
|
|
30
|
+
* coverage less a small penalty for being visibly incomplete.
|
|
31
|
+
*/
|
|
32
|
+
export declare function truncationAwareGrader(options?: TruncationOptions): ReplayGrader;
|
|
33
|
+
export type WeightedGrader = {
|
|
34
|
+
grader: ReplayGrader;
|
|
35
|
+
weight: number;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Weighted mean of several graders.
|
|
39
|
+
*
|
|
40
|
+
* A reduction can fail in unrelated ways at once: trimming descriptions changes
|
|
41
|
+
* which tool is called, capping output shortens the answer. One number has to
|
|
42
|
+
* account for both, and a single grader only sees one of them.
|
|
43
|
+
*/
|
|
44
|
+
export declare function compositeGrader(parts: WeightedGrader[]): ReplayGrader;
|
|
45
|
+
/**
|
|
46
|
+
* Fails outright when the tool choice changed, and otherwise scores by how
|
|
47
|
+
* complete the answer is.
|
|
48
|
+
*
|
|
49
|
+
* The right default for an agent workload. A changed tool call is a different
|
|
50
|
+
* action taken in the world and no amount of eloquence redeems it, so it is a
|
|
51
|
+
* hard zero. A shorter answer is a matter of degree.
|
|
52
|
+
*/
|
|
53
|
+
export declare function agentGrader(options?: TruncationOptions): ReplayGrader;
|