@harrym1707/sdk 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +96 -0
- package/README.md +48 -2
- package/bin/mesh-demo.mjs +105 -0
- package/bin/mesh-proxy.mjs +39 -0
- package/bin/mesh-quote.mjs +45 -0
- package/bin/mesh-report.mjs +47 -0
- package/index.cjs +4923 -0
- package/index.js +4850 -0
- package/package.json +39 -4
- package/types/blind-features.d.ts +28 -0
- package/types/cache-ttl-advisor.d.ts +31 -0
- package/types/cache-volatility-guard.d.ts +17 -0
- package/types/core/cache-simulator.d.ts +271 -0
- package/types/core/confidence.d.ts +36 -0
- package/types/core/cost.d.ts +16 -0
- package/types/core/evaluator.d.ts +6 -0
- package/types/core/gateway.d.ts +75 -0
- package/types/core/geo-visibility.d.ts +25 -0
- package/types/core/helicone.d.ts +6 -0
- package/types/core/index.d.ts +26 -0
- package/types/core/langfuse.d.ts +7 -0
- package/types/core/litellm.d.ts +15 -0
- package/types/core/local-tier.d.ts +82 -0
- package/types/core/model-prices.d.ts +25 -0
- package/types/core/output-diagnostics.d.ts +39 -0
- package/types/core/price-catalogue.d.ts +168 -0
- package/types/core/provider-usage.d.ts +84 -0
- package/types/core/quote-report.d.ts +54 -0
- package/types/core/replay-runner.d.ts +11 -0
- package/types/core/replay.d.ts +2 -0
- package/types/core/router.d.ts +9 -0
- package/types/core/routing-simulation.d.ts +24 -0
- package/types/core/savings-ledger.d.ts +349 -0
- package/types/core/savings-model.d.ts +132 -0
- package/types/core/stripe-billing.d.ts +67 -0
- package/types/core/stripe-sink.d.ts +77 -0
- package/types/core/theo-coverage.d.ts +49 -0
- package/types/core/token-reduction.d.ts +208 -0
- package/types/core/tool-schema.d.ts +64 -0
- package/types/core/traffic.d.ts +182 -0
- package/types/core/triage.d.ts +70 -0
- package/types/core/types.d.ts +138 -0
- package/types/evaluation-capture.d.ts +11 -0
- package/types/exact-cache.d.ts +20 -0
- package/types/graders.d.ts +53 -0
- package/types/index.d.ts +26 -0
- package/types/openai.d.ts +23 -0
- package/types/proxy.d.ts +63 -0
- package/types/reduce-request.d.ts +247 -0
- package/types/replay-gate.d.ts +120 -0
- package/types/report.d.ts +33 -0
- package/types/routing.d.ts +11 -0
- package/types/savings-ledger.d.ts +12 -0
- package/types/semantic-compaction.d.ts +84 -0
- package/types/telemetry.d.ts +6 -0
- package/types/theo-policy.d.ts +119 -0
- package/types/types.d.ts +263 -0
- package/types/wrap.d.ts +52 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { MeshExactCacheConfig } from "./types.js";
|
|
2
|
+
export type ExactCacheEntry = {
|
|
3
|
+
response: unknown;
|
|
4
|
+
model: string;
|
|
5
|
+
inputTokens?: number;
|
|
6
|
+
uncachedInputTokens?: number;
|
|
7
|
+
cachedInputTokens?: number;
|
|
8
|
+
cacheWriteInputTokens?: number;
|
|
9
|
+
outputTokens?: number;
|
|
10
|
+
costUsd?: number;
|
|
11
|
+
expiresAt: number;
|
|
12
|
+
};
|
|
13
|
+
export declare class ExactResponseCache {
|
|
14
|
+
private readonly config;
|
|
15
|
+
private readonly entries;
|
|
16
|
+
constructor(config: MeshExactCacheConfig);
|
|
17
|
+
keyFor(request: unknown, scopeKey: string | undefined): string | undefined;
|
|
18
|
+
get(key: string): ExactCacheEntry | undefined;
|
|
19
|
+
set(key: string, entry: Omit<ExactCacheEntry, "expiresAt">): void;
|
|
20
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type ReplayGrader } from "./replay-gate.js";
|
|
2
|
+
/**
|
|
3
|
+
* Harmonic mean of precision and recall over word overlap.
|
|
4
|
+
*
|
|
5
|
+
* Standard summarisation-style scoring, and it degrades gracefully in exactly
|
|
6
|
+
* the way truncation does: an answer carrying 90% of the baseline's content
|
|
7
|
+
* scores near 0.95 rather than 0, because its precision is intact and only its
|
|
8
|
+
* recall fell.
|
|
9
|
+
*/
|
|
10
|
+
export declare const textSimilarityGrader: ReplayGrader;
|
|
11
|
+
export type TruncationOptions = {
|
|
12
|
+
/**
|
|
13
|
+
* Share of the baseline's content below which a truncated answer counts as
|
|
14
|
+
* useless rather than merely shorter. Default 0.5.
|
|
15
|
+
*/
|
|
16
|
+
uselessBelowCoverage?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Penalty applied to an answer the provider marked as cut off, on top of its
|
|
19
|
+
* coverage. Default 0.05, so being visibly truncated costs a little even when
|
|
20
|
+
* almost everything survived.
|
|
21
|
+
*/
|
|
22
|
+
truncationPenalty?: number;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Scores a possibly-truncated answer by how much of the baseline's content it
|
|
26
|
+
* still carries, rather than failing it outright.
|
|
27
|
+
*
|
|
28
|
+
* Below `uselessBelowCoverage` the answer is treated as a failure, because an
|
|
29
|
+
* answer cut off in its first third usually is one. Above it, the score is the
|
|
30
|
+
* coverage less a small penalty for being visibly incomplete.
|
|
31
|
+
*/
|
|
32
|
+
export declare function truncationAwareGrader(options?: TruncationOptions): ReplayGrader;
|
|
33
|
+
export type WeightedGrader = {
|
|
34
|
+
grader: ReplayGrader;
|
|
35
|
+
weight: number;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Weighted mean of several graders.
|
|
39
|
+
*
|
|
40
|
+
* A reduction can fail in unrelated ways at once: trimming descriptions changes
|
|
41
|
+
* which tool is called, capping output shortens the answer. One number has to
|
|
42
|
+
* account for both, and a single grader only sees one of them.
|
|
43
|
+
*/
|
|
44
|
+
export declare function compositeGrader(parts: WeightedGrader[]): ReplayGrader;
|
|
45
|
+
/**
|
|
46
|
+
* Fails outright when the tool choice changed, and otherwise scores by how
|
|
47
|
+
* complete the answer is.
|
|
48
|
+
*
|
|
49
|
+
* The right default for an agent workload. A changed tool call is a different
|
|
50
|
+
* action taken in the world and no amount of eloquence redeems it, so it is a
|
|
51
|
+
* hard zero. A shorter answer is a matter of degree.
|
|
52
|
+
*/
|
|
53
|
+
export declare function agentGrader(options?: TruncationOptions): ReplayGrader;
|
package/types/index.d.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export { serviceTierOf, withMesh } from "./openai.js";
|
|
2
|
+
export * from "./types.js";
|
|
3
|
+
export { chooseRoute } from "./routing.js";
|
|
4
|
+
export { ExactResponseCache } from "./exact-cache.js";
|
|
5
|
+
export { createLocalJsonlCapture } from "./evaluation-capture.js";
|
|
6
|
+
export { billableEventFromRequest, createSavingsLedger } from "./savings-ledger.js";
|
|
7
|
+
export type { SavingsLedger } from "./savings-ledger.js";
|
|
8
|
+
export { startMeshProxy, usageFromSse } from "./proxy.js";
|
|
9
|
+
export type { MeshProxy, MeshProxyOptions } from "./proxy.js";
|
|
10
|
+
export { buildQuoteReport, parseGatewayExport, renderQuoteReport } from "./core/index.js";
|
|
11
|
+
export type { QuoteExportFormat, QuoteReport, QuoteStreamReport } from "./core/index.js";
|
|
12
|
+
export { auditEventPricing, canonicalJson, catalogueVersion, chainLedgerRecord, ledgerEventHash, LEDGER_GENESIS_HASH, priceLedgerUsage, publishedModelPrices, recomputeInvoice, verifyLedgerChain, } from "./core/index.js";
|
|
13
|
+
export type { BillableEvent, Invoice, InvoiceLineItem, InvoiceOptions, InvoiceUnbilledItem, LedgerChainVerification, LedgerExecutionMode, LedgerGateVerdict, LedgerLever, LedgerLeverKind, LedgerPricing, LedgerPricingAudit, LedgerPricingInput, LedgerRecord, LedgerUsage, PriceCatalogue, } from "./core/index.js";
|
|
14
|
+
export { capToolResultBody, digestStaleBody, reduceRequest } from "./reduce-request.js";
|
|
15
|
+
export { wrapMesh, type WrapMeshOptions, type MeshSavingsSummary, type MeshHandle, } from "./wrap.js";
|
|
16
|
+
export { extractBlindFeatures, type BlindFeatures } from "./blind-features.js";
|
|
17
|
+
export { policyToReductionConfig, withheldLevers, publishLever, defaultPolicy, knownLeverIds, type TheoPolicy, type TheoLever, type TheoGateProof, type TheoLeverClass, type WithheldLever, } from "./theo-policy.js";
|
|
18
|
+
export { compactStaleContext, reduceRequestAsync } from "./semantic-compaction.js";
|
|
19
|
+
export type { SemanticCompactionConfig, SemanticCompactionResult, SemanticCompactor, } from "./semantic-compaction.js";
|
|
20
|
+
export { CachePrefixVolatilityGuard, type CacheStampDecision } from "./cache-volatility-guard.js";
|
|
21
|
+
export { renderLedgerReport, type LedgerReportOptions, type LedgerReportSpool } from "./report.js";
|
|
22
|
+
export type { MeshHistoryCompactionConfig, MeshReductionChange, MeshReductionSummary, MeshTokenReductionConfig, } from "./reduce-request.js";
|
|
23
|
+
export { exactTextGrader, replayGate, replayUsage, responseText, responseToolCalls, toolNameGrader, toolSelectionGrader, } from "./replay-gate.js";
|
|
24
|
+
export type { ReplayCandidate, ReplayCandidateResult, ReplayComparison, ReplayGateOptions, ReplayGateResult, ReplayGrader, ReplayUsage, ToolCall, } from "./replay-gate.js";
|
|
25
|
+
export { agentGrader, compositeGrader, textSimilarityGrader, truncationAwareGrader, } from "./graders.js";
|
|
26
|
+
export type { TruncationOptions, WeightedGrader } from "./graders.js";
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { MeshOptions } from "./types.js";
|
|
2
|
+
type AnyObject = Record<PropertyKey, any>;
|
|
3
|
+
/**
|
|
4
|
+
* The served-tier detection the ledger event is built from: the tier the
|
|
5
|
+
* provider ANSWERED with (OpenAI's top-level response.service_tier, Anthropic's
|
|
6
|
+
* usage.service_tier), never the tier the request asked for. Exported so the
|
|
7
|
+
* measurement harness prices the SERVED tier through this exact code path
|
|
8
|
+
* rather than reimplementing it: a flex request answered "default" must bill
|
|
9
|
+
* standard everywhere, and a response with no tier evidence reports undefined.
|
|
10
|
+
*/
|
|
11
|
+
export declare function serviceTierOf(response: AnyObject | undefined): string | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* Wrap a provider client and preserve its response shapes.
|
|
14
|
+
*
|
|
15
|
+
* Three surfaces route through ONE shared handler: OpenAI Responses
|
|
16
|
+
* (responses.create), OpenAI Chat Completions (chat.completions.create), and
|
|
17
|
+
* an Anthropic-SDK-shaped client (messages.create). Reduction, routing, the
|
|
18
|
+
* exact cache, telemetry, evaluation capture and the savings ledger are all
|
|
19
|
+
* the same code on every surface - the adapters differ only in which output-cap
|
|
20
|
+
* field a routing candidate uses and which provider label the event records.
|
|
21
|
+
*/
|
|
22
|
+
export declare function withMesh<T extends AnyObject>(client: T, options: MeshOptions): T;
|
|
23
|
+
export {};
|
package/types/proxy.d.ts
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { type MeshTokenReductionConfig } from "./reduce-request.js";
|
|
2
|
+
/**
|
|
3
|
+
* MESH proxy mode: point an application's base_url here instead of changing
|
|
4
|
+
* its code. The proxy applies the configured token reduction to each request,
|
|
5
|
+
* forwards it to the real provider with the CALLER's own credentials, returns
|
|
6
|
+
* the provider's bytes untouched, and appends one hash-chained ledger event
|
|
7
|
+
* per completed request — the same recomputable record the SDK wrapper writes.
|
|
8
|
+
*
|
|
9
|
+
* The contract, enforced rather than promised:
|
|
10
|
+
* - MESH never holds keys. Auth headers pass through verbatim; nothing is
|
|
11
|
+
* stored. Prompts are analysed in-process and never written anywhere — the
|
|
12
|
+
* ledger records usage numbers, lever names and hashes, not content.
|
|
13
|
+
* - Nothing here may break a request. Reduction never throws (a failure
|
|
14
|
+
* returns the original request); a ledger failure silently disables the
|
|
15
|
+
* meter; only an unreachable upstream produces an error, and it is the
|
|
16
|
+
* upstream's own status and body wherever one exists.
|
|
17
|
+
* - Responses stream through as they arrive. Usage for the ledger is parsed
|
|
18
|
+
* from the SSE events after the last byte has already been forwarded; a
|
|
19
|
+
* stream whose usage cannot be read produces an UNPRICED event, never a
|
|
20
|
+
* guessed one.
|
|
21
|
+
* - Lossless levers only by default. Lossy levers and tier/cache stamps are
|
|
22
|
+
* per-deployment configuration, exactly as in the SDK.
|
|
23
|
+
*/
|
|
24
|
+
type AnyObject = Record<string, unknown>;
|
|
25
|
+
export type MeshProxyOptions = {
|
|
26
|
+
/** Port to listen on; 0 picks a free one (reported by start()). */
|
|
27
|
+
port?: number;
|
|
28
|
+
/** Directory for the hash-chained ledger spools (one per app + provider). */
|
|
29
|
+
ledgerDirectory: string;
|
|
30
|
+
/** Reduction config applied to every proxied request. Lossless-only default. */
|
|
31
|
+
tokenReduction?: MeshTokenReductionConfig;
|
|
32
|
+
/** Override upstreams, e.g. a mock in tests or an OpenAI-compatible host. */
|
|
33
|
+
anthropicUpstream?: string;
|
|
34
|
+
openaiUpstream?: string;
|
|
35
|
+
/**
|
|
36
|
+
* Enable the provably-blind learning stream: when set, each request also
|
|
37
|
+
* appends an IRREVERSIBLE feature record (keyed HMAC hashes + structural
|
|
38
|
+
* counts, never any prompt content) to a `<app>-<provider>.features.jsonl`
|
|
39
|
+
* spool. This is the hmac key; it must live only in the gateway (ideally
|
|
40
|
+
* sealed inside an enclave). Off when unset — no features are computed.
|
|
41
|
+
*/
|
|
42
|
+
blindFeaturesKey?: string;
|
|
43
|
+
};
|
|
44
|
+
export type MeshProxy = {
|
|
45
|
+
port: number;
|
|
46
|
+
url: string;
|
|
47
|
+
close: () => Promise<void>;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* Merge the usage scattered across a surface's SSE events into one response-
|
|
51
|
+
* shaped object normalizeProviderUsage can read. Returns undefined when the
|
|
52
|
+
* stream carried no readable usage — the event is then unpriced, never guessed.
|
|
53
|
+
*/
|
|
54
|
+
export declare function usageFromSse(provider: "anthropic" | "openai", sseText: string): AnyObject | undefined;
|
|
55
|
+
/**
|
|
56
|
+
* Assistant text of a streamed response, for output-waste metering: the
|
|
57
|
+
* concatenated text deltas of whichever surface produced the stream. Used
|
|
58
|
+
* only after the bytes were already forwarded; never stored.
|
|
59
|
+
*/
|
|
60
|
+
export declare function outputTextFromSse(provider: "anthropic" | "openai", sseText: string): string;
|
|
61
|
+
/** Start the MESH proxy. Resolves once the server is listening. */
|
|
62
|
+
export declare function startMeshProxy(options: MeshProxyOptions): Promise<MeshProxy>;
|
|
63
|
+
export {};
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { type TokenCounter } from "../../core/dist/src/token-reduction.js";
|
|
2
|
+
/**
|
|
3
|
+
* Reduce a request before it is sent.
|
|
4
|
+
*
|
|
5
|
+
* This is the part of MESH that actually lowers a bill rather than measuring how
|
|
6
|
+
* much it could be lowered. It runs on every request, independent of model
|
|
7
|
+
* routing: the levers here spend fewer tokens on the model the caller already
|
|
8
|
+
* chose, so coupling them to a model swap would be backwards.
|
|
9
|
+
*
|
|
10
|
+
* Every transform here is lossless by default. Tool definitions keep their
|
|
11
|
+
* descriptions, only formatting and schema plumbing go, duplicate blocks are
|
|
12
|
+
* exact duplicates, and prefix repair moves text rather than deleting it. The
|
|
13
|
+
* lossy options exist but are opt-in and reported as such, because descriptions
|
|
14
|
+
* are how a model chooses between tools.
|
|
15
|
+
*
|
|
16
|
+
* Nothing here may break a request. Any failure returns the original untouched.
|
|
17
|
+
*/
|
|
18
|
+
export type MeshReductionChange = {
|
|
19
|
+
lever: string;
|
|
20
|
+
lossless: boolean;
|
|
21
|
+
estimatedTokensSaved: number;
|
|
22
|
+
detail: string;
|
|
23
|
+
};
|
|
24
|
+
/** Reasoning/output effort levels Anthropic's `output_config.effort` accepts. */
|
|
25
|
+
export type MeshOutputEffort = "low" | "medium" | "high" | "xhigh" | "max";
|
|
26
|
+
export type MeshReductionSummary = {
|
|
27
|
+
/** Estimated input tokens before reduction, by MESH's own counter. */
|
|
28
|
+
estimatedTokensBefore: number;
|
|
29
|
+
estimatedTokensAfter: number;
|
|
30
|
+
estimatedTokensSaved: number;
|
|
31
|
+
/** Of the above, the share from transforms that cannot change the answer. */
|
|
32
|
+
estimatedLosslessTokensSaved: number;
|
|
33
|
+
changes: MeshReductionChange[];
|
|
34
|
+
/** True when reduction was attempted but the request was returned unchanged. */
|
|
35
|
+
skipped: boolean;
|
|
36
|
+
skippedReason?: string;
|
|
37
|
+
};
|
|
38
|
+
export type MeshHistoryCompactionConfig = {
|
|
39
|
+
/**
|
|
40
|
+
* Compaction starts once the conversation has at least this many user turns.
|
|
41
|
+
* Short conversations are left alone: there is nothing stale to compact.
|
|
42
|
+
*/
|
|
43
|
+
minimumTurns?: number;
|
|
44
|
+
/** The most recent turns whose content is always left verbatim. */
|
|
45
|
+
keepRecentTurns?: number;
|
|
46
|
+
/**
|
|
47
|
+
* The compaction boundary only advances in steps of this many turns, so the
|
|
48
|
+
* set of compacted messages is identical across several consecutive requests
|
|
49
|
+
* and the provider's exact-prefix cache can keep matching through them.
|
|
50
|
+
*/
|
|
51
|
+
epochTurns?: number;
|
|
52
|
+
};
|
|
53
|
+
export type MeshTokenReductionConfig = {
|
|
54
|
+
/**
|
|
55
|
+
* Tool and MCP schema minification. `true` applies the lossless tier only.
|
|
56
|
+
* Pass an object to opt into description trimming, which is a behaviour change.
|
|
57
|
+
*/
|
|
58
|
+
toolSchemas?: boolean | {
|
|
59
|
+
descriptions?: "keep" | "truncate" | "drop";
|
|
60
|
+
maximumDescriptionTokens?: number;
|
|
61
|
+
};
|
|
62
|
+
/** Remove exact duplicate blocks from instructions and message text. */
|
|
63
|
+
duplicateContext?: boolean;
|
|
64
|
+
/**
|
|
65
|
+
* Move per-request values (timestamps, request ids) out of the front of the
|
|
66
|
+
* instructions so the stable remainder becomes a cacheable prefix. Providers
|
|
67
|
+
* match an exact prefix, so one changing value at the top of an otherwise
|
|
68
|
+
* identical system prompt costs the whole prompt's cache discount.
|
|
69
|
+
*/
|
|
70
|
+
repairCachePrefix?: boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Replace stale tool-result and assistant bodies in old turns with short
|
|
73
|
+
* deterministic digests. Error messages, file paths and the headline sentence
|
|
74
|
+
* survive verbatim; the rest is summarised by a byte count. The boundary only
|
|
75
|
+
* moves in frozen epochs, so consecutive requests re-emit the compacted
|
|
76
|
+
* history byte-identically and the provider cache re-writes once per epoch,
|
|
77
|
+
* never once per turn. `true` uses the defaults; pass an object to tune.
|
|
78
|
+
*/
|
|
79
|
+
historyCompaction?: boolean | MeshHistoryCompactionConfig;
|
|
80
|
+
/**
|
|
81
|
+
* Replace a repeated LEADING block in same-role messages after the first
|
|
82
|
+
* with a short reference marker, keeping each message's differing
|
|
83
|
+
* remainder verbatim. This is the shape a continuous-learning /
|
|
84
|
+
* continuous-memory agent produces: the application glues the same
|
|
85
|
+
* memory or persistent-instruction block to the front of every turn, so
|
|
86
|
+
* the history carries N-1 byte-identical copies that no existing lever
|
|
87
|
+
* catches (`duplicateContext` only ever sees one message's text at a
|
|
88
|
+
* time, never the text several messages back that a later one repeats).
|
|
89
|
+
*
|
|
90
|
+
* LOSSY, unlike `duplicateContext`: the duplicate removed here is never
|
|
91
|
+
* adjacent to its twin, so a model that attends less to context far back
|
|
92
|
+
* in a long history may behave differently with the block present once
|
|
93
|
+
* near the start versus restated fresh every turn. This must be proven
|
|
94
|
+
* on the customer's own traffic through `replayGate` before it is ever
|
|
95
|
+
* billed, and is never enabled by default. The cut is always snapped to
|
|
96
|
+
* a line boundary (see `dedupeRepeatedLeadingBlock`), so a false match on
|
|
97
|
+
* two messages that merely open with the same few words can never
|
|
98
|
+
* truncate into the line where they actually differ. `true` uses the
|
|
99
|
+
* default 64-token minimum block size; pass an object to tune it. Applies
|
|
100
|
+
* only to messages whose content is a plain string or whose first content
|
|
101
|
+
* block is `{type: "text", text}`; anything else is left untouched.
|
|
102
|
+
*/
|
|
103
|
+
crossTurnMemoryDedup?: boolean | {
|
|
104
|
+
minimumBlockTokens?: number;
|
|
105
|
+
};
|
|
106
|
+
/**
|
|
107
|
+
* Cap oversized tool-result and assistant bodies at injection, BEFORE first
|
|
108
|
+
* send. A fresh oversized result is billed once at the FULL input rate and
|
|
109
|
+
* only its cached rebills are discounted, so this reaches tokens that
|
|
110
|
+
* history compaction (which touches already-cached turns) never can. The
|
|
111
|
+
* headline sentence, error mentions, file paths and MESH's own markers
|
|
112
|
+
* survive verbatim; the rest collapses to a char-count marker. Pure,
|
|
113
|
+
* deterministic and idempotent, so consecutive requests re-emit identical
|
|
114
|
+
* bytes and the provider cache keeps matching. Lossy by definition and
|
|
115
|
+
* reported as such. `true` uses the 600-token default; pass an object to
|
|
116
|
+
* tune `maximumTokens` (integer, at least 64).
|
|
117
|
+
*/
|
|
118
|
+
freshToolResultCap?: boolean | {
|
|
119
|
+
maximumTokens?: number;
|
|
120
|
+
};
|
|
121
|
+
/**
|
|
122
|
+
* Output budget, applied through the field each surface honours: Responses
|
|
123
|
+
* max_output_tokens (set only when absent), Chat Completions
|
|
124
|
+
* max_completion_tokens (set only when the caller set no cap in any
|
|
125
|
+
* spelling), Anthropic Messages max_tokens (downward-only clamp of the
|
|
126
|
+
* caller's required value). Output is the dearer half of the bill.
|
|
127
|
+
*/
|
|
128
|
+
maxOutputTokens?: number;
|
|
129
|
+
/**
|
|
130
|
+
* Output effort governor: Anthropic's `output_config.effort`
|
|
131
|
+
* ("low" | "medium" | "high" | "xhigh" | "max"; default "high"). The one
|
|
132
|
+
* output-side lever that is PROVIDER-SANCTIONED rather than a prompt hack — a
|
|
133
|
+
* lower effort makes the model think less and answer terser (less preamble,
|
|
134
|
+
* fewer and more-consolidated tool calls), cutting the dearest half of the
|
|
135
|
+
* bill: output tokens price 4-8x input. This is MESH's honest equivalent of a
|
|
136
|
+
* competitor that "trims the model's replies": the model decides how to be
|
|
137
|
+
* briefer, we do not truncate or rewrite its words.
|
|
138
|
+
*
|
|
139
|
+
* It is a TOP-LEVEL request field, outside the cached prefix (tools -> system
|
|
140
|
+
* -> messages), so a STABLE value never invalidates a prompt cache — unlike
|
|
141
|
+
* editing the system prompt to beg for brevity, which rewrites the cached
|
|
142
|
+
* prefix every turn. (A value that CHANGED between turns would invalidate the
|
|
143
|
+
* messages cache, so this is a per-workload setting, never flipped mid-run.)
|
|
144
|
+
*
|
|
145
|
+
* LOSSY: it changes the answer's depth and length, so it is off by default,
|
|
146
|
+
* must clear a TASK-COMPLETION replay gate before any saving is billed
|
|
147
|
+
* (exactTextGrader is meaningless here — a terser answer is different text by
|
|
148
|
+
* definition), and is only ever the caller's per-workload flag. Applied only
|
|
149
|
+
* on Claude models whose published effort range includes the requested level
|
|
150
|
+
* (an unsupported level is a 400, and this reducer must never break a
|
|
151
|
+
* request); an unsupported pairing is recorded and skipped, never stamped. A
|
|
152
|
+
* caller who set `output_config.effort` themselves is left untouched. The
|
|
153
|
+
* OpenAI `reasoning.effort` sibling is a planned follow-on, not wired here.
|
|
154
|
+
*/
|
|
155
|
+
outputEffort?: MeshOutputEffort;
|
|
156
|
+
/**
|
|
157
|
+
* Execution tier for requests the caller has flagged latency-tolerant.
|
|
158
|
+
* "flex" sets OpenAI's service_tier on the request (batch-level pricing, 50%
|
|
159
|
+
* off input and output, on a synchronous API with a possible 429 fallback to
|
|
160
|
+
* standard). The answer is unchanged - same model, same tokens - so this is
|
|
161
|
+
* deterministic for cost; only latency changes, which is why it must be the
|
|
162
|
+
* CALLER's flag per workload and is never set by default. Applied only when
|
|
163
|
+
* the request does not already carry a service_tier, only on OpenAI-shaped
|
|
164
|
+
* requests (service_tier is not an Anthropic field), and only for models the
|
|
165
|
+
* published catalogue resolves - a tier delta on an unpriceable model could
|
|
166
|
+
* never be verified, so it is never stamped.
|
|
167
|
+
*/
|
|
168
|
+
executionTier?: "flex";
|
|
169
|
+
/**
|
|
170
|
+
* Anthropic prompt-cache stamping. Anthropic's cache is OPT-IN: a Claude
|
|
171
|
+
* request with no cache_control gets no caching at all and re-pays full
|
|
172
|
+
* rate for its entire history every turn (the first live export measured a
|
|
173
|
+
* five-turn agent with zero cache traffic). This lever stamps
|
|
174
|
+
* `cache_control: {type: "ephemeral", ttl}` on the STABLE prefix of an
|
|
175
|
+
* Anthropic-shaped request (a `claude-*` model with a `messages` array):
|
|
176
|
+
* always on the system prompt when one exists, and on the last message only
|
|
177
|
+
* for multi-turn requests, whose history is re-sent verbatim next turn. A
|
|
178
|
+
* one-shot request with no system prompt is never stamped — its prefix
|
|
179
|
+
* never repeats, so a breakpoint could only buy write premiums (the first
|
|
180
|
+
* live export measured exactly that loss).
|
|
181
|
+
*
|
|
182
|
+
* Pick the TTL by call spacing. "5m" (writes 1.25x base, reads 0.1x) is the
|
|
183
|
+
* multi-turn agent setting: turns arrive seconds apart, so every turn after
|
|
184
|
+
* the first reads its history at a tenth of the price. "1h" (writes 2x)
|
|
185
|
+
* pays exactly when calls arrive MORE than ~5 minutes but less than an hour
|
|
186
|
+
* apart - the regime where a 5m entry expires between calls and every call
|
|
187
|
+
* rewrites. Applied only when the request carries no cache_control of its
|
|
188
|
+
* own: a caller who placed breakpoints has chosen their own caching and
|
|
189
|
+
* silently raising their write rate would change their bill. Never set by
|
|
190
|
+
* default; like executionTier, it must be the caller's per-workload flag.
|
|
191
|
+
*/
|
|
192
|
+
cacheTtl?: "5m" | "1h";
|
|
193
|
+
/**
|
|
194
|
+
* Cross-turn volatility guard for the cache stamp (see
|
|
195
|
+
* cache-volatility-guard.ts). The reducer is a pure per-request function
|
|
196
|
+
* and cannot see that a conversation's prefix churns between turns; the
|
|
197
|
+
* guard can, and a churning prefix turns the stamp into pure write premium
|
|
198
|
+
* (the second live export measured exactly that loss). The proxy and
|
|
199
|
+
* withMesh install one automatically whenever cacheTtl is set; it is
|
|
200
|
+
* consulted with the RAW request before any stamp is placed.
|
|
201
|
+
*/
|
|
202
|
+
cacheVolatilityGuard?: {
|
|
203
|
+
observe(request: unknown): {
|
|
204
|
+
stamp: boolean;
|
|
205
|
+
reason?: string;
|
|
206
|
+
};
|
|
207
|
+
};
|
|
208
|
+
countTokens?: TokenCounter;
|
|
209
|
+
onReduction?: (summary: MeshReductionSummary) => void;
|
|
210
|
+
};
|
|
211
|
+
/**
|
|
212
|
+
* Digest one stale body. Pure function of the text, so every later request in
|
|
213
|
+
* the same epoch re-emits exactly the same bytes, which is what lets the
|
|
214
|
+
* provider's exact-prefix cache match across the compacted region.
|
|
215
|
+
*
|
|
216
|
+
* Kept verbatim: the headline (first sentence), any sentence that mentions an
|
|
217
|
+
* error, and any sentence containing a file path. Everything else collapses to
|
|
218
|
+
* a byte count. If that would not actually shorten the text, the original is
|
|
219
|
+
* returned untouched.
|
|
220
|
+
*/
|
|
221
|
+
export declare function digestStaleBody(text: string): string;
|
|
222
|
+
/**
|
|
223
|
+
* Cap one oversized tool-result body at injection time, BEFORE it is first
|
|
224
|
+
* sent. A fresh result is billed once at the FULL input rate; only its cached
|
|
225
|
+
* rebills get the provider discount, so this is the one lever that removes
|
|
226
|
+
* full-rate tokens rather than nibbling 0.1x rebills of an already-cached copy.
|
|
227
|
+
*
|
|
228
|
+
* Pure and idempotent: a body at or under `maximumTokens` is returned
|
|
229
|
+
* untouched (identity, no marker), and capping a capped body changes nothing,
|
|
230
|
+
* because every surviving segment - the headline, error mentions, file paths
|
|
231
|
+
* and MESH's own markers - is a keeper on the second pass too. The same keep
|
|
232
|
+
* rules as stale compaction apply, so later turns still find the reference
|
|
233
|
+
* they depend on; a capper that lost the latest result's headline would fail
|
|
234
|
+
* tasks, which is exactly the quality bar working.
|
|
235
|
+
*/
|
|
236
|
+
export declare function capToolResultBody(text: string, maximumTokens: number, countTokens: TokenCounter): string;
|
|
237
|
+
/**
|
|
238
|
+
* Apply reduction to one request.
|
|
239
|
+
*
|
|
240
|
+
* Returns the original request and a `skipped` summary if anything goes wrong,
|
|
241
|
+
* because a cheaper request that fails is worth less than an expensive one that
|
|
242
|
+
* works.
|
|
243
|
+
*/
|
|
244
|
+
export declare function reduceRequest(request: unknown, config?: MeshTokenReductionConfig): {
|
|
245
|
+
request: unknown;
|
|
246
|
+
summary: MeshReductionSummary;
|
|
247
|
+
};
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { type TokenPrice } from "../../core/dist/src/cost.js";
|
|
2
|
+
import { type MeshTokenReductionConfig } from "./reduce-request.js";
|
|
3
|
+
export type ReplayComparison = {
|
|
4
|
+
request: unknown;
|
|
5
|
+
baselineResponse: unknown;
|
|
6
|
+
candidateResponse: unknown;
|
|
7
|
+
};
|
|
8
|
+
/** Returns 1 when the candidate is as good as the baseline, 0 when unusable. */
|
|
9
|
+
export type ReplayGrader = (comparison: ReplayComparison) => number | Promise<number>;
|
|
10
|
+
export type ReplayCandidate = {
|
|
11
|
+
label: string;
|
|
12
|
+
config: MeshTokenReductionConfig;
|
|
13
|
+
};
|
|
14
|
+
export type ReplayGateOptions = {
|
|
15
|
+
/** A representative sample of real requests. */
|
|
16
|
+
requests: unknown[];
|
|
17
|
+
/** Sends a request and returns the provider's response. */
|
|
18
|
+
call: (request: unknown) => Promise<unknown>;
|
|
19
|
+
candidates: ReplayCandidate[];
|
|
20
|
+
grade: ReplayGrader;
|
|
21
|
+
/** Mean quality the gate may give up, against the unreduced baseline. Default 0.02. */
|
|
22
|
+
maximumQualityDrop?: number;
|
|
23
|
+
bootstrapSamples?: number;
|
|
24
|
+
seed?: string;
|
|
25
|
+
/**
|
|
26
|
+
* Group label per request. Several requests drawn from one prompt or task are
|
|
27
|
+
* correlated, and treating them as independent overstates the bound.
|
|
28
|
+
*/
|
|
29
|
+
groupOf?: (request: unknown, index: number) => string;
|
|
30
|
+
/** Recorded baseline responses, to avoid paying for the baseline run again. */
|
|
31
|
+
baselineResponses?: unknown[];
|
|
32
|
+
/** Rate used to express savings in money as well as tokens. */
|
|
33
|
+
price?: TokenPrice;
|
|
34
|
+
/**
|
|
35
|
+
* Called before the baseline run and before each candidate run.
|
|
36
|
+
*
|
|
37
|
+
* Providers hold state between calls: a prefix cache is cold on the first
|
|
38
|
+
* request and warm afterwards. Without a reset between runs the baseline
|
|
39
|
+
* warms the cache that the candidates then benefit from, which flatters every
|
|
40
|
+
* candidate and makes the baseline look worse than it is. Use this to clear
|
|
41
|
+
* whatever state would otherwise leak across runs.
|
|
42
|
+
*/
|
|
43
|
+
beforeRun?: (label: string) => void | Promise<void>;
|
|
44
|
+
};
|
|
45
|
+
export type ReplayUsage = {
|
|
46
|
+
/** Total input billed, inclusive of anything served from cache. */
|
|
47
|
+
inputTokens: number;
|
|
48
|
+
outputTokens: number;
|
|
49
|
+
/** Input served from the provider's cache, billed at the cached rate. */
|
|
50
|
+
cachedInputTokens: number;
|
|
51
|
+
/** Input written into the cache, which some providers bill at a premium. */
|
|
52
|
+
cacheWriteInputTokens: number;
|
|
53
|
+
};
|
|
54
|
+
export type ReplayCandidateResult = {
|
|
55
|
+
label: string;
|
|
56
|
+
config: MeshTokenReductionConfig;
|
|
57
|
+
graded: number;
|
|
58
|
+
/** Requests where either side failed, excluded from the verdict. */
|
|
59
|
+
failed: number;
|
|
60
|
+
meanQualityDelta: number;
|
|
61
|
+
lower95QualityDelta: number;
|
|
62
|
+
approved: boolean;
|
|
63
|
+
rejectionReason?: string;
|
|
64
|
+
baselineUsage: ReplayUsage;
|
|
65
|
+
candidateUsage: ReplayUsage;
|
|
66
|
+
inputTokensSaved: number;
|
|
67
|
+
outputTokensSaved: number;
|
|
68
|
+
tokenSavingPercent: number;
|
|
69
|
+
costSavedUsd?: number;
|
|
70
|
+
costSavingPercent?: number;
|
|
71
|
+
};
|
|
72
|
+
export type ReplayGateResult = {
|
|
73
|
+
candidates: ReplayCandidateResult[];
|
|
74
|
+
/** The most aggressive candidate that held quality, if any did. */
|
|
75
|
+
approved?: ReplayCandidateResult;
|
|
76
|
+
maximumQualityDrop: number;
|
|
77
|
+
/** Provider calls this replay made, so its cost is visible. */
|
|
78
|
+
calls: number;
|
|
79
|
+
notes: string[];
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* Pull usage out of a Responses, Chat Completions or Anthropic Messages shape.
|
|
83
|
+
*
|
|
84
|
+
* Cached tokens must be read, not ignored. Pricing all input at the full rate
|
|
85
|
+
* makes prefix caching worth exactly zero to the gate, which would mean the
|
|
86
|
+
* largest lossless lever could never be measured or credited.
|
|
87
|
+
*
|
|
88
|
+
* The parsing itself now lives in packages/core (normalizeProviderUsage), the
|
|
89
|
+
* single shared normaliser the SDK wrapper and the ledger read through too, so
|
|
90
|
+
* the gate and the meter can never disagree about what a response cost. This
|
|
91
|
+
* wrapper keeps the gate's historical contract: the usage, or undefined.
|
|
92
|
+
*/
|
|
93
|
+
export declare function replayUsage(response: unknown): ReplayUsage | undefined;
|
|
94
|
+
/** Concatenate the assistant text of a response, ignoring structure. */
|
|
95
|
+
export declare function responseText(response: unknown): string;
|
|
96
|
+
export type ToolCall = {
|
|
97
|
+
name: string;
|
|
98
|
+
argumentsJson: string;
|
|
99
|
+
};
|
|
100
|
+
/** Tool calls a response made, normalised so two responses can be compared. */
|
|
101
|
+
export declare function responseToolCalls(response: unknown): ToolCall[];
|
|
102
|
+
/** Scores 1 only when the assistant text matches exactly. */
|
|
103
|
+
export declare const exactTextGrader: ReplayGrader;
|
|
104
|
+
/**
|
|
105
|
+
* Scores on whether the same tools were called with the same arguments.
|
|
106
|
+
*
|
|
107
|
+
* The grader that matters for tool-description trimming, because that lever's
|
|
108
|
+
* risk is precisely that the model picks a different tool. Text graders miss it:
|
|
109
|
+
* two different tool choices can produce similarly-worded replies.
|
|
110
|
+
*/
|
|
111
|
+
export declare const toolSelectionGrader: ReplayGrader;
|
|
112
|
+
/** Scores 1 when the same tool names were called, ignoring their arguments. */
|
|
113
|
+
export declare const toolNameGrader: ReplayGrader;
|
|
114
|
+
/**
|
|
115
|
+
* Replay a sample both ways and report which candidates may be enabled.
|
|
116
|
+
*
|
|
117
|
+
* Returns every candidate's verdict, not only the winner, so a rejection can be
|
|
118
|
+
* read as rejected-at-what-cost rather than merely absent.
|
|
119
|
+
*/
|
|
120
|
+
export declare function replayGate(options: ReplayGateOptions): Promise<ReplayGateResult>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type LedgerRecord } from "./core/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* The customer-facing monthly statement: one self-contained HTML page
|
|
4
|
+
* rendered from the customer's own ledger spools.
|
|
5
|
+
*
|
|
6
|
+
* This is the invoice as MESH means it: every figure on the page is the
|
|
7
|
+
* output of `recomputeInvoice` over records the customer holds, the chain
|
|
8
|
+
* verification status is printed at the top (a broken chain renders DO NOT
|
|
9
|
+
* PAY, not a number), exclusions and unbilled levers are itemised rather
|
|
10
|
+
* than hidden, negative levers appear as negative, estimates carry their
|
|
11
|
+
* label, and the command to recompute everything independently is printed
|
|
12
|
+
* on the statement itself. No network access, no external assets.
|
|
13
|
+
*/
|
|
14
|
+
export type LedgerReportSpool = {
|
|
15
|
+
/** Display name, usually the spool filename. */
|
|
16
|
+
name: string;
|
|
17
|
+
records: readonly LedgerRecord[];
|
|
18
|
+
};
|
|
19
|
+
export type LedgerReportOptions = {
|
|
20
|
+
/** Restrict to one invoice period (YYYY-MM). Omitted: all events. */
|
|
21
|
+
period?: string;
|
|
22
|
+
/** MESH's contracted share of net verified savings. Default 1 (percent). */
|
|
23
|
+
feePercent?: number;
|
|
24
|
+
/**
|
|
25
|
+
* Verified-savings deficit carried in from prior statements (cumulative
|
|
26
|
+
* netting: fees never exceed aggregate verified savings). Applied at the
|
|
27
|
+
* statement total, shown as its own line.
|
|
28
|
+
*/
|
|
29
|
+
carryForwardDeficitUsd?: number;
|
|
30
|
+
/** Timestamp printed on the statement; injectable for deterministic tests. */
|
|
31
|
+
generatedAt?: string;
|
|
32
|
+
};
|
|
33
|
+
export declare function renderLedgerReport(spools: readonly LedgerReportSpool[], options?: LedgerReportOptions): string;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { MeshRequestContext, MeshRoutingConfig } from "./types.js";
|
|
2
|
+
export type MeshRoute = {
|
|
3
|
+
model: string;
|
|
4
|
+
requestedModel: string;
|
|
5
|
+
policyId?: string;
|
|
6
|
+
reason: "no_matching_policy" | "missing_stable_key" | "baseline" | "candidate_quality_and_cost_gates_passed";
|
|
7
|
+
estimatedCostSavingsPercent?: number;
|
|
8
|
+
maxOutputTokens?: number;
|
|
9
|
+
contextProfileId?: string;
|
|
10
|
+
};
|
|
11
|
+
export declare function chooseRoute(config: MeshRoutingConfig | undefined, request: Record<PropertyKey, any>, taskClass: string | undefined, context?: MeshRequestContext): MeshRoute;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type PriceCatalogue } from "../../core/dist/src/price-catalogue.js";
|
|
2
|
+
import { type BillableEvent } from "../../core/dist/src/savings-ledger.js";
|
|
3
|
+
import type { MeshRequestEvent, MeshSavingsLedgerConfig } from "./types.js";
|
|
4
|
+
/** Build the append-only billable record for one completed request. Pure. */
|
|
5
|
+
export declare function billableEventFromRequest(event: MeshRequestEvent, config: MeshSavingsLedgerConfig, catalogue: PriceCatalogue, pinnedCatalogueVersion: string): BillableEvent;
|
|
6
|
+
export type SavingsLedger = {
|
|
7
|
+
/** Synchronously appends one chained record. Never throws. */
|
|
8
|
+
emit: (event: MeshRequestEvent) => void;
|
|
9
|
+
readonly path: string;
|
|
10
|
+
};
|
|
11
|
+
/** Create the opt-in JSONL savings meter. Construction never throws on I/O. */
|
|
12
|
+
export declare function createSavingsLedger(config: MeshSavingsLedgerConfig): SavingsLedger;
|