@harrym1707/sdk 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +98 -0
- package/README.md +48 -2
- package/bin/mesh-demo.mjs +105 -0
- package/bin/mesh-proxy.mjs +39 -0
- package/bin/mesh-quote.mjs +45 -0
- package/bin/mesh-report.mjs +47 -0
- package/index.cjs +6306 -0
- package/index.js +6204 -0
- package/package.json +39 -4
- package/types/blind-features.d.ts +28 -0
- package/types/cache-volatility-guard.d.ts +31 -0
- package/types/core/account-store.d.ts +7 -0
- package/types/core/accounts.d.ts +134 -0
- package/types/core/billing-worker.d.ts +113 -0
- package/types/core/budget.d.ts +216 -0
- package/types/core/cache-simulator.d.ts +271 -0
- package/types/core/confidence.d.ts +36 -0
- package/types/core/cost.d.ts +16 -0
- package/types/core/evaluator.d.ts +16 -0
- package/types/core/gateway.d.ts +75 -0
- package/types/core/geo-visibility.d.ts +25 -0
- package/types/core/github-oauth.d.ts +41 -0
- package/types/core/helicone.d.ts +6 -0
- package/types/core/index.d.ts +36 -0
- package/types/core/langfuse.d.ts +7 -0
- package/types/core/ledger-store.d.ts +62 -0
- package/types/core/litellm.d.ts +15 -0
- package/types/core/local-tier.d.ts +82 -0
- package/types/core/model-prices.d.ts +25 -0
- package/types/core/output-diagnostics.d.ts +39 -0
- package/types/core/price-catalogue.d.ts +168 -0
- package/types/core/provider-usage.d.ts +84 -0
- package/types/core/quote-report.d.ts +54 -0
- package/types/core/replay.d.ts +2 -0
- package/types/core/router.d.ts +9 -0
- package/types/core/routing.d.ts +282 -0
- package/types/core/savings-ledger.d.ts +349 -0
- package/types/core/savings-model.d.ts +132 -0
- package/types/core/stripe-billing.d.ts +67 -0
- package/types/core/stripe-client.d.ts +117 -0
- package/types/core/stripe-sink.d.ts +77 -0
- package/types/core/theo-coverage.d.ts +49 -0
- package/types/core/token-reduction.d.ts +219 -0
- package/types/core/tool-schema.d.ts +64 -0
- package/types/core/traffic.d.ts +182 -0
- package/types/core/triage.d.ts +70 -0
- package/types/core/types.d.ts +149 -0
- package/types/core/workflow-sessions.d.ts +50 -0
- package/types/cta.d.ts +60 -0
- package/types/evaluation-capture.d.ts +11 -0
- package/types/exact-cache.d.ts +20 -0
- package/types/graders.d.ts +53 -0
- package/types/index.d.ts +31 -0
- package/types/ledger-mirror.d.ts +38 -0
- package/types/openai.d.ts +23 -0
- package/types/proxy.d.ts +104 -0
- package/types/reduce-request.d.ts +272 -0
- package/types/replay-gate.d.ts +120 -0
- package/types/report.d.ts +33 -0
- package/types/routing.d.ts +11 -0
- package/types/savings-ledger.d.ts +22 -0
- package/types/savings-snapshot.d.ts +11 -0
- package/types/semantic-compaction.d.ts +84 -0
- package/types/telemetry.d.ts +6 -0
- package/types/theo-agent-store.d.ts +27 -0
- package/types/theo-agent.d.ts +247 -0
- package/types/theo-gate.d.ts +75 -0
- package/types/theo-policy.d.ts +219 -0
- package/types/types.d.ts +287 -0
- package/types/wrap.d.ts +60 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type BillableEvent } from "./core/index.js";
|
|
2
|
+
import type { SavingsSnapshot } from "./cta.js";
|
|
3
|
+
/** Fold a set of ledger events into a snapshot. `chainOk` carries through. */
|
|
4
|
+
export declare function savingsSnapshotFromEvents(events: readonly BillableEvent[], chainOk?: boolean): SavingsSnapshot;
|
|
5
|
+
/**
|
|
6
|
+
* Summarise every ledger spool in a directory (the proxy writes one JSONL per
|
|
7
|
+
* app + provider). Any spool whose hash chain does not verify is excluded and
|
|
8
|
+
* flips `chainOk` to false, so a tampered or truncated spool is never silently
|
|
9
|
+
* folded into the number.
|
|
10
|
+
*/
|
|
11
|
+
export declare function savingsSnapshotFromSpoolDir(dir: string): SavingsSnapshot;
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { type TokenCounter } from "./core/index.js";
|
|
2
|
+
import { type MeshReductionChange, type MeshReductionSummary, type MeshTokenReductionConfig } from "./reduce-request.js";
|
|
3
|
+
import type { LedgerGateVerdict } from "./core/index.js";
|
|
4
|
+
/**
|
|
5
|
+
* The pluggable adapter. The customer supplies a LOCAL model behind this
|
|
6
|
+
* interface; MESH bundles none. `compact` may be sync or async — a pre-pass can
|
|
7
|
+
* await either. It is called ONLY to compress stale context, never model output
|
|
8
|
+
* and never the live user turn (this module enforces that; the compactor itself
|
|
9
|
+
* only ever sees the text MESH decided was eligible).
|
|
10
|
+
*/
|
|
11
|
+
export type SemanticCompactor = {
|
|
12
|
+
compact(input: {
|
|
13
|
+
/** The stale body text to compress. */
|
|
14
|
+
text: string;
|
|
15
|
+
/** Target ceiling for the compressed result, in MESH's approximate tokens. */
|
|
16
|
+
maxTokens: number;
|
|
17
|
+
/** What kind of body this is, so a compactor can prompt itself accordingly. */
|
|
18
|
+
kind: "tool-result";
|
|
19
|
+
}): Promise<string> | string;
|
|
20
|
+
};
|
|
21
|
+
export type SemanticCompactionConfig = {
|
|
22
|
+
/** The customer's LOCAL compactor. Absent => the lever is off (no-op). */
|
|
23
|
+
compactor?: SemanticCompactor;
|
|
24
|
+
/**
|
|
25
|
+
* Replay-gate verdict for the `semantic-compaction` lever. REQUIRED and must
|
|
26
|
+
* be "approved" for the pre-pass to touch ANY content. Absent or "rejected"
|
|
27
|
+
* => the pre-pass is a byte-identical no-op and records why. This is the
|
|
28
|
+
* reducer-level half of the double gate; the ledger still independently
|
|
29
|
+
* refuses to bill the lever without an approved verdict.
|
|
30
|
+
*/
|
|
31
|
+
gateVerdict?: LedgerGateVerdict;
|
|
32
|
+
/**
|
|
33
|
+
* Target ceiling for a compacted body, approximate tokens. Default 500.
|
|
34
|
+
*
|
|
35
|
+
* Calibrated from real agent trajectories (docs/rnd/knowledge/
|
|
36
|
+
* 2026-10-27-dataset-findings.md): tool-result median is ~270–302 tok across
|
|
37
|
+
* SWE-rebench/SWE-smith/tau-bench, so the old 200 floor flagged 58–68% of ALL
|
|
38
|
+
* tool-results — compacting the typical observation, not the oversized tail.
|
|
39
|
+
* 500 fires on only the genuinely-oversized 3.5–34.5% tail. This lever runs
|
|
40
|
+
* only on multi-turn agent-shaped conversations (minimumTurns gate), so the
|
|
41
|
+
* agent-trajectory distribution is the right calibration population.
|
|
42
|
+
*/
|
|
43
|
+
maxTokens?: number;
|
|
44
|
+
/** Only compact once the conversation has at least this many user turns. Default 3. */
|
|
45
|
+
minimumTurns?: number;
|
|
46
|
+
/** The most recent user turns whose content is always left verbatim. Default 2. */
|
|
47
|
+
keepRecentTurns?: number;
|
|
48
|
+
/** MESH's token counter; defaults to the shared approximate counter. */
|
|
49
|
+
countTokens?: TokenCounter;
|
|
50
|
+
};
|
|
51
|
+
export type SemanticCompactionResult = {
|
|
52
|
+
request: unknown;
|
|
53
|
+
/** The lossy change record when compaction ran; absent when nothing changed. */
|
|
54
|
+
change?: MeshReductionChange;
|
|
55
|
+
/** MESH's approximate input-token count of the targeted fields, before and after. */
|
|
56
|
+
tokensBefore: number;
|
|
57
|
+
tokensAfter: number;
|
|
58
|
+
/** Set when the pre-pass was a no-op, saying why (off, no verdict, nothing stale). */
|
|
59
|
+
skippedReason?: string;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* The async PRE-PASS. Compacts stale tool-result bodies with the customer's
|
|
63
|
+
* local compactor and returns a new request, leaving the original untouched.
|
|
64
|
+
*
|
|
65
|
+
* Hard guard: without an APPROVED gate verdict in the config, this returns the
|
|
66
|
+
* request byte-identical and records why — it refuses to call the compactor at
|
|
67
|
+
* all. A compacted body that is not actually shorter is discarded (the original
|
|
68
|
+
* bytes are kept), so compaction can never make a request larger.
|
|
69
|
+
*/
|
|
70
|
+
export declare function compactStaleContext(request: unknown, config?: SemanticCompactionConfig): Promise<SemanticCompactionResult>;
|
|
71
|
+
/**
|
|
72
|
+
* Async entry point: the semantic-compaction PRE-PASS followed by the
|
|
73
|
+
* untouched synchronous reduceRequest, with the compaction change merged into
|
|
74
|
+
* the summary so before/after spans the whole pipeline.
|
|
75
|
+
*
|
|
76
|
+
* This is the only place the async lever meets the sync core. reduceRequest
|
|
77
|
+
* itself stays pure and synchronous (see the design note at the top of this
|
|
78
|
+
* file); callers that do not use semantic compaction keep calling reduceRequest
|
|
79
|
+
* directly and never take on a promise.
|
|
80
|
+
*/
|
|
81
|
+
export declare function reduceRequestAsync(request: unknown, config?: MeshTokenReductionConfig, semantic?: SemanticCompactionConfig): Promise<{
|
|
82
|
+
request: unknown;
|
|
83
|
+
summary: MeshReductionSummary;
|
|
84
|
+
}>;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { TheoMemory } from "./theo-agent.js";
|
|
2
|
+
/**
|
|
3
|
+
* The impure edge of the Theo agent: load and save his per-customer memory, and
|
|
4
|
+
* read realised savings from the ledger. The reasoning in `theo-agent.ts` stays
|
|
5
|
+
* pure; everything that touches the disk is here.
|
|
6
|
+
*
|
|
7
|
+
* Memory is a single JSON document per customer, kept alongside the ledger it is
|
|
8
|
+
* about (default `<ledgerDir>/theo/<customer>.json`). It never contains prompt or
|
|
9
|
+
* response content — only Theo's policy, hypotheses, experiment log and proposals.
|
|
10
|
+
*/
|
|
11
|
+
/** Default location for a customer's Theo memory, beside their ledger. */
|
|
12
|
+
export declare function defaultTheoMemoryPath(ledgerDir: string, customer: string): string;
|
|
13
|
+
/** Load Theo's memory, or null if it is absent or unreadable. */
|
|
14
|
+
export declare function loadTheoMemory(path: string): TheoMemory | null;
|
|
15
|
+
/** Persist Theo's memory, creating the directory if needed. */
|
|
16
|
+
export declare function saveTheoMemory(path: string, memory: TheoMemory): void;
|
|
17
|
+
/**
|
|
18
|
+
* Realised, verified figures for the report, read from the ledger spools the
|
|
19
|
+
* proxy wrote. A tampered or truncated spool is excluded by the chain check
|
|
20
|
+
* inside `savingsSnapshotFromSpoolDir`, so these numbers are never inflated.
|
|
21
|
+
*/
|
|
22
|
+
export declare function realisedFromSpoolDir(ledgerDir: string): {
|
|
23
|
+
savingsUsd: number;
|
|
24
|
+
spendUsd: number;
|
|
25
|
+
requests: number;
|
|
26
|
+
chainOk: boolean;
|
|
27
|
+
};
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import type { TokenPrice } from "../../core/dist/src/cost.js";
|
|
2
|
+
import { type WorkloadProfile } from "../../core/dist/src/savings-model.js";
|
|
3
|
+
import type { BlindFeatures } from "./blind-features.js";
|
|
4
|
+
import { type TheoLeverClass, type TheoPolicy } from "./theo-policy.js";
|
|
5
|
+
/**
|
|
6
|
+
* Theo — the persistent, per-customer token-engineer agent.
|
|
7
|
+
*
|
|
8
|
+
* This is the running loop the R&D sketch (docs/rnd/theo-token-engineer-agent.md)
|
|
9
|
+
* describes, built on primitives that already exist: blind features (Observe),
|
|
10
|
+
* the savings model (Hypothesise), the replay gate (Prove), and the per-user
|
|
11
|
+
* policy the proxy enforces (Decide). The prototype in evals/theo-loop.ts proved
|
|
12
|
+
* the shape offline; this turns it into a stateful agent with memory and a
|
|
13
|
+
* configurable autonomy level.
|
|
14
|
+
*
|
|
15
|
+
* Two invariants hold in code, not by trust:
|
|
16
|
+
*
|
|
17
|
+
* 1. **This module is pure.** It never reads the clock, the disk or the
|
|
18
|
+
* network, and it never calls a provider. Persistence lives in
|
|
19
|
+
* `theo-agent-store.ts`; proving a lever lives at the call site (the replay
|
|
20
|
+
* gate). The cycle CONSUMES proofs it is handed and never invents one.
|
|
21
|
+
*
|
|
22
|
+
* 2. **Theo never enables a saving he has not proven.** Only the three truly
|
|
23
|
+
* lossless levers run unconditionally (they cannot change the answer). Every
|
|
24
|
+
* other lever is withheld until a replay-gate proof shows, on the customer's
|
|
25
|
+
* own traffic, that quality held within the route's ceiling — and even then
|
|
26
|
+
* Theo only auto-publishes it at the `optimise` autonomy level. The
|
|
27
|
+
* enforcement point is `publishLever` in theo-policy.ts, which refuses a
|
|
28
|
+
* lossy lever with no gate proof.
|
|
29
|
+
*/
|
|
30
|
+
/**
|
|
31
|
+
* How much Theo may do on his own, lowest to highest. This is the "Improve" dial
|
|
32
|
+
* from the product guide:
|
|
33
|
+
*
|
|
34
|
+
* observe — identify opportunities, change nothing.
|
|
35
|
+
* recommend — auto-apply lossless; surface proven lossy levers as proposals.
|
|
36
|
+
* test — recommend, and additionally initiate gate experiments (emit the
|
|
37
|
+
* requests for the next lossy candidates to prove).
|
|
38
|
+
* optimise — auto-publish a lossy lever once it has a passing proof within the
|
|
39
|
+
* route's quality ceiling (pre-approved, reversible).
|
|
40
|
+
*/
|
|
41
|
+
export type TheoAutonomyLevel = "observe" | "recommend" | "test" | "optimise";
|
|
42
|
+
/** A lossy lever Theo believes is worth proving next. The share is a prior for
|
|
43
|
+
* ORDERING only — it is never shown to the customer as a saving. */
|
|
44
|
+
export type TheoHypothesis = {
|
|
45
|
+
lever: string;
|
|
46
|
+
class: TheoLeverClass;
|
|
47
|
+
priorSharePercent: number;
|
|
48
|
+
rationale: string;
|
|
49
|
+
};
|
|
50
|
+
/** An opportunity the savings model sees that the proxy cannot yet enforce as a
|
|
51
|
+
* discrete lever (new reducer code or an app-side change — the parked frontier). */
|
|
52
|
+
export type TheoFrontierOpportunity = {
|
|
53
|
+
concept: string;
|
|
54
|
+
priorSharePercent: number;
|
|
55
|
+
note: string;
|
|
56
|
+
};
|
|
57
|
+
/** One replay-gate outcome, kept so a lever is never re-tested from scratch. */
|
|
58
|
+
export type TheoExperiment = {
|
|
59
|
+
lever: string;
|
|
60
|
+
ranAt: string;
|
|
61
|
+
passed: boolean;
|
|
62
|
+
savingPercent?: number;
|
|
63
|
+
qualityDeltaPercent?: number;
|
|
64
|
+
grader?: string;
|
|
65
|
+
note?: string;
|
|
66
|
+
};
|
|
67
|
+
/** A proven lossy lever awaiting the customer's approval (recommend/test modes). */
|
|
68
|
+
export type TheoProposal = {
|
|
69
|
+
lever: string;
|
|
70
|
+
savingPercent: number;
|
|
71
|
+
qualityDeltaPercent: number;
|
|
72
|
+
grader: string;
|
|
73
|
+
provenAt: string;
|
|
74
|
+
message: string;
|
|
75
|
+
};
|
|
76
|
+
export type TheoCycleLogEntry = {
|
|
77
|
+
at: string;
|
|
78
|
+
autonomy: TheoAutonomyLevel;
|
|
79
|
+
observed: string;
|
|
80
|
+
actions: number;
|
|
81
|
+
};
|
|
82
|
+
/** Everything Theo remembers about one customer's environment. Persisted as JSON. */
|
|
83
|
+
export type TheoMemory = {
|
|
84
|
+
schemaVersion: 1;
|
|
85
|
+
customer: string;
|
|
86
|
+
autonomy: TheoAutonomyLevel;
|
|
87
|
+
/** The live policy the proxy enforces. Theo's only lever on the world. */
|
|
88
|
+
policy: TheoPolicy;
|
|
89
|
+
/** The most recently observed workload shape. */
|
|
90
|
+
profile?: WorkloadProfile;
|
|
91
|
+
/** Ranked lossy candidates not yet active. */
|
|
92
|
+
hypotheses: TheoHypothesis[];
|
|
93
|
+
/** Opportunities beyond the enforceable repertoire. */
|
|
94
|
+
frontier: TheoFrontierOpportunity[];
|
|
95
|
+
/** Append-only log of gate outcomes. */
|
|
96
|
+
experiments: TheoExperiment[];
|
|
97
|
+
/** Open proposals the customer has not yet accepted or rejected. */
|
|
98
|
+
proposals: TheoProposal[];
|
|
99
|
+
/** Per applied lossy lever: the retry and failure rates measured when it was
|
|
100
|
+
* proven, so a later rise in either can be detected as quality drift. */
|
|
101
|
+
driftBaselines: Record<string, {
|
|
102
|
+
retryRate: number;
|
|
103
|
+
failureRate: number;
|
|
104
|
+
at: string;
|
|
105
|
+
}>;
|
|
106
|
+
/** Levers Theo auto-rolled-back after drift; not re-proposed this session. */
|
|
107
|
+
rolledBack: string[];
|
|
108
|
+
/** Proposals the customer turned down. Theo neither re-proves nor re-proposes
|
|
109
|
+
* these until the customer asks him to reconsider. Optional so memory files
|
|
110
|
+
* written before this field existed still load. */
|
|
111
|
+
declined?: string[];
|
|
112
|
+
log: TheoCycleLogEntry[];
|
|
113
|
+
updatedAt: string;
|
|
114
|
+
};
|
|
115
|
+
/** What Theo sees this cycle. The profile comes from Observe (blind features);
|
|
116
|
+
* the realised figures come from the ledger and feed the report only. */
|
|
117
|
+
export type TheoObservation = {
|
|
118
|
+
profile: WorkloadProfile;
|
|
119
|
+
price: TokenPrice;
|
|
120
|
+
realisedSavingsUsd?: number;
|
|
121
|
+
realisedSpendUsd?: number;
|
|
122
|
+
/** Fraction of recent requests that were retries (duplicate of a prior one) —
|
|
123
|
+
* a behavioural-drift signal. When it rises after a lever is applied, Theo
|
|
124
|
+
* rolls that lever back. Omit it and that signal simply does not fire. */
|
|
125
|
+
retryRate?: number;
|
|
126
|
+
/** Fraction of recent requests that FAILED (provider error) — a second drift
|
|
127
|
+
* signal, caught the same way. Omit it to ignore it. */
|
|
128
|
+
failureRate?: number;
|
|
129
|
+
};
|
|
130
|
+
/** A measured gate result the caller obtained (from `replayGate`) and hands to
|
|
131
|
+
* the cycle. Theo trusts the measurement; he never produces one himself. */
|
|
132
|
+
export type TheoProof = {
|
|
133
|
+
lever: string;
|
|
134
|
+
/** Measured saving on the customer's own traffic, from provider usage. */
|
|
135
|
+
savingPercent: number;
|
|
136
|
+
/** Measured quality delta; its magnitude is compared to the route ceiling. */
|
|
137
|
+
qualityDeltaPercent: number;
|
|
138
|
+
grader: string;
|
|
139
|
+
provenAt: string;
|
|
140
|
+
};
|
|
141
|
+
export type TheoAction = {
|
|
142
|
+
kind: "lossless-active";
|
|
143
|
+
levers: string[];
|
|
144
|
+
} | {
|
|
145
|
+
kind: "policy-published";
|
|
146
|
+
version: number;
|
|
147
|
+
lever: string;
|
|
148
|
+
} | {
|
|
149
|
+
kind: "proposal-opened";
|
|
150
|
+
lever: string;
|
|
151
|
+
savingPercent: number;
|
|
152
|
+
qualityDeltaPercent: number;
|
|
153
|
+
} | {
|
|
154
|
+
kind: "gate-requested";
|
|
155
|
+
levers: string[];
|
|
156
|
+
} | {
|
|
157
|
+
kind: "proof-rejected";
|
|
158
|
+
lever: string;
|
|
159
|
+
reason: string;
|
|
160
|
+
} | {
|
|
161
|
+
kind: "rolled-back";
|
|
162
|
+
lever: string;
|
|
163
|
+
reason: string;
|
|
164
|
+
};
|
|
165
|
+
export type TheoCycleResult = {
|
|
166
|
+
memory: TheoMemory;
|
|
167
|
+
actions: TheoAction[];
|
|
168
|
+
report: string;
|
|
169
|
+
};
|
|
170
|
+
/** How far the retry rate may rise above an applied lever's baseline before Theo
|
|
171
|
+
* treats it as quality drift and rolls the lever back (absolute fraction).
|
|
172
|
+
* Exported so surfaces (e.g. the dashboard's health panel) flag "watch" against
|
|
173
|
+
* the SAME line Theo actually rolls back on — one source of truth, not a copy. */
|
|
174
|
+
export declare const DRIFT_RISE_THRESHOLD = 0.05;
|
|
175
|
+
/** How many of the MOST RECENT ledger events the drift signals are measured
|
|
176
|
+
* over. A whole-history rate dilutes a fresh regression until it can no longer
|
|
177
|
+
* cross the threshold; a bounded window caps that dilution, so a rise shows
|
|
178
|
+
* within this many requests however long the history is. Untuned, like the
|
|
179
|
+
* threshold above. */
|
|
180
|
+
export declare const DRIFT_WINDOW_EVENTS = 200;
|
|
181
|
+
export type DriftRates = {
|
|
182
|
+
/** Events the rates were measured over (at most the window). */
|
|
183
|
+
sampleSize: number;
|
|
184
|
+
retryRate: number;
|
|
185
|
+
failureRate: number;
|
|
186
|
+
};
|
|
187
|
+
/**
|
|
188
|
+
* The two behavioural-drift signals over the most recent `windowEvents` ledger
|
|
189
|
+
* events: the share that were byte-identical retries and the share that failed
|
|
190
|
+
* at the provider. Pure. The ONE definition shared by Theo's rollback check and
|
|
191
|
+
* the dashboard's health panel, so the two can never disagree.
|
|
192
|
+
*/
|
|
193
|
+
export declare function recentDriftRates(events: readonly {
|
|
194
|
+
timestamp: string;
|
|
195
|
+
status: string;
|
|
196
|
+
duplicateOfRequestId?: string;
|
|
197
|
+
}[], windowEvents?: number): DriftRates;
|
|
198
|
+
/**
|
|
199
|
+
* OBSERVE — fold blind features into a workload profile. Learned from irreversible
|
|
200
|
+
* features only; no prompt, code or response is ever read. Output tokens are not
|
|
201
|
+
* present in a request, so they are supplied from the ledger (mean per request).
|
|
202
|
+
*/
|
|
203
|
+
export declare function observeWorkload(features: readonly BlindFeatures[], options?: {
|
|
204
|
+
name?: string;
|
|
205
|
+
outputTokensMean?: number;
|
|
206
|
+
}): WorkloadProfile;
|
|
207
|
+
/** A fresh agent for a new customer: strict, lossless-only, nothing proven yet. */
|
|
208
|
+
export declare function createTheoMemory(customer: string, autonomy: TheoAutonomyLevel, at: string): TheoMemory;
|
|
209
|
+
/**
|
|
210
|
+
* Run one Theo cycle: Observe -> Hypothesise -> (consume) Prove -> Decide -> Report.
|
|
211
|
+
* Pure: returns a new memory, the actions taken, and a human-readable report.
|
|
212
|
+
* `now` is the caller's clock; `input.proofs` are gate results the caller obtained.
|
|
213
|
+
*/
|
|
214
|
+
export declare function runTheoCycle(memory: TheoMemory, input: {
|
|
215
|
+
observation: TheoObservation;
|
|
216
|
+
proofs?: readonly TheoProof[];
|
|
217
|
+
}, now: string): TheoCycleResult;
|
|
218
|
+
export type TheoProposalDecision = {
|
|
219
|
+
ok: true;
|
|
220
|
+
memory: TheoMemory;
|
|
221
|
+
message: string;
|
|
222
|
+
} | {
|
|
223
|
+
ok: false;
|
|
224
|
+
reason: string;
|
|
225
|
+
};
|
|
226
|
+
/**
|
|
227
|
+
* The customer accepts an open proposal: the lever is published into the policy
|
|
228
|
+
* carrying the proof it was proposed with, and its drift baselines are recorded
|
|
229
|
+
* so a later rise rolls it back. Pure — returns a new memory.
|
|
230
|
+
*
|
|
231
|
+
* Acceptance re-checks the proof against the CURRENT ceiling (it may have been
|
|
232
|
+
* tightened since the proposal opened) and refuses a lever that was rolled back
|
|
233
|
+
* after drift, so accepting can never enable something the gate would not.
|
|
234
|
+
*/
|
|
235
|
+
export declare function acceptProposal(memory: TheoMemory, lever: string, rates: {
|
|
236
|
+
retryRate?: number;
|
|
237
|
+
failureRate?: number;
|
|
238
|
+
}, now: string): TheoProposalDecision;
|
|
239
|
+
/**
|
|
240
|
+
* The customer turns an open proposal down: it is closed and the lever is
|
|
241
|
+
* remembered as declined, so Theo neither re-proves nor re-proposes it. Pure.
|
|
242
|
+
*/
|
|
243
|
+
export declare function rejectProposal(memory: TheoMemory, lever: string, now: string): TheoProposalDecision;
|
|
244
|
+
/** Undo a decline: Theo may prove and propose the lever again. Pure. */
|
|
245
|
+
export declare function reconsiderLever(memory: TheoMemory, lever: string, now: string): TheoProposalDecision;
|
|
246
|
+
/** A short, honest status line set for the customer — no unproven numbers. */
|
|
247
|
+
export declare function renderTheoReport(memory: TheoMemory, observation: TheoObservation, actions: readonly TheoAction[]): string;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { TokenPrice } from "../../core/dist/src/cost.js";
|
|
2
|
+
import { type ReplayGrader } from "./replay-gate.js";
|
|
3
|
+
import type { TheoProof } from "./theo-agent.js";
|
|
4
|
+
/**
|
|
5
|
+
* The replay gate, wired to Theo.
|
|
6
|
+
*
|
|
7
|
+
* `runTheoCycle` consumes proofs but never produces one. This is where a proof
|
|
8
|
+
* comes from: replay a sample of the customer's OWN requests both ways — baseline
|
|
9
|
+
* and candidate-reduced — against the real provider, and measure what actually
|
|
10
|
+
* happened. Token savings and cost come from the provider's reported usage on
|
|
11
|
+
* both runs; quality from a grader comparing the two responses. A candidate is a
|
|
12
|
+
* proof only if the bill fell AND the measured quality drop stayed within budget.
|
|
13
|
+
*
|
|
14
|
+
* This is the one impure corner of the agent: it needs a `call` that reaches the
|
|
15
|
+
* provider (with the customer's key) and a sample of real requests. Both stay on
|
|
16
|
+
* the customer's machine; nothing is uploaded. Replay costs money — each request
|
|
17
|
+
* is sent again — so the call count is reported, and recorded baseline responses
|
|
18
|
+
* can be supplied to avoid paying for the baseline run twice.
|
|
19
|
+
*/
|
|
20
|
+
export type ProveCandidatesOptions = {
|
|
21
|
+
/** Enforceable lossy lever ids to prove — typically Theo's ranked hypotheses. */
|
|
22
|
+
levers: readonly string[];
|
|
23
|
+
/** A representative sample of the customer's real requests. */
|
|
24
|
+
requests: readonly unknown[];
|
|
25
|
+
/** Sends a request to the provider and returns the response. */
|
|
26
|
+
call: (request: unknown) => Promise<unknown>;
|
|
27
|
+
/** Clock for each proof's provenAt. */
|
|
28
|
+
now: string;
|
|
29
|
+
/**
|
|
30
|
+
* Mean quality the gate may give up, as a FRACTION (0.02 = a 2% drop). Pass the
|
|
31
|
+
* route's quality ceiling / 100 so the gate and Theo's ceiling agree. Default 0.02.
|
|
32
|
+
*/
|
|
33
|
+
maximumQualityDrop?: number;
|
|
34
|
+
/** Recorded baseline responses (one per request) to avoid paying for the baseline run. */
|
|
35
|
+
baselineResponses?: readonly unknown[];
|
|
36
|
+
/** Rate, so the gate judges the BILL (cache writes/reads included), not only tokens. */
|
|
37
|
+
price?: TokenPrice;
|
|
38
|
+
/** Quality grader. Default `agentGrader()`: a changed tool call fails hard, else coverage. */
|
|
39
|
+
grader?: ReplayGrader;
|
|
40
|
+
/** A short name for the grader, recorded on the proof. Default "agent". */
|
|
41
|
+
graderName?: string;
|
|
42
|
+
/** Group correlated requests (e.g. by task) so the quality bound is not overstated. */
|
|
43
|
+
groupOf?: (request: unknown, index: number) => string;
|
|
44
|
+
/**
|
|
45
|
+
* Exploration budget: the maximum number of paid provider calls this run may
|
|
46
|
+
* make. The gate sends roughly requests × (candidates + 1 baseline) calls
|
|
47
|
+
* (the baseline is free when recorded responses are supplied), so the sample
|
|
48
|
+
* is trimmed to fit. Omit for no cap. Protects against a runaway provider bill
|
|
49
|
+
* when Theo proves on its own.
|
|
50
|
+
*/
|
|
51
|
+
maxProviderCalls?: number;
|
|
52
|
+
};
|
|
53
|
+
export type ProveCandidatesResult = {
|
|
54
|
+
/** One proof per candidate the gate approved — ready to hand to `runTheoCycle`. */
|
|
55
|
+
proofs: TheoProof[];
|
|
56
|
+
/** Candidates the gate rejected, with why (quality or no saving). */
|
|
57
|
+
rejections: Array<{
|
|
58
|
+
lever: string;
|
|
59
|
+
reason: string;
|
|
60
|
+
}>;
|
|
61
|
+
/** Provider calls this replay made, so its cost is visible. */
|
|
62
|
+
calls: number;
|
|
63
|
+
/** Set when the exploration budget trimmed the sample. */
|
|
64
|
+
note?: string;
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* Prove a set of candidate lossy levers on the customer's own traffic.
|
|
68
|
+
*
|
|
69
|
+
* Runs ONE replay gate across all candidates so the baseline is paid for once and
|
|
70
|
+
* the quality bound holds simultaneously across the search (picking the best of
|
|
71
|
+
* several otherwise flatters the winner). Each approved candidate becomes a
|
|
72
|
+
* {@link TheoProof}; the measured quality bound (one-sided, ≤ 0) is carried as a
|
|
73
|
+
* percentage so Theo's ceiling check compares like with like.
|
|
74
|
+
*/
|
|
75
|
+
export declare function proveCandidates(options: ProveCandidatesOptions): Promise<ProveCandidatesResult>;
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import type { MeshTokenReductionConfig } from "./reduce-request.js";
|
|
2
|
+
/**
|
|
3
|
+
* Theo's per-user policy — the control mechanism between Theo and MESH.
|
|
4
|
+
*
|
|
5
|
+
* Theo never sits in the request path. His ONLY action is to publish a new version
|
|
6
|
+
* of this document; the proxy reads it and enforces it synchronously. A policy is
|
|
7
|
+
* self-documenting: every lossy lever carries the gate proof (measured saving + the
|
|
8
|
+
* measured quality delta Y + the grader) that justifies it.
|
|
9
|
+
*
|
|
10
|
+
* The load-bearing safety property is in `policyToReductionConfig`: a lossy lever is
|
|
11
|
+
* applied ONLY if it carries a passing gate proof within the route's quality ceiling.
|
|
12
|
+
* A policy therefore CANNOT enable a saving Theo has not proven — the "never degrade
|
|
13
|
+
* the product" rule is enforced here, in code, not by trust.
|
|
14
|
+
*
|
|
15
|
+
* Pure and dependency-free (no node:*), so it loads in any JavaScript runtime.
|
|
16
|
+
*/
|
|
17
|
+
export type TheoLeverClass = "lossless" | "lossy";
|
|
18
|
+
/** The gate proof for a lossy lever: measured on the user's OWN traffic. */
|
|
19
|
+
export type TheoGateProof = {
|
|
20
|
+
/** Measured saving on the user's traffic, from provider-reported usage. */
|
|
21
|
+
savingPercent: number;
|
|
22
|
+
/** Measured quality delta Y (<= 0; e.g. -0.3 = a 0.3% drop). */
|
|
23
|
+
qualityDeltaPercent: number;
|
|
24
|
+
/** What graded quality: "task-completion" | "tests" | "behavioural" | a custom name. */
|
|
25
|
+
grader: string;
|
|
26
|
+
provenAt: string;
|
|
27
|
+
};
|
|
28
|
+
export type TheoLever = {
|
|
29
|
+
/** The reducer lever id (e.g. "tool-schema-compaction", "cache-ttl-5m", "output-effort-low"). */
|
|
30
|
+
lever: string;
|
|
31
|
+
class: TheoLeverClass;
|
|
32
|
+
/** Required for a lossy lever before it can be enforced; absent for lossless. */
|
|
33
|
+
gate?: TheoGateProof;
|
|
34
|
+
/** Lever-specific params (e.g. { maximumTokens: 600 }, { maximumDescriptionTokens: 40 }). */
|
|
35
|
+
params?: Record<string, unknown>;
|
|
36
|
+
};
|
|
37
|
+
export type TheoPolicy = {
|
|
38
|
+
schemaVersion: 1;
|
|
39
|
+
user: string;
|
|
40
|
+
/** Optional per-route policy; a default route when omitted. */
|
|
41
|
+
route?: string;
|
|
42
|
+
/** Monotonic — publishing a new version is Theo's only action. Rollback = publish a prior one. */
|
|
43
|
+
version: number;
|
|
44
|
+
/** Max quality drop (percent) this route tolerates. Default STRICT; loosened only by the user. */
|
|
45
|
+
qualityCeilingPercent: number;
|
|
46
|
+
levers: TheoLever[];
|
|
47
|
+
updatedAt: string;
|
|
48
|
+
};
|
|
49
|
+
/** Why a lever in the policy is NOT currently enforced — for Theo's transparency to the user. */
|
|
50
|
+
export type WithheldLever = {
|
|
51
|
+
lever: string;
|
|
52
|
+
reason: string;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Hard upper bound on any route's quality ceiling. Enforcement clamps to this, so
|
|
56
|
+
* however loose a policy's `qualityCeilingPercent` is set (by config, a tier dial,
|
|
57
|
+
* or a tampered file), a lossy lever whose measured quality drop exceeds this is
|
|
58
|
+
* never enforced. A floor under "never degrade" that no misconfiguration crosses.
|
|
59
|
+
*/
|
|
60
|
+
export declare const MAX_QUALITY_CEILING_PERCENT = 5;
|
|
61
|
+
/** The effective ceiling: the route's, clamped to the hard maximum (and ≥ 0). */
|
|
62
|
+
export declare function effectiveQualityCeiling(ceilingPercent: number): number;
|
|
63
|
+
/**
|
|
64
|
+
* Turn a policy into the reduction config the proxy enforces.
|
|
65
|
+
*
|
|
66
|
+
* Lossless levers apply unconditionally (they cannot change the answer). A lossy
|
|
67
|
+
* lever applies ONLY if it carries a gate proof whose measured quality delta is
|
|
68
|
+
* within the route's ceiling; otherwise it is silently withheld. This is the
|
|
69
|
+
* enforcement point of "never enable a saving we have not proven within budget."
|
|
70
|
+
*/
|
|
71
|
+
export declare function policyToReductionConfig(policy: TheoPolicy): MeshTokenReductionConfig;
|
|
72
|
+
/** A ledger gate verdict, structurally the ledger's `LedgerGateVerdict` (kept
|
|
73
|
+
* local so this module stays dependency-free). */
|
|
74
|
+
export type PolicyGateVerdict = {
|
|
75
|
+
ref: string;
|
|
76
|
+
verdict: "approved";
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* The replay-gate verdicts behind the lossy levers this policy ENFORCES, keyed by
|
|
80
|
+
* lever id, for the savings ledger. Exactly the levers `policyToReductionConfig`
|
|
81
|
+
* applies: a lossy lever with a proof inside the (hard-capped) ceiling. A lever
|
|
82
|
+
* that is withheld gets no verdict, so its savings can never bill. The ref names
|
|
83
|
+
* the proof — grader and the time it was measured — so an invoice line can be
|
|
84
|
+
* traced back to the gate run that justified it.
|
|
85
|
+
*/
|
|
86
|
+
export declare function policyGateVerdicts(policy: TheoPolicy): Record<string, PolicyGateVerdict>;
|
|
87
|
+
/**
|
|
88
|
+
* Build the reducer config for a SINGLE lever, bypassing the policy's proof gate.
|
|
89
|
+
*
|
|
90
|
+
* This exists to CONSTRUCT a candidate for the replay gate to measure — it is not
|
|
91
|
+
* an enforcement path. Production enforcement still goes only through
|
|
92
|
+
* `policyToReductionConfig`, which withholds any lossy lever that lacks a passing
|
|
93
|
+
* proof within the route's ceiling. Reuses `applyLever` so the gate measures the
|
|
94
|
+
* exact transform the proxy would later enforce.
|
|
95
|
+
*/
|
|
96
|
+
export declare function leverToReductionConfig(lever: TheoLever): MeshTokenReductionConfig;
|
|
97
|
+
/**
|
|
98
|
+
* A bundle of policies for one user: a DEFAULT policy plus zero or more
|
|
99
|
+
* route-specific overrides. A "route" is matched against the request's model id,
|
|
100
|
+
* so a user can run a strict ceiling on their agent's planning model and a looser
|
|
101
|
+
* one on a cheap summariser — each route carries its OWN ceiling, levers and
|
|
102
|
+
* proofs, and the hard cap (`MAX_QUALITY_CEILING_PERCENT`) still binds every one.
|
|
103
|
+
*
|
|
104
|
+
* Back-compatible: a stored single `TheoPolicy` is read as a bundle whose default
|
|
105
|
+
* is that policy and whose route list is empty (`asBundle`), so existing policy
|
|
106
|
+
* files and the single-policy enforcement path keep working unchanged.
|
|
107
|
+
*/
|
|
108
|
+
export type TheoPolicyBundle = {
|
|
109
|
+
schemaVersion: 1;
|
|
110
|
+
user: string;
|
|
111
|
+
/** Applied to any request whose model id matches no route. */
|
|
112
|
+
default: TheoPolicy;
|
|
113
|
+
/** Route-specific policies; each MUST carry a `route` matcher. First match wins. */
|
|
114
|
+
routes?: TheoPolicy[];
|
|
115
|
+
updatedAt: string;
|
|
116
|
+
};
|
|
117
|
+
/**
|
|
118
|
+
* Whether a route matcher applies to a model id. Three forms, deliberately small:
|
|
119
|
+
* - `"*"` or `""` — matches every model (a catch-all route).
|
|
120
|
+
* - a trailing `*` — prefix match (`"claude-*"` matches `"claude-sonnet-5-5"`).
|
|
121
|
+
* - anything else — an exact model-id match.
|
|
122
|
+
* No regex: a policy file is config a user hand-edits, so matching stays obvious.
|
|
123
|
+
*/
|
|
124
|
+
export declare function routeMatches(pattern: string, modelId: string): boolean;
|
|
125
|
+
/** Is this stored value a bundle (has a `default` policy) rather than a single policy? */
|
|
126
|
+
export declare function isPolicyBundle(value: unknown): value is TheoPolicyBundle;
|
|
127
|
+
/**
|
|
128
|
+
* Normalise a stored document to a bundle. A single `TheoPolicy` becomes a bundle
|
|
129
|
+
* with that policy as its default and no routes; a bundle passes through. One read
|
|
130
|
+
* path for both the legacy single-policy file and the new per-route bundle.
|
|
131
|
+
*/
|
|
132
|
+
export declare function asBundle(stored: TheoPolicy | TheoPolicyBundle): TheoPolicyBundle;
|
|
133
|
+
/**
|
|
134
|
+
* The policy governing a given model id: the FIRST route whose matcher applies,
|
|
135
|
+
* else the bundle's default. Order in `routes` is the precedence, so put specific
|
|
136
|
+
* matchers before catch-alls.
|
|
137
|
+
*/
|
|
138
|
+
export declare function policyForRoute(bundle: TheoPolicyBundle, modelId: string): TheoPolicy;
|
|
139
|
+
/**
|
|
140
|
+
* The reduction config the proxy enforces for a given model id under a bundle.
|
|
141
|
+
* Routes through `policyForRoute` then the single-policy enforcement gate, so the
|
|
142
|
+
* per-route ceiling and every safety guarantee (proof required, hard cap) apply to
|
|
143
|
+
* the SELECTED route exactly as they do to a single policy.
|
|
144
|
+
*/
|
|
145
|
+
export declare function bundleToReductionConfig(bundle: TheoPolicyBundle, modelId: string): MeshTokenReductionConfig;
|
|
146
|
+
/** The gate verdicts for the route governing a model id (see `policyGateVerdicts`). */
|
|
147
|
+
export declare function bundleGateVerdicts(bundle: TheoPolicyBundle, modelId: string): Record<string, PolicyGateVerdict>;
|
|
148
|
+
/**
|
|
149
|
+
* Insert or replace a route policy in a bundle, returning a NEW bundle (never
|
|
150
|
+
* mutating the input). The policy must carry a `route`; a policy for an existing
|
|
151
|
+
* route replaces it, matched by the exact matcher string.
|
|
152
|
+
*/
|
|
153
|
+
export declare function upsertRoutePolicy(bundle: TheoPolicyBundle, policy: TheoPolicy, at: string): TheoPolicyBundle;
|
|
154
|
+
/**
|
|
155
|
+
* Land a policy Theo has evolved into a bundle WITHOUT clobbering the rest of it,
|
|
156
|
+
* returning a NEW bundle. With a `route`, the policy becomes that route's override
|
|
157
|
+
* (every other route and the default preserved); without one, it replaces the
|
|
158
|
+
* bundle's default (every route preserved). This is what `mesh theo --apply`
|
|
159
|
+
* writes: Theo optimises one model at a time, so he can publish to that model's
|
|
160
|
+
* route and leave the others exactly as they were.
|
|
161
|
+
*/
|
|
162
|
+
export declare function mergePolicyIntoBundle(existing: TheoPolicyBundle | undefined, policy: TheoPolicy, route: string | undefined, at: string): TheoPolicyBundle;
|
|
163
|
+
/** The lossy levers present in the policy but NOT enforced, with the reason — for Theo's UX. */
|
|
164
|
+
export declare function withheldLevers(policy: TheoPolicy): WithheldLever[];
|
|
165
|
+
/**
|
|
166
|
+
* Publish a lever into a policy, returning a NEW version (never mutating the input).
|
|
167
|
+
* A lossless lever is added as-is; a lossy lever must carry a gate proof. The version
|
|
168
|
+
* bumps and updatedAt is set by the caller's clock (passed in, so this stays pure).
|
|
169
|
+
*/
|
|
170
|
+
export declare function publishLever(policy: TheoPolicy, lever: TheoLever, at: string): TheoPolicy;
|
|
171
|
+
/** A fresh, STRICT default policy for a new user — lossless-only until Theo proves more. */
|
|
172
|
+
export declare function defaultPolicy(user: string, at: string): TheoPolicy;
|
|
173
|
+
/** The reducer-lever ids Theo can enforce. */
|
|
174
|
+
export declare function knownLeverIds(): string[];
|
|
175
|
+
export type EnforceableLever = {
|
|
176
|
+
id: string;
|
|
177
|
+
label: string;
|
|
178
|
+
members: readonly string[];
|
|
179
|
+
};
|
|
180
|
+
/**
|
|
181
|
+
* The levers the proxy can enforce, grouped so mutually-exclusive variants count
|
|
182
|
+
* once: a policy cannot run output-effort-low AND output-effort-high, nor both
|
|
183
|
+
* cache TTLs, so counting each id separately would understate coverage. Every id
|
|
184
|
+
* in knownLeverIds() belongs to exactly one group here, and vice versa — the test
|
|
185
|
+
* `ENFORCEABLE_LEVERS stays in sync with knownLeverIds` holds the two together.
|
|
186
|
+
*/
|
|
187
|
+
export declare const ENFORCEABLE_LEVERS: readonly EnforceableLever[];
|
|
188
|
+
/**
|
|
189
|
+
* Where an enforceable lever stands for this user right now:
|
|
190
|
+
* - `active`: Theo is enforcing it (lossless, or lossy with a proof within the ceiling).
|
|
191
|
+
* - `held`: it is in the policy but a lossy proof does not yet clear the ceiling.
|
|
192
|
+
* - `available`: Theo can enforce it but has not adopted it yet — a candidate for the live gate.
|
|
193
|
+
*/
|
|
194
|
+
export type LeverStandState = "active" | "held" | "available";
|
|
195
|
+
export type LeverStand = {
|
|
196
|
+
lever: string;
|
|
197
|
+
label: string;
|
|
198
|
+
state: LeverStandState;
|
|
199
|
+
class?: TheoLeverClass;
|
|
200
|
+
reason?: string;
|
|
201
|
+
};
|
|
202
|
+
export type PolicyCoverage = {
|
|
203
|
+
/** Enforceable levers in the repertoire (grouped). */
|
|
204
|
+
enforceableCount: number;
|
|
205
|
+
activeCount: number;
|
|
206
|
+
heldCount: number;
|
|
207
|
+
availableCount: number;
|
|
208
|
+
/** activeCount / enforceableCount * 100 — the share of the repertoire live right now. */
|
|
209
|
+
activeCoveragePercent: number;
|
|
210
|
+
stands: LeverStand[];
|
|
211
|
+
};
|
|
212
|
+
/**
|
|
213
|
+
* How much of the proxy's enforceable repertoire Theo is actually using for this
|
|
214
|
+
* user. The honest denominator for the live dashboard: it shows not just the
|
|
215
|
+
* levers Theo applies and holds, but every lever he COULD apply and has not
|
|
216
|
+
* adopted yet — so a user can see how complete the coverage is, not only what is
|
|
217
|
+
* on. Pure: the same policy always yields the same coverage.
|
|
218
|
+
*/
|
|
219
|
+
export declare function policyCoverage(policy: TheoPolicy): PolicyCoverage;
|