@harrym1707/sdk 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -2
- package/index.cjs +1547 -164
- package/index.js +1520 -166
- package/package.json +1 -1
- package/types/cache-volatility-guard.d.ts +18 -4
- package/types/core/account-store.d.ts +7 -0
- package/types/core/accounts.d.ts +134 -0
- package/types/core/billing-worker.d.ts +113 -0
- package/types/core/budget.d.ts +216 -0
- package/types/core/evaluator.d.ts +10 -0
- package/types/core/github-oauth.d.ts +41 -0
- package/types/core/index.d.ts +10 -0
- package/types/core/ledger-store.d.ts +62 -0
- package/types/core/routing.d.ts +282 -0
- package/types/core/stripe-client.d.ts +117 -0
- package/types/core/token-reduction.d.ts +11 -0
- package/types/core/types.d.ts +11 -0
- package/types/core/workflow-sessions.d.ts +50 -0
- package/types/cta.d.ts +60 -0
- package/types/index.d.ts +6 -1
- package/types/ledger-mirror.d.ts +38 -0
- package/types/proxy.d.ts +43 -2
- package/types/reduce-request.d.ts +27 -2
- package/types/savings-ledger.d.ts +12 -2
- package/types/savings-snapshot.d.ts +11 -0
- package/types/theo-agent-store.d.ts +27 -0
- package/types/theo-agent.d.ts +247 -0
- package/types/theo-gate.d.ts +75 -0
- package/types/theo-policy.d.ts +100 -0
- package/types/types.d.ts +24 -0
- package/types/wrap.d.ts +8 -0
- package/types/cache-ttl-advisor.d.ts +0 -31
- package/types/core/replay-runner.d.ts +0 -11
- package/types/core/routing-simulation.d.ts +0 -24
package/index.cjs
CHANGED
|
@@ -21,12 +21,18 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
23
|
CachePrefixVolatilityGuard: () => CachePrefixVolatilityGuard,
|
|
24
|
+
DRIFT_RISE_THRESHOLD: () => DRIFT_RISE_THRESHOLD,
|
|
25
|
+
DRIFT_WINDOW_EVENTS: () => DRIFT_WINDOW_EVENTS,
|
|
24
26
|
ExactResponseCache: () => ExactResponseCache,
|
|
25
27
|
LEDGER_GENESIS_HASH: () => LEDGER_GENESIS_HASH,
|
|
28
|
+
MESH_SIGNUP_URL: () => MESH_SIGNUP_URL,
|
|
29
|
+
acceptProposal: () => acceptProposal,
|
|
26
30
|
agentGrader: () => agentGrader,
|
|
31
|
+
asBundle: () => asBundle,
|
|
27
32
|
auditEventPricing: () => auditEventPricing,
|
|
28
33
|
billableEventFromRequest: () => billableEventFromRequest,
|
|
29
34
|
buildQuoteReport: () => buildQuoteReport,
|
|
35
|
+
bundleToReductionConfig: () => bundleToReductionConfig,
|
|
30
36
|
canonicalJson: () => canonicalJson,
|
|
31
37
|
capToolResultBody: () => capToolResultBody,
|
|
32
38
|
catalogueVersion: () => catalogueVersion,
|
|
@@ -36,32 +42,55 @@ __export(index_exports, {
|
|
|
36
42
|
compositeGrader: () => compositeGrader,
|
|
37
43
|
createLocalJsonlCapture: () => createLocalJsonlCapture,
|
|
38
44
|
createSavingsLedger: () => createSavingsLedger,
|
|
45
|
+
createTheoMemory: () => createTheoMemory,
|
|
39
46
|
defaultPolicy: () => defaultPolicy,
|
|
47
|
+
defaultTheoMemoryPath: () => defaultTheoMemoryPath,
|
|
40
48
|
digestStaleBody: () => digestStaleBody,
|
|
41
49
|
exactTextGrader: () => exactTextGrader,
|
|
42
50
|
extractBlindFeatures: () => extractBlindFeatures,
|
|
51
|
+
formatSavingsCta: () => formatSavingsCta,
|
|
52
|
+
isPolicyBundle: () => isPolicyBundle,
|
|
43
53
|
knownLeverIds: () => knownLeverIds,
|
|
44
54
|
ledgerEventHash: () => ledgerEventHash,
|
|
55
|
+
leverToReductionConfig: () => leverToReductionConfig,
|
|
56
|
+
loadTheoMemory: () => loadTheoMemory,
|
|
57
|
+
mergePolicyIntoBundle: () => mergePolicyIntoBundle,
|
|
58
|
+
observeWorkload: () => observeWorkload,
|
|
45
59
|
parseGatewayExport: () => parseGatewayExport,
|
|
60
|
+
policyForRoute: () => policyForRoute,
|
|
46
61
|
policyToReductionConfig: () => policyToReductionConfig,
|
|
47
62
|
priceLedgerUsage: () => priceLedgerUsage,
|
|
63
|
+
proveCandidates: () => proveCandidates,
|
|
48
64
|
publishLever: () => publishLever,
|
|
49
65
|
publishedModelPrices: () => publishedModelPrices,
|
|
66
|
+
realisedFromSpoolDir: () => realisedFromSpoolDir,
|
|
67
|
+
recentDriftRates: () => recentDriftRates,
|
|
50
68
|
recomputeInvoice: () => recomputeInvoice,
|
|
69
|
+
reconsiderLever: () => reconsiderLever,
|
|
51
70
|
reduceRequest: () => reduceRequest,
|
|
52
71
|
reduceRequestAsync: () => reduceRequestAsync,
|
|
72
|
+
rejectProposal: () => rejectProposal,
|
|
53
73
|
renderLedgerReport: () => renderLedgerReport,
|
|
54
74
|
renderQuoteReport: () => renderQuoteReport,
|
|
75
|
+
renderTheoReport: () => renderTheoReport,
|
|
55
76
|
replayGate: () => replayGate,
|
|
56
77
|
replayUsage: () => replayUsage,
|
|
57
78
|
responseText: () => responseText,
|
|
58
79
|
responseToolCalls: () => responseToolCalls,
|
|
80
|
+
routeMatches: () => routeMatches,
|
|
81
|
+
runTheoCycle: () => runTheoCycle,
|
|
82
|
+
saveTheoMemory: () => saveTheoMemory,
|
|
83
|
+
savingsBadge: () => savingsBadge,
|
|
84
|
+
savingsPercent: () => savingsPercent,
|
|
85
|
+
savingsSnapshotFromEvents: () => savingsSnapshotFromEvents,
|
|
86
|
+
savingsSnapshotFromSpoolDir: () => savingsSnapshotFromSpoolDir,
|
|
59
87
|
serviceTierOf: () => serviceTierOf,
|
|
60
88
|
startMeshProxy: () => startMeshProxy,
|
|
61
89
|
textSimilarityGrader: () => textSimilarityGrader,
|
|
62
90
|
toolNameGrader: () => toolNameGrader,
|
|
63
91
|
toolSelectionGrader: () => toolSelectionGrader,
|
|
64
92
|
truncationAwareGrader: () => truncationAwareGrader,
|
|
93
|
+
upsertRoutePolicy: () => upsertRoutePolicy,
|
|
65
94
|
usageFromSse: () => usageFromSse,
|
|
66
95
|
verifyLedgerChain: () => verifyLedgerChain,
|
|
67
96
|
withMesh: () => withMesh,
|
|
@@ -73,6 +102,10 @@ module.exports = __toCommonJS(index_exports);
|
|
|
73
102
|
// packages/sdk-js/src/openai.ts
|
|
74
103
|
var import_node_crypto8 = require("node:crypto");
|
|
75
104
|
|
|
105
|
+
// packages/core/dist/src/routing.js
|
|
106
|
+
var ROLLOUT_LADDER = [0.01, 0.05, 0.1, 0.25, 0.5, 1];
|
|
107
|
+
var ROLLOUT_LADDER_TOP = ROLLOUT_LADDER.length - 1;
|
|
108
|
+
|
|
76
109
|
// packages/core/dist/src/cost.js
|
|
77
110
|
function estimateTokenCost(inputTokens, outputTokens, price, cacheUsage = {}) {
|
|
78
111
|
const cached = cacheUsage.cachedInputTokens ?? 0;
|
|
@@ -645,7 +678,33 @@ function resolveModelId(catalogue, provider, rawModelId) {
|
|
|
645
678
|
}
|
|
646
679
|
|
|
647
680
|
// packages/core/dist/src/tool-schema.js
|
|
648
|
-
var PLUMBING_KEYS = /* @__PURE__ */ new Set(["$schema", "$id", "$comment"
|
|
681
|
+
var PLUMBING_KEYS = /* @__PURE__ */ new Set(["$schema", "$id", "$comment"]);
|
|
682
|
+
var NAME_MAP_KEYS = /* @__PURE__ */ new Set([
|
|
683
|
+
"properties",
|
|
684
|
+
"patternProperties",
|
|
685
|
+
"$defs",
|
|
686
|
+
"definitions",
|
|
687
|
+
"dependentSchemas",
|
|
688
|
+
"dependencies"
|
|
689
|
+
]);
|
|
690
|
+
var DATA_KEYS = /* @__PURE__ */ new Set(["default", "const", "enum", "examples", "example", "input_examples"]);
|
|
691
|
+
var REFERENCE_KEYS = /* @__PURE__ */ new Set(["$ref", "$dynamicRef", "$recursiveRef"]);
|
|
692
|
+
function usesReferences(node) {
|
|
693
|
+
if (Array.isArray(node))
|
|
694
|
+
return node.some(usesReferences);
|
|
695
|
+
if (typeof node !== "object" || node === null)
|
|
696
|
+
return false;
|
|
697
|
+
for (const [key, value] of Object.entries(node)) {
|
|
698
|
+
if (REFERENCE_KEYS.has(key) && typeof value === "string")
|
|
699
|
+
return true;
|
|
700
|
+
if (usesReferences(value))
|
|
701
|
+
return true;
|
|
702
|
+
}
|
|
703
|
+
return false;
|
|
704
|
+
}
|
|
705
|
+
function isPlainRecord(value) {
|
|
706
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
707
|
+
}
|
|
649
708
|
function toolNameOf(tool, index) {
|
|
650
709
|
if (typeof tool === "object" && tool !== null) {
|
|
651
710
|
const row = tool;
|
|
@@ -691,6 +750,7 @@ function minifyToolSchemas(tools, options = {}) {
|
|
|
691
750
|
const tokensBefore = countTokens(JSON.stringify(tools));
|
|
692
751
|
const keep = options.keepTools === void 0 ? void 0 : new Set(options.keepTools);
|
|
693
752
|
const retained = [];
|
|
753
|
+
let keepIdentifiers = false;
|
|
694
754
|
tools.forEach((tool, index) => {
|
|
695
755
|
const name = toolNameOf(tool, index);
|
|
696
756
|
if (keep !== void 0 && !keep.has(name)) {
|
|
@@ -703,17 +763,23 @@ function minifyToolSchemas(tools, options = {}) {
|
|
|
703
763
|
});
|
|
704
764
|
return;
|
|
705
765
|
}
|
|
766
|
+
keepIdentifiers = usesReferences(tool);
|
|
706
767
|
retained.push(prune(tool, name));
|
|
707
768
|
});
|
|
708
|
-
function prune(node, path) {
|
|
769
|
+
function prune(node, path, names = false) {
|
|
709
770
|
if (Array.isArray(node))
|
|
710
771
|
return node.map((item, index) => prune(item, `${path}[${index}]`));
|
|
711
772
|
if (typeof node !== "object" || node === null)
|
|
712
773
|
return node;
|
|
713
774
|
const row = node;
|
|
714
775
|
const output = {};
|
|
776
|
+
if (names) {
|
|
777
|
+
for (const [key, value] of Object.entries(row))
|
|
778
|
+
output[key] = prune(value, `${path}.${key}`);
|
|
779
|
+
return output;
|
|
780
|
+
}
|
|
715
781
|
for (const [key, value] of Object.entries(row)) {
|
|
716
|
-
if (PLUMBING_KEYS.has(key)) {
|
|
782
|
+
if (PLUMBING_KEYS.has(key) && !(key === "$id" && keepIdentifiers)) {
|
|
717
783
|
changes.push({
|
|
718
784
|
kind: "plumbing",
|
|
719
785
|
lossless: true,
|
|
@@ -770,7 +836,11 @@ function minifyToolSchemas(tools, options = {}) {
|
|
|
770
836
|
output[key] = trimmed;
|
|
771
837
|
continue;
|
|
772
838
|
}
|
|
773
|
-
|
|
839
|
+
if (DATA_KEYS.has(key)) {
|
|
840
|
+
output[key] = structuredClone(value);
|
|
841
|
+
continue;
|
|
842
|
+
}
|
|
843
|
+
output[key] = prune(value, `${path}.${key}`, NAME_MAP_KEYS.has(key) && isPlainRecord(value));
|
|
774
844
|
}
|
|
775
845
|
return output;
|
|
776
846
|
}
|
|
@@ -809,6 +879,271 @@ function auditToolCatalogue(tools, options = {}) {
|
|
|
809
879
|
};
|
|
810
880
|
}
|
|
811
881
|
|
|
882
|
+
// packages/core/dist/src/savings-model.js
|
|
883
|
+
var INPUT_SEGMENTS = [
|
|
884
|
+
"toolSchemas",
|
|
885
|
+
"systemPrompt",
|
|
886
|
+
"retrievedContext",
|
|
887
|
+
"history",
|
|
888
|
+
"userTurn"
|
|
889
|
+
];
|
|
890
|
+
function assertFraction(value, label) {
|
|
891
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) {
|
|
892
|
+
throw new Error(`${label} must be between 0 and 1.`);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
var PUBLISHED_LEVERS = {
|
|
896
|
+
toolSchemaCompaction: {
|
|
897
|
+
id: "tool-schema-compaction",
|
|
898
|
+
description: "Remove JSON indentation, schema plumbing the model never reads ($schema, $id, $comment), and `title` fields that only restate their property name. The decoded schema is semantically identical.",
|
|
899
|
+
risk: "deterministic",
|
|
900
|
+
targets: ["toolSchemas"],
|
|
901
|
+
// Measured per catalogue by auditToolCatalogue; this is a conservative stand-in.
|
|
902
|
+
reduction: 0.15,
|
|
903
|
+
evidence: {
|
|
904
|
+
source: "Measured directly from the request's own tool definitions by auditToolCatalogue",
|
|
905
|
+
claim: "Formatting and plumbing only. Tool selection cannot change because the decoded schema is unchanged."
|
|
906
|
+
}
|
|
907
|
+
},
|
|
908
|
+
toolSchemaMinification: {
|
|
909
|
+
id: "tool-schema-description-trim",
|
|
910
|
+
description: "Trim prose descriptions and docstrings in tool/MCP JSON Schema, keeping names, types and required fields. Agent runtimes resend the whole catalogue on every turn because the APIs are stateless, so a 20-turn task pays for it 20 times. Descriptions are how the model chooses between tools, so this is a behaviour change.",
|
|
911
|
+
risk: "needs-verification",
|
|
912
|
+
targets: ["toolSchemas"],
|
|
913
|
+
// Sources report >76% on a single schema and 30-60% across large catalogues.
|
|
914
|
+
reduction: 0.6,
|
|
915
|
+
evidence: {
|
|
916
|
+
source: "github.com/olivomarco/github-copilot-token-optimization, InfoQ on GitHub's agent audits",
|
|
917
|
+
claim: "Schema minification took one catalogue from 3,198 to under 750 tokens (>76%); tool-schema compression reduces per-request overhead 30-60%; GitHub cut agent workflow token spend up to 62% with MCP pruning."
|
|
918
|
+
}
|
|
919
|
+
},
|
|
920
|
+
unusedToolRemoval: {
|
|
921
|
+
id: "unused-tool-removal",
|
|
922
|
+
description: "Drop tools the workload never calls. A typical MCP server exposes 20-30 tools at 300-500 tokens each, and six connected servers run to 60,000-90,000 tokens of catalogue per request. Removing a tool removes the model's ability to call it, so this holds only where the tool is genuinely unreachable for the workload.",
|
|
923
|
+
risk: "needs-verification",
|
|
924
|
+
targets: ["toolSchemas"],
|
|
925
|
+
reduction: 0.3,
|
|
926
|
+
evidence: {
|
|
927
|
+
source: "mindstudio.ai MCP optimisation, buildmvpfast MCP token cost analysis",
|
|
928
|
+
claim: "Removing unused MCP entries cuts 8-12 KB of context per call; unused tools are the most common inefficiency."
|
|
929
|
+
}
|
|
930
|
+
},
|
|
931
|
+
progressiveDisclosure: {
|
|
932
|
+
id: "progressive-disclosure",
|
|
933
|
+
description: "Replace the tool catalogue with a search/meta tool the model queries for the definitions it needs, so the full catalogue is never sent. Cuts the prompt hard, but adds a round trip per novel tool invocation.",
|
|
934
|
+
risk: "needs-verification",
|
|
935
|
+
targets: ["toolSchemas"],
|
|
936
|
+
// Solo.io measured 10,877 -> 970 prompt tokens, a 91.1% cut.
|
|
937
|
+
reduction: 0.91,
|
|
938
|
+
// The reason this lever usually disappoints. Archestra measured ~77% modelled
|
|
939
|
+
// against ~13% in production on a 20-step run; 1.5 is a conservative middle.
|
|
940
|
+
roundTripMultiplier: 1.5,
|
|
941
|
+
evidence: {
|
|
942
|
+
source: "solo.io agentgateway progressive disclosure; archestra.ai modelled-vs-measured bench",
|
|
943
|
+
claim: "Prompt overhead fell 10,877 to 970 tokens (91.1%) on Claude Sonnet 4.6. But a router adds a hop: the same approach modelled at ~77% measured ~13% in production, because every novel tool call costs an extra round trip and an extra model decision."
|
|
944
|
+
}
|
|
945
|
+
},
|
|
946
|
+
codeMode: {
|
|
947
|
+
id: "code-mode",
|
|
948
|
+
description: "Expose a handful of meta-tools (list, describe, call, call-sequential) and let the model write code that orchestrates the real tools in a sandbox, so it never sees the full definitions. Cuts the prompt and the number of calls at the same time.",
|
|
949
|
+
risk: "needs-verification",
|
|
950
|
+
targets: ["toolSchemas"],
|
|
951
|
+
// Bifrost measured 92.8% at 508 tools; Anthropic reported 150k -> 2k (98.7%).
|
|
952
|
+
reduction: 0.9,
|
|
953
|
+
// The reason this one holds up: 3-4x fewer round trips, modelled at the
|
|
954
|
+
// conservative end of that range.
|
|
955
|
+
roundTripMultiplier: 0.35,
|
|
956
|
+
evidence: {
|
|
957
|
+
source: "Bifrost code mode benchmarks; Anthropic engineering on code execution with MCP",
|
|
958
|
+
claim: "Bifrost cut input tokens up to 92.8% at 508 tools with a 100% pass rate and ran ~40% faster, needing 3-4x fewer round trips. Anthropic reported a workflow falling from 150,000 to 2,000 tokens (98.7%) rewritten as code execution. Requires a sandbox and changes how the agent is built."
|
|
959
|
+
}
|
|
960
|
+
},
|
|
961
|
+
retrievedContextPruning: {
|
|
962
|
+
id: "retrieved-context-pruning",
|
|
963
|
+
description: "Rerank retrieved chunks and keep only those that add information, instead of passing a fixed large top-k.",
|
|
964
|
+
risk: "needs-verification",
|
|
965
|
+
targets: ["retrievedContext"],
|
|
966
|
+
// Sources span 50-90%; held well below that.
|
|
967
|
+
reduction: 0.6,
|
|
968
|
+
evidence: {
|
|
969
|
+
source: "arXiv:2601.17532 (Information Gain Pruning), AWS Bedrock query-aware compression",
|
|
970
|
+
claim: "IGP cut final-stage input tokens 76-79% while improving average F1 by 12-20%; rerank plus compression reached 10.1x fewer tokens at 36% cost saving; right-sized chunks with top-k 3-8 cut context 50-90%."
|
|
971
|
+
}
|
|
972
|
+
},
|
|
973
|
+
promptCompression: {
|
|
974
|
+
id: "prompt-compression",
|
|
975
|
+
description: "Token-level compression of the variable prompt body, dropping tokens a small model judges predictable.",
|
|
976
|
+
risk: "needs-verification",
|
|
977
|
+
targets: ["history", "retrievedContext"],
|
|
978
|
+
// LLMLingua reports up to 20x (95%); 50% is far inside that.
|
|
979
|
+
reduction: 0.5,
|
|
980
|
+
evidence: {
|
|
981
|
+
source: "arXiv:2403.12968 (LLMLingua-2), Microsoft Research",
|
|
982
|
+
claim: "Up to 20x compression with a 1.5 point drop on GSM8K and BBH, and under 2% quality loss on CoQA, HotpotQA and TriviaQA."
|
|
983
|
+
}
|
|
984
|
+
},
|
|
985
|
+
historyCompaction: {
|
|
986
|
+
id: "history-compaction",
|
|
987
|
+
description: "Summarise or window old conversation turns rather than resending every turn verbatim.",
|
|
988
|
+
risk: "needs-verification",
|
|
989
|
+
targets: ["history"],
|
|
990
|
+
reduction: 0.4,
|
|
991
|
+
evidence: {
|
|
992
|
+
source: "Provider guidance on context management; measured per workload by replay",
|
|
993
|
+
claim: "Workload-specific. Conservative placeholder until measured on real traffic."
|
|
994
|
+
}
|
|
995
|
+
},
|
|
996
|
+
outputBudget: {
|
|
997
|
+
id: "output-budget",
|
|
998
|
+
description: "Match reasoning and output budget to task difficulty via reasoning_effort or budget_tokens, instead of letting every request reason at full effort. Output is the expensive half of the bill: 5x input on Sonnet 5.5 and 8x on gpt-5.",
|
|
999
|
+
risk: "needs-verification",
|
|
1000
|
+
targets: ["output"],
|
|
1001
|
+
// TALE-EP reports 67%; held slightly below.
|
|
1002
|
+
reduction: 0.6,
|
|
1003
|
+
evidence: {
|
|
1004
|
+
source: "TALE-EP; arXiv:2508.17196 (BudgetThinker); provider reasoning-effort controls",
|
|
1005
|
+
claim: "67% average output token reduction at under 3% accuracy drop across seven datasets. On GSM8K accuracy rose from 81.35% to 84.46% while mean output fell from 318 to 77 tokens, so shorter was also better."
|
|
1006
|
+
}
|
|
1007
|
+
},
|
|
1008
|
+
duplicateContextRemoval: {
|
|
1009
|
+
id: "duplicate-context-removal",
|
|
1010
|
+
description: "Remove exact duplicate context blocks. Overlapping retrieval chunks and system prompts echoed into the first user turn are routine; the model already has the text.",
|
|
1011
|
+
risk: "deterministic",
|
|
1012
|
+
targets: ["retrievedContext", "history"],
|
|
1013
|
+
reduction: 0.1,
|
|
1014
|
+
evidence: {
|
|
1015
|
+
source: "Measured directly from the customer's own traffic by dedupeRepeatedBlocks",
|
|
1016
|
+
claim: "Exact duplicates only, so removal cannot change the prompt's meaning."
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
};
|
|
1020
|
+
function projectSavings(profile, levers, price, options = {}) {
|
|
1021
|
+
if (!Number.isInteger(profile.requests) || profile.requests < 1) {
|
|
1022
|
+
throw new Error("A workload profile needs at least one request.");
|
|
1023
|
+
}
|
|
1024
|
+
for (const [segment, tokens] of Object.entries(profile.segments)) {
|
|
1025
|
+
if (!Number.isFinite(tokens) || tokens < 0) {
|
|
1026
|
+
throw new Error(`Segment ${segment} must hold a non-negative token count.`);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
for (const lever of levers) {
|
|
1030
|
+
assertFraction(lever.reduction, `Lever ${lever.id} reduction`);
|
|
1031
|
+
if (lever.roundTripMultiplier !== void 0 && (!Number.isFinite(lever.roundTripMultiplier) || lever.roundTripMultiplier <= 0)) {
|
|
1032
|
+
throw new Error(`Lever ${lever.id} roundTripMultiplier must be a positive finite number.`);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
const minimumCacheable = options.minimumCacheableTokens ?? 1024;
|
|
1036
|
+
const warnings = [];
|
|
1037
|
+
const remaining = { ...profile.segments };
|
|
1038
|
+
const inputTokensBefore = INPUT_SEGMENTS.reduce((sum, segment) => sum + profile.segments[segment], 0);
|
|
1039
|
+
const outputTokensBefore = profile.segments.output;
|
|
1040
|
+
let callMultiplier = 1;
|
|
1041
|
+
const costOf = (input, output) => estimateTokenCost(input, output, price) * profile.requests * callMultiplier;
|
|
1042
|
+
const baselineCostUsd = costOf(inputTokensBefore, outputTokensBefore);
|
|
1043
|
+
let runningCost = baselineCostUsd;
|
|
1044
|
+
const contributions = [];
|
|
1045
|
+
for (const lever of levers) {
|
|
1046
|
+
let tokensRemoved = 0;
|
|
1047
|
+
for (const segment of lever.targets) {
|
|
1048
|
+
const removed = remaining[segment] * lever.reduction;
|
|
1049
|
+
remaining[segment] -= removed;
|
|
1050
|
+
tokensRemoved += removed;
|
|
1051
|
+
}
|
|
1052
|
+
callMultiplier *= lever.roundTripMultiplier ?? 1;
|
|
1053
|
+
const input = INPUT_SEGMENTS.reduce((sum, segment) => sum + remaining[segment], 0);
|
|
1054
|
+
const cost = costOf(input, remaining.output);
|
|
1055
|
+
const costRemovedUsd = runningCost - cost;
|
|
1056
|
+
contributions.push({
|
|
1057
|
+
lever: lever.id,
|
|
1058
|
+
risk: lever.risk,
|
|
1059
|
+
tokensRemoved: tokensRemoved * profile.requests,
|
|
1060
|
+
costRemovedUsd,
|
|
1061
|
+
sharePercent: baselineCostUsd === 0 ? 0 : costRemovedUsd / baselineCostUsd * 100,
|
|
1062
|
+
evidence: lever.evidence
|
|
1063
|
+
});
|
|
1064
|
+
runningCost = cost;
|
|
1065
|
+
}
|
|
1066
|
+
let cacheableTokens = 0;
|
|
1067
|
+
for (const segment of profile.stableSegments) {
|
|
1068
|
+
if (!INPUT_SEGMENTS.includes(segment)) {
|
|
1069
|
+
warnings.push(`Segment ${segment} cannot be part of a cached prefix and was ignored for caching.`);
|
|
1070
|
+
continue;
|
|
1071
|
+
}
|
|
1072
|
+
cacheableTokens += remaining[segment];
|
|
1073
|
+
}
|
|
1074
|
+
const inputTokensAfter = INPUT_SEGMENTS.reduce((sum, segment) => sum + remaining[segment], 0);
|
|
1075
|
+
if (price.cachedInputPerMillion === void 0) {
|
|
1076
|
+
if (cacheableTokens > 0) {
|
|
1077
|
+
warnings.push("The supplied price has no cached-input rate, so no caching saving is projected.");
|
|
1078
|
+
}
|
|
1079
|
+
cacheableTokens = 0;
|
|
1080
|
+
} else if (cacheableTokens < minimumCacheable) {
|
|
1081
|
+
if (cacheableTokens > 0) {
|
|
1082
|
+
warnings.push(`The stable prefix is ${Math.round(cacheableTokens)} tokens, below the provider's ${minimumCacheable}-token cacheable minimum, so no caching saving is projected.`);
|
|
1083
|
+
}
|
|
1084
|
+
cacheableTokens = 0;
|
|
1085
|
+
}
|
|
1086
|
+
if (profile.requests === 1 && cacheableTokens > 0) {
|
|
1087
|
+
warnings.push("A single request cannot benefit from a prefix cache; it only populates it.");
|
|
1088
|
+
cacheableTokens = 0;
|
|
1089
|
+
}
|
|
1090
|
+
let projectedCostUsd = runningCost;
|
|
1091
|
+
if (cacheableTokens > 0) {
|
|
1092
|
+
const body = inputTokensAfter - cacheableTokens;
|
|
1093
|
+
const write = price.cacheWriteInputPerMillion === void 0 ? estimateTokenCost(cacheableTokens, 0, price) : estimateTokenCost(cacheableTokens, 0, price, { cacheWriteInputTokens: cacheableTokens });
|
|
1094
|
+
const read = estimateTokenCost(cacheableTokens, 0, price, { cachedInputTokens: cacheableTokens });
|
|
1095
|
+
const perRequestRest = estimateTokenCost(body, remaining.output, price);
|
|
1096
|
+
const calls = profile.requests * callMultiplier;
|
|
1097
|
+
const cached = write + Math.max(0, calls - 1) * read + calls * perRequestRest;
|
|
1098
|
+
contributions.push({
|
|
1099
|
+
lever: "prefix-caching",
|
|
1100
|
+
risk: "deterministic",
|
|
1101
|
+
tokensRemoved: 0,
|
|
1102
|
+
costRemovedUsd: projectedCostUsd - cached,
|
|
1103
|
+
sharePercent: baselineCostUsd === 0 ? 0 : (projectedCostUsd - cached) / baselineCostUsd * 100,
|
|
1104
|
+
evidence: {
|
|
1105
|
+
source: "Provider published rates, captured in model-prices.ts",
|
|
1106
|
+
claim: "Cached input is billed at a fraction of fresh input (Sonnet 5.5 $2.00 to $0.10; gpt-5 $1.25 to $0.125). The same tokens are sent and the same answer returns, so only the rate changes."
|
|
1107
|
+
}
|
|
1108
|
+
});
|
|
1109
|
+
projectedCostUsd = cached;
|
|
1110
|
+
}
|
|
1111
|
+
const savingUsd = baselineCostUsd - projectedCostUsd;
|
|
1112
|
+
const share = (risk) => contributions.filter((contribution) => contribution.risk === risk).reduce((sum, contribution) => sum + contribution.sharePercent, 0);
|
|
1113
|
+
return {
|
|
1114
|
+
workload: profile.name,
|
|
1115
|
+
requests: profile.requests,
|
|
1116
|
+
baselineCostUsd,
|
|
1117
|
+
projectedCostUsd,
|
|
1118
|
+
savingUsd,
|
|
1119
|
+
savingPercent: baselineCostUsd === 0 ? 0 : savingUsd / baselineCostUsd * 100,
|
|
1120
|
+
deterministicSavingPercent: share("deterministic"),
|
|
1121
|
+
needsVerificationSavingPercent: share("needs-verification"),
|
|
1122
|
+
contributions,
|
|
1123
|
+
inputTokensBefore: inputTokensBefore * profile.requests,
|
|
1124
|
+
inputTokensAfter: inputTokensAfter * profile.requests * callMultiplier,
|
|
1125
|
+
outputTokensBefore: outputTokensBefore * profile.requests,
|
|
1126
|
+
outputTokensAfter: remaining.output * profile.requests * callMultiplier,
|
|
1127
|
+
cachedInputTokens: cacheableTokens * profile.requests * callMultiplier,
|
|
1128
|
+
roundTripsBefore: profile.requests,
|
|
1129
|
+
roundTripsAfter: profile.requests * callMultiplier,
|
|
1130
|
+
warnings
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
function leversFor(profile) {
|
|
1134
|
+
const order = [
|
|
1135
|
+
PUBLISHED_LEVERS.toolSchemaCompaction,
|
|
1136
|
+
PUBLISHED_LEVERS.unusedToolRemoval,
|
|
1137
|
+
PUBLISHED_LEVERS.toolSchemaMinification,
|
|
1138
|
+
PUBLISHED_LEVERS.duplicateContextRemoval,
|
|
1139
|
+
PUBLISHED_LEVERS.retrievedContextPruning,
|
|
1140
|
+
PUBLISHED_LEVERS.historyCompaction,
|
|
1141
|
+
PUBLISHED_LEVERS.promptCompression,
|
|
1142
|
+
PUBLISHED_LEVERS.outputBudget
|
|
1143
|
+
];
|
|
1144
|
+
return order.filter((lever) => lever.targets.some((segment) => (profile.segments[segment] ?? 0) > 0));
|
|
1145
|
+
}
|
|
1146
|
+
|
|
812
1147
|
// packages/core/dist/src/savings-ledger.js
|
|
813
1148
|
var import_node_crypto3 = require("node:crypto");
|
|
814
1149
|
var LEDGER_GENESIS_HASH = "0".repeat(64);
|
|
@@ -2093,6 +2428,12 @@ function measureOutputReEmission(requestText, outputText, countTokens = approxim
|
|
|
2093
2428
|
};
|
|
2094
2429
|
}
|
|
2095
2430
|
|
|
2431
|
+
// packages/core/dist/src/security/hash-chain.js
|
|
2432
|
+
var CHAIN_GENESIS_HASH = "0".repeat(64);
|
|
2433
|
+
|
|
2434
|
+
// packages/core/dist/src/github-oauth.js
|
|
2435
|
+
var STATE_MAX_AGE_MS = 10 * 60 * 1e3;
|
|
2436
|
+
|
|
2096
2437
|
// packages/sdk-js/src/telemetry.ts
|
|
2097
2438
|
var TelemetryClient = class {
|
|
2098
2439
|
constructor(config = {}) {
|
|
@@ -2266,6 +2607,14 @@ ${serialized}`).digest("hex");
|
|
|
2266
2607
|
var import_node_crypto7 = require("node:crypto");
|
|
2267
2608
|
var isPlainObject5 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2268
2609
|
var digest = (value) => (0, import_node_crypto7.createHash)("sha256").update(JSON.stringify(value ?? null) ?? "null").digest("hex").slice(0, 32);
|
|
2610
|
+
function advance(prior, churned) {
|
|
2611
|
+
if (churned) return { track: { volatile: true, stableStreak: 0 }, ok: false, proving: false };
|
|
2612
|
+
if (!prior.volatile) return { track: prior, ok: true, proving: false };
|
|
2613
|
+
const stableStreak = prior.stableStreak + 1;
|
|
2614
|
+
if (stableStreak >= 2) return { track: { volatile: false, stableStreak: 0 }, ok: true, proving: false };
|
|
2615
|
+
return { track: { volatile: true, stableStreak }, ok: false, proving: true };
|
|
2616
|
+
}
|
|
2617
|
+
var PROVING = "this conversation's prefix changed on an earlier turn; waiting for it to prove stable before stamping again";
|
|
2269
2618
|
var CachePrefixVolatilityGuard = class {
|
|
2270
2619
|
constructor(maxConversations = 1e3) {
|
|
2271
2620
|
this.maxConversations = maxConversations;
|
|
@@ -2277,48 +2626,49 @@ var CachePrefixVolatilityGuard = class {
|
|
|
2277
2626
|
/** Insertion-ordered; re-inserted on touch, so the first key is the LRU. */
|
|
2278
2627
|
conversations = /* @__PURE__ */ new Map();
|
|
2279
2628
|
/**
|
|
2280
|
-
* Record this request and decide
|
|
2281
|
-
*
|
|
2629
|
+
* Record this request and decide which cache breakpoints its conversation
|
|
2630
|
+
* has earned. Call once per request, with the raw (unreduced) request and,
|
|
2631
|
+
* when known, the content the system breakpoint would cover.
|
|
2282
2632
|
*/
|
|
2283
|
-
observe(request) {
|
|
2633
|
+
observe(request, context = {}) {
|
|
2284
2634
|
if (!isPlainObject5(request) || !Array.isArray(request.messages) || request.messages.length === 0) {
|
|
2285
2635
|
return { stamp: true };
|
|
2286
2636
|
}
|
|
2287
2637
|
const key = digest([request.model ?? null, request.messages[0]]);
|
|
2288
|
-
const prefixHash = digest([request.tools ?? null, request.system ?? null])
|
|
2638
|
+
const prefixHash = context.stampedPrefix !== void 0 ? `span:${digest(context.stampedPrefix)}` : `raw:${digest([request.tools ?? null, request.system ?? null])}`;
|
|
2289
2639
|
const messageHashes = request.messages.map(digest);
|
|
2290
2640
|
const prior = this.conversations.get(key);
|
|
2291
2641
|
if (prior !== void 0) this.conversations.delete(key);
|
|
2292
2642
|
if (prior === void 0) {
|
|
2293
|
-
this.conversations.set(key, {
|
|
2643
|
+
this.conversations.set(key, {
|
|
2644
|
+
prefixHash,
|
|
2645
|
+
messageHashes,
|
|
2646
|
+
prefix: { volatile: false, stableStreak: 0 },
|
|
2647
|
+
history: { volatile: false, stableStreak: 0 }
|
|
2648
|
+
});
|
|
2294
2649
|
this.evict();
|
|
2295
2650
|
return { stamp: true };
|
|
2296
2651
|
}
|
|
2297
2652
|
const prefixChanged = prior.prefixHash !== prefixHash;
|
|
2298
2653
|
const historyExtends = messageHashes.length >= prior.messageHashes.length && prior.messageHashes.every((hash, index) => messageHashes[index] === hash);
|
|
2299
|
-
const
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
decision = {
|
|
2654
|
+
const prefix = advance(prior.prefix, prefixChanged);
|
|
2655
|
+
const history = advance(prior.history, !historyExtends);
|
|
2656
|
+
this.conversations.set(key, { prefixHash, messageHashes, prefix: prefix.track, history: history.track });
|
|
2657
|
+
if (!prefix.ok) {
|
|
2658
|
+
return {
|
|
2305
2659
|
stamp: false,
|
|
2306
|
-
|
|
2660
|
+
messageBreakpoint: false,
|
|
2661
|
+
reason: prefixChanged ? "tools/system changed between turns of this conversation, so a cached prefix could never be read back" : PROVING
|
|
2307
2662
|
};
|
|
2308
|
-
} else if (entry2.volatile) {
|
|
2309
|
-
entry2.stableStreak += 1;
|
|
2310
|
-
if (entry2.stableStreak >= 2) {
|
|
2311
|
-
entry2.volatile = false;
|
|
2312
|
-
entry2.stableStreak = 0;
|
|
2313
|
-
} else {
|
|
2314
|
-
decision = {
|
|
2315
|
-
stamp: false,
|
|
2316
|
-
reason: "this conversation's prefix changed on an earlier turn; waiting for it to prove stable before stamping again"
|
|
2317
|
-
};
|
|
2318
|
-
}
|
|
2319
2663
|
}
|
|
2320
|
-
|
|
2321
|
-
|
|
2664
|
+
if (!history.ok) {
|
|
2665
|
+
return {
|
|
2666
|
+
stamp: true,
|
|
2667
|
+
messageBreakpoint: false,
|
|
2668
|
+
reason: history.proving ? "this conversation's history was rewritten on an earlier turn; waiting for it to prove stable before stamping the newest message again" : "the conversation history was rewritten between turns, so a breakpoint on the newest message could never be read back"
|
|
2669
|
+
};
|
|
2670
|
+
}
|
|
2671
|
+
return { stamp: true };
|
|
2322
2672
|
}
|
|
2323
2673
|
evict() {
|
|
2324
2674
|
while (this.conversations.size > this.maxConversations) {
|
|
@@ -2613,7 +2963,7 @@ function reduceMessageContent(content, config, countTokens, changes) {
|
|
|
2613
2963
|
}
|
|
2614
2964
|
return content;
|
|
2615
2965
|
}
|
|
2616
|
-
function stampAnthropicCacheTtl(next, ttl, systemVolatileTail, countTokens) {
|
|
2966
|
+
function stampAnthropicCacheTtl(next, ttl, systemVolatileTail, countTokens, allowMessageBreakpoint = true) {
|
|
2617
2967
|
const model = typeof next.model === "string" ? next.model.toLowerCase() : "";
|
|
2618
2968
|
if (!model.includes("claude")) return { stamped: false };
|
|
2619
2969
|
if (!Array.isArray(next.messages) || next.messages.length === 0) return { stamped: false };
|
|
@@ -2682,7 +3032,7 @@ ${tailText}`))) {
|
|
|
2682
3032
|
}
|
|
2683
3033
|
}
|
|
2684
3034
|
}
|
|
2685
|
-
if (!volatileTailPresent && next.messages.length >= 2) {
|
|
3035
|
+
if (allowMessageBreakpoint && !volatileTailPresent && next.messages.length >= 2) {
|
|
2686
3036
|
const messages = [...next.messages];
|
|
2687
3037
|
const lastIndex = messages.length - 1;
|
|
2688
3038
|
const last = messages[lastIndex];
|
|
@@ -2720,6 +3070,20 @@ ${tailText}`))) {
|
|
|
2720
3070
|
}
|
|
2721
3071
|
return stamped ? { stamped: true } : { stamped: false, belowMinimum };
|
|
2722
3072
|
}
|
|
3073
|
+
function stampedSystemPrefix(stamped) {
|
|
3074
|
+
if (!Array.isArray(stamped.system)) return void 0;
|
|
3075
|
+
let last = -1;
|
|
3076
|
+
stamped.system.forEach((block, index) => {
|
|
3077
|
+
if (isPlainObject6(block) && block.cache_control !== void 0) last = index;
|
|
3078
|
+
});
|
|
3079
|
+
if (last < 0) return void 0;
|
|
3080
|
+
const span = stamped.system.slice(0, last + 1).map((block) => {
|
|
3081
|
+
if (!isPlainObject6(block)) return block;
|
|
3082
|
+
const { cache_control: _marker, ...rest } = block;
|
|
3083
|
+
return rest;
|
|
3084
|
+
});
|
|
3085
|
+
return [stamped.model ?? null, stamped.tools ?? null, span];
|
|
3086
|
+
}
|
|
2723
3087
|
function reduceRequest(request, config = {}) {
|
|
2724
3088
|
const countTokens = config.countTokens ?? approximateTokenCount;
|
|
2725
3089
|
const changes = [];
|
|
@@ -2746,6 +3110,35 @@ function reduceRequest(request, config = {}) {
|
|
|
2746
3110
|
}
|
|
2747
3111
|
};
|
|
2748
3112
|
const before = measure(next.instructions) + measure(next.system) + measure(next.input) + measure(next.messages) + measure(next.tools);
|
|
3113
|
+
if (Array.isArray(next.tools) && next.tools.length > 0 && Array.isArray(config.dropTools) && config.dropTools.length > 0) {
|
|
3114
|
+
const drop = new Set(config.dropTools.filter((name) => typeof name === "string"));
|
|
3115
|
+
if (drop.size > 0) {
|
|
3116
|
+
const nameOf = (tool) => {
|
|
3117
|
+
if (!isPlainObject6(tool)) return void 0;
|
|
3118
|
+
if (typeof tool.name === "string") return tool.name;
|
|
3119
|
+
if (isPlainObject6(tool.function) && typeof tool.function.name === "string") return tool.function.name;
|
|
3120
|
+
return void 0;
|
|
3121
|
+
};
|
|
3122
|
+
const toolsBefore = measure(next.tools);
|
|
3123
|
+
const kept = next.tools.filter((tool) => {
|
|
3124
|
+
const name = nameOf(tool);
|
|
3125
|
+
return name === void 0 || !drop.has(name);
|
|
3126
|
+
});
|
|
3127
|
+
const removed = next.tools.length - kept.length;
|
|
3128
|
+
if (removed > 0) {
|
|
3129
|
+
next.tools = kept;
|
|
3130
|
+
const saved = toolsBefore - measure(next.tools);
|
|
3131
|
+
if (saved > 0) {
|
|
3132
|
+
changes.push({
|
|
3133
|
+
lever: "unused-tool-removal",
|
|
3134
|
+
lossless: false,
|
|
3135
|
+
estimatedTokensSaved: saved,
|
|
3136
|
+
detail: `Removed ${removed} tool definition(s) the workload does not call; proven on your traffic before billing.`
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
2749
3142
|
if (Array.isArray(next.tools) && next.tools.length > 0 && config.toolSchemas !== false) {
|
|
2750
3143
|
const toolOptions = isPlainObject6(config.toolSchemas) ? config.toolSchemas : {};
|
|
2751
3144
|
const minified = minifyToolSchemas(next.tools, {
|
|
@@ -2863,8 +3256,32 @@ function reduceRequest(request, config = {}) {
|
|
|
2863
3256
|
}
|
|
2864
3257
|
if (config.cacheTtl === "5m" || config.cacheTtl === "1h") {
|
|
2865
3258
|
const ttl = config.cacheTtl;
|
|
2866
|
-
const
|
|
2867
|
-
const
|
|
3259
|
+
const trial = { ...next };
|
|
3260
|
+
const trialResult = stampAnthropicCacheTtl(trial, ttl, systemVolatileTail, countTokens);
|
|
3261
|
+
const decision = config.cacheVolatilityGuard?.observe(request, {
|
|
3262
|
+
stampedPrefix: stampedSystemPrefix(trial)
|
|
3263
|
+
}) ?? { stamp: true };
|
|
3264
|
+
const messageBreakpointWithheld = decision.stamp && decision.messageBreakpoint === false && trial.messages !== next.messages;
|
|
3265
|
+
let result = { stamped: false };
|
|
3266
|
+
if (decision.stamp) {
|
|
3267
|
+
const systemStamped = trial.system !== next.system;
|
|
3268
|
+
if (systemStamped) next.system = trial.system;
|
|
3269
|
+
if (decision.messageBreakpoint === false) {
|
|
3270
|
+
result = systemStamped ? { stamped: true } : { stamped: false, belowMinimum: trialResult.belowMinimum };
|
|
3271
|
+
} else {
|
|
3272
|
+
if (trial.messages !== next.messages) next.messages = trial.messages;
|
|
3273
|
+
result = trialResult;
|
|
3274
|
+
}
|
|
3275
|
+
}
|
|
3276
|
+
if (messageBreakpointWithheld) {
|
|
3277
|
+
changes.push({
|
|
3278
|
+
lever: "cache-ttl-suppressed",
|
|
3279
|
+
lossless: true,
|
|
3280
|
+
// Nothing was changed; this records WHY one breakpoint was not placed.
|
|
3281
|
+
estimatedTokensSaved: 0,
|
|
3282
|
+
detail: `Cache breakpoint on the newest message withheld: ${decision.reason ?? "the conversation's history is not stable"}. The tools + system breakpoint is unaffected: it sits before the history and still reads.`
|
|
3283
|
+
});
|
|
3284
|
+
}
|
|
2868
3285
|
if (!decision.stamp) {
|
|
2869
3286
|
changes.push({
|
|
2870
3287
|
lever: "cache-ttl-suppressed",
|
|
@@ -2980,6 +3397,94 @@ function reduceRequest(request, config = {}) {
|
|
|
2980
3397
|
// packages/sdk-js/src/savings-ledger.ts
|
|
2981
3398
|
var import_node_fs = require("node:fs");
|
|
2982
3399
|
var import_node_path = require("node:path");
|
|
3400
|
+
|
|
3401
|
+
// packages/sdk-js/src/ledger-mirror.ts
|
|
3402
|
+
var LedgerMirror = class {
|
|
3403
|
+
endpoint;
|
|
3404
|
+
accountKey;
|
|
3405
|
+
doFetch;
|
|
3406
|
+
maxRetries;
|
|
3407
|
+
maxPending;
|
|
3408
|
+
/** Records with seq > confirmedTailSeq, awaiting confirmation. */
|
|
3409
|
+
pending = [];
|
|
3410
|
+
confirmedTailSeq = -1;
|
|
3411
|
+
disabled = false;
|
|
3412
|
+
/** A flush is requested again if records arrive while one is in flight. */
|
|
3413
|
+
dirty = false;
|
|
3414
|
+
/** The in-flight flush, so callers (and tests) can await completion. */
|
|
3415
|
+
current;
|
|
3416
|
+
constructor(config) {
|
|
3417
|
+
this.endpoint = `${config.apiUrl.replace(/\/$/, "")}/v1/ledger/events`;
|
|
3418
|
+
this.accountKey = config.accountKey;
|
|
3419
|
+
this.doFetch = config.fetch ?? fetch;
|
|
3420
|
+
this.maxRetries = config.maxRetries ?? 3;
|
|
3421
|
+
this.maxPending = config.maxPending ?? 5e3;
|
|
3422
|
+
}
|
|
3423
|
+
/** Buffer a record and schedule a flush. Never blocks or throws. */
|
|
3424
|
+
enqueue(record) {
|
|
3425
|
+
if (this.disabled) return;
|
|
3426
|
+
if (record.seq <= this.confirmedTailSeq) return;
|
|
3427
|
+
this.pending.push(record);
|
|
3428
|
+
if (this.pending.length > this.maxPending) {
|
|
3429
|
+
this.disabled = true;
|
|
3430
|
+
this.pending = [];
|
|
3431
|
+
return;
|
|
3432
|
+
}
|
|
3433
|
+
void this.flush();
|
|
3434
|
+
}
|
|
3435
|
+
/** Flush the pending buffer to the API, retrying a few times. Awaitable. */
|
|
3436
|
+
flush() {
|
|
3437
|
+
if (this.disabled) return Promise.resolve();
|
|
3438
|
+
if (this.current) {
|
|
3439
|
+
this.dirty = true;
|
|
3440
|
+
return this.current;
|
|
3441
|
+
}
|
|
3442
|
+
this.current = this.run().finally(() => {
|
|
3443
|
+
this.current = void 0;
|
|
3444
|
+
});
|
|
3445
|
+
return this.current;
|
|
3446
|
+
}
|
|
3447
|
+
async run() {
|
|
3448
|
+
do {
|
|
3449
|
+
this.dirty = false;
|
|
3450
|
+
const batch = this.pending.filter((record) => record.seq > this.confirmedTailSeq);
|
|
3451
|
+
if (batch.length === 0) break;
|
|
3452
|
+
const confirmed = await this.send(batch);
|
|
3453
|
+
if (confirmed === void 0) break;
|
|
3454
|
+
this.confirmedTailSeq = Math.max(this.confirmedTailSeq, confirmed);
|
|
3455
|
+
this.pending = this.pending.filter((record) => record.seq > this.confirmedTailSeq);
|
|
3456
|
+
} while (this.dirty || this.pending.length > 0);
|
|
3457
|
+
}
|
|
3458
|
+
/** POST one batch; returns the server's tailSeq on success, or undefined. */
|
|
3459
|
+
async send(batch) {
|
|
3460
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt += 1) {
|
|
3461
|
+
try {
|
|
3462
|
+
const res = await this.doFetch(this.endpoint, {
|
|
3463
|
+
method: "POST",
|
|
3464
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${this.accountKey}` },
|
|
3465
|
+
body: JSON.stringify({ records: batch }),
|
|
3466
|
+
signal: AbortSignal.timeout(1e4)
|
|
3467
|
+
});
|
|
3468
|
+
if (res.ok) {
|
|
3469
|
+
const json = await res.json().catch(() => ({}));
|
|
3470
|
+
return typeof json.tailSeq === "number" ? json.tailSeq : batch[batch.length - 1].seq;
|
|
3471
|
+
}
|
|
3472
|
+
if (res.status !== 429 && res.status < 500) {
|
|
3473
|
+
if (res.status === 401 || res.status === 403) this.disabled = true;
|
|
3474
|
+
return void 0;
|
|
3475
|
+
}
|
|
3476
|
+
} catch {
|
|
3477
|
+
}
|
|
3478
|
+
if (attempt < this.maxRetries) await delay(250 * 2 ** attempt);
|
|
3479
|
+
}
|
|
3480
|
+
return void 0;
|
|
3481
|
+
}
|
|
3482
|
+
};
|
|
3483
|
+
function delay(ms) {
|
|
3484
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
3485
|
+
}
|
|
3486
|
+
|
|
3487
|
+
// packages/sdk-js/src/savings-ledger.ts
|
|
2983
3488
|
var EXECUTION_TIER_LEVERS = /* @__PURE__ */ new Set(["execution-tier-flex"]);
|
|
2984
3489
|
var CACHE_TTL_LEVERS = /* @__PURE__ */ new Map([["cache-ttl-5m", "5m"], ["cache-ttl-1h", "1h"]]);
|
|
2985
3490
|
var MEASURED_COMPOSITE_LEVER = "token-reduction-measured";
|
|
@@ -3158,17 +3663,20 @@ function createSavingsLedger(config) {
|
|
|
3158
3663
|
const tail = resumeChainTail(config.path);
|
|
3159
3664
|
let previous = tail.previous;
|
|
3160
3665
|
let disabled = tail.disabled;
|
|
3666
|
+
const mirror = config.mirror ? new LedgerMirror(config.mirror) : void 0;
|
|
3161
3667
|
return {
|
|
3162
3668
|
path: config.path,
|
|
3163
|
-
emit(event) {
|
|
3669
|
+
emit(event, context) {
|
|
3164
3670
|
if (disabled) return;
|
|
3165
3671
|
try {
|
|
3166
|
-
const
|
|
3672
|
+
const effective = context?.gateVerdicts === void 0 ? config : { ...config, gateVerdicts: { ...config.gateVerdicts ?? {}, ...context.gateVerdicts } };
|
|
3673
|
+
const billable = billableEventFromRequest(event, effective, catalogue, pinnedCatalogueVersion);
|
|
3167
3674
|
const record = chainLedgerRecord(billable, previous);
|
|
3168
3675
|
(0, import_node_fs.mkdirSync)((0, import_node_path.dirname)(config.path), { recursive: true, mode: 448 });
|
|
3169
3676
|
(0, import_node_fs.appendFileSync)(config.path, `${JSON.stringify(record)}
|
|
3170
3677
|
`, { encoding: "utf8", mode: 384 });
|
|
3171
3678
|
previous = record;
|
|
3679
|
+
mirror?.enqueue(record);
|
|
3172
3680
|
} catch {
|
|
3173
3681
|
disabled = true;
|
|
3174
3682
|
}
|
|
@@ -3562,6 +4070,7 @@ function createLocalJsonlCapture(path, redact, hooks = {}) {
|
|
|
3562
4070
|
|
|
3563
4071
|
// packages/sdk-js/src/proxy.ts
|
|
3564
4072
|
var import_node_http = require("node:http");
|
|
4073
|
+
var import_node_stream = require("node:stream");
|
|
3565
4074
|
var import_node_crypto10 = require("node:crypto");
|
|
3566
4075
|
var import_node_fs2 = require("node:fs");
|
|
3567
4076
|
var import_node_path3 = require("node:path");
|
|
@@ -3796,6 +4305,7 @@ async function replayGate(options) {
|
|
|
3796
4305
|
const costBaseline = priceUsageRow(row.baselineUsage);
|
|
3797
4306
|
const costCandidate = priceUsageRow(row.candidateUsage);
|
|
3798
4307
|
const costFell = costBaseline !== void 0 && costCandidate !== void 0 && costCandidate < costBaseline;
|
|
4308
|
+
const priced = costBaseline !== void 0 && costCandidate !== void 0;
|
|
3799
4309
|
let approved2 = lower >= -maximumQualityDrop;
|
|
3800
4310
|
let rejectionReason;
|
|
3801
4311
|
if (row.graded === 0) {
|
|
@@ -3803,9 +4313,14 @@ async function replayGate(options) {
|
|
|
3803
4313
|
rejectionReason = "No gradeable comparison was produced.";
|
|
3804
4314
|
} else if (!approved2) {
|
|
3805
4315
|
rejectionReason = `Quality lower bound ${(lower * 100).toFixed(2)} pp breaches the ${(maximumQualityDrop * 100).toFixed(2)} pp limit.`;
|
|
3806
|
-
} else if (
|
|
4316
|
+
} else if (priced) {
|
|
4317
|
+
if (!costFell) {
|
|
4318
|
+
approved2 = false;
|
|
4319
|
+
rejectionReason = "Quality held but the bill did not fall once cache writes and reads are priced in (e.g. compacting a prefix the provider was still serving from cache), so there is nothing to enable.";
|
|
4320
|
+
}
|
|
4321
|
+
} else if (inputSaved + outputSaved <= 0) {
|
|
3807
4322
|
approved2 = false;
|
|
3808
|
-
rejectionReason = "Quality held but
|
|
4323
|
+
rejectionReason = "Quality held but no tokens were removed, so there is nothing to enable.";
|
|
3809
4324
|
}
|
|
3810
4325
|
return {
|
|
3811
4326
|
label: row.candidate.label,
|
|
@@ -3919,8 +4434,16 @@ var HOP_HEADERS = /* @__PURE__ */ new Set([
|
|
|
3919
4434
|
"te",
|
|
3920
4435
|
"trailer",
|
|
3921
4436
|
"keep-alive",
|
|
3922
|
-
"proxy-authorization"
|
|
4437
|
+
"proxy-authorization",
|
|
4438
|
+
// `Expect: 100-continue` is settled between the client and this proxy (Node
|
|
4439
|
+
// answers it); forwarding it makes the upstream call fail outright. curl adds
|
|
4440
|
+
// it to every body over 1 MB.
|
|
4441
|
+
"expect",
|
|
4442
|
+
// Cookies belong to whatever else runs on this machine's localhost (cookies
|
|
4443
|
+
// are scoped by host, not port). No provider API authenticates with one.
|
|
4444
|
+
"cookie"
|
|
3923
4445
|
]);
|
|
4446
|
+
var RESPONSE_DROP_HEADERS = /* @__PURE__ */ new Set([...HOP_HEADERS, "content-encoding", "set-cookie"]);
|
|
3924
4447
|
function forwardHeaders(req, provider) {
|
|
3925
4448
|
const out = {};
|
|
3926
4449
|
for (const [name, value] of Object.entries(req.headers)) {
|
|
@@ -3969,6 +4492,16 @@ function surfaceOf(path) {
|
|
|
3969
4492
|
if (path === "/v1/chat/completions") return { provider: "openai", upstreamPath: "/v1/chat/completions" };
|
|
3970
4493
|
return void 0;
|
|
3971
4494
|
}
|
|
4495
|
+
function passthroughProvider(req, path) {
|
|
4496
|
+
if (req.headers["anthropic-version"] !== void 0 || req.headers["anthropic-beta"] !== void 0) return "anthropic";
|
|
4497
|
+
if (req.headers["x-api-key"] !== void 0) return "anthropic";
|
|
4498
|
+
if (path === "/v1/messages" || path.startsWith("/v1/messages/") || path === "/v1/complete") return "anthropic";
|
|
4499
|
+
return path.startsWith("/v1/") ? "openai" : void 0;
|
|
4500
|
+
}
|
|
4501
|
+
function meshError(message) {
|
|
4502
|
+
return JSON.stringify({ type: "error", error: { type: "mesh_proxy_error", message } });
|
|
4503
|
+
}
|
|
4504
|
+
var PASSTHROUGH_BUFFER_CAP = 16 * 1024 * 1024;
|
|
3972
4505
|
function usageFromSse(provider, sseText) {
|
|
3973
4506
|
const events = [];
|
|
3974
4507
|
for (const line of sseText.split(/\r?\n/)) {
|
|
@@ -4027,8 +4560,22 @@ function outputTextFromSse(provider, sseText) {
|
|
|
4027
4560
|
return parts.join("");
|
|
4028
4561
|
}
|
|
4029
4562
|
function startMeshProxy(options) {
|
|
4030
|
-
const
|
|
4031
|
-
const
|
|
4563
|
+
const reductionOption = options.tokenReduction;
|
|
4564
|
+
const resolveBaseReduction = typeof reductionOption === "function" ? reductionOption : () => reductionOption ?? LOSSLESS_DEFAULT;
|
|
4565
|
+
let volatilityGuard;
|
|
4566
|
+
const currentReduction = (modelId) => {
|
|
4567
|
+
let base;
|
|
4568
|
+
try {
|
|
4569
|
+
base = resolveBaseReduction(modelId) ?? LOSSLESS_DEFAULT;
|
|
4570
|
+
} catch {
|
|
4571
|
+
base = LOSSLESS_DEFAULT;
|
|
4572
|
+
}
|
|
4573
|
+
if (base.cacheTtl !== void 0 && base.cacheVolatilityGuard === void 0) {
|
|
4574
|
+
volatilityGuard ??= new CachePrefixVolatilityGuard();
|
|
4575
|
+
return { ...base, cacheVolatilityGuard: volatilityGuard };
|
|
4576
|
+
}
|
|
4577
|
+
return base;
|
|
4578
|
+
};
|
|
4032
4579
|
const upstreams = {
|
|
4033
4580
|
anthropic: (options.anthropicUpstream ?? "https://api.anthropic.com").replace(/\/$/, ""),
|
|
4034
4581
|
openai: (options.openaiUpstream ?? "https://api.openai.com").replace(/\/$/, "")
|
|
@@ -4054,7 +4601,8 @@ ${provider}`;
|
|
|
4054
4601
|
}
|
|
4055
4602
|
const ledger = createSavingsLedger({
|
|
4056
4603
|
path: (0, import_node_path3.join)(options.ledgerDirectory, `${safeApp}-${provider}.jsonl`),
|
|
4057
|
-
provider
|
|
4604
|
+
provider,
|
|
4605
|
+
...options.meshApiUrl && options.meshAccountKey ? { mirror: { apiUrl: options.meshApiUrl, accountKey: options.meshAccountKey } } : {}
|
|
4058
4606
|
});
|
|
4059
4607
|
ledgers.set(key, ledger);
|
|
4060
4608
|
return ledger;
|
|
@@ -4066,55 +4614,140 @@ ${provider}`;
|
|
|
4066
4614
|
(0, import_node_fs2.appendFileSync)((0, import_node_path3.join)(options.ledgerDirectory, `${safeApp}-${provider}.features.jsonl`), `${JSON.stringify(record)}
|
|
4067
4615
|
`, { mode: 384 });
|
|
4068
4616
|
};
|
|
4617
|
+
const requestCapture = options.requestCapture;
|
|
4618
|
+
const captureLimit = requestCapture?.limit ?? 50;
|
|
4619
|
+
const captureMaxBytes = requestCapture?.maxBytes ?? 5 * 1024 * 1024;
|
|
4620
|
+
let capturedRequests = 0;
|
|
4621
|
+
let capturedBytes = (() => {
|
|
4622
|
+
if (requestCapture === void 0) return 0;
|
|
4623
|
+
try {
|
|
4624
|
+
return (0, import_node_fs2.existsSync)(requestCapture.path) ? (0, import_node_fs2.statSync)(requestCapture.path).size : 0;
|
|
4625
|
+
} catch {
|
|
4626
|
+
return 0;
|
|
4627
|
+
}
|
|
4628
|
+
})();
|
|
4629
|
+
const writeCapturedRequest = (body) => {
|
|
4630
|
+
if (requestCapture === void 0 || body === void 0 || capturedRequests >= captureLimit) return;
|
|
4631
|
+
try {
|
|
4632
|
+
const line = `${JSON.stringify({ request: body })}
|
|
4633
|
+
`;
|
|
4634
|
+
const lineBytes = Buffer.byteLength(line);
|
|
4635
|
+
if (capturedBytes + lineBytes > captureMaxBytes) return;
|
|
4636
|
+
(0, import_node_fs2.mkdirSync)((0, import_node_path3.dirname)(requestCapture.path), { recursive: true, mode: 448 });
|
|
4637
|
+
(0, import_node_fs2.appendFileSync)(requestCapture.path, line, { mode: 384 });
|
|
4638
|
+
capturedRequests += 1;
|
|
4639
|
+
capturedBytes += lineBytes;
|
|
4640
|
+
} catch {
|
|
4641
|
+
}
|
|
4642
|
+
};
|
|
4069
4643
|
const handle = async (req, res) => {
|
|
4070
4644
|
const path = (req.url ?? "").split("?")[0];
|
|
4071
4645
|
if (!hostAllowed(req)) {
|
|
4072
4646
|
res.writeHead(403, { "content-type": "application/json" });
|
|
4073
|
-
res.end(
|
|
4647
|
+
res.end(meshError("MESH proxy serves loopback clients only."));
|
|
4074
4648
|
return;
|
|
4075
4649
|
}
|
|
4076
4650
|
const fetchSite = req.headers["sec-fetch-site"];
|
|
4077
4651
|
if (fetchSite === "cross-site" || fetchSite === "same-site") {
|
|
4078
4652
|
res.writeHead(403, { "content-type": "application/json" });
|
|
4079
|
-
res.end(
|
|
4653
|
+
res.end(meshError("MESH proxy does not serve cross-origin browser requests."));
|
|
4654
|
+
return;
|
|
4655
|
+
}
|
|
4656
|
+
const surface = req.method === "POST" ? surfaceOf(path) : void 0;
|
|
4657
|
+
if (path === "/mesh/health") {
|
|
4658
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
4659
|
+
res.end(req.method === "HEAD" ? void 0 : JSON.stringify({ status: "ok", service: "mesh-proxy" }));
|
|
4660
|
+
return;
|
|
4661
|
+
}
|
|
4662
|
+
const passProvider = surface === void 0 ? passthroughProvider(req, path) : void 0;
|
|
4663
|
+
if (surface === void 0 && (path === "/" || path === "" || passProvider === void 0)) {
|
|
4664
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
4665
|
+
res.end(meshError(
|
|
4666
|
+
"This is the MESH proxy. Point your client's base URL at this server: Anthropic clients at the bare address, OpenAI-style clients at /v1. MESH reduces POST /v1/messages, /v1/responses and /v1/chat/completions and forwards every other request to your provider unchanged."
|
|
4667
|
+
));
|
|
4080
4668
|
return;
|
|
4081
4669
|
}
|
|
4082
|
-
if (
|
|
4083
|
-
const provider =
|
|
4670
|
+
if (surface === void 0 && passProvider !== void 0) {
|
|
4671
|
+
const provider = passProvider;
|
|
4672
|
+
const hasBody = req.method !== "GET" && req.method !== "HEAD";
|
|
4673
|
+
const declaredLength = Number.parseInt(String(req.headers["content-length"] ?? ""), 10);
|
|
4674
|
+
const passControl = new AbortController();
|
|
4675
|
+
const onPassClose = () => {
|
|
4676
|
+
if (!res.writableEnded) passControl.abort(new Error("client disconnected"));
|
|
4677
|
+
};
|
|
4678
|
+
res.on("close", onPassClose);
|
|
4084
4679
|
let upstreamResponse2;
|
|
4085
4680
|
try {
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4681
|
+
const passRequestHeaders = forwardHeaders(req, provider);
|
|
4682
|
+
let init = {
|
|
4683
|
+
method: req.method ?? "GET",
|
|
4684
|
+
headers: passRequestHeaders,
|
|
4685
|
+
signal: passControl.signal,
|
|
4686
|
+
// A redirect is the client's to follow. Followed here, the caller's
|
|
4687
|
+
// key would travel to wherever the provider pointed.
|
|
4688
|
+
redirect: "manual"
|
|
4689
|
+
};
|
|
4690
|
+
if (hasBody) {
|
|
4691
|
+
if (Number.isFinite(declaredLength) && declaredLength <= PASSTHROUGH_BUFFER_CAP) {
|
|
4692
|
+
init = { ...init, body: new Uint8Array(await readBody(req)) };
|
|
4693
|
+
} else {
|
|
4694
|
+
if (Number.isFinite(declaredLength)) passRequestHeaders["content-length"] = String(declaredLength);
|
|
4695
|
+
init = { ...init, body: import_node_stream.Readable.toWeb(req), duplex: "half" };
|
|
4696
|
+
}
|
|
4697
|
+
}
|
|
4698
|
+
upstreamResponse2 = await fetch(`${upstreams[provider]}${req.url ?? path}`, init);
|
|
4090
4699
|
} catch (cause) {
|
|
4091
|
-
res.
|
|
4092
|
-
res.
|
|
4700
|
+
res.off("close", onPassClose);
|
|
4701
|
+
if (!res.headersSent && !res.destroyed) {
|
|
4702
|
+
res.writeHead(502, { "content-type": "application/json" });
|
|
4703
|
+
res.end(meshError(`Upstream unreachable: ${cause.message}`));
|
|
4704
|
+
}
|
|
4093
4705
|
return;
|
|
4094
4706
|
}
|
|
4095
4707
|
const passHeaders = {};
|
|
4096
4708
|
upstreamResponse2.headers.forEach((value, name) => {
|
|
4097
|
-
if (!
|
|
4709
|
+
if (!RESPONSE_DROP_HEADERS.has(name.toLowerCase())) passHeaders[name] = value;
|
|
4098
4710
|
});
|
|
4099
4711
|
res.writeHead(upstreamResponse2.status, passHeaders);
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4712
|
+
if (upstreamResponse2.body !== null) {
|
|
4713
|
+
const reader = upstreamResponse2.body.getReader();
|
|
4714
|
+
try {
|
|
4715
|
+
for (; ; ) {
|
|
4716
|
+
const { done, value } = await reader.read();
|
|
4717
|
+
if (done || res.destroyed) break;
|
|
4718
|
+
if (!res.write(Buffer.from(value))) {
|
|
4719
|
+
await new Promise((proceed) => {
|
|
4720
|
+
const onDrain = () => {
|
|
4721
|
+
res.off("close", onGone);
|
|
4722
|
+
proceed();
|
|
4723
|
+
};
|
|
4724
|
+
const onGone = () => {
|
|
4725
|
+
res.off("drain", onDrain);
|
|
4726
|
+
proceed();
|
|
4727
|
+
};
|
|
4728
|
+
res.once("drain", onDrain);
|
|
4729
|
+
res.once("close", onGone);
|
|
4730
|
+
});
|
|
4731
|
+
}
|
|
4732
|
+
}
|
|
4733
|
+
} catch {
|
|
4734
|
+
res.off("close", onPassClose);
|
|
4735
|
+
if (!res.destroyed) res.destroy();
|
|
4736
|
+
return;
|
|
4737
|
+
}
|
|
4738
|
+
}
|
|
4739
|
+
res.off("close", onPassClose);
|
|
4740
|
+
if (!res.destroyed) res.end();
|
|
4109
4741
|
return;
|
|
4110
4742
|
}
|
|
4743
|
+
if (surface === void 0) return;
|
|
4111
4744
|
let rawBody;
|
|
4112
4745
|
try {
|
|
4113
4746
|
rawBody = await readBody(req);
|
|
4114
4747
|
} catch (cause) {
|
|
4115
4748
|
if (cause instanceof BodyTooLargeError && !res.headersSent) {
|
|
4116
4749
|
res.writeHead(413, { "content-type": "application/json" });
|
|
4117
|
-
res.end(
|
|
4750
|
+
res.end(meshError(cause.message));
|
|
4118
4751
|
}
|
|
4119
4752
|
return;
|
|
4120
4753
|
}
|
|
@@ -4124,17 +4757,22 @@ ${provider}`;
|
|
|
4124
4757
|
} catch {
|
|
4125
4758
|
body = void 0;
|
|
4126
4759
|
}
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4760
|
+
const readable = typeof body === "object" && body !== null && !Array.isArray(body);
|
|
4761
|
+
if (!readable) body = void 0;
|
|
4762
|
+
const requestModelId = typeof body?.model === "string" ? body.model : void 0;
|
|
4763
|
+
const { request: reducedRequest, summary } = reduceRequest(body, currentReduction(requestModelId));
|
|
4764
|
+
let requestGateVerdicts;
|
|
4765
|
+
try {
|
|
4766
|
+
requestGateVerdicts = options.gateVerdicts?.(requestModelId);
|
|
4767
|
+
} catch {
|
|
4768
|
+
requestGateVerdicts = void 0;
|
|
4131
4769
|
}
|
|
4132
|
-
const
|
|
4133
|
-
const sendBody = JSON.stringify(
|
|
4770
|
+
const reducedText = JSON.stringify(reducedRequest);
|
|
4771
|
+
const sendBody = !readable || reducedText === JSON.stringify(body) ? new Uint8Array(rawBody) : reducedText;
|
|
4134
4772
|
const applicationIdHeader = req.headers["x-mesh-application"];
|
|
4135
4773
|
const applicationId = ((Array.isArray(applicationIdHeader) ? applicationIdHeader[0] : applicationIdHeader) ?? "proxy").slice(0, 128);
|
|
4136
4774
|
const headers = forwardHeaders(req, surface.provider);
|
|
4137
|
-
headers["content-type"] = "application/json";
|
|
4775
|
+
if (readable) headers["content-type"] = "application/json";
|
|
4138
4776
|
const upstreamControl = new AbortController();
|
|
4139
4777
|
const abortUpstream = (reason) => upstreamControl.abort(new Error(reason));
|
|
4140
4778
|
const onClientClose = () => {
|
|
@@ -4158,24 +4796,27 @@ ${provider}`;
|
|
|
4158
4796
|
const startedAt = Date.now();
|
|
4159
4797
|
let upstreamResponse;
|
|
4160
4798
|
try {
|
|
4161
|
-
|
|
4799
|
+
const query = (req.url ?? "").includes("?") ? (req.url ?? "").slice((req.url ?? "").indexOf("?")) : "";
|
|
4800
|
+
upstreamResponse = await fetch(`${upstreams[surface.provider]}${surface.upstreamPath}${query}`, {
|
|
4162
4801
|
method: "POST",
|
|
4163
4802
|
headers,
|
|
4164
4803
|
body: sendBody,
|
|
4165
|
-
signal: upstreamControl.signal
|
|
4804
|
+
signal: upstreamControl.signal,
|
|
4805
|
+
redirect: "manual"
|
|
4806
|
+
// never carry the caller's key to a redirect target
|
|
4166
4807
|
});
|
|
4167
4808
|
} catch (cause) {
|
|
4168
4809
|
clearTimeout(phaseTimer);
|
|
4169
4810
|
if (!res.headersSent && !res.destroyed) {
|
|
4170
4811
|
res.writeHead(502, { "content-type": "application/json" });
|
|
4171
|
-
res.end(
|
|
4812
|
+
res.end(meshError(`Upstream unreachable: ${cause.message}`));
|
|
4172
4813
|
}
|
|
4173
4814
|
return;
|
|
4174
4815
|
}
|
|
4175
4816
|
clearTimeout(phaseTimer);
|
|
4176
4817
|
const responseHeaders = {};
|
|
4177
4818
|
upstreamResponse.headers.forEach((value, name) => {
|
|
4178
|
-
if (!
|
|
4819
|
+
if (!RESPONSE_DROP_HEADERS.has(name.toLowerCase())) responseHeaders[name] = value;
|
|
4179
4820
|
});
|
|
4180
4821
|
if (res.destroyed) {
|
|
4181
4822
|
abortUpstream("client disconnected");
|
|
@@ -4186,6 +4827,7 @@ ${provider}`;
|
|
|
4186
4827
|
let accumulated = 0;
|
|
4187
4828
|
const ACCUMULATION_CAP = 32 * 1024 * 1024;
|
|
4188
4829
|
let clientLost = false;
|
|
4830
|
+
let upstreamBroke = false;
|
|
4189
4831
|
if (upstreamResponse.body !== null) {
|
|
4190
4832
|
const reader = upstreamResponse.body.getReader();
|
|
4191
4833
|
try {
|
|
@@ -4227,16 +4869,18 @@ ${provider}`;
|
|
|
4227
4869
|
}
|
|
4228
4870
|
} catch {
|
|
4229
4871
|
clientLost = res.destroyed;
|
|
4872
|
+
upstreamBroke = !res.destroyed;
|
|
4230
4873
|
}
|
|
4231
4874
|
}
|
|
4232
4875
|
res.off("close", onClientClose);
|
|
4233
|
-
if (
|
|
4876
|
+
if (upstreamBroke && !res.destroyed) res.destroy();
|
|
4877
|
+
else if (!clientLost && !res.destroyed) res.end();
|
|
4234
4878
|
try {
|
|
4235
4879
|
const text = Buffer.concat(chunks).toString("utf8");
|
|
4236
4880
|
const contentType = upstreamResponse.headers.get("content-type") ?? "";
|
|
4237
4881
|
if (process.env.MESH_PROXY_DEBUG === "1" && !upstreamResponse.ok) {
|
|
4238
4882
|
console.warn(
|
|
4239
|
-
`[mesh-debug] upstream ${upstreamResponse.status} model=${JSON.stringify(body
|
|
4883
|
+
`[mesh-debug] upstream ${upstreamResponse.status} model=${JSON.stringify(body?.model)} path=${path} anthropic-beta=${JSON.stringify(req.headers["anthropic-beta"] ?? null)} errbody=${text.slice(0, 500)}`
|
|
4240
4884
|
);
|
|
4241
4885
|
}
|
|
4242
4886
|
let responseShaped;
|
|
@@ -4256,14 +4900,14 @@ ${provider}`;
|
|
|
4256
4900
|
if (usage !== void 0 && usage.outputTokens >= 200 && responseShaped !== void 0) {
|
|
4257
4901
|
const outputText = contentType.includes("text/event-stream") ? outputTextFromSse(surface.provider, text) : responseText(responseShaped);
|
|
4258
4902
|
if (outputText.length > 0) {
|
|
4259
|
-
outputDiagnostics = measureOutputReEmission(collectRequestText(body), outputText);
|
|
4903
|
+
outputDiagnostics = measureOutputReEmission(collectRequestText(body ?? {}), outputText);
|
|
4260
4904
|
}
|
|
4261
4905
|
}
|
|
4262
4906
|
const event = {
|
|
4263
4907
|
requestId,
|
|
4264
4908
|
applicationId,
|
|
4265
4909
|
provider: `${surface.provider}-compatible`,
|
|
4266
|
-
model: typeof body
|
|
4910
|
+
model: typeof body?.model === "string" ? body.model : "unknown",
|
|
4267
4911
|
latencyMs: Date.now() - startedAt,
|
|
4268
4912
|
status: upstreamResponse.ok ? "succeeded" : "failed",
|
|
4269
4913
|
timestamp: new Date(startedAt).toISOString(),
|
|
@@ -4283,8 +4927,11 @@ ${provider}`;
|
|
|
4283
4927
|
...outputDiagnostics === void 0 ? {} : { outputDiagnostics },
|
|
4284
4928
|
...duplicateOfRequestId === void 0 ? {} : { duplicateOfRequestId }
|
|
4285
4929
|
};
|
|
4286
|
-
ledgerFor(applicationId, surface.provider).emit(
|
|
4287
|
-
|
|
4930
|
+
ledgerFor(applicationId, surface.provider).emit(
|
|
4931
|
+
event,
|
|
4932
|
+
requestGateVerdicts === void 0 ? void 0 : { gateVerdicts: requestGateVerdicts }
|
|
4933
|
+
);
|
|
4934
|
+
if (blindFeaturesKey !== void 0 && body !== void 0) {
|
|
4288
4935
|
const features = extractBlindFeatures(body, { hmacKey: blindFeaturesKey });
|
|
4289
4936
|
writeBlindFeatures(applicationId, surface.provider, {
|
|
4290
4937
|
requestId,
|
|
@@ -4293,6 +4940,7 @@ ${provider}`;
|
|
|
4293
4940
|
features
|
|
4294
4941
|
});
|
|
4295
4942
|
}
|
|
4943
|
+
if (body !== void 0) writeCapturedRequest(body);
|
|
4296
4944
|
} catch {
|
|
4297
4945
|
}
|
|
4298
4946
|
};
|
|
@@ -4300,7 +4948,7 @@ ${provider}`;
|
|
|
4300
4948
|
handle(req, res).catch(() => {
|
|
4301
4949
|
if (!res.headersSent) {
|
|
4302
4950
|
res.writeHead(500, { "content-type": "application/json" });
|
|
4303
|
-
res.end(
|
|
4951
|
+
res.end(meshError("MESH proxy internal error."));
|
|
4304
4952
|
} else {
|
|
4305
4953
|
res.end();
|
|
4306
4954
|
}
|
|
@@ -4344,10 +4992,13 @@ function wrapMesh(client, options = {}) {
|
|
|
4344
4992
|
...options.tokenReduction ?? LOSSLESS_DEFAULT2,
|
|
4345
4993
|
...options.cacheTtl ? { cacheTtl: options.cacheTtl } : {}
|
|
4346
4994
|
};
|
|
4995
|
+
const meshApiUrl = options.meshApiUrl ?? process.env.MESH_API_URL;
|
|
4996
|
+
const meshAccountKey = options.meshAccountKey ?? process.env.MESH_ACCOUNT_KEY;
|
|
4997
|
+
const mirror = meshApiUrl && meshAccountKey ? { apiUrl: meshApiUrl, accountKey: meshAccountKey } : void 0;
|
|
4347
4998
|
const wrapped = withMesh(client, {
|
|
4348
4999
|
applicationId: app,
|
|
4349
5000
|
tokenReduction,
|
|
4350
|
-
ledger: { path: ledgerPath, provider }
|
|
5001
|
+
ledger: { path: ledgerPath, provider, ...mirror ? { mirror } : {} }
|
|
4351
5002
|
});
|
|
4352
5003
|
const mesh = {
|
|
4353
5004
|
ledgerPath,
|
|
@@ -4387,10 +5038,15 @@ function wrapMesh(client, options = {}) {
|
|
|
4387
5038
|
}
|
|
4388
5039
|
|
|
4389
5040
|
// packages/sdk-js/src/theo-policy.ts
|
|
5041
|
+
var MAX_QUALITY_CEILING_PERCENT = 5;
|
|
5042
|
+
function effectiveQualityCeiling(ceilingPercent) {
|
|
5043
|
+
const c = Number.isFinite(ceilingPercent) ? ceilingPercent : 0;
|
|
5044
|
+
return Math.max(0, Math.min(c, MAX_QUALITY_CEILING_PERCENT));
|
|
5045
|
+
}
|
|
4390
5046
|
function proofClearsCeiling(lever, ceilingPercent) {
|
|
4391
5047
|
if (lever.gate === void 0) return false;
|
|
4392
5048
|
const y = lever.gate.qualityDeltaPercent;
|
|
4393
|
-
return Number.isFinite(y) && Math.abs(y) <= ceilingPercent;
|
|
5049
|
+
return Number.isFinite(y) && Math.abs(y) <= effectiveQualityCeiling(ceilingPercent);
|
|
4394
5050
|
}
|
|
4395
5051
|
function applyLever(config, lever) {
|
|
4396
5052
|
const p = lever.params ?? {};
|
|
@@ -4404,6 +5060,11 @@ function applyLever(config, lever) {
|
|
|
4404
5060
|
...typeof p.maximumDescriptionTokens === "number" ? { maximumDescriptionTokens: p.maximumDescriptionTokens } : {}
|
|
4405
5061
|
};
|
|
4406
5062
|
return;
|
|
5063
|
+
case "unused-tool-removal": {
|
|
5064
|
+
const names = Array.isArray(p.tools) ? p.tools.filter((name) => typeof name === "string") : [];
|
|
5065
|
+
if (names.length > 0) config.dropTools = names;
|
|
5066
|
+
return;
|
|
5067
|
+
}
|
|
4407
5068
|
case "duplicate-context-removal":
|
|
4408
5069
|
config.duplicateContext = true;
|
|
4409
5070
|
return;
|
|
@@ -4450,6 +5111,51 @@ function policyToReductionConfig(policy) {
|
|
|
4450
5111
|
}
|
|
4451
5112
|
return config;
|
|
4452
5113
|
}
|
|
5114
|
+
function leverToReductionConfig(lever) {
|
|
5115
|
+
const config = {};
|
|
5116
|
+
applyLever(config, lever);
|
|
5117
|
+
return config;
|
|
5118
|
+
}
|
|
5119
|
+
function routeMatches(pattern, modelId) {
|
|
5120
|
+
if (pattern === "*" || pattern === "") return true;
|
|
5121
|
+
if (pattern.endsWith("*")) return modelId.startsWith(pattern.slice(0, -1));
|
|
5122
|
+
return pattern === modelId;
|
|
5123
|
+
}
|
|
5124
|
+
function isPolicyBundle(value) {
|
|
5125
|
+
return typeof value === "object" && value !== null && "default" in value && typeof value.default === "object";
|
|
5126
|
+
}
|
|
5127
|
+
function asBundle(stored) {
|
|
5128
|
+
if (isPolicyBundle(stored)) return stored;
|
|
5129
|
+
const policy = stored;
|
|
5130
|
+
return { schemaVersion: 1, user: policy.user, default: policy, updatedAt: policy.updatedAt };
|
|
5131
|
+
}
|
|
5132
|
+
function policyForRoute(bundle, modelId) {
|
|
5133
|
+
for (const route of bundle.routes ?? []) {
|
|
5134
|
+
if (route.route !== void 0 && routeMatches(route.route, modelId)) return route;
|
|
5135
|
+
}
|
|
5136
|
+
return bundle.default;
|
|
5137
|
+
}
|
|
5138
|
+
function bundleToReductionConfig(bundle, modelId) {
|
|
5139
|
+
return policyToReductionConfig(policyForRoute(bundle, modelId));
|
|
5140
|
+
}
|
|
5141
|
+
function upsertRoutePolicy(bundle, policy, at) {
|
|
5142
|
+
if (policy.route === void 0 || policy.route === "") {
|
|
5143
|
+
throw new Error("Refusing to add a route policy without a `route` matcher.");
|
|
5144
|
+
}
|
|
5145
|
+
const routes = (bundle.routes ?? []).filter((existing) => existing.route !== policy.route);
|
|
5146
|
+
routes.push(policy);
|
|
5147
|
+
return { ...bundle, routes, updatedAt: at };
|
|
5148
|
+
}
|
|
5149
|
+
function mergePolicyIntoBundle(existing, policy, route, at) {
|
|
5150
|
+
if (route !== void 0 && route !== "") {
|
|
5151
|
+
const base2 = existing ?? asBundle(defaultPolicy(policy.user, at));
|
|
5152
|
+
return upsertRoutePolicy(base2, { ...policy, route }, at);
|
|
5153
|
+
}
|
|
5154
|
+
const base = existing ?? asBundle(policy);
|
|
5155
|
+
const def = { ...policy };
|
|
5156
|
+
delete def.route;
|
|
5157
|
+
return { ...base, default: def, updatedAt: at };
|
|
5158
|
+
}
|
|
4453
5159
|
function withheldLevers(policy) {
|
|
4454
5160
|
const out = [];
|
|
4455
5161
|
for (const lever of policy.levers) {
|
|
@@ -4492,6 +5198,7 @@ function knownLeverIds() {
|
|
|
4492
5198
|
return [
|
|
4493
5199
|
"tool-schema-compaction",
|
|
4494
5200
|
"tool-schema-description-trim",
|
|
5201
|
+
"unused-tool-removal",
|
|
4495
5202
|
"duplicate-context-removal",
|
|
4496
5203
|
"cache-prefix-repair",
|
|
4497
5204
|
"stale-history-compaction",
|
|
@@ -4508,6 +5215,683 @@ function knownLeverIds() {
|
|
|
4508
5215
|
"output-effort-max"
|
|
4509
5216
|
];
|
|
4510
5217
|
}
|
|
5218
|
+
var ENFORCEABLE_LEVERS = [
|
|
5219
|
+
{ id: "tool-schema-compaction", label: "Tool-schema compaction", members: ["tool-schema-compaction"] },
|
|
5220
|
+
{ id: "tool-schema-description-trim", label: "Tool-schema description trim", members: ["tool-schema-description-trim"] },
|
|
5221
|
+
{ id: "unused-tool-removal", label: "Unused-tool removal", members: ["unused-tool-removal"] },
|
|
5222
|
+
{ id: "duplicate-context-removal", label: "Duplicate-context removal", members: ["duplicate-context-removal"] },
|
|
5223
|
+
{ id: "cache-prefix-repair", label: "Cache-prefix repair", members: ["cache-prefix-repair"] },
|
|
5224
|
+
{ id: "stale-history-compaction", label: "Stale-history compaction", members: ["stale-history-compaction"] },
|
|
5225
|
+
{ id: "cross-turn-memory-dedup", label: "Cross-turn memory dedup", members: ["cross-turn-memory-dedup"] },
|
|
5226
|
+
{ id: "fresh-tool-result-cap", label: "Fresh tool-result cap", members: ["fresh-tool-result-cap"] },
|
|
5227
|
+
{ id: "output-cap", label: "Output cap", members: ["output-cap"] },
|
|
5228
|
+
{ id: "execution-tier-flex", label: "Execution-tier flex", members: ["execution-tier-flex"] },
|
|
5229
|
+
{ id: "cache-ttl", label: "Cache TTL", members: ["cache-ttl-5m", "cache-ttl-1h"] },
|
|
5230
|
+
{
|
|
5231
|
+
id: "output-effort",
|
|
5232
|
+
label: "Output effort",
|
|
5233
|
+
members: ["output-effort-low", "output-effort-medium", "output-effort-high", "output-effort-xhigh", "output-effort-max"]
|
|
5234
|
+
}
|
|
5235
|
+
];
|
|
5236
|
+
function policyCoverage(policy) {
|
|
5237
|
+
const byId = new Map(policy.levers.map((lever) => [lever.lever, lever]));
|
|
5238
|
+
const heldById = new Map(withheldLevers(policy).map((withheld) => [withheld.lever, withheld.reason]));
|
|
5239
|
+
const stands = ENFORCEABLE_LEVERS.map((group) => {
|
|
5240
|
+
let state = "available";
|
|
5241
|
+
let reason;
|
|
5242
|
+
let leverClass;
|
|
5243
|
+
for (const member of group.members) {
|
|
5244
|
+
const lever = byId.get(member);
|
|
5245
|
+
if (lever === void 0) continue;
|
|
5246
|
+
leverClass = lever.class;
|
|
5247
|
+
const enforced = lever.class === "lossless" || proofClearsCeiling(lever, policy.qualityCeilingPercent);
|
|
5248
|
+
if (enforced) {
|
|
5249
|
+
state = "active";
|
|
5250
|
+
reason = void 0;
|
|
5251
|
+
break;
|
|
5252
|
+
}
|
|
5253
|
+
state = "held";
|
|
5254
|
+
reason = heldById.get(member);
|
|
5255
|
+
}
|
|
5256
|
+
return {
|
|
5257
|
+
lever: group.id,
|
|
5258
|
+
label: group.label,
|
|
5259
|
+
state,
|
|
5260
|
+
...leverClass ? { class: leverClass } : {},
|
|
5261
|
+
...reason ? { reason } : {}
|
|
5262
|
+
};
|
|
5263
|
+
});
|
|
5264
|
+
const activeCount = stands.filter((stand) => stand.state === "active").length;
|
|
5265
|
+
const heldCount = stands.filter((stand) => stand.state === "held").length;
|
|
5266
|
+
const availableCount = stands.filter((stand) => stand.state === "available").length;
|
|
5267
|
+
const enforceableCount = ENFORCEABLE_LEVERS.length;
|
|
5268
|
+
return {
|
|
5269
|
+
enforceableCount,
|
|
5270
|
+
activeCount,
|
|
5271
|
+
heldCount,
|
|
5272
|
+
availableCount,
|
|
5273
|
+
activeCoveragePercent: enforceableCount === 0 ? 0 : activeCount / enforceableCount * 100,
|
|
5274
|
+
stands
|
|
5275
|
+
};
|
|
5276
|
+
}
|
|
5277
|
+
|
|
5278
|
+
// packages/sdk-js/src/theo-agent.ts
|
|
5279
|
+
var ENFORCEABLE_BY_CONCEPT = {
|
|
5280
|
+
"tool-schema-compaction": { lever: "tool-schema-compaction", class: "lossless" },
|
|
5281
|
+
"duplicate-context-removal": { lever: "duplicate-context-removal", class: "lossless" },
|
|
5282
|
+
"tool-schema-description-trim": { lever: "tool-schema-description-trim", class: "lossy" },
|
|
5283
|
+
"history-compaction": { lever: "stale-history-compaction", class: "lossy" },
|
|
5284
|
+
"output-budget": { lever: "output-effort-low", class: "lossy" }
|
|
5285
|
+
};
|
|
5286
|
+
var ALREADY_COVERED_CONCEPTS = /* @__PURE__ */ new Set(["prefix-caching"]);
|
|
5287
|
+
var OPERATOR_CONFIGURED_CONCEPTS = {
|
|
5288
|
+
"unused-tool-removal": "An enforceable lever now exists (`unused-tool-removal`). Theo is content-blind and cannot name the tools, so enable it with the tool names your workload never calls \u2014 the replay gate proves it on your traffic before billing."
|
|
5289
|
+
};
|
|
5290
|
+
var ENFORCEABLE_LOSSY = new Set(
|
|
5291
|
+
Object.values(ENFORCEABLE_BY_CONCEPT).filter((entry2) => entry2.class === "lossy").map((entry2) => entry2.lever)
|
|
5292
|
+
);
|
|
5293
|
+
var DRIFT_RISE_THRESHOLD = 0.05;
|
|
5294
|
+
var DRIFT_WINDOW_EVENTS = 200;
|
|
5295
|
+
function recentDriftRates(events, windowEvents = DRIFT_WINDOW_EVENTS) {
|
|
5296
|
+
const size = Math.max(1, Math.floor(windowEvents));
|
|
5297
|
+
const recent = [...events].sort((a, b) => {
|
|
5298
|
+
const left = a.timestamp ?? "";
|
|
5299
|
+
const right = b.timestamp ?? "";
|
|
5300
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
5301
|
+
}).slice(-size);
|
|
5302
|
+
const n = recent.length;
|
|
5303
|
+
if (n === 0) return { sampleSize: 0, retryRate: 0, failureRate: 0 };
|
|
5304
|
+
return {
|
|
5305
|
+
sampleSize: n,
|
|
5306
|
+
retryRate: recent.filter((event) => event.duplicateOfRequestId !== void 0).length / n,
|
|
5307
|
+
failureRate: recent.filter((event) => event.status === "failed").length / n
|
|
5308
|
+
};
|
|
5309
|
+
}
|
|
5310
|
+
function zeroSegments() {
|
|
5311
|
+
return { toolSchemas: 0, systemPrompt: 0, retrievedContext: 0, history: 0, userTurn: 0, output: 0 };
|
|
5312
|
+
}
|
|
5313
|
+
function observeWorkload(features, options = {}) {
|
|
5314
|
+
const name = options.name ?? "observed";
|
|
5315
|
+
const n = features.length;
|
|
5316
|
+
if (n === 0) {
|
|
5317
|
+
return { name, requests: 1, segments: zeroSegments(), stableSegments: [] };
|
|
5318
|
+
}
|
|
5319
|
+
const mean2 = (nums) => nums.reduce((sum, value) => sum + value, 0) / nums.length;
|
|
5320
|
+
const segments = {
|
|
5321
|
+
toolSchemas: mean2(features.map((f) => f.toolSchemaTokensApprox)),
|
|
5322
|
+
systemPrompt: mean2(features.map((f) => f.roleTokensApprox.system ?? 0)),
|
|
5323
|
+
retrievedContext: 0,
|
|
5324
|
+
history: mean2(features.map((f) => (f.roleTokensApprox.assistant ?? 0) + (f.roleTokensApprox.tool ?? 0))),
|
|
5325
|
+
userTurn: mean2(features.map((f) => f.roleTokensApprox.user ?? 0)),
|
|
5326
|
+
output: Math.max(0, options.outputTokensMean ?? 0)
|
|
5327
|
+
};
|
|
5328
|
+
const prefixReuse = n > 1 && new Set(features.map((f) => f.stablePrefixHash)).size === 1;
|
|
5329
|
+
const stableSegments = prefixReuse ? ["toolSchemas", "systemPrompt"] : [];
|
|
5330
|
+
return { name, requests: n, segments, stableSegments };
|
|
5331
|
+
}
|
|
5332
|
+
function createTheoMemory(customer, autonomy, at) {
|
|
5333
|
+
return {
|
|
5334
|
+
schemaVersion: 1,
|
|
5335
|
+
customer,
|
|
5336
|
+
autonomy,
|
|
5337
|
+
policy: defaultPolicy(customer, at),
|
|
5338
|
+
hypotheses: [],
|
|
5339
|
+
frontier: [],
|
|
5340
|
+
experiments: [],
|
|
5341
|
+
proposals: [],
|
|
5342
|
+
driftBaselines: {},
|
|
5343
|
+
rolledBack: [],
|
|
5344
|
+
log: [],
|
|
5345
|
+
updatedAt: at
|
|
5346
|
+
};
|
|
5347
|
+
}
|
|
5348
|
+
function losslessLevers(policy) {
|
|
5349
|
+
return policy.levers.filter((lever) => lever.class === "lossless").map((lever) => lever.lever);
|
|
5350
|
+
}
|
|
5351
|
+
function hypothesise(profile, price, activeLossy, rolledBack, declined = /* @__PURE__ */ new Set()) {
|
|
5352
|
+
let contributions;
|
|
5353
|
+
try {
|
|
5354
|
+
contributions = projectSavings(profile, leversFor(profile), price).contributions;
|
|
5355
|
+
} catch {
|
|
5356
|
+
return { hypotheses: [], frontier: [] };
|
|
5357
|
+
}
|
|
5358
|
+
const ranked = [...contributions].sort((a, b) => b.sharePercent - a.sharePercent);
|
|
5359
|
+
const hypotheses = [];
|
|
5360
|
+
const frontier = [];
|
|
5361
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5362
|
+
for (const contribution of ranked) {
|
|
5363
|
+
if (contribution.sharePercent <= 0) continue;
|
|
5364
|
+
if (ALREADY_COVERED_CONCEPTS.has(contribution.lever)) continue;
|
|
5365
|
+
const mapped = ENFORCEABLE_BY_CONCEPT[contribution.lever];
|
|
5366
|
+
if (mapped === void 0) {
|
|
5367
|
+
frontier.push({
|
|
5368
|
+
concept: contribution.lever,
|
|
5369
|
+
priorSharePercent: contribution.sharePercent,
|
|
5370
|
+
note: OPERATOR_CONFIGURED_CONCEPTS[contribution.lever] ?? "Opportunity with no enforceable proxy lever yet \u2014 needs new reduction code or an app-side change."
|
|
5371
|
+
});
|
|
5372
|
+
continue;
|
|
5373
|
+
}
|
|
5374
|
+
if (mapped.class === "lossless") continue;
|
|
5375
|
+
if (rolledBack.has(mapped.lever)) continue;
|
|
5376
|
+
if (declined.has(mapped.lever)) continue;
|
|
5377
|
+
if (activeLossy.has(mapped.lever) || seen.has(mapped.lever)) continue;
|
|
5378
|
+
seen.add(mapped.lever);
|
|
5379
|
+
hypotheses.push({
|
|
5380
|
+
lever: mapped.lever,
|
|
5381
|
+
class: "lossy",
|
|
5382
|
+
priorSharePercent: contribution.sharePercent,
|
|
5383
|
+
rationale: contribution.evidence.claim
|
|
5384
|
+
});
|
|
5385
|
+
}
|
|
5386
|
+
return { hypotheses, frontier };
|
|
5387
|
+
}
|
|
5388
|
+
function proposalMessage(proof) {
|
|
5389
|
+
const drop = Math.abs(proof.qualityDeltaPercent);
|
|
5390
|
+
return `${proof.lever}: proven to save ~${proof.savingPercent.toFixed(1)}% on your traffic, measured quality cost ~${drop.toFixed(2)}% (grader: ${proof.grader}). Apply it?`;
|
|
5391
|
+
}
|
|
5392
|
+
function runTheoCycle(memory, input, now) {
|
|
5393
|
+
const { observation } = input;
|
|
5394
|
+
const proofs = input.proofs ?? [];
|
|
5395
|
+
const autonomy = memory.autonomy;
|
|
5396
|
+
const actions = [];
|
|
5397
|
+
let policy = memory.policy;
|
|
5398
|
+
const experiments = [...memory.experiments];
|
|
5399
|
+
const proposals = [...memory.proposals];
|
|
5400
|
+
const driftBaselines = { ...memory.driftBaselines ?? {} };
|
|
5401
|
+
const rolledBack = [...memory.rolledBack ?? []];
|
|
5402
|
+
const declined = new Set(memory.declined ?? []);
|
|
5403
|
+
const profile = observation.profile;
|
|
5404
|
+
const retryRate = observation.retryRate;
|
|
5405
|
+
const failureRate = observation.failureRate;
|
|
5406
|
+
if (retryRate !== void 0 || failureRate !== void 0) {
|
|
5407
|
+
const pct = (value) => `${(value * 100).toFixed(1)}%`;
|
|
5408
|
+
for (const lever of policy.levers) {
|
|
5409
|
+
if (lever.class !== "lossy" || lever.gate === void 0) continue;
|
|
5410
|
+
const baseline = driftBaselines[lever.lever];
|
|
5411
|
+
if (baseline === void 0) continue;
|
|
5412
|
+
const retryRose = retryRate !== void 0 && retryRate > baseline.retryRate + DRIFT_RISE_THRESHOLD;
|
|
5413
|
+
const failRose = failureRate !== void 0 && failureRate > (baseline.failureRate ?? 0) + DRIFT_RISE_THRESHOLD;
|
|
5414
|
+
if (!retryRose && !failRose) continue;
|
|
5415
|
+
const reason = retryRose ? `retry rate rose ${pct(baseline.retryRate)} \u2192 ${pct(retryRate)}` : `failure rate rose ${pct(baseline.failureRate ?? 0)} \u2192 ${pct(failureRate)}`;
|
|
5416
|
+
policy = {
|
|
5417
|
+
...policy,
|
|
5418
|
+
levers: policy.levers.filter((other) => other.lever !== lever.lever),
|
|
5419
|
+
version: policy.version + 1,
|
|
5420
|
+
updatedAt: now
|
|
5421
|
+
};
|
|
5422
|
+
delete driftBaselines[lever.lever];
|
|
5423
|
+
if (!rolledBack.includes(lever.lever)) rolledBack.push(lever.lever);
|
|
5424
|
+
experiments.push({ lever: lever.lever, ranAt: now, passed: false, note: `rolled back: ${reason}` });
|
|
5425
|
+
actions.push({ kind: "rolled-back", lever: lever.lever, reason });
|
|
5426
|
+
}
|
|
5427
|
+
}
|
|
5428
|
+
const activeLossy = new Set(
|
|
5429
|
+
policy.levers.filter((lever) => lever.class === "lossy" && lever.gate !== void 0).map((l) => l.lever)
|
|
5430
|
+
);
|
|
5431
|
+
const { hypotheses, frontier } = hypothesise(profile, observation.price, activeLossy, new Set(rolledBack), declined);
|
|
5432
|
+
const lossless = losslessLevers(policy);
|
|
5433
|
+
if (autonomy !== "observe") actions.push({ kind: "lossless-active", levers: lossless });
|
|
5434
|
+
for (const proof of proofs) {
|
|
5435
|
+
if (!ENFORCEABLE_LOSSY.has(proof.lever)) {
|
|
5436
|
+
actions.push({ kind: "proof-rejected", lever: proof.lever, reason: "not an enforceable lossy lever" });
|
|
5437
|
+
continue;
|
|
5438
|
+
}
|
|
5439
|
+
if (rolledBack.includes(proof.lever)) {
|
|
5440
|
+
actions.push({ kind: "proof-rejected", lever: proof.lever, reason: "auto-rolled-back earlier after quality drift" });
|
|
5441
|
+
continue;
|
|
5442
|
+
}
|
|
5443
|
+
if (declined.has(proof.lever)) {
|
|
5444
|
+
actions.push({ kind: "proof-rejected", lever: proof.lever, reason: "declined by the customer" });
|
|
5445
|
+
continue;
|
|
5446
|
+
}
|
|
5447
|
+
const ceiling = effectiveQualityCeiling(policy.qualityCeilingPercent);
|
|
5448
|
+
const withinCeiling = Math.abs(proof.qualityDeltaPercent) <= ceiling;
|
|
5449
|
+
const saves = proof.savingPercent > 0;
|
|
5450
|
+
const passed = withinCeiling && saves;
|
|
5451
|
+
experiments.push({
|
|
5452
|
+
lever: proof.lever,
|
|
5453
|
+
ranAt: proof.provenAt,
|
|
5454
|
+
passed,
|
|
5455
|
+
savingPercent: proof.savingPercent,
|
|
5456
|
+
qualityDeltaPercent: proof.qualityDeltaPercent,
|
|
5457
|
+
grader: proof.grader,
|
|
5458
|
+
...passed ? {} : {
|
|
5459
|
+
note: !saves ? "no saving once priced, so nothing to enable" : `quality drop ${Math.abs(proof.qualityDeltaPercent).toFixed(2)}% exceeds the ${ceiling}% ceiling`
|
|
5460
|
+
}
|
|
5461
|
+
});
|
|
5462
|
+
if (!passed) {
|
|
5463
|
+
actions.push({
|
|
5464
|
+
kind: "proof-rejected",
|
|
5465
|
+
lever: proof.lever,
|
|
5466
|
+
reason: saves ? "quality drop exceeds the route ceiling" : "no verified saving"
|
|
5467
|
+
});
|
|
5468
|
+
continue;
|
|
5469
|
+
}
|
|
5470
|
+
if (autonomy === "optimise") {
|
|
5471
|
+
policy = publishLever(
|
|
5472
|
+
policy,
|
|
5473
|
+
{
|
|
5474
|
+
lever: proof.lever,
|
|
5475
|
+
class: "lossy",
|
|
5476
|
+
gate: {
|
|
5477
|
+
savingPercent: proof.savingPercent,
|
|
5478
|
+
qualityDeltaPercent: proof.qualityDeltaPercent,
|
|
5479
|
+
grader: proof.grader,
|
|
5480
|
+
provenAt: proof.provenAt
|
|
5481
|
+
}
|
|
5482
|
+
},
|
|
5483
|
+
now
|
|
5484
|
+
);
|
|
5485
|
+
actions.push({ kind: "policy-published", version: policy.version, lever: proof.lever });
|
|
5486
|
+
driftBaselines[proof.lever] = {
|
|
5487
|
+
retryRate: observation.retryRate ?? 0,
|
|
5488
|
+
failureRate: observation.failureRate ?? 0,
|
|
5489
|
+
at: now
|
|
5490
|
+
};
|
|
5491
|
+
} else if (autonomy === "recommend" || autonomy === "test") {
|
|
5492
|
+
if (!proposals.some((existing) => existing.lever === proof.lever)) {
|
|
5493
|
+
proposals.push({
|
|
5494
|
+
lever: proof.lever,
|
|
5495
|
+
savingPercent: proof.savingPercent,
|
|
5496
|
+
qualityDeltaPercent: proof.qualityDeltaPercent,
|
|
5497
|
+
grader: proof.grader,
|
|
5498
|
+
provenAt: proof.provenAt,
|
|
5499
|
+
message: proposalMessage(proof)
|
|
5500
|
+
});
|
|
5501
|
+
actions.push({
|
|
5502
|
+
kind: "proposal-opened",
|
|
5503
|
+
lever: proof.lever,
|
|
5504
|
+
savingPercent: proof.savingPercent,
|
|
5505
|
+
qualityDeltaPercent: proof.qualityDeltaPercent
|
|
5506
|
+
});
|
|
5507
|
+
}
|
|
5508
|
+
}
|
|
5509
|
+
}
|
|
5510
|
+
const provenThisCycle = new Set(proofs.map((proof) => proof.lever));
|
|
5511
|
+
const openHypotheses = hypotheses.filter((hypothesis) => !provenThisCycle.has(hypothesis.lever));
|
|
5512
|
+
if ((autonomy === "test" || autonomy === "optimise") && openHypotheses.length > 0) {
|
|
5513
|
+
actions.push({ kind: "gate-requested", levers: openHypotheses.slice(0, 3).map((h) => h.lever) });
|
|
5514
|
+
}
|
|
5515
|
+
const nextMemory = {
|
|
5516
|
+
...memory,
|
|
5517
|
+
policy,
|
|
5518
|
+
profile,
|
|
5519
|
+
hypotheses: openHypotheses,
|
|
5520
|
+
frontier,
|
|
5521
|
+
experiments,
|
|
5522
|
+
proposals,
|
|
5523
|
+
driftBaselines,
|
|
5524
|
+
rolledBack,
|
|
5525
|
+
log: [
|
|
5526
|
+
...memory.log,
|
|
5527
|
+
{
|
|
5528
|
+
at: now,
|
|
5529
|
+
autonomy,
|
|
5530
|
+
observed: `${profile.name}: ${profile.requests} req, ${Math.round(sumSegments(profile))} tok/req`,
|
|
5531
|
+
actions: actions.length
|
|
5532
|
+
}
|
|
5533
|
+
],
|
|
5534
|
+
updatedAt: now
|
|
5535
|
+
};
|
|
5536
|
+
return { memory: nextMemory, actions, report: renderTheoReport(nextMemory, observation, actions) };
|
|
5537
|
+
}
|
|
5538
|
+
function openProposalList(memory) {
|
|
5539
|
+
return memory.proposals.length === 0 ? "There are no open proposals." : `Open proposals: ${memory.proposals.map((proposal) => proposal.lever).join(", ")}.`;
|
|
5540
|
+
}
|
|
5541
|
+
function acceptProposal(memory, lever, rates, now) {
|
|
5542
|
+
const proposal = memory.proposals.find((candidate) => candidate.lever === lever);
|
|
5543
|
+
if (proposal === void 0) {
|
|
5544
|
+
return { ok: false, reason: `No open proposal for '${lever}'. ${openProposalList(memory)}` };
|
|
5545
|
+
}
|
|
5546
|
+
if (!ENFORCEABLE_LOSSY.has(lever)) {
|
|
5547
|
+
return { ok: false, reason: `'${lever}' is not a lever Theo can enforce.` };
|
|
5548
|
+
}
|
|
5549
|
+
if ((memory.rolledBack ?? []).includes(lever)) {
|
|
5550
|
+
return { ok: false, reason: `'${lever}' was rolled back after quality drift and cannot be re-applied.` };
|
|
5551
|
+
}
|
|
5552
|
+
const ceiling = effectiveQualityCeiling(memory.policy.qualityCeilingPercent);
|
|
5553
|
+
const drop = Math.abs(proposal.qualityDeltaPercent);
|
|
5554
|
+
if (!Number.isFinite(drop) || drop > ceiling) {
|
|
5555
|
+
return {
|
|
5556
|
+
ok: false,
|
|
5557
|
+
reason: `'${lever}' measured a ${drop.toFixed(2)}% quality drop, above the current ${ceiling}% ceiling.`
|
|
5558
|
+
};
|
|
5559
|
+
}
|
|
5560
|
+
if (!(proposal.savingPercent > 0)) {
|
|
5561
|
+
return { ok: false, reason: `'${lever}' has no verified saving to enable.` };
|
|
5562
|
+
}
|
|
5563
|
+
const policy = publishLever(
|
|
5564
|
+
memory.policy,
|
|
5565
|
+
{
|
|
5566
|
+
lever,
|
|
5567
|
+
class: "lossy",
|
|
5568
|
+
gate: {
|
|
5569
|
+
savingPercent: proposal.savingPercent,
|
|
5570
|
+
qualityDeltaPercent: proposal.qualityDeltaPercent,
|
|
5571
|
+
grader: proposal.grader,
|
|
5572
|
+
provenAt: proposal.provenAt
|
|
5573
|
+
}
|
|
5574
|
+
},
|
|
5575
|
+
now
|
|
5576
|
+
);
|
|
5577
|
+
return {
|
|
5578
|
+
ok: true,
|
|
5579
|
+
memory: {
|
|
5580
|
+
...memory,
|
|
5581
|
+
policy,
|
|
5582
|
+
proposals: memory.proposals.filter((candidate) => candidate.lever !== lever),
|
|
5583
|
+
hypotheses: memory.hypotheses.filter((hypothesis) => hypothesis.lever !== lever),
|
|
5584
|
+
driftBaselines: {
|
|
5585
|
+
...memory.driftBaselines ?? {},
|
|
5586
|
+
[lever]: { retryRate: rates.retryRate ?? 0, failureRate: rates.failureRate ?? 0, at: now }
|
|
5587
|
+
},
|
|
5588
|
+
declined: (memory.declined ?? []).filter((other) => other !== lever),
|
|
5589
|
+
updatedAt: now
|
|
5590
|
+
},
|
|
5591
|
+
message: `${lever} accepted \u2014 policy version ${policy.version}. Theo rolls it back if retries or failures rise.`
|
|
5592
|
+
};
|
|
5593
|
+
}
|
|
5594
|
+
function rejectProposal(memory, lever, now) {
|
|
5595
|
+
if (!memory.proposals.some((candidate) => candidate.lever === lever)) {
|
|
5596
|
+
return { ok: false, reason: `No open proposal for '${lever}'. ${openProposalList(memory)}` };
|
|
5597
|
+
}
|
|
5598
|
+
const declined = memory.declined ?? [];
|
|
5599
|
+
return {
|
|
5600
|
+
ok: true,
|
|
5601
|
+
memory: {
|
|
5602
|
+
...memory,
|
|
5603
|
+
proposals: memory.proposals.filter((candidate) => candidate.lever !== lever),
|
|
5604
|
+
hypotheses: memory.hypotheses.filter((hypothesis) => hypothesis.lever !== lever),
|
|
5605
|
+
declined: declined.includes(lever) ? declined : [...declined, lever],
|
|
5606
|
+
updatedAt: now
|
|
5607
|
+
},
|
|
5608
|
+
message: `${lever} declined \u2014 Theo will not propose it again unless you ask him to reconsider.`
|
|
5609
|
+
};
|
|
5610
|
+
}
|
|
5611
|
+
function reconsiderLever(memory, lever, now) {
|
|
5612
|
+
if (!(memory.declined ?? []).includes(lever)) {
|
|
5613
|
+
return { ok: false, reason: `'${lever}' is not on the declined list.` };
|
|
5614
|
+
}
|
|
5615
|
+
return {
|
|
5616
|
+
ok: true,
|
|
5617
|
+
memory: {
|
|
5618
|
+
...memory,
|
|
5619
|
+
declined: (memory.declined ?? []).filter((other) => other !== lever),
|
|
5620
|
+
updatedAt: now
|
|
5621
|
+
},
|
|
5622
|
+
message: `${lever} is back in play \u2014 Theo will prove it again on the next gate run.`
|
|
5623
|
+
};
|
|
5624
|
+
}
|
|
5625
|
+
function sumSegments(profile) {
|
|
5626
|
+
return Object.values(profile.segments).reduce((sum, value) => sum + value, 0);
|
|
5627
|
+
}
|
|
5628
|
+
function renderTheoReport(memory, observation, actions) {
|
|
5629
|
+
const coverage = policyCoverage(memory.policy);
|
|
5630
|
+
const published = actions.filter((a) => a.kind === "policy-published").length;
|
|
5631
|
+
const proposed = memory.proposals.length;
|
|
5632
|
+
const lines = [];
|
|
5633
|
+
lines.push(`Theo \u2014 ${memory.customer} (autonomy: ${memory.autonomy})`);
|
|
5634
|
+
lines.push(
|
|
5635
|
+
`OBSERVE: ${observation.profile.name}, ${observation.profile.requests} requests, ${Math.round(sumSegments(observation.profile))} tok/req (from blind features only).`
|
|
5636
|
+
);
|
|
5637
|
+
lines.push(
|
|
5638
|
+
`ACTIVE: ${coverage.activeCount}/${coverage.enforceableCount} enforceable levers (${coverage.activeCoveragePercent.toFixed(0)}%), lossless on by default.`
|
|
5639
|
+
);
|
|
5640
|
+
if (memory.hypotheses.length > 0) {
|
|
5641
|
+
lines.push(`QUEUE: ${memory.hypotheses.length} lossy candidate(s) ranked to prove (priors, not claims).`);
|
|
5642
|
+
}
|
|
5643
|
+
if (memory.frontier.length > 0) {
|
|
5644
|
+
lines.push(`FRONTIER: ${memory.frontier.length} opportunity(ies) needing new code \u2014 reported, not touched.`);
|
|
5645
|
+
}
|
|
5646
|
+
if (published > 0) lines.push(`APPLIED: ${published} proven lever(s) published to the policy this cycle.`);
|
|
5647
|
+
const rolledBackNow = actions.filter((a) => a.kind === "rolled-back").length;
|
|
5648
|
+
if (rolledBackNow > 0) lines.push(`ROLLED BACK: ${rolledBackNow} lever(s) reverted after a rise in retries or failures (quality drift).`);
|
|
5649
|
+
if (proposed > 0) lines.push(`PROPOSED: ${proposed} proven lever(s) awaiting your approval.`);
|
|
5650
|
+
if (observation.realisedSavingsUsd !== void 0) {
|
|
5651
|
+
lines.push(`LEDGER: $${observation.realisedSavingsUsd.toFixed(2)} verified savings recorded so far.`);
|
|
5652
|
+
}
|
|
5653
|
+
lines.push("Theo never enables a saving he has not proven on your own traffic within your quality ceiling.");
|
|
5654
|
+
return lines.join("\n");
|
|
5655
|
+
}
|
|
5656
|
+
|
|
5657
|
+
// packages/sdk-js/src/theo-agent-store.ts
|
|
5658
|
+
var import_node_fs5 = require("node:fs");
|
|
5659
|
+
var import_node_path6 = require("node:path");
|
|
5660
|
+
|
|
5661
|
+
// packages/sdk-js/src/savings-snapshot.ts
|
|
5662
|
+
var import_node_fs4 = require("node:fs");
|
|
5663
|
+
var import_node_path5 = require("node:path");
|
|
5664
|
+
var EMPTY = {
|
|
5665
|
+
requests: 0,
|
|
5666
|
+
savedUsd: 0,
|
|
5667
|
+
spendUsd: 0,
|
|
5668
|
+
verdict: "do_not_charge",
|
|
5669
|
+
chainOk: true
|
|
5670
|
+
};
|
|
5671
|
+
function savingsSnapshotFromEvents(events, chainOk = true) {
|
|
5672
|
+
if (!chainOk) return { ...EMPTY, chainOk: false };
|
|
5673
|
+
if (events.length === 0) return EMPTY;
|
|
5674
|
+
const invoice = recomputeInvoice(events, { feePercent: 1 });
|
|
5675
|
+
const spendUsd = events.reduce((sum, event) => sum + (event.actualUsd ?? 0), 0);
|
|
5676
|
+
return {
|
|
5677
|
+
requests: invoice.eventsConsidered,
|
|
5678
|
+
savedUsd: invoice.netVerifiedSavingsUsd,
|
|
5679
|
+
spendUsd,
|
|
5680
|
+
verdict: invoice.verdict,
|
|
5681
|
+
chainOk: true
|
|
5682
|
+
};
|
|
5683
|
+
}
|
|
5684
|
+
function readLedgerRecords(path) {
|
|
5685
|
+
try {
|
|
5686
|
+
const lines = (0, import_node_fs4.readFileSync)(path, "utf8").trim().split(/\r?\n/).filter(Boolean);
|
|
5687
|
+
return lines.map((line) => JSON.parse(line));
|
|
5688
|
+
} catch {
|
|
5689
|
+
return null;
|
|
5690
|
+
}
|
|
5691
|
+
}
|
|
5692
|
+
function savingsSnapshotFromSpoolDir(dir) {
|
|
5693
|
+
let files;
|
|
5694
|
+
try {
|
|
5695
|
+
files = (0, import_node_fs4.readdirSync)(dir).filter(
|
|
5696
|
+
(f) => f.endsWith(".jsonl") && !f.endsWith(".features.jsonl")
|
|
5697
|
+
);
|
|
5698
|
+
} catch {
|
|
5699
|
+
return EMPTY;
|
|
5700
|
+
}
|
|
5701
|
+
if (files.length === 0) return EMPTY;
|
|
5702
|
+
const events = [];
|
|
5703
|
+
let chainOk = true;
|
|
5704
|
+
for (const file of files) {
|
|
5705
|
+
const records = readLedgerRecords((0, import_node_path5.join)(dir, file));
|
|
5706
|
+
if (records === null || records.length === 0) continue;
|
|
5707
|
+
if (!verifyLedgerChain(records).ok) {
|
|
5708
|
+
chainOk = false;
|
|
5709
|
+
continue;
|
|
5710
|
+
}
|
|
5711
|
+
for (const record of records) events.push(record.event);
|
|
5712
|
+
}
|
|
5713
|
+
if (events.length === 0) return { ...EMPTY, chainOk };
|
|
5714
|
+
return { ...savingsSnapshotFromEvents(events, true), chainOk };
|
|
5715
|
+
}
|
|
5716
|
+
|
|
5717
|
+
// packages/sdk-js/src/theo-agent-store.ts
|
|
5718
|
+
function defaultTheoMemoryPath(ledgerDir, customer) {
|
|
5719
|
+
const safe = customer.replace(/[^A-Za-z0-9._-]/g, "_") || "default";
|
|
5720
|
+
return (0, import_node_path6.join)(ledgerDir, "theo", `${safe}.json`);
|
|
5721
|
+
}
|
|
5722
|
+
function loadTheoMemory(path) {
|
|
5723
|
+
try {
|
|
5724
|
+
const parsed = JSON.parse((0, import_node_fs5.readFileSync)(path, "utf8"));
|
|
5725
|
+
if (parsed && typeof parsed === "object" && parsed.schemaVersion === 1) return parsed;
|
|
5726
|
+
return null;
|
|
5727
|
+
} catch {
|
|
5728
|
+
return null;
|
|
5729
|
+
}
|
|
5730
|
+
}
|
|
5731
|
+
function saveTheoMemory(path, memory) {
|
|
5732
|
+
(0, import_node_fs5.mkdirSync)((0, import_node_path6.dirname)(path), { recursive: true });
|
|
5733
|
+
(0, import_node_fs5.writeFileSync)(path, `${JSON.stringify(memory, null, 2)}
|
|
5734
|
+
`, "utf8");
|
|
5735
|
+
}
|
|
5736
|
+
function realisedFromSpoolDir(ledgerDir) {
|
|
5737
|
+
const snapshot = savingsSnapshotFromSpoolDir(ledgerDir);
|
|
5738
|
+
return {
|
|
5739
|
+
savingsUsd: snapshot.savedUsd,
|
|
5740
|
+
spendUsd: snapshot.spendUsd,
|
|
5741
|
+
requests: snapshot.requests,
|
|
5742
|
+
chainOk: snapshot.chainOk
|
|
5743
|
+
};
|
|
5744
|
+
}
|
|
5745
|
+
|
|
5746
|
+
// packages/sdk-js/src/graders.ts
|
|
5747
|
+
function words(text) {
|
|
5748
|
+
return text.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? [];
|
|
5749
|
+
}
|
|
5750
|
+
function overlap(left, right) {
|
|
5751
|
+
const counts = /* @__PURE__ */ new Map();
|
|
5752
|
+
for (const word of left) counts.set(word, (counts.get(word) ?? 0) + 1);
|
|
5753
|
+
let shared = 0;
|
|
5754
|
+
for (const word of right) {
|
|
5755
|
+
const remaining = counts.get(word) ?? 0;
|
|
5756
|
+
if (remaining > 0) {
|
|
5757
|
+
shared += 1;
|
|
5758
|
+
counts.set(word, remaining - 1);
|
|
5759
|
+
}
|
|
5760
|
+
}
|
|
5761
|
+
return shared;
|
|
5762
|
+
}
|
|
5763
|
+
var textSimilarityGrader = ({ baselineResponse, candidateResponse }) => {
|
|
5764
|
+
const baseline = words(responseText(baselineResponse));
|
|
5765
|
+
const candidate = words(responseText(candidateResponse));
|
|
5766
|
+
if (baseline.length === 0 && candidate.length === 0) return 1;
|
|
5767
|
+
if (baseline.length === 0 || candidate.length === 0) return 0;
|
|
5768
|
+
const shared = overlap(baseline, candidate);
|
|
5769
|
+
if (shared === 0) return 0;
|
|
5770
|
+
const precision = shared / candidate.length;
|
|
5771
|
+
const recall = shared / baseline.length;
|
|
5772
|
+
return 2 * precision * recall / (precision + recall);
|
|
5773
|
+
};
|
|
5774
|
+
function wasTruncated(response) {
|
|
5775
|
+
if (typeof response !== "object" || response === null) return false;
|
|
5776
|
+
const row = response;
|
|
5777
|
+
if (row.finish_reason === "length" || row.stop_reason === "max_tokens") return true;
|
|
5778
|
+
if (row.incomplete_details !== null && typeof row.incomplete_details === "object") {
|
|
5779
|
+
const reason = row.incomplete_details.reason;
|
|
5780
|
+
if (reason === "max_output_tokens" || reason === "max_tokens") return true;
|
|
5781
|
+
}
|
|
5782
|
+
if (Array.isArray(row.choices)) {
|
|
5783
|
+
return row.choices.some((choice) => typeof choice === "object" && choice !== null && choice.finish_reason === "length");
|
|
5784
|
+
}
|
|
5785
|
+
return false;
|
|
5786
|
+
}
|
|
5787
|
+
function truncationAwareGrader(options = {}) {
|
|
5788
|
+
const floor = options.uselessBelowCoverage ?? 0.5;
|
|
5789
|
+
const penalty = options.truncationPenalty ?? 0.05;
|
|
5790
|
+
if (floor < 0 || floor > 1) throw new Error("uselessBelowCoverage must be between 0 and 1.");
|
|
5791
|
+
if (penalty < 0 || penalty > 1) throw new Error("truncationPenalty must be between 0 and 1.");
|
|
5792
|
+
return (comparison) => {
|
|
5793
|
+
const baseline = words(responseText(comparison.baselineResponse));
|
|
5794
|
+
const candidate = words(responseText(comparison.candidateResponse));
|
|
5795
|
+
const truncated = wasTruncated(comparison.candidateResponse);
|
|
5796
|
+
if (!truncated) return 1;
|
|
5797
|
+
if (baseline.length === 0) return 1;
|
|
5798
|
+
const coverage = Math.min(1, candidate.length / baseline.length);
|
|
5799
|
+
if (coverage < floor) return 0;
|
|
5800
|
+
return Math.max(0, coverage - penalty);
|
|
5801
|
+
};
|
|
5802
|
+
}
|
|
5803
|
+
function compositeGrader(parts) {
|
|
5804
|
+
if (parts.length === 0) throw new Error("A composite grader needs at least one part.");
|
|
5805
|
+
if (parts.some((part) => !Number.isFinite(part.weight) || part.weight <= 0)) {
|
|
5806
|
+
throw new Error("Composite grader weights must be positive finite numbers.");
|
|
5807
|
+
}
|
|
5808
|
+
const total = parts.reduce((sum, part) => sum + part.weight, 0);
|
|
5809
|
+
return async (comparison) => {
|
|
5810
|
+
let score = 0;
|
|
5811
|
+
for (const part of parts) {
|
|
5812
|
+
const value = await part.grader(comparison);
|
|
5813
|
+
score += (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0) * part.weight;
|
|
5814
|
+
}
|
|
5815
|
+
return score / total;
|
|
5816
|
+
};
|
|
5817
|
+
}
|
|
5818
|
+
function agentGrader(options = {}) {
|
|
5819
|
+
const truncation = truncationAwareGrader(options);
|
|
5820
|
+
return async (comparison) => {
|
|
5821
|
+
const baselineCalls = responseToolCalls(comparison.baselineResponse);
|
|
5822
|
+
const candidateCalls = responseToolCalls(comparison.candidateResponse);
|
|
5823
|
+
if (baselineCalls.length > 0 || candidateCalls.length > 0) {
|
|
5824
|
+
const tool = await toolSelectionGrader(comparison);
|
|
5825
|
+
if (tool < 1) return tool;
|
|
5826
|
+
}
|
|
5827
|
+
return truncation(comparison);
|
|
5828
|
+
};
|
|
5829
|
+
}
|
|
5830
|
+
|
|
5831
|
+
// packages/sdk-js/src/theo-gate.ts
|
|
5832
|
+
async function proveCandidates(options) {
|
|
5833
|
+
const levers = [...new Set(options.levers)];
|
|
5834
|
+
if (levers.length === 0) return { proofs: [], rejections: [], calls: 0 };
|
|
5835
|
+
if (options.requests.length === 0) {
|
|
5836
|
+
return { proofs: [], rejections: levers.map((lever) => ({ lever, reason: "no request sample to replay" })), calls: 0 };
|
|
5837
|
+
}
|
|
5838
|
+
const candidates = levers.map((lever) => ({
|
|
5839
|
+
label: lever,
|
|
5840
|
+
config: leverToReductionConfig({ lever, class: "lossy" })
|
|
5841
|
+
}));
|
|
5842
|
+
const perRequestCalls = candidates.length + (options.baselineResponses ? 0 : 1);
|
|
5843
|
+
let requests = [...options.requests];
|
|
5844
|
+
let baselineResponses = options.baselineResponses ? [...options.baselineResponses] : void 0;
|
|
5845
|
+
let budgetNote;
|
|
5846
|
+
if (options.maxProviderCalls !== void 0) {
|
|
5847
|
+
const maxRequests = Math.floor(options.maxProviderCalls / Math.max(1, perRequestCalls));
|
|
5848
|
+
if (maxRequests < 1) {
|
|
5849
|
+
return {
|
|
5850
|
+
proofs: [],
|
|
5851
|
+
rejections: levers.map((lever) => ({
|
|
5852
|
+
lever,
|
|
5853
|
+
reason: `exploration budget of ${options.maxProviderCalls} call(s) is too small to prove it`
|
|
5854
|
+
})),
|
|
5855
|
+
calls: 0,
|
|
5856
|
+
note: `skipped: ${perRequestCalls} call(s)/request exceeds the ${options.maxProviderCalls}-call budget`
|
|
5857
|
+
};
|
|
5858
|
+
}
|
|
5859
|
+
if (requests.length > maxRequests) {
|
|
5860
|
+
requests = requests.slice(0, maxRequests);
|
|
5861
|
+
baselineResponses = baselineResponses?.slice(0, maxRequests);
|
|
5862
|
+
budgetNote = `capped to ${maxRequests} request(s) to stay within the ${options.maxProviderCalls}-call budget`;
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
5865
|
+
const result = await replayGate({
|
|
5866
|
+
requests,
|
|
5867
|
+
call: options.call,
|
|
5868
|
+
candidates,
|
|
5869
|
+
grade: options.grader ?? agentGrader(),
|
|
5870
|
+
maximumQualityDrop: options.maximumQualityDrop ?? 0.02,
|
|
5871
|
+
...baselineResponses ? { baselineResponses } : {},
|
|
5872
|
+
...options.price ? { price: options.price } : {},
|
|
5873
|
+
...options.groupOf ? { groupOf: options.groupOf } : {}
|
|
5874
|
+
});
|
|
5875
|
+
const graderName = options.graderName ?? "agent";
|
|
5876
|
+
const proofs = [];
|
|
5877
|
+
const rejections = [];
|
|
5878
|
+
for (const candidate of result.candidates) {
|
|
5879
|
+
if (candidate.approved) {
|
|
5880
|
+
proofs.push({
|
|
5881
|
+
lever: candidate.label,
|
|
5882
|
+
// Prefer the cost saving (the bill) when priced; fall back to tokens.
|
|
5883
|
+
savingPercent: candidate.costSavingPercent ?? candidate.tokenSavingPercent,
|
|
5884
|
+
// One-sided 95% quality bound, a fraction ≤ 0; expressed as a signed percent.
|
|
5885
|
+
qualityDeltaPercent: candidate.lower95QualityDelta * 100,
|
|
5886
|
+
grader: graderName,
|
|
5887
|
+
provenAt: options.now
|
|
5888
|
+
});
|
|
5889
|
+
} else {
|
|
5890
|
+
rejections.push({ lever: candidate.label, reason: candidate.rejectionReason ?? "not approved by the gate" });
|
|
5891
|
+
}
|
|
5892
|
+
}
|
|
5893
|
+
return { proofs, rejections, calls: result.calls, ...budgetNote ? { note: budgetNote } : {} };
|
|
5894
|
+
}
|
|
4511
5895
|
|
|
4512
5896
|
// packages/sdk-js/src/semantic-compaction.ts
|
|
4513
5897
|
function isPlainObject9(value) {
|
|
@@ -4779,7 +6163,7 @@ independently, without MESH and without accepting any terms:
|
|
|
4779
6163
|
recomputation disagrees with this statement, yours wins. Negative levers are
|
|
4780
6164
|
shown negative and reduce the bill; savings you already had are itemised above
|
|
4781
6165
|
and never billed. Methodology: https://www.mesh-technology.com/methodology.html<br>
|
|
4782
|
-
MESH is a
|
|
6166
|
+
MESH is a trading name of Flux AI Technologies Limited, registered in England and Wales, company no. 17543982.
|
|
4783
6167
|
</footer>
|
|
4784
6168
|
</main>
|
|
4785
6169
|
</body>
|
|
@@ -4787,99 +6171,75 @@ MESH is a product of Mesh AI Technologies Limited, registered in England and Wal
|
|
|
4787
6171
|
`;
|
|
4788
6172
|
}
|
|
4789
6173
|
|
|
4790
|
-
// packages/sdk-js/src/
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
const
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
}
|
|
4818
|
-
function
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
const baseline = words(responseText(comparison.baselineResponse));
|
|
4838
|
-
const candidate = words(responseText(comparison.candidateResponse));
|
|
4839
|
-
const truncated = wasTruncated(comparison.candidateResponse);
|
|
4840
|
-
if (!truncated) return 1;
|
|
4841
|
-
if (baseline.length === 0) return 1;
|
|
4842
|
-
const coverage = Math.min(1, candidate.length / baseline.length);
|
|
4843
|
-
if (coverage < floor) return 0;
|
|
4844
|
-
return Math.max(0, coverage - penalty);
|
|
4845
|
-
};
|
|
4846
|
-
}
|
|
4847
|
-
function compositeGrader(parts) {
|
|
4848
|
-
if (parts.length === 0) throw new Error("A composite grader needs at least one part.");
|
|
4849
|
-
if (parts.some((part) => !Number.isFinite(part.weight) || part.weight <= 0)) {
|
|
4850
|
-
throw new Error("Composite grader weights must be positive finite numbers.");
|
|
6174
|
+
// packages/sdk-js/src/cta.ts
|
|
6175
|
+
var MESH_SIGNUP_URL = "https://www.mesh-technology.com";
|
|
6176
|
+
function savingsPercent(snap) {
|
|
6177
|
+
const baseline = snap.spendUsd + snap.savedUsd;
|
|
6178
|
+
if (baseline <= 0) return null;
|
|
6179
|
+
return snap.savedUsd / baseline * 100;
|
|
6180
|
+
}
|
|
6181
|
+
function formatSavingsCta(snap, opts = {}) {
|
|
6182
|
+
const mark = opts.mark ?? "\u2B22 MESH";
|
|
6183
|
+
const url = opts.signupUrl ?? MESH_SIGNUP_URL;
|
|
6184
|
+
if (!snap.chainOk) {
|
|
6185
|
+
return `${mark}: a ledger spool failed to verify \u2014 savings withheld. Recompute with \`mesh verify\`.`;
|
|
6186
|
+
}
|
|
6187
|
+
if (snap.requests === 0) {
|
|
6188
|
+
return `${mark}: no requests metered yet. Point a client's base_url here and your verified savings print on exit.`;
|
|
6189
|
+
}
|
|
6190
|
+
if (snap.verdict === "do_not_charge" || snap.savedUsd <= 0) {
|
|
6191
|
+
return [
|
|
6192
|
+
`${mark}: ${snap.requests} requests metered \xB7 verified savings $${snap.savedUsd.toFixed(4)}.`,
|
|
6193
|
+
`Your stack is already lean \u2014 verdict do_not_charge. You owe MESH nothing.`,
|
|
6194
|
+
`Recompute it yourself: \`mesh verify\`.`
|
|
6195
|
+
].join("\n");
|
|
6196
|
+
}
|
|
6197
|
+
const pct = savingsPercent(snap);
|
|
6198
|
+
const pctText = pct === null ? "" : ` (${pct.toFixed(1)}% lower)`;
|
|
6199
|
+
return [
|
|
6200
|
+
``,
|
|
6201
|
+
`${mark} saved you $${snap.savedUsd.toFixed(4)}${pctText} across ${snap.requests} verified requests.`,
|
|
6202
|
+
`This number is a pure function of a hash-chained ledger you hold \u2014 recompute it: \`mesh verify\`.`,
|
|
6203
|
+
`Claim your savings and keep them metered: ${url}`,
|
|
6204
|
+
``
|
|
6205
|
+
].join("\n");
|
|
6206
|
+
}
|
|
6207
|
+
function savingsBadge(snap, opts = {}) {
|
|
6208
|
+
const url = opts.signupUrl ?? MESH_SIGNUP_URL;
|
|
6209
|
+
const pct = savingsPercent(snap);
|
|
6210
|
+
let message;
|
|
6211
|
+
let color;
|
|
6212
|
+
if (!snap.chainOk) {
|
|
6213
|
+
message = "unverified";
|
|
6214
|
+
color = "lightgrey";
|
|
6215
|
+
} else if (snap.verdict === "do_not_charge" || snap.savedUsd <= 0) {
|
|
6216
|
+
message = "already lean";
|
|
6217
|
+
color = "blue";
|
|
6218
|
+
} else {
|
|
6219
|
+
message = pct === null ? `$${snap.savedUsd.toFixed(2)} saved` : `${pct.toFixed(0)}% lower`;
|
|
6220
|
+
color = "brightgreen";
|
|
4851
6221
|
}
|
|
4852
|
-
const
|
|
4853
|
-
return
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
const value = await part.grader(comparison);
|
|
4857
|
-
score += (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0) * part.weight;
|
|
4858
|
-
}
|
|
4859
|
-
return score / total;
|
|
4860
|
-
};
|
|
4861
|
-
}
|
|
4862
|
-
function agentGrader(options = {}) {
|
|
4863
|
-
const truncation = truncationAwareGrader(options);
|
|
4864
|
-
return async (comparison) => {
|
|
4865
|
-
const baselineCalls = responseToolCalls(comparison.baselineResponse);
|
|
4866
|
-
const candidateCalls = responseToolCalls(comparison.candidateResponse);
|
|
4867
|
-
if (baselineCalls.length > 0 || candidateCalls.length > 0) {
|
|
4868
|
-
const tool = await toolSelectionGrader(comparison);
|
|
4869
|
-
if (tool < 1) return tool;
|
|
4870
|
-
}
|
|
4871
|
-
return truncation(comparison);
|
|
6222
|
+
const imageUrl = `https://img.shields.io/static/v1?label=${encodeURIComponent("LLM cost")}&message=${encodeURIComponent(message)}&color=${color}`;
|
|
6223
|
+
return {
|
|
6224
|
+
imageUrl,
|
|
6225
|
+
markdown: `[](${url})`
|
|
4872
6226
|
};
|
|
4873
6227
|
}
|
|
4874
6228
|
// Annotate the CommonJS export names for ESM import in node:
|
|
4875
6229
|
0 && (module.exports = {
|
|
4876
6230
|
CachePrefixVolatilityGuard,
|
|
6231
|
+
DRIFT_RISE_THRESHOLD,
|
|
6232
|
+
DRIFT_WINDOW_EVENTS,
|
|
4877
6233
|
ExactResponseCache,
|
|
4878
6234
|
LEDGER_GENESIS_HASH,
|
|
6235
|
+
MESH_SIGNUP_URL,
|
|
6236
|
+
acceptProposal,
|
|
4879
6237
|
agentGrader,
|
|
6238
|
+
asBundle,
|
|
4880
6239
|
auditEventPricing,
|
|
4881
6240
|
billableEventFromRequest,
|
|
4882
6241
|
buildQuoteReport,
|
|
6242
|
+
bundleToReductionConfig,
|
|
4883
6243
|
canonicalJson,
|
|
4884
6244
|
capToolResultBody,
|
|
4885
6245
|
catalogueVersion,
|
|
@@ -4889,32 +6249,55 @@ function agentGrader(options = {}) {
|
|
|
4889
6249
|
compositeGrader,
|
|
4890
6250
|
createLocalJsonlCapture,
|
|
4891
6251
|
createSavingsLedger,
|
|
6252
|
+
createTheoMemory,
|
|
4892
6253
|
defaultPolicy,
|
|
6254
|
+
defaultTheoMemoryPath,
|
|
4893
6255
|
digestStaleBody,
|
|
4894
6256
|
exactTextGrader,
|
|
4895
6257
|
extractBlindFeatures,
|
|
6258
|
+
formatSavingsCta,
|
|
6259
|
+
isPolicyBundle,
|
|
4896
6260
|
knownLeverIds,
|
|
4897
6261
|
ledgerEventHash,
|
|
6262
|
+
leverToReductionConfig,
|
|
6263
|
+
loadTheoMemory,
|
|
6264
|
+
mergePolicyIntoBundle,
|
|
6265
|
+
observeWorkload,
|
|
4898
6266
|
parseGatewayExport,
|
|
6267
|
+
policyForRoute,
|
|
4899
6268
|
policyToReductionConfig,
|
|
4900
6269
|
priceLedgerUsage,
|
|
6270
|
+
proveCandidates,
|
|
4901
6271
|
publishLever,
|
|
4902
6272
|
publishedModelPrices,
|
|
6273
|
+
realisedFromSpoolDir,
|
|
6274
|
+
recentDriftRates,
|
|
4903
6275
|
recomputeInvoice,
|
|
6276
|
+
reconsiderLever,
|
|
4904
6277
|
reduceRequest,
|
|
4905
6278
|
reduceRequestAsync,
|
|
6279
|
+
rejectProposal,
|
|
4906
6280
|
renderLedgerReport,
|
|
4907
6281
|
renderQuoteReport,
|
|
6282
|
+
renderTheoReport,
|
|
4908
6283
|
replayGate,
|
|
4909
6284
|
replayUsage,
|
|
4910
6285
|
responseText,
|
|
4911
6286
|
responseToolCalls,
|
|
6287
|
+
routeMatches,
|
|
6288
|
+
runTheoCycle,
|
|
6289
|
+
saveTheoMemory,
|
|
6290
|
+
savingsBadge,
|
|
6291
|
+
savingsPercent,
|
|
6292
|
+
savingsSnapshotFromEvents,
|
|
6293
|
+
savingsSnapshotFromSpoolDir,
|
|
4912
6294
|
serviceTierOf,
|
|
4913
6295
|
startMeshProxy,
|
|
4914
6296
|
textSimilarityGrader,
|
|
4915
6297
|
toolNameGrader,
|
|
4916
6298
|
toolSelectionGrader,
|
|
4917
6299
|
truncationAwareGrader,
|
|
6300
|
+
upsertRoutePolicy,
|
|
4918
6301
|
usageFromSse,
|
|
4919
6302
|
verifyLedgerChain,
|
|
4920
6303
|
withMesh,
|