@harrym1707/sdk 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -2
- package/index.cjs +1547 -164
- package/index.js +1520 -166
- package/package.json +1 -1
- package/types/cache-volatility-guard.d.ts +18 -4
- package/types/core/account-store.d.ts +7 -0
- package/types/core/accounts.d.ts +134 -0
- package/types/core/billing-worker.d.ts +113 -0
- package/types/core/budget.d.ts +216 -0
- package/types/core/evaluator.d.ts +10 -0
- package/types/core/github-oauth.d.ts +41 -0
- package/types/core/index.d.ts +10 -0
- package/types/core/ledger-store.d.ts +62 -0
- package/types/core/routing.d.ts +282 -0
- package/types/core/stripe-client.d.ts +117 -0
- package/types/core/token-reduction.d.ts +11 -0
- package/types/core/types.d.ts +11 -0
- package/types/core/workflow-sessions.d.ts +50 -0
- package/types/cta.d.ts +60 -0
- package/types/index.d.ts +6 -1
- package/types/ledger-mirror.d.ts +38 -0
- package/types/proxy.d.ts +43 -2
- package/types/reduce-request.d.ts +27 -2
- package/types/savings-ledger.d.ts +12 -2
- package/types/savings-snapshot.d.ts +11 -0
- package/types/theo-agent-store.d.ts +27 -0
- package/types/theo-agent.d.ts +247 -0
- package/types/theo-gate.d.ts +75 -0
- package/types/theo-policy.d.ts +100 -0
- package/types/types.d.ts +24 -0
- package/types/wrap.d.ts +8 -0
- package/types/cache-ttl-advisor.d.ts +0 -31
- package/types/core/replay-runner.d.ts +0 -11
- package/types/core/routing-simulation.d.ts +0 -24
package/index.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
// packages/sdk-js/src/openai.ts
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
|
|
4
|
+
// packages/core/dist/src/routing.js
|
|
5
|
+
var ROLLOUT_LADDER = [0.01, 0.05, 0.1, 0.25, 0.5, 1];
|
|
6
|
+
var ROLLOUT_LADDER_TOP = ROLLOUT_LADDER.length - 1;
|
|
7
|
+
|
|
4
8
|
// packages/core/dist/src/cost.js
|
|
5
9
|
function estimateTokenCost(inputTokens, outputTokens, price, cacheUsage = {}) {
|
|
6
10
|
const cached = cacheUsage.cachedInputTokens ?? 0;
|
|
@@ -573,7 +577,33 @@ function resolveModelId(catalogue, provider, rawModelId) {
|
|
|
573
577
|
}
|
|
574
578
|
|
|
575
579
|
// packages/core/dist/src/tool-schema.js
|
|
576
|
-
var PLUMBING_KEYS = /* @__PURE__ */ new Set(["$schema", "$id", "$comment"
|
|
580
|
+
var PLUMBING_KEYS = /* @__PURE__ */ new Set(["$schema", "$id", "$comment"]);
|
|
581
|
+
var NAME_MAP_KEYS = /* @__PURE__ */ new Set([
|
|
582
|
+
"properties",
|
|
583
|
+
"patternProperties",
|
|
584
|
+
"$defs",
|
|
585
|
+
"definitions",
|
|
586
|
+
"dependentSchemas",
|
|
587
|
+
"dependencies"
|
|
588
|
+
]);
|
|
589
|
+
var DATA_KEYS = /* @__PURE__ */ new Set(["default", "const", "enum", "examples", "example", "input_examples"]);
|
|
590
|
+
var REFERENCE_KEYS = /* @__PURE__ */ new Set(["$ref", "$dynamicRef", "$recursiveRef"]);
|
|
591
|
+
function usesReferences(node) {
|
|
592
|
+
if (Array.isArray(node))
|
|
593
|
+
return node.some(usesReferences);
|
|
594
|
+
if (typeof node !== "object" || node === null)
|
|
595
|
+
return false;
|
|
596
|
+
for (const [key, value] of Object.entries(node)) {
|
|
597
|
+
if (REFERENCE_KEYS.has(key) && typeof value === "string")
|
|
598
|
+
return true;
|
|
599
|
+
if (usesReferences(value))
|
|
600
|
+
return true;
|
|
601
|
+
}
|
|
602
|
+
return false;
|
|
603
|
+
}
|
|
604
|
+
function isPlainRecord(value) {
|
|
605
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
606
|
+
}
|
|
577
607
|
function toolNameOf(tool, index) {
|
|
578
608
|
if (typeof tool === "object" && tool !== null) {
|
|
579
609
|
const row = tool;
|
|
@@ -619,6 +649,7 @@ function minifyToolSchemas(tools, options = {}) {
|
|
|
619
649
|
const tokensBefore = countTokens(JSON.stringify(tools));
|
|
620
650
|
const keep = options.keepTools === void 0 ? void 0 : new Set(options.keepTools);
|
|
621
651
|
const retained = [];
|
|
652
|
+
let keepIdentifiers = false;
|
|
622
653
|
tools.forEach((tool, index) => {
|
|
623
654
|
const name = toolNameOf(tool, index);
|
|
624
655
|
if (keep !== void 0 && !keep.has(name)) {
|
|
@@ -631,17 +662,23 @@ function minifyToolSchemas(tools, options = {}) {
|
|
|
631
662
|
});
|
|
632
663
|
return;
|
|
633
664
|
}
|
|
665
|
+
keepIdentifiers = usesReferences(tool);
|
|
634
666
|
retained.push(prune(tool, name));
|
|
635
667
|
});
|
|
636
|
-
function prune(node, path) {
|
|
668
|
+
function prune(node, path, names = false) {
|
|
637
669
|
if (Array.isArray(node))
|
|
638
670
|
return node.map((item, index) => prune(item, `${path}[${index}]`));
|
|
639
671
|
if (typeof node !== "object" || node === null)
|
|
640
672
|
return node;
|
|
641
673
|
const row = node;
|
|
642
674
|
const output = {};
|
|
675
|
+
if (names) {
|
|
676
|
+
for (const [key, value] of Object.entries(row))
|
|
677
|
+
output[key] = prune(value, `${path}.${key}`);
|
|
678
|
+
return output;
|
|
679
|
+
}
|
|
643
680
|
for (const [key, value] of Object.entries(row)) {
|
|
644
|
-
if (PLUMBING_KEYS.has(key)) {
|
|
681
|
+
if (PLUMBING_KEYS.has(key) && !(key === "$id" && keepIdentifiers)) {
|
|
645
682
|
changes.push({
|
|
646
683
|
kind: "plumbing",
|
|
647
684
|
lossless: true,
|
|
@@ -698,7 +735,11 @@ function minifyToolSchemas(tools, options = {}) {
|
|
|
698
735
|
output[key] = trimmed;
|
|
699
736
|
continue;
|
|
700
737
|
}
|
|
701
|
-
|
|
738
|
+
if (DATA_KEYS.has(key)) {
|
|
739
|
+
output[key] = structuredClone(value);
|
|
740
|
+
continue;
|
|
741
|
+
}
|
|
742
|
+
output[key] = prune(value, `${path}.${key}`, NAME_MAP_KEYS.has(key) && isPlainRecord(value));
|
|
702
743
|
}
|
|
703
744
|
return output;
|
|
704
745
|
}
|
|
@@ -737,6 +778,271 @@ function auditToolCatalogue(tools, options = {}) {
|
|
|
737
778
|
};
|
|
738
779
|
}
|
|
739
780
|
|
|
781
|
+
// packages/core/dist/src/savings-model.js
|
|
782
|
+
var INPUT_SEGMENTS = [
|
|
783
|
+
"toolSchemas",
|
|
784
|
+
"systemPrompt",
|
|
785
|
+
"retrievedContext",
|
|
786
|
+
"history",
|
|
787
|
+
"userTurn"
|
|
788
|
+
];
|
|
789
|
+
function assertFraction(value, label) {
|
|
790
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) {
|
|
791
|
+
throw new Error(`${label} must be between 0 and 1.`);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
var PUBLISHED_LEVERS = {
|
|
795
|
+
toolSchemaCompaction: {
|
|
796
|
+
id: "tool-schema-compaction",
|
|
797
|
+
description: "Remove JSON indentation, schema plumbing the model never reads ($schema, $id, $comment), and `title` fields that only restate their property name. The decoded schema is semantically identical.",
|
|
798
|
+
risk: "deterministic",
|
|
799
|
+
targets: ["toolSchemas"],
|
|
800
|
+
// Measured per catalogue by auditToolCatalogue; this is a conservative stand-in.
|
|
801
|
+
reduction: 0.15,
|
|
802
|
+
evidence: {
|
|
803
|
+
source: "Measured directly from the request's own tool definitions by auditToolCatalogue",
|
|
804
|
+
claim: "Formatting and plumbing only. Tool selection cannot change because the decoded schema is unchanged."
|
|
805
|
+
}
|
|
806
|
+
},
|
|
807
|
+
toolSchemaMinification: {
|
|
808
|
+
id: "tool-schema-description-trim",
|
|
809
|
+
description: "Trim prose descriptions and docstrings in tool/MCP JSON Schema, keeping names, types and required fields. Agent runtimes resend the whole catalogue on every turn because the APIs are stateless, so a 20-turn task pays for it 20 times. Descriptions are how the model chooses between tools, so this is a behaviour change.",
|
|
810
|
+
risk: "needs-verification",
|
|
811
|
+
targets: ["toolSchemas"],
|
|
812
|
+
// Sources report >76% on a single schema and 30-60% across large catalogues.
|
|
813
|
+
reduction: 0.6,
|
|
814
|
+
evidence: {
|
|
815
|
+
source: "github.com/olivomarco/github-copilot-token-optimization, InfoQ on GitHub's agent audits",
|
|
816
|
+
claim: "Schema minification took one catalogue from 3,198 to under 750 tokens (>76%); tool-schema compression reduces per-request overhead 30-60%; GitHub cut agent workflow token spend up to 62% with MCP pruning."
|
|
817
|
+
}
|
|
818
|
+
},
|
|
819
|
+
unusedToolRemoval: {
|
|
820
|
+
id: "unused-tool-removal",
|
|
821
|
+
description: "Drop tools the workload never calls. A typical MCP server exposes 20-30 tools at 300-500 tokens each, and six connected servers run to 60,000-90,000 tokens of catalogue per request. Removing a tool removes the model's ability to call it, so this holds only where the tool is genuinely unreachable for the workload.",
|
|
822
|
+
risk: "needs-verification",
|
|
823
|
+
targets: ["toolSchemas"],
|
|
824
|
+
reduction: 0.3,
|
|
825
|
+
evidence: {
|
|
826
|
+
source: "mindstudio.ai MCP optimisation, buildmvpfast MCP token cost analysis",
|
|
827
|
+
claim: "Removing unused MCP entries cuts 8-12 KB of context per call; unused tools are the most common inefficiency."
|
|
828
|
+
}
|
|
829
|
+
},
|
|
830
|
+
progressiveDisclosure: {
|
|
831
|
+
id: "progressive-disclosure",
|
|
832
|
+
description: "Replace the tool catalogue with a search/meta tool the model queries for the definitions it needs, so the full catalogue is never sent. Cuts the prompt hard, but adds a round trip per novel tool invocation.",
|
|
833
|
+
risk: "needs-verification",
|
|
834
|
+
targets: ["toolSchemas"],
|
|
835
|
+
// Solo.io measured 10,877 -> 970 prompt tokens, a 91.1% cut.
|
|
836
|
+
reduction: 0.91,
|
|
837
|
+
// The reason this lever usually disappoints. Archestra measured ~77% modelled
|
|
838
|
+
// against ~13% in production on a 20-step run; 1.5 is a conservative middle.
|
|
839
|
+
roundTripMultiplier: 1.5,
|
|
840
|
+
evidence: {
|
|
841
|
+
source: "solo.io agentgateway progressive disclosure; archestra.ai modelled-vs-measured bench",
|
|
842
|
+
claim: "Prompt overhead fell 10,877 to 970 tokens (91.1%) on Claude Sonnet 4.6. But a router adds a hop: the same approach modelled at ~77% measured ~13% in production, because every novel tool call costs an extra round trip and an extra model decision."
|
|
843
|
+
}
|
|
844
|
+
},
|
|
845
|
+
codeMode: {
|
|
846
|
+
id: "code-mode",
|
|
847
|
+
description: "Expose a handful of meta-tools (list, describe, call, call-sequential) and let the model write code that orchestrates the real tools in a sandbox, so it never sees the full definitions. Cuts the prompt and the number of calls at the same time.",
|
|
848
|
+
risk: "needs-verification",
|
|
849
|
+
targets: ["toolSchemas"],
|
|
850
|
+
// Bifrost measured 92.8% at 508 tools; Anthropic reported 150k -> 2k (98.7%).
|
|
851
|
+
reduction: 0.9,
|
|
852
|
+
// The reason this one holds up: 3-4x fewer round trips, modelled at the
|
|
853
|
+
// conservative end of that range.
|
|
854
|
+
roundTripMultiplier: 0.35,
|
|
855
|
+
evidence: {
|
|
856
|
+
source: "Bifrost code mode benchmarks; Anthropic engineering on code execution with MCP",
|
|
857
|
+
claim: "Bifrost cut input tokens up to 92.8% at 508 tools with a 100% pass rate and ran ~40% faster, needing 3-4x fewer round trips. Anthropic reported a workflow falling from 150,000 to 2,000 tokens (98.7%) rewritten as code execution. Requires a sandbox and changes how the agent is built."
|
|
858
|
+
}
|
|
859
|
+
},
|
|
860
|
+
retrievedContextPruning: {
|
|
861
|
+
id: "retrieved-context-pruning",
|
|
862
|
+
description: "Rerank retrieved chunks and keep only those that add information, instead of passing a fixed large top-k.",
|
|
863
|
+
risk: "needs-verification",
|
|
864
|
+
targets: ["retrievedContext"],
|
|
865
|
+
// Sources span 50-90%; held well below that.
|
|
866
|
+
reduction: 0.6,
|
|
867
|
+
evidence: {
|
|
868
|
+
source: "arXiv:2601.17532 (Information Gain Pruning), AWS Bedrock query-aware compression",
|
|
869
|
+
claim: "IGP cut final-stage input tokens 76-79% while improving average F1 by 12-20%; rerank plus compression reached 10.1x fewer tokens at 36% cost saving; right-sized chunks with top-k 3-8 cut context 50-90%."
|
|
870
|
+
}
|
|
871
|
+
},
|
|
872
|
+
promptCompression: {
|
|
873
|
+
id: "prompt-compression",
|
|
874
|
+
description: "Token-level compression of the variable prompt body, dropping tokens a small model judges predictable.",
|
|
875
|
+
risk: "needs-verification",
|
|
876
|
+
targets: ["history", "retrievedContext"],
|
|
877
|
+
// LLMLingua reports up to 20x (95%); 50% is far inside that.
|
|
878
|
+
reduction: 0.5,
|
|
879
|
+
evidence: {
|
|
880
|
+
source: "arXiv:2403.12968 (LLMLingua-2), Microsoft Research",
|
|
881
|
+
claim: "Up to 20x compression with a 1.5 point drop on GSM8K and BBH, and under 2% quality loss on CoQA, HotpotQA and TriviaQA."
|
|
882
|
+
}
|
|
883
|
+
},
|
|
884
|
+
historyCompaction: {
|
|
885
|
+
id: "history-compaction",
|
|
886
|
+
description: "Summarise or window old conversation turns rather than resending every turn verbatim.",
|
|
887
|
+
risk: "needs-verification",
|
|
888
|
+
targets: ["history"],
|
|
889
|
+
reduction: 0.4,
|
|
890
|
+
evidence: {
|
|
891
|
+
source: "Provider guidance on context management; measured per workload by replay",
|
|
892
|
+
claim: "Workload-specific. Conservative placeholder until measured on real traffic."
|
|
893
|
+
}
|
|
894
|
+
},
|
|
895
|
+
outputBudget: {
|
|
896
|
+
id: "output-budget",
|
|
897
|
+
description: "Match reasoning and output budget to task difficulty via reasoning_effort or budget_tokens, instead of letting every request reason at full effort. Output is the expensive half of the bill: 5x input on Sonnet 5.5 and 8x on gpt-5.",
|
|
898
|
+
risk: "needs-verification",
|
|
899
|
+
targets: ["output"],
|
|
900
|
+
// TALE-EP reports 67%; held slightly below.
|
|
901
|
+
reduction: 0.6,
|
|
902
|
+
evidence: {
|
|
903
|
+
source: "TALE-EP; arXiv:2508.17196 (BudgetThinker); provider reasoning-effort controls",
|
|
904
|
+
claim: "67% average output token reduction at under 3% accuracy drop across seven datasets. On GSM8K accuracy rose from 81.35% to 84.46% while mean output fell from 318 to 77 tokens, so shorter was also better."
|
|
905
|
+
}
|
|
906
|
+
},
|
|
907
|
+
duplicateContextRemoval: {
|
|
908
|
+
id: "duplicate-context-removal",
|
|
909
|
+
description: "Remove exact duplicate context blocks. Overlapping retrieval chunks and system prompts echoed into the first user turn are routine; the model already has the text.",
|
|
910
|
+
risk: "deterministic",
|
|
911
|
+
targets: ["retrievedContext", "history"],
|
|
912
|
+
reduction: 0.1,
|
|
913
|
+
evidence: {
|
|
914
|
+
source: "Measured directly from the customer's own traffic by dedupeRepeatedBlocks",
|
|
915
|
+
claim: "Exact duplicates only, so removal cannot change the prompt's meaning."
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
function projectSavings(profile, levers, price, options = {}) {
|
|
920
|
+
if (!Number.isInteger(profile.requests) || profile.requests < 1) {
|
|
921
|
+
throw new Error("A workload profile needs at least one request.");
|
|
922
|
+
}
|
|
923
|
+
for (const [segment, tokens] of Object.entries(profile.segments)) {
|
|
924
|
+
if (!Number.isFinite(tokens) || tokens < 0) {
|
|
925
|
+
throw new Error(`Segment ${segment} must hold a non-negative token count.`);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
for (const lever of levers) {
|
|
929
|
+
assertFraction(lever.reduction, `Lever ${lever.id} reduction`);
|
|
930
|
+
if (lever.roundTripMultiplier !== void 0 && (!Number.isFinite(lever.roundTripMultiplier) || lever.roundTripMultiplier <= 0)) {
|
|
931
|
+
throw new Error(`Lever ${lever.id} roundTripMultiplier must be a positive finite number.`);
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const minimumCacheable = options.minimumCacheableTokens ?? 1024;
|
|
935
|
+
const warnings = [];
|
|
936
|
+
const remaining = { ...profile.segments };
|
|
937
|
+
const inputTokensBefore = INPUT_SEGMENTS.reduce((sum, segment) => sum + profile.segments[segment], 0);
|
|
938
|
+
const outputTokensBefore = profile.segments.output;
|
|
939
|
+
let callMultiplier = 1;
|
|
940
|
+
const costOf = (input, output) => estimateTokenCost(input, output, price) * profile.requests * callMultiplier;
|
|
941
|
+
const baselineCostUsd = costOf(inputTokensBefore, outputTokensBefore);
|
|
942
|
+
let runningCost = baselineCostUsd;
|
|
943
|
+
const contributions = [];
|
|
944
|
+
for (const lever of levers) {
|
|
945
|
+
let tokensRemoved = 0;
|
|
946
|
+
for (const segment of lever.targets) {
|
|
947
|
+
const removed = remaining[segment] * lever.reduction;
|
|
948
|
+
remaining[segment] -= removed;
|
|
949
|
+
tokensRemoved += removed;
|
|
950
|
+
}
|
|
951
|
+
callMultiplier *= lever.roundTripMultiplier ?? 1;
|
|
952
|
+
const input = INPUT_SEGMENTS.reduce((sum, segment) => sum + remaining[segment], 0);
|
|
953
|
+
const cost = costOf(input, remaining.output);
|
|
954
|
+
const costRemovedUsd = runningCost - cost;
|
|
955
|
+
contributions.push({
|
|
956
|
+
lever: lever.id,
|
|
957
|
+
risk: lever.risk,
|
|
958
|
+
tokensRemoved: tokensRemoved * profile.requests,
|
|
959
|
+
costRemovedUsd,
|
|
960
|
+
sharePercent: baselineCostUsd === 0 ? 0 : costRemovedUsd / baselineCostUsd * 100,
|
|
961
|
+
evidence: lever.evidence
|
|
962
|
+
});
|
|
963
|
+
runningCost = cost;
|
|
964
|
+
}
|
|
965
|
+
let cacheableTokens = 0;
|
|
966
|
+
for (const segment of profile.stableSegments) {
|
|
967
|
+
if (!INPUT_SEGMENTS.includes(segment)) {
|
|
968
|
+
warnings.push(`Segment ${segment} cannot be part of a cached prefix and was ignored for caching.`);
|
|
969
|
+
continue;
|
|
970
|
+
}
|
|
971
|
+
cacheableTokens += remaining[segment];
|
|
972
|
+
}
|
|
973
|
+
const inputTokensAfter = INPUT_SEGMENTS.reduce((sum, segment) => sum + remaining[segment], 0);
|
|
974
|
+
if (price.cachedInputPerMillion === void 0) {
|
|
975
|
+
if (cacheableTokens > 0) {
|
|
976
|
+
warnings.push("The supplied price has no cached-input rate, so no caching saving is projected.");
|
|
977
|
+
}
|
|
978
|
+
cacheableTokens = 0;
|
|
979
|
+
} else if (cacheableTokens < minimumCacheable) {
|
|
980
|
+
if (cacheableTokens > 0) {
|
|
981
|
+
warnings.push(`The stable prefix is ${Math.round(cacheableTokens)} tokens, below the provider's ${minimumCacheable}-token cacheable minimum, so no caching saving is projected.`);
|
|
982
|
+
}
|
|
983
|
+
cacheableTokens = 0;
|
|
984
|
+
}
|
|
985
|
+
if (profile.requests === 1 && cacheableTokens > 0) {
|
|
986
|
+
warnings.push("A single request cannot benefit from a prefix cache; it only populates it.");
|
|
987
|
+
cacheableTokens = 0;
|
|
988
|
+
}
|
|
989
|
+
let projectedCostUsd = runningCost;
|
|
990
|
+
if (cacheableTokens > 0) {
|
|
991
|
+
const body = inputTokensAfter - cacheableTokens;
|
|
992
|
+
const write = price.cacheWriteInputPerMillion === void 0 ? estimateTokenCost(cacheableTokens, 0, price) : estimateTokenCost(cacheableTokens, 0, price, { cacheWriteInputTokens: cacheableTokens });
|
|
993
|
+
const read = estimateTokenCost(cacheableTokens, 0, price, { cachedInputTokens: cacheableTokens });
|
|
994
|
+
const perRequestRest = estimateTokenCost(body, remaining.output, price);
|
|
995
|
+
const calls = profile.requests * callMultiplier;
|
|
996
|
+
const cached = write + Math.max(0, calls - 1) * read + calls * perRequestRest;
|
|
997
|
+
contributions.push({
|
|
998
|
+
lever: "prefix-caching",
|
|
999
|
+
risk: "deterministic",
|
|
1000
|
+
tokensRemoved: 0,
|
|
1001
|
+
costRemovedUsd: projectedCostUsd - cached,
|
|
1002
|
+
sharePercent: baselineCostUsd === 0 ? 0 : (projectedCostUsd - cached) / baselineCostUsd * 100,
|
|
1003
|
+
evidence: {
|
|
1004
|
+
source: "Provider published rates, captured in model-prices.ts",
|
|
1005
|
+
claim: "Cached input is billed at a fraction of fresh input (Sonnet 5.5 $2.00 to $0.10; gpt-5 $1.25 to $0.125). The same tokens are sent and the same answer returns, so only the rate changes."
|
|
1006
|
+
}
|
|
1007
|
+
});
|
|
1008
|
+
projectedCostUsd = cached;
|
|
1009
|
+
}
|
|
1010
|
+
const savingUsd = baselineCostUsd - projectedCostUsd;
|
|
1011
|
+
const share = (risk) => contributions.filter((contribution) => contribution.risk === risk).reduce((sum, contribution) => sum + contribution.sharePercent, 0);
|
|
1012
|
+
return {
|
|
1013
|
+
workload: profile.name,
|
|
1014
|
+
requests: profile.requests,
|
|
1015
|
+
baselineCostUsd,
|
|
1016
|
+
projectedCostUsd,
|
|
1017
|
+
savingUsd,
|
|
1018
|
+
savingPercent: baselineCostUsd === 0 ? 0 : savingUsd / baselineCostUsd * 100,
|
|
1019
|
+
deterministicSavingPercent: share("deterministic"),
|
|
1020
|
+
needsVerificationSavingPercent: share("needs-verification"),
|
|
1021
|
+
contributions,
|
|
1022
|
+
inputTokensBefore: inputTokensBefore * profile.requests,
|
|
1023
|
+
inputTokensAfter: inputTokensAfter * profile.requests * callMultiplier,
|
|
1024
|
+
outputTokensBefore: outputTokensBefore * profile.requests,
|
|
1025
|
+
outputTokensAfter: remaining.output * profile.requests * callMultiplier,
|
|
1026
|
+
cachedInputTokens: cacheableTokens * profile.requests * callMultiplier,
|
|
1027
|
+
roundTripsBefore: profile.requests,
|
|
1028
|
+
roundTripsAfter: profile.requests * callMultiplier,
|
|
1029
|
+
warnings
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
function leversFor(profile) {
|
|
1033
|
+
const order = [
|
|
1034
|
+
PUBLISHED_LEVERS.toolSchemaCompaction,
|
|
1035
|
+
PUBLISHED_LEVERS.unusedToolRemoval,
|
|
1036
|
+
PUBLISHED_LEVERS.toolSchemaMinification,
|
|
1037
|
+
PUBLISHED_LEVERS.duplicateContextRemoval,
|
|
1038
|
+
PUBLISHED_LEVERS.retrievedContextPruning,
|
|
1039
|
+
PUBLISHED_LEVERS.historyCompaction,
|
|
1040
|
+
PUBLISHED_LEVERS.promptCompression,
|
|
1041
|
+
PUBLISHED_LEVERS.outputBudget
|
|
1042
|
+
];
|
|
1043
|
+
return order.filter((lever) => lever.targets.some((segment) => (profile.segments[segment] ?? 0) > 0));
|
|
1044
|
+
}
|
|
1045
|
+
|
|
740
1046
|
// packages/core/dist/src/savings-ledger.js
|
|
741
1047
|
import { createHash as createHash3 } from "node:crypto";
|
|
742
1048
|
var LEDGER_GENESIS_HASH = "0".repeat(64);
|
|
@@ -2021,6 +2327,12 @@ function measureOutputReEmission(requestText, outputText, countTokens = approxim
|
|
|
2021
2327
|
};
|
|
2022
2328
|
}
|
|
2023
2329
|
|
|
2330
|
+
// packages/core/dist/src/security/hash-chain.js
|
|
2331
|
+
var CHAIN_GENESIS_HASH = "0".repeat(64);
|
|
2332
|
+
|
|
2333
|
+
// packages/core/dist/src/github-oauth.js
|
|
2334
|
+
var STATE_MAX_AGE_MS = 10 * 60 * 1e3;
|
|
2335
|
+
|
|
2024
2336
|
// packages/sdk-js/src/telemetry.ts
|
|
2025
2337
|
var TelemetryClient = class {
|
|
2026
2338
|
constructor(config = {}) {
|
|
@@ -2194,6 +2506,14 @@ ${serialized}`).digest("hex");
|
|
|
2194
2506
|
import { createHash as createHash7 } from "node:crypto";
|
|
2195
2507
|
var isPlainObject5 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2196
2508
|
var digest = (value) => createHash7("sha256").update(JSON.stringify(value ?? null) ?? "null").digest("hex").slice(0, 32);
|
|
2509
|
+
function advance(prior, churned) {
|
|
2510
|
+
if (churned) return { track: { volatile: true, stableStreak: 0 }, ok: false, proving: false };
|
|
2511
|
+
if (!prior.volatile) return { track: prior, ok: true, proving: false };
|
|
2512
|
+
const stableStreak = prior.stableStreak + 1;
|
|
2513
|
+
if (stableStreak >= 2) return { track: { volatile: false, stableStreak: 0 }, ok: true, proving: false };
|
|
2514
|
+
return { track: { volatile: true, stableStreak }, ok: false, proving: true };
|
|
2515
|
+
}
|
|
2516
|
+
var PROVING = "this conversation's prefix changed on an earlier turn; waiting for it to prove stable before stamping again";
|
|
2197
2517
|
var CachePrefixVolatilityGuard = class {
|
|
2198
2518
|
constructor(maxConversations = 1e3) {
|
|
2199
2519
|
this.maxConversations = maxConversations;
|
|
@@ -2205,48 +2525,49 @@ var CachePrefixVolatilityGuard = class {
|
|
|
2205
2525
|
/** Insertion-ordered; re-inserted on touch, so the first key is the LRU. */
|
|
2206
2526
|
conversations = /* @__PURE__ */ new Map();
|
|
2207
2527
|
/**
|
|
2208
|
-
* Record this request and decide
|
|
2209
|
-
*
|
|
2528
|
+
* Record this request and decide which cache breakpoints its conversation
|
|
2529
|
+
* has earned. Call once per request, with the raw (unreduced) request and,
|
|
2530
|
+
* when known, the content the system breakpoint would cover.
|
|
2210
2531
|
*/
|
|
2211
|
-
observe(request) {
|
|
2532
|
+
observe(request, context = {}) {
|
|
2212
2533
|
if (!isPlainObject5(request) || !Array.isArray(request.messages) || request.messages.length === 0) {
|
|
2213
2534
|
return { stamp: true };
|
|
2214
2535
|
}
|
|
2215
2536
|
const key = digest([request.model ?? null, request.messages[0]]);
|
|
2216
|
-
const prefixHash = digest([request.tools ?? null, request.system ?? null])
|
|
2537
|
+
const prefixHash = context.stampedPrefix !== void 0 ? `span:${digest(context.stampedPrefix)}` : `raw:${digest([request.tools ?? null, request.system ?? null])}`;
|
|
2217
2538
|
const messageHashes = request.messages.map(digest);
|
|
2218
2539
|
const prior = this.conversations.get(key);
|
|
2219
2540
|
if (prior !== void 0) this.conversations.delete(key);
|
|
2220
2541
|
if (prior === void 0) {
|
|
2221
|
-
this.conversations.set(key, {
|
|
2542
|
+
this.conversations.set(key, {
|
|
2543
|
+
prefixHash,
|
|
2544
|
+
messageHashes,
|
|
2545
|
+
prefix: { volatile: false, stableStreak: 0 },
|
|
2546
|
+
history: { volatile: false, stableStreak: 0 }
|
|
2547
|
+
});
|
|
2222
2548
|
this.evict();
|
|
2223
2549
|
return { stamp: true };
|
|
2224
2550
|
}
|
|
2225
2551
|
const prefixChanged = prior.prefixHash !== prefixHash;
|
|
2226
2552
|
const historyExtends = messageHashes.length >= prior.messageHashes.length && prior.messageHashes.every((hash, index) => messageHashes[index] === hash);
|
|
2227
|
-
const
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
decision = {
|
|
2553
|
+
const prefix = advance(prior.prefix, prefixChanged);
|
|
2554
|
+
const history = advance(prior.history, !historyExtends);
|
|
2555
|
+
this.conversations.set(key, { prefixHash, messageHashes, prefix: prefix.track, history: history.track });
|
|
2556
|
+
if (!prefix.ok) {
|
|
2557
|
+
return {
|
|
2233
2558
|
stamp: false,
|
|
2234
|
-
|
|
2559
|
+
messageBreakpoint: false,
|
|
2560
|
+
reason: prefixChanged ? "tools/system changed between turns of this conversation, so a cached prefix could never be read back" : PROVING
|
|
2235
2561
|
};
|
|
2236
|
-
} else if (entry2.volatile) {
|
|
2237
|
-
entry2.stableStreak += 1;
|
|
2238
|
-
if (entry2.stableStreak >= 2) {
|
|
2239
|
-
entry2.volatile = false;
|
|
2240
|
-
entry2.stableStreak = 0;
|
|
2241
|
-
} else {
|
|
2242
|
-
decision = {
|
|
2243
|
-
stamp: false,
|
|
2244
|
-
reason: "this conversation's prefix changed on an earlier turn; waiting for it to prove stable before stamping again"
|
|
2245
|
-
};
|
|
2246
|
-
}
|
|
2247
2562
|
}
|
|
2248
|
-
|
|
2249
|
-
|
|
2563
|
+
if (!history.ok) {
|
|
2564
|
+
return {
|
|
2565
|
+
stamp: true,
|
|
2566
|
+
messageBreakpoint: false,
|
|
2567
|
+
reason: history.proving ? "this conversation's history was rewritten on an earlier turn; waiting for it to prove stable before stamping the newest message again" : "the conversation history was rewritten between turns, so a breakpoint on the newest message could never be read back"
|
|
2568
|
+
};
|
|
2569
|
+
}
|
|
2570
|
+
return { stamp: true };
|
|
2250
2571
|
}
|
|
2251
2572
|
evict() {
|
|
2252
2573
|
while (this.conversations.size > this.maxConversations) {
|
|
@@ -2541,7 +2862,7 @@ function reduceMessageContent(content, config, countTokens, changes) {
|
|
|
2541
2862
|
}
|
|
2542
2863
|
return content;
|
|
2543
2864
|
}
|
|
2544
|
-
function stampAnthropicCacheTtl(next, ttl, systemVolatileTail, countTokens) {
|
|
2865
|
+
function stampAnthropicCacheTtl(next, ttl, systemVolatileTail, countTokens, allowMessageBreakpoint = true) {
|
|
2545
2866
|
const model = typeof next.model === "string" ? next.model.toLowerCase() : "";
|
|
2546
2867
|
if (!model.includes("claude")) return { stamped: false };
|
|
2547
2868
|
if (!Array.isArray(next.messages) || next.messages.length === 0) return { stamped: false };
|
|
@@ -2610,7 +2931,7 @@ ${tailText}`))) {
|
|
|
2610
2931
|
}
|
|
2611
2932
|
}
|
|
2612
2933
|
}
|
|
2613
|
-
if (!volatileTailPresent && next.messages.length >= 2) {
|
|
2934
|
+
if (allowMessageBreakpoint && !volatileTailPresent && next.messages.length >= 2) {
|
|
2614
2935
|
const messages = [...next.messages];
|
|
2615
2936
|
const lastIndex = messages.length - 1;
|
|
2616
2937
|
const last = messages[lastIndex];
|
|
@@ -2648,6 +2969,20 @@ ${tailText}`))) {
|
|
|
2648
2969
|
}
|
|
2649
2970
|
return stamped ? { stamped: true } : { stamped: false, belowMinimum };
|
|
2650
2971
|
}
|
|
2972
|
+
function stampedSystemPrefix(stamped) {
|
|
2973
|
+
if (!Array.isArray(stamped.system)) return void 0;
|
|
2974
|
+
let last = -1;
|
|
2975
|
+
stamped.system.forEach((block, index) => {
|
|
2976
|
+
if (isPlainObject6(block) && block.cache_control !== void 0) last = index;
|
|
2977
|
+
});
|
|
2978
|
+
if (last < 0) return void 0;
|
|
2979
|
+
const span = stamped.system.slice(0, last + 1).map((block) => {
|
|
2980
|
+
if (!isPlainObject6(block)) return block;
|
|
2981
|
+
const { cache_control: _marker, ...rest } = block;
|
|
2982
|
+
return rest;
|
|
2983
|
+
});
|
|
2984
|
+
return [stamped.model ?? null, stamped.tools ?? null, span];
|
|
2985
|
+
}
|
|
2651
2986
|
function reduceRequest(request, config = {}) {
|
|
2652
2987
|
const countTokens = config.countTokens ?? approximateTokenCount;
|
|
2653
2988
|
const changes = [];
|
|
@@ -2674,6 +3009,35 @@ function reduceRequest(request, config = {}) {
|
|
|
2674
3009
|
}
|
|
2675
3010
|
};
|
|
2676
3011
|
const before = measure(next.instructions) + measure(next.system) + measure(next.input) + measure(next.messages) + measure(next.tools);
|
|
3012
|
+
if (Array.isArray(next.tools) && next.tools.length > 0 && Array.isArray(config.dropTools) && config.dropTools.length > 0) {
|
|
3013
|
+
const drop = new Set(config.dropTools.filter((name) => typeof name === "string"));
|
|
3014
|
+
if (drop.size > 0) {
|
|
3015
|
+
const nameOf = (tool) => {
|
|
3016
|
+
if (!isPlainObject6(tool)) return void 0;
|
|
3017
|
+
if (typeof tool.name === "string") return tool.name;
|
|
3018
|
+
if (isPlainObject6(tool.function) && typeof tool.function.name === "string") return tool.function.name;
|
|
3019
|
+
return void 0;
|
|
3020
|
+
};
|
|
3021
|
+
const toolsBefore = measure(next.tools);
|
|
3022
|
+
const kept = next.tools.filter((tool) => {
|
|
3023
|
+
const name = nameOf(tool);
|
|
3024
|
+
return name === void 0 || !drop.has(name);
|
|
3025
|
+
});
|
|
3026
|
+
const removed = next.tools.length - kept.length;
|
|
3027
|
+
if (removed > 0) {
|
|
3028
|
+
next.tools = kept;
|
|
3029
|
+
const saved = toolsBefore - measure(next.tools);
|
|
3030
|
+
if (saved > 0) {
|
|
3031
|
+
changes.push({
|
|
3032
|
+
lever: "unused-tool-removal",
|
|
3033
|
+
lossless: false,
|
|
3034
|
+
estimatedTokensSaved: saved,
|
|
3035
|
+
detail: `Removed ${removed} tool definition(s) the workload does not call; proven on your traffic before billing.`
|
|
3036
|
+
});
|
|
3037
|
+
}
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
2677
3041
|
if (Array.isArray(next.tools) && next.tools.length > 0 && config.toolSchemas !== false) {
|
|
2678
3042
|
const toolOptions = isPlainObject6(config.toolSchemas) ? config.toolSchemas : {};
|
|
2679
3043
|
const minified = minifyToolSchemas(next.tools, {
|
|
@@ -2791,8 +3155,32 @@ function reduceRequest(request, config = {}) {
|
|
|
2791
3155
|
}
|
|
2792
3156
|
if (config.cacheTtl === "5m" || config.cacheTtl === "1h") {
|
|
2793
3157
|
const ttl = config.cacheTtl;
|
|
2794
|
-
const
|
|
2795
|
-
const
|
|
3158
|
+
const trial = { ...next };
|
|
3159
|
+
const trialResult = stampAnthropicCacheTtl(trial, ttl, systemVolatileTail, countTokens);
|
|
3160
|
+
const decision = config.cacheVolatilityGuard?.observe(request, {
|
|
3161
|
+
stampedPrefix: stampedSystemPrefix(trial)
|
|
3162
|
+
}) ?? { stamp: true };
|
|
3163
|
+
const messageBreakpointWithheld = decision.stamp && decision.messageBreakpoint === false && trial.messages !== next.messages;
|
|
3164
|
+
let result = { stamped: false };
|
|
3165
|
+
if (decision.stamp) {
|
|
3166
|
+
const systemStamped = trial.system !== next.system;
|
|
3167
|
+
if (systemStamped) next.system = trial.system;
|
|
3168
|
+
if (decision.messageBreakpoint === false) {
|
|
3169
|
+
result = systemStamped ? { stamped: true } : { stamped: false, belowMinimum: trialResult.belowMinimum };
|
|
3170
|
+
} else {
|
|
3171
|
+
if (trial.messages !== next.messages) next.messages = trial.messages;
|
|
3172
|
+
result = trialResult;
|
|
3173
|
+
}
|
|
3174
|
+
}
|
|
3175
|
+
if (messageBreakpointWithheld) {
|
|
3176
|
+
changes.push({
|
|
3177
|
+
lever: "cache-ttl-suppressed",
|
|
3178
|
+
lossless: true,
|
|
3179
|
+
// Nothing was changed; this records WHY one breakpoint was not placed.
|
|
3180
|
+
estimatedTokensSaved: 0,
|
|
3181
|
+
detail: `Cache breakpoint on the newest message withheld: ${decision.reason ?? "the conversation's history is not stable"}. The tools + system breakpoint is unaffected: it sits before the history and still reads.`
|
|
3182
|
+
});
|
|
3183
|
+
}
|
|
2796
3184
|
if (!decision.stamp) {
|
|
2797
3185
|
changes.push({
|
|
2798
3186
|
lever: "cache-ttl-suppressed",
|
|
@@ -2908,6 +3296,94 @@ function reduceRequest(request, config = {}) {
|
|
|
2908
3296
|
// packages/sdk-js/src/savings-ledger.ts
|
|
2909
3297
|
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2910
3298
|
import { dirname } from "node:path";
|
|
3299
|
+
|
|
3300
|
+
// packages/sdk-js/src/ledger-mirror.ts
|
|
3301
|
+
var LedgerMirror = class {
|
|
3302
|
+
endpoint;
|
|
3303
|
+
accountKey;
|
|
3304
|
+
doFetch;
|
|
3305
|
+
maxRetries;
|
|
3306
|
+
maxPending;
|
|
3307
|
+
/** Records with seq > confirmedTailSeq, awaiting confirmation. */
|
|
3308
|
+
pending = [];
|
|
3309
|
+
confirmedTailSeq = -1;
|
|
3310
|
+
disabled = false;
|
|
3311
|
+
/** A flush is requested again if records arrive while one is in flight. */
|
|
3312
|
+
dirty = false;
|
|
3313
|
+
/** The in-flight flush, so callers (and tests) can await completion. */
|
|
3314
|
+
current;
|
|
3315
|
+
constructor(config) {
|
|
3316
|
+
this.endpoint = `${config.apiUrl.replace(/\/$/, "")}/v1/ledger/events`;
|
|
3317
|
+
this.accountKey = config.accountKey;
|
|
3318
|
+
this.doFetch = config.fetch ?? fetch;
|
|
3319
|
+
this.maxRetries = config.maxRetries ?? 3;
|
|
3320
|
+
this.maxPending = config.maxPending ?? 5e3;
|
|
3321
|
+
}
|
|
3322
|
+
/** Buffer a record and schedule a flush. Never blocks or throws. */
|
|
3323
|
+
enqueue(record) {
|
|
3324
|
+
if (this.disabled) return;
|
|
3325
|
+
if (record.seq <= this.confirmedTailSeq) return;
|
|
3326
|
+
this.pending.push(record);
|
|
3327
|
+
if (this.pending.length > this.maxPending) {
|
|
3328
|
+
this.disabled = true;
|
|
3329
|
+
this.pending = [];
|
|
3330
|
+
return;
|
|
3331
|
+
}
|
|
3332
|
+
void this.flush();
|
|
3333
|
+
}
|
|
3334
|
+
/** Flush the pending buffer to the API, retrying a few times. Awaitable. */
|
|
3335
|
+
flush() {
|
|
3336
|
+
if (this.disabled) return Promise.resolve();
|
|
3337
|
+
if (this.current) {
|
|
3338
|
+
this.dirty = true;
|
|
3339
|
+
return this.current;
|
|
3340
|
+
}
|
|
3341
|
+
this.current = this.run().finally(() => {
|
|
3342
|
+
this.current = void 0;
|
|
3343
|
+
});
|
|
3344
|
+
return this.current;
|
|
3345
|
+
}
|
|
3346
|
+
async run() {
|
|
3347
|
+
do {
|
|
3348
|
+
this.dirty = false;
|
|
3349
|
+
const batch = this.pending.filter((record) => record.seq > this.confirmedTailSeq);
|
|
3350
|
+
if (batch.length === 0) break;
|
|
3351
|
+
const confirmed = await this.send(batch);
|
|
3352
|
+
if (confirmed === void 0) break;
|
|
3353
|
+
this.confirmedTailSeq = Math.max(this.confirmedTailSeq, confirmed);
|
|
3354
|
+
this.pending = this.pending.filter((record) => record.seq > this.confirmedTailSeq);
|
|
3355
|
+
} while (this.dirty || this.pending.length > 0);
|
|
3356
|
+
}
|
|
3357
|
+
/** POST one batch; returns the server's tailSeq on success, or undefined. */
|
|
3358
|
+
async send(batch) {
|
|
3359
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt += 1) {
|
|
3360
|
+
try {
|
|
3361
|
+
const res = await this.doFetch(this.endpoint, {
|
|
3362
|
+
method: "POST",
|
|
3363
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${this.accountKey}` },
|
|
3364
|
+
body: JSON.stringify({ records: batch }),
|
|
3365
|
+
signal: AbortSignal.timeout(1e4)
|
|
3366
|
+
});
|
|
3367
|
+
if (res.ok) {
|
|
3368
|
+
const json = await res.json().catch(() => ({}));
|
|
3369
|
+
return typeof json.tailSeq === "number" ? json.tailSeq : batch[batch.length - 1].seq;
|
|
3370
|
+
}
|
|
3371
|
+
if (res.status !== 429 && res.status < 500) {
|
|
3372
|
+
if (res.status === 401 || res.status === 403) this.disabled = true;
|
|
3373
|
+
return void 0;
|
|
3374
|
+
}
|
|
3375
|
+
} catch {
|
|
3376
|
+
}
|
|
3377
|
+
if (attempt < this.maxRetries) await delay(250 * 2 ** attempt);
|
|
3378
|
+
}
|
|
3379
|
+
return void 0;
|
|
3380
|
+
}
|
|
3381
|
+
};
|
|
3382
|
+
function delay(ms) {
|
|
3383
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
3384
|
+
}
|
|
3385
|
+
|
|
3386
|
+
// packages/sdk-js/src/savings-ledger.ts
|
|
2911
3387
|
var EXECUTION_TIER_LEVERS = /* @__PURE__ */ new Set(["execution-tier-flex"]);
|
|
2912
3388
|
var CACHE_TTL_LEVERS = /* @__PURE__ */ new Map([["cache-ttl-5m", "5m"], ["cache-ttl-1h", "1h"]]);
|
|
2913
3389
|
var MEASURED_COMPOSITE_LEVER = "token-reduction-measured";
|
|
@@ -3086,17 +3562,20 @@ function createSavingsLedger(config) {
|
|
|
3086
3562
|
const tail = resumeChainTail(config.path);
|
|
3087
3563
|
let previous = tail.previous;
|
|
3088
3564
|
let disabled = tail.disabled;
|
|
3565
|
+
const mirror = config.mirror ? new LedgerMirror(config.mirror) : void 0;
|
|
3089
3566
|
return {
|
|
3090
3567
|
path: config.path,
|
|
3091
|
-
emit(event) {
|
|
3568
|
+
emit(event, context) {
|
|
3092
3569
|
if (disabled) return;
|
|
3093
3570
|
try {
|
|
3094
|
-
const
|
|
3571
|
+
const effective = context?.gateVerdicts === void 0 ? config : { ...config, gateVerdicts: { ...config.gateVerdicts ?? {}, ...context.gateVerdicts } };
|
|
3572
|
+
const billable = billableEventFromRequest(event, effective, catalogue, pinnedCatalogueVersion);
|
|
3095
3573
|
const record = chainLedgerRecord(billable, previous);
|
|
3096
3574
|
mkdirSync(dirname(config.path), { recursive: true, mode: 448 });
|
|
3097
3575
|
appendFileSync(config.path, `${JSON.stringify(record)}
|
|
3098
3576
|
`, { encoding: "utf8", mode: 384 });
|
|
3099
3577
|
previous = record;
|
|
3578
|
+
mirror?.enqueue(record);
|
|
3100
3579
|
} catch {
|
|
3101
3580
|
disabled = true;
|
|
3102
3581
|
}
|
|
@@ -3490,9 +3969,10 @@ function createLocalJsonlCapture(path, redact, hooks = {}) {
|
|
|
3490
3969
|
|
|
3491
3970
|
// packages/sdk-js/src/proxy.ts
|
|
3492
3971
|
import { createServer } from "node:http";
|
|
3972
|
+
import { Readable } from "node:stream";
|
|
3493
3973
|
import { createHash as createHash8, randomUUID as randomUUID2 } from "node:crypto";
|
|
3494
|
-
import { appendFileSync as appendFileSync2, mkdirSync as mkdirSync2 } from "node:fs";
|
|
3495
|
-
import { join } from "node:path";
|
|
3974
|
+
import { appendFileSync as appendFileSync2, existsSync as existsSync2, mkdirSync as mkdirSync2, statSync } from "node:fs";
|
|
3975
|
+
import { dirname as dirname3, join } from "node:path";
|
|
3496
3976
|
|
|
3497
3977
|
// packages/sdk-js/src/replay-gate.ts
|
|
3498
3978
|
function isPlainObject7(value) {
|
|
@@ -3724,6 +4204,7 @@ async function replayGate(options) {
|
|
|
3724
4204
|
const costBaseline = priceUsageRow(row.baselineUsage);
|
|
3725
4205
|
const costCandidate = priceUsageRow(row.candidateUsage);
|
|
3726
4206
|
const costFell = costBaseline !== void 0 && costCandidate !== void 0 && costCandidate < costBaseline;
|
|
4207
|
+
const priced = costBaseline !== void 0 && costCandidate !== void 0;
|
|
3727
4208
|
let approved2 = lower >= -maximumQualityDrop;
|
|
3728
4209
|
let rejectionReason;
|
|
3729
4210
|
if (row.graded === 0) {
|
|
@@ -3731,9 +4212,14 @@ async function replayGate(options) {
|
|
|
3731
4212
|
rejectionReason = "No gradeable comparison was produced.";
|
|
3732
4213
|
} else if (!approved2) {
|
|
3733
4214
|
rejectionReason = `Quality lower bound ${(lower * 100).toFixed(2)} pp breaches the ${(maximumQualityDrop * 100).toFixed(2)} pp limit.`;
|
|
3734
|
-
} else if (
|
|
4215
|
+
} else if (priced) {
|
|
4216
|
+
if (!costFell) {
|
|
4217
|
+
approved2 = false;
|
|
4218
|
+
rejectionReason = "Quality held but the bill did not fall once cache writes and reads are priced in (e.g. compacting a prefix the provider was still serving from cache), so there is nothing to enable.";
|
|
4219
|
+
}
|
|
4220
|
+
} else if (inputSaved + outputSaved <= 0) {
|
|
3735
4221
|
approved2 = false;
|
|
3736
|
-
rejectionReason = "Quality held but
|
|
4222
|
+
rejectionReason = "Quality held but no tokens were removed, so there is nothing to enable.";
|
|
3737
4223
|
}
|
|
3738
4224
|
return {
|
|
3739
4225
|
label: row.candidate.label,
|
|
@@ -3847,8 +4333,16 @@ var HOP_HEADERS = /* @__PURE__ */ new Set([
|
|
|
3847
4333
|
"te",
|
|
3848
4334
|
"trailer",
|
|
3849
4335
|
"keep-alive",
|
|
3850
|
-
"proxy-authorization"
|
|
4336
|
+
"proxy-authorization",
|
|
4337
|
+
// `Expect: 100-continue` is settled between the client and this proxy (Node
|
|
4338
|
+
// answers it); forwarding it makes the upstream call fail outright. curl adds
|
|
4339
|
+
// it to every body over 1 MB.
|
|
4340
|
+
"expect",
|
|
4341
|
+
// Cookies belong to whatever else runs on this machine's localhost (cookies
|
|
4342
|
+
// are scoped by host, not port). No provider API authenticates with one.
|
|
4343
|
+
"cookie"
|
|
3851
4344
|
]);
|
|
4345
|
+
var RESPONSE_DROP_HEADERS = /* @__PURE__ */ new Set([...HOP_HEADERS, "content-encoding", "set-cookie"]);
|
|
3852
4346
|
function forwardHeaders(req, provider) {
|
|
3853
4347
|
const out = {};
|
|
3854
4348
|
for (const [name, value] of Object.entries(req.headers)) {
|
|
@@ -3897,6 +4391,16 @@ function surfaceOf(path) {
|
|
|
3897
4391
|
if (path === "/v1/chat/completions") return { provider: "openai", upstreamPath: "/v1/chat/completions" };
|
|
3898
4392
|
return void 0;
|
|
3899
4393
|
}
|
|
4394
|
+
function passthroughProvider(req, path) {
|
|
4395
|
+
if (req.headers["anthropic-version"] !== void 0 || req.headers["anthropic-beta"] !== void 0) return "anthropic";
|
|
4396
|
+
if (req.headers["x-api-key"] !== void 0) return "anthropic";
|
|
4397
|
+
if (path === "/v1/messages" || path.startsWith("/v1/messages/") || path === "/v1/complete") return "anthropic";
|
|
4398
|
+
return path.startsWith("/v1/") ? "openai" : void 0;
|
|
4399
|
+
}
|
|
4400
|
+
function meshError(message) {
|
|
4401
|
+
return JSON.stringify({ type: "error", error: { type: "mesh_proxy_error", message } });
|
|
4402
|
+
}
|
|
4403
|
+
var PASSTHROUGH_BUFFER_CAP = 16 * 1024 * 1024;
|
|
3900
4404
|
function usageFromSse(provider, sseText) {
|
|
3901
4405
|
const events = [];
|
|
3902
4406
|
for (const line of sseText.split(/\r?\n/)) {
|
|
@@ -3955,8 +4459,22 @@ function outputTextFromSse(provider, sseText) {
|
|
|
3955
4459
|
return parts.join("");
|
|
3956
4460
|
}
|
|
3957
4461
|
function startMeshProxy(options) {
|
|
3958
|
-
const
|
|
3959
|
-
const
|
|
4462
|
+
const reductionOption = options.tokenReduction;
|
|
4463
|
+
const resolveBaseReduction = typeof reductionOption === "function" ? reductionOption : () => reductionOption ?? LOSSLESS_DEFAULT;
|
|
4464
|
+
let volatilityGuard;
|
|
4465
|
+
const currentReduction = (modelId) => {
|
|
4466
|
+
let base;
|
|
4467
|
+
try {
|
|
4468
|
+
base = resolveBaseReduction(modelId) ?? LOSSLESS_DEFAULT;
|
|
4469
|
+
} catch {
|
|
4470
|
+
base = LOSSLESS_DEFAULT;
|
|
4471
|
+
}
|
|
4472
|
+
if (base.cacheTtl !== void 0 && base.cacheVolatilityGuard === void 0) {
|
|
4473
|
+
volatilityGuard ??= new CachePrefixVolatilityGuard();
|
|
4474
|
+
return { ...base, cacheVolatilityGuard: volatilityGuard };
|
|
4475
|
+
}
|
|
4476
|
+
return base;
|
|
4477
|
+
};
|
|
3960
4478
|
const upstreams = {
|
|
3961
4479
|
anthropic: (options.anthropicUpstream ?? "https://api.anthropic.com").replace(/\/$/, ""),
|
|
3962
4480
|
openai: (options.openaiUpstream ?? "https://api.openai.com").replace(/\/$/, "")
|
|
@@ -3982,7 +4500,8 @@ ${provider}`;
|
|
|
3982
4500
|
}
|
|
3983
4501
|
const ledger = createSavingsLedger({
|
|
3984
4502
|
path: join(options.ledgerDirectory, `${safeApp}-${provider}.jsonl`),
|
|
3985
|
-
provider
|
|
4503
|
+
provider,
|
|
4504
|
+
...options.meshApiUrl && options.meshAccountKey ? { mirror: { apiUrl: options.meshApiUrl, accountKey: options.meshAccountKey } } : {}
|
|
3986
4505
|
});
|
|
3987
4506
|
ledgers.set(key, ledger);
|
|
3988
4507
|
return ledger;
|
|
@@ -3994,55 +4513,140 @@ ${provider}`;
|
|
|
3994
4513
|
appendFileSync2(join(options.ledgerDirectory, `${safeApp}-${provider}.features.jsonl`), `${JSON.stringify(record)}
|
|
3995
4514
|
`, { mode: 384 });
|
|
3996
4515
|
};
|
|
4516
|
+
const requestCapture = options.requestCapture;
|
|
4517
|
+
const captureLimit = requestCapture?.limit ?? 50;
|
|
4518
|
+
const captureMaxBytes = requestCapture?.maxBytes ?? 5 * 1024 * 1024;
|
|
4519
|
+
let capturedRequests = 0;
|
|
4520
|
+
let capturedBytes = (() => {
|
|
4521
|
+
if (requestCapture === void 0) return 0;
|
|
4522
|
+
try {
|
|
4523
|
+
return existsSync2(requestCapture.path) ? statSync(requestCapture.path).size : 0;
|
|
4524
|
+
} catch {
|
|
4525
|
+
return 0;
|
|
4526
|
+
}
|
|
4527
|
+
})();
|
|
4528
|
+
const writeCapturedRequest = (body) => {
|
|
4529
|
+
if (requestCapture === void 0 || body === void 0 || capturedRequests >= captureLimit) return;
|
|
4530
|
+
try {
|
|
4531
|
+
const line = `${JSON.stringify({ request: body })}
|
|
4532
|
+
`;
|
|
4533
|
+
const lineBytes = Buffer.byteLength(line);
|
|
4534
|
+
if (capturedBytes + lineBytes > captureMaxBytes) return;
|
|
4535
|
+
mkdirSync2(dirname3(requestCapture.path), { recursive: true, mode: 448 });
|
|
4536
|
+
appendFileSync2(requestCapture.path, line, { mode: 384 });
|
|
4537
|
+
capturedRequests += 1;
|
|
4538
|
+
capturedBytes += lineBytes;
|
|
4539
|
+
} catch {
|
|
4540
|
+
}
|
|
4541
|
+
};
|
|
3997
4542
|
const handle = async (req, res) => {
|
|
3998
4543
|
const path = (req.url ?? "").split("?")[0];
|
|
3999
4544
|
if (!hostAllowed(req)) {
|
|
4000
4545
|
res.writeHead(403, { "content-type": "application/json" });
|
|
4001
|
-
res.end(
|
|
4546
|
+
res.end(meshError("MESH proxy serves loopback clients only."));
|
|
4002
4547
|
return;
|
|
4003
4548
|
}
|
|
4004
4549
|
const fetchSite = req.headers["sec-fetch-site"];
|
|
4005
4550
|
if (fetchSite === "cross-site" || fetchSite === "same-site") {
|
|
4006
4551
|
res.writeHead(403, { "content-type": "application/json" });
|
|
4007
|
-
res.end(
|
|
4552
|
+
res.end(meshError("MESH proxy does not serve cross-origin browser requests."));
|
|
4553
|
+
return;
|
|
4554
|
+
}
|
|
4555
|
+
const surface = req.method === "POST" ? surfaceOf(path) : void 0;
|
|
4556
|
+
if (path === "/mesh/health") {
|
|
4557
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
4558
|
+
res.end(req.method === "HEAD" ? void 0 : JSON.stringify({ status: "ok", service: "mesh-proxy" }));
|
|
4559
|
+
return;
|
|
4560
|
+
}
|
|
4561
|
+
const passProvider = surface === void 0 ? passthroughProvider(req, path) : void 0;
|
|
4562
|
+
if (surface === void 0 && (path === "/" || path === "" || passProvider === void 0)) {
|
|
4563
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
4564
|
+
res.end(meshError(
|
|
4565
|
+
"This is the MESH proxy. Point your client's base URL at this server: Anthropic clients at the bare address, OpenAI-style clients at /v1. MESH reduces POST /v1/messages, /v1/responses and /v1/chat/completions and forwards every other request to your provider unchanged."
|
|
4566
|
+
));
|
|
4008
4567
|
return;
|
|
4009
4568
|
}
|
|
4010
|
-
if (
|
|
4011
|
-
const provider =
|
|
4569
|
+
if (surface === void 0 && passProvider !== void 0) {
|
|
4570
|
+
const provider = passProvider;
|
|
4571
|
+
const hasBody = req.method !== "GET" && req.method !== "HEAD";
|
|
4572
|
+
const declaredLength = Number.parseInt(String(req.headers["content-length"] ?? ""), 10);
|
|
4573
|
+
const passControl = new AbortController();
|
|
4574
|
+
const onPassClose = () => {
|
|
4575
|
+
if (!res.writableEnded) passControl.abort(new Error("client disconnected"));
|
|
4576
|
+
};
|
|
4577
|
+
res.on("close", onPassClose);
|
|
4012
4578
|
let upstreamResponse2;
|
|
4013
4579
|
try {
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4580
|
+
const passRequestHeaders = forwardHeaders(req, provider);
|
|
4581
|
+
let init = {
|
|
4582
|
+
method: req.method ?? "GET",
|
|
4583
|
+
headers: passRequestHeaders,
|
|
4584
|
+
signal: passControl.signal,
|
|
4585
|
+
// A redirect is the client's to follow. Followed here, the caller's
|
|
4586
|
+
// key would travel to wherever the provider pointed.
|
|
4587
|
+
redirect: "manual"
|
|
4588
|
+
};
|
|
4589
|
+
if (hasBody) {
|
|
4590
|
+
if (Number.isFinite(declaredLength) && declaredLength <= PASSTHROUGH_BUFFER_CAP) {
|
|
4591
|
+
init = { ...init, body: new Uint8Array(await readBody(req)) };
|
|
4592
|
+
} else {
|
|
4593
|
+
if (Number.isFinite(declaredLength)) passRequestHeaders["content-length"] = String(declaredLength);
|
|
4594
|
+
init = { ...init, body: Readable.toWeb(req), duplex: "half" };
|
|
4595
|
+
}
|
|
4596
|
+
}
|
|
4597
|
+
upstreamResponse2 = await fetch(`${upstreams[provider]}${req.url ?? path}`, init);
|
|
4018
4598
|
} catch (cause) {
|
|
4019
|
-
res.
|
|
4020
|
-
res.
|
|
4599
|
+
res.off("close", onPassClose);
|
|
4600
|
+
if (!res.headersSent && !res.destroyed) {
|
|
4601
|
+
res.writeHead(502, { "content-type": "application/json" });
|
|
4602
|
+
res.end(meshError(`Upstream unreachable: ${cause.message}`));
|
|
4603
|
+
}
|
|
4021
4604
|
return;
|
|
4022
4605
|
}
|
|
4023
4606
|
const passHeaders = {};
|
|
4024
4607
|
upstreamResponse2.headers.forEach((value, name) => {
|
|
4025
|
-
if (!
|
|
4608
|
+
if (!RESPONSE_DROP_HEADERS.has(name.toLowerCase())) passHeaders[name] = value;
|
|
4026
4609
|
});
|
|
4027
4610
|
res.writeHead(upstreamResponse2.status, passHeaders);
|
|
4028
|
-
|
|
4029
|
-
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
4611
|
+
if (upstreamResponse2.body !== null) {
|
|
4612
|
+
const reader = upstreamResponse2.body.getReader();
|
|
4613
|
+
try {
|
|
4614
|
+
for (; ; ) {
|
|
4615
|
+
const { done, value } = await reader.read();
|
|
4616
|
+
if (done || res.destroyed) break;
|
|
4617
|
+
if (!res.write(Buffer.from(value))) {
|
|
4618
|
+
await new Promise((proceed) => {
|
|
4619
|
+
const onDrain = () => {
|
|
4620
|
+
res.off("close", onGone);
|
|
4621
|
+
proceed();
|
|
4622
|
+
};
|
|
4623
|
+
const onGone = () => {
|
|
4624
|
+
res.off("drain", onDrain);
|
|
4625
|
+
proceed();
|
|
4626
|
+
};
|
|
4627
|
+
res.once("drain", onDrain);
|
|
4628
|
+
res.once("close", onGone);
|
|
4629
|
+
});
|
|
4630
|
+
}
|
|
4631
|
+
}
|
|
4632
|
+
} catch {
|
|
4633
|
+
res.off("close", onPassClose);
|
|
4634
|
+
if (!res.destroyed) res.destroy();
|
|
4635
|
+
return;
|
|
4636
|
+
}
|
|
4637
|
+
}
|
|
4638
|
+
res.off("close", onPassClose);
|
|
4639
|
+
if (!res.destroyed) res.end();
|
|
4037
4640
|
return;
|
|
4038
4641
|
}
|
|
4642
|
+
if (surface === void 0) return;
|
|
4039
4643
|
let rawBody;
|
|
4040
4644
|
try {
|
|
4041
4645
|
rawBody = await readBody(req);
|
|
4042
4646
|
} catch (cause) {
|
|
4043
4647
|
if (cause instanceof BodyTooLargeError && !res.headersSent) {
|
|
4044
4648
|
res.writeHead(413, { "content-type": "application/json" });
|
|
4045
|
-
res.end(
|
|
4649
|
+
res.end(meshError(cause.message));
|
|
4046
4650
|
}
|
|
4047
4651
|
return;
|
|
4048
4652
|
}
|
|
@@ -4052,17 +4656,22 @@ ${provider}`;
|
|
|
4052
4656
|
} catch {
|
|
4053
4657
|
body = void 0;
|
|
4054
4658
|
}
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4659
|
+
const readable = typeof body === "object" && body !== null && !Array.isArray(body);
|
|
4660
|
+
if (!readable) body = void 0;
|
|
4661
|
+
const requestModelId = typeof body?.model === "string" ? body.model : void 0;
|
|
4662
|
+
const { request: reducedRequest, summary } = reduceRequest(body, currentReduction(requestModelId));
|
|
4663
|
+
let requestGateVerdicts;
|
|
4664
|
+
try {
|
|
4665
|
+
requestGateVerdicts = options.gateVerdicts?.(requestModelId);
|
|
4666
|
+
} catch {
|
|
4667
|
+
requestGateVerdicts = void 0;
|
|
4059
4668
|
}
|
|
4060
|
-
const
|
|
4061
|
-
const sendBody = JSON.stringify(
|
|
4669
|
+
const reducedText = JSON.stringify(reducedRequest);
|
|
4670
|
+
const sendBody = !readable || reducedText === JSON.stringify(body) ? new Uint8Array(rawBody) : reducedText;
|
|
4062
4671
|
const applicationIdHeader = req.headers["x-mesh-application"];
|
|
4063
4672
|
const applicationId = ((Array.isArray(applicationIdHeader) ? applicationIdHeader[0] : applicationIdHeader) ?? "proxy").slice(0, 128);
|
|
4064
4673
|
const headers = forwardHeaders(req, surface.provider);
|
|
4065
|
-
headers["content-type"] = "application/json";
|
|
4674
|
+
if (readable) headers["content-type"] = "application/json";
|
|
4066
4675
|
const upstreamControl = new AbortController();
|
|
4067
4676
|
const abortUpstream = (reason) => upstreamControl.abort(new Error(reason));
|
|
4068
4677
|
const onClientClose = () => {
|
|
@@ -4086,24 +4695,27 @@ ${provider}`;
|
|
|
4086
4695
|
const startedAt = Date.now();
|
|
4087
4696
|
let upstreamResponse;
|
|
4088
4697
|
try {
|
|
4089
|
-
|
|
4698
|
+
const query = (req.url ?? "").includes("?") ? (req.url ?? "").slice((req.url ?? "").indexOf("?")) : "";
|
|
4699
|
+
upstreamResponse = await fetch(`${upstreams[surface.provider]}${surface.upstreamPath}${query}`, {
|
|
4090
4700
|
method: "POST",
|
|
4091
4701
|
headers,
|
|
4092
4702
|
body: sendBody,
|
|
4093
|
-
signal: upstreamControl.signal
|
|
4703
|
+
signal: upstreamControl.signal,
|
|
4704
|
+
redirect: "manual"
|
|
4705
|
+
// never carry the caller's key to a redirect target
|
|
4094
4706
|
});
|
|
4095
4707
|
} catch (cause) {
|
|
4096
4708
|
clearTimeout(phaseTimer);
|
|
4097
4709
|
if (!res.headersSent && !res.destroyed) {
|
|
4098
4710
|
res.writeHead(502, { "content-type": "application/json" });
|
|
4099
|
-
res.end(
|
|
4711
|
+
res.end(meshError(`Upstream unreachable: ${cause.message}`));
|
|
4100
4712
|
}
|
|
4101
4713
|
return;
|
|
4102
4714
|
}
|
|
4103
4715
|
clearTimeout(phaseTimer);
|
|
4104
4716
|
const responseHeaders = {};
|
|
4105
4717
|
upstreamResponse.headers.forEach((value, name) => {
|
|
4106
|
-
if (!
|
|
4718
|
+
if (!RESPONSE_DROP_HEADERS.has(name.toLowerCase())) responseHeaders[name] = value;
|
|
4107
4719
|
});
|
|
4108
4720
|
if (res.destroyed) {
|
|
4109
4721
|
abortUpstream("client disconnected");
|
|
@@ -4114,6 +4726,7 @@ ${provider}`;
|
|
|
4114
4726
|
let accumulated = 0;
|
|
4115
4727
|
const ACCUMULATION_CAP = 32 * 1024 * 1024;
|
|
4116
4728
|
let clientLost = false;
|
|
4729
|
+
let upstreamBroke = false;
|
|
4117
4730
|
if (upstreamResponse.body !== null) {
|
|
4118
4731
|
const reader = upstreamResponse.body.getReader();
|
|
4119
4732
|
try {
|
|
@@ -4155,16 +4768,18 @@ ${provider}`;
|
|
|
4155
4768
|
}
|
|
4156
4769
|
} catch {
|
|
4157
4770
|
clientLost = res.destroyed;
|
|
4771
|
+
upstreamBroke = !res.destroyed;
|
|
4158
4772
|
}
|
|
4159
4773
|
}
|
|
4160
4774
|
res.off("close", onClientClose);
|
|
4161
|
-
if (
|
|
4775
|
+
if (upstreamBroke && !res.destroyed) res.destroy();
|
|
4776
|
+
else if (!clientLost && !res.destroyed) res.end();
|
|
4162
4777
|
try {
|
|
4163
4778
|
const text = Buffer.concat(chunks).toString("utf8");
|
|
4164
4779
|
const contentType = upstreamResponse.headers.get("content-type") ?? "";
|
|
4165
4780
|
if (process.env.MESH_PROXY_DEBUG === "1" && !upstreamResponse.ok) {
|
|
4166
4781
|
console.warn(
|
|
4167
|
-
`[mesh-debug] upstream ${upstreamResponse.status} model=${JSON.stringify(body
|
|
4782
|
+
`[mesh-debug] upstream ${upstreamResponse.status} model=${JSON.stringify(body?.model)} path=${path} anthropic-beta=${JSON.stringify(req.headers["anthropic-beta"] ?? null)} errbody=${text.slice(0, 500)}`
|
|
4168
4783
|
);
|
|
4169
4784
|
}
|
|
4170
4785
|
let responseShaped;
|
|
@@ -4184,14 +4799,14 @@ ${provider}`;
|
|
|
4184
4799
|
if (usage !== void 0 && usage.outputTokens >= 200 && responseShaped !== void 0) {
|
|
4185
4800
|
const outputText = contentType.includes("text/event-stream") ? outputTextFromSse(surface.provider, text) : responseText(responseShaped);
|
|
4186
4801
|
if (outputText.length > 0) {
|
|
4187
|
-
outputDiagnostics = measureOutputReEmission(collectRequestText(body), outputText);
|
|
4802
|
+
outputDiagnostics = measureOutputReEmission(collectRequestText(body ?? {}), outputText);
|
|
4188
4803
|
}
|
|
4189
4804
|
}
|
|
4190
4805
|
const event = {
|
|
4191
4806
|
requestId,
|
|
4192
4807
|
applicationId,
|
|
4193
4808
|
provider: `${surface.provider}-compatible`,
|
|
4194
|
-
model: typeof body
|
|
4809
|
+
model: typeof body?.model === "string" ? body.model : "unknown",
|
|
4195
4810
|
latencyMs: Date.now() - startedAt,
|
|
4196
4811
|
status: upstreamResponse.ok ? "succeeded" : "failed",
|
|
4197
4812
|
timestamp: new Date(startedAt).toISOString(),
|
|
@@ -4211,8 +4826,11 @@ ${provider}`;
|
|
|
4211
4826
|
...outputDiagnostics === void 0 ? {} : { outputDiagnostics },
|
|
4212
4827
|
...duplicateOfRequestId === void 0 ? {} : { duplicateOfRequestId }
|
|
4213
4828
|
};
|
|
4214
|
-
ledgerFor(applicationId, surface.provider).emit(
|
|
4215
|
-
|
|
4829
|
+
ledgerFor(applicationId, surface.provider).emit(
|
|
4830
|
+
event,
|
|
4831
|
+
requestGateVerdicts === void 0 ? void 0 : { gateVerdicts: requestGateVerdicts }
|
|
4832
|
+
);
|
|
4833
|
+
if (blindFeaturesKey !== void 0 && body !== void 0) {
|
|
4216
4834
|
const features = extractBlindFeatures(body, { hmacKey: blindFeaturesKey });
|
|
4217
4835
|
writeBlindFeatures(applicationId, surface.provider, {
|
|
4218
4836
|
requestId,
|
|
@@ -4221,6 +4839,7 @@ ${provider}`;
|
|
|
4221
4839
|
features
|
|
4222
4840
|
});
|
|
4223
4841
|
}
|
|
4842
|
+
if (body !== void 0) writeCapturedRequest(body);
|
|
4224
4843
|
} catch {
|
|
4225
4844
|
}
|
|
4226
4845
|
};
|
|
@@ -4228,7 +4847,7 @@ ${provider}`;
|
|
|
4228
4847
|
handle(req, res).catch(() => {
|
|
4229
4848
|
if (!res.headersSent) {
|
|
4230
4849
|
res.writeHead(500, { "content-type": "application/json" });
|
|
4231
|
-
res.end(
|
|
4850
|
+
res.end(meshError("MESH proxy internal error."));
|
|
4232
4851
|
} else {
|
|
4233
4852
|
res.end();
|
|
4234
4853
|
}
|
|
@@ -4272,10 +4891,13 @@ function wrapMesh(client, options = {}) {
|
|
|
4272
4891
|
...options.tokenReduction ?? LOSSLESS_DEFAULT2,
|
|
4273
4892
|
...options.cacheTtl ? { cacheTtl: options.cacheTtl } : {}
|
|
4274
4893
|
};
|
|
4894
|
+
const meshApiUrl = options.meshApiUrl ?? process.env.MESH_API_URL;
|
|
4895
|
+
const meshAccountKey = options.meshAccountKey ?? process.env.MESH_ACCOUNT_KEY;
|
|
4896
|
+
const mirror = meshApiUrl && meshAccountKey ? { apiUrl: meshApiUrl, accountKey: meshAccountKey } : void 0;
|
|
4275
4897
|
const wrapped = withMesh(client, {
|
|
4276
4898
|
applicationId: app,
|
|
4277
4899
|
tokenReduction,
|
|
4278
|
-
ledger: { path: ledgerPath, provider }
|
|
4900
|
+
ledger: { path: ledgerPath, provider, ...mirror ? { mirror } : {} }
|
|
4279
4901
|
});
|
|
4280
4902
|
const mesh = {
|
|
4281
4903
|
ledgerPath,
|
|
@@ -4315,10 +4937,15 @@ function wrapMesh(client, options = {}) {
|
|
|
4315
4937
|
}
|
|
4316
4938
|
|
|
4317
4939
|
// packages/sdk-js/src/theo-policy.ts
|
|
4940
|
+
var MAX_QUALITY_CEILING_PERCENT = 5;
|
|
4941
|
+
function effectiveQualityCeiling(ceilingPercent) {
|
|
4942
|
+
const c = Number.isFinite(ceilingPercent) ? ceilingPercent : 0;
|
|
4943
|
+
return Math.max(0, Math.min(c, MAX_QUALITY_CEILING_PERCENT));
|
|
4944
|
+
}
|
|
4318
4945
|
function proofClearsCeiling(lever, ceilingPercent) {
|
|
4319
4946
|
if (lever.gate === void 0) return false;
|
|
4320
4947
|
const y = lever.gate.qualityDeltaPercent;
|
|
4321
|
-
return Number.isFinite(y) && Math.abs(y) <= ceilingPercent;
|
|
4948
|
+
return Number.isFinite(y) && Math.abs(y) <= effectiveQualityCeiling(ceilingPercent);
|
|
4322
4949
|
}
|
|
4323
4950
|
function applyLever(config, lever) {
|
|
4324
4951
|
const p = lever.params ?? {};
|
|
@@ -4332,6 +4959,11 @@ function applyLever(config, lever) {
|
|
|
4332
4959
|
...typeof p.maximumDescriptionTokens === "number" ? { maximumDescriptionTokens: p.maximumDescriptionTokens } : {}
|
|
4333
4960
|
};
|
|
4334
4961
|
return;
|
|
4962
|
+
case "unused-tool-removal": {
|
|
4963
|
+
const names = Array.isArray(p.tools) ? p.tools.filter((name) => typeof name === "string") : [];
|
|
4964
|
+
if (names.length > 0) config.dropTools = names;
|
|
4965
|
+
return;
|
|
4966
|
+
}
|
|
4335
4967
|
case "duplicate-context-removal":
|
|
4336
4968
|
config.duplicateContext = true;
|
|
4337
4969
|
return;
|
|
@@ -4378,6 +5010,51 @@ function policyToReductionConfig(policy) {
|
|
|
4378
5010
|
}
|
|
4379
5011
|
return config;
|
|
4380
5012
|
}
|
|
5013
|
+
function leverToReductionConfig(lever) {
|
|
5014
|
+
const config = {};
|
|
5015
|
+
applyLever(config, lever);
|
|
5016
|
+
return config;
|
|
5017
|
+
}
|
|
5018
|
+
function routeMatches(pattern, modelId) {
|
|
5019
|
+
if (pattern === "*" || pattern === "") return true;
|
|
5020
|
+
if (pattern.endsWith("*")) return modelId.startsWith(pattern.slice(0, -1));
|
|
5021
|
+
return pattern === modelId;
|
|
5022
|
+
}
|
|
5023
|
+
function isPolicyBundle(value) {
|
|
5024
|
+
return typeof value === "object" && value !== null && "default" in value && typeof value.default === "object";
|
|
5025
|
+
}
|
|
5026
|
+
function asBundle(stored) {
|
|
5027
|
+
if (isPolicyBundle(stored)) return stored;
|
|
5028
|
+
const policy = stored;
|
|
5029
|
+
return { schemaVersion: 1, user: policy.user, default: policy, updatedAt: policy.updatedAt };
|
|
5030
|
+
}
|
|
5031
|
+
function policyForRoute(bundle, modelId) {
|
|
5032
|
+
for (const route of bundle.routes ?? []) {
|
|
5033
|
+
if (route.route !== void 0 && routeMatches(route.route, modelId)) return route;
|
|
5034
|
+
}
|
|
5035
|
+
return bundle.default;
|
|
5036
|
+
}
|
|
5037
|
+
function bundleToReductionConfig(bundle, modelId) {
|
|
5038
|
+
return policyToReductionConfig(policyForRoute(bundle, modelId));
|
|
5039
|
+
}
|
|
5040
|
+
function upsertRoutePolicy(bundle, policy, at) {
|
|
5041
|
+
if (policy.route === void 0 || policy.route === "") {
|
|
5042
|
+
throw new Error("Refusing to add a route policy without a `route` matcher.");
|
|
5043
|
+
}
|
|
5044
|
+
const routes = (bundle.routes ?? []).filter((existing) => existing.route !== policy.route);
|
|
5045
|
+
routes.push(policy);
|
|
5046
|
+
return { ...bundle, routes, updatedAt: at };
|
|
5047
|
+
}
|
|
5048
|
+
function mergePolicyIntoBundle(existing, policy, route, at) {
|
|
5049
|
+
if (route !== void 0 && route !== "") {
|
|
5050
|
+
const base2 = existing ?? asBundle(defaultPolicy(policy.user, at));
|
|
5051
|
+
return upsertRoutePolicy(base2, { ...policy, route }, at);
|
|
5052
|
+
}
|
|
5053
|
+
const base = existing ?? asBundle(policy);
|
|
5054
|
+
const def = { ...policy };
|
|
5055
|
+
delete def.route;
|
|
5056
|
+
return { ...base, default: def, updatedAt: at };
|
|
5057
|
+
}
|
|
4381
5058
|
function withheldLevers(policy) {
|
|
4382
5059
|
const out = [];
|
|
4383
5060
|
for (const lever of policy.levers) {
|
|
@@ -4420,6 +5097,7 @@ function knownLeverIds() {
|
|
|
4420
5097
|
return [
|
|
4421
5098
|
"tool-schema-compaction",
|
|
4422
5099
|
"tool-schema-description-trim",
|
|
5100
|
+
"unused-tool-removal",
|
|
4423
5101
|
"duplicate-context-removal",
|
|
4424
5102
|
"cache-prefix-repair",
|
|
4425
5103
|
"stale-history-compaction",
|
|
@@ -4436,6 +5114,683 @@ function knownLeverIds() {
|
|
|
4436
5114
|
"output-effort-max"
|
|
4437
5115
|
];
|
|
4438
5116
|
}
|
|
5117
|
+
var ENFORCEABLE_LEVERS = [
|
|
5118
|
+
{ id: "tool-schema-compaction", label: "Tool-schema compaction", members: ["tool-schema-compaction"] },
|
|
5119
|
+
{ id: "tool-schema-description-trim", label: "Tool-schema description trim", members: ["tool-schema-description-trim"] },
|
|
5120
|
+
{ id: "unused-tool-removal", label: "Unused-tool removal", members: ["unused-tool-removal"] },
|
|
5121
|
+
{ id: "duplicate-context-removal", label: "Duplicate-context removal", members: ["duplicate-context-removal"] },
|
|
5122
|
+
{ id: "cache-prefix-repair", label: "Cache-prefix repair", members: ["cache-prefix-repair"] },
|
|
5123
|
+
{ id: "stale-history-compaction", label: "Stale-history compaction", members: ["stale-history-compaction"] },
|
|
5124
|
+
{ id: "cross-turn-memory-dedup", label: "Cross-turn memory dedup", members: ["cross-turn-memory-dedup"] },
|
|
5125
|
+
{ id: "fresh-tool-result-cap", label: "Fresh tool-result cap", members: ["fresh-tool-result-cap"] },
|
|
5126
|
+
{ id: "output-cap", label: "Output cap", members: ["output-cap"] },
|
|
5127
|
+
{ id: "execution-tier-flex", label: "Execution-tier flex", members: ["execution-tier-flex"] },
|
|
5128
|
+
{ id: "cache-ttl", label: "Cache TTL", members: ["cache-ttl-5m", "cache-ttl-1h"] },
|
|
5129
|
+
{
|
|
5130
|
+
id: "output-effort",
|
|
5131
|
+
label: "Output effort",
|
|
5132
|
+
members: ["output-effort-low", "output-effort-medium", "output-effort-high", "output-effort-xhigh", "output-effort-max"]
|
|
5133
|
+
}
|
|
5134
|
+
];
|
|
5135
|
+
function policyCoverage(policy) {
|
|
5136
|
+
const byId = new Map(policy.levers.map((lever) => [lever.lever, lever]));
|
|
5137
|
+
const heldById = new Map(withheldLevers(policy).map((withheld) => [withheld.lever, withheld.reason]));
|
|
5138
|
+
const stands = ENFORCEABLE_LEVERS.map((group) => {
|
|
5139
|
+
let state = "available";
|
|
5140
|
+
let reason;
|
|
5141
|
+
let leverClass;
|
|
5142
|
+
for (const member of group.members) {
|
|
5143
|
+
const lever = byId.get(member);
|
|
5144
|
+
if (lever === void 0) continue;
|
|
5145
|
+
leverClass = lever.class;
|
|
5146
|
+
const enforced = lever.class === "lossless" || proofClearsCeiling(lever, policy.qualityCeilingPercent);
|
|
5147
|
+
if (enforced) {
|
|
5148
|
+
state = "active";
|
|
5149
|
+
reason = void 0;
|
|
5150
|
+
break;
|
|
5151
|
+
}
|
|
5152
|
+
state = "held";
|
|
5153
|
+
reason = heldById.get(member);
|
|
5154
|
+
}
|
|
5155
|
+
return {
|
|
5156
|
+
lever: group.id,
|
|
5157
|
+
label: group.label,
|
|
5158
|
+
state,
|
|
5159
|
+
...leverClass ? { class: leverClass } : {},
|
|
5160
|
+
...reason ? { reason } : {}
|
|
5161
|
+
};
|
|
5162
|
+
});
|
|
5163
|
+
const activeCount = stands.filter((stand) => stand.state === "active").length;
|
|
5164
|
+
const heldCount = stands.filter((stand) => stand.state === "held").length;
|
|
5165
|
+
const availableCount = stands.filter((stand) => stand.state === "available").length;
|
|
5166
|
+
const enforceableCount = ENFORCEABLE_LEVERS.length;
|
|
5167
|
+
return {
|
|
5168
|
+
enforceableCount,
|
|
5169
|
+
activeCount,
|
|
5170
|
+
heldCount,
|
|
5171
|
+
availableCount,
|
|
5172
|
+
activeCoveragePercent: enforceableCount === 0 ? 0 : activeCount / enforceableCount * 100,
|
|
5173
|
+
stands
|
|
5174
|
+
};
|
|
5175
|
+
}
|
|
5176
|
+
|
|
5177
|
+
// packages/sdk-js/src/theo-agent.ts
|
|
5178
|
+
var ENFORCEABLE_BY_CONCEPT = {
|
|
5179
|
+
"tool-schema-compaction": { lever: "tool-schema-compaction", class: "lossless" },
|
|
5180
|
+
"duplicate-context-removal": { lever: "duplicate-context-removal", class: "lossless" },
|
|
5181
|
+
"tool-schema-description-trim": { lever: "tool-schema-description-trim", class: "lossy" },
|
|
5182
|
+
"history-compaction": { lever: "stale-history-compaction", class: "lossy" },
|
|
5183
|
+
"output-budget": { lever: "output-effort-low", class: "lossy" }
|
|
5184
|
+
};
|
|
5185
|
+
var ALREADY_COVERED_CONCEPTS = /* @__PURE__ */ new Set(["prefix-caching"]);
|
|
5186
|
+
var OPERATOR_CONFIGURED_CONCEPTS = {
|
|
5187
|
+
"unused-tool-removal": "An enforceable lever now exists (`unused-tool-removal`). Theo is content-blind and cannot name the tools, so enable it with the tool names your workload never calls \u2014 the replay gate proves it on your traffic before billing."
|
|
5188
|
+
};
|
|
5189
|
+
var ENFORCEABLE_LOSSY = new Set(
|
|
5190
|
+
Object.values(ENFORCEABLE_BY_CONCEPT).filter((entry2) => entry2.class === "lossy").map((entry2) => entry2.lever)
|
|
5191
|
+
);
|
|
5192
|
+
var DRIFT_RISE_THRESHOLD = 0.05;
|
|
5193
|
+
var DRIFT_WINDOW_EVENTS = 200;
|
|
5194
|
+
function recentDriftRates(events, windowEvents = DRIFT_WINDOW_EVENTS) {
|
|
5195
|
+
const size = Math.max(1, Math.floor(windowEvents));
|
|
5196
|
+
const recent = [...events].sort((a, b) => {
|
|
5197
|
+
const left = a.timestamp ?? "";
|
|
5198
|
+
const right = b.timestamp ?? "";
|
|
5199
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
5200
|
+
}).slice(-size);
|
|
5201
|
+
const n = recent.length;
|
|
5202
|
+
if (n === 0) return { sampleSize: 0, retryRate: 0, failureRate: 0 };
|
|
5203
|
+
return {
|
|
5204
|
+
sampleSize: n,
|
|
5205
|
+
retryRate: recent.filter((event) => event.duplicateOfRequestId !== void 0).length / n,
|
|
5206
|
+
failureRate: recent.filter((event) => event.status === "failed").length / n
|
|
5207
|
+
};
|
|
5208
|
+
}
|
|
5209
|
+
function zeroSegments() {
|
|
5210
|
+
return { toolSchemas: 0, systemPrompt: 0, retrievedContext: 0, history: 0, userTurn: 0, output: 0 };
|
|
5211
|
+
}
|
|
5212
|
+
function observeWorkload(features, options = {}) {
|
|
5213
|
+
const name = options.name ?? "observed";
|
|
5214
|
+
const n = features.length;
|
|
5215
|
+
if (n === 0) {
|
|
5216
|
+
return { name, requests: 1, segments: zeroSegments(), stableSegments: [] };
|
|
5217
|
+
}
|
|
5218
|
+
const mean2 = (nums) => nums.reduce((sum, value) => sum + value, 0) / nums.length;
|
|
5219
|
+
const segments = {
|
|
5220
|
+
toolSchemas: mean2(features.map((f) => f.toolSchemaTokensApprox)),
|
|
5221
|
+
systemPrompt: mean2(features.map((f) => f.roleTokensApprox.system ?? 0)),
|
|
5222
|
+
retrievedContext: 0,
|
|
5223
|
+
history: mean2(features.map((f) => (f.roleTokensApprox.assistant ?? 0) + (f.roleTokensApprox.tool ?? 0))),
|
|
5224
|
+
userTurn: mean2(features.map((f) => f.roleTokensApprox.user ?? 0)),
|
|
5225
|
+
output: Math.max(0, options.outputTokensMean ?? 0)
|
|
5226
|
+
};
|
|
5227
|
+
const prefixReuse = n > 1 && new Set(features.map((f) => f.stablePrefixHash)).size === 1;
|
|
5228
|
+
const stableSegments = prefixReuse ? ["toolSchemas", "systemPrompt"] : [];
|
|
5229
|
+
return { name, requests: n, segments, stableSegments };
|
|
5230
|
+
}
|
|
5231
|
+
function createTheoMemory(customer, autonomy, at) {
|
|
5232
|
+
return {
|
|
5233
|
+
schemaVersion: 1,
|
|
5234
|
+
customer,
|
|
5235
|
+
autonomy,
|
|
5236
|
+
policy: defaultPolicy(customer, at),
|
|
5237
|
+
hypotheses: [],
|
|
5238
|
+
frontier: [],
|
|
5239
|
+
experiments: [],
|
|
5240
|
+
proposals: [],
|
|
5241
|
+
driftBaselines: {},
|
|
5242
|
+
rolledBack: [],
|
|
5243
|
+
log: [],
|
|
5244
|
+
updatedAt: at
|
|
5245
|
+
};
|
|
5246
|
+
}
|
|
5247
|
+
function losslessLevers(policy) {
|
|
5248
|
+
return policy.levers.filter((lever) => lever.class === "lossless").map((lever) => lever.lever);
|
|
5249
|
+
}
|
|
5250
|
+
function hypothesise(profile, price, activeLossy, rolledBack, declined = /* @__PURE__ */ new Set()) {
|
|
5251
|
+
let contributions;
|
|
5252
|
+
try {
|
|
5253
|
+
contributions = projectSavings(profile, leversFor(profile), price).contributions;
|
|
5254
|
+
} catch {
|
|
5255
|
+
return { hypotheses: [], frontier: [] };
|
|
5256
|
+
}
|
|
5257
|
+
const ranked = [...contributions].sort((a, b) => b.sharePercent - a.sharePercent);
|
|
5258
|
+
const hypotheses = [];
|
|
5259
|
+
const frontier = [];
|
|
5260
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5261
|
+
for (const contribution of ranked) {
|
|
5262
|
+
if (contribution.sharePercent <= 0) continue;
|
|
5263
|
+
if (ALREADY_COVERED_CONCEPTS.has(contribution.lever)) continue;
|
|
5264
|
+
const mapped = ENFORCEABLE_BY_CONCEPT[contribution.lever];
|
|
5265
|
+
if (mapped === void 0) {
|
|
5266
|
+
frontier.push({
|
|
5267
|
+
concept: contribution.lever,
|
|
5268
|
+
priorSharePercent: contribution.sharePercent,
|
|
5269
|
+
note: OPERATOR_CONFIGURED_CONCEPTS[contribution.lever] ?? "Opportunity with no enforceable proxy lever yet \u2014 needs new reduction code or an app-side change."
|
|
5270
|
+
});
|
|
5271
|
+
continue;
|
|
5272
|
+
}
|
|
5273
|
+
if (mapped.class === "lossless") continue;
|
|
5274
|
+
if (rolledBack.has(mapped.lever)) continue;
|
|
5275
|
+
if (declined.has(mapped.lever)) continue;
|
|
5276
|
+
if (activeLossy.has(mapped.lever) || seen.has(mapped.lever)) continue;
|
|
5277
|
+
seen.add(mapped.lever);
|
|
5278
|
+
hypotheses.push({
|
|
5279
|
+
lever: mapped.lever,
|
|
5280
|
+
class: "lossy",
|
|
5281
|
+
priorSharePercent: contribution.sharePercent,
|
|
5282
|
+
rationale: contribution.evidence.claim
|
|
5283
|
+
});
|
|
5284
|
+
}
|
|
5285
|
+
return { hypotheses, frontier };
|
|
5286
|
+
}
|
|
5287
|
+
function proposalMessage(proof) {
|
|
5288
|
+
const drop = Math.abs(proof.qualityDeltaPercent);
|
|
5289
|
+
return `${proof.lever}: proven to save ~${proof.savingPercent.toFixed(1)}% on your traffic, measured quality cost ~${drop.toFixed(2)}% (grader: ${proof.grader}). Apply it?`;
|
|
5290
|
+
}
|
|
5291
|
+
function runTheoCycle(memory, input, now) {
|
|
5292
|
+
const { observation } = input;
|
|
5293
|
+
const proofs = input.proofs ?? [];
|
|
5294
|
+
const autonomy = memory.autonomy;
|
|
5295
|
+
const actions = [];
|
|
5296
|
+
let policy = memory.policy;
|
|
5297
|
+
const experiments = [...memory.experiments];
|
|
5298
|
+
const proposals = [...memory.proposals];
|
|
5299
|
+
const driftBaselines = { ...memory.driftBaselines ?? {} };
|
|
5300
|
+
const rolledBack = [...memory.rolledBack ?? []];
|
|
5301
|
+
const declined = new Set(memory.declined ?? []);
|
|
5302
|
+
const profile = observation.profile;
|
|
5303
|
+
const retryRate = observation.retryRate;
|
|
5304
|
+
const failureRate = observation.failureRate;
|
|
5305
|
+
if (retryRate !== void 0 || failureRate !== void 0) {
|
|
5306
|
+
const pct = (value) => `${(value * 100).toFixed(1)}%`;
|
|
5307
|
+
for (const lever of policy.levers) {
|
|
5308
|
+
if (lever.class !== "lossy" || lever.gate === void 0) continue;
|
|
5309
|
+
const baseline = driftBaselines[lever.lever];
|
|
5310
|
+
if (baseline === void 0) continue;
|
|
5311
|
+
const retryRose = retryRate !== void 0 && retryRate > baseline.retryRate + DRIFT_RISE_THRESHOLD;
|
|
5312
|
+
const failRose = failureRate !== void 0 && failureRate > (baseline.failureRate ?? 0) + DRIFT_RISE_THRESHOLD;
|
|
5313
|
+
if (!retryRose && !failRose) continue;
|
|
5314
|
+
const reason = retryRose ? `retry rate rose ${pct(baseline.retryRate)} \u2192 ${pct(retryRate)}` : `failure rate rose ${pct(baseline.failureRate ?? 0)} \u2192 ${pct(failureRate)}`;
|
|
5315
|
+
policy = {
|
|
5316
|
+
...policy,
|
|
5317
|
+
levers: policy.levers.filter((other) => other.lever !== lever.lever),
|
|
5318
|
+
version: policy.version + 1,
|
|
5319
|
+
updatedAt: now
|
|
5320
|
+
};
|
|
5321
|
+
delete driftBaselines[lever.lever];
|
|
5322
|
+
if (!rolledBack.includes(lever.lever)) rolledBack.push(lever.lever);
|
|
5323
|
+
experiments.push({ lever: lever.lever, ranAt: now, passed: false, note: `rolled back: ${reason}` });
|
|
5324
|
+
actions.push({ kind: "rolled-back", lever: lever.lever, reason });
|
|
5325
|
+
}
|
|
5326
|
+
}
|
|
5327
|
+
const activeLossy = new Set(
|
|
5328
|
+
policy.levers.filter((lever) => lever.class === "lossy" && lever.gate !== void 0).map((l) => l.lever)
|
|
5329
|
+
);
|
|
5330
|
+
const { hypotheses, frontier } = hypothesise(profile, observation.price, activeLossy, new Set(rolledBack), declined);
|
|
5331
|
+
const lossless = losslessLevers(policy);
|
|
5332
|
+
if (autonomy !== "observe") actions.push({ kind: "lossless-active", levers: lossless });
|
|
5333
|
+
for (const proof of proofs) {
|
|
5334
|
+
if (!ENFORCEABLE_LOSSY.has(proof.lever)) {
|
|
5335
|
+
actions.push({ kind: "proof-rejected", lever: proof.lever, reason: "not an enforceable lossy lever" });
|
|
5336
|
+
continue;
|
|
5337
|
+
}
|
|
5338
|
+
if (rolledBack.includes(proof.lever)) {
|
|
5339
|
+
actions.push({ kind: "proof-rejected", lever: proof.lever, reason: "auto-rolled-back earlier after quality drift" });
|
|
5340
|
+
continue;
|
|
5341
|
+
}
|
|
5342
|
+
if (declined.has(proof.lever)) {
|
|
5343
|
+
actions.push({ kind: "proof-rejected", lever: proof.lever, reason: "declined by the customer" });
|
|
5344
|
+
continue;
|
|
5345
|
+
}
|
|
5346
|
+
const ceiling = effectiveQualityCeiling(policy.qualityCeilingPercent);
|
|
5347
|
+
const withinCeiling = Math.abs(proof.qualityDeltaPercent) <= ceiling;
|
|
5348
|
+
const saves = proof.savingPercent > 0;
|
|
5349
|
+
const passed = withinCeiling && saves;
|
|
5350
|
+
experiments.push({
|
|
5351
|
+
lever: proof.lever,
|
|
5352
|
+
ranAt: proof.provenAt,
|
|
5353
|
+
passed,
|
|
5354
|
+
savingPercent: proof.savingPercent,
|
|
5355
|
+
qualityDeltaPercent: proof.qualityDeltaPercent,
|
|
5356
|
+
grader: proof.grader,
|
|
5357
|
+
...passed ? {} : {
|
|
5358
|
+
note: !saves ? "no saving once priced, so nothing to enable" : `quality drop ${Math.abs(proof.qualityDeltaPercent).toFixed(2)}% exceeds the ${ceiling}% ceiling`
|
|
5359
|
+
}
|
|
5360
|
+
});
|
|
5361
|
+
if (!passed) {
|
|
5362
|
+
actions.push({
|
|
5363
|
+
kind: "proof-rejected",
|
|
5364
|
+
lever: proof.lever,
|
|
5365
|
+
reason: saves ? "quality drop exceeds the route ceiling" : "no verified saving"
|
|
5366
|
+
});
|
|
5367
|
+
continue;
|
|
5368
|
+
}
|
|
5369
|
+
if (autonomy === "optimise") {
|
|
5370
|
+
policy = publishLever(
|
|
5371
|
+
policy,
|
|
5372
|
+
{
|
|
5373
|
+
lever: proof.lever,
|
|
5374
|
+
class: "lossy",
|
|
5375
|
+
gate: {
|
|
5376
|
+
savingPercent: proof.savingPercent,
|
|
5377
|
+
qualityDeltaPercent: proof.qualityDeltaPercent,
|
|
5378
|
+
grader: proof.grader,
|
|
5379
|
+
provenAt: proof.provenAt
|
|
5380
|
+
}
|
|
5381
|
+
},
|
|
5382
|
+
now
|
|
5383
|
+
);
|
|
5384
|
+
actions.push({ kind: "policy-published", version: policy.version, lever: proof.lever });
|
|
5385
|
+
driftBaselines[proof.lever] = {
|
|
5386
|
+
retryRate: observation.retryRate ?? 0,
|
|
5387
|
+
failureRate: observation.failureRate ?? 0,
|
|
5388
|
+
at: now
|
|
5389
|
+
};
|
|
5390
|
+
} else if (autonomy === "recommend" || autonomy === "test") {
|
|
5391
|
+
if (!proposals.some((existing) => existing.lever === proof.lever)) {
|
|
5392
|
+
proposals.push({
|
|
5393
|
+
lever: proof.lever,
|
|
5394
|
+
savingPercent: proof.savingPercent,
|
|
5395
|
+
qualityDeltaPercent: proof.qualityDeltaPercent,
|
|
5396
|
+
grader: proof.grader,
|
|
5397
|
+
provenAt: proof.provenAt,
|
|
5398
|
+
message: proposalMessage(proof)
|
|
5399
|
+
});
|
|
5400
|
+
actions.push({
|
|
5401
|
+
kind: "proposal-opened",
|
|
5402
|
+
lever: proof.lever,
|
|
5403
|
+
savingPercent: proof.savingPercent,
|
|
5404
|
+
qualityDeltaPercent: proof.qualityDeltaPercent
|
|
5405
|
+
});
|
|
5406
|
+
}
|
|
5407
|
+
}
|
|
5408
|
+
}
|
|
5409
|
+
const provenThisCycle = new Set(proofs.map((proof) => proof.lever));
|
|
5410
|
+
const openHypotheses = hypotheses.filter((hypothesis) => !provenThisCycle.has(hypothesis.lever));
|
|
5411
|
+
if ((autonomy === "test" || autonomy === "optimise") && openHypotheses.length > 0) {
|
|
5412
|
+
actions.push({ kind: "gate-requested", levers: openHypotheses.slice(0, 3).map((h) => h.lever) });
|
|
5413
|
+
}
|
|
5414
|
+
const nextMemory = {
|
|
5415
|
+
...memory,
|
|
5416
|
+
policy,
|
|
5417
|
+
profile,
|
|
5418
|
+
hypotheses: openHypotheses,
|
|
5419
|
+
frontier,
|
|
5420
|
+
experiments,
|
|
5421
|
+
proposals,
|
|
5422
|
+
driftBaselines,
|
|
5423
|
+
rolledBack,
|
|
5424
|
+
log: [
|
|
5425
|
+
...memory.log,
|
|
5426
|
+
{
|
|
5427
|
+
at: now,
|
|
5428
|
+
autonomy,
|
|
5429
|
+
observed: `${profile.name}: ${profile.requests} req, ${Math.round(sumSegments(profile))} tok/req`,
|
|
5430
|
+
actions: actions.length
|
|
5431
|
+
}
|
|
5432
|
+
],
|
|
5433
|
+
updatedAt: now
|
|
5434
|
+
};
|
|
5435
|
+
return { memory: nextMemory, actions, report: renderTheoReport(nextMemory, observation, actions) };
|
|
5436
|
+
}
|
|
5437
|
+
function openProposalList(memory) {
|
|
5438
|
+
return memory.proposals.length === 0 ? "There are no open proposals." : `Open proposals: ${memory.proposals.map((proposal) => proposal.lever).join(", ")}.`;
|
|
5439
|
+
}
|
|
5440
|
+
function acceptProposal(memory, lever, rates, now) {
|
|
5441
|
+
const proposal = memory.proposals.find((candidate) => candidate.lever === lever);
|
|
5442
|
+
if (proposal === void 0) {
|
|
5443
|
+
return { ok: false, reason: `No open proposal for '${lever}'. ${openProposalList(memory)}` };
|
|
5444
|
+
}
|
|
5445
|
+
if (!ENFORCEABLE_LOSSY.has(lever)) {
|
|
5446
|
+
return { ok: false, reason: `'${lever}' is not a lever Theo can enforce.` };
|
|
5447
|
+
}
|
|
5448
|
+
if ((memory.rolledBack ?? []).includes(lever)) {
|
|
5449
|
+
return { ok: false, reason: `'${lever}' was rolled back after quality drift and cannot be re-applied.` };
|
|
5450
|
+
}
|
|
5451
|
+
const ceiling = effectiveQualityCeiling(memory.policy.qualityCeilingPercent);
|
|
5452
|
+
const drop = Math.abs(proposal.qualityDeltaPercent);
|
|
5453
|
+
if (!Number.isFinite(drop) || drop > ceiling) {
|
|
5454
|
+
return {
|
|
5455
|
+
ok: false,
|
|
5456
|
+
reason: `'${lever}' measured a ${drop.toFixed(2)}% quality drop, above the current ${ceiling}% ceiling.`
|
|
5457
|
+
};
|
|
5458
|
+
}
|
|
5459
|
+
if (!(proposal.savingPercent > 0)) {
|
|
5460
|
+
return { ok: false, reason: `'${lever}' has no verified saving to enable.` };
|
|
5461
|
+
}
|
|
5462
|
+
const policy = publishLever(
|
|
5463
|
+
memory.policy,
|
|
5464
|
+
{
|
|
5465
|
+
lever,
|
|
5466
|
+
class: "lossy",
|
|
5467
|
+
gate: {
|
|
5468
|
+
savingPercent: proposal.savingPercent,
|
|
5469
|
+
qualityDeltaPercent: proposal.qualityDeltaPercent,
|
|
5470
|
+
grader: proposal.grader,
|
|
5471
|
+
provenAt: proposal.provenAt
|
|
5472
|
+
}
|
|
5473
|
+
},
|
|
5474
|
+
now
|
|
5475
|
+
);
|
|
5476
|
+
return {
|
|
5477
|
+
ok: true,
|
|
5478
|
+
memory: {
|
|
5479
|
+
...memory,
|
|
5480
|
+
policy,
|
|
5481
|
+
proposals: memory.proposals.filter((candidate) => candidate.lever !== lever),
|
|
5482
|
+
hypotheses: memory.hypotheses.filter((hypothesis) => hypothesis.lever !== lever),
|
|
5483
|
+
driftBaselines: {
|
|
5484
|
+
...memory.driftBaselines ?? {},
|
|
5485
|
+
[lever]: { retryRate: rates.retryRate ?? 0, failureRate: rates.failureRate ?? 0, at: now }
|
|
5486
|
+
},
|
|
5487
|
+
declined: (memory.declined ?? []).filter((other) => other !== lever),
|
|
5488
|
+
updatedAt: now
|
|
5489
|
+
},
|
|
5490
|
+
message: `${lever} accepted \u2014 policy version ${policy.version}. Theo rolls it back if retries or failures rise.`
|
|
5491
|
+
};
|
|
5492
|
+
}
|
|
5493
|
+
function rejectProposal(memory, lever, now) {
|
|
5494
|
+
if (!memory.proposals.some((candidate) => candidate.lever === lever)) {
|
|
5495
|
+
return { ok: false, reason: `No open proposal for '${lever}'. ${openProposalList(memory)}` };
|
|
5496
|
+
}
|
|
5497
|
+
const declined = memory.declined ?? [];
|
|
5498
|
+
return {
|
|
5499
|
+
ok: true,
|
|
5500
|
+
memory: {
|
|
5501
|
+
...memory,
|
|
5502
|
+
proposals: memory.proposals.filter((candidate) => candidate.lever !== lever),
|
|
5503
|
+
hypotheses: memory.hypotheses.filter((hypothesis) => hypothesis.lever !== lever),
|
|
5504
|
+
declined: declined.includes(lever) ? declined : [...declined, lever],
|
|
5505
|
+
updatedAt: now
|
|
5506
|
+
},
|
|
5507
|
+
message: `${lever} declined \u2014 Theo will not propose it again unless you ask him to reconsider.`
|
|
5508
|
+
};
|
|
5509
|
+
}
|
|
5510
|
+
function reconsiderLever(memory, lever, now) {
|
|
5511
|
+
if (!(memory.declined ?? []).includes(lever)) {
|
|
5512
|
+
return { ok: false, reason: `'${lever}' is not on the declined list.` };
|
|
5513
|
+
}
|
|
5514
|
+
return {
|
|
5515
|
+
ok: true,
|
|
5516
|
+
memory: {
|
|
5517
|
+
...memory,
|
|
5518
|
+
declined: (memory.declined ?? []).filter((other) => other !== lever),
|
|
5519
|
+
updatedAt: now
|
|
5520
|
+
},
|
|
5521
|
+
message: `${lever} is back in play \u2014 Theo will prove it again on the next gate run.`
|
|
5522
|
+
};
|
|
5523
|
+
}
|
|
5524
|
+
function sumSegments(profile) {
|
|
5525
|
+
return Object.values(profile.segments).reduce((sum, value) => sum + value, 0);
|
|
5526
|
+
}
|
|
5527
|
+
function renderTheoReport(memory, observation, actions) {
|
|
5528
|
+
const coverage = policyCoverage(memory.policy);
|
|
5529
|
+
const published = actions.filter((a) => a.kind === "policy-published").length;
|
|
5530
|
+
const proposed = memory.proposals.length;
|
|
5531
|
+
const lines = [];
|
|
5532
|
+
lines.push(`Theo \u2014 ${memory.customer} (autonomy: ${memory.autonomy})`);
|
|
5533
|
+
lines.push(
|
|
5534
|
+
`OBSERVE: ${observation.profile.name}, ${observation.profile.requests} requests, ${Math.round(sumSegments(observation.profile))} tok/req (from blind features only).`
|
|
5535
|
+
);
|
|
5536
|
+
lines.push(
|
|
5537
|
+
`ACTIVE: ${coverage.activeCount}/${coverage.enforceableCount} enforceable levers (${coverage.activeCoveragePercent.toFixed(0)}%), lossless on by default.`
|
|
5538
|
+
);
|
|
5539
|
+
if (memory.hypotheses.length > 0) {
|
|
5540
|
+
lines.push(`QUEUE: ${memory.hypotheses.length} lossy candidate(s) ranked to prove (priors, not claims).`);
|
|
5541
|
+
}
|
|
5542
|
+
if (memory.frontier.length > 0) {
|
|
5543
|
+
lines.push(`FRONTIER: ${memory.frontier.length} opportunity(ies) needing new code \u2014 reported, not touched.`);
|
|
5544
|
+
}
|
|
5545
|
+
if (published > 0) lines.push(`APPLIED: ${published} proven lever(s) published to the policy this cycle.`);
|
|
5546
|
+
const rolledBackNow = actions.filter((a) => a.kind === "rolled-back").length;
|
|
5547
|
+
if (rolledBackNow > 0) lines.push(`ROLLED BACK: ${rolledBackNow} lever(s) reverted after a rise in retries or failures (quality drift).`);
|
|
5548
|
+
if (proposed > 0) lines.push(`PROPOSED: ${proposed} proven lever(s) awaiting your approval.`);
|
|
5549
|
+
if (observation.realisedSavingsUsd !== void 0) {
|
|
5550
|
+
lines.push(`LEDGER: $${observation.realisedSavingsUsd.toFixed(2)} verified savings recorded so far.`);
|
|
5551
|
+
}
|
|
5552
|
+
lines.push("Theo never enables a saving he has not proven on your own traffic within your quality ceiling.");
|
|
5553
|
+
return lines.join("\n");
|
|
5554
|
+
}
|
|
5555
|
+
|
|
5556
|
+
// packages/sdk-js/src/theo-agent-store.ts
|
|
5557
|
+
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4, writeFileSync } from "node:fs";
|
|
5558
|
+
import { dirname as dirname4, join as join4 } from "node:path";
|
|
5559
|
+
|
|
5560
|
+
// packages/sdk-js/src/savings-snapshot.ts
|
|
5561
|
+
import { readdirSync, readFileSync as readFileSync3 } from "node:fs";
|
|
5562
|
+
import { join as join3 } from "node:path";
|
|
5563
|
+
var EMPTY = {
|
|
5564
|
+
requests: 0,
|
|
5565
|
+
savedUsd: 0,
|
|
5566
|
+
spendUsd: 0,
|
|
5567
|
+
verdict: "do_not_charge",
|
|
5568
|
+
chainOk: true
|
|
5569
|
+
};
|
|
5570
|
+
function savingsSnapshotFromEvents(events, chainOk = true) {
|
|
5571
|
+
if (!chainOk) return { ...EMPTY, chainOk: false };
|
|
5572
|
+
if (events.length === 0) return EMPTY;
|
|
5573
|
+
const invoice = recomputeInvoice(events, { feePercent: 1 });
|
|
5574
|
+
const spendUsd = events.reduce((sum, event) => sum + (event.actualUsd ?? 0), 0);
|
|
5575
|
+
return {
|
|
5576
|
+
requests: invoice.eventsConsidered,
|
|
5577
|
+
savedUsd: invoice.netVerifiedSavingsUsd,
|
|
5578
|
+
spendUsd,
|
|
5579
|
+
verdict: invoice.verdict,
|
|
5580
|
+
chainOk: true
|
|
5581
|
+
};
|
|
5582
|
+
}
|
|
5583
|
+
function readLedgerRecords(path) {
|
|
5584
|
+
try {
|
|
5585
|
+
const lines = readFileSync3(path, "utf8").trim().split(/\r?\n/).filter(Boolean);
|
|
5586
|
+
return lines.map((line) => JSON.parse(line));
|
|
5587
|
+
} catch {
|
|
5588
|
+
return null;
|
|
5589
|
+
}
|
|
5590
|
+
}
|
|
5591
|
+
function savingsSnapshotFromSpoolDir(dir) {
|
|
5592
|
+
let files;
|
|
5593
|
+
try {
|
|
5594
|
+
files = readdirSync(dir).filter(
|
|
5595
|
+
(f) => f.endsWith(".jsonl") && !f.endsWith(".features.jsonl")
|
|
5596
|
+
);
|
|
5597
|
+
} catch {
|
|
5598
|
+
return EMPTY;
|
|
5599
|
+
}
|
|
5600
|
+
if (files.length === 0) return EMPTY;
|
|
5601
|
+
const events = [];
|
|
5602
|
+
let chainOk = true;
|
|
5603
|
+
for (const file of files) {
|
|
5604
|
+
const records = readLedgerRecords(join3(dir, file));
|
|
5605
|
+
if (records === null || records.length === 0) continue;
|
|
5606
|
+
if (!verifyLedgerChain(records).ok) {
|
|
5607
|
+
chainOk = false;
|
|
5608
|
+
continue;
|
|
5609
|
+
}
|
|
5610
|
+
for (const record of records) events.push(record.event);
|
|
5611
|
+
}
|
|
5612
|
+
if (events.length === 0) return { ...EMPTY, chainOk };
|
|
5613
|
+
return { ...savingsSnapshotFromEvents(events, true), chainOk };
|
|
5614
|
+
}
|
|
5615
|
+
|
|
5616
|
+
// packages/sdk-js/src/theo-agent-store.ts
|
|
5617
|
+
function defaultTheoMemoryPath(ledgerDir, customer) {
|
|
5618
|
+
const safe = customer.replace(/[^A-Za-z0-9._-]/g, "_") || "default";
|
|
5619
|
+
return join4(ledgerDir, "theo", `${safe}.json`);
|
|
5620
|
+
}
|
|
5621
|
+
function loadTheoMemory(path) {
|
|
5622
|
+
try {
|
|
5623
|
+
const parsed = JSON.parse(readFileSync4(path, "utf8"));
|
|
5624
|
+
if (parsed && typeof parsed === "object" && parsed.schemaVersion === 1) return parsed;
|
|
5625
|
+
return null;
|
|
5626
|
+
} catch {
|
|
5627
|
+
return null;
|
|
5628
|
+
}
|
|
5629
|
+
}
|
|
5630
|
+
function saveTheoMemory(path, memory) {
|
|
5631
|
+
mkdirSync3(dirname4(path), { recursive: true });
|
|
5632
|
+
writeFileSync(path, `${JSON.stringify(memory, null, 2)}
|
|
5633
|
+
`, "utf8");
|
|
5634
|
+
}
|
|
5635
|
+
function realisedFromSpoolDir(ledgerDir) {
|
|
5636
|
+
const snapshot = savingsSnapshotFromSpoolDir(ledgerDir);
|
|
5637
|
+
return {
|
|
5638
|
+
savingsUsd: snapshot.savedUsd,
|
|
5639
|
+
spendUsd: snapshot.spendUsd,
|
|
5640
|
+
requests: snapshot.requests,
|
|
5641
|
+
chainOk: snapshot.chainOk
|
|
5642
|
+
};
|
|
5643
|
+
}
|
|
5644
|
+
|
|
5645
|
+
// packages/sdk-js/src/graders.ts
|
|
5646
|
+
function words(text) {
|
|
5647
|
+
return text.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? [];
|
|
5648
|
+
}
|
|
5649
|
+
function overlap(left, right) {
|
|
5650
|
+
const counts = /* @__PURE__ */ new Map();
|
|
5651
|
+
for (const word of left) counts.set(word, (counts.get(word) ?? 0) + 1);
|
|
5652
|
+
let shared = 0;
|
|
5653
|
+
for (const word of right) {
|
|
5654
|
+
const remaining = counts.get(word) ?? 0;
|
|
5655
|
+
if (remaining > 0) {
|
|
5656
|
+
shared += 1;
|
|
5657
|
+
counts.set(word, remaining - 1);
|
|
5658
|
+
}
|
|
5659
|
+
}
|
|
5660
|
+
return shared;
|
|
5661
|
+
}
|
|
5662
|
+
var textSimilarityGrader = ({ baselineResponse, candidateResponse }) => {
|
|
5663
|
+
const baseline = words(responseText(baselineResponse));
|
|
5664
|
+
const candidate = words(responseText(candidateResponse));
|
|
5665
|
+
if (baseline.length === 0 && candidate.length === 0) return 1;
|
|
5666
|
+
if (baseline.length === 0 || candidate.length === 0) return 0;
|
|
5667
|
+
const shared = overlap(baseline, candidate);
|
|
5668
|
+
if (shared === 0) return 0;
|
|
5669
|
+
const precision = shared / candidate.length;
|
|
5670
|
+
const recall = shared / baseline.length;
|
|
5671
|
+
return 2 * precision * recall / (precision + recall);
|
|
5672
|
+
};
|
|
5673
|
+
function wasTruncated(response) {
|
|
5674
|
+
if (typeof response !== "object" || response === null) return false;
|
|
5675
|
+
const row = response;
|
|
5676
|
+
if (row.finish_reason === "length" || row.stop_reason === "max_tokens") return true;
|
|
5677
|
+
if (row.incomplete_details !== null && typeof row.incomplete_details === "object") {
|
|
5678
|
+
const reason = row.incomplete_details.reason;
|
|
5679
|
+
if (reason === "max_output_tokens" || reason === "max_tokens") return true;
|
|
5680
|
+
}
|
|
5681
|
+
if (Array.isArray(row.choices)) {
|
|
5682
|
+
return row.choices.some((choice) => typeof choice === "object" && choice !== null && choice.finish_reason === "length");
|
|
5683
|
+
}
|
|
5684
|
+
return false;
|
|
5685
|
+
}
|
|
5686
|
+
function truncationAwareGrader(options = {}) {
|
|
5687
|
+
const floor = options.uselessBelowCoverage ?? 0.5;
|
|
5688
|
+
const penalty = options.truncationPenalty ?? 0.05;
|
|
5689
|
+
if (floor < 0 || floor > 1) throw new Error("uselessBelowCoverage must be between 0 and 1.");
|
|
5690
|
+
if (penalty < 0 || penalty > 1) throw new Error("truncationPenalty must be between 0 and 1.");
|
|
5691
|
+
return (comparison) => {
|
|
5692
|
+
const baseline = words(responseText(comparison.baselineResponse));
|
|
5693
|
+
const candidate = words(responseText(comparison.candidateResponse));
|
|
5694
|
+
const truncated = wasTruncated(comparison.candidateResponse);
|
|
5695
|
+
if (!truncated) return 1;
|
|
5696
|
+
if (baseline.length === 0) return 1;
|
|
5697
|
+
const coverage = Math.min(1, candidate.length / baseline.length);
|
|
5698
|
+
if (coverage < floor) return 0;
|
|
5699
|
+
return Math.max(0, coverage - penalty);
|
|
5700
|
+
};
|
|
5701
|
+
}
|
|
5702
|
+
function compositeGrader(parts) {
|
|
5703
|
+
if (parts.length === 0) throw new Error("A composite grader needs at least one part.");
|
|
5704
|
+
if (parts.some((part) => !Number.isFinite(part.weight) || part.weight <= 0)) {
|
|
5705
|
+
throw new Error("Composite grader weights must be positive finite numbers.");
|
|
5706
|
+
}
|
|
5707
|
+
const total = parts.reduce((sum, part) => sum + part.weight, 0);
|
|
5708
|
+
return async (comparison) => {
|
|
5709
|
+
let score = 0;
|
|
5710
|
+
for (const part of parts) {
|
|
5711
|
+
const value = await part.grader(comparison);
|
|
5712
|
+
score += (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0) * part.weight;
|
|
5713
|
+
}
|
|
5714
|
+
return score / total;
|
|
5715
|
+
};
|
|
5716
|
+
}
|
|
5717
|
+
function agentGrader(options = {}) {
|
|
5718
|
+
const truncation = truncationAwareGrader(options);
|
|
5719
|
+
return async (comparison) => {
|
|
5720
|
+
const baselineCalls = responseToolCalls(comparison.baselineResponse);
|
|
5721
|
+
const candidateCalls = responseToolCalls(comparison.candidateResponse);
|
|
5722
|
+
if (baselineCalls.length > 0 || candidateCalls.length > 0) {
|
|
5723
|
+
const tool = await toolSelectionGrader(comparison);
|
|
5724
|
+
if (tool < 1) return tool;
|
|
5725
|
+
}
|
|
5726
|
+
return truncation(comparison);
|
|
5727
|
+
};
|
|
5728
|
+
}
|
|
5729
|
+
|
|
5730
|
+
// packages/sdk-js/src/theo-gate.ts
|
|
5731
|
+
async function proveCandidates(options) {
|
|
5732
|
+
const levers = [...new Set(options.levers)];
|
|
5733
|
+
if (levers.length === 0) return { proofs: [], rejections: [], calls: 0 };
|
|
5734
|
+
if (options.requests.length === 0) {
|
|
5735
|
+
return { proofs: [], rejections: levers.map((lever) => ({ lever, reason: "no request sample to replay" })), calls: 0 };
|
|
5736
|
+
}
|
|
5737
|
+
const candidates = levers.map((lever) => ({
|
|
5738
|
+
label: lever,
|
|
5739
|
+
config: leverToReductionConfig({ lever, class: "lossy" })
|
|
5740
|
+
}));
|
|
5741
|
+
const perRequestCalls = candidates.length + (options.baselineResponses ? 0 : 1);
|
|
5742
|
+
let requests = [...options.requests];
|
|
5743
|
+
let baselineResponses = options.baselineResponses ? [...options.baselineResponses] : void 0;
|
|
5744
|
+
let budgetNote;
|
|
5745
|
+
if (options.maxProviderCalls !== void 0) {
|
|
5746
|
+
const maxRequests = Math.floor(options.maxProviderCalls / Math.max(1, perRequestCalls));
|
|
5747
|
+
if (maxRequests < 1) {
|
|
5748
|
+
return {
|
|
5749
|
+
proofs: [],
|
|
5750
|
+
rejections: levers.map((lever) => ({
|
|
5751
|
+
lever,
|
|
5752
|
+
reason: `exploration budget of ${options.maxProviderCalls} call(s) is too small to prove it`
|
|
5753
|
+
})),
|
|
5754
|
+
calls: 0,
|
|
5755
|
+
note: `skipped: ${perRequestCalls} call(s)/request exceeds the ${options.maxProviderCalls}-call budget`
|
|
5756
|
+
};
|
|
5757
|
+
}
|
|
5758
|
+
if (requests.length > maxRequests) {
|
|
5759
|
+
requests = requests.slice(0, maxRequests);
|
|
5760
|
+
baselineResponses = baselineResponses?.slice(0, maxRequests);
|
|
5761
|
+
budgetNote = `capped to ${maxRequests} request(s) to stay within the ${options.maxProviderCalls}-call budget`;
|
|
5762
|
+
}
|
|
5763
|
+
}
|
|
5764
|
+
const result = await replayGate({
|
|
5765
|
+
requests,
|
|
5766
|
+
call: options.call,
|
|
5767
|
+
candidates,
|
|
5768
|
+
grade: options.grader ?? agentGrader(),
|
|
5769
|
+
maximumQualityDrop: options.maximumQualityDrop ?? 0.02,
|
|
5770
|
+
...baselineResponses ? { baselineResponses } : {},
|
|
5771
|
+
...options.price ? { price: options.price } : {},
|
|
5772
|
+
...options.groupOf ? { groupOf: options.groupOf } : {}
|
|
5773
|
+
});
|
|
5774
|
+
const graderName = options.graderName ?? "agent";
|
|
5775
|
+
const proofs = [];
|
|
5776
|
+
const rejections = [];
|
|
5777
|
+
for (const candidate of result.candidates) {
|
|
5778
|
+
if (candidate.approved) {
|
|
5779
|
+
proofs.push({
|
|
5780
|
+
lever: candidate.label,
|
|
5781
|
+
// Prefer the cost saving (the bill) when priced; fall back to tokens.
|
|
5782
|
+
savingPercent: candidate.costSavingPercent ?? candidate.tokenSavingPercent,
|
|
5783
|
+
// One-sided 95% quality bound, a fraction ≤ 0; expressed as a signed percent.
|
|
5784
|
+
qualityDeltaPercent: candidate.lower95QualityDelta * 100,
|
|
5785
|
+
grader: graderName,
|
|
5786
|
+
provenAt: options.now
|
|
5787
|
+
});
|
|
5788
|
+
} else {
|
|
5789
|
+
rejections.push({ lever: candidate.label, reason: candidate.rejectionReason ?? "not approved by the gate" });
|
|
5790
|
+
}
|
|
5791
|
+
}
|
|
5792
|
+
return { proofs, rejections, calls: result.calls, ...budgetNote ? { note: budgetNote } : {} };
|
|
5793
|
+
}
|
|
4439
5794
|
|
|
4440
5795
|
// packages/sdk-js/src/semantic-compaction.ts
|
|
4441
5796
|
function isPlainObject9(value) {
|
|
@@ -4707,7 +6062,7 @@ independently, without MESH and without accepting any terms:
|
|
|
4707
6062
|
recomputation disagrees with this statement, yours wins. Negative levers are
|
|
4708
6063
|
shown negative and reduce the bill; savings you already had are itemised above
|
|
4709
6064
|
and never billed. Methodology: https://www.mesh-technology.com/methodology.html<br>
|
|
4710
|
-
MESH is a
|
|
6065
|
+
MESH is a trading name of Flux AI Technologies Limited, registered in England and Wales, company no. 17543982.
|
|
4711
6066
|
</footer>
|
|
4712
6067
|
</main>
|
|
4713
6068
|
</body>
|
|
@@ -4715,98 +6070,74 @@ MESH is a product of Mesh AI Technologies Limited, registered in England and Wal
|
|
|
4715
6070
|
`;
|
|
4716
6071
|
}
|
|
4717
6072
|
|
|
4718
|
-
// packages/sdk-js/src/
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
const
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
}
|
|
4746
|
-
function
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
const baseline = words(responseText(comparison.baselineResponse));
|
|
4766
|
-
const candidate = words(responseText(comparison.candidateResponse));
|
|
4767
|
-
const truncated = wasTruncated(comparison.candidateResponse);
|
|
4768
|
-
if (!truncated) return 1;
|
|
4769
|
-
if (baseline.length === 0) return 1;
|
|
4770
|
-
const coverage = Math.min(1, candidate.length / baseline.length);
|
|
4771
|
-
if (coverage < floor) return 0;
|
|
4772
|
-
return Math.max(0, coverage - penalty);
|
|
4773
|
-
};
|
|
4774
|
-
}
|
|
4775
|
-
function compositeGrader(parts) {
|
|
4776
|
-
if (parts.length === 0) throw new Error("A composite grader needs at least one part.");
|
|
4777
|
-
if (parts.some((part) => !Number.isFinite(part.weight) || part.weight <= 0)) {
|
|
4778
|
-
throw new Error("Composite grader weights must be positive finite numbers.");
|
|
6073
|
+
// packages/sdk-js/src/cta.ts
|
|
6074
|
+
var MESH_SIGNUP_URL = "https://www.mesh-technology.com";
|
|
6075
|
+
function savingsPercent(snap) {
|
|
6076
|
+
const baseline = snap.spendUsd + snap.savedUsd;
|
|
6077
|
+
if (baseline <= 0) return null;
|
|
6078
|
+
return snap.savedUsd / baseline * 100;
|
|
6079
|
+
}
|
|
6080
|
+
function formatSavingsCta(snap, opts = {}) {
|
|
6081
|
+
const mark = opts.mark ?? "\u2B22 MESH";
|
|
6082
|
+
const url = opts.signupUrl ?? MESH_SIGNUP_URL;
|
|
6083
|
+
if (!snap.chainOk) {
|
|
6084
|
+
return `${mark}: a ledger spool failed to verify \u2014 savings withheld. Recompute with \`mesh verify\`.`;
|
|
6085
|
+
}
|
|
6086
|
+
if (snap.requests === 0) {
|
|
6087
|
+
return `${mark}: no requests metered yet. Point a client's base_url here and your verified savings print on exit.`;
|
|
6088
|
+
}
|
|
6089
|
+
if (snap.verdict === "do_not_charge" || snap.savedUsd <= 0) {
|
|
6090
|
+
return [
|
|
6091
|
+
`${mark}: ${snap.requests} requests metered \xB7 verified savings $${snap.savedUsd.toFixed(4)}.`,
|
|
6092
|
+
`Your stack is already lean \u2014 verdict do_not_charge. You owe MESH nothing.`,
|
|
6093
|
+
`Recompute it yourself: \`mesh verify\`.`
|
|
6094
|
+
].join("\n");
|
|
6095
|
+
}
|
|
6096
|
+
const pct = savingsPercent(snap);
|
|
6097
|
+
const pctText = pct === null ? "" : ` (${pct.toFixed(1)}% lower)`;
|
|
6098
|
+
return [
|
|
6099
|
+
``,
|
|
6100
|
+
`${mark} saved you $${snap.savedUsd.toFixed(4)}${pctText} across ${snap.requests} verified requests.`,
|
|
6101
|
+
`This number is a pure function of a hash-chained ledger you hold \u2014 recompute it: \`mesh verify\`.`,
|
|
6102
|
+
`Claim your savings and keep them metered: ${url}`,
|
|
6103
|
+
``
|
|
6104
|
+
].join("\n");
|
|
6105
|
+
}
|
|
6106
|
+
function savingsBadge(snap, opts = {}) {
|
|
6107
|
+
const url = opts.signupUrl ?? MESH_SIGNUP_URL;
|
|
6108
|
+
const pct = savingsPercent(snap);
|
|
6109
|
+
let message;
|
|
6110
|
+
let color;
|
|
6111
|
+
if (!snap.chainOk) {
|
|
6112
|
+
message = "unverified";
|
|
6113
|
+
color = "lightgrey";
|
|
6114
|
+
} else if (snap.verdict === "do_not_charge" || snap.savedUsd <= 0) {
|
|
6115
|
+
message = "already lean";
|
|
6116
|
+
color = "blue";
|
|
6117
|
+
} else {
|
|
6118
|
+
message = pct === null ? `$${snap.savedUsd.toFixed(2)} saved` : `${pct.toFixed(0)}% lower`;
|
|
6119
|
+
color = "brightgreen";
|
|
4779
6120
|
}
|
|
4780
|
-
const
|
|
4781
|
-
return
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
const value = await part.grader(comparison);
|
|
4785
|
-
score += (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0) * part.weight;
|
|
4786
|
-
}
|
|
4787
|
-
return score / total;
|
|
4788
|
-
};
|
|
4789
|
-
}
|
|
4790
|
-
function agentGrader(options = {}) {
|
|
4791
|
-
const truncation = truncationAwareGrader(options);
|
|
4792
|
-
return async (comparison) => {
|
|
4793
|
-
const baselineCalls = responseToolCalls(comparison.baselineResponse);
|
|
4794
|
-
const candidateCalls = responseToolCalls(comparison.candidateResponse);
|
|
4795
|
-
if (baselineCalls.length > 0 || candidateCalls.length > 0) {
|
|
4796
|
-
const tool = await toolSelectionGrader(comparison);
|
|
4797
|
-
if (tool < 1) return tool;
|
|
4798
|
-
}
|
|
4799
|
-
return truncation(comparison);
|
|
6121
|
+
const imageUrl = `https://img.shields.io/static/v1?label=${encodeURIComponent("LLM cost")}&message=${encodeURIComponent(message)}&color=${color}`;
|
|
6122
|
+
return {
|
|
6123
|
+
imageUrl,
|
|
6124
|
+
markdown: `[](${url})`
|
|
4800
6125
|
};
|
|
4801
6126
|
}
|
|
4802
6127
|
export {
|
|
4803
6128
|
CachePrefixVolatilityGuard,
|
|
6129
|
+
DRIFT_RISE_THRESHOLD,
|
|
6130
|
+
DRIFT_WINDOW_EVENTS,
|
|
4804
6131
|
ExactResponseCache,
|
|
4805
6132
|
LEDGER_GENESIS_HASH,
|
|
6133
|
+
MESH_SIGNUP_URL,
|
|
6134
|
+
acceptProposal,
|
|
4806
6135
|
agentGrader,
|
|
6136
|
+
asBundle,
|
|
4807
6137
|
auditEventPricing,
|
|
4808
6138
|
billableEventFromRequest,
|
|
4809
6139
|
buildQuoteReport,
|
|
6140
|
+
bundleToReductionConfig,
|
|
4810
6141
|
canonicalJson,
|
|
4811
6142
|
capToolResultBody,
|
|
4812
6143
|
catalogueVersion,
|
|
@@ -4816,32 +6147,55 @@ export {
|
|
|
4816
6147
|
compositeGrader,
|
|
4817
6148
|
createLocalJsonlCapture,
|
|
4818
6149
|
createSavingsLedger,
|
|
6150
|
+
createTheoMemory,
|
|
4819
6151
|
defaultPolicy,
|
|
6152
|
+
defaultTheoMemoryPath,
|
|
4820
6153
|
digestStaleBody,
|
|
4821
6154
|
exactTextGrader,
|
|
4822
6155
|
extractBlindFeatures,
|
|
6156
|
+
formatSavingsCta,
|
|
6157
|
+
isPolicyBundle,
|
|
4823
6158
|
knownLeverIds,
|
|
4824
6159
|
ledgerEventHash,
|
|
6160
|
+
leverToReductionConfig,
|
|
6161
|
+
loadTheoMemory,
|
|
6162
|
+
mergePolicyIntoBundle,
|
|
6163
|
+
observeWorkload,
|
|
4825
6164
|
parseGatewayExport,
|
|
6165
|
+
policyForRoute,
|
|
4826
6166
|
policyToReductionConfig,
|
|
4827
6167
|
priceLedgerUsage,
|
|
6168
|
+
proveCandidates,
|
|
4828
6169
|
publishLever,
|
|
4829
6170
|
publishedModelPrices,
|
|
6171
|
+
realisedFromSpoolDir,
|
|
6172
|
+
recentDriftRates,
|
|
4830
6173
|
recomputeInvoice,
|
|
6174
|
+
reconsiderLever,
|
|
4831
6175
|
reduceRequest,
|
|
4832
6176
|
reduceRequestAsync,
|
|
6177
|
+
rejectProposal,
|
|
4833
6178
|
renderLedgerReport,
|
|
4834
6179
|
renderQuoteReport,
|
|
6180
|
+
renderTheoReport,
|
|
4835
6181
|
replayGate,
|
|
4836
6182
|
replayUsage,
|
|
4837
6183
|
responseText,
|
|
4838
6184
|
responseToolCalls,
|
|
6185
|
+
routeMatches,
|
|
6186
|
+
runTheoCycle,
|
|
6187
|
+
saveTheoMemory,
|
|
6188
|
+
savingsBadge,
|
|
6189
|
+
savingsPercent,
|
|
6190
|
+
savingsSnapshotFromEvents,
|
|
6191
|
+
savingsSnapshotFromSpoolDir,
|
|
4839
6192
|
serviceTierOf,
|
|
4840
6193
|
startMeshProxy,
|
|
4841
6194
|
textSimilarityGrader,
|
|
4842
6195
|
toolNameGrader,
|
|
4843
6196
|
toolSelectionGrader,
|
|
4844
6197
|
truncationAwareGrader,
|
|
6198
|
+
upsertRoutePolicy,
|
|
4845
6199
|
usageFromSse,
|
|
4846
6200
|
verifyLedgerChain,
|
|
4847
6201
|
withMesh,
|