@mupt-ai/dari-router 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +100 -0
- package/dist/anonymous_actions.d.ts +55 -0
- package/dist/anonymous_actions.js +277 -0
- package/dist/anonymous_actions.js.map +1 -0
- package/dist/auto_router.d.ts +10 -0
- package/dist/auto_router.js +56 -0
- package/dist/auto_router.js.map +1 -0
- package/dist/cache_behavior.d.ts +14 -0
- package/dist/cache_behavior.js +112 -0
- package/dist/cache_behavior.js.map +1 -0
- package/dist/cache_scope.d.ts +5 -0
- package/dist/cache_scope.js +49 -0
- package/dist/cache_scope.js.map +1 -0
- package/dist/cache_switch_decision.d.ts +24 -0
- package/dist/cache_switch_decision.js +21 -0
- package/dist/cache_switch_decision.js.map +1 -0
- package/dist/candidate_pruning.d.ts +45 -0
- package/dist/candidate_pruning.js +154 -0
- package/dist/candidate_pruning.js.map +1 -0
- package/dist/candidate_resolution.d.ts +41 -0
- package/dist/candidate_resolution.js +111 -0
- package/dist/candidate_resolution.js.map +1 -0
- package/dist/compatibility.d.ts +22 -0
- package/dist/compatibility.js +76 -0
- package/dist/compatibility.js.map +1 -0
- package/dist/continuation_state.d.ts +36 -0
- package/dist/continuation_state.js +249 -0
- package/dist/continuation_state.js.map +1 -0
- package/dist/cost.d.ts +31 -0
- package/dist/cost.js +314 -0
- package/dist/cost.js.map +1 -0
- package/dist/dari_policy.d.ts +34 -0
- package/dist/dari_policy.js +267 -0
- package/dist/dari_policy.js.map +1 -0
- package/dist/errors.d.ts +7 -0
- package/dist/errors.js +14 -0
- package/dist/errors.js.map +1 -0
- package/dist/fingerprint.d.ts +13 -0
- package/dist/fingerprint.js +130 -0
- package/dist/fingerprint.js.map +1 -0
- package/dist/fixed_turn_cost_config.d.ts +9 -0
- package/dist/fixed_turn_cost_config.js +22 -0
- package/dist/fixed_turn_cost_config.js.map +1 -0
- package/dist/framework_error.d.ts +8 -0
- package/dist/framework_error.js +24 -0
- package/dist/framework_error.js.map +1 -0
- package/dist/framework_lifecycle.d.ts +11 -0
- package/dist/framework_lifecycle.js +57 -0
- package/dist/framework_lifecycle.js.map +1 -0
- package/dist/framework_models.d.ts +3 -0
- package/dist/framework_models.js +120 -0
- package/dist/framework_models.js.map +1 -0
- package/dist/framework_router.d.ts +2 -0
- package/dist/framework_router.js +384 -0
- package/dist/framework_router.js.map +1 -0
- package/dist/framework_streaming.d.ts +26 -0
- package/dist/framework_streaming.js +486 -0
- package/dist/framework_streaming.js.map +1 -0
- package/dist/framework_types.d.ts +272 -0
- package/dist/framework_types.js +5 -0
- package/dist/framework_types.js.map +1 -0
- package/dist/framework_validation.d.ts +7 -0
- package/dist/framework_validation.js +169 -0
- package/dist/framework_validation.js.map +1 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +38 -0
- package/dist/index.js.map +1 -0
- package/dist/json.d.ts +10 -0
- package/dist/json.js +29 -0
- package/dist/json.js.map +1 -0
- package/dist/model_ids.d.ts +4 -0
- package/dist/model_ids.js +26 -0
- package/dist/model_ids.js.map +1 -0
- package/dist/pi_context.d.ts +4 -0
- package/dist/pi_context.js +206 -0
- package/dist/pi_context.js.map +1 -0
- package/dist/pi_options.d.ts +3 -0
- package/dist/pi_options.js +240 -0
- package/dist/pi_options.js.map +1 -0
- package/dist/pi_output.d.ts +4 -0
- package/dist/pi_output.js +328 -0
- package/dist/pi_output.js.map +1 -0
- package/dist/pi_reasoning_signature.d.ts +10 -0
- package/dist/pi_reasoning_signature.js +33 -0
- package/dist/pi_reasoning_signature.js.map +1 -0
- package/dist/pi_runtime.d.ts +3 -0
- package/dist/pi_runtime.js +155 -0
- package/dist/pi_runtime.js.map +1 -0
- package/dist/pi_types.d.ts +35 -0
- package/dist/pi_types.js +2 -0
- package/dist/pi_types.js.map +1 -0
- package/dist/portable_reasoning_state.d.ts +7 -0
- package/dist/portable_reasoning_state.js +81 -0
- package/dist/portable_reasoning_state.js.map +1 -0
- package/dist/previous_decision.d.ts +13 -0
- package/dist/previous_decision.js +82 -0
- package/dist/previous_decision.js.map +1 -0
- package/dist/prompts.d.ts +3 -0
- package/dist/prompts.js +65 -0
- package/dist/prompts.js.map +1 -0
- package/dist/protocol_anthropic.d.ts +18 -0
- package/dist/protocol_anthropic.js +443 -0
- package/dist/protocol_anthropic.js.map +1 -0
- package/dist/protocol_openai_chat.d.ts +25 -0
- package/dist/protocol_openai_chat.js +503 -0
- package/dist/protocol_openai_chat.js.map +1 -0
- package/dist/protocol_validation.d.ts +23 -0
- package/dist/protocol_validation.js +134 -0
- package/dist/protocol_validation.js.map +1 -0
- package/dist/route.d.ts +105 -0
- package/dist/route.js +226 -0
- package/dist/route.js.map +1 -0
- package/dist/routing_history.d.ts +7 -0
- package/dist/routing_history.js +132 -0
- package/dist/routing_history.js.map +1 -0
- package/dist/selector_input.d.ts +42 -0
- package/dist/selector_input.js +153 -0
- package/dist/selector_input.js.map +1 -0
- package/dist/selector_parse.d.ts +13 -0
- package/dist/selector_parse.js +95 -0
- package/dist/selector_parse.js.map +1 -0
- package/dist/selector_request.d.ts +26 -0
- package/dist/selector_request.js +85 -0
- package/dist/selector_request.js.map +1 -0
- package/dist/selector_truncation.d.ts +2 -0
- package/dist/selector_truncation.js +48 -0
- package/dist/selector_truncation.js.map +1 -0
- package/dist/selector_wire.d.ts +10 -0
- package/dist/selector_wire.js +113 -0
- package/dist/selector_wire.js.map +1 -0
- package/dist/types.d.ts +152 -0
- package/dist/types.js +14 -0
- package/dist/types.js.map +1 -0
- package/package.json +64 -0
- package/src/anonymous_actions.ts +434 -0
- package/src/auto_router.ts +95 -0
- package/src/cache_behavior.ts +130 -0
- package/src/cache_scope.ts +58 -0
- package/src/cache_switch_decision.ts +53 -0
- package/src/candidate_pruning.ts +257 -0
- package/src/candidate_resolution.ts +208 -0
- package/src/compatibility.ts +162 -0
- package/src/continuation_state.ts +368 -0
- package/src/cost.ts +455 -0
- package/src/dari_policy.ts +360 -0
- package/src/errors.ts +27 -0
- package/src/fingerprint.ts +148 -0
- package/src/fixed_turn_cost_config.ts +24 -0
- package/src/framework_error.ts +38 -0
- package/src/framework_lifecycle.ts +74 -0
- package/src/framework_models.ts +186 -0
- package/src/framework_router.ts +490 -0
- package/src/framework_streaming.ts +613 -0
- package/src/framework_types.ts +372 -0
- package/src/framework_validation.ts +235 -0
- package/src/index.ts +37 -0
- package/src/json.ts +43 -0
- package/src/model_ids.ts +29 -0
- package/src/pi_context.ts +251 -0
- package/src/pi_options.ts +295 -0
- package/src/pi_output.ts +387 -0
- package/src/pi_reasoning_signature.ts +41 -0
- package/src/pi_runtime.ts +238 -0
- package/src/pi_types.ts +71 -0
- package/src/portable_reasoning_state.ts +129 -0
- package/src/previous_decision.ts +109 -0
- package/src/prompts.ts +69 -0
- package/src/protocol_anthropic.ts +571 -0
- package/src/protocol_openai_chat.ts +645 -0
- package/src/protocol_validation.ts +202 -0
- package/src/route.ts +490 -0
- package/src/routing_history.ts +157 -0
- package/src/selector_input.ts +246 -0
- package/src/selector_parse.ts +138 -0
- package/src/selector_request.ts +130 -0
- package/src/selector_truncation.ts +48 -0
- package/src/selector_wire.ts +167 -0
- package/src/types.ts +231 -0
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
export declare function isRecord(value: unknown): value is Record<string, unknown>;
|
|
2
|
+
export type Provider = string;
|
|
3
|
+
export type ChatMessage = {
|
|
4
|
+
role: string;
|
|
5
|
+
content?: string | Array<Record<string, unknown>> | null;
|
|
6
|
+
tool_call_id?: string;
|
|
7
|
+
tool_calls?: Array<Record<string, unknown>>;
|
|
8
|
+
reasoning_content?: string;
|
|
9
|
+
reasoning?: string;
|
|
10
|
+
reasoning_text?: string;
|
|
11
|
+
reasoning_details?: Array<Record<string, unknown>>;
|
|
12
|
+
};
|
|
13
|
+
export declare const REASONING_EFFORTS: readonly ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
|
|
14
|
+
export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
|
|
15
|
+
export declare function isReasoningEffort(value: unknown): value is ReasoningEffort;
|
|
16
|
+
export type ResponseFormat = {
|
|
17
|
+
type: "text";
|
|
18
|
+
} | {
|
|
19
|
+
type: "json_object";
|
|
20
|
+
} | {
|
|
21
|
+
type: "json_schema";
|
|
22
|
+
json_schema: {
|
|
23
|
+
name: string;
|
|
24
|
+
schema: Record<string, unknown>;
|
|
25
|
+
strict?: boolean;
|
|
26
|
+
description?: string;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export type ChatCompletionRequest = {
|
|
30
|
+
model?: string;
|
|
31
|
+
messages?: ChatMessage[];
|
|
32
|
+
stream?: boolean;
|
|
33
|
+
source_protocol?: "anthropic_messages" | "openai_chat" | "openai_responses";
|
|
34
|
+
prompt_cache_key?: string;
|
|
35
|
+
temperature?: number;
|
|
36
|
+
top_p?: number;
|
|
37
|
+
max_tokens?: number;
|
|
38
|
+
max_completion_tokens?: number;
|
|
39
|
+
stop?: string | string[];
|
|
40
|
+
tools?: Array<Record<string, unknown>>;
|
|
41
|
+
tool_choice?: string | Record<string, unknown>;
|
|
42
|
+
parallel_tool_calls?: boolean;
|
|
43
|
+
response_format?: ResponseFormat;
|
|
44
|
+
reasoning_effort?: ReasoningEffort;
|
|
45
|
+
metadata?: Record<string, unknown>;
|
|
46
|
+
user?: string;
|
|
47
|
+
};
|
|
48
|
+
export type RoutingCandidate = {
|
|
49
|
+
model: string;
|
|
50
|
+
reasoningEffort: ReasoningEffort;
|
|
51
|
+
};
|
|
52
|
+
export declare function routingCandidateKey(candidate: RoutingCandidate): string;
|
|
53
|
+
export declare function sameRoutingCandidate(left: RoutingCandidate, right: RoutingCandidate): boolean;
|
|
54
|
+
export type RoutingDecision = {
|
|
55
|
+
selectedModel: string;
|
|
56
|
+
reasoningEffort: ReasoningEffort;
|
|
57
|
+
reason: string;
|
|
58
|
+
leaseTurnsRemaining?: number;
|
|
59
|
+
};
|
|
60
|
+
export type RouterEvalScore = {
|
|
61
|
+
model_id: string;
|
|
62
|
+
score: number;
|
|
63
|
+
notes?: string | null;
|
|
64
|
+
thinking_level?: ReasoningEffort | null;
|
|
65
|
+
};
|
|
66
|
+
export type RouterEval = {
|
|
67
|
+
id: string;
|
|
68
|
+
name: string;
|
|
69
|
+
description?: string | null;
|
|
70
|
+
min_score: number;
|
|
71
|
+
max_score: number;
|
|
72
|
+
scores: RouterEvalScore[];
|
|
73
|
+
};
|
|
74
|
+
export type RouterPrefixHit = {
|
|
75
|
+
hash: string;
|
|
76
|
+
loose_hash?: string | null;
|
|
77
|
+
conversation_id: string;
|
|
78
|
+
model: string;
|
|
79
|
+
next_model?: string | null;
|
|
80
|
+
next_reasoning_effort?: ReasoningEffort | null;
|
|
81
|
+
message_depth: number;
|
|
82
|
+
block_depth: number;
|
|
83
|
+
provider_block_depth?: number | null;
|
|
84
|
+
prompt_anchor_depth?: number | null;
|
|
85
|
+
prompt_tokens: number;
|
|
86
|
+
benchmark_replay_depth?: number | null;
|
|
87
|
+
input_tokens: number;
|
|
88
|
+
cache_read_tokens: number;
|
|
89
|
+
cache_write_tokens: number;
|
|
90
|
+
tool_choice_fp: string;
|
|
91
|
+
response_format_fp: string;
|
|
92
|
+
reasoning_bucket?: string | null;
|
|
93
|
+
lease_turns_remaining?: number | null;
|
|
94
|
+
pending_lease_model?: string | null;
|
|
95
|
+
pending_lease_reasoning_effort?: ReasoningEffort | null;
|
|
96
|
+
pending_lease_turns?: number | null;
|
|
97
|
+
pending_lease_reason?: string | null;
|
|
98
|
+
pending_lease_output_text?: string | null;
|
|
99
|
+
reason: string | null;
|
|
100
|
+
updated_at: string;
|
|
101
|
+
};
|
|
102
|
+
export type RoutingStrategy = "slm" | "custom";
|
|
103
|
+
export type CustomRouterRule = {
|
|
104
|
+
when: string;
|
|
105
|
+
use: string;
|
|
106
|
+
thinking_level?: ReasoningEffort | null;
|
|
107
|
+
};
|
|
108
|
+
export type CustomRouterConfig = {
|
|
109
|
+
rules: CustomRouterRule[];
|
|
110
|
+
default?: string | null;
|
|
111
|
+
default_thinking_level?: ReasoningEffort | null;
|
|
112
|
+
};
|
|
113
|
+
export type RouterModelPrice = {
|
|
114
|
+
input: number;
|
|
115
|
+
output: number;
|
|
116
|
+
cached_input?: number | null;
|
|
117
|
+
cache_write?: number | null;
|
|
118
|
+
};
|
|
119
|
+
export type PrefixHit = {
|
|
120
|
+
entry: RouterPrefixHit;
|
|
121
|
+
depth: number;
|
|
122
|
+
};
|
|
123
|
+
export type ChainHits = {
|
|
124
|
+
deepest?: PrefixHit;
|
|
125
|
+
perModel: Map<string, PrefixHit>;
|
|
126
|
+
perModelBuckets: Map<string, Map<string | null, PrefixHit>>;
|
|
127
|
+
};
|
|
128
|
+
export type TurnCostProjection = {
|
|
129
|
+
projected_turns: number;
|
|
130
|
+
total_cost_usd: number;
|
|
131
|
+
};
|
|
132
|
+
export type FixedTurnCostEstimate = {
|
|
133
|
+
output_tokens_per_turn: number;
|
|
134
|
+
assumed_reasoning_effort: ReasoningEffort;
|
|
135
|
+
projections: TurnCostProjection[];
|
|
136
|
+
};
|
|
137
|
+
export type CandidateCostEstimate = {
|
|
138
|
+
model: string;
|
|
139
|
+
reasoning_effort: ReasoningEffort;
|
|
140
|
+
warm_tokens: number;
|
|
141
|
+
est_prompt_tokens: number;
|
|
142
|
+
est_input_cost_usd: number | null;
|
|
143
|
+
output_cost_per_mtok: number | null;
|
|
144
|
+
pricing_known: boolean;
|
|
145
|
+
fixed_turn_cost_estimate: FixedTurnCostEstimate | null;
|
|
146
|
+
};
|
|
147
|
+
export type PreviousDecision = {
|
|
148
|
+
model: string;
|
|
149
|
+
reasoningEffort: ReasoningEffort;
|
|
150
|
+
reason: string;
|
|
151
|
+
};
|
|
152
|
+
export type HardCapability = "image_input" | "structured_output" | "openai_hosted_web_search";
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function isRecord(value) {
|
|
2
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3
|
+
}
|
|
4
|
+
export const REASONING_EFFORTS = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
|
|
5
|
+
export function isReasoningEffort(value) {
|
|
6
|
+
return typeof value === "string" && REASONING_EFFORTS.some((effort) => effort === value);
|
|
7
|
+
}
|
|
8
|
+
export function routingCandidateKey(candidate) {
|
|
9
|
+
return `${candidate.model}\u0000${candidate.reasoningEffort}`;
|
|
10
|
+
}
|
|
11
|
+
export function sameRoutingCandidate(left, right) {
|
|
12
|
+
return left.model === right.model && left.reasoningEffort === right.reasoningEffort;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAiBD,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAU,CAAC;AAGtG,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;AAC3F,CAAC;AAwCD,MAAM,UAAU,mBAAmB,CAAC,SAA2B;IAC7D,OAAO,GAAG,SAAS,CAAC,KAAK,SAAS,SAAS,CAAC,eAAe,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAsB,EACtB,KAAuB;IAEvB,OAAO,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe,CAAC;AACtF,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@mupt-ai/dari-router",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "End-to-end TypeScript framework for routing across LLMs with built-in Pi AI execution and pluggable policies and executors.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/mupt-ai/dari-router-core.git"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://docs.dari.dev/framework/overview",
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/mupt-ai/dari-router-core/issues"
|
|
13
|
+
},
|
|
14
|
+
"keywords": [
|
|
15
|
+
"llm",
|
|
16
|
+
"router",
|
|
17
|
+
"llm-router",
|
|
18
|
+
"model-routing",
|
|
19
|
+
"openai",
|
|
20
|
+
"anthropic",
|
|
21
|
+
"chat-completions",
|
|
22
|
+
"dari"
|
|
23
|
+
],
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public"
|
|
26
|
+
},
|
|
27
|
+
"type": "module",
|
|
28
|
+
"engines": {
|
|
29
|
+
"node": ">=22.19.0"
|
|
30
|
+
},
|
|
31
|
+
"main": "./dist/index.js",
|
|
32
|
+
"types": "./dist/index.d.ts",
|
|
33
|
+
"exports": {
|
|
34
|
+
".": {
|
|
35
|
+
"types": "./dist/index.d.ts",
|
|
36
|
+
"default": "./dist/index.js"
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"files": [
|
|
40
|
+
"dist",
|
|
41
|
+
"src",
|
|
42
|
+
"README.md",
|
|
43
|
+
"LICENSE"
|
|
44
|
+
],
|
|
45
|
+
"scripts": {
|
|
46
|
+
"build": "tsc -p tsconfig.build.json",
|
|
47
|
+
"prepublishOnly": "bun run typecheck && bun test && bun run build",
|
|
48
|
+
"typecheck": "tsc --noEmit",
|
|
49
|
+
"test": "bun test",
|
|
50
|
+
"example": "bun run examples/create_router.ts",
|
|
51
|
+
"example:pi": "bun run examples/pi_runtime.ts",
|
|
52
|
+
"example:dari-policy": "bun run examples/dari_policy.ts",
|
|
53
|
+
"example:decision-pipeline": "bun run examples/basic_route.ts",
|
|
54
|
+
"example:lease-stores": "bun run examples/lease_stores.ts"
|
|
55
|
+
},
|
|
56
|
+
"dependencies": {
|
|
57
|
+
"@mupt-ai/pi-ai": "0.80.6-mupt.2"
|
|
58
|
+
},
|
|
59
|
+
"devDependencies": {
|
|
60
|
+
"@types/bun": "^1.3.4",
|
|
61
|
+
"@types/node": "^24.10.2",
|
|
62
|
+
"typescript": "^5.6.3"
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
// Anonymous candidate-action selector protocol. Candidates are shuffled into
|
|
2
|
+
// anonymous action slots ("A", "B", ..., "AA", ...) and every model-identifying
|
|
3
|
+
// field in the selector input is replaced by its slot, so a trained policy
|
|
4
|
+
// cannot memorize model names or list positions. This is the canonical
|
|
5
|
+
// open-weight selector protocol used by Dari's SLM routing strategy; the
|
|
6
|
+
// named-candidate protocol in selector_input.ts/prompts.ts remains available
|
|
7
|
+
// for custom routers that use a general-purpose LLM selector.
|
|
8
|
+
|
|
9
|
+
import { ANONYMOUS_ACTION_SYSTEM_PROMPT } from "./prompts.js";
|
|
10
|
+
import {
|
|
11
|
+
jsonRecord,
|
|
12
|
+
jsonValue,
|
|
13
|
+
requiredString,
|
|
14
|
+
type JsonObject,
|
|
15
|
+
type JsonValue,
|
|
16
|
+
} from "./json.js";
|
|
17
|
+
import type { TurnCostProjection } from "./types.js";
|
|
18
|
+
|
|
19
|
+
// Deterministic random source in [0, 1). Callers own seeding so experiment
|
|
20
|
+
// harnesses can reproduce recorded permutations with their own PRNG.
|
|
21
|
+
export type Rng = () => number;
|
|
22
|
+
|
|
23
|
+
export type AnonymousActionCandidate = {
|
|
24
|
+
model: string;
|
|
25
|
+
reasoningEffort: string;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type AnonymousActionSlot = {
|
|
29
|
+
action: string;
|
|
30
|
+
candidate: AnonymousActionCandidate;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type AnonymousSelectorMessage = {
|
|
34
|
+
role: "system" | "user";
|
|
35
|
+
content: string;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export type AnonymousPolicyPrompt = {
|
|
39
|
+
messages: [system: AnonymousSelectorMessage, user: AnonymousSelectorMessage];
|
|
40
|
+
actions: string[];
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export type AnonymousActionSelection = {
|
|
44
|
+
action: string;
|
|
45
|
+
candidate: AnonymousActionCandidate;
|
|
46
|
+
// Turns the selection commits to when the caller offered a lease menu;
|
|
47
|
+
// absent for single-turn selectors.
|
|
48
|
+
turns?: number;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
// The de-identified selector input. Every field is JSON-serializable so it can
|
|
52
|
+
// cross a process boundary, but it is a concrete type rather than a bag: it is
|
|
53
|
+
// produced by anonymizeSelectorInput and consumed by the prompt formatter, so
|
|
54
|
+
// nothing downstream needs to re-parse it.
|
|
55
|
+
export type AnonymousEvalScore = {
|
|
56
|
+
action: string;
|
|
57
|
+
rank: number;
|
|
58
|
+
rank_total: number;
|
|
59
|
+
z_score: number;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
export type AnonymousEvalCard = {
|
|
63
|
+
name: string;
|
|
64
|
+
description: string | null;
|
|
65
|
+
scores: AnonymousEvalScore[];
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
export type AnonymousActionCost = {
|
|
69
|
+
action: string;
|
|
70
|
+
projections: TurnCostProjection[];
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
export type AnonymousSelectorInput = {
|
|
74
|
+
candidate_actions: Array<{ action: string }>;
|
|
75
|
+
imported_evals: AnonymousEvalCard[];
|
|
76
|
+
previous_action: { action: string } | null;
|
|
77
|
+
cost_estimates: AnonymousActionCost[];
|
|
78
|
+
messages: JsonValue;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
// Shuffles candidates into spreadsheet-style action slots A..Z, AA..AZ, ...
|
|
82
|
+
// with the injected rng.
|
|
83
|
+
export function assignAnonymousActions(
|
|
84
|
+
candidates: readonly AnonymousActionCandidate[],
|
|
85
|
+
rng: Rng,
|
|
86
|
+
): AnonymousActionSlot[] {
|
|
87
|
+
if (candidates.length < 2) {
|
|
88
|
+
throw new Error(`Anonymous action routing requires at least two candidates, got ${candidates.length}`);
|
|
89
|
+
}
|
|
90
|
+
const keys = new Set<string>();
|
|
91
|
+
for (const candidate of candidates) {
|
|
92
|
+
const key = candidateKey(candidate.model, candidate.reasoningEffort);
|
|
93
|
+
if (keys.has(key)) throw new Error(`Duplicate anonymous action candidate ${key}`);
|
|
94
|
+
keys.add(key);
|
|
95
|
+
}
|
|
96
|
+
return shuffled(candidates, rng).map((candidate, index) => ({
|
|
97
|
+
action: actionForIndex(index),
|
|
98
|
+
candidate,
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function actionForIndex(index: number): string {
|
|
103
|
+
let remaining = index;
|
|
104
|
+
let action = "";
|
|
105
|
+
do {
|
|
106
|
+
action = String.fromCharCode("A".charCodeAt(0) + (remaining % 26)) + action;
|
|
107
|
+
remaining = Math.floor(remaining / 26) - 1;
|
|
108
|
+
} while (remaining >= 0);
|
|
109
|
+
return action;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Rewrites a named-candidate selector input (see selector_input.ts) into its
|
|
113
|
+
// anonymous-action form. The input must reference exactly the candidates in
|
|
114
|
+
// `slots`; any mismatch is an error rather than a silent partial rewrite.
|
|
115
|
+
export function anonymizeSelectorInput(
|
|
116
|
+
selectorInput: JsonObject,
|
|
117
|
+
slots: readonly AnonymousActionSlot[],
|
|
118
|
+
): AnonymousSelectorInput {
|
|
119
|
+
const input = jsonRecord(structuredClone(selectorInput), "selector input");
|
|
120
|
+
const byCandidate = new Map(
|
|
121
|
+
slots.map((slot) => [candidateKey(slot.candidate.model, slot.candidate.reasoningEffort), slot.action]),
|
|
122
|
+
);
|
|
123
|
+
assertSelectorCandidates(input["candidate_pairs"], byCandidate);
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
candidate_actions: slots.map((slot) => ({ action: slot.action })),
|
|
127
|
+
imported_evals: anonymizeEvals(input["imported_evals"], byCandidate),
|
|
128
|
+
previous_action: anonymizePreviousDecision(input["previous_decision"], byCandidate),
|
|
129
|
+
cost_estimates: anonymizeCosts(input["cost_estimates"], byCandidate),
|
|
130
|
+
messages: jsonValue(input["messages"], "selector input messages"),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function buildAnonymousPolicyPrompt(
|
|
135
|
+
anonymousSelectorInput: AnonymousSelectorInput,
|
|
136
|
+
slots: readonly AnonymousActionSlot[],
|
|
137
|
+
): AnonymousPolicyPrompt {
|
|
138
|
+
return {
|
|
139
|
+
messages: [
|
|
140
|
+
{ role: "system", content: ANONYMOUS_ACTION_SYSTEM_PROMPT },
|
|
141
|
+
{ role: "user", content: formatAnonymousPolicyInput(anonymousSelectorInput) },
|
|
142
|
+
],
|
|
143
|
+
actions: slots.map((slot) => slot.action),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Renders the anonymous selector input as sectioned text rather than raw JSON.
|
|
148
|
+
// One block per action carries that action's benchmark standing and its
|
|
149
|
+
// projected cost, so the policy compares like against like instead of
|
|
150
|
+
// re-deriving comparisons from scores on incomparable scales and six cost
|
|
151
|
+
// fields. How to read those blocks lives in ANONYMOUS_ACTION_SYSTEM_PROMPT,
|
|
152
|
+
// keeping this message pure data. The conversation goes last: it is the only
|
|
153
|
+
// section that grows turn over turn, so keeping it at the end preserves the
|
|
154
|
+
// shared prefix.
|
|
155
|
+
export function formatAnonymousPolicyInput(input: AnonymousSelectorInput): string {
|
|
156
|
+
const sections = [
|
|
157
|
+
...(input.imported_evals.length > 0
|
|
158
|
+
? [section("benchmarks", input.imported_evals.map(glossaryLine).join("\n"))]
|
|
159
|
+
: []),
|
|
160
|
+
section(
|
|
161
|
+
"candidates",
|
|
162
|
+
input.candidate_actions
|
|
163
|
+
.map((candidate) => candidateBlock(candidate.action, input))
|
|
164
|
+
.join("\n\n"),
|
|
165
|
+
),
|
|
166
|
+
...(input.previous_action === null
|
|
167
|
+
? []
|
|
168
|
+
: [section("previous_action", input.previous_action.action)]),
|
|
169
|
+
section("conversation", JSON.stringify(input.messages)),
|
|
170
|
+
];
|
|
171
|
+
return sections.join("\n\n");
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function section(name: string, body: string): string {
|
|
175
|
+
return `<${name}>\n${body}\n</${name}>`;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function candidateBlock(action: string, input: AnonymousSelectorInput): string {
|
|
179
|
+
const cost = input.cost_estimates.find((entry) => entry.action === action);
|
|
180
|
+
const lines = [`## Action ${action}`];
|
|
181
|
+
if (cost !== undefined) lines.push(costLine(cost));
|
|
182
|
+
for (const card of input.imported_evals) {
|
|
183
|
+
const score = card.scores.find((entry) => entry.action === action);
|
|
184
|
+
if (score !== undefined) {
|
|
185
|
+
lines.push(
|
|
186
|
+
`- ${card.name}: Rank ${score.rank}/${score.rank_total}, Z ${signed(score.z_score)}`,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return lines.join("\n");
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// One line, one unit, the same horizons for every action.
|
|
194
|
+
function costLine(cost: AnonymousActionCost): string {
|
|
195
|
+
const horizons = cost.projections.map(
|
|
196
|
+
(projection) =>
|
|
197
|
+
`${projection.projected_turns} turn${projection.projected_turns === 1 ? "" : "s"} ${usd(projection.total_cost_usd)}`,
|
|
198
|
+
);
|
|
199
|
+
return `- Cost: ${horizons.join(", ")}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function signed(value: number): string {
|
|
203
|
+
return value >= 0 ? `+${value.toFixed(2)}` : value.toFixed(2);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function usd(value: number): string {
|
|
207
|
+
if (value >= 1) return `$${value.toFixed(2)}`;
|
|
208
|
+
const trimmed = value.toPrecision(3).replace(/0+$/, "").replace(/\.$/, "");
|
|
209
|
+
return `$${trimmed}`;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function glossaryLine(card: AnonymousEvalCard): string {
|
|
213
|
+
return card.description === null ? `- ${card.name}` : `- ${card.name}: ${card.description}`;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function previousAction(value: unknown): string | null {
|
|
217
|
+
if (value === null || value === undefined) return null;
|
|
218
|
+
return requiredString(
|
|
219
|
+
jsonRecord(value, "anonymous previous_action")["action"],
|
|
220
|
+
"anonymous previous_action.action",
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function isRecordValue(value: unknown): value is Record<string, JsonValue> {
|
|
225
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Parses a policy completion against the permutation's slots. Without a lease
|
|
229
|
+
// menu the exact shape is {"action":"B"}; with one it is
|
|
230
|
+
// {"action":"B","turns":10} and turns must be on the menu.
|
|
231
|
+
export function parseAnonymousActionSelection(
|
|
232
|
+
completion: string,
|
|
233
|
+
slots: readonly AnonymousActionSlot[],
|
|
234
|
+
leaseTurns?: readonly number[],
|
|
235
|
+
): AnonymousActionSelection {
|
|
236
|
+
let parsed: unknown;
|
|
237
|
+
try {
|
|
238
|
+
parsed = JSON.parse(completion);
|
|
239
|
+
} catch {
|
|
240
|
+
throw new Error("Anonymous action completion is not valid JSON");
|
|
241
|
+
}
|
|
242
|
+
const record = jsonRecord(parsed, "anonymous action completion");
|
|
243
|
+
const keys = Object.keys(record);
|
|
244
|
+
// An empty menu means the caller routes one turn at a time, same as no menu.
|
|
245
|
+
const offeredLeaseTurns =
|
|
246
|
+
leaseTurns !== undefined && leaseTurns.length > 0 ? leaseTurns : undefined;
|
|
247
|
+
const expected = offeredLeaseTurns === undefined ? ["action"] : ["action", "turns"];
|
|
248
|
+
if (keys.length !== expected.length || expected.some((key, i) => keys[i] !== key)) {
|
|
249
|
+
throw new Error(
|
|
250
|
+
`Anonymous action completion must contain exactly {${expected
|
|
251
|
+
.map((key) => `"${key}":...`)
|
|
252
|
+
.join(",")}}`,
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
const action = record["action"];
|
|
256
|
+
if (typeof action !== "string") {
|
|
257
|
+
throw new Error("Anonymous action completion action must be a string");
|
|
258
|
+
}
|
|
259
|
+
const slot = slots.find((candidate) => candidate.action === action);
|
|
260
|
+
if (slot === undefined) {
|
|
261
|
+
throw new Error(`Anonymous action completion selected unavailable action ${action}`);
|
|
262
|
+
}
|
|
263
|
+
if (offeredLeaseTurns === undefined) return { action, candidate: slot.candidate };
|
|
264
|
+
const turns = record["turns"];
|
|
265
|
+
if (typeof turns !== "number" || !offeredLeaseTurns.includes(turns)) {
|
|
266
|
+
throw new Error(
|
|
267
|
+
`Anonymous action completion turns must be one of ${offeredLeaseTurns.join(", ")}`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
return { action, candidate: slot.candidate, turns };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Fisher-Yates copy, drawing from the end, matching d3-array's shuffler so an
|
|
274
|
+
// injected rng reproduces permutations recorded by harnesses that used d3.
|
|
275
|
+
function shuffled<T>(items: readonly T[], rng: Rng): T[] {
|
|
276
|
+
const array = [...items];
|
|
277
|
+
let m = array.length;
|
|
278
|
+
while (m) {
|
|
279
|
+
const i = (rng() * m--) | 0;
|
|
280
|
+
const t = array[m]!;
|
|
281
|
+
array[m] = array[i]!;
|
|
282
|
+
array[i] = t;
|
|
283
|
+
}
|
|
284
|
+
return array;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function assertSelectorCandidates(
|
|
288
|
+
value: unknown,
|
|
289
|
+
byCandidate: ReadonlyMap<string, string>,
|
|
290
|
+
): void {
|
|
291
|
+
if (!Array.isArray(value)) {
|
|
292
|
+
throw new Error("Selector input candidate_pairs must be an array");
|
|
293
|
+
}
|
|
294
|
+
const selectorKeys = new Set(
|
|
295
|
+
value.map((candidate, index) => {
|
|
296
|
+
const record = jsonRecord(candidate, `selector candidate[${index}]`);
|
|
297
|
+
return candidateKey(
|
|
298
|
+
requiredString(record["model"], `selector candidate[${index}].model`),
|
|
299
|
+
requiredString(
|
|
300
|
+
record["thinking_level"],
|
|
301
|
+
`selector candidate[${index}].thinking_level`,
|
|
302
|
+
),
|
|
303
|
+
);
|
|
304
|
+
}),
|
|
305
|
+
);
|
|
306
|
+
if (selectorKeys.size !== byCandidate.size || [...byCandidate.keys()].some((key) => !selectorKeys.has(key))) {
|
|
307
|
+
throw new Error("Routing and selector candidates do not match");
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function anonymizeEvals(
|
|
312
|
+
value: unknown,
|
|
313
|
+
byCandidate: ReadonlyMap<string, string>,
|
|
314
|
+
): AnonymousEvalCard[] {
|
|
315
|
+
if (!Array.isArray(value)) {
|
|
316
|
+
throw new Error("Selector input imported_evals must be an array");
|
|
317
|
+
}
|
|
318
|
+
return value.map((entry, evalIndex) => {
|
|
319
|
+
const card = jsonRecord(entry, `selector eval[${evalIndex}]`);
|
|
320
|
+
const scores = card["scores"];
|
|
321
|
+
if (!Array.isArray(scores)) {
|
|
322
|
+
throw new Error(`selector eval[${evalIndex}].scores must be an array`);
|
|
323
|
+
}
|
|
324
|
+
return {
|
|
325
|
+
name: requiredString(card["name"], `selector eval[${evalIndex}].name`),
|
|
326
|
+
description: typeof card["description"] === "string" ? card["description"] : null,
|
|
327
|
+
// Explicit allowlist: notes and any future field can name the model.
|
|
328
|
+
scores: scores.map((entryScore, scoreIndex) => {
|
|
329
|
+
const label = `selector eval[${evalIndex}].scores[${scoreIndex}]`;
|
|
330
|
+
const score = jsonRecord(entryScore, label);
|
|
331
|
+
return {
|
|
332
|
+
action: actionForCandidate(
|
|
333
|
+
byCandidate,
|
|
334
|
+
requiredString(score["model_id"], `${label}.model_id`),
|
|
335
|
+
requiredString(score["thinking_level"], `${label}.thinking_level`),
|
|
336
|
+
),
|
|
337
|
+
rank: requiredNumber(score["rank"], `${label}.rank`),
|
|
338
|
+
rank_total: requiredNumber(score["rank_total"], `${label}.rank_total`),
|
|
339
|
+
z_score: requiredNumber(score["z_score"], `${label}.z_score`),
|
|
340
|
+
};
|
|
341
|
+
}),
|
|
342
|
+
};
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function anonymizePreviousDecision(
|
|
347
|
+
value: unknown,
|
|
348
|
+
byCandidate: ReadonlyMap<string, string>,
|
|
349
|
+
): { action: string } | null {
|
|
350
|
+
if (value === null || value === undefined) return null;
|
|
351
|
+
const previous = jsonRecord(value, "selector previous_decision");
|
|
352
|
+
const model = requiredString(previous["model"], "selector previous_decision.model");
|
|
353
|
+
const thinkingLevel = requiredString(
|
|
354
|
+
previous["thinking_level"],
|
|
355
|
+
"selector previous_decision.thinking_level",
|
|
356
|
+
);
|
|
357
|
+
return {
|
|
358
|
+
action: actionForCandidate(byCandidate, model, thinkingLevel),
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function anonymizeCosts(
|
|
363
|
+
value: unknown,
|
|
364
|
+
byCandidate: ReadonlyMap<string, string>,
|
|
365
|
+
): AnonymousActionCost[] {
|
|
366
|
+
if (value === null || value === undefined) return [];
|
|
367
|
+
if (!Array.isArray(value)) {
|
|
368
|
+
throw new Error("Selector input cost_estimates must be an array or null");
|
|
369
|
+
}
|
|
370
|
+
// Only the loop projections survive. The per-request fields the prompt does
|
|
371
|
+
// not show would be dead weight, and assumed_reasoning_effort names the
|
|
372
|
+
// candidate's thinking level, which the protocol exists to hide.
|
|
373
|
+
return value.flatMap((entry, index) => {
|
|
374
|
+
const label = `selector cost_estimates[${index}]`;
|
|
375
|
+
const cost = jsonRecord(entry, label);
|
|
376
|
+
const action = actionForCandidate(
|
|
377
|
+
byCandidate,
|
|
378
|
+
requiredString(cost["model"], `${label}.model`),
|
|
379
|
+
requiredString(cost["reasoning_effort"], `${label}.reasoning_effort`),
|
|
380
|
+
);
|
|
381
|
+
const fixedTurn = cost["fixed_turn_cost_estimate"];
|
|
382
|
+
if (fixedTurn === null || fixedTurn === undefined) return [];
|
|
383
|
+
if (!isRecordValue(fixedTurn) || !Array.isArray(fixedTurn["projections"])) {
|
|
384
|
+
// A present estimate without projections is the pre-projection schema.
|
|
385
|
+
// Reject it loudly: silently dropping it would show "Cost: unavailable"
|
|
386
|
+
// for a candidate whose cost the caller thinks it supplied.
|
|
387
|
+
throw new Error(
|
|
388
|
+
`${label}.fixed_turn_cost_estimate must carry a projections array`,
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
return [
|
|
392
|
+
{
|
|
393
|
+
action,
|
|
394
|
+
projections: fixedTurn["projections"].map((projection, projectionIndex) => {
|
|
395
|
+
const projectionLabel = `${label}.fixed_turn_cost_estimate.projections[${projectionIndex}]`;
|
|
396
|
+
const record = jsonRecord(projection, projectionLabel);
|
|
397
|
+
return {
|
|
398
|
+
projected_turns: requiredNumber(
|
|
399
|
+
record["projected_turns"],
|
|
400
|
+
`${projectionLabel}.projected_turns`,
|
|
401
|
+
),
|
|
402
|
+
total_cost_usd: requiredNumber(
|
|
403
|
+
record["total_cost_usd"],
|
|
404
|
+
`${projectionLabel}.total_cost_usd`,
|
|
405
|
+
),
|
|
406
|
+
};
|
|
407
|
+
}),
|
|
408
|
+
},
|
|
409
|
+
];
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function requiredNumber(value: JsonValue | undefined, label: string): number {
|
|
414
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
415
|
+
throw new Error(`${label} must be a finite number`);
|
|
416
|
+
}
|
|
417
|
+
return value;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function actionForCandidate(
|
|
421
|
+
byCandidate: ReadonlyMap<string, string>,
|
|
422
|
+
model: string,
|
|
423
|
+
thinkingLevel: string,
|
|
424
|
+
): string {
|
|
425
|
+
const action = byCandidate.get(candidateKey(model, thinkingLevel));
|
|
426
|
+
if (action === undefined) {
|
|
427
|
+
throw new Error(`Selector field references unavailable candidate ${model}/${thinkingLevel}`);
|
|
428
|
+
}
|
|
429
|
+
return action;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function candidateKey(model: string, thinkingLevel: string): string {
|
|
433
|
+
return `${model}\u0000${thinkingLevel}`;
|
|
434
|
+
}
|