@x12i/ai-profiles 4.3.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -32
- package/dist/buildCatalogFromOpenRouterApi.d.ts +3 -1
- package/dist/buildCatalogFromOpenRouterApi.d.ts.map +1 -1
- package/dist/buildCatalogFromOpenRouterApi.js +11 -69
- package/dist/buildCatalogFromOpenRouterApi.js.map +1 -1
- package/dist/constants.d.ts +5 -3
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +15 -2
- package/dist/constants.js.map +1 -1
- package/dist/data/models-catalog.json +5036 -2055
- package/dist/data/models-profiles.json +3442 -866
- package/dist/data/models-registry.json +4594 -2772
- package/dist/ensureRemoteOpenRouterBackend.d.ts +4 -0
- package/dist/ensureRemoteOpenRouterBackend.d.ts.map +1 -0
- package/dist/ensureRemoteOpenRouterBackend.js +33 -0
- package/dist/ensureRemoteOpenRouterBackend.js.map +1 -0
- package/dist/evidence/adapterTypes.d.ts +14 -0
- package/dist/evidence/adapterTypes.d.ts.map +1 -0
- package/dist/evidence/adapterTypes.js +2 -0
- package/dist/evidence/adapterTypes.js.map +1 -0
- package/dist/evidence/adapters.d.ts +28 -0
- package/dist/evidence/adapters.d.ts.map +1 -0
- package/dist/evidence/adapters.js +102 -0
- package/dist/evidence/adapters.js.map +1 -0
- package/dist/evidence/index.d.ts +13 -0
- package/dist/evidence/index.d.ts.map +1 -0
- package/dist/evidence/index.js +9 -0
- package/dist/evidence/index.js.map +1 -0
- package/dist/evidence/local/probes.d.ts +58 -0
- package/dist/evidence/local/probes.d.ts.map +1 -0
- package/dist/evidence/local/probes.js +210 -0
- package/dist/evidence/local/probes.js.map +1 -0
- package/dist/evidence/openRouter.d.ts +11 -0
- package/dist/evidence/openRouter.d.ts.map +1 -0
- package/dist/evidence/openRouter.js +80 -0
- package/dist/evidence/openRouter.js.map +1 -0
- package/dist/evidence/patch.d.ts +8 -0
- package/dist/evidence/patch.d.ts.map +1 -0
- package/dist/evidence/patch.js +50 -0
- package/dist/evidence/patch.js.map +1 -0
- package/dist/evidence/recommend.d.ts +20 -0
- package/dist/evidence/recommend.d.ts.map +1 -0
- package/dist/evidence/recommend.js +142 -0
- package/dist/evidence/recommend.js.map +1 -0
- package/dist/evidence/refreshModelEvidence.d.ts +25 -0
- package/dist/evidence/refreshModelEvidence.d.ts.map +1 -0
- package/dist/evidence/refreshModelEvidence.js +104 -0
- package/dist/evidence/refreshModelEvidence.js.map +1 -0
- package/dist/evidence/scoreModelForTask.d.ts +13 -0
- package/dist/evidence/scoreModelForTask.d.ts.map +1 -0
- package/dist/evidence/scoreModelForTask.js +137 -0
- package/dist/evidence/scoreModelForTask.js.map +1 -0
- package/dist/evidence/scoringConfig.d.ts +11 -0
- package/dist/evidence/scoringConfig.d.ts.map +1 -0
- package/dist/evidence/scoringConfig.js +259 -0
- package/dist/evidence/scoringConfig.js.map +1 -0
- package/dist/evidence/types.d.ts +218 -0
- package/dist/evidence/types.d.ts.map +1 -0
- package/dist/evidence/types.js +2 -0
- package/dist/evidence/types.js.map +1 -0
- package/dist/finalProfileModelMappingPatch.d.ts +31 -0
- package/dist/finalProfileModelMappingPatch.d.ts.map +1 -0
- package/dist/finalProfileModelMappingPatch.js +405 -0
- package/dist/finalProfileModelMappingPatch.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/dist/inferModelFamily.d.ts +8 -0
- package/dist/inferModelFamily.d.ts.map +1 -0
- package/dist/inferModelFamily.js +242 -0
- package/dist/inferModelFamily.js.map +1 -0
- package/dist/modelIdentity.d.ts.map +1 -1
- package/dist/modelIdentity.js +2 -1
- package/dist/modelIdentity.js.map +1 -1
- package/dist/modelsCatalog.d.ts +2 -2
- package/dist/modelsCatalog.d.ts.map +1 -1
- package/dist/modelsCatalog.js +36 -34
- package/dist/modelsCatalog.js.map +1 -1
- package/dist/openRouterProviderMap.d.ts +57 -0
- package/dist/openRouterProviderMap.d.ts.map +1 -0
- package/dist/openRouterProviderMap.js +120 -0
- package/dist/openRouterProviderMap.js.map +1 -0
- package/dist/profileRegistryV3.d.ts.map +1 -1
- package/dist/profileRegistryV3.js +30 -23
- package/dist/profileRegistryV3.js.map +1 -1
- package/dist/resolveInstructionTierAndBackend.d.ts.map +1 -1
- package/dist/resolveInstructionTierAndBackend.js +18 -5
- package/dist/resolveInstructionTierAndBackend.js.map +1 -1
- package/dist/resolveLocalRuntime.d.ts +22 -0
- package/dist/resolveLocalRuntime.d.ts.map +1 -0
- package/dist/resolveLocalRuntime.js +60 -0
- package/dist/resolveLocalRuntime.js.map +1 -0
- package/dist/resolveModelRouting.d.ts.map +1 -1
- package/dist/resolveModelRouting.js +8 -1
- package/dist/resolveModelRouting.js.map +1 -1
- package/dist/sourceCatalogProviders.d.ts +9 -0
- package/dist/sourceCatalogProviders.d.ts.map +1 -0
- package/dist/sourceCatalogProviders.js +51 -0
- package/dist/sourceCatalogProviders.js.map +1 -0
- package/dist/types.d.ts +6 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/validateFunctionalRouting.d.ts +14 -0
- package/dist/validateFunctionalRouting.d.ts.map +1 -0
- package/dist/validateFunctionalRouting.js +178 -0
- package/dist/validateFunctionalRouting.js.map +1 -0
- package/dist/validateProfileTransportPolicy.d.ts.map +1 -1
- package/dist/validateProfileTransportPolicy.js +19 -5
- package/dist/validateProfileTransportPolicy.js.map +1 -1
- package/dist/validation/openRouterAlignment.d.ts +1 -1
- package/dist/validation/openRouterAlignment.d.ts.map +1 -1
- package/dist/validation/validateModelsProfilesJson.d.ts.map +1 -1
- package/dist/validation/validateModelsProfilesJson.js +6 -3
- package/dist/validation/validateModelsProfilesJson.js.map +1 -1
- package/dist/vendorModelId.d.ts +1 -1
- package/dist/vendorModelId.d.ts.map +1 -1
- package/dist/vendorModelId.js +4 -3
- package/dist/vendorModelId.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
export const defaultProfileTaskShapes = {
|
|
2
|
+
cheap: { inputTokens: 1_500, outputTokens: 250 },
|
|
3
|
+
routing: { inputTokens: 1_000, outputTokens: 80 },
|
|
4
|
+
extraction: { inputTokens: 4_000, outputTokens: 350 },
|
|
5
|
+
json: { inputTokens: 3_000, outputTokens: 500 },
|
|
6
|
+
fast: { inputTokens: 1_500, outputTokens: 400 },
|
|
7
|
+
balanced: { inputTokens: 4_000, outputTokens: 900 },
|
|
8
|
+
value: { inputTokens: 8_000, outputTokens: 1_200 },
|
|
9
|
+
deep: { inputTokens: 20_000, outputTokens: 2_000, reasoningTokens: 2_000 },
|
|
10
|
+
pro: { inputTokens: 25_000, outputTokens: 2_500, reasoningTokens: 5_000 },
|
|
11
|
+
agentic: {
|
|
12
|
+
inputTokens: 12_000,
|
|
13
|
+
outputTokens: 1_500,
|
|
14
|
+
reasoningTokens: 2_000,
|
|
15
|
+
requests: 5,
|
|
16
|
+
},
|
|
17
|
+
research: {
|
|
18
|
+
inputTokens: 60_000,
|
|
19
|
+
outputTokens: 2_500,
|
|
20
|
+
reasoningTokens: 3_000,
|
|
21
|
+
},
|
|
22
|
+
cyber: {
|
|
23
|
+
inputTokens: 40_000,
|
|
24
|
+
outputTokens: 1_500,
|
|
25
|
+
reasoningTokens: 2_000,
|
|
26
|
+
},
|
|
27
|
+
coding: {
|
|
28
|
+
inputTokens: 30_000,
|
|
29
|
+
outputTokens: 2_000,
|
|
30
|
+
reasoningTokens: 3_000,
|
|
31
|
+
},
|
|
32
|
+
content: { inputTokens: 2_500, outputTokens: 2_500 },
|
|
33
|
+
summarization: { inputTokens: 80_000, outputTokens: 1_200 },
|
|
34
|
+
long_context: {
|
|
35
|
+
inputTokens: 200_000,
|
|
36
|
+
outputTokens: 1_500,
|
|
37
|
+
reasoningTokens: 2_000,
|
|
38
|
+
},
|
|
39
|
+
local: { inputTokens: 2_000, outputTokens: 500 },
|
|
40
|
+
vol: { inputTokens: 2_000, outputTokens: 200 },
|
|
41
|
+
};
|
|
42
|
+
export const taskRequirements = {
|
|
43
|
+
cheap: { requiresTextOutput: true, maxCostUsd: 0.002 },
|
|
44
|
+
routing: { requiresStructuredOutputs: true, maxCostUsd: 0.001 },
|
|
45
|
+
extraction: { requiresStructuredOutputs: true, maxCostUsd: 0.004 },
|
|
46
|
+
json: { requiresStructuredOutputs: true },
|
|
47
|
+
fast: { requiresTextOutput: true, maxLatencyMs: 2_000 },
|
|
48
|
+
balanced: { requiresTextOutput: true },
|
|
49
|
+
value: { requiresTextOutput: true },
|
|
50
|
+
deep: { requiresReasoning: true, minContextTokens: 128_000 },
|
|
51
|
+
pro: {
|
|
52
|
+
requiresReasoning: true,
|
|
53
|
+
minContextTokens: 128_000,
|
|
54
|
+
minQualityScore: 0.85,
|
|
55
|
+
},
|
|
56
|
+
agentic: {
|
|
57
|
+
requiresTools: true,
|
|
58
|
+
requiresReasoning: true,
|
|
59
|
+
minContextTokens: 128_000,
|
|
60
|
+
},
|
|
61
|
+
research: { requiresReasoning: true, minContextTokens: 128_000 },
|
|
62
|
+
cyber: { requiresReasoning: true, minContextTokens: 128_000 },
|
|
63
|
+
coding: { requiresReasoning: true, minContextTokens: 128_000 },
|
|
64
|
+
content: { requiresTextOutput: true },
|
|
65
|
+
summarization: { minContextTokens: 128_000 },
|
|
66
|
+
long_context: { minContextTokens: 1_000_000 },
|
|
67
|
+
local: { requiresTextOutput: true },
|
|
68
|
+
vol: { requiresTextOutput: true, maxCostUsd: 0.002 },
|
|
69
|
+
};
|
|
70
|
+
export const scoreWeightsByTask = {
|
|
71
|
+
cheap: {
|
|
72
|
+
quality: 0.25,
|
|
73
|
+
cost: 0.45,
|
|
74
|
+
latency: 0.15,
|
|
75
|
+
reliability: 0.1,
|
|
76
|
+
context: 0.05,
|
|
77
|
+
benchmarkConfidence: 0.05,
|
|
78
|
+
},
|
|
79
|
+
routing: {
|
|
80
|
+
quality: 0.3,
|
|
81
|
+
cost: 0.25,
|
|
82
|
+
latency: 0.15,
|
|
83
|
+
reliability: 0.25,
|
|
84
|
+
context: 0.05,
|
|
85
|
+
benchmarkConfidence: 0.05,
|
|
86
|
+
},
|
|
87
|
+
extraction: {
|
|
88
|
+
quality: 0.35,
|
|
89
|
+
cost: 0.2,
|
|
90
|
+
latency: 0.1,
|
|
91
|
+
reliability: 0.3,
|
|
92
|
+
context: 0.05,
|
|
93
|
+
benchmarkConfidence: 0.05,
|
|
94
|
+
},
|
|
95
|
+
json: {
|
|
96
|
+
quality: 0.3,
|
|
97
|
+
cost: 0.15,
|
|
98
|
+
latency: 0.1,
|
|
99
|
+
reliability: 0.4,
|
|
100
|
+
context: 0.05,
|
|
101
|
+
benchmarkConfidence: 0.05,
|
|
102
|
+
},
|
|
103
|
+
fast: {
|
|
104
|
+
quality: 0.3,
|
|
105
|
+
cost: 0.15,
|
|
106
|
+
latency: 0.35,
|
|
107
|
+
reliability: 0.15,
|
|
108
|
+
context: 0.05,
|
|
109
|
+
benchmarkConfidence: 0.05,
|
|
110
|
+
},
|
|
111
|
+
balanced: {
|
|
112
|
+
quality: 0.45,
|
|
113
|
+
cost: 0.2,
|
|
114
|
+
latency: 0.1,
|
|
115
|
+
reliability: 0.15,
|
|
116
|
+
context: 0.1,
|
|
117
|
+
benchmarkConfidence: 0.05,
|
|
118
|
+
},
|
|
119
|
+
value: {
|
|
120
|
+
quality: 0.45,
|
|
121
|
+
cost: 0.3,
|
|
122
|
+
latency: 0.05,
|
|
123
|
+
reliability: 0.1,
|
|
124
|
+
context: 0.1,
|
|
125
|
+
benchmarkConfidence: 0.05,
|
|
126
|
+
},
|
|
127
|
+
deep: {
|
|
128
|
+
quality: 0.55,
|
|
129
|
+
cost: 0.1,
|
|
130
|
+
latency: 0.05,
|
|
131
|
+
reliability: 0.15,
|
|
132
|
+
context: 0.15,
|
|
133
|
+
benchmarkConfidence: 0.1,
|
|
134
|
+
},
|
|
135
|
+
pro: {
|
|
136
|
+
quality: 0.7,
|
|
137
|
+
cost: 0.05,
|
|
138
|
+
latency: 0.03,
|
|
139
|
+
reliability: 0.12,
|
|
140
|
+
context: 0.1,
|
|
141
|
+
benchmarkConfidence: 0.1,
|
|
142
|
+
},
|
|
143
|
+
agentic: {
|
|
144
|
+
quality: 0.45,
|
|
145
|
+
cost: 0.1,
|
|
146
|
+
latency: 0.1,
|
|
147
|
+
reliability: 0.2,
|
|
148
|
+
context: 0.15,
|
|
149
|
+
benchmarkConfidence: 0.1,
|
|
150
|
+
},
|
|
151
|
+
research: {
|
|
152
|
+
quality: 0.55,
|
|
153
|
+
cost: 0.1,
|
|
154
|
+
latency: 0.03,
|
|
155
|
+
reliability: 0.17,
|
|
156
|
+
context: 0.15,
|
|
157
|
+
benchmarkConfidence: 0.1,
|
|
158
|
+
},
|
|
159
|
+
cyber: {
|
|
160
|
+
quality: 0.55,
|
|
161
|
+
cost: 0.1,
|
|
162
|
+
latency: 0.03,
|
|
163
|
+
reliability: 0.17,
|
|
164
|
+
context: 0.15,
|
|
165
|
+
benchmarkConfidence: 0.1,
|
|
166
|
+
},
|
|
167
|
+
coding: {
|
|
168
|
+
quality: 0.55,
|
|
169
|
+
cost: 0.1,
|
|
170
|
+
latency: 0.05,
|
|
171
|
+
reliability: 0.15,
|
|
172
|
+
context: 0.15,
|
|
173
|
+
benchmarkConfidence: 0.1,
|
|
174
|
+
},
|
|
175
|
+
content: {
|
|
176
|
+
quality: 0.6,
|
|
177
|
+
cost: 0.15,
|
|
178
|
+
latency: 0.05,
|
|
179
|
+
reliability: 0.1,
|
|
180
|
+
context: 0.1,
|
|
181
|
+
benchmarkConfidence: 0.05,
|
|
182
|
+
},
|
|
183
|
+
summarization: {
|
|
184
|
+
quality: 0.4,
|
|
185
|
+
cost: 0.2,
|
|
186
|
+
latency: 0.05,
|
|
187
|
+
reliability: 0.1,
|
|
188
|
+
context: 0.25,
|
|
189
|
+
benchmarkConfidence: 0.05,
|
|
190
|
+
},
|
|
191
|
+
long_context: {
|
|
192
|
+
quality: 0.35,
|
|
193
|
+
cost: 0.15,
|
|
194
|
+
latency: 0.03,
|
|
195
|
+
reliability: 0.12,
|
|
196
|
+
context: 0.35,
|
|
197
|
+
benchmarkConfidence: 0.05,
|
|
198
|
+
},
|
|
199
|
+
local: {
|
|
200
|
+
quality: 0.3,
|
|
201
|
+
cost: 0.3,
|
|
202
|
+
latency: 0.1,
|
|
203
|
+
reliability: 0.2,
|
|
204
|
+
context: 0.1,
|
|
205
|
+
benchmarkConfidence: 0.05,
|
|
206
|
+
},
|
|
207
|
+
vol: {
|
|
208
|
+
quality: 0.25,
|
|
209
|
+
cost: 0.45,
|
|
210
|
+
latency: 0.15,
|
|
211
|
+
reliability: 0.1,
|
|
212
|
+
context: 0.05,
|
|
213
|
+
benchmarkConfidence: 0.05,
|
|
214
|
+
},
|
|
215
|
+
};
|
|
216
|
+
export function estimateTaskCostUsd(pricing, task) {
|
|
217
|
+
const input = ((pricing.inputUsdPer1M ?? 0) * task.inputTokens) / 1_000_000;
|
|
218
|
+
const output = ((pricing.outputUsdPer1M ?? 0) * task.outputTokens) / 1_000_000;
|
|
219
|
+
const reasoning = ((pricing.reasoningUsdPer1M ?? 0) * (task.reasoningTokens ?? 0)) / 1_000_000;
|
|
220
|
+
const cacheRead = ((pricing.cacheReadUsdPer1M ?? 0) * (task.cacheReadTokens ?? 0)) / 1_000_000;
|
|
221
|
+
const cacheWrite = ((pricing.cacheWriteUsdPer1M ?? 0) * (task.cacheWriteTokens ?? 0)) /
|
|
222
|
+
1_000_000;
|
|
223
|
+
const request = (pricing.requestUsd ?? 0) * (task.requests ?? 1);
|
|
224
|
+
const web = (pricing.webSearchUsd ?? 0) * (task.webSearches ?? 0);
|
|
225
|
+
return input + output + reasoning + cacheRead + cacheWrite + request + web;
|
|
226
|
+
}
|
|
227
|
+
export function clamp01(value) {
|
|
228
|
+
return Math.max(0, Math.min(1, value));
|
|
229
|
+
}
|
|
230
|
+
export function inverseCostScore(costUsd, maxAcceptableCostUsd) {
|
|
231
|
+
if (costUsd <= 0)
|
|
232
|
+
return 1;
|
|
233
|
+
return clamp01(1 - costUsd / maxAcceptableCostUsd);
|
|
234
|
+
}
|
|
235
|
+
export function contextScore(contextLength, required) {
|
|
236
|
+
if (!required)
|
|
237
|
+
return 1;
|
|
238
|
+
if (!contextLength)
|
|
239
|
+
return 0;
|
|
240
|
+
return clamp01(contextLength / required);
|
|
241
|
+
}
|
|
242
|
+
export function issuePenalty(issues) {
|
|
243
|
+
if (issues.length === 0)
|
|
244
|
+
return 0;
|
|
245
|
+
const severityWeight = {
|
|
246
|
+
low: 0.05,
|
|
247
|
+
medium: 0.15,
|
|
248
|
+
high: 0.35,
|
|
249
|
+
critical: 0.75,
|
|
250
|
+
};
|
|
251
|
+
const total = issues.reduce((sum, issue) => sum + issue.score * severityWeight[issue.severity], 0);
|
|
252
|
+
return clamp01(total);
|
|
253
|
+
}
|
|
254
|
+
export function average(values) {
|
|
255
|
+
if (values.length === 0)
|
|
256
|
+
return 0.5;
|
|
257
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
258
|
+
}
|
|
259
|
+
//# sourceMappingURL=scoringConfig.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scoringConfig.js","sourceRoot":"","sources":["../../src/evidence/scoringConfig.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,wBAAwB,GAA4C;IAC/E,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;IAChD,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,EAAE;IACjD,UAAU,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;IACrD,IAAI,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;IAC/C,IAAI,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;IAC/C,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;IACnD,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE;IAClD,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE;IAC1E,GAAG,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE;IACzE,OAAO,EAAE;QACP,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,KAAK;QACnB,eAAe,EAAE,KAAK;QACtB,QAAQ,EAAE,CAAC;KACZ;IACD,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,KAAK;QACnB,eAAe,EAAE,KAAK;KACvB;IACD,KAAK,EAAE;QACL,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,KAAK;QACnB,eAAe,EAAE,KAAK;KACvB;IACD,MAAM,EAAE;QACN,WAAW,EAAE,MAAM;QACnB,YAAY,EAAE,KAAK;QACnB,eAAe,EAAE,KAAK;KACvB;IACD,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE;IACpD,aAAa,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE;IAC3D,YAAY,EAAE;QACZ,WAAW,EAAE,OAAO;QACpB,YAAY,EAAE,KAAK;QACnB,eAAe,EAAE,KAAK;KACvB;IACD,KAAK,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;IAChD,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE;CAC/C,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAA0C;IACrE,KAAK,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE;IACtD,OAAO,EAAE,EAAE,yBAAyB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE;IAC/D,UAAU,EAAE,EAAE,yBAAyB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE;IAClE,IAAI,EAAE,EAAE,yBAAyB,EAAE,IAAI,EAAE;IACzC,IAAI,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE;IACvD,QAAQ,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE;IACtC,KAAK,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE;IACnC,IAAI,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE;IAC5D,GAAG,EAAE;QACH,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,OAAO;QACzB,eAAe,EAAE,IAAI;KACtB;IACD,OAAO,EAAE;QACP,aAAa,EAAE,IAAI;QACnB,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,OAAO;KAC1B;IACD,QAAQ,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE;IAChE,KAAK,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE;IAC7D,MAAM,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE;IAC9D,OAAO,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE;IACrC,aAAa,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE;IAC5C,YAAY,EAAE,EAAE,gBAAgB,EAAE,SAAS,EAAE;IAC7C,KAAK,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE;IACnC,GAAG,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE;CACrD,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAsC;IACnE,KAAK,EAAE;QACL,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,OAAO,EAAE;QACP,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,UAAU,EAAE;QACV,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,IAAI,EAAE;QACJ,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,IAAI,EAAE;QACJ,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,QAAQ,EAAE;QACR,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,GAAG;QACZ,mBAAmB,EAAE,IAAI;KAC1B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,GAAG;QACZ,mBAAmB,EAAE,IAAI;KAC1B;IACD,IAAI,EAAE;QACJ,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,GAAG;KACzB;IACD,GAAG,EAAE;QACH,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,GAAG;QACZ,mBAAmB,EAAE,GAAG;KACzB;IACD,OAAO,EAAE;QACP,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,GAAG;KACzB;IACD,QAAQ,EAAE;QACR,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,GAAG;KACzB;IACD,KAAK,EAAE;QACL,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,GAAG;KACzB;IACD,MAAM,EAAE;QACN,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,GAAG;KACzB;IACD,OAAO,EAAE;QACP,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,GAAG;QACZ,mBAAmB,EAAE,IAAI;KAC1B;IACD,aAAa,EAAE;QACb,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,YAAY,EAAE;QACZ,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,GAAG;QACZ,IAAI,EAAE,GAAG;QACT,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,GAAG;QACZ,mBAAmB,EAAE,IAAI;KAC1B;IACD,GAAG,EAAE;QACH,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,GAAG;QAChB,OAAO,EAAE,IAAI;QACb,mBAAmB,EAAE,IAAI;KAC1B;CACF,CAAC;AAEF,MAAM,UAAU,mBAAmB,CACjC,OAAqB,EACrB,IAAwB;IAExB,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,SAAS,CAAC;IAC5E,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC;IAC/E,MAAM,SAAS,GACb,CAAC,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;IAC/E,MAAM,SAAS,GACb,CAAC,CAAC,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;IAC/E,MAAM,UAAU,GACd,CAAC,CAAC,OAAO,CAAC,kBAAkB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;QAClE,SAAS,CAAC;IACZ,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAClE,OAAO,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,GAAG,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAa;IACnC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,oBAA4B;IAE5B,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC,CAAC,GAAG,OAAO,GAAG,oBAAoB,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,aAAiC,EACjC,QAA4B;IAE5B,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,CAAC,aAAa;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,aAAa,GAAG,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAA0B;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,cAAc,GAAG;QACrB,GAAG,EAAE,IAAI;QACT,MAAM,EAAE,IAAI;QACZ,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,IAAI;KACN,CAAC;IACX,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CACzB,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,EAClE,CAAC,CACF,CAAC;IACF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,MAAgB;IACtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACpC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import type { AIProvider, ProviderId } from "../types.js";
|
|
2
|
+
export type BenchmarkSourceId = "openrouter" | "artificial_analysis" | "huggingface_dataset" | "lmarena" | "swe_bench" | "bfcl" | "terminal_bench" | "simpleqa" | "internal_eval";
|
|
3
|
+
export type TaskProfile = "cheap" | "routing" | "extraction" | "json" | "fast" | "balanced" | "value" | "deep" | "pro" | "agentic" | "research" | "cyber" | "coding" | "content" | "summarization" | "long_context" | "local" | "vol";
|
|
4
|
+
export type ProfileChoiceKey = `${string}/${string}`;
|
|
5
|
+
export type ModelIdentity = {
|
|
6
|
+
provider: ProviderId;
|
|
7
|
+
modelId: string;
|
|
8
|
+
openRouterId?: string;
|
|
9
|
+
canonicalSlug?: string;
|
|
10
|
+
displayName?: string;
|
|
11
|
+
};
|
|
12
|
+
export type OpenRouterModel = {
|
|
13
|
+
id: string;
|
|
14
|
+
canonical_slug?: string;
|
|
15
|
+
name: string;
|
|
16
|
+
created?: number;
|
|
17
|
+
description?: string;
|
|
18
|
+
context_length?: number;
|
|
19
|
+
architecture?: {
|
|
20
|
+
input_modalities?: string[];
|
|
21
|
+
output_modalities?: string[];
|
|
22
|
+
tokenizer?: string;
|
|
23
|
+
instruct_type?: string | null;
|
|
24
|
+
};
|
|
25
|
+
pricing?: {
|
|
26
|
+
prompt?: string;
|
|
27
|
+
completion?: string;
|
|
28
|
+
request?: string;
|
|
29
|
+
image?: string;
|
|
30
|
+
web_search?: string;
|
|
31
|
+
internal_reasoning?: string;
|
|
32
|
+
input_cache_read?: string;
|
|
33
|
+
input_cache_write?: string;
|
|
34
|
+
};
|
|
35
|
+
top_provider?: {
|
|
36
|
+
context_length?: number;
|
|
37
|
+
max_completion_tokens?: number;
|
|
38
|
+
is_moderated?: boolean;
|
|
39
|
+
};
|
|
40
|
+
supported_parameters?: string[];
|
|
41
|
+
default_parameters?: Record<string, unknown> | null;
|
|
42
|
+
expiration_date?: string | null;
|
|
43
|
+
benchmarks?: Record<string, unknown>;
|
|
44
|
+
};
|
|
45
|
+
export type OpenRouterModelResponse = {
|
|
46
|
+
data: OpenRouterModel[];
|
|
47
|
+
};
|
|
48
|
+
export type ModelCapabilities = {
|
|
49
|
+
inputModalities: string[];
|
|
50
|
+
outputModalities: string[];
|
|
51
|
+
contextLength?: number;
|
|
52
|
+
maxOutputTokens?: number;
|
|
53
|
+
supportsTools: boolean;
|
|
54
|
+
supportsToolChoice: boolean;
|
|
55
|
+
supportsStructuredOutputs: boolean;
|
|
56
|
+
supportsResponseFormat: boolean;
|
|
57
|
+
supportsReasoning: boolean;
|
|
58
|
+
supportsIncludeReasoning: boolean;
|
|
59
|
+
supportsTemperature: boolean;
|
|
60
|
+
supportsSeed: boolean;
|
|
61
|
+
isDeprecated: boolean;
|
|
62
|
+
deprecationDate?: string;
|
|
63
|
+
};
|
|
64
|
+
export type ModelPricing = {
|
|
65
|
+
inputUsdPer1M?: number;
|
|
66
|
+
outputUsdPer1M?: number;
|
|
67
|
+
requestUsd?: number;
|
|
68
|
+
imageUsd?: number;
|
|
69
|
+
webSearchUsd?: number;
|
|
70
|
+
reasoningUsdPer1M?: number;
|
|
71
|
+
cacheReadUsdPer1M?: number;
|
|
72
|
+
cacheWriteUsdPer1M?: number;
|
|
73
|
+
};
|
|
74
|
+
export type NormalizedModel = {
|
|
75
|
+
identity: ModelIdentity;
|
|
76
|
+
source: "openrouter" | "local" | "direct";
|
|
77
|
+
capabilities: ModelCapabilities;
|
|
78
|
+
pricing: ModelPricing;
|
|
79
|
+
raw?: {
|
|
80
|
+
openRouter?: OpenRouterModel;
|
|
81
|
+
};
|
|
82
|
+
fetchedAt: string;
|
|
83
|
+
};
|
|
84
|
+
export type BenchmarkMetricName = "intelligence" | "coding" | "agentic" | "functionCalling" | "factuality" | "nonHallucination" | "longContext" | "mathReasoning" | "scienceReasoning" | "instructionFollowing" | "writingPreference" | "jsonReliability" | "terminalUse" | "securityReasoning" | "costPerTask" | "outputTokensPerTask" | "tokensPerSecond" | "timeToFirstTokenMs" | "latencyP50Ms" | "latencyP95Ms";
|
|
85
|
+
export type BenchmarkSignal = {
|
|
86
|
+
source: BenchmarkSourceId;
|
|
87
|
+
sourceUrl?: string;
|
|
88
|
+
provider?: ProviderId;
|
|
89
|
+
modelId?: string;
|
|
90
|
+
openRouterId?: string;
|
|
91
|
+
displayName?: string;
|
|
92
|
+
metric: BenchmarkMetricName;
|
|
93
|
+
value: number;
|
|
94
|
+
normalizedValue?: number;
|
|
95
|
+
higherIsBetter: boolean;
|
|
96
|
+
taskProfiles: TaskProfile[];
|
|
97
|
+
fetchedAt: string;
|
|
98
|
+
publishedAt?: string;
|
|
99
|
+
confidence: number;
|
|
100
|
+
notes?: string;
|
|
101
|
+
};
|
|
102
|
+
export type ModelIssueSignalName = "hallucinationRisk" | "overRefusalRisk" | "jsonBreakageRisk" | "toolCallFailureRisk" | "longContextDegradationRisk" | "highOutputCostRisk" | "latencyRisk" | "providerStabilityRisk" | "deprecationRisk" | "moderationRisk" | "benchmarkContaminationRisk" | "licenseRisk" | "localInstallRisk";
|
|
103
|
+
export type ModelIssueSignal = {
|
|
104
|
+
source: BenchmarkSourceId | "internal_eval";
|
|
105
|
+
issue: ModelIssueSignalName;
|
|
106
|
+
provider?: ProviderId;
|
|
107
|
+
modelId?: string;
|
|
108
|
+
openRouterId?: string;
|
|
109
|
+
severity: "low" | "medium" | "high" | "critical";
|
|
110
|
+
score: number;
|
|
111
|
+
evidence: string;
|
|
112
|
+
sourceUrl?: string;
|
|
113
|
+
fetchedAt: string;
|
|
114
|
+
};
|
|
115
|
+
export type InternalEvalTaskId = "json_extraction" | "classification" | "routing_decision" | "cyber_analysis" | "long_input_summary" | "tool_calling" | "coding_patch" | "content_generation" | "final_executive_answer";
|
|
116
|
+
export type InternalEvalResult = {
|
|
117
|
+
provider: ProviderId;
|
|
118
|
+
modelId: string;
|
|
119
|
+
openRouterId?: string;
|
|
120
|
+
taskId: InternalEvalTaskId;
|
|
121
|
+
passRate?: number;
|
|
122
|
+
qualityScore?: number;
|
|
123
|
+
jsonValidRate?: number;
|
|
124
|
+
schemaValidRate?: number;
|
|
125
|
+
toolCallSuccessRate?: number;
|
|
126
|
+
hallucinationRate?: number;
|
|
127
|
+
retryRate?: number;
|
|
128
|
+
avgInputTokens?: number;
|
|
129
|
+
avgOutputTokens?: number;
|
|
130
|
+
avgReasoningTokens?: number;
|
|
131
|
+
avgLatencyMs?: number;
|
|
132
|
+
p95LatencyMs?: number;
|
|
133
|
+
avgCostUsd?: number;
|
|
134
|
+
sampleSize: number;
|
|
135
|
+
evaluatedAt: string;
|
|
136
|
+
};
|
|
137
|
+
export type EstimatedTaskShape = {
|
|
138
|
+
inputTokens: number;
|
|
139
|
+
outputTokens: number;
|
|
140
|
+
reasoningTokens?: number;
|
|
141
|
+
cacheReadTokens?: number;
|
|
142
|
+
cacheWriteTokens?: number;
|
|
143
|
+
requests?: number;
|
|
144
|
+
webSearches?: number;
|
|
145
|
+
};
|
|
146
|
+
export type TaskRequirements = {
|
|
147
|
+
minContextTokens?: number;
|
|
148
|
+
requiresTools?: boolean;
|
|
149
|
+
requiresStructuredOutputs?: boolean;
|
|
150
|
+
requiresReasoning?: boolean;
|
|
151
|
+
requiresTextOutput?: boolean;
|
|
152
|
+
maxCostUsd?: number;
|
|
153
|
+
maxLatencyMs?: number;
|
|
154
|
+
minQualityScore?: number;
|
|
155
|
+
allowDeprecated?: boolean;
|
|
156
|
+
};
|
|
157
|
+
export type ScoreWeights = {
|
|
158
|
+
quality: number;
|
|
159
|
+
cost: number;
|
|
160
|
+
latency: number;
|
|
161
|
+
reliability: number;
|
|
162
|
+
context: number;
|
|
163
|
+
benchmarkConfidence: number;
|
|
164
|
+
};
|
|
165
|
+
export type ModelTaskScore = {
|
|
166
|
+
model: ModelIdentity;
|
|
167
|
+
taskProfile: TaskProfile;
|
|
168
|
+
qualityScore: number;
|
|
169
|
+
costScore: number;
|
|
170
|
+
latencyScore: number;
|
|
171
|
+
reliabilityScore: number;
|
|
172
|
+
contextScore: number;
|
|
173
|
+
benchmarkConfidenceScore: number;
|
|
174
|
+
finalScore: number;
|
|
175
|
+
estimatedCostUsd: number;
|
|
176
|
+
evidence: Array<BenchmarkSignal | InternalEvalResult>;
|
|
177
|
+
issues: ModelIssueSignal[];
|
|
178
|
+
disqualifiedReasons: string[];
|
|
179
|
+
};
|
|
180
|
+
export type ProfileModelRecommendation = {
|
|
181
|
+
profileChoiceKey: ProfileChoiceKey;
|
|
182
|
+
current?: {
|
|
183
|
+
provider: ProviderId;
|
|
184
|
+
modelId: string;
|
|
185
|
+
};
|
|
186
|
+
recommended: {
|
|
187
|
+
provider: ProviderId;
|
|
188
|
+
modelId: string;
|
|
189
|
+
modelStatus: "verified";
|
|
190
|
+
};
|
|
191
|
+
taskProfile: TaskProfile;
|
|
192
|
+
score: number;
|
|
193
|
+
estimatedCostUsd: number;
|
|
194
|
+
reason: string;
|
|
195
|
+
confidence: number;
|
|
196
|
+
evidenceSummary: string[];
|
|
197
|
+
issuesSummary: string[];
|
|
198
|
+
requiresHumanApproval: boolean;
|
|
199
|
+
};
|
|
200
|
+
export type PatchValidationError = {
|
|
201
|
+
profileChoiceKey: string;
|
|
202
|
+
message: string;
|
|
203
|
+
};
|
|
204
|
+
export type ModelEvidenceFile = {
|
|
205
|
+
generatedAt: string;
|
|
206
|
+
sourceVersions: {
|
|
207
|
+
openrouterFetchedAt: string;
|
|
208
|
+
artificialAnalysisFetchedAt?: string;
|
|
209
|
+
huggingFaceFetchedAt?: string;
|
|
210
|
+
internalEvalFetchedAt?: string;
|
|
211
|
+
};
|
|
212
|
+
models: NormalizedModel[];
|
|
213
|
+
benchmarkSignals: BenchmarkSignal[];
|
|
214
|
+
issues: ModelIssueSignal[];
|
|
215
|
+
recommendations: ProfileModelRecommendation[];
|
|
216
|
+
};
|
|
217
|
+
export type { AIProvider, ProviderId };
|
|
218
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evidence/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE1D,MAAM,MAAM,iBAAiB,GACzB,YAAY,GACZ,qBAAqB,GACrB,qBAAqB,GACrB,SAAS,GACT,WAAW,GACX,MAAM,GACN,gBAAgB,GAChB,UAAU,GACV,eAAe,CAAC;AAEpB,MAAM,MAAM,WAAW,GACnB,OAAO,GACP,SAAS,GACT,YAAY,GACZ,MAAM,GACN,MAAM,GACN,UAAU,GACV,OAAO,GACP,MAAM,GACN,KAAK,GACL,SAAS,GACT,UAAU,GACV,OAAO,GACP,QAAQ,GACR,SAAS,GACT,eAAe,GACf,cAAc,GACd,OAAO,GACP,KAAK,CAAC;AAEV,MAAM,MAAM,gBAAgB,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;AAErD,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE;QACb,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;IACF,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,YAAY,CAAC,EAAE;QACb,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,CAAC;IACF,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,IAAI,EAAE,eAAe,EAAE,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,yBAAyB,EAAE,OAAO,CAAC;IACnC,sBAAsB,EAAE,OAAO,CAAC;IAChC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,wBAAwB,EAAE,OAAO,CAAC;IAClC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,YAAY,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC1C,YAAY,EAAE,iBAAiB,CAAC;IAChC,OAAO,EAAE,YAAY,CAAC;IACtB,GAAG,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,eAAe,CAAA;KAAE,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAC3B,cAAc,GACd,QAAQ,GACR,SAAS,GACT,iBAAiB,GACjB,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,GACf,kBAAkB,GAClB,sBAAsB,GACtB,mBAAmB,GACnB,iBAAiB,GACjB,aAAa,GACb,mBAAmB,GACnB,aAAa,GACb,qBAAqB,GACrB,iBAAiB,GACjB,oBAAoB,GACpB,cAAc,GACd,cAAc,CAAC;AAEnB,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,iBAAiB,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;IACxB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAC5B,mBAAmB,GACnB,iBAAiB,GACjB,kBAAkB,GAClB,qBAAqB,GACrB,4BAA4B,GAC5B,oBAAoB,GACpB,aAAa,GACb,uBAAuB,GACvB,iBAAiB,GACjB,gBAAgB,GAChB,4BAA4B,GAC5B,aAAa,GACb,kBAAkB,CAAC;AAEvB,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;IAC5C,KAAK,EAAE,oBAAoB,CAAC;IAC5B,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IACjD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAC1B,iBAAiB,GACjB,gBAAgB,GAChB,kBAAkB,GAClB,gBAAgB,GAChB,oBAAoB,GACpB,cAAc,GACd,cAAc,GACd,oBAAoB,GACpB,wBAAwB,CAAC;AAE7B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,EAAE,WAAW,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,KAAK,CAAC,eAAe,GAAG,kBAAkB,CAAC,CAAC;IACtD,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,OAAO,CAAC,EAAE;QAAE,QAAQ,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACpD,WAAW,EAAE;QACX,QAAQ,EAAE,UAAU,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,UAAU,CAAC;KACzB,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE;QACd,mBAAmB,EAAE,MAAM,CAAC;QAC5B,2BAA2B,CAAC,EAAE,MAAM,CAAC;QACrC,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,qBAAqB,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,eAAe,EAAE,0BAA0B,EAAE,CAAC;CAC/C,CAAC;AAEF,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/evidence/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { AIProvider, AIRuntimeDefaults, ModelsCatalogJson, ModelsProfilesJson, ModelAvailabilityStatus } from "./types.js";
|
|
2
|
+
export type ProfileChoiceKey = `${string}/${string}`;
|
|
3
|
+
export type FinalProfileModelMappingPatchEntry = {
|
|
4
|
+
provider: AIProvider;
|
|
5
|
+
modelId: string;
|
|
6
|
+
modelStatus: ModelAvailabilityStatus;
|
|
7
|
+
reason?: string;
|
|
8
|
+
displayName?: string;
|
|
9
|
+
description?: string;
|
|
10
|
+
runtime?: Partial<AIRuntimeDefaults>;
|
|
11
|
+
};
|
|
12
|
+
export type FinalProfileModelMappingPatch = Record<ProfileChoiceKey, FinalProfileModelMappingPatchEntry>;
|
|
13
|
+
/**
|
|
14
|
+
* Resolve the authoritative patch against the active OpenRouter-backed catalog.
|
|
15
|
+
* Preferred Gemini (and other) targets win when present; otherwise catalog-valid fallbacks.
|
|
16
|
+
*/
|
|
17
|
+
export declare function resolveFinalProfileModelMappingPatch(catalog?: ModelsCatalogJson): FinalProfileModelMappingPatch;
|
|
18
|
+
/** Authoritative patch (preferred targets; may include Gemini before catalog resolve). */
|
|
19
|
+
export declare const finalProfileModelMappingPatch: FinalProfileModelMappingPatch;
|
|
20
|
+
export declare function parseProfileChoiceKey(key: string): {
|
|
21
|
+
profile: string;
|
|
22
|
+
choice: string;
|
|
23
|
+
} | null;
|
|
24
|
+
/**
|
|
25
|
+
* Apply canonical routing patch. Only mutates provider/modelId/modelStatus/reason
|
|
26
|
+
* and optional displayName/description/runtime. Never renames profile or choice keys.
|
|
27
|
+
*/
|
|
28
|
+
export declare function applyFinalProfileModelMappingPatch(profiles: ModelsProfilesJson, patch?: FinalProfileModelMappingPatch): ModelsProfilesJson;
|
|
29
|
+
/** Keys that the sync remapper must not overwrite after the canonical patch. */
|
|
30
|
+
export declare function patchedProfileChoiceKeys(patch?: FinalProfileModelMappingPatch): Set<string>;
|
|
31
|
+
//# sourceMappingURL=finalProfileModelMappingPatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"finalProfileModelMappingPatch.d.ts","sourceRoot":"","sources":["../src/finalProfileModelMappingPatch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAEV,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,uBAAuB,EACxB,MAAM,YAAY,CAAC;AAGpB,MAAM,MAAM,gBAAgB,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;AAErD,MAAM,MAAM,kCAAkC,GAAG;IAC/C,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,uBAAuB,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG,MAAM,CAChD,gBAAgB,EAChB,kCAAkC,CACnC,CAAC;AAiWF;;;GAGG;AACH,wBAAgB,oCAAoC,CAClD,OAAO,CAAC,EAAE,iBAAiB,GAC1B,6BAA6B,CAM/B;AAED,0FAA0F;AAC1F,eAAO,MAAM,6BAA6B,EAAE,6BACJ,CAAC;AAEzC,wBAAgB,qBAAqB,CACnC,GAAG,EAAE,MAAM,GACV;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAO5C;AAYD;;;GAGG;AACH,wBAAgB,kCAAkC,CAChD,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,GAAE,6BAA6D,GACnE,kBAAkB,CAkDpB;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CACtC,KAAK,GAAE,6BAA6D,GACnE,GAAG,CAAC,MAAM,CAAC,CAEb"}
|