@aliou/pi-synthetic 0.17.4 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -2
- package/package.json +5 -4
- package/src/extensions/provider/index.ts +38 -11
- package/src/extensions/provider/models.ts +126 -8
- package/src/extensions/provider/index.test.ts +0 -46
- package/src/extensions/provider/models.test.ts +0 -217
- package/src/services/quota-store.test.ts +0 -211
- package/src/services/quota-warnings.test.ts +0 -393
- package/src/utils/quotas-severity.test.ts +0 -295
package/README.md
CHANGED
|
@@ -56,6 +56,8 @@ Once installed, select `synthetic` as your provider and choose from available mo
|
|
|
56
56
|
|
|
57
57
|
All models are accessed through Synthetic's API. Some models are hosted by Synthetic directly (`provider: "synthetic"` in the model config); others are proxied by Synthetic to upstream backends such as Fireworks or Together.
|
|
58
58
|
|
|
59
|
+
Synthetic also provides permanent aliases (`syn:large:text`, `syn:small:text`, `syn:large:vision`, `syn:small:vision`) that route to the current best model for each category. These aliases are stable across model rotations — using an alias means no reconfiguration when models change. Alias models are always visible even when Proxied Models is disabled.
|
|
60
|
+
|
|
59
61
|
By default, new installs show only Synthetic-hosted models. You can enable proxied models in `/synthetic:settings` under **Models > Proxied Models**. Existing configurations keep proxied models enabled to preserve prior behavior.
|
|
60
62
|
|
|
61
63
|
The `provider` field in `src/extensions/provider/models.ts` is for maintenance only and is stripped before registering models with Pi, so users always select the `synthetic` provider.
|
|
@@ -109,13 +111,24 @@ The **Proxied Models** setting is not a loadable extension feature. It is a regu
|
|
|
109
111
|
|
|
110
112
|
## Adding or Updating Models
|
|
111
113
|
|
|
112
|
-
Models are hardcoded in `src/extensions/provider/models.ts`.
|
|
114
|
+
Models are hardcoded in `src/extensions/provider/models.ts`. Entries are a union of concrete models and thin aliases (`syn:*` IDs).
|
|
115
|
+
|
|
116
|
+
### Adding a concrete model
|
|
113
117
|
|
|
114
118
|
1. Edit `src/extensions/provider/models.ts`
|
|
115
|
-
2.
|
|
119
|
+
2. Append a concrete model following the `SyntheticModelConfig` interface
|
|
116
120
|
3. Set `provider` to the upstream backend Synthetic uses for that model, such as `synthetic`, `fireworks`, or `together`
|
|
117
121
|
4. Run `pnpm run typecheck` to verify
|
|
118
122
|
|
|
123
|
+
### Adding an alias model
|
|
124
|
+
|
|
125
|
+
1. Add a thin `{ id, name, aliasFor }` entry at the top of `SYNTHETIC_MODELS`
|
|
126
|
+
2. Set `id` and `name` from the Synthetic API
|
|
127
|
+
3. Set `aliasFor` to `"hf:" + hugging_face_id` from the Synthetic API
|
|
128
|
+
4. The resolved alias inherits all fields from the target at build time
|
|
129
|
+
|
|
130
|
+
When Synthetic changes which model an alias routes to, update only the `aliasFor` field.
|
|
131
|
+
|
|
119
132
|
## Development
|
|
120
133
|
|
|
121
134
|
### Setup
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aliou/pi-synthetic",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
},
|
|
30
30
|
"files": [
|
|
31
31
|
"src",
|
|
32
|
-
"README.md"
|
|
32
|
+
"README.md",
|
|
33
|
+
"!src/**/*.test.ts"
|
|
33
34
|
],
|
|
34
35
|
"peerDependencies": {
|
|
35
36
|
"@earendil-works/pi-coding-agent": "0.74.0",
|
|
@@ -40,8 +41,8 @@
|
|
|
40
41
|
"@aliou/pi-utils-ui": "^0.4.0"
|
|
41
42
|
},
|
|
42
43
|
"devDependencies": {
|
|
43
|
-
"@aliou/biome-plugins": "^0.
|
|
44
|
-
"@biomejs/biome": "^2.4.
|
|
44
|
+
"@aliou/biome-plugins": "^0.8.1",
|
|
45
|
+
"@biomejs/biome": "^2.4.15",
|
|
45
46
|
"@changesets/cli": "^2.27.11",
|
|
46
47
|
"@earendil-works/pi-coding-agent": "0.74.0",
|
|
47
48
|
"typebox": "^1.1.37",
|
|
@@ -28,19 +28,46 @@ import {
|
|
|
28
28
|
} from "../../types/quotas";
|
|
29
29
|
import { fetchQuotas } from "../../utils/quotas";
|
|
30
30
|
import { SYNTHETIC_OVERFLOW_PATTERN } from "./context-overflow";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
type ConcreteSyntheticModelConfig,
|
|
33
|
+
isAlias,
|
|
34
|
+
SYNTHETIC_MODELS,
|
|
35
|
+
} from "./models";
|
|
32
36
|
|
|
33
37
|
export function buildSyntheticProviderModels(includeProxiedModels: boolean) {
|
|
34
|
-
|
|
35
|
-
(
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
const concreteModels = SYNTHETIC_MODELS.filter(
|
|
39
|
+
(m): m is ConcreteSyntheticModelConfig => !isAlias(m),
|
|
40
|
+
);
|
|
41
|
+
const byId = new Map(concreteModels.map((m) => [m.id, m]));
|
|
42
|
+
|
|
43
|
+
const resolved = SYNTHETIC_MODELS.map((entry) => {
|
|
44
|
+
if (!isAlias(entry)) return entry;
|
|
45
|
+
|
|
46
|
+
const target = byId.get(entry.aliasFor);
|
|
47
|
+
if (!target) {
|
|
48
|
+
throw new Error(
|
|
49
|
+
`Synthetic alias "${entry.id}" references missing model "${entry.aliasFor}"`,
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return {
|
|
54
|
+
...target,
|
|
55
|
+
id: entry.id,
|
|
56
|
+
name: entry.name,
|
|
57
|
+
provider: "synthetic" as const,
|
|
58
|
+
};
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
return resolved
|
|
62
|
+
.filter((model) => includeProxiedModels || model.provider === "synthetic")
|
|
63
|
+
.map(({ provider: _provider, ...model }) => ({
|
|
64
|
+
...model,
|
|
65
|
+
compat: {
|
|
66
|
+
supportsDeveloperRole: false,
|
|
67
|
+
maxTokensField: "max_tokens" as const,
|
|
68
|
+
...model.compat,
|
|
69
|
+
},
|
|
70
|
+
}));
|
|
44
71
|
}
|
|
45
72
|
|
|
46
73
|
interface RegisterSyntheticProviderOptions {
|
|
@@ -9,14 +9,68 @@ export interface SyntheticModelConfig extends ProviderModelConfig {
|
|
|
9
9
|
provider: string;
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
/** A thin alias that resolves to a concrete model at build time. */
|
|
13
|
+
export interface SyntheticModelAliasConfig {
|
|
14
|
+
id: string;
|
|
15
|
+
name: string;
|
|
16
|
+
/** Full model ID of the concrete target this alias resolves to. */
|
|
17
|
+
aliasFor: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Concrete model with full spec; aliases are excluded. */
|
|
21
|
+
export type ConcreteSyntheticModelConfig = SyntheticModelConfig & {
|
|
22
|
+
aliasFor?: never;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
export function isAlias(
|
|
26
|
+
entry: ConcreteSyntheticModelConfig | SyntheticModelAliasConfig,
|
|
27
|
+
): entry is SyntheticModelAliasConfig {
|
|
28
|
+
return "aliasFor" in entry;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export type SyntheticModelEntry =
|
|
32
|
+
| ConcreteSyntheticModelConfig
|
|
33
|
+
| SyntheticModelAliasConfig;
|
|
34
|
+
|
|
35
|
+
export const SYNTHETIC_MODELS: SyntheticModelEntry[] = [
|
|
36
|
+
// API: syn:large:text → alias for hf:zai-org/GLM-5.1
|
|
37
|
+
{
|
|
38
|
+
id: "syn:large:text",
|
|
39
|
+
name: "syn:large:text",
|
|
40
|
+
aliasFor: "hf:zai-org/GLM-5.1",
|
|
41
|
+
},
|
|
42
|
+
// API: syn:small:text → alias for hf:zai-org/GLM-4.7-Flash
|
|
43
|
+
{
|
|
44
|
+
id: "syn:small:text",
|
|
45
|
+
name: "syn:small:text",
|
|
46
|
+
aliasFor: "hf:zai-org/GLM-4.7-Flash",
|
|
47
|
+
},
|
|
48
|
+
// API: syn:large:vision → alias for hf:moonshotai/Kimi-K2.6
|
|
49
|
+
{
|
|
50
|
+
id: "syn:large:vision",
|
|
51
|
+
name: "syn:large:vision",
|
|
52
|
+
aliasFor: "hf:moonshotai/Kimi-K2.6",
|
|
53
|
+
},
|
|
54
|
+
// API: syn:small:vision → alias for hf:moonshotai/Kimi-K2.6
|
|
55
|
+
{
|
|
56
|
+
id: "syn:small:vision",
|
|
57
|
+
name: "syn:small:vision",
|
|
58
|
+
aliasFor: "hf:moonshotai/Kimi-K2.6",
|
|
59
|
+
},
|
|
13
60
|
// API: hf:zai-org/GLM-4.7 → ctx=202752
|
|
14
61
|
{
|
|
15
62
|
id: "hf:zai-org/GLM-4.7",
|
|
16
63
|
name: "zai-org/GLM-4.7",
|
|
17
64
|
provider: "synthetic",
|
|
18
65
|
reasoning: true,
|
|
19
|
-
thinkingLevelMap: {
|
|
66
|
+
thinkingLevelMap: {
|
|
67
|
+
off: "none",
|
|
68
|
+
minimal: null,
|
|
69
|
+
low: null,
|
|
70
|
+
medium: "medium",
|
|
71
|
+
high: null,
|
|
72
|
+
xhigh: null,
|
|
73
|
+
},
|
|
20
74
|
compat: {
|
|
21
75
|
supportsReasoningEffort: true,
|
|
22
76
|
},
|
|
@@ -36,7 +90,14 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
36
90
|
name: "zai-org/GLM-5",
|
|
37
91
|
provider: "synthetic",
|
|
38
92
|
reasoning: true,
|
|
39
|
-
thinkingLevelMap: {
|
|
93
|
+
thinkingLevelMap: {
|
|
94
|
+
off: "none",
|
|
95
|
+
minimal: null,
|
|
96
|
+
low: null,
|
|
97
|
+
medium: "medium",
|
|
98
|
+
high: null,
|
|
99
|
+
xhigh: null,
|
|
100
|
+
},
|
|
40
101
|
compat: {
|
|
41
102
|
supportsReasoningEffort: true,
|
|
42
103
|
},
|
|
@@ -56,7 +117,14 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
56
117
|
name: "zai-org/GLM-5.1",
|
|
57
118
|
provider: "synthetic",
|
|
58
119
|
reasoning: true,
|
|
59
|
-
thinkingLevelMap: {
|
|
120
|
+
thinkingLevelMap: {
|
|
121
|
+
off: "none",
|
|
122
|
+
minimal: null,
|
|
123
|
+
low: null,
|
|
124
|
+
medium: "medium",
|
|
125
|
+
high: null,
|
|
126
|
+
xhigh: null,
|
|
127
|
+
},
|
|
60
128
|
compat: {
|
|
61
129
|
supportsReasoningEffort: true,
|
|
62
130
|
supportsDeveloperRole: false,
|
|
@@ -77,7 +145,14 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
77
145
|
name: "zai-org/GLM-4.7-Flash",
|
|
78
146
|
provider: "synthetic",
|
|
79
147
|
reasoning: true,
|
|
80
|
-
thinkingLevelMap: {
|
|
148
|
+
thinkingLevelMap: {
|
|
149
|
+
off: "none",
|
|
150
|
+
minimal: null,
|
|
151
|
+
low: null,
|
|
152
|
+
medium: "medium",
|
|
153
|
+
high: null,
|
|
154
|
+
xhigh: null,
|
|
155
|
+
},
|
|
81
156
|
compat: {
|
|
82
157
|
supportsReasoningEffort: true,
|
|
83
158
|
},
|
|
@@ -97,6 +172,17 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
97
172
|
name: "deepseek-ai/DeepSeek-V3.2",
|
|
98
173
|
provider: "fireworks",
|
|
99
174
|
reasoning: true,
|
|
175
|
+
thinkingLevelMap: {
|
|
176
|
+
off: "none",
|
|
177
|
+
minimal: null,
|
|
178
|
+
low: null,
|
|
179
|
+
medium: "medium",
|
|
180
|
+
high: null,
|
|
181
|
+
xhigh: null,
|
|
182
|
+
},
|
|
183
|
+
compat: {
|
|
184
|
+
supportsReasoningEffort: true,
|
|
185
|
+
},
|
|
100
186
|
input: ["text"],
|
|
101
187
|
cost: {
|
|
102
188
|
input: 0.56,
|
|
@@ -145,7 +231,14 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
145
231
|
name: "moonshotai/Kimi-K2.6",
|
|
146
232
|
provider: "synthetic",
|
|
147
233
|
reasoning: true,
|
|
148
|
-
thinkingLevelMap: {
|
|
234
|
+
thinkingLevelMap: {
|
|
235
|
+
off: "none",
|
|
236
|
+
minimal: null,
|
|
237
|
+
low: null,
|
|
238
|
+
medium: "medium",
|
|
239
|
+
high: null,
|
|
240
|
+
xhigh: null,
|
|
241
|
+
},
|
|
149
242
|
compat: {
|
|
150
243
|
supportsReasoningEffort: true,
|
|
151
244
|
},
|
|
@@ -165,6 +258,17 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
165
258
|
name: "Qwen/Qwen3.5-397B-A17B",
|
|
166
259
|
provider: "together",
|
|
167
260
|
reasoning: true,
|
|
261
|
+
thinkingLevelMap: {
|
|
262
|
+
off: "none",
|
|
263
|
+
minimal: null,
|
|
264
|
+
low: null,
|
|
265
|
+
medium: "medium",
|
|
266
|
+
high: null,
|
|
267
|
+
xhigh: null,
|
|
268
|
+
},
|
|
269
|
+
compat: {
|
|
270
|
+
supportsReasoningEffort: true,
|
|
271
|
+
},
|
|
168
272
|
input: ["text", "image"],
|
|
169
273
|
cost: {
|
|
170
274
|
input: 0.6,
|
|
@@ -181,7 +285,14 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
181
285
|
name: "MiniMaxAI/MiniMax-M2.5",
|
|
182
286
|
provider: "synthetic",
|
|
183
287
|
reasoning: true,
|
|
184
|
-
thinkingLevelMap: {
|
|
288
|
+
thinkingLevelMap: {
|
|
289
|
+
off: null,
|
|
290
|
+
minimal: null,
|
|
291
|
+
low: null,
|
|
292
|
+
medium: "medium",
|
|
293
|
+
high: null,
|
|
294
|
+
xhigh: null,
|
|
295
|
+
},
|
|
185
296
|
input: ["text"],
|
|
186
297
|
cost: {
|
|
187
298
|
input: 0.4,
|
|
@@ -202,7 +313,14 @@ export const SYNTHETIC_MODELS: SyntheticModelConfig[] = [
|
|
|
202
313
|
name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
|
|
203
314
|
provider: "synthetic",
|
|
204
315
|
reasoning: true,
|
|
205
|
-
thinkingLevelMap: {
|
|
316
|
+
thinkingLevelMap: {
|
|
317
|
+
off: "none",
|
|
318
|
+
minimal: null,
|
|
319
|
+
low: null,
|
|
320
|
+
medium: "medium",
|
|
321
|
+
high: null,
|
|
322
|
+
xhigh: null,
|
|
323
|
+
},
|
|
206
324
|
compat: {
|
|
207
325
|
supportsReasoningEffort: true,
|
|
208
326
|
},
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { buildSyntheticProviderModels } from "./index";
|
|
3
|
-
import { SYNTHETIC_MODELS } from "./models";
|
|
4
|
-
|
|
5
|
-
describe("buildSyntheticProviderModels", () => {
|
|
6
|
-
it("excludes proxied models when includeProxiedModels is false", () => {
|
|
7
|
-
const models = buildSyntheticProviderModels(false);
|
|
8
|
-
for (const model of models) {
|
|
9
|
-
const source = SYNTHETIC_MODELS.find((m) => m.id === model.id);
|
|
10
|
-
expect(source).toBeDefined();
|
|
11
|
-
expect(source?.provider).toBe("synthetic");
|
|
12
|
-
}
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
it("includes all models when includeProxiedModels is true", () => {
|
|
16
|
-
const models = buildSyntheticProviderModels(true);
|
|
17
|
-
expect(models).toHaveLength(SYNTHETIC_MODELS.length);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
it("does not expose the internal provider field", () => {
|
|
21
|
-
const models = buildSyntheticProviderModels(true);
|
|
22
|
-
for (const model of models) {
|
|
23
|
-
expect(model).not.toHaveProperty("provider");
|
|
24
|
-
}
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
it("sets default compat fields on every model", () => {
|
|
28
|
-
const models = buildSyntheticProviderModels(true);
|
|
29
|
-
for (const model of models) {
|
|
30
|
-
expect(model.compat).toMatchObject({
|
|
31
|
-
supportsDeveloperRole: false,
|
|
32
|
-
});
|
|
33
|
-
expect(model.compat).toHaveProperty("maxTokensField");
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
it("preserves model-specific compat overrides", () => {
|
|
38
|
-
const models = buildSyntheticProviderModels(true);
|
|
39
|
-
const miniMax = models.find((m) => m.id === "hf:MiniMaxAI/MiniMax-M2.5");
|
|
40
|
-
expect(miniMax).toBeDefined();
|
|
41
|
-
expect(miniMax?.compat).toMatchObject({
|
|
42
|
-
supportsDeveloperRole: false,
|
|
43
|
-
maxTokensField: "max_completion_tokens",
|
|
44
|
-
});
|
|
45
|
-
});
|
|
46
|
-
});
|
|
@@ -1,217 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { SYNTHETIC_MODELS } from "./models";
|
|
3
|
-
|
|
4
|
-
interface ApiModel {
|
|
5
|
-
id: string;
|
|
6
|
-
name: string;
|
|
7
|
-
provider: string | null;
|
|
8
|
-
input_modalities: string[];
|
|
9
|
-
output_modalities: string[];
|
|
10
|
-
context_length: number;
|
|
11
|
-
max_output_length: number;
|
|
12
|
-
pricing: {
|
|
13
|
-
prompt: string;
|
|
14
|
-
completion: string;
|
|
15
|
-
input_cache_reads: string;
|
|
16
|
-
input_cache_writes: string;
|
|
17
|
-
};
|
|
18
|
-
supported_features?: string[];
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
interface ApiResponse {
|
|
22
|
-
data: ApiModel[];
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
interface Discrepancy {
|
|
26
|
-
model: string;
|
|
27
|
-
field: string;
|
|
28
|
-
hardcoded: unknown;
|
|
29
|
-
api: unknown;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
async function fetchApiModels(): Promise<ApiModel[]> {
|
|
33
|
-
// Making ourselves known
|
|
34
|
-
const response = await fetch("https://api.synthetic.new/openai/v1/models", {
|
|
35
|
-
headers: {
|
|
36
|
-
Referer: "https://github.com/aliou/pi-synthetic",
|
|
37
|
-
},
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
if (!response.ok) {
|
|
41
|
-
throw new Error(
|
|
42
|
-
`API request failed: ${response.status} ${response.statusText}`,
|
|
43
|
-
);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
const data: ApiResponse = await response.json();
|
|
47
|
-
return data.data;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function parsePrice(priceStr: string): number {
|
|
51
|
-
// Convert "$0.0000006" to 0.6 (dollars per million tokens)
|
|
52
|
-
const match = priceStr.match(/\$?(\d+\.?\d*)/);
|
|
53
|
-
if (!match) return 0;
|
|
54
|
-
const pricePerToken = Number.parseFloat(match[1]);
|
|
55
|
-
// API prices are per token, hardcoded prices are per million tokens
|
|
56
|
-
return pricePerToken * 1_000_000;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
function compareModels(
|
|
60
|
-
apiModels: ApiModel[],
|
|
61
|
-
hardcodedModels: typeof SYNTHETIC_MODELS,
|
|
62
|
-
): Discrepancy[] {
|
|
63
|
-
const discrepancies: Discrepancy[] = [];
|
|
64
|
-
|
|
65
|
-
for (const hardcoded of hardcodedModels) {
|
|
66
|
-
const apiModel = apiModels.find((m) => m.id === hardcoded.id);
|
|
67
|
-
|
|
68
|
-
if (!apiModel) {
|
|
69
|
-
discrepancies.push({
|
|
70
|
-
model: hardcoded.id,
|
|
71
|
-
field: "exists",
|
|
72
|
-
hardcoded: true,
|
|
73
|
-
api: false,
|
|
74
|
-
});
|
|
75
|
-
continue;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
// Check input modalities (text vs image support)
|
|
79
|
-
const apiInputs = apiModel.input_modalities.sort();
|
|
80
|
-
const hardcodedInputs = [...hardcoded.input].sort();
|
|
81
|
-
if (JSON.stringify(apiInputs) !== JSON.stringify(hardcodedInputs)) {
|
|
82
|
-
discrepancies.push({
|
|
83
|
-
model: hardcoded.id,
|
|
84
|
-
field: "input",
|
|
85
|
-
hardcoded: hardcodedInputs,
|
|
86
|
-
api: apiInputs,
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
// Check context window
|
|
91
|
-
if (apiModel.context_length !== hardcoded.contextWindow) {
|
|
92
|
-
discrepancies.push({
|
|
93
|
-
model: hardcoded.id,
|
|
94
|
-
field: "contextWindow",
|
|
95
|
-
hardcoded: hardcoded.contextWindow,
|
|
96
|
-
api: apiModel.context_length,
|
|
97
|
-
});
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// Check max output tokens (skip if API doesn't provide it)
|
|
101
|
-
if (
|
|
102
|
-
apiModel.max_output_length !== undefined &&
|
|
103
|
-
apiModel.max_output_length !== hardcoded.maxTokens
|
|
104
|
-
) {
|
|
105
|
-
discrepancies.push({
|
|
106
|
-
model: hardcoded.id,
|
|
107
|
-
field: "maxTokens",
|
|
108
|
-
hardcoded: hardcoded.maxTokens,
|
|
109
|
-
api: apiModel.max_output_length,
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
// Check input cost (convert API price to per-million rate)
|
|
114
|
-
const apiInputCost = parsePrice(apiModel.pricing.prompt);
|
|
115
|
-
const epsilon = 0.001; // Small tolerance for floating point
|
|
116
|
-
if (Math.abs(apiInputCost - hardcoded.cost.input) > epsilon) {
|
|
117
|
-
discrepancies.push({
|
|
118
|
-
model: hardcoded.id,
|
|
119
|
-
field: "cost.input",
|
|
120
|
-
hardcoded: hardcoded.cost.input,
|
|
121
|
-
api: apiInputCost,
|
|
122
|
-
});
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// Check output cost
|
|
126
|
-
const apiOutputCost = parsePrice(apiModel.pricing.completion);
|
|
127
|
-
if (Math.abs(apiOutputCost - hardcoded.cost.output) > epsilon) {
|
|
128
|
-
discrepancies.push({
|
|
129
|
-
model: hardcoded.id,
|
|
130
|
-
field: "cost.output",
|
|
131
|
-
hardcoded: hardcoded.cost.output,
|
|
132
|
-
api: apiOutputCost,
|
|
133
|
-
});
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Check cache read cost
|
|
137
|
-
const apiCacheReadCost = parsePrice(apiModel.pricing.input_cache_reads);
|
|
138
|
-
if (Math.abs(apiCacheReadCost - hardcoded.cost.cacheRead) > epsilon) {
|
|
139
|
-
discrepancies.push({
|
|
140
|
-
model: hardcoded.id,
|
|
141
|
-
field: "cost.cacheRead",
|
|
142
|
-
hardcoded: hardcoded.cost.cacheRead,
|
|
143
|
-
api: apiCacheReadCost,
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// Check reasoning capability from supported_features (skip if API doesn't provide it)
|
|
148
|
-
if (apiModel.supported_features !== undefined) {
|
|
149
|
-
const apiSupportsReasoning =
|
|
150
|
-
apiModel.supported_features.includes("reasoning");
|
|
151
|
-
if (apiSupportsReasoning !== hardcoded.reasoning) {
|
|
152
|
-
discrepancies.push({
|
|
153
|
-
model: hardcoded.id,
|
|
154
|
-
field: "reasoning",
|
|
155
|
-
hardcoded: hardcoded.reasoning,
|
|
156
|
-
api: apiSupportsReasoning,
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
// Check provider
|
|
162
|
-
if (
|
|
163
|
-
apiModel.provider !== null &&
|
|
164
|
-
apiModel.provider !== hardcoded.provider
|
|
165
|
-
) {
|
|
166
|
-
discrepancies.push({
|
|
167
|
-
model: hardcoded.id,
|
|
168
|
-
field: "provider",
|
|
169
|
-
hardcoded: hardcoded.provider,
|
|
170
|
-
api: apiModel.provider,
|
|
171
|
-
});
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Check for API models not in hardcoded list
|
|
176
|
-
for (const apiModel of apiModels) {
|
|
177
|
-
const hardcoded = hardcodedModels.find((m) => m.id === apiModel.id);
|
|
178
|
-
if (!hardcoded) {
|
|
179
|
-
discrepancies.push({
|
|
180
|
-
model: apiModel.id,
|
|
181
|
-
field: "exists",
|
|
182
|
-
hardcoded: false,
|
|
183
|
-
api: true,
|
|
184
|
-
});
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
return discrepancies;
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
describe("Synthetic models", () => {
|
|
192
|
-
it("should match API model definitions", { timeout: 30000 }, async () => {
|
|
193
|
-
const apiModels = await fetchApiModels();
|
|
194
|
-
const discrepancies = compareModels(apiModels, SYNTHETIC_MODELS);
|
|
195
|
-
|
|
196
|
-
if (discrepancies.length > 0) {
|
|
197
|
-
console.error("\nModel discrepancies found:");
|
|
198
|
-
console.error("==========================");
|
|
199
|
-
for (const d of discrepancies) {
|
|
200
|
-
if (d.field === "exists") {
|
|
201
|
-
if (d.hardcoded) {
|
|
202
|
-
console.error(` ${d.model}: Missing from API`);
|
|
203
|
-
} else {
|
|
204
|
-
console.error(` ${d.model}: Missing from hardcoded models (NEW)`);
|
|
205
|
-
}
|
|
206
|
-
} else {
|
|
207
|
-
console.error(` ${d.model}.${d.field}:`);
|
|
208
|
-
console.error(` hardcoded: ${JSON.stringify(d.hardcoded)}`);
|
|
209
|
-
console.error(` api: ${JSON.stringify(d.api)}`);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
console.error("==========================\n");
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
expect(discrepancies).toHaveLength(0);
|
|
216
|
-
});
|
|
217
|
-
});
|