@f5-sales-demo/pi-ai 22.4.11 → 22.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/api-registry.ts +2 -0
- package/src/codex-model-interaction.ts +20 -4
- package/src/deferred.ts +198 -0
- package/src/index.ts +3 -0
- package/src/model-cache.ts +34 -1
- package/src/model-manager.ts +27 -9
- package/src/model-thinking.ts +94 -2
- package/src/models.json +11193 -6758
- package/src/models.ts +48 -8
- package/src/operation-catalog.ts +103 -0
- package/src/operation-models.json +1462 -0
- package/src/operations.ts +71 -0
- package/src/provider-models/descriptors.ts +4 -2
- package/src/provider-models/openai-compat.ts +186 -60
- package/src/provider-models/special.ts +2 -2
- package/src/provider-models/upstream-provider-routes.ts +107 -0
- package/src/provider-models/upstream-providers.ts +64 -0
- package/src/providers/amazon-bedrock.ts +36 -4
- package/src/providers/anthropic.ts +121 -5
- package/src/providers/azure-openai-responses.ts +55 -159
- package/src/providers/cloudflare-route.ts +82 -0
- package/src/providers/cloudflare-workers-ai-system-one.ts +43 -0
- package/src/providers/completions-reasoning-details.ts +38 -0
- package/src/providers/google-vertex.ts +7 -2
- package/src/providers/google.ts +15 -6
- package/src/providers/llama-cpp-classify.ts +461 -0
- package/src/providers/mistral-conversations.ts +914 -0
- package/src/providers/openai-codex/live-steering.ts +237 -0
- package/src/providers/openai-codex-responses.ts +287 -21
- package/src/providers/openai-completions-compat.ts +84 -11
- package/src/providers/openai-completions.ts +273 -59
- package/src/providers/openai-responses-shared.ts +185 -24
- package/src/providers/openai-responses.ts +240 -18
- package/src/providers/openrouter-images.ts +209 -0
- package/src/providers/pi-messages.ts +439 -0
- package/src/providers/sol-request-boundary.ts +31 -0
- package/src/providers/system-one-shared.ts +241 -0
- package/src/providers/transform-messages.ts +13 -1
- package/src/providers/typesafe-system-one.ts +21 -0
- package/src/sol-model.ts +90 -0
- package/src/stream.ts +61 -3
- package/src/types.ts +310 -8
- package/src/utils/discovery/codex.ts +34 -0
- package/src/utils/discovery/openai-compatible.ts +3 -1
- package/src/utils/discovery/operations.ts +150 -0
- package/src/utils/mistral-compat.ts +34 -0
- package/src/utils/oauth/device-code.ts +98 -0
- package/src/utils/oauth/index.ts +33 -2
- package/src/utils/oauth/meta.ts +202 -0
- package/src/utils/oauth/openrouter.ts +66 -0
- package/src/utils/oauth/radius-callback.ts +151 -0
- package/src/utils/oauth/radius.ts +308 -0
- package/src/utils/oauth/types.ts +4 -0
- package/src/utils/oauth/xai.ts +235 -0
- package/src/utils/operation-http.ts +39 -0
- package/src/utils/provider-retry.ts +125 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@f5-sales-demo/pi-ai",
|
|
4
|
-
"version": "22.
|
|
4
|
+
"version": "22.5.1",
|
|
5
5
|
"description": "Unified LLM API with automatic model discovery and provider configuration",
|
|
6
6
|
"homepage": "https://github.com/f5-sales-demo/xcsh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -43,17 +43,17 @@
|
|
|
43
43
|
"generate-models": "bun scripts/generate-models.ts"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
|
-
"@anthropic-ai/sdk": "0.
|
|
47
|
-
"@aws-sdk/client-bedrock-runtime": "3.
|
|
46
|
+
"@anthropic-ai/sdk": "0.124.0",
|
|
47
|
+
"@aws-sdk/client-bedrock-runtime": "3.1127.0",
|
|
48
48
|
"@bufbuild/protobuf": "2.14.1",
|
|
49
|
-
"@f5-sales-demo/pi-utils": "22.
|
|
49
|
+
"@f5-sales-demo/pi-utils": "22.5.1",
|
|
50
50
|
"@google/genai": "2.21.0",
|
|
51
51
|
"@sinclair/typebox": "0.34.52",
|
|
52
52
|
"@smithy/node-http-handler": "4.12.1",
|
|
53
53
|
"ajv": "^8.20",
|
|
54
54
|
"ajv-formats": "^3.0",
|
|
55
55
|
"google-auth-library": "10.9.1",
|
|
56
|
-
"openai": "7.
|
|
56
|
+
"openai": "7.19.0",
|
|
57
57
|
"partial-json": "^0.1",
|
|
58
58
|
"zod": "4.5.4"
|
|
59
59
|
},
|
package/src/api-registry.ts
CHANGED
|
@@ -64,6 +64,7 @@ function metadata(
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
export const CODEX_MODEL_INTERACTION_METADATA: Readonly<Record<string, CodexModelInteractionMetadata>> = {
|
|
67
|
+
"gpt-6.1-sol": metadata("medium", ["low", "medium", "high", "xhigh", "max", "ultra"], "default", 872_000, 922_000),
|
|
67
68
|
"gpt-6-astra": metadata(
|
|
68
69
|
"low",
|
|
69
70
|
["low", "medium", "high", "xhigh", "max", "ultra"],
|
|
@@ -141,11 +142,26 @@ export function applyCodexInteractionMetadata<TApi extends Api>(
|
|
|
141
142
|
providerContextWindow ??
|
|
142
143
|
model.providerContextWindow ??
|
|
143
144
|
(model.provider === "openai-codex" ? interaction.maxContextWindow : model.contextWindow);
|
|
144
|
-
const advertisedLimit = Math.min(
|
|
145
|
+
const advertisedLimit = Math.min(
|
|
146
|
+
rawProviderLimit,
|
|
147
|
+
model.maxInputTokens ?? Number.POSITIVE_INFINITY,
|
|
148
|
+
interaction.internalProviderMaxContextWindow,
|
|
149
|
+
);
|
|
145
150
|
const budget = resolveCodexContextBudget(model.id, "standard", advertisedLimit)!;
|
|
146
151
|
return {
|
|
147
152
|
...model,
|
|
148
|
-
|
|
153
|
+
...(model.provider === "openai-codex" && model.id === "gpt-6.1-sol"
|
|
154
|
+
? { compat: { supportsWebSocketSteering: true, ...model.compat } as Model<TApi>["compat"] }
|
|
155
|
+
: {}),
|
|
156
|
+
thinking:
|
|
157
|
+
model.provider === "openai-codex"
|
|
158
|
+
? (model.thinking ?? interaction.thinking)
|
|
159
|
+
: model.id === "gpt-6.1-sol"
|
|
160
|
+
? {
|
|
161
|
+
...interaction.thinking,
|
|
162
|
+
supportedLevels: interaction.thinking.supportedLevels.filter(level => level.effort !== "ultra"),
|
|
163
|
+
}
|
|
164
|
+
: interaction.thinking,
|
|
149
165
|
contextWindow: budget.contextWindow,
|
|
150
166
|
maxContextWindow: interaction.maxContextWindow,
|
|
151
167
|
maxTokens: interaction.outputLimit,
|
|
@@ -156,8 +172,8 @@ export function applyCodexInteractionMetadata<TApi extends Api>(
|
|
|
156
172
|
autoCompactTokenLimit: budget.autoCompactTokenLimit,
|
|
157
173
|
defaultReasoningSummary: interaction.defaultReasoningSummary,
|
|
158
174
|
defaultVerbosity: interaction.defaultVerbosity,
|
|
159
|
-
serviceTiers: interaction.serviceTiers,
|
|
160
|
-
defaultServiceTier: interaction.defaultServiceTier,
|
|
175
|
+
serviceTiers: model.serviceTiers ?? interaction.serviceTiers,
|
|
176
|
+
defaultServiceTier: model.defaultServiceTier ?? interaction.defaultServiceTier,
|
|
161
177
|
truncationPolicy: interaction.truncationPolicy,
|
|
162
178
|
supportsParallelToolCalls: interaction.supportsParallelToolCalls,
|
|
163
179
|
};
|
package/src/deferred.ts
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { buildParams } from "./providers/openai-responses";
|
|
2
|
+
import { processResponsesStream } from "./providers/openai-responses-shared";
|
|
3
|
+
import { validateFinalResponsesRequest } from "./providers/sol-request-boundary";
|
|
4
|
+
import { getEnvApiKey } from "./stream";
|
|
5
|
+
import type { AssistantMessage, Context, DeferredHandle, Model, OperationOptions } from "./types";
|
|
6
|
+
import { createOpenAIResponsesHistoryPayload } from "./utils";
|
|
7
|
+
import { AssistantMessageEventStream } from "./utils/event-stream";
|
|
8
|
+
import { retryProviderRequest } from "./utils/provider-retry";
|
|
9
|
+
|
|
10
|
+
export type DeferredRequestOptions = OperationOptions &
|
|
11
|
+
Pick<
|
|
12
|
+
import("./providers/openai-responses").OpenAIResponsesOptions,
|
|
13
|
+
"reasoning" | "reasoningSummary" | "promptCache" | "serviceTier" | "toolChoice" | "maxTokens"
|
|
14
|
+
>;
|
|
15
|
+
|
|
16
|
+
export interface DeferredResponse {
|
|
17
|
+
id: string;
|
|
18
|
+
status: "queued" | "in_progress" | "completed" | "incomplete" | "failed" | "cancelled";
|
|
19
|
+
output: Record<string, unknown>[];
|
|
20
|
+
}
|
|
21
|
+
function validateRoute(model: Model, handle?: DeferredHandle): void {
|
|
22
|
+
if (model.type !== undefined && model.type !== "chat") throw new Error("Deferred operations require a chat model");
|
|
23
|
+
if (model.api !== "openai-responses" || model.provider === "openai-codex")
|
|
24
|
+
throw new Error("Deferred Responses unsupported on this route");
|
|
25
|
+
if (
|
|
26
|
+
handle &&
|
|
27
|
+
(handle.provider !== model.provider ||
|
|
28
|
+
handle.modelId !== model.id ||
|
|
29
|
+
handle.api !== model.api ||
|
|
30
|
+
handle.baseUrl !== model.baseUrl)
|
|
31
|
+
)
|
|
32
|
+
throw new Error("Deferred handle belongs to a different route");
|
|
33
|
+
if (handle?.expiresAt && handle.expiresAt < Date.now()) throw new Error("Deferred handle expired");
|
|
34
|
+
}
|
|
35
|
+
async function request(
|
|
36
|
+
model: Model,
|
|
37
|
+
path: string,
|
|
38
|
+
method: string,
|
|
39
|
+
options: OperationOptions,
|
|
40
|
+
payload?: unknown,
|
|
41
|
+
): Promise<DeferredResponse> {
|
|
42
|
+
options.signal?.throwIfAborted();
|
|
43
|
+
const apiKey = options.apiKey ?? getEnvApiKey(model.provider);
|
|
44
|
+
if (!apiKey) throw new Error(`No API key for provider: ${model.provider}`);
|
|
45
|
+
const timeoutSignal = options.timeoutMs !== undefined ? AbortSignal.timeout(options.timeoutMs) : undefined;
|
|
46
|
+
const signal =
|
|
47
|
+
options.signal && timeoutSignal
|
|
48
|
+
? AbortSignal.any([options.signal, timeoutSignal])
|
|
49
|
+
: (options.signal ?? timeoutSignal);
|
|
50
|
+
const response = await retryProviderRequest(
|
|
51
|
+
async () => {
|
|
52
|
+
const result = await (options.fetch ?? globalThis.fetch)(
|
|
53
|
+
`${model.baseUrl.replace(/\/+$/, "")}/responses${path}`,
|
|
54
|
+
{
|
|
55
|
+
method,
|
|
56
|
+
headers: {
|
|
57
|
+
...model.headers,
|
|
58
|
+
...options.headers,
|
|
59
|
+
Authorization: `Bearer ${apiKey}`,
|
|
60
|
+
"Content-Type": "application/json",
|
|
61
|
+
},
|
|
62
|
+
...(payload === undefined ? {} : { body: JSON.stringify(payload) }),
|
|
63
|
+
signal,
|
|
64
|
+
},
|
|
65
|
+
);
|
|
66
|
+
if (!result.ok)
|
|
67
|
+
throw Object.assign(new Error(`Deferred Responses HTTP ${result.status}`), {
|
|
68
|
+
status: result.status,
|
|
69
|
+
headers: result.headers,
|
|
70
|
+
});
|
|
71
|
+
return result;
|
|
72
|
+
},
|
|
73
|
+
{ ...options, signal },
|
|
74
|
+
);
|
|
75
|
+
await options.onResponse?.(
|
|
76
|
+
{ status: response.status, headers: Object.fromEntries(response.headers.entries()) },
|
|
77
|
+
model,
|
|
78
|
+
);
|
|
79
|
+
const body = (await response.json()) as DeferredResponse;
|
|
80
|
+
if (
|
|
81
|
+
!body ||
|
|
82
|
+
typeof body.id !== "string" ||
|
|
83
|
+
!Array.isArray(body.output) ||
|
|
84
|
+
!["queued", "in_progress", "completed", "incomplete", "failed", "cancelled"].includes(body.status)
|
|
85
|
+
)
|
|
86
|
+
throw new Error("Malformed deferred response");
|
|
87
|
+
return body;
|
|
88
|
+
}
|
|
89
|
+
export async function requestDeferred(
|
|
90
|
+
model: Model,
|
|
91
|
+
context: Context,
|
|
92
|
+
options: DeferredRequestOptions = {},
|
|
93
|
+
): Promise<DeferredHandle> {
|
|
94
|
+
options.signal?.throwIfAborted();
|
|
95
|
+
validateRoute(model);
|
|
96
|
+
let payload: Record<string, unknown> = {
|
|
97
|
+
...buildParams(
|
|
98
|
+
model as Model<"openai-responses">,
|
|
99
|
+
context,
|
|
100
|
+
options as import("./providers/openai-responses").OpenAIResponsesOptions,
|
|
101
|
+
undefined,
|
|
102
|
+
).params,
|
|
103
|
+
background: true,
|
|
104
|
+
store: true,
|
|
105
|
+
stream: false,
|
|
106
|
+
};
|
|
107
|
+
const replacement = await options.onPayload?.(payload, model);
|
|
108
|
+
if (replacement !== undefined) payload = replacement as Record<string, unknown>;
|
|
109
|
+
validateFinalResponsesRequest(model, payload);
|
|
110
|
+
Object.assign(payload, { background: true, store: true, stream: false });
|
|
111
|
+
const response = await request(model, "", "POST", options, payload);
|
|
112
|
+
return {
|
|
113
|
+
provider: model.provider,
|
|
114
|
+
modelId: model.id,
|
|
115
|
+
api: model.api,
|
|
116
|
+
baseUrl: model.baseUrl,
|
|
117
|
+
id: response.id,
|
|
118
|
+
pollAfterMs: 1000,
|
|
119
|
+
data: {
|
|
120
|
+
grammarTools: (context.tools ?? [])
|
|
121
|
+
.filter(tool => tool.constrainedSampling && tool.constrainedSampling.type === "grammar")
|
|
122
|
+
.map(tool => [tool.name, (tool.parameters as { required?: string[] }).required?.[0] ?? "input"]),
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
export async function fetchDeferred(
|
|
127
|
+
model: Model,
|
|
128
|
+
handle: DeferredHandle,
|
|
129
|
+
options: OperationOptions = {},
|
|
130
|
+
): Promise<AssistantMessage> {
|
|
131
|
+
validateRoute(model, handle);
|
|
132
|
+
const response = await request(model, `/${encodeURIComponent(handle.id)}`, "GET", options);
|
|
133
|
+
if (response.id !== handle.id) throw new Error("Deferred response identity does not match its handle");
|
|
134
|
+
const output: AssistantMessage = {
|
|
135
|
+
role: "assistant",
|
|
136
|
+
api: model.api,
|
|
137
|
+
provider: model.provider,
|
|
138
|
+
model: model.id,
|
|
139
|
+
responseId: response.id,
|
|
140
|
+
content: [],
|
|
141
|
+
stopReason: "stop",
|
|
142
|
+
timestamp: Date.now(),
|
|
143
|
+
usage: {
|
|
144
|
+
input: 0,
|
|
145
|
+
output: 0,
|
|
146
|
+
cacheRead: 0,
|
|
147
|
+
cacheWrite: 0,
|
|
148
|
+
totalTokens: 0,
|
|
149
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
if (response.status === "queued" || response.status === "in_progress") {
|
|
153
|
+
output.deferred = handle;
|
|
154
|
+
return output;
|
|
155
|
+
}
|
|
156
|
+
if (response.status === "cancelled" || response.status === "failed") {
|
|
157
|
+
output.stopReason = "error";
|
|
158
|
+
output.errorMessage = `Deferred response ${response.status}`;
|
|
159
|
+
return output;
|
|
160
|
+
}
|
|
161
|
+
async function* events() {
|
|
162
|
+
for (const [output_index, item] of response.output.entries()) {
|
|
163
|
+
if (!item || typeof item !== "object" || typeof item.type !== "string")
|
|
164
|
+
throw new Error("Malformed deferred output item");
|
|
165
|
+
yield { type: "response.output_item.added", output_index, item };
|
|
166
|
+
yield { type: "response.output_item.done", output_index, item };
|
|
167
|
+
}
|
|
168
|
+
yield { type: response.status === "incomplete" ? "response.incomplete" : "response.completed", response };
|
|
169
|
+
}
|
|
170
|
+
const stream = new AssistantMessageEventStream();
|
|
171
|
+
try {
|
|
172
|
+
await processResponsesStream(
|
|
173
|
+
events() as AsyncIterable<import("openai").default.Responses.ResponseStreamEvent>,
|
|
174
|
+
output,
|
|
175
|
+
stream,
|
|
176
|
+
model,
|
|
177
|
+
{
|
|
178
|
+
signal: options.signal,
|
|
179
|
+
grammarToolInputProperties: new Map(
|
|
180
|
+
(handle.data as { grammarTools?: [string, string][] } | undefined)?.grammarTools ?? [],
|
|
181
|
+
),
|
|
182
|
+
},
|
|
183
|
+
);
|
|
184
|
+
} finally {
|
|
185
|
+
stream.end(output);
|
|
186
|
+
}
|
|
187
|
+
output.providerPayload = createOpenAIResponsesHistoryPayload(model.provider, response.output);
|
|
188
|
+
return output;
|
|
189
|
+
}
|
|
190
|
+
export async function cancelDeferred(
|
|
191
|
+
model: Model,
|
|
192
|
+
handle: DeferredHandle,
|
|
193
|
+
options: OperationOptions = {},
|
|
194
|
+
): Promise<void> {
|
|
195
|
+
validateRoute(model, handle);
|
|
196
|
+
const response = await request(model, `/${encodeURIComponent(handle.id)}/cancel`, "POST", options);
|
|
197
|
+
if (response.id !== handle.id) throw new Error("Deferred cancellation identity does not match its handle");
|
|
198
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -3,10 +3,13 @@ export { Type } from "@sinclair/typebox";
|
|
|
3
3
|
export * from "./api-registry";
|
|
4
4
|
export * from "./auth-storage";
|
|
5
5
|
export * from "./codex-model-interaction";
|
|
6
|
+
export * from "./deferred";
|
|
6
7
|
export * from "./model-cache";
|
|
7
8
|
export * from "./model-manager";
|
|
8
9
|
export * from "./model-thinking";
|
|
9
10
|
export * from "./models";
|
|
11
|
+
export * from "./operation-catalog";
|
|
12
|
+
export * from "./operations";
|
|
10
13
|
export * from "./provider-details";
|
|
11
14
|
export * from "./provider-models";
|
|
12
15
|
export * from "./providers/anthropic";
|
package/src/model-cache.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { Database } from "bun:sqlite";
|
|
6
6
|
import { getModelDbPath } from "@f5-sales-demo/pi-utils";
|
|
7
|
-
import type { Api, Model } from "./types";
|
|
7
|
+
import type { AnyModel, Api, Model } from "./types";
|
|
8
8
|
|
|
9
9
|
const CACHE_SCHEMA_VERSION = 2;
|
|
10
10
|
|
|
@@ -95,3 +95,36 @@ export function writeModelCache<TApi extends Api>(
|
|
|
95
95
|
// Cache writes are best-effort; failures should not break model resolution.
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
|
+
|
|
99
|
+
/** Operation catalogs share atomic storage but use distinct rows from chat discovery. */
|
|
100
|
+
export function readOperationModelCache(provider: string, dbPath: string): AnyModel[] | undefined {
|
|
101
|
+
try {
|
|
102
|
+
const row = getDb(dbPath)
|
|
103
|
+
.query<CacheRow, [string]>("SELECT * FROM model_cache WHERE provider_id = ?")
|
|
104
|
+
.get(`operations:${provider}`);
|
|
105
|
+
if (!row || row.version !== CACHE_SCHEMA_VERSION) return undefined;
|
|
106
|
+
const models: unknown = JSON.parse(row.models);
|
|
107
|
+
if (
|
|
108
|
+
!Array.isArray(models) ||
|
|
109
|
+
!models.every(
|
|
110
|
+
model =>
|
|
111
|
+
model?.provider === provider &&
|
|
112
|
+
["image", "classifier"].includes(model.type) &&
|
|
113
|
+
typeof model.id === "string" &&
|
|
114
|
+
typeof model.api === "string" &&
|
|
115
|
+
typeof model.baseUrl === "string",
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
return undefined;
|
|
119
|
+
return models;
|
|
120
|
+
} catch {
|
|
121
|
+
return undefined;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function writeOperationModelCache(provider: string, models: readonly AnyModel[], dbPath: string): void {
|
|
126
|
+
getDb(dbPath).run(
|
|
127
|
+
"INSERT OR REPLACE INTO model_cache (provider_id, version, updated_at, authoritative, models) VALUES (?, ?, ?, ?, ?)",
|
|
128
|
+
[`operations:${provider}`, CACHE_SCHEMA_VERSION, Date.now(), 1, JSON.stringify(models)],
|
|
129
|
+
);
|
|
130
|
+
}
|
package/src/model-manager.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readModelCache, writeModelCache } from "./model-cache";
|
|
2
|
-
import { enrichModelThinking } from "./model-thinking";
|
|
2
|
+
import { applyGeneratedModelPolicies, enrichModelThinking } from "./model-thinking";
|
|
3
3
|
import { type GeneratedProvider, getBundledModels } from "./models";
|
|
4
4
|
import type { Api, Model, Provider } from "./types";
|
|
5
5
|
import { isRecord } from "./utils";
|
|
@@ -17,7 +17,7 @@ export type ModelRefreshStrategy = "online" | "offline" | "online-if-uncached";
|
|
|
17
17
|
*/
|
|
18
18
|
export interface ModelsDevFallback<TApi extends Api = Api, TPayload = unknown> {
|
|
19
19
|
/** Fetches raw fallback payload (for example from models.dev). */
|
|
20
|
-
fetch(): Promise<TPayload>;
|
|
20
|
+
fetch(signal?: AbortSignal): Promise<TPayload>;
|
|
21
21
|
/** Maps payload into provider models. */
|
|
22
22
|
map(payload: TPayload, providerId: Provider): readonly Model<TApi>[];
|
|
23
23
|
}
|
|
@@ -35,7 +35,7 @@ export interface ModelManagerOptions<TApi extends Api = Api, TModelsDevPayload =
|
|
|
35
35
|
/** Maximum cache age in milliseconds before considered stale. Default: 24h. */
|
|
36
36
|
cacheTtlMs?: number;
|
|
37
37
|
/** Optional dynamic endpoint fetcher. */
|
|
38
|
-
fetchDynamicModels?: () => Promise<readonly Model<TApi>[] | null>;
|
|
38
|
+
fetchDynamicModels?: (signal?: AbortSignal) => Promise<readonly Model<TApi>[] | null>;
|
|
39
39
|
/** Optional models.dev fallback hook. */
|
|
40
40
|
modelsDev?: ModelsDevFallback<TApi, TModelsDevPayload>;
|
|
41
41
|
/** Clock override for deterministic tests. */
|
|
@@ -59,7 +59,7 @@ export interface ModelResolutionResult<TApi extends Api = Api> {
|
|
|
59
59
|
* Stateful facade over provider model resolution.
|
|
60
60
|
*/
|
|
61
61
|
export interface ModelManager<TApi extends Api = Api> {
|
|
62
|
-
refresh(strategy?: ModelRefreshStrategy): Promise<ModelResolutionResult<TApi>>;
|
|
62
|
+
refresh(strategy?: ModelRefreshStrategy, options?: { signal?: AbortSignal }): Promise<ModelResolutionResult<TApi>>;
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
/**
|
|
@@ -68,9 +68,14 @@ export interface ModelManager<TApi extends Api = Api> {
|
|
|
68
68
|
export function createModelManager<TApi extends Api = Api, TModelsDevPayload = unknown>(
|
|
69
69
|
options: ModelManagerOptions<TApi, TModelsDevPayload>,
|
|
70
70
|
): ModelManager<TApi> {
|
|
71
|
+
let generation = 0;
|
|
71
72
|
return {
|
|
72
|
-
refresh(strategy: ModelRefreshStrategy = "online-if-uncached") {
|
|
73
|
-
|
|
73
|
+
refresh(strategy: ModelRefreshStrategy = "online-if-uncached", request?: { signal?: AbortSignal }) {
|
|
74
|
+
const revision = ++generation;
|
|
75
|
+
return resolveProviderModels(options, strategy, {
|
|
76
|
+
signal: request?.signal,
|
|
77
|
+
isCurrent: () => generation === revision,
|
|
78
|
+
});
|
|
74
79
|
},
|
|
75
80
|
};
|
|
76
81
|
}
|
|
@@ -84,7 +89,9 @@ export function createModelManager<TApi extends Api = Api, TModelsDevPayload = u
|
|
|
84
89
|
export async function resolveProviderModels<TApi extends Api = Api, TModelsDevPayload = unknown>(
|
|
85
90
|
options: ModelManagerOptions<TApi, TModelsDevPayload>,
|
|
86
91
|
strategy: ModelRefreshStrategy = "online-if-uncached",
|
|
92
|
+
request?: { signal?: AbortSignal; isCurrent?: () => boolean },
|
|
87
93
|
): Promise<ModelResolutionResult<TApi>> {
|
|
94
|
+
request?.signal?.throwIfAborted();
|
|
88
95
|
const now = options.now ?? Date.now;
|
|
89
96
|
const ttlMs = options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
|
|
90
97
|
const dbPath = options.cacheDbPath;
|
|
@@ -103,8 +110,13 @@ export async function resolveProviderModels<TApi extends Api = Api, TModelsDevPa
|
|
|
103
110
|
cacheAgeMs,
|
|
104
111
|
);
|
|
105
112
|
const [fetchedModelsDevModels, fetchedDynamicModels] = shouldFetchFromNetwork
|
|
106
|
-
? await Promise.all([
|
|
113
|
+
? await Promise.all([
|
|
114
|
+
fetchModelsDev(options, request?.signal),
|
|
115
|
+
dynamicFetcher ? fetchDynamicModels(() => dynamicFetcher(request?.signal)) : null,
|
|
116
|
+
])
|
|
107
117
|
: [null, null];
|
|
118
|
+
request?.signal?.throwIfAborted();
|
|
119
|
+
if (request?.isCurrent && !request.isCurrent()) throw new Error("Model refresh was superseded");
|
|
108
120
|
const modelsDevModels = normalizeModelList<TApi>(fetchedModelsDevModels ?? []);
|
|
109
121
|
const shouldUseFreshCacheAsAuthoritative =
|
|
110
122
|
strategy === "online-if-uncached" && (cache?.fresh ?? false) && hasAuthoritativeCache;
|
|
@@ -142,13 +154,14 @@ export async function resolveProviderModels<TApi extends Api = Api, TModelsDevPa
|
|
|
142
154
|
|
|
143
155
|
async function fetchModelsDev<TApi extends Api, TModelsDevPayload>(
|
|
144
156
|
options: ModelManagerOptions<TApi, TModelsDevPayload>,
|
|
157
|
+
signal?: AbortSignal,
|
|
145
158
|
): Promise<Model<TApi>[] | null> {
|
|
146
159
|
if (!options.modelsDev) {
|
|
147
160
|
return null;
|
|
148
161
|
}
|
|
149
162
|
|
|
150
163
|
try {
|
|
151
|
-
const payload = await options.modelsDev.fetch();
|
|
164
|
+
const payload = await options.modelsDev.fetch(signal);
|
|
152
165
|
return normalizeModelList<TApi>(options.modelsDev.map(payload, options.providerId));
|
|
153
166
|
} catch {
|
|
154
167
|
return null;
|
|
@@ -234,10 +247,12 @@ function mergeDynamicModel<TApi extends Api>(existingModel: Model<TApi>, dynamic
|
|
|
234
247
|
reasoning: existingModel.reasoning || dynamicModel.reasoning,
|
|
235
248
|
input: supportsImage ? ["text", "image"] : ["text"],
|
|
236
249
|
cost: {
|
|
250
|
+
pricingKnown: dynamicModel.cost.pricingKnown ?? existingModel.cost.pricingKnown,
|
|
237
251
|
input: preferDiscoveryCost(dynamicModel.cost.input, existingModel.cost.input),
|
|
238
252
|
output: preferDiscoveryCost(dynamicModel.cost.output, existingModel.cost.output),
|
|
239
253
|
cacheRead: preferDiscoveryCost(dynamicModel.cost.cacheRead, existingModel.cost.cacheRead),
|
|
240
254
|
cacheWrite: preferDiscoveryCost(dynamicModel.cost.cacheWrite, existingModel.cost.cacheWrite),
|
|
255
|
+
tiers: dynamicModel.cost.tiers ?? existingModel.cost.tiers,
|
|
241
256
|
},
|
|
242
257
|
contextWindow: preferDiscoveryLimit(dynamicModel.contextWindow, existingModel.contextWindow),
|
|
243
258
|
maxTokens: preferDiscoveryLimit(dynamicModel.maxTokens, existingModel.maxTokens),
|
|
@@ -282,7 +297,10 @@ function normalizeModelList<TApi extends Api>(value: unknown): Model<TApi>[] {
|
|
|
282
297
|
const models: Model<TApi>[] = [];
|
|
283
298
|
for (const item of value) {
|
|
284
299
|
if (isModelLike(item)) {
|
|
285
|
-
|
|
300
|
+
const model = enrichModelThinking({ ...item } as Model<TApi>);
|
|
301
|
+
const normalized = [model];
|
|
302
|
+
applyGeneratedModelPolicies(normalized, { preserveDiscoveredThinking: true });
|
|
303
|
+
models.push(normalized[0]!);
|
|
286
304
|
}
|
|
287
305
|
}
|
|
288
306
|
return models;
|
package/src/model-thinking.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { enrichCloudflareModel } from "./providers/cloudflare-route";
|
|
1
2
|
import { resolveOpenAICompat } from "./providers/openai-completions-compat";
|
|
3
|
+
import { SOL_API_COST, SOL_API_THINKING, SOL_MODEL_ID } from "./sol-model";
|
|
2
4
|
import type { Api, Model as ApiModel, ThinkingConfig } from "./types";
|
|
3
5
|
|
|
4
6
|
/**
|
|
@@ -210,6 +212,14 @@ export const CLOUDFLARE_FALLBACK_MODEL: ApiModel<"anthropic-messages"> = {
|
|
|
210
212
|
* trust `model.thinking` and avoid inferring capabilities on demand.
|
|
211
213
|
*/
|
|
212
214
|
export function enrichModelThinking<TApi extends Api>(model: ApiModel<TApi>): ApiModel<TApi> {
|
|
215
|
+
model = enrichCloudflareModel(model);
|
|
216
|
+
if (
|
|
217
|
+
model.provider === "baseten" &&
|
|
218
|
+
["zai-org/GLM-5.2", "zai-org/GLM-5.2-Fast", "moonshotai/Kimi-K2.6"].includes(model.id)
|
|
219
|
+
) {
|
|
220
|
+
model = { ...model };
|
|
221
|
+
applyGeneratedModelPolicy(model);
|
|
222
|
+
}
|
|
213
223
|
const normalizedThinking = normalizeThinkingConfig(model.thinking);
|
|
214
224
|
if (!model.reasoning) {
|
|
215
225
|
return normalizedThinking === undefined && model.thinking === undefined
|
|
@@ -253,11 +263,12 @@ export function applyGeneratedModelPolicies(
|
|
|
253
263
|
// Subscription discovery is authoritative for the effort enum. Re-inferring
|
|
254
264
|
// it from a model family can add values rejected by the backing model.
|
|
255
265
|
const model =
|
|
256
|
-
options.preserveDiscoveredThinking &&
|
|
266
|
+
options.preserveDiscoveredThinking &&
|
|
267
|
+
(source.api === "openai-codex-responses" || source.thinking !== undefined)
|
|
257
268
|
? enrichModelThinking(source)
|
|
258
269
|
: refreshModelThinking(source);
|
|
259
270
|
applyGeneratedModelPolicy(model);
|
|
260
|
-
models[index] = model;
|
|
271
|
+
models[index] = enrichCloudflareModel(model);
|
|
261
272
|
}
|
|
262
273
|
}
|
|
263
274
|
|
|
@@ -402,6 +413,66 @@ export function mapEffortToAnthropicAdaptiveEffort<TApi extends Api>(
|
|
|
402
413
|
}
|
|
403
414
|
|
|
404
415
|
function applyGeneratedModelPolicy(model: ApiModel<Api>): void {
|
|
416
|
+
if (model.provider === "fireworks") {
|
|
417
|
+
const completion = model.id.includes("glm-") || model.id.includes("kimi-k3");
|
|
418
|
+
model.api = completion ? "openai-completions" : "anthropic-messages";
|
|
419
|
+
model.baseUrl = model.baseUrl.replace(/\/v1\/?$/, "") + (completion ? "/v1" : "");
|
|
420
|
+
model.compat = {
|
|
421
|
+
...model.compat,
|
|
422
|
+
supportsStore: false,
|
|
423
|
+
supportsDeveloperRole: false,
|
|
424
|
+
sendSessionAffinityHeaders: true,
|
|
425
|
+
...(!completion ? { allowEmptySignature: true, supportsCacheControlOnTools: false } : {}),
|
|
426
|
+
} as ApiModel<Api>["compat"];
|
|
427
|
+
if (!completion && model.compat?.forceAdaptiveThinking && model.thinking)
|
|
428
|
+
model.thinking = { ...model.thinking, mode: "anthropic-adaptive" };
|
|
429
|
+
}
|
|
430
|
+
if (model.provider === "xai") model.api = "openai-responses";
|
|
431
|
+
if (model.provider === "xiaomi" && /api\.xiaomimimo\.com/.test(model.baseUrl)) {
|
|
432
|
+
model.api = "openai-completions";
|
|
433
|
+
model.baseUrl = model.baseUrl.replace(/\/anthropic\/?$/, "/v1");
|
|
434
|
+
}
|
|
435
|
+
if (
|
|
436
|
+
model.provider === "baseten" &&
|
|
437
|
+
["zai-org/GLM-5.2", "zai-org/GLM-5.2-Fast", "moonshotai/Kimi-K2.6"].includes(model.id)
|
|
438
|
+
) {
|
|
439
|
+
const glm = model.id.startsWith("zai-org/GLM-5.2");
|
|
440
|
+
model.compat = {
|
|
441
|
+
...model.compat,
|
|
442
|
+
thinkingFormat: "baseten",
|
|
443
|
+
supportsReasoningEffort: glm,
|
|
444
|
+
supportsStore: false,
|
|
445
|
+
supportsDeveloperRole: false,
|
|
446
|
+
maxTokensField: "max_tokens",
|
|
447
|
+
sendSessionAffinityHeaders: true,
|
|
448
|
+
chatTemplateArgs: { enable_thinking: { $var: "thinking.enabled" } },
|
|
449
|
+
} as ApiModel<Api>["compat"];
|
|
450
|
+
model.thinking = {
|
|
451
|
+
mode: "effort",
|
|
452
|
+
defaultLevel: "high",
|
|
453
|
+
supportedLevels: (glm ? [Effort.High, Effort.Max] : [Effort.High]).map(effort => ({
|
|
454
|
+
effort,
|
|
455
|
+
description: EFFORT_DESCRIPTIONS[effort],
|
|
456
|
+
})),
|
|
457
|
+
};
|
|
458
|
+
if (glm) model.input = ["text"];
|
|
459
|
+
}
|
|
460
|
+
if (model.provider === "ant-ling" && model.id.startsWith("Ring-")) {
|
|
461
|
+
model.thinking = {
|
|
462
|
+
mode: "effort",
|
|
463
|
+
defaultLevel: "high",
|
|
464
|
+
supportedLevels: [
|
|
465
|
+
{ effort: "high", description: "Deep reasoning" },
|
|
466
|
+
{ effort: "xhigh", description: "Very deep reasoning" },
|
|
467
|
+
],
|
|
468
|
+
};
|
|
469
|
+
model.compat = {
|
|
470
|
+
...model.compat,
|
|
471
|
+
thinkingFormat: "ant-ling",
|
|
472
|
+
supportsReasoningEffort: false,
|
|
473
|
+
} as ApiModel<Api>["compat"];
|
|
474
|
+
}
|
|
475
|
+
if (model.provider === "mistral") model.api = "mistral-conversations";
|
|
405
476
|
const parsedModel = parseKnownModel(model.id);
|
|
406
477
|
if (parsedModel.family === "anthropic") {
|
|
407
478
|
applyAnthropicCatalogPolicy(model, parsedModel);
|
|
@@ -435,6 +506,26 @@ function applyAnthropicCatalogPolicy(model: ApiModel<Api>, parsedModel: Anthropi
|
|
|
435
506
|
}
|
|
436
507
|
|
|
437
508
|
function applyOpenAICatalogPolicy(model: ApiModel<Api>, parsedModel: OpenAIModel): void {
|
|
509
|
+
if (model.id === SOL_MODEL_ID) {
|
|
510
|
+
model.name = "GPT-6.1 Sol";
|
|
511
|
+
if (model.provider === "openai")
|
|
512
|
+
model.compat = {
|
|
513
|
+
supportsExplicitPromptCacheMode: true,
|
|
514
|
+
supportsAdditionalTools: true,
|
|
515
|
+
supportsToolSearch: true,
|
|
516
|
+
supportsOpenAIGrammarTools: true,
|
|
517
|
+
supportsCachedReasoningUpdates: true,
|
|
518
|
+
...model.compat,
|
|
519
|
+
} as ApiModel<Api>["compat"];
|
|
520
|
+
model.input = ["text", "image"];
|
|
521
|
+
model.maxInputTokens = Math.min(model.maxInputTokens ?? 922_000, 922_000);
|
|
522
|
+
model.maxTokens = Math.min(model.maxTokens, 128_000);
|
|
523
|
+
if (model.provider === "openai") {
|
|
524
|
+
model.api = "openai-responses";
|
|
525
|
+
model.contextWindow = 1_050_000;
|
|
526
|
+
model.cost = SOL_API_COST;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
438
529
|
// The ChatGPT Codex subscription transport defaults to the lower-cost
|
|
439
530
|
// short-context tier. The coding-agent registry can opt Luna and Sol into
|
|
440
531
|
// their full published window without changing generated catalog defaults.
|
|
@@ -549,6 +640,7 @@ function inferOpenAISupportedEfforts<TApi extends Api>(
|
|
|
549
640
|
model: OpenAIModel,
|
|
550
641
|
catalogModel: ApiModel<TApi>,
|
|
551
642
|
): readonly ReasoningEffort[] {
|
|
643
|
+
if (catalogModel.id === SOL_MODEL_ID) return SOL_API_THINKING.supportedLevels.map(level => level.effort);
|
|
552
644
|
if (semverGte(model.version, "5.6") && catalogModel.api === "openai-codex-responses") {
|
|
553
645
|
return [ReasoningEffort.None, Effort.Low, Effort.Medium, Effort.High, Effort.XHigh, Effort.Max];
|
|
554
646
|
}
|