@xandreed/plugin-models 0.4.0 → 0.8.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/auth/anthropicOAuth.js +1 -1
- package/dist/auth/localAuth.js +1 -1
- package/dist/auth/openAiCodexOAuth.js +3 -3
- package/dist/llm/compat.d.ts +7 -3
- package/dist/llm/compat.js +46 -60
- package/dist/llm/openAiCodex.d.ts +4 -8
- package/dist/llm/openAiCodex.js +31 -21
- package/dist/llm/openAiCodexWebSocket.d.ts +1 -1
- package/dist/llm/openAiCodexWebSocket.js +12 -16
- package/dist/llm/providers.d.ts +10 -6
- package/dist/llm/providers.js +37 -29
- package/dist/llm/retry.d.ts +4 -4
- package/dist/llm/retry.js +25 -28
- package/dist/llm/router.d.ts +2 -2
- package/dist/llm/router.js +53 -37
- package/dist/llm/sse.d.ts +12 -7
- package/dist/llm/sse.js +18 -16
- package/dist/llm/utilityLlm.js +6 -14
- package/dist/plugin.adapter.d.ts +13 -1
- package/dist/plugin.adapter.js +3 -3
- package/package.json +5 -8
package/README.md
CHANGED
|
@@ -63,7 +63,7 @@ export const generatePkce = Effect.promise(async () => {
|
|
|
63
63
|
code_verifier: verifier,
|
|
64
64
|
state: verifier
|
|
65
65
|
}, "code exchange");
|
|
66
|
-
const fromNullableTrimmed = (value) => Option.filter(Option.
|
|
66
|
+
const fromNullableTrimmed = (value) => Option.filter(Option.fromNullishOr(value), (v) => v.trim().length > 0);
|
|
67
67
|
export const parseAuthorizationInput = (input) => {
|
|
68
68
|
const value = input.trim();
|
|
69
69
|
if (value.length === 0)
|
package/dist/auth/localAuth.js
CHANGED
|
@@ -55,7 +55,7 @@ export const authPaths = (cwd, home, directory = ".efferent") => [
|
|
|
55
55
|
}),
|
|
56
56
|
get: (provider) => Effect.gen(function* () {
|
|
57
57
|
const files = yield* Effect.forEach(paths, readAuthFile), merged = new Map(files.flatMap((f) => [...f.entries]));
|
|
58
|
-
return Option.
|
|
58
|
+
return Option.fromNullishOr(merged.get(provider));
|
|
59
59
|
}),
|
|
60
60
|
resolveKey: (provider) => Effect.gen(function* () {
|
|
61
61
|
const holder = [...yield* Effect.forEach(paths, readAuthFile)].reverse().find((f) => f.entries.has(provider)), cred = holder?.entries.get(provider);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Effect,
|
|
1
|
+
import { Effect, Result, Option } from "effect";
|
|
2
2
|
import { AuthError } from "@xandreed/core";
|
|
3
3
|
import { generatePkce } from "./anthropicOAuth.js";
|
|
4
4
|
export const OPENAI_CODEX_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann";
|
|
@@ -10,12 +10,12 @@ const SCOPE = "openid profile email offline_access", ACCOUNT_CLAIM = "https://ap
|
|
|
10
10
|
};
|
|
11
11
|
export const openAiCodexAccountId = (accessToken) => {
|
|
12
12
|
const segment = accessToken.split(".")[1];
|
|
13
|
-
return Option.flatMap(Option.
|
|
13
|
+
return Option.flatMap(Option.fromNullishOr(segment), (encoded) => Option.flatMap(Result.getSuccess(Result.try(() => base64UrlJson(encoded))), (payload) => {
|
|
14
14
|
const claim = payload[ACCOUNT_CLAIM];
|
|
15
15
|
if (typeof claim !== "object" || claim === null)
|
|
16
16
|
return Option.none();
|
|
17
17
|
const accountId = claim.chatgpt_account_id;
|
|
18
|
-
return Option.filter(Option.
|
|
18
|
+
return Option.filter(Option.fromNullishOr(typeof accountId === "string" ? accountId : void 0), (value) => value.length > 0);
|
|
19
19
|
}));
|
|
20
20
|
};
|
|
21
21
|
const postToken = (body, stage) => Effect.tryPromise({
|
package/dist/llm/compat.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AiError, LanguageModel, Tool } from "
|
|
2
|
-
import type { Prompt } from "
|
|
1
|
+
import { AiError, LanguageModel, Tool } from "effect/ai";
|
|
2
|
+
import type { Prompt } from "effect/ai";
|
|
3
3
|
import { Effect } from "effect";
|
|
4
4
|
import type { CompletionUsage } from "./sse.js";
|
|
5
5
|
/**
|
|
@@ -28,6 +28,10 @@ export interface CompatConfig {
|
|
|
28
28
|
/** Gateway reasoning vocabulary, independent of provider-native thinking. */
|
|
29
29
|
readonly reasoningEffort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
|
|
30
30
|
readonly temperature?: number;
|
|
31
|
+
/** Send standard chat-completions fields only — no prompt-cache key,
|
|
32
|
+
* thinking defaults or call policy — for endpoints that reject the
|
|
33
|
+
* gateway extensions (Gemini's OpenAI-compatible API). */
|
|
34
|
+
readonly standardOnly?: boolean;
|
|
31
35
|
/** Injectable for tests; defaults to global fetch. */
|
|
32
36
|
readonly fetchImpl?: typeof fetch;
|
|
33
37
|
/** Host-owned admission (e.g. reserve a worst-case cost) before every call. */
|
|
@@ -73,5 +77,5 @@ interface ChatCompletion {
|
|
|
73
77
|
}
|
|
74
78
|
/** Parse one non-streaming completion into `@effect/ai` encoded parts. */
|
|
75
79
|
export declare const fromChatCompletion: (moduleName: string, body: ChatCompletion) => Effect.Effect<ReadonlyArray<unknown>, AiError.AiError>;
|
|
76
|
-
export declare const makeCompatLanguageModel: (config: CompatConfig) => Effect.Effect<LanguageModel.
|
|
80
|
+
export declare const makeCompatLanguageModel: (config: CompatConfig) => Effect.Effect<LanguageModel.LanguageModel>;
|
|
77
81
|
export {};
|
package/dist/llm/compat.js
CHANGED
|
@@ -1,32 +1,28 @@
|
|
|
1
|
-
import { AiError, LanguageModel, Tool } from "
|
|
2
|
-
import { Effect,
|
|
3
|
-
import { CurrentModelCallPolicy, CurrentPromptCacheKey } from "@xandreed/core";
|
|
1
|
+
import { AiError, LanguageModel, Tool } from "effect/ai";
|
|
2
|
+
import { Effect, Result, Option, Stream } from "effect";
|
|
3
|
+
import { CurrentModelCallPolicy, CurrentPromptCacheKey, describeModel, strictJsonSchema, toolParametersSchema } from "@xandreed/core";
|
|
4
4
|
import { finishReasonFromWire, sseStreamParts, usageFromCompletion } from "./sse.js";
|
|
5
|
-
const record = (value) => typeof value === "object" && value !== null ? value : {}, semanticGatewayError = (moduleName, method, body) => Option.flatMap(Option.
|
|
5
|
+
const record = (value) => typeof value === "object" && value !== null ? value : {}, semanticGatewayError = (moduleName, method, body) => Option.flatMap(Option.fromNullishOr(Result.getOrUndefined(Result.try(() => JSON.parse(body)))), (decoded) => {
|
|
6
6
|
const error = record(record(decoded).error), type = typeof error.type === "string" ? error.type : "", message = typeof error.message === "string" ? error.message : "";
|
|
7
7
|
if (type === "ModelError" || /model .*not supported/i.test(message))
|
|
8
|
-
return Option.some(
|
|
8
|
+
return Option.some(AiError.make({
|
|
9
9
|
module: moduleName,
|
|
10
10
|
method,
|
|
11
|
-
|
|
11
|
+
reason: new AiError.InvalidRequestError({
|
|
12
|
+
description: message.length > 0 ? message : "the selected model is not supported"
|
|
13
|
+
})
|
|
12
14
|
}));
|
|
13
15
|
if (type === "CreditsError" || /insufficient (?:balance|credits)|usage limit|quota/i.test(message))
|
|
14
|
-
return Option.some(
|
|
16
|
+
return Option.some(AiError.make({
|
|
15
17
|
module: moduleName,
|
|
16
18
|
method,
|
|
17
|
-
description: `CreditsError: ${message || "provider quota exhausted"}`
|
|
19
|
+
reason: new AiError.UnknownError({ description: `CreditsError: ${message || "provider quota exhausted"}` })
|
|
18
20
|
}));
|
|
19
21
|
return Option.none();
|
|
20
|
-
}), aiUnknown = (moduleName, method, e) => new AiError.UnknownError({
|
|
21
|
-
module: moduleName,
|
|
22
|
-
method,
|
|
23
|
-
description: String(e),
|
|
24
|
-
cause: e
|
|
25
|
-
}), requestInfo = (chatUrl) => ({
|
|
22
|
+
}), aiUnknown = (moduleName, method, e) => AiError.make({ module: moduleName, method, reason: new AiError.UnknownError({ description: String(e) }) }), invalidOutput = (moduleName, method, description) => AiError.make({ module: moduleName, method, reason: new AiError.InvalidOutputError({ description }) }), requestInfo = (chatUrl) => ({
|
|
26
23
|
method: "POST",
|
|
27
24
|
url: chatUrl,
|
|
28
25
|
urlParams: [],
|
|
29
|
-
hash: Option.none(),
|
|
30
26
|
headers: { "content-type": "application/json" }
|
|
31
27
|
});
|
|
32
28
|
export const toChatMessages = (prompt) => prompt.content.flatMap((message) => {
|
|
@@ -63,7 +59,7 @@ export const toChatMessages = (prompt) => prompt.content.flatMap((message) => {
|
|
|
63
59
|
function: {
|
|
64
60
|
name: tool.name,
|
|
65
61
|
description: Tool.getDescription(tool),
|
|
66
|
-
parameters:
|
|
62
|
+
parameters: toolParametersSchema(tool)
|
|
67
63
|
}
|
|
68
64
|
}
|
|
69
65
|
] : []);
|
|
@@ -88,11 +84,7 @@ export const thinkingParams = (model) => {
|
|
|
88
84
|
}, fromChatCompletion = (moduleName, body) => Effect.gen(function* () {
|
|
89
85
|
const choice = body.choices?.[0];
|
|
90
86
|
if (choice === void 0)
|
|
91
|
-
return yield* Effect.fail(
|
|
92
|
-
module: moduleName,
|
|
93
|
-
method: "generateText",
|
|
94
|
-
description: "the completion carried no choices"
|
|
95
|
-
}));
|
|
87
|
+
return yield* Effect.fail(invalidOutput(moduleName, "generateText", "the completion carried no choices"));
|
|
96
88
|
const message = choice.message ?? {}, reasoning = message.reasoning_content ?? message.reasoning, text = message.content, toolCalls = yield* Effect.forEach(message.tool_calls ?? [], (tc) => Effect.try({
|
|
97
89
|
try: () => ({
|
|
98
90
|
type: "tool-call",
|
|
@@ -100,11 +92,7 @@ export const thinkingParams = (model) => {
|
|
|
100
92
|
name: tc.function?.name ?? "",
|
|
101
93
|
params: JSON.parse(tc.function?.arguments ?? "{}")
|
|
102
94
|
}),
|
|
103
|
-
catch: () =>
|
|
104
|
-
module: moduleName,
|
|
105
|
-
method: "generateText",
|
|
106
|
-
description: `tool call ${tc.function?.name ?? "?"} carried unparseable JSON arguments`
|
|
107
|
-
})
|
|
95
|
+
catch: () => invalidOutput(moduleName, "generateText", `tool call ${tc.function?.name ?? "?"} carried unparseable JSON arguments`)
|
|
108
96
|
}));
|
|
109
97
|
return [
|
|
110
98
|
...reasoning !== null && reasoning !== void 0 && reasoning.length > 0 ? [{ type: "reasoning", text: reasoning }] : [],
|
|
@@ -117,22 +105,10 @@ export const thinkingParams = (model) => {
|
|
|
117
105
|
}
|
|
118
106
|
];
|
|
119
107
|
});
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
...config.temperature === void 0 ? {} : { temperature: config.temperature },
|
|
125
|
-
messages: toChatMessages(options.prompt),
|
|
126
|
-
stream: streaming,
|
|
127
|
-
...options.responseFormat.type === "json" ? { response_format: {
|
|
128
|
-
type: "json_schema",
|
|
129
|
-
json_schema: { name: options.responseFormat.objectName, strict: !0, schema: JSONSchema.make(options.responseFormat.schema) }
|
|
130
|
-
} } : {},
|
|
131
|
-
...streaming ? { stream_options: { include_usage: !0 } } : {},
|
|
132
|
-
...Option.match(cacheKey, {
|
|
133
|
-
onNone: () => ({}),
|
|
134
|
-
onSome: (key) => ({ prompt_cache_key: key })
|
|
135
|
-
}),
|
|
108
|
+
const requestSettings = (config, cacheKey, policy) => ({
|
|
109
|
+
sampling: config.temperature === void 0 ? {} : { temperature: config.temperature },
|
|
110
|
+
extensions: config.standardOnly === !0 ? {} : {
|
|
111
|
+
...Option.match(cacheKey, { onNone: () => ({}), onSome: (key) => ({ prompt_cache_key: key }) }),
|
|
136
112
|
...config.thinking === void 0 ? thinkingParams(config.model) : { thinking: { type: config.thinking } },
|
|
137
113
|
...config.reasoningEffort === void 0 ? {} : { reasoning: { effort: config.reasoningEffort } },
|
|
138
114
|
...Option.match(policy, {
|
|
@@ -141,11 +117,25 @@ const chatRequestBody = (config, options, streaming) => Effect.all({ cacheKey: F
|
|
|
141
117
|
...value.maxOutputTokens === void 0 ? {} : { max_tokens: value.maxOutputTokens },
|
|
142
118
|
...config.thinking === "disabled" || config.reasoningEffort !== void 0 ? {} : { reasoning_effort: value.effort }
|
|
143
119
|
})
|
|
144
|
-
})
|
|
120
|
+
})
|
|
121
|
+
}
|
|
122
|
+
}), chatRequestBody = (config, options, streaming) => Effect.all({ cacheKey: Effect.service(CurrentPromptCacheKey), policy: Effect.service(CurrentModelCallPolicy) }).pipe(Effect.map(({ cacheKey, policy }) => {
|
|
123
|
+
const tools = toChatTools(options.tools), settings = requestSettings(config, cacheKey, policy);
|
|
124
|
+
return {
|
|
125
|
+
model: config.model,
|
|
126
|
+
...settings.sampling,
|
|
127
|
+
messages: toChatMessages(options.prompt),
|
|
128
|
+
stream: streaming,
|
|
129
|
+
...options.responseFormat.type === "json" ? { response_format: {
|
|
130
|
+
type: "json_schema",
|
|
131
|
+
json_schema: { name: options.responseFormat.objectName, strict: !0, schema: strictJsonSchema(options.responseFormat.schema) }
|
|
132
|
+
} } : {},
|
|
133
|
+
...streaming ? { stream_options: { include_usage: !0 } } : {},
|
|
134
|
+
...settings.extensions,
|
|
145
135
|
...tools.length > 0 ? { tools, tool_choice: toToolChoice(options.toolChoice) } : {}
|
|
146
136
|
};
|
|
147
137
|
})), postChat = (config, method, body, buffered = !1) => {
|
|
148
|
-
const request = (config.beforeRequest?.(body) ?? Effect.void).pipe(Effect.
|
|
138
|
+
const request = (config.beforeRequest?.(body) ?? Effect.void).pipe(Effect.andThen(Effect.tryPromise({
|
|
149
139
|
try: async (signal) => {
|
|
150
140
|
const response = await (config.fetchImpl ?? fetch)(config.chatUrl, {
|
|
151
141
|
signal,
|
|
@@ -172,13 +162,14 @@ const chatRequestBody = (config, options, streaming) => Effect.all({ cacheKey: F
|
|
|
172
162
|
res.headers.forEach((value, headerName) => {
|
|
173
163
|
headers[headerName] = value;
|
|
174
164
|
});
|
|
175
|
-
return yield* Effect.fail(
|
|
165
|
+
return yield* Effect.fail(AiError.make({
|
|
176
166
|
module: config.moduleName,
|
|
177
167
|
method,
|
|
178
|
-
reason:
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
168
|
+
reason: AiError.reasonFromHttpStatus({
|
|
169
|
+
status: res.status,
|
|
170
|
+
http: { request: requestInfo(config.chatUrl), response: { status: res.status, headers }, body: text.slice(0, 500) },
|
|
171
|
+
description: text.slice(0, 500)
|
|
172
|
+
})
|
|
182
173
|
}));
|
|
183
174
|
});
|
|
184
175
|
export const makeCompatLanguageModel = (config) => LanguageModel.make({
|
|
@@ -191,11 +182,7 @@ export const makeCompatLanguageModel = (config) => LanguageModel.make({
|
|
|
191
182
|
catch: (e) => aiUnknown(config.moduleName, "generateText", e)
|
|
192
183
|
}), parsed = yield* Effect.try({
|
|
193
184
|
try: () => JSON.parse(text),
|
|
194
|
-
catch: () =>
|
|
195
|
-
module: config.moduleName,
|
|
196
|
-
method: "generateText",
|
|
197
|
-
description: `the completion body was not JSON: ${text.slice(0, 200)}`
|
|
198
|
-
})
|
|
185
|
+
catch: () => invalidOutput(config.moduleName, "generateText", `the completion body was not JSON: ${text.slice(0, 200)}`)
|
|
199
186
|
});
|
|
200
187
|
return yield* fromChatCompletion(config.moduleName, parsed);
|
|
201
188
|
}),
|
|
@@ -205,11 +192,10 @@ export const makeCompatLanguageModel = (config) => LanguageModel.make({
|
|
|
205
192
|
return yield* failStatus(config, "streamText", res);
|
|
206
193
|
const body = res.body;
|
|
207
194
|
if (body === null)
|
|
208
|
-
return yield* Effect.fail(
|
|
209
|
-
module: config.moduleName,
|
|
210
|
-
method: "streamText",
|
|
211
|
-
description: "the streaming response carried no body"
|
|
212
|
-
}));
|
|
195
|
+
return yield* Effect.fail(invalidOutput(config.moduleName, "streamText", "the streaming response carried no body"));
|
|
213
196
|
return sseStreamParts({ moduleName: config.moduleName, body });
|
|
214
197
|
}))
|
|
215
|
-
})
|
|
198
|
+
}).pipe(Effect.map((model) => describeModel(model, Effect.all({ cacheKey: Effect.service(CurrentPromptCacheKey), policy: Effect.service(CurrentModelCallPolicy) }).pipe(Effect.map(({ cacheKey, policy }) => {
|
|
199
|
+
const settings = requestSettings(config, cacheKey, policy);
|
|
200
|
+
return { provider: config.moduleName, model: config.model, settings: { ...settings.sampling, ...settings.extensions } };
|
|
201
|
+
})))));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { LanguageModel } from "
|
|
2
|
-
import { HttpClient } from "
|
|
3
|
-
import { Effect, Redacted
|
|
1
|
+
import { LanguageModel } from "effect/ai";
|
|
2
|
+
import { HttpClient } from "effect/http";
|
|
3
|
+
import { Effect, Redacted } from "effect";
|
|
4
4
|
export declare const OPENAI_CODEX_API_URL = "https://chatgpt.com/backend-api/codex";
|
|
5
5
|
type Json = Record<string, unknown>;
|
|
6
6
|
/** Translate the standard Responses request into the Codex subscription dialect. */
|
|
@@ -10,9 +10,5 @@ export declare const makeOpenAiCodexLanguageModel: (args: {
|
|
|
10
10
|
readonly model: string;
|
|
11
11
|
readonly accessToken: Redacted.Redacted<string>;
|
|
12
12
|
readonly accountId: string;
|
|
13
|
-
}) => Effect.Effect<
|
|
14
|
-
streamText: <Options extends import("effect/Types").NoExcessProperties<LanguageModel.GenerateTextOptions<any>, Options>, Tools extends Record<string, import("@effect/ai/Tool").Any> = {}>(options: Options & LanguageModel.GenerateTextOptions<Tools>) => Stream.Stream<import("@effect/ai/Response").StreamPart<Tools>, LanguageModel.ExtractError<Options>, LanguageModel.ExtractContext<Options>>;
|
|
15
|
-
generateText: LanguageModel.Service["generateText"];
|
|
16
|
-
generateObject: <A, I extends Record<string, unknown>, R, Options extends import("effect/Types").NoExcessProperties<LanguageModel.GenerateObjectOptions<any, A, I, R>, Options>, Tools extends Record<string, import("@effect/ai/Tool").Any> = {}>(options: Options & LanguageModel.GenerateObjectOptions<Tools, A, I, R>) => Effect.Effect<LanguageModel.GenerateObjectResponse<Tools, A>, LanguageModel.ExtractError<Options>, R | LanguageModel.ExtractContext<Options>>;
|
|
17
|
-
}, never, import("effect/Scope").Scope | HttpClient.HttpClient>;
|
|
13
|
+
}) => Effect.Effect<LanguageModel.LanguageModel, never, HttpClient.HttpClient>;
|
|
18
14
|
export {};
|
package/dist/llm/openAiCodex.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { AiError, LanguageModel } from "
|
|
1
|
+
import { AiError, LanguageModel } from "effect/ai";
|
|
2
2
|
import { OpenAiClient, OpenAiLanguageModel } from "@effect/ai-openai";
|
|
3
|
-
import { HttpClient, HttpClientRequest } from "
|
|
4
|
-
import { Effect,
|
|
5
|
-
import { CurrentModelCallPolicy, CurrentPromptCacheKey, foldStreamParts } from "@xandreed/core";
|
|
3
|
+
import { HttpClient, HttpClientRequest } from "effect/http";
|
|
4
|
+
import { Effect, Result, Option, Redacted, Stream } from "effect";
|
|
5
|
+
import { CurrentModelCallPolicy, CurrentPromptCacheKey, describeModel, foldStreamParts } from "@xandreed/core";
|
|
6
6
|
import { OpenAiCodexWebSocketHttpClient } from "./openAiCodexWebSocket.js";
|
|
7
7
|
export const OPENAI_CODEX_API_URL = "https://chatgpt.com/backend-api/codex";
|
|
8
8
|
const record = (value) => typeof value === "object" && value !== null ? value : {}, instructionText = (message) => {
|
|
@@ -46,27 +46,26 @@ const transformRequest = (accountId, reasoningEffort) => (request) => {
|
|
|
46
46
|
if (request.body._tag !== "Uint8Array")
|
|
47
47
|
return withHeaders;
|
|
48
48
|
const encodedBody = request.body.body;
|
|
49
|
-
return Option.match(
|
|
49
|
+
return Option.match(Result.getSuccess(Result.try(() => JSON.parse(new TextDecoder().decode(encodedBody)))), {
|
|
50
50
|
onNone: () => withHeaders,
|
|
51
|
-
onSome: (decoded) => HttpClientRequest.
|
|
51
|
+
onSome: (decoded) => HttpClientRequest.bodyJsonUnsafe(withHeaders, toOpenAiCodexRequestBody(decoded, reasoningEffort))
|
|
52
52
|
});
|
|
53
|
-
}, responseStatus = (error) =>
|
|
54
|
-
|
|
55
|
-
return typeof response.status === "number" ? response.status : void 0;
|
|
56
|
-
};
|
|
57
|
-
export const mapOpenAiCodexError = (model, error) => responseStatus(error) === 404 ? new AiError.MalformedInput({
|
|
53
|
+
}, responseStatus = (error) => AiError.isAiError(error) && ("http" in error.reason) ? error.reason.http?.response?.status : void 0;
|
|
54
|
+
export const mapOpenAiCodexError = (model, error) => responseStatus(error) === 404 ? AiError.make({
|
|
58
55
|
module: "OpenAiCodex",
|
|
59
56
|
method: "responses",
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
})
|
|
57
|
+
reason: new AiError.InvalidRequestError({
|
|
58
|
+
description: `model ${model} is not available to the configured ChatGPT subscription account`
|
|
59
|
+
})
|
|
60
|
+
}) : responseStatus(error) === 200 ? AiError.make({
|
|
63
61
|
module: "OpenAiCodex",
|
|
64
62
|
method: "responses",
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
reason: new AiError.InvalidOutputError({
|
|
64
|
+
description: `the ${model} subscription stream returned a rejected or incompatible event`
|
|
65
|
+
})
|
|
67
66
|
}) : error, makeOpenAiCodexLanguageModel = (args) => Effect.all({
|
|
68
|
-
cacheKey:
|
|
69
|
-
policy:
|
|
67
|
+
cacheKey: Effect.service(CurrentPromptCacheKey),
|
|
68
|
+
policy: Effect.service(CurrentModelCallPolicy)
|
|
70
69
|
}).pipe(Effect.flatMap(({ cacheKey, policy }) => OpenAiClient.make({
|
|
71
70
|
apiKey: args.accessToken,
|
|
72
71
|
apiUrl: OPENAI_CODEX_API_URL,
|
|
@@ -74,7 +73,7 @@ export const mapOpenAiCodexError = (model, error) => responseStatus(error) === 4
|
|
|
74
73
|
}).pipe(Effect.flatMap((client) => OpenAiLanguageModel.make({
|
|
75
74
|
model: args.model,
|
|
76
75
|
config: {
|
|
77
|
-
|
|
76
|
+
strictJsonSchema: !1,
|
|
78
77
|
store: !1,
|
|
79
78
|
prompt_cache_key: Option.getOrElse(cacheKey, () => "efferent"),
|
|
80
79
|
text: { verbosity: "low" },
|
|
@@ -90,10 +89,21 @@ export const mapOpenAiCodexError = (model, error) => responseStatus(error) === 4
|
|
|
90
89
|
})
|
|
91
90
|
}
|
|
92
91
|
}).pipe(Effect.provideService(OpenAiClient.OpenAiClient, client))), Effect.map((streaming) => {
|
|
93
|
-
const streamText = (options) => streaming.streamText(options).pipe(Stream.mapError((error) => mapOpenAiCodexError(args.model, error)))
|
|
94
|
-
return {
|
|
92
|
+
const streamText = (options) => streaming.streamText(options).pipe(Stream.mapError((error) => mapOpenAiCodexError(args.model, error))), model = {
|
|
95
93
|
...streaming,
|
|
96
94
|
streamText,
|
|
97
95
|
generateText: (options) => foldStreamParts(streamText(options), () => Effect.void).pipe(Effect.map((turn) => new LanguageModel.GenerateTextResponse(turn.content)))
|
|
98
96
|
};
|
|
97
|
+
return describeModel(model, {
|
|
98
|
+
provider: "openai-codex",
|
|
99
|
+
model: args.model,
|
|
100
|
+
settings: {
|
|
101
|
+
strictJsonSchema: !1,
|
|
102
|
+
store: !1,
|
|
103
|
+
stream: !0,
|
|
104
|
+
prompt_cache_key: Option.getOrElse(cacheKey, () => "efferent"),
|
|
105
|
+
text: { verbosity: "low" },
|
|
106
|
+
...Option.match(policy, { onNone: () => ({}), onSome: (value) => ({ reasoning: { effort: value.effort, summary: "auto" } }) })
|
|
107
|
+
}
|
|
108
|
+
});
|
|
99
109
|
}))));
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { HttpClient } from "
|
|
1
|
+
import { HttpClient } from "effect/http";
|
|
2
2
|
/** ChatGPT's Luna router currently keys its internal route on Pi's
|
|
3
3
|
* time-ordered UUIDv7 session shape; UUIDv4 reaches a nonexistent rollout
|
|
4
4
|
* alias. Keep one v7 id per Efferent conversation/cache key. */
|
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
HttpClientError,
|
|
4
|
-
HttpClientRequest,
|
|
5
|
-
HttpClientResponse
|
|
6
|
-
} from "@effect/platform";
|
|
7
|
-
import { Effect, Either } from "effect";
|
|
1
|
+
import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
|
|
2
|
+
import { Effect, Result } from "effect";
|
|
8
3
|
import { release } from "node:os";
|
|
9
4
|
import WebSocket from "ws";
|
|
10
5
|
const WEBSOCKET_BETA = "responses_websockets=2026-02-06", sessionIds = new Map, decodeBody = (request) => {
|
|
@@ -78,17 +73,17 @@ export const normalizeOpenAiCodexWebSocketEvent = (value) => {
|
|
|
78
73
|
}, abort = () => fail(Error("OpenAI subscription request aborted"));
|
|
79
74
|
signal.addEventListener("abort", abort, { once: !0 });
|
|
80
75
|
socket.addEventListener("open", () => {
|
|
81
|
-
|
|
76
|
+
Result.match(Result.try(() => {
|
|
82
77
|
socket.send(JSON.stringify({ type: "response.create", ...body }));
|
|
83
78
|
state.settled = !0;
|
|
84
79
|
return HttpClientResponse.fromWeb(safeRequest, new Response(stream, {
|
|
85
80
|
status: 200,
|
|
86
81
|
headers: { "content-type": "text/event-stream" }
|
|
87
82
|
}));
|
|
88
|
-
}), {
|
|
83
|
+
}), { onFailure: fail, onSuccess: resolve });
|
|
89
84
|
});
|
|
90
85
|
socket.addEventListener("message", (event) => {
|
|
91
|
-
|
|
86
|
+
Result.match(Result.try(() => {
|
|
92
87
|
const text = eventText(event.data);
|
|
93
88
|
if (text === void 0)
|
|
94
89
|
return;
|
|
@@ -107,7 +102,7 @@ export const normalizeOpenAiCodexWebSocketEvent = (value) => {
|
|
|
107
102
|
signal.removeEventListener("abort", abort);
|
|
108
103
|
socket.close(1000, "done");
|
|
109
104
|
}
|
|
110
|
-
}), {
|
|
105
|
+
}), { onFailure: fail, onSuccess: () => {
|
|
111
106
|
return;
|
|
112
107
|
} });
|
|
113
108
|
});
|
|
@@ -117,10 +112,11 @@ export const normalizeOpenAiCodexWebSocketEvent = (value) => {
|
|
|
117
112
|
fail(Error(`OpenAI subscription WebSocket closed ${event.code}: ${event.reason}`));
|
|
118
113
|
});
|
|
119
114
|
}),
|
|
120
|
-
catch: (cause) => new HttpClientError.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
115
|
+
catch: (cause) => new HttpClientError.HttpClientError({
|
|
116
|
+
reason: new HttpClientError.TransportError({
|
|
117
|
+
request: request.pipe(HttpClientRequest.setHeader("authorization", "[redacted]"), HttpClientRequest.setHeader("chatgpt-account-id", "[redacted]")),
|
|
118
|
+
description: `OpenAI subscription WebSocket failed: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
119
|
+
cause
|
|
120
|
+
})
|
|
125
121
|
})
|
|
126
122
|
}));
|
package/dist/llm/providers.d.ts
CHANGED
|
@@ -1,25 +1,28 @@
|
|
|
1
|
-
import { LanguageModel } from "
|
|
2
|
-
import { HttpClient } from "
|
|
1
|
+
import { LanguageModel } from "effect/ai";
|
|
2
|
+
import { HttpClient } from "effect/http";
|
|
3
3
|
import { Effect, Redacted } from "effect";
|
|
4
4
|
import type { Scope } from "effect";
|
|
5
5
|
import { AuthError } from "@xandreed/core";
|
|
6
6
|
import type { Credential, ModelSelection } from "@xandreed/core";
|
|
7
7
|
/**
|
|
8
|
-
* Per-provider `LanguageModel
|
|
8
|
+
* Per-provider `LanguageModel` construction. Built PER REQUEST from a
|
|
9
9
|
* freshly-resolved key (never captured at layer build), so a credential or
|
|
10
10
|
* model switch applies on the next call.
|
|
11
11
|
*
|
|
12
|
-
* v1 providers: opencode (OpenAI-compatible gateway — the default), google
|
|
13
|
-
*
|
|
12
|
+
* v1 providers: opencode (OpenAI-compatible gateway — the default), google
|
|
13
|
+
* (Gemini's OpenAI-compatible endpoint), anthropic (api key or subscription
|
|
14
|
+
* OAuth), openai (api key). OpenAI-OAuth
|
|
14
15
|
* (Codex) and ollama are deferred until an agent needs them.
|
|
15
16
|
*/
|
|
16
17
|
export declare const OPENCODE_CHAT_URL = "https://opencode.ai/zen/go/v1/chat/completions";
|
|
17
18
|
/** OpenCode serves GPT models through the OpenAI Responses protocol. */
|
|
18
19
|
export declare const OPENCODE_RESPONSES_API_URL = "https://opencode.ai/zen/v1";
|
|
20
|
+
/** Gemini through its OpenAI-compatible chat completions (Bearer API key). */
|
|
21
|
+
export declare const GOOGLE_CHAT_URL = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions";
|
|
19
22
|
/** Pure protocol routing decision; provider details stay inside this adapter. */
|
|
20
23
|
export declare const usesOpenCodeResponses: (modelId: string) => boolean;
|
|
21
24
|
export interface BuiltProvider {
|
|
22
|
-
readonly svc: LanguageModel.
|
|
25
|
+
readonly svc: LanguageModel.LanguageModel;
|
|
23
26
|
/** Anthropic subscription auth requires the Claude Code system block first. */
|
|
24
27
|
readonly prependClaudeCode: boolean;
|
|
25
28
|
}
|
|
@@ -34,4 +37,5 @@ export declare const prependClaudeCode: (options: unknown) => unknown;
|
|
|
34
37
|
* alone.
|
|
35
38
|
*/
|
|
36
39
|
export declare const withAnthropicCacheBreakpoints: (options: unknown) => unknown;
|
|
40
|
+
/** Preserve adapter descriptors and describe the public configuration of SDK-backed models. */
|
|
37
41
|
export declare const buildProvider: (selection: ModelSelection, credential: Credential | undefined, key: Redacted.Redacted<string> | undefined) => Effect.Effect<BuiltProvider, AuthError, HttpClient.HttpClient | Scope.Scope>;
|
package/dist/llm/providers.js
CHANGED
|
@@ -1,22 +1,21 @@
|
|
|
1
|
-
import { LanguageModel, Prompt } from "
|
|
1
|
+
import { LanguageModel, Prompt } from "effect/ai";
|
|
2
2
|
import { AnthropicClient, AnthropicLanguageModel } from "@effect/ai-anthropic";
|
|
3
|
-
import { GoogleClient, GoogleLanguageModel } from "@effect/ai-google";
|
|
4
3
|
import { OpenAiClient, OpenAiLanguageModel } from "@effect/ai-openai";
|
|
5
|
-
import { HttpClient, HttpClientRequest } from "
|
|
6
|
-
import { Effect,
|
|
7
|
-
import { AuthError, CurrentModelCallPolicy, CurrentPromptCacheKey } from "@xandreed/core";
|
|
4
|
+
import { HttpClient, HttpClientRequest } from "effect/http";
|
|
5
|
+
import { Effect, Option, Redacted } from "effect";
|
|
6
|
+
import { AuthError, CurrentModelCallPolicy, CurrentPromptCacheKey, describeModel, modelRequestDescriptorOf } from "@xandreed/core";
|
|
8
7
|
import { ANTHROPIC_OAUTH_BETA, CLAUDE_CODE_SYSTEM } from "../auth/anthropicOAuth.js";
|
|
9
8
|
import { makeCompatLanguageModel } from "./compat.js";
|
|
10
9
|
import { makeOpenAiCodexLanguageModel } from "./openAiCodex.js";
|
|
11
10
|
import { openAiCodexAccountId } from "../auth/openAiCodexOAuth.js";
|
|
12
|
-
export const OPENCODE_CHAT_URL = "https://opencode.ai/zen/go/v1/chat/completions", OPENCODE_RESPONSES_API_URL = "https://opencode.ai/zen/v1", usesOpenCodeResponses = (modelId) => /^gpt-/i.test(modelId);
|
|
11
|
+
export const OPENCODE_CHAT_URL = "https://opencode.ai/zen/go/v1/chat/completions", OPENCODE_RESPONSES_API_URL = "https://opencode.ai/zen/v1", GOOGLE_CHAT_URL = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions", usesOpenCodeResponses = (modelId) => /^gpt-/i.test(modelId);
|
|
13
12
|
const anthropicOAuthTransform = (access) => (client) => client.pipe(HttpClient.mapRequest((req) => req.pipe(HttpClientRequest.setHeaders({
|
|
14
13
|
Authorization: `Bearer ${Redacted.value(access)}`,
|
|
15
14
|
"anthropic-beta": ANTHROPIC_OAUTH_BETA
|
|
16
15
|
})))), claudeCodePrompt = Prompt.make([{ role: "system", content: CLAUDE_CODE_SYSTEM }]);
|
|
17
16
|
export const prependClaudeCode = (options) => ({
|
|
18
17
|
...options,
|
|
19
|
-
prompt: Prompt.
|
|
18
|
+
prompt: Prompt.concat(claudeCodePrompt, Prompt.make(options.prompt))
|
|
20
19
|
}), withAnthropicCacheBreakpoints = (options) => {
|
|
21
20
|
const messages = Prompt.make(options.prompt).content;
|
|
22
21
|
if (messages.length === 0)
|
|
@@ -46,31 +45,30 @@ export const prependClaudeCode = (options) => ({
|
|
|
46
45
|
const missingKey = (selection) => new AuthError({
|
|
47
46
|
provider: selection.provider,
|
|
48
47
|
message: `no credential for ${selection.provider} \u2014 add one to ~/.efferent/auth.json`
|
|
49
|
-
})
|
|
50
|
-
|
|
48
|
+
}), openCodeResponsesConfig = (cacheKey, policy) => ({
|
|
49
|
+
strictJsonSchema: !1,
|
|
50
|
+
prompt_cache_key: Option.getOrElse(cacheKey, () => "efferent"),
|
|
51
|
+
...Option.match(policy, {
|
|
52
|
+
onNone: () => ({}),
|
|
53
|
+
onSome: (value) => ({
|
|
54
|
+
...value.maxOutputTokens === void 0 ? {} : { max_output_tokens: value.maxOutputTokens },
|
|
55
|
+
reasoning: {
|
|
56
|
+
effort: value.effort === "xhigh" || value.effort === "max" ? "high" : value.effort === "none" ? "low" : value.effort
|
|
57
|
+
}
|
|
58
|
+
})
|
|
59
|
+
})
|
|
60
|
+
}), buildProviderService = (selection, credential, key) => {
|
|
51
61
|
const oauth = credential?.type === "oauth";
|
|
52
62
|
if (selection.provider === "opencode") {
|
|
53
63
|
if (key === void 0)
|
|
54
64
|
return Effect.fail(missingKey(selection));
|
|
55
65
|
if (usesOpenCodeResponses(selection.modelId))
|
|
56
66
|
return Effect.all({
|
|
57
|
-
cacheKey:
|
|
58
|
-
policy:
|
|
67
|
+
cacheKey: Effect.service(CurrentPromptCacheKey),
|
|
68
|
+
policy: Effect.service(CurrentModelCallPolicy)
|
|
59
69
|
}).pipe(Effect.flatMap(({ cacheKey, policy }) => OpenAiClient.make({ apiKey: key, apiUrl: OPENCODE_RESPONSES_API_URL }).pipe(Effect.flatMap((client) => OpenAiLanguageModel.make({
|
|
60
70
|
model: selection.modelId,
|
|
61
|
-
config:
|
|
62
|
-
strict: !1,
|
|
63
|
-
prompt_cache_key: Option.getOrElse(cacheKey, () => "efferent"),
|
|
64
|
-
...Option.match(policy, {
|
|
65
|
-
onNone: () => ({}),
|
|
66
|
-
onSome: (value) => ({
|
|
67
|
-
...value.maxOutputTokens === void 0 ? {} : { max_output_tokens: value.maxOutputTokens },
|
|
68
|
-
reasoning: {
|
|
69
|
-
effort: value.effort === "xhigh" || value.effort === "max" ? "high" : value.effort === "none" ? "low" : value.effort
|
|
70
|
-
}
|
|
71
|
-
})
|
|
72
|
-
})
|
|
73
|
-
}
|
|
71
|
+
config: openCodeResponsesConfig(cacheKey, policy)
|
|
74
72
|
}).pipe(Effect.provideService(OpenAiClient.OpenAiClient, client))), Effect.map((svc) => ({ svc, prependClaudeCode: !1 })))));
|
|
75
73
|
return makeCompatLanguageModel({
|
|
76
74
|
moduleName: "OpenCode",
|
|
@@ -80,10 +78,13 @@ export const buildProvider = (selection, credential, key) => {
|
|
|
80
78
|
}).pipe(Effect.map((svc) => ({ svc, prependClaudeCode: !1 })));
|
|
81
79
|
}
|
|
82
80
|
if (selection.provider === "google")
|
|
83
|
-
return key === void 0 ? Effect.fail(missingKey(selection)) :
|
|
81
|
+
return key === void 0 ? Effect.fail(missingKey(selection)) : makeCompatLanguageModel({
|
|
82
|
+
moduleName: "Google",
|
|
83
|
+
chatUrl: GOOGLE_CHAT_URL,
|
|
84
|
+
apiKey: Redacted.value(key),
|
|
84
85
|
model: selection.modelId,
|
|
85
|
-
|
|
86
|
-
}).pipe(Effect.
|
|
86
|
+
standardOnly: !0
|
|
87
|
+
}).pipe(Effect.map((svc) => ({ svc, prependClaudeCode: !1 })));
|
|
87
88
|
if (selection.provider === "anthropic") {
|
|
88
89
|
if (key === void 0)
|
|
89
90
|
return Effect.fail(missingKey(selection));
|
|
@@ -95,13 +96,13 @@ export const buildProvider = (selection, credential, key) => {
|
|
|
95
96
|
if (selection.provider === "openai-codex") {
|
|
96
97
|
if (key === void 0 || credential?.type !== "oauth")
|
|
97
98
|
return Effect.fail(missingKey(selection));
|
|
98
|
-
const accountId = Option.orElse(Option.
|
|
99
|
+
const accountId = Option.orElse(Option.fromNullishOr(credential.accountId), () => openAiCodexAccountId(Redacted.value(key)));
|
|
99
100
|
if (Option.isNone(accountId))
|
|
100
101
|
return Effect.fail(new AuthError({ provider: selection.provider, message: "the OpenAI subscription token has no ChatGPT account id \u2014 sign in again with Smith :login" }));
|
|
101
102
|
return makeOpenAiCodexLanguageModel({ model: selection.modelId, accessToken: key, accountId: accountId.value }).pipe(Effect.map((svc) => ({ svc, prependClaudeCode: !1 })));
|
|
102
103
|
}
|
|
103
104
|
if (selection.provider === "openai")
|
|
104
|
-
return key === void 0 ? Effect.fail(missingKey(selection)) : Effect.flatMap(
|
|
105
|
+
return key === void 0 ? Effect.fail(missingKey(selection)) : Effect.flatMap(Effect.service(CurrentPromptCacheKey), (cacheKey) => OpenAiClient.make({ apiKey: key }).pipe(Effect.flatMap((client) => OpenAiLanguageModel.make({
|
|
105
106
|
model: selection.modelId,
|
|
106
107
|
config: {
|
|
107
108
|
prompt_cache_key: Option.getOrElse(cacheKey, () => "efferent")
|
|
@@ -112,3 +113,10 @@ export const buildProvider = (selection, credential, key) => {
|
|
|
112
113
|
message: `provider "${selection.provider}" is not wired on the new line (v1: opencode, google, anthropic, openai, openai-codex)`
|
|
113
114
|
}));
|
|
114
115
|
};
|
|
116
|
+
export const buildProvider = (selection, credential, key) => buildProviderService(selection, credential, key).pipe(Effect.flatMap((built) => Effect.gen(function* () {
|
|
117
|
+
const cacheKey = yield* Effect.service(CurrentPromptCacheKey), policy = yield* Effect.service(CurrentModelCallPolicy), responses = selection.provider === "opencode" && usesOpenCodeResponses(selection.modelId), settings = selection.provider === "anthropic" ? { systemPrelude: built.prependClaudeCode ? CLAUDE_CODE_SYSTEM : null } : responses ? openCodeResponsesConfig(cacheKey, policy) : { prompt_cache_key: Option.getOrElse(cacheKey, () => "efferent") }, descriptor = { provider: selection.provider, model: selection.modelId, settings };
|
|
118
|
+
return {
|
|
119
|
+
...built,
|
|
120
|
+
svc: describeModel(built.svc, modelRequestDescriptorOf(built.svc).pipe(Effect.map((existing) => Option.match(existing, { onNone: () => descriptor, onSome: (value) => ({ ...value, provider: selection.provider }) }))))
|
|
121
|
+
};
|
|
122
|
+
})));
|
package/dist/llm/retry.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AiError } from "
|
|
1
|
+
import { AiError } from "effect/ai";
|
|
2
2
|
import { Effect, Stream } from "effect";
|
|
3
3
|
/**
|
|
4
4
|
* Transient-failure resilience for every routed LLM call — the survival
|
|
@@ -32,9 +32,9 @@ export declare const classifyLlmError: (error: unknown) => ErrorClass;
|
|
|
32
32
|
* it would park the turn in the patient outage ladder. */
|
|
33
33
|
export declare const rejectEmptyResponse: (label: string) => <A extends {
|
|
34
34
|
readonly content: ReadonlyArray<unknown>;
|
|
35
|
-
}, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | AiError.
|
|
35
|
+
}, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | AiError.AiError, R>;
|
|
36
36
|
/** Timeout + transient-only retries around one routed LLM connect. */
|
|
37
|
-
export declare const retryableLlm: (label: string) => <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | AiError.
|
|
37
|
+
export declare const retryableLlm: (label: string) => <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E | AiError.AiError, R>;
|
|
38
38
|
/**
|
|
39
39
|
* The STREAM twin of {@link retryableLlm}, with semantics a settled call
|
|
40
40
|
* doesn't need:
|
|
@@ -51,4 +51,4 @@ export declare const retryableLlm: (label: string) => <A, E, R>(effect: Effect.E
|
|
|
51
51
|
* mid-stream hang), not whole-request — a long healthy stream never trips
|
|
52
52
|
* it while tokens flow.
|
|
53
53
|
*/
|
|
54
|
-
export declare const retryableLlmStream: (label: string, timeoutMs?: number) => <A, E, R>(stream: Stream.Stream<A, E, R>) => Stream.Stream<A, E | AiError.
|
|
54
|
+
export declare const retryableLlmStream: (label: string, timeoutMs?: number) => <A, E, R>(stream: Stream.Stream<A, E, R>) => Stream.Stream<A, E | AiError.AiError, R>;
|
package/dist/llm/retry.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AiError } from "
|
|
2
|
-
import { Duration, Effect,
|
|
1
|
+
import { AiError } from "effect/ai";
|
|
2
|
+
import { Duration, Effect, Ref, Schedule, Stream } from "effect";
|
|
3
3
|
import { CurrentEmptyResponseTolerance } from "@xandreed/core";
|
|
4
4
|
export const LLM_REQUEST_TIMEOUT_MS = 300000, MAX_HONORED_RETRY_AFTER_MS = 60000;
|
|
5
5
|
const retryAfterMillis = (headers) => {
|
|
@@ -11,38 +11,32 @@ const retryAfterMillis = (headers) => {
|
|
|
11
11
|
return seconds * 1000;
|
|
12
12
|
const at = Date.parse(raw);
|
|
13
13
|
return Number.isNaN(at) ? void 0 : Math.max(0, at - Date.now());
|
|
14
|
-
};
|
|
14
|
+
}, rateLimited = (waitMs) => waitMs !== void 0 && waitMs > MAX_HONORED_RETRY_AFTER_MS ? "permanent" : "transient";
|
|
15
15
|
export const classifyLlmError = (error) => {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
return "permanent";
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
return
|
|
24
|
-
|
|
25
|
-
return status >= 500 ? "transient" : "permanent";
|
|
16
|
+
if (!AiError.isAiError(error)) {
|
|
17
|
+
const e = error;
|
|
18
|
+
return e?._tag === "HttpClientError" && e.reason?._tag === "TransportError" ? "transient" : "permanent";
|
|
19
|
+
}
|
|
20
|
+
const reason = error.reason, response = "http" in reason ? reason.http?.response : void 0;
|
|
21
|
+
if (response !== void 0) {
|
|
22
|
+
if (response.status === 429)
|
|
23
|
+
return rateLimited(retryAfterMillis(response.headers));
|
|
24
|
+
return response.status >= 500 ? "transient" : "permanent";
|
|
26
25
|
}
|
|
27
|
-
if (
|
|
28
|
-
return
|
|
29
|
-
return "permanent";
|
|
26
|
+
if (reason._tag === "RateLimitError")
|
|
27
|
+
return rateLimited(reason.retryAfter === void 0 ? void 0 : Duration.toMillis(reason.retryAfter));
|
|
28
|
+
return reason._tag === "InternalProviderError" || reason._tag === "NetworkError" || reason._tag === "UnknownError" ? "transient" : "permanent";
|
|
30
29
|
};
|
|
31
|
-
const fastRetries = Schedule.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
}), emptyError = (label, method) => new AiError.UnknownError({
|
|
36
|
-
module: "Router",
|
|
37
|
-
method,
|
|
38
|
-
description: `${label} returned an empty response (no text, tool call, or reasoning) \u2014 treated as a transient provider failure`
|
|
39
|
-
}), isContentPart = (part) => {
|
|
30
|
+
const fastRetries = Schedule.max([
|
|
31
|
+
Schedule.exponential(Duration.seconds(1)).pipe(Schedule.jittered),
|
|
32
|
+
Schedule.recurs(3)
|
|
33
|
+
]), routerError = (method, description) => AiError.make({ module: "Router", method, reason: new AiError.UnknownError({ description }) }), timeoutError = (label, method, timeoutMs) => routerError(method, `the ${label} request exceeded ${timeoutMs / 1000}s and was cut off`), emptyError = (label, method) => routerError(method, `${label} returned an empty response (no text, tool call, or reasoning) \u2014 treated as a transient provider failure`), isContentPart = (part) => {
|
|
40
34
|
const t = part?.type;
|
|
41
35
|
return t === "text" || t === "tool-call" || t === "reasoning";
|
|
42
36
|
};
|
|
43
|
-
export const rejectEmptyResponse = (label) => (effect) =>
|
|
37
|
+
export const rejectEmptyResponse = (label) => (effect) => Effect.service(CurrentEmptyResponseTolerance).pipe(Effect.flatMap((tolerant) => tolerant ? effect : effect.pipe(Effect.filterOrFail((res) => res.content.some(isContentPart), () => emptyError(label, "generateText"))))), retryableLlm = (label) => (effect) => effect.pipe(Effect.timeoutOrElse({
|
|
44
38
|
duration: Duration.millis(LLM_REQUEST_TIMEOUT_MS),
|
|
45
|
-
|
|
39
|
+
orElse: () => Effect.fail(timeoutError(label, "generateText", LLM_REQUEST_TIMEOUT_MS))
|
|
46
40
|
}), Effect.retry({
|
|
47
41
|
schedule: fastRetries,
|
|
48
42
|
while: (error) => classifyLlmError(error) === "transient"
|
|
@@ -57,5 +51,8 @@ const isContentPartEncoded = (part) => {
|
|
|
57
51
|
}, isFinishPartEncoded = (part) => part?.type === "finish";
|
|
58
52
|
export const retryableLlmStream = (label, timeoutMs = LLM_REQUEST_TIMEOUT_MS) => (stream) => Stream.unwrap(Effect.gen(function* () {
|
|
59
53
|
const contentSeen = yield* Ref.make(!1);
|
|
60
|
-
return stream.pipe(Stream.mapEffect((part) => isContentPartEncoded(part) ? Ref.set(contentSeen, !0).pipe(Effect.as(part)) : isFinishPartEncoded(part) ? Ref.get(contentSeen).pipe(Effect.flatMap((seen) => seen ? Effect.succeed(part) : Effect.fail(emptyError(label, "streamText")))) : Effect.succeed(part)), Stream.
|
|
54
|
+
return stream.pipe(Stream.mapEffect((part) => isContentPartEncoded(part) ? Ref.set(contentSeen, !0).pipe(Effect.as(part)) : isFinishPartEncoded(part) ? Ref.get(contentSeen).pipe(Effect.flatMap((seen) => seen ? Effect.succeed(part) : Effect.fail(emptyError(label, "streamText")))) : Effect.succeed(part)), Stream.timeoutOrElse({
|
|
55
|
+
duration: Duration.millis(timeoutMs),
|
|
56
|
+
orElse: () => Stream.fail(timeoutError(label, "streamText", timeoutMs))
|
|
57
|
+
}), Stream.retry(fastRetries.pipe(Schedule.while(({ input }) => Ref.get(contentSeen).pipe(Effect.map((seen) => !seen && classifyLlmError(input) === "transient"))))), Stream.tapError((error) => classifyLlmError(error) === "transient" ? Effect.logWarning(`${label}: transient stream failure was final: ${String(error)}`) : Effect.void));
|
|
61
58
|
}));
|
package/dist/llm/router.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { LanguageModel } from "
|
|
2
|
-
import { HttpClient } from "
|
|
1
|
+
import { LanguageModel } from "effect/ai";
|
|
2
|
+
import { HttpClient } from "effect/http";
|
|
3
3
|
import { Effect, Layer, Option, Stream } from "effect";
|
|
4
4
|
import { AuthStore, SettingsStore } from "@xandreed/core";
|
|
5
5
|
import type { ModelSelection } from "@xandreed/core";
|
package/dist/llm/router.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { AiError, LanguageModel } from "
|
|
2
|
-
import { FetchHttpClient, HttpClient } from "
|
|
3
|
-
import { Clock, Duration, Effect,
|
|
4
|
-
import { AuthStore, CurrentModelCallPolicy, formatModelSelection, parseModelSelection, SettingsStore } from "@xandreed/core";
|
|
1
|
+
import { AiError, LanguageModel } from "effect/ai";
|
|
2
|
+
import { FetchHttpClient, HttpClient } from "effect/http";
|
|
3
|
+
import { Clock, Duration, Effect, Layer, Metric, Option, Ref, Stream } from "effect";
|
|
4
|
+
import { AuthStore, CurrentModelCallPolicy, describeModel, formatModelSelection, HarnessError, parseModelSelection, resolvingModel, SettingsStore } from "@xandreed/core";
|
|
5
5
|
import { buildProvider, prependClaudeCode, withAnthropicCacheBreakpoints } from "./providers.js";
|
|
6
6
|
import {
|
|
7
7
|
classifyLlmError,
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
retryableLlm,
|
|
10
10
|
retryableLlmStream
|
|
11
11
|
} from "./retry.js";
|
|
12
|
-
const configError = (message) =>
|
|
12
|
+
const configError = (message) => AiError.make({ module: "Router", method: "selection", reason: new AiError.UnknownError({ description: message }) }), llmInputTokens = Metric.counter("llm.usage.input_tokens", {
|
|
13
13
|
description: "prompt tokens consumed by routed LLM calls",
|
|
14
14
|
incremental: !0
|
|
15
15
|
}), llmOutputTokens = Metric.counter("llm.usage.output_tokens", {
|
|
@@ -18,7 +18,15 @@ const configError = (message) => new AiError.UnknownError({ module: "Router", me
|
|
|
18
18
|
}), llmRequests = Metric.counter("llm.requests", {
|
|
19
19
|
description: "routed LLM calls by final outcome (after retries)",
|
|
20
20
|
incremental: !0
|
|
21
|
-
}), llmDuration = Metric.
|
|
21
|
+
}), llmDuration = Metric.timer("llm.request.duration", {
|
|
22
|
+
boundaries: [100, 250, 500, 1000, 2500, 5000, 1e4, 30000, 60000, 120000, 300000]
|
|
23
|
+
}), byModel = (metric, label) => Metric.withAttributes(metric, { "llm.model": label }), countRequest = (label, outcome, isFallback) => Metric.update(Metric.withAttributes(byModel(llmRequests, label), {
|
|
24
|
+
outcome,
|
|
25
|
+
...isFallback === void 0 ? {} : { fallback: isFallback ? "true" : "false" }
|
|
26
|
+
}), 1), usageTotals = (usage) => {
|
|
27
|
+
const u = usage;
|
|
28
|
+
return { input: u?.inputTokens?.total ?? 0, output: u?.outputTokens?.total ?? 0 };
|
|
29
|
+
}, tracedContent = (content) => process.env.EFFERENT_TRACE_CONTENT === "1" ? content.slice(0, 500) : "[redacted]", shapeOptions = (selection, shouldPrependClaudeCode, options) => {
|
|
22
30
|
const cached = selection.provider === "anthropic" ? withAnthropicCacheBreakpoints(options) : options;
|
|
23
31
|
return shouldPrependClaudeCode ? prependClaudeCode(cached) : cached;
|
|
24
32
|
}, currentSelection = Effect.gen(function* () {
|
|
@@ -31,13 +39,20 @@ const configError = (message) => new AiError.UnknownError({ module: "Router", me
|
|
|
31
39
|
fallback: Option.flatMap(loaded.fallbackModel, parseModelSelection),
|
|
32
40
|
effort: loaded.reasoningEffort
|
|
33
41
|
};
|
|
34
|
-
}), withConfiguredEffort = (effect, effort) =>
|
|
42
|
+
}), withConfiguredEffort = (effect, effort) => Effect.service(CurrentModelCallPolicy).pipe(Effect.flatMap((current) => Option.isSome(current) || Option.isNone(effort) ? effect : effect.pipe(Effect.provideService(CurrentModelCallPolicy, Option.some({ effort: effort.value }))))), describeSelection = (primary, fallback, effort) => Effect.service(CurrentModelCallPolicy).pipe(Effect.map((current) => ({
|
|
43
|
+
provider: primary.provider,
|
|
44
|
+
model: primary.modelId,
|
|
45
|
+
settings: {
|
|
46
|
+
fallback: Option.getOrElse(Option.map(fallback, formatModelSelection), () => null),
|
|
47
|
+
policy: Option.getOrElse(Option.orElse(current, () => Option.map(effort, (value) => ({ effort: value }))), () => null)
|
|
48
|
+
}
|
|
49
|
+
})));
|
|
35
50
|
export const generateWith = (selection, options, isFallback = !1) => Effect.gen(function* () {
|
|
36
51
|
const auth = yield* AuthStore, label = formatModelSelection(selection), credential = Option.getOrUndefined(yield* auth.get(selection.provider)), key = Option.getOrUndefined(yield* auth.resolveKey(selection.provider)), built = yield* buildProvider(selection, credential, key);
|
|
37
52
|
return yield* built.svc.generateText(shapeOptions(selection, built.prependClaudeCode, options)).pipe(rejectEmptyResponse(label), retryableLlm(label), Effect.map((res) => stampResponse(res, label)), Effect.tap((res) => Effect.annotateCurrentSpan({
|
|
38
53
|
"llm.finish_reason": String(res.finishReason),
|
|
39
|
-
"llm.usage.input_tokens": res.usage.
|
|
40
|
-
"llm.usage.output_tokens": res.usage.
|
|
54
|
+
"llm.usage.input_tokens": usageTotals(res.usage).input,
|
|
55
|
+
"llm.usage.output_tokens": usageTotals(res.usage).output,
|
|
41
56
|
"llm.response_chars": res.text.length,
|
|
42
57
|
"llm.tool_calls": res.content.flatMap((part) => {
|
|
43
58
|
const p = part;
|
|
@@ -46,17 +61,14 @@ export const generateWith = (selection, options, isFallback = !1) => Effect.gen(
|
|
|
46
61
|
"llm.reasoning": tracedContent(res.reasoningText ?? ""),
|
|
47
62
|
"llm.fallback": isFallback
|
|
48
63
|
})), Effect.tap((res) => Effect.all([
|
|
49
|
-
Metric.
|
|
50
|
-
Metric.
|
|
51
|
-
], { discard: !0 })), Effect.
|
|
52
|
-
onFailure: () => Metric.increment(Metric.tagged(Metric.tagged(byModel(llmRequests, label), "outcome", "error"), "fallback", isFallback ? "true" : "false")),
|
|
53
|
-
onSuccess: () => Metric.increment(Metric.tagged(Metric.tagged(byModel(llmRequests, label), "outcome", "ok"), "fallback", isFallback ? "true" : "false"))
|
|
54
|
-
}), Metric.trackDuration(byModel(llmDuration, label)), Effect.withSpan("providers.generate", {
|
|
64
|
+
Metric.update(byModel(llmInputTokens, label), usageTotals(res.usage).input),
|
|
65
|
+
Metric.update(byModel(llmOutputTokens, label), usageTotals(res.usage).output)
|
|
66
|
+
], { discard: !0 })), Effect.tap(() => countRequest(label, "ok", isFallback)), Effect.tapError(() => countRequest(label, "error", isFallback)), Effect.timed, Effect.tap(([duration]) => Metric.update(byModel(llmDuration, label), duration)), Effect.map(([, res]) => res), Effect.withSpan("providers.generate", {
|
|
55
67
|
attributes: { "llm.model": label }
|
|
56
68
|
}));
|
|
57
|
-
}).pipe(Effect.scoped), withFallbackRung = (primary, fallback, call) => call(primary, !1).pipe(Effect.
|
|
69
|
+
}).pipe(Effect.scoped), withFallbackRung = (primary, fallback, call) => call(primary, !1).pipe(Effect.catch((error) => Option.match(Option.filter(fallback, (fb) => formatModelSelection(fb) !== formatModelSelection(primary) && classifyLlmError(error) === "transient"), {
|
|
58
70
|
onNone: () => Effect.fail(error),
|
|
59
|
-
onSome: (fb) => Effect.logWarning(`${formatModelSelection(primary)} exhausted retries (${String(error).slice(0, 200)}) \u2014 falling back to ${formatModelSelection(fb)}`).pipe(Effect.
|
|
71
|
+
onSome: (fb) => Effect.logWarning(`${formatModelSelection(primary)} exhausted retries (${String(error).slice(0, 200)}) \u2014 falling back to ${formatModelSelection(fb)}`).pipe(Effect.andThen(call(fb, !0)))
|
|
60
72
|
})));
|
|
61
73
|
const stampModel = (part, label) => {
|
|
62
74
|
if (typeof part !== "object" || part === null)
|
|
@@ -90,36 +102,40 @@ export const tapStreamTelemetry = (label) => (stream) => Stream.unwrap(Effect.ge
|
|
|
90
102
|
return Ref.update(stats, (s) => accumulateStats(s, part)).pipe(Effect.as(part));
|
|
91
103
|
return Ref.get(stats).pipe(Effect.flatMap((s) => Effect.annotateCurrentSpan({
|
|
92
104
|
"llm.finish_reason": String(p.reason),
|
|
93
|
-
"llm.usage.input_tokens": p.usage
|
|
94
|
-
"llm.usage.output_tokens": p.usage
|
|
105
|
+
"llm.usage.input_tokens": usageTotals(p.usage).input,
|
|
106
|
+
"llm.usage.output_tokens": usageTotals(p.usage).output,
|
|
95
107
|
"llm.response_chars": s.textChars,
|
|
96
108
|
"llm.tool_calls": s.toolNames.join(","),
|
|
97
109
|
"llm.reasoning": tracedContent(s.reasoning)
|
|
98
|
-
})), Effect.
|
|
99
|
-
Metric.
|
|
100
|
-
Metric.
|
|
110
|
+
})), Effect.andThen(Effect.all([
|
|
111
|
+
Metric.update(byModel(llmInputTokens, label), usageTotals(p.usage).input),
|
|
112
|
+
Metric.update(byModel(llmOutputTokens, label), usageTotals(p.usage).output)
|
|
101
113
|
], { discard: !0 })), Effect.as(part));
|
|
102
|
-
}), Stream.
|
|
103
|
-
})), streamWith = (selection, options) => Stream.
|
|
114
|
+
}), Stream.onEnd(countRequest(label, "ok").pipe(Effect.andThen(recordDuration))), Stream.tapCause(() => countRequest(label, "error").pipe(Effect.andThen(recordDuration))));
|
|
115
|
+
})), streamWith = (selection, options) => Stream.unwrap(Effect.gen(function* () {
|
|
104
116
|
const auth = yield* AuthStore, label = formatModelSelection(selection), credential = Option.getOrUndefined(yield* auth.get(selection.provider)), key = Option.getOrUndefined(yield* auth.resolveKey(selection.provider)), built = yield* buildProvider(selection, credential, key);
|
|
105
117
|
return built.svc.streamText(shapeOptions(selection, built.prependClaudeCode, options)).pipe(retryableLlmStream(label), Stream.map((part) => stampModel(part, label)), tapStreamTelemetry(label), Stream.withSpan("providers.generate", {
|
|
106
118
|
attributes: { "llm.model": label }
|
|
107
119
|
}));
|
|
108
|
-
}))
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
120
|
+
}));
|
|
121
|
+
const withConfiguredEffortStream = (stream, effort) => Stream.unwrap(Effect.service(CurrentModelCallPolicy).pipe(Effect.map((current) => Option.isSome(current) || Option.isNone(effort) ? stream : stream.pipe(Stream.provideService(CurrentModelCallPolicy, Option.some({ effort: effort.value })))))), selectionModel = (primary, fallback, effort, context, http) => {
|
|
122
|
+
const service = {
|
|
123
|
+
[LanguageModel.TypeId]: LanguageModel.TypeId,
|
|
124
|
+
generateText: (options) => withConfiguredEffort(withFallbackRung(primary, fallback, (selection, isFallback) => generateWith(selection, options, isFallback)), effort).pipe(Effect.provide(context), Effect.provideService(HttpClient.HttpClient, http)),
|
|
112
125
|
generateObject: () => Effect.fail(configError("generateObject is not wired on the new line yet")),
|
|
113
|
-
streamText: (options) =>
|
|
114
|
-
const stream = streamWith(primary, options);
|
|
115
|
-
return Option.isSome(current) || Option.isNone(effort) ? Effect.succeed(stream) : Effect.locallyScoped(CurrentModelCallPolicy, Option.some({ effort: effort.value })).pipe(Effect.as(stream));
|
|
116
|
-
}))))).pipe(Stream.provideSomeContext(context), Stream.provideService(HttpClient.HttpClient, http))
|
|
126
|
+
streamText: (options) => withConfiguredEffortStream(streamWith(primary, options), effort).pipe(Stream.provideContext(context), Stream.provideService(HttpClient.HttpClient, http))
|
|
117
127
|
};
|
|
128
|
+
return describeModel(service, describeSelection(primary, fallback, effort));
|
|
129
|
+
};
|
|
130
|
+
export const LanguageModelLive = Layer.effect(LanguageModel.LanguageModel, Effect.gen(function* () {
|
|
131
|
+
const context = yield* Effect.context(), http = yield* HttpClient.HttpClient, selected = currentSelection.pipe(Effect.map(({ primary, fallback, effort }) => selectionModel(primary, fallback, effort, context, http)), Effect.provide(context)), service = {
|
|
132
|
+
[LanguageModel.TypeId]: LanguageModel.TypeId,
|
|
133
|
+
generateText: (options) => selected.pipe(Effect.flatMap((model) => model.generateText(options))),
|
|
134
|
+
generateObject: () => Effect.fail(configError("generateObject is not wired on the new line yet")),
|
|
135
|
+
streamText: (options) => Stream.unwrap(selected.pipe(Effect.map((model) => model.streamText(options))))
|
|
136
|
+
}, asHarness = (error) => new HarnessError({ code: "request.model", message: String(error) });
|
|
137
|
+
return resolvingModel(describeModel(service, currentSelection.pipe(Effect.flatMap(({ primary, fallback, effort }) => describeSelection(primary, fallback, effort)), Effect.provide(context), Effect.mapError(asHarness))), selected.pipe(Effect.mapError(asHarness)));
|
|
118
138
|
})).pipe(Layer.provide(FetchHttpClient.layer)), LanguageModelSelectionLive = (primary, fallback) => Layer.effect(LanguageModel.LanguageModel, Effect.gen(function* () {
|
|
119
139
|
const context = yield* Effect.context(), http = yield* HttpClient.HttpClient;
|
|
120
|
-
return
|
|
121
|
-
generateText: (options) => withFallbackRung(primary, fallback, (selection, isFallback) => generateWith(selection, options, isFallback)).pipe(Effect.provide(context), Effect.provideService(HttpClient.HttpClient, http)),
|
|
122
|
-
generateObject: () => Effect.fail(configError("generateObject is not wired on the new line yet")),
|
|
123
|
-
streamText: (options) => streamWith(primary, options).pipe(Stream.provideSomeContext(context), Stream.provideService(HttpClient.HttpClient, http))
|
|
124
|
-
};
|
|
140
|
+
return selectionModel(primary, fallback, Option.none(), context, http);
|
|
125
141
|
})).pipe(Layer.provide(FetchHttpClient.layer));
|
package/dist/llm/sse.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AiError } from "
|
|
2
|
-
import type { Response } from "
|
|
1
|
+
import { AiError } from "effect/ai";
|
|
2
|
+
import type { Response } from "effect/ai";
|
|
3
3
|
import { Stream } from "effect";
|
|
4
4
|
/**
|
|
5
5
|
* The OpenAI-compatible SSE wire → `@effect/ai` `StreamPartEncoded`s, as a
|
|
@@ -30,12 +30,17 @@ export interface CompletionUsage {
|
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
32
|
/** The vendor cached-token fallback chain, shared with the non-streaming
|
|
33
|
-
* parser — the gateway fronts upstreams with three cache vocabularies.
|
|
33
|
+
* parser — the gateway fronts upstreams with three cache vocabularies.
|
|
34
|
+
* `prompt_tokens` already counts the cached tokens: it is the full input. */
|
|
34
35
|
export declare const usageFromCompletion: (usage: CompletionUsage | undefined) => {
|
|
35
|
-
inputTokens:
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
inputTokens: {
|
|
37
|
+
total: number;
|
|
38
|
+
uncached: number;
|
|
39
|
+
cacheRead: number;
|
|
40
|
+
};
|
|
41
|
+
outputTokens: {
|
|
42
|
+
total: number;
|
|
43
|
+
};
|
|
39
44
|
};
|
|
40
45
|
export declare const finishReasonFromWire: (raw: string | undefined) => "stop" | "length" | "content-filter" | "tool-calls";
|
|
41
46
|
/** Parse a streaming chat-completions body into encoded stream parts. */
|
package/dist/llm/sse.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { AiError } from "
|
|
2
|
-
import { Effect, Option, Stream } from "effect";
|
|
3
|
-
export const usageFromCompletion = (usage) =>
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
}
|
|
1
|
+
import { AiError } from "effect/ai";
|
|
2
|
+
import { Effect, Option, Result, Stream } from "effect";
|
|
3
|
+
export const usageFromCompletion = (usage) => {
|
|
4
|
+
const input = usage?.prompt_tokens ?? 0, cached = usage?.prompt_cache_hit_tokens ?? usage?.cached_tokens ?? usage?.prompt_tokens_details?.cached_tokens ?? 0;
|
|
5
|
+
return {
|
|
6
|
+
inputTokens: { total: input, uncached: Math.max(0, input - cached), cacheRead: cached },
|
|
7
|
+
outputTokens: { total: usage?.completion_tokens ?? 0 }
|
|
8
|
+
};
|
|
9
|
+
}, finishReasonFromWire = (raw) => {
|
|
9
10
|
if (raw === "tool_calls")
|
|
10
11
|
return "tool-calls";
|
|
11
12
|
if (raw === "length")
|
|
@@ -66,12 +67,14 @@ const initialState = {
|
|
|
66
67
|
name: call.name.length > 0 ? call.name : fragment.function?.name ?? "",
|
|
67
68
|
args: call.args + (fragment.function?.arguments ?? "")
|
|
68
69
|
} : call);
|
|
69
|
-
}, malformed = (moduleName, description) =>
|
|
70
|
+
}, malformed = (moduleName, description) => AiError.make({ module: moduleName, method: "streamText", reason: new AiError.InvalidOutputError({ description }) }), applyChunk = (moduleName, state, chunk) => {
|
|
70
71
|
if (chunk.error !== void 0 && chunk.error !== null)
|
|
71
|
-
return Effect.fail(
|
|
72
|
+
return Effect.fail(AiError.make({
|
|
72
73
|
module: moduleName,
|
|
73
74
|
method: "streamText",
|
|
74
|
-
|
|
75
|
+
reason: new AiError.UnknownError({
|
|
76
|
+
description: `the stream carried an error chunk: ${JSON.stringify(chunk.error).slice(0, 500)}`
|
|
77
|
+
})
|
|
75
78
|
}));
|
|
76
79
|
const choice = chunk.choices?.[0], delta = choice?.delta, reasoning = delta?.reasoning_content ?? delta?.reasoning, content = delta?.content, afterReasoning = typeof reasoning === "string" && reasoning.length > 0 ? emitReasoning(state, reasoning) : [state, []], afterText = emitThen(afterReasoning, (s) => typeof content === "string" && content.length > 0 ? emitText(s, content) : [s, []]), [folded, parts] = afterText, next = {
|
|
77
80
|
...folded,
|
|
@@ -104,13 +107,12 @@ const initialState = {
|
|
|
104
107
|
}).pipe(Effect.flatMap((chunk) => applyChunk(moduleName, state, chunk)));
|
|
105
108
|
export const sseStreamParts = (options) => Stream.fromReadableStream({
|
|
106
109
|
evaluate: () => options.body,
|
|
107
|
-
onError: (cause) =>
|
|
110
|
+
onError: (cause) => AiError.make({
|
|
108
111
|
module: options.moduleName,
|
|
109
112
|
method: "streamText",
|
|
110
|
-
description: `the response body stream failed: ${String(cause)}
|
|
111
|
-
cause
|
|
113
|
+
reason: new AiError.UnknownError({ description: `the response body stream failed: ${String(cause)}` })
|
|
112
114
|
})
|
|
113
115
|
}).pipe(Stream.decodeText(), Stream.splitLines, Stream.filterMap((line) => {
|
|
114
116
|
const trimmed = line.trim();
|
|
115
|
-
return trimmed.startsWith("data:") ?
|
|
116
|
-
}), Stream.concat(Stream.succeed(DONE)), Stream.mapAccumEffect(initialState, (state, payload) => step(options.moduleName, state, payload))
|
|
117
|
+
return trimmed.startsWith("data:") ? Result.succeed(trimmed.slice(5).trim()) : Result.fail(line);
|
|
118
|
+
}), Stream.concat(Stream.succeed(DONE)), Stream.mapAccumEffect(() => initialState, (state, payload) => step(options.moduleName, state, payload)));
|
package/dist/llm/utilityLlm.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { Prompt } from "
|
|
2
|
-
import { FetchHttpClient, HttpClient } from "
|
|
1
|
+
import { Prompt } from "effect/ai";
|
|
2
|
+
import { FetchHttpClient, HttpClient } from "effect/http";
|
|
3
3
|
import { Effect, Layer, Option } from "effect";
|
|
4
|
-
import { AuthStore, parseModelSelection, SettingsStore, UtilityError, UtilityLlm } from "@xandreed/core";
|
|
4
|
+
import { AuthStore, extractUsage, parseModelSelection, SettingsStore, UtilityError, UtilityLlm } from "@xandreed/core";
|
|
5
5
|
import { generateWith } from "./router.js";
|
|
6
6
|
export const UtilityLlmLive = Layer.effect(UtilityLlm, Effect.gen(function* () {
|
|
7
7
|
const context = yield* Effect.context(), http = yield* HttpClient.HttpClient, settings = yield* SettingsStore;
|
|
@@ -15,19 +15,11 @@ export const UtilityLlmLive = Layer.effect(UtilityLlm, Effect.gen(function* () {
|
|
|
15
15
|
tools: [],
|
|
16
16
|
toolChoice: "none",
|
|
17
17
|
responseFormat: { type: "text" }
|
|
18
|
-
}).pipe(Effect.mapError((e) => new UtilityError({ message: String(e) })), Effect.provide(context), Effect.provideService(HttpClient.HttpClient, http))
|
|
18
|
+
}).pipe(Effect.mapError((e) => new UtilityError({ message: String(e) })), Effect.provide(context), Effect.provideService(HttpClient.HttpClient, http));
|
|
19
|
+
return { text: res.content.flatMap((p) => {
|
|
19
20
|
const part = p;
|
|
20
21
|
return part.type === "text" ? [part.text ?? ""] : [];
|
|
21
|
-
}).join(""), usage
|
|
22
|
-
return {
|
|
23
|
-
text,
|
|
24
|
-
usage: {
|
|
25
|
-
inputTokens: usage?.inputTokens ?? 0,
|
|
26
|
-
outputTokens: usage?.outputTokens ?? 0,
|
|
27
|
-
totalTokens: usage?.totalTokens ?? 0,
|
|
28
|
-
cacheReadTokens: usage?.cachedInputTokens ?? 0
|
|
29
|
-
}
|
|
30
|
-
};
|
|
22
|
+
}).join(""), usage: extractUsage(res.usage, res.content) };
|
|
31
23
|
})
|
|
32
24
|
};
|
|
33
25
|
})).pipe(Layer.provide(FetchHttpClient.layer));
|
package/dist/plugin.adapter.d.ts
CHANGED
|
@@ -1,2 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
import { LanguageModel } from "effect/ai";
|
|
2
|
+
import { AuthStore, ModelCatalog, SessionEnvironment, SettingsError, SettingsStore, UtilityLlm } from "@xandreed/core";
|
|
3
|
+
export declare const modelsPlugin: import("@xandreed/core").TypedPlugin<{
|
|
4
|
+
readonly model: string;
|
|
5
|
+
readonly fastModel: string;
|
|
6
|
+
readonly fallbackModel: string;
|
|
7
|
+
readonly inheritPrevious: boolean;
|
|
8
|
+
}, {
|
|
9
|
+
readonly model: string;
|
|
10
|
+
readonly fastModel: string;
|
|
11
|
+
readonly fallbackModel: string;
|
|
12
|
+
readonly inheritPrevious: boolean;
|
|
13
|
+
}, SettingsStore | AuthStore | ModelCatalog | UtilityLlm | LanguageModel.LanguageModel, SettingsError, SessionEnvironment>;
|
|
2
14
|
export default modelsPlugin;
|
package/dist/plugin.adapter.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { homedir } from "node:os";
|
|
2
|
-
import { LanguageModel } from "
|
|
2
|
+
import { LanguageModel } from "effect/ai";
|
|
3
3
|
import { Effect, Layer, Option, Schema } from "effect";
|
|
4
4
|
import { AuthStore, definePlugin, EngineSettings, ModelCatalog, SessionEnvironment, SettingsError, SettingsStore, UtilityLlm } from "@xandreed/core";
|
|
5
5
|
import { LocalAuthStoreLive } from "./auth/localAuth.js";
|
|
@@ -10,12 +10,12 @@ import { LocalSettingsStoreLive } from "./settings/localSettings.js";
|
|
|
10
10
|
const Config = Schema.Struct({ model: Schema.String, fastModel: Schema.String, fallbackModel: Schema.String, inheritPrevious: Schema.Boolean });
|
|
11
11
|
export const modelsPlugin = definePlugin({
|
|
12
12
|
id: "@xandreed/plugin-models",
|
|
13
|
-
version: "0.
|
|
13
|
+
version: "0.8.0-next.0",
|
|
14
14
|
config: Config,
|
|
15
15
|
defaults: { model: "", fastModel: "", fallbackModel: "", inheritPrevious: !0 },
|
|
16
16
|
requires: [SessionEnvironment],
|
|
17
17
|
provides: [LanguageModel.LanguageModel, UtilityLlm, AuthStore, SettingsStore, ModelCatalog],
|
|
18
|
-
layer: (config) => Layer.
|
|
18
|
+
layer: (config) => Layer.unwrap(Effect.gen(function* () {
|
|
19
19
|
const { workspace } = yield* SessionEnvironment, previous = config.inheritPrevious ? yield* SettingsStore.pipe(Effect.flatMap((settings) => settings.load), Effect.provide(LocalSettingsStoreLive(workspace, homedir()))) : new EngineSettings({}), optional = (value) => value.length === 0 ? Option.none() : Option.some(value), settings = Layer.succeed(SettingsStore, {
|
|
20
20
|
load: Effect.succeed(new EngineSettings({ ...previous, model: Option.orElse(optional(config.model), () => previous.model), fastModel: Option.orElse(optional(config.fastModel), () => previous.fastModel), fallbackModel: Option.orElse(optional(config.fallbackModel), () => previous.fallbackModel) })),
|
|
21
21
|
setRole: () => Effect.fail(new SettingsError({ message: "Edit the models plugin configuration to change a model" })),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xandreed/plugin-models",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0-next.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -14,13 +14,10 @@
|
|
|
14
14
|
}
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@xandreed/core": "0.
|
|
18
|
-
"effect": "
|
|
19
|
-
"@effect/ai": "0.
|
|
20
|
-
"@effect/ai-
|
|
21
|
-
"@effect/ai-google": "^0.14.0",
|
|
22
|
-
"@effect/ai-openai": "^0.39.2",
|
|
23
|
-
"@effect/platform": "0.96.1",
|
|
17
|
+
"@xandreed/core": "0.8.0-next.0",
|
|
18
|
+
"effect": "4.0.0-rc.118",
|
|
19
|
+
"@effect/ai-anthropic": "4.0.0-rc.118",
|
|
20
|
+
"@effect/ai-openai": "4.0.0-rc.118",
|
|
24
21
|
"ws": "^8.21.0"
|
|
25
22
|
},
|
|
26
23
|
"private": false,
|