@astratra/ai 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -2
- package/package.json +2 -1
- package/src/agentLoop.js +88 -11
- package/src/answerText.js +137 -0
- package/src/askLimit.js +73 -0
- package/src/breakers.js +97 -0
- package/src/eventStream.js +47 -0
- package/src/gemini.js +152 -0
- package/src/index.d.ts +452 -3
- package/src/index.js +13 -1
- package/src/language.js +85 -0
- package/src/masking.js +279 -0
- package/src/modelAnswers.js +119 -0
- package/src/nativeTools.js +256 -0
- package/src/openaiCompatible.js +158 -0
- package/src/passages.js +101 -0
- package/src/providerRouter.js +224 -44
- package/src/sources.js +243 -0
- package/src/webSearch.js +71 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Providers that speak the OpenAI chat format (Groq, Mistral, Cloudflare
|
|
3
|
+
* Workers AI, gateways…): one request shape, one way to read the answer.
|
|
4
|
+
*
|
|
5
|
+
* Three things real traffic taught:
|
|
6
|
+
*
|
|
7
|
+
* - a model sometimes returns tool arguments that are not JSON. Parsing them
|
|
8
|
+
* blindly threw inside the turn and lost the whole answer; a malformed call
|
|
9
|
+
* is now reported as such and the loop decides;
|
|
10
|
+
* - reasoning models write their thinking between think tags inside the
|
|
11
|
+
* answer — never shown, even when the closing tag is missing;
|
|
12
|
+
* - an error carries the HTTP status (a 429 must reach the router's cooldown)
|
|
13
|
+
* and never the key.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const { providerResult, readToolArguments, readWrittenAnswer, readWrittenToolCalls, toolCallOf, withoutThinking } = require('./modelAnswers');
|
|
17
|
+
|
|
18
|
+
function toTranscript(messages) {
|
|
19
|
+
return (Array.isArray(messages) ? messages : []).map((message) => {
|
|
20
|
+
if (message.role === 'assistant') {
|
|
21
|
+
return {
|
|
22
|
+
role: 'assistant',
|
|
23
|
+
content: message.text === undefined ? message.content || '' : message.text,
|
|
24
|
+
...(message.toolCalls && message.toolCalls.length ? {
|
|
25
|
+
tool_calls: message.toolCalls.map((call) => ({
|
|
26
|
+
id: call.id,
|
|
27
|
+
type: 'function',
|
|
28
|
+
/* Des arguments illisibles (null) repartent en objet vide : le fournisseur refuserait « null ». */
|
|
29
|
+
function: { name: call.name, arguments: JSON.stringify(call.args && typeof call.args === 'object' ? call.args : {}) }
|
|
30
|
+
}))
|
|
31
|
+
} : {})
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
if (message.role === 'tool') {
|
|
35
|
+
return { role: 'tool', tool_call_id: message.toolCallId, content: typeof message.result === 'string' ? message.result : JSON.stringify(message.result) };
|
|
36
|
+
}
|
|
37
|
+
const text = message.text === undefined ? message.content || '' : message.text;
|
|
38
|
+
/* A photo joined by the person: an image part with a data address. Only
|
|
39
|
+
send it to models that see. */
|
|
40
|
+
if (message.images && message.images.length) {
|
|
41
|
+
return {
|
|
42
|
+
role: 'user',
|
|
43
|
+
content: [
|
|
44
|
+
{ type: 'text', text },
|
|
45
|
+
...message.images.map(({ mimeType, data }) => ({ type: 'image_url', image_url: { url: `data:${mimeType};base64,${data}` } }))
|
|
46
|
+
]
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
return { role: message.role === 'system' ? 'system' : 'user', content: text };
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Ask one model.
|
|
55
|
+
* @param {{ url: string, key: string, model: string, extra?: object }} target
|
|
56
|
+
* @param {{ system?: string, messages: object[], tools?: object[], maxTokens?: number, temperature?: number }} request
|
|
57
|
+
* @param {{ fetch: Function, signal?: AbortSignal, timeoutMs?: number }} io
|
|
58
|
+
* @returns {Promise<{ status: number, text?: string|null, toolCalls?: object[], cut?: boolean }>}
|
|
59
|
+
* A non-2xx answer resolves `{ status }` — nothing is thrown for it.
|
|
60
|
+
*/
|
|
61
|
+
async function askChatModel(target, request, io = {}) {
|
|
62
|
+
if (typeof io.fetch !== 'function') throw new Error('askChatModel requires io.fetch.');
|
|
63
|
+
const { url, key, model, extra = {} } = target || {};
|
|
64
|
+
const timeoutMs = io.timeoutMs || 45_000;
|
|
65
|
+
const Signal = globalThis.AbortSignal;
|
|
66
|
+
const timeout = Signal.timeout(timeoutMs);
|
|
67
|
+
const signal = io.signal && typeof Signal.any === 'function' ? Signal.any([io.signal, timeout]) : (io.signal || timeout);
|
|
68
|
+
const messages = [
|
|
69
|
+
...(request.system ? [{ role: 'system', content: request.system }] : []),
|
|
70
|
+
...toTranscript(request.messages)
|
|
71
|
+
];
|
|
72
|
+
const response = await io.fetch(url, {
|
|
73
|
+
method: 'POST',
|
|
74
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
|
|
75
|
+
body: JSON.stringify({
|
|
76
|
+
model,
|
|
77
|
+
messages,
|
|
78
|
+
temperature: request.temperature === undefined ? 0.4 : request.temperature,
|
|
79
|
+
max_tokens: request.maxTokens || 1200,
|
|
80
|
+
...(request.tools && request.tools.length ? { tools: request.tools.map((tool) => ({ type: 'function', function: tool })) } : {}),
|
|
81
|
+
...extra
|
|
82
|
+
}),
|
|
83
|
+
signal
|
|
84
|
+
});
|
|
85
|
+
if (!response.ok) return { status: response.status };
|
|
86
|
+
const answer = await response.json();
|
|
87
|
+
const choice = (answer && answer.choices && answer.choices[0]) || {};
|
|
88
|
+
const message = choice.message || {};
|
|
89
|
+
if (Array.isArray(message.tool_calls) && message.tool_calls.length) {
|
|
90
|
+
return {
|
|
91
|
+
status: response.status,
|
|
92
|
+
text: null,
|
|
93
|
+
toolCalls: message.tool_calls.map((call) => toolCallOf(call.id, call.function && call.function.name, call.function && call.function.arguments))
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
const text = withoutThinking(message.content);
|
|
97
|
+
const written = readWrittenAnswer(text, Boolean(request.tools && request.tools.length));
|
|
98
|
+
if (written) return { status: response.status, ...written };
|
|
99
|
+
return { status: response.status, text, cut: choice.finish_reason === 'length' };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* A provider for createProviderRouter, over an OpenAI-compatible endpoint.
|
|
104
|
+
*
|
|
105
|
+
* @param {object} options
|
|
106
|
+
* @param {string} options.id
|
|
107
|
+
* @param {string|Function} options.url or (ctx) => url, read at every call (an account id kept with the keys).
|
|
108
|
+
* @param {Function} options.getKey (ctx) => key, read at every call: a key changed
|
|
109
|
+
* from the settings screen takes effect without a restart.
|
|
110
|
+
* @param {object[]} options.models router model entries ({ id, rpm, rpd, tpd, complexity, extra? }).
|
|
111
|
+
* A model's `extra` joins the provider's for that model only.
|
|
112
|
+
* @param {Function} options.fetch `ctx.fetch`, when given, replaces it for one call.
|
|
113
|
+
* @param {boolean} [options.external] false for a model on this machine: the router then does not mask.
|
|
114
|
+
* @param {number} [options.timeoutMs] `ctx.timeoutMs` replaces it for one call.
|
|
115
|
+
* @param {object} [options.extra] joined to each request.
|
|
116
|
+
* @param {Function} [options.lane] (ctx) => string | null — a key of its own for this use: the router rests it apart.
|
|
117
|
+
* @param {boolean} [options.detailed] true: the call resolves { text, toolCalls, cut } (an empty text
|
|
118
|
+
* included, for the router's `accepts` to judge) instead of the text alone.
|
|
119
|
+
* @param {Function} [options.toRequest] (prompt, ctx) => { system?, messages, tools?, maxTokens? }. Default: the prompt as one user message.
|
|
120
|
+
*/
|
|
121
|
+
function createOpenAICompatibleProvider(options = {}) {
|
|
122
|
+
if (typeof options.id !== 'string' || !options.id) throw new Error('createOpenAICompatibleProvider requires options.id.');
|
|
123
|
+
if (!(typeof options.url === 'function' || (typeof options.url === 'string' && options.url))) {
|
|
124
|
+
throw new Error('createOpenAICompatibleProvider requires options.url.');
|
|
125
|
+
}
|
|
126
|
+
if (typeof options.getKey !== 'function') throw new Error('createOpenAICompatibleProvider requires options.getKey.');
|
|
127
|
+
if (typeof options.fetch !== 'function') throw new Error('createOpenAICompatibleProvider requires options.fetch.');
|
|
128
|
+
const toRequest = options.toRequest || ((prompt) => ({ messages: [{ role: 'user', text: String(prompt) }] }));
|
|
129
|
+
const urlOf = (ctx) => (typeof options.url === 'function' ? options.url(ctx) : options.url);
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
id: options.id,
|
|
133
|
+
models: options.models || [],
|
|
134
|
+
...(options.external === false ? { external: false } : {}),
|
|
135
|
+
...(typeof options.lane === 'function' ? { lane: options.lane } : {}),
|
|
136
|
+
/* Sans clé (ou sans adresse), le routeur saute ce fournisseur sans rien lui demander. */
|
|
137
|
+
async available(ctx = {}) {
|
|
138
|
+
return Boolean(urlOf(ctx)) && Boolean(await options.getKey(ctx));
|
|
139
|
+
},
|
|
140
|
+
async call(prompt, ctx = {}, model = {}) {
|
|
141
|
+
const key = await options.getKey(ctx);
|
|
142
|
+
const url = urlOf(ctx);
|
|
143
|
+
if (!key || !url) {
|
|
144
|
+
const missing = new Error(`Provider "${options.id}" has no key.`);
|
|
145
|
+
missing.statusCode = 401;
|
|
146
|
+
throw missing;
|
|
147
|
+
}
|
|
148
|
+
const answer = await askChatModel(
|
|
149
|
+
{ url, key, model: model.id, extra: { ...(options.extra || {}), ...(model.extra || {}) } },
|
|
150
|
+
{ maxTokens: model.maxTokens, ...toRequest(prompt, ctx) },
|
|
151
|
+
{ fetch: typeof ctx.fetch === 'function' ? ctx.fetch : options.fetch, signal: ctx.signal, timeoutMs: ctx.timeoutMs || options.timeoutMs }
|
|
152
|
+
);
|
|
153
|
+
return providerResult(options.id, answer, options.detailed);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
module.exports = { askChatModel, createOpenAICompatibleProvider, readToolArguments, readWrittenToolCalls, withoutThinking };
|
package/src/passages.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Handing the model passages that may not be in the reader's language.
|
|
3
|
+
*
|
|
4
|
+
* A question in English may be best answered from a text that exists only in
|
|
5
|
+
* French. Dropping it loses the answer; slipping it in unmarked gets it quoted
|
|
6
|
+
* in French, or "translated" with a reference the reader cannot find. So each
|
|
7
|
+
* passage says its language, in a tag the model cannot miss — `[fr]` — and one
|
|
8
|
+
* instruction says what to do with a tagged passage: translate what you use,
|
|
9
|
+
* keep the ORIGINAL reference, so the reader can go and check it.
|
|
10
|
+
*
|
|
11
|
+
* Language tags are compared on their primary subtag: `fr-FR` and `fr` are
|
|
12
|
+
* the same language, and marking one as foreign to the other would have the
|
|
13
|
+
* model "translate" French into French.
|
|
14
|
+
*
|
|
15
|
+
* No wording lives here: the header, footer, instruction and "nothing found"
|
|
16
|
+
* lines are the caller's, per language. `{languages}` in the instruction is
|
|
17
|
+
* replaced by the tags found.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** 'fr-FR', 'FR', ' fr_ca ' -> 'fr'. Null for nothing usable. */
|
|
21
|
+
function primaryLanguage(tag) {
|
|
22
|
+
if (typeof tag !== 'string') return null;
|
|
23
|
+
const primary = tag.trim().toLowerCase().split(/[-_]/)[0];
|
|
24
|
+
return /^[a-z]{2,3}$/.test(primary) ? primary : null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function isForeign(passageLanguage, readerLanguage) {
|
|
28
|
+
const passage = primaryLanguage(passageLanguage);
|
|
29
|
+
const reader = primaryLanguage(readerLanguage);
|
|
30
|
+
return Boolean(passage && reader && passage !== reader);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const defaultTag = (language) => `[${language}]`;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @param {Array<{title?: string, text: string, lang?: string}>} passages
|
|
37
|
+
* @param {object} options
|
|
38
|
+
* @param {string} options.lang the reader's language.
|
|
39
|
+
* @param {Function} [options.tag] (primaryLanguage) => string. Default "[fr]".
|
|
40
|
+
* @returns {{ blocks: string[], languages: string[] }} one block per passage;
|
|
41
|
+
* `languages` lists the foreign tags present, in order of appearance.
|
|
42
|
+
*/
|
|
43
|
+
function markForeignPassages(passages, { lang, tag = defaultTag } = {}) {
|
|
44
|
+
const languages = [];
|
|
45
|
+
const blocks = (Array.isArray(passages) ? passages : []).filter((passage) => passage && passage.text).map((passage) => {
|
|
46
|
+
const foreign = isForeign(passage.lang, lang);
|
|
47
|
+
const primary = foreign ? primaryLanguage(passage.lang) : null;
|
|
48
|
+
if (primary && !languages.includes(primary)) languages.push(primary);
|
|
49
|
+
const title = [passage.title, primary ? tag(primary) : null].filter(Boolean).join(' ');
|
|
50
|
+
return title ? `${title}\n${passage.text}` : String(passage.text);
|
|
51
|
+
});
|
|
52
|
+
return { blocks, languages };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The passages block of a system prompt.
|
|
57
|
+
*
|
|
58
|
+
* @param {object} options
|
|
59
|
+
* @param {Array|null} options.passages null when the SEARCH failed — then
|
|
60
|
+
* nothing is said about the library, rather than "nothing found", which
|
|
61
|
+
* would be a lie; [] when it found nothing.
|
|
62
|
+
* @param {string} options.lang
|
|
63
|
+
* @param {object} options.texts { header?, footer?, foreign (required when a
|
|
64
|
+
* passage can be foreign), empty? } — the caller's words.
|
|
65
|
+
* @param {Function} [options.tag]
|
|
66
|
+
* @returns {string[]} lines to join into the system prompt.
|
|
67
|
+
*/
|
|
68
|
+
function buildPassagesContext({ passages, lang, texts = {}, tag } = {}) {
|
|
69
|
+
if (passages === null || passages === undefined) return [];
|
|
70
|
+
const { blocks, languages } = markForeignPassages(passages, { lang, tag });
|
|
71
|
+
if (!blocks.length) return texts.empty ? [texts.empty] : [];
|
|
72
|
+
if (languages.length && !(typeof texts.foreign === 'string' && texts.foreign.trim())) {
|
|
73
|
+
/* A tagged passage without the instruction is worse than no tag: the
|
|
74
|
+
model sees "[fr]" and guesses what it means. */
|
|
75
|
+
throw new Error('buildPassagesContext: a passage is in another language but texts.foreign is missing.');
|
|
76
|
+
}
|
|
77
|
+
return [
|
|
78
|
+
...(texts.header ? [texts.header] : []),
|
|
79
|
+
...blocks,
|
|
80
|
+
...(texts.footer ? [texts.footer] : []),
|
|
81
|
+
...(languages.length ? [texts.foreign.replace(/\{languages\}/g, () => languages.map((language) => (tag || defaultTag)(language)).join(', '))] : [])
|
|
82
|
+
];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The source card of a passage: its reference untouched — the original, even
|
|
87
|
+
* when the answer is a translation — and a short excerpt.
|
|
88
|
+
*/
|
|
89
|
+
function passageSource(passage, { excerptMax = 300 } = {}) {
|
|
90
|
+
const text = String((passage && passage.text) || '');
|
|
91
|
+
return {
|
|
92
|
+
...(passage.kind ? { kind: passage.kind } : {}),
|
|
93
|
+
...(passage.title ? { title: passage.title } : {}),
|
|
94
|
+
...(passage.ref !== undefined ? { ref: passage.ref } : {}),
|
|
95
|
+
...(passage.url ? { url: passage.url } : {}),
|
|
96
|
+
...(passage.lang ? { lang: passage.lang } : {}),
|
|
97
|
+
excerpt: text.length > excerptMax ? text.slice(0, excerptMax) : text
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
module.exports = { primaryLanguage, isForeignLanguage: isForeign, markForeignPassages, buildPassagesContext, passageSource };
|
package/src/providerRouter.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const { AppError } = require('@astratra/core');
|
|
2
2
|
const { randomUUID } = require('crypto');
|
|
3
|
+
const { createBreakerPool, isCircuitOpen } = require('./breakers');
|
|
3
4
|
|
|
4
5
|
const DEFAULT_COOLDOWN_MS = 60 * 1000;
|
|
5
6
|
const DEFAULT_COOLDOWN_JITTER_MS = 8 * 1000;
|
|
@@ -44,71 +45,160 @@ function createProviderRouter(config = {}) {
|
|
|
44
45
|
maxFailures: config.maxFailures ?? DEFAULT_MAX_FAILURES,
|
|
45
46
|
degradedMs: config.degradedMs ?? DEFAULT_DEGRADED_MS,
|
|
46
47
|
intentRouting: config.intentRouting || {},
|
|
47
|
-
redisKeyPrefix: config.redisKeyPrefix || 'astratra:ai:provider'
|
|
48
|
+
redisKeyPrefix: config.redisKeyPrefix || 'astratra:ai:provider',
|
|
49
|
+
/* Ce qui met un modèle au repos. Par défaut un 429 ; un projet peut y
|
|
50
|
+
ajouter un 503 « surchargé » ou un délai dépassé. */
|
|
51
|
+
cooldownOn: typeof config.cooldownOn === 'function' ? config.cooldownOn : isRateLimitError,
|
|
52
|
+
/* 'try' : quand tous les candidats se reposent, on les essaie quand même —
|
|
53
|
+
un repos est une supposition, pas un refus. */
|
|
54
|
+
whenAllCooling: config.whenAllCooling === 'try' ? 'try' : 'skip',
|
|
55
|
+
now: typeof config.now === 'function' ? config.now : Date.now
|
|
48
56
|
};
|
|
57
|
+
/* One circuit per PROVIDER, never one for all: a provider that is down must
|
|
58
|
+
not take the others with it. Optional — the per-model cooldown and
|
|
59
|
+
degradation below work without it. Resolved BEFORE any timer or
|
|
60
|
+
connection starts: a wiring error must not leave a timer behind. */
|
|
61
|
+
const breakers = resolveBreakers(config.breakers);
|
|
49
62
|
let midnightTimer = null;
|
|
50
63
|
const resetDailyAtMidnight = () => {
|
|
51
64
|
resetDailyUsage(state);
|
|
52
|
-
midnightTimer = setTimeout(resetDailyAtMidnight, msUntilNextMidnight());
|
|
65
|
+
midnightTimer = unref(setTimeout(resetDailyAtMidnight, msUntilNextMidnight()));
|
|
53
66
|
};
|
|
54
|
-
|
|
67
|
+
/* Le minuteur ne garde jamais le processus en vie : un routeur créé au
|
|
68
|
+
chargement d'un module ne doit pas empêcher un script ou des tests de finir. */
|
|
69
|
+
midnightTimer = unref(setTimeout(resetDailyAtMidnight, msUntilNextMidnight()));
|
|
55
70
|
|
|
56
71
|
const redis = createRedisLink(config, options);
|
|
57
72
|
const redisReady = redis.connect().then(() => redis.restoreState(state)).catch(() => {});
|
|
58
73
|
|
|
59
|
-
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Comme ask, mais dit aussi QUI a répondu : { value, provider, model, key, partial }.
|
|
77
|
+
* `key` est la clé d'état ("fournisseur:modèle", suivie de "@voie" quand le
|
|
78
|
+
* fournisseur a une voie pour cet appel).
|
|
79
|
+
*/
|
|
80
|
+
async function route(prompt, request = {}, ctx = {}) {
|
|
60
81
|
await redisReady;
|
|
61
|
-
const
|
|
82
|
+
const signal = ctx && ctx.signal;
|
|
83
|
+
const masker = ctx && ctx.masker && typeof ctx.masker.mask === 'function' ? ctx.masker : null;
|
|
84
|
+
const providerCtx = masker ? withoutMasker(ctx) : ctx;
|
|
85
|
+
const accepts = typeof request.accepts === 'function' ? request.accepts : null;
|
|
86
|
+
const partial = typeof request.partial === 'function' ? request.partial : null;
|
|
87
|
+
|
|
88
|
+
/* Un fournisseur sans clé (ou éteint) est sauté sans bruit : ni quota, ni échec. */
|
|
89
|
+
const reachable = [];
|
|
90
|
+
for (const candidate of selectCandidates(providers, options.intentRouting, request)) {
|
|
91
|
+
if (typeof candidate.provider.available === 'function' && !(await candidate.provider.available(providerCtx))) continue;
|
|
92
|
+
reachable.push(candidate);
|
|
93
|
+
}
|
|
94
|
+
if (!reachable.length) throw unavailable('AI_NO_PROVIDER', 'No AI provider is available for this request.');
|
|
95
|
+
const matching = typeof request.select === 'function'
|
|
96
|
+
? reachable.filter(({ provider, model }) => request.select(model, provider))
|
|
97
|
+
: reachable;
|
|
98
|
+
if (!matching.length) throw unavailable('AI_NO_MATCH', 'No available AI model matches this request.');
|
|
99
|
+
|
|
100
|
+
const keyed = matching.map((candidate) => ({ ...candidate, key: laneKey(candidate.provider, candidate.model, providerCtx) }));
|
|
101
|
+
const awake = keyed.filter(({ key, model }) => isModelAvailable(state, key, model, request, options.now()));
|
|
102
|
+
const tried = awake.length || options.whenAllCooling !== 'try' ? awake : keyed;
|
|
103
|
+
|
|
62
104
|
const errors = [];
|
|
105
|
+
let last = null;
|
|
106
|
+
let kept = null;
|
|
107
|
+
for (const { provider, model, key } of tried) {
|
|
108
|
+
stopIfAborted(signal);
|
|
109
|
+
const reservation = await redis.reserveUsage(key, model, request);
|
|
110
|
+
if (!reservation) continue;
|
|
63
111
|
|
|
64
|
-
|
|
65
|
-
const { provider, model } = candidate;
|
|
66
|
-
const key = modelKey(provider, model);
|
|
67
|
-
if (!isModelAvailable(state, key, model, request)) continue;
|
|
68
|
-
if (!await redis.reserveUsage(key, model, request)) continue;
|
|
112
|
+
const undo = reserveUsage(state, key, request, options.now());
|
|
69
113
|
|
|
70
|
-
|
|
114
|
+
/* Masked at the moment it leaves: a provider that runs on this machine
|
|
115
|
+
(`external: false`) gets the text as it is. */
|
|
116
|
+
const outbound = masker && provider.external !== false;
|
|
117
|
+
const send = async () => provider.call(outbound ? await masker.maskAsync(prompt) : prompt, providerCtx, model);
|
|
71
118
|
|
|
119
|
+
let raw;
|
|
72
120
|
try {
|
|
73
|
-
|
|
74
|
-
clearFailures(state, key);
|
|
75
|
-
redis.mirrorFailures(key, state.models[key]);
|
|
76
|
-
return result;
|
|
121
|
+
raw = breakers ? await breakers.run(provider.id, send) : await send();
|
|
77
122
|
} catch (error) {
|
|
123
|
+
/* La personne est partie : rien n'a échoué, on s'arrête là. */
|
|
124
|
+
stopIfAborted(signal);
|
|
78
125
|
errors.push(error);
|
|
126
|
+
last = `${key}: ${error && error.message}`;
|
|
127
|
+
if (isCircuitOpen(error)) {
|
|
128
|
+
/* The breaker refused before anything was sent: nothing was used,
|
|
129
|
+
and the model itself did not fail. */
|
|
130
|
+
undo();
|
|
131
|
+
redis.releaseUsage(reservation);
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
79
134
|
markFailure(state, key, error, options);
|
|
80
135
|
redis.mirrorFailure(key, state.models[key]);
|
|
136
|
+
continue;
|
|
81
137
|
}
|
|
138
|
+
clearFailures(state, key);
|
|
139
|
+
redis.mirrorFailures(key, state.models[key]);
|
|
140
|
+
const value = outbound ? unmaskResult(masker, raw) : raw;
|
|
141
|
+
/* Une réponse coupée à sa longueur n'est pas donnée tant qu'un autre
|
|
142
|
+
modèle peut la finir : gardée en dernier recours. */
|
|
143
|
+
if (partial && partial(value) && (!accepts || accepts(value))) {
|
|
144
|
+
kept = kept || { value, provider: provider.id, model: model.id, key, partial: true };
|
|
145
|
+
last = `${key} stopped short`;
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (accepts && !accepts(value)) {
|
|
149
|
+
/* Une réponse que l'appelant refuse (outil inconnu, texte vide) passe
|
|
150
|
+
au modèle suivant ; ce n'est pas une panne du modèle. */
|
|
151
|
+
last = `${key} gave a response that was not accepted`;
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
return { value, provider: provider.id, model: model.id, key, partial: false };
|
|
82
155
|
}
|
|
83
156
|
|
|
84
|
-
|
|
85
|
-
|
|
157
|
+
if (kept) return kept;
|
|
158
|
+
const error = unavailable('AI_UNAVAILABLE', `No available AI provider/model for this request.${last ? ` Last error: ${last}` : ''}`);
|
|
159
|
+
error.errors = errors;
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function ask(prompt, request = {}, ctx = {}) {
|
|
164
|
+
return (await route(prompt, request, ctx)).value;
|
|
86
165
|
}
|
|
87
166
|
|
|
88
167
|
function getStats() {
|
|
89
|
-
const now =
|
|
168
|
+
const now = options.now();
|
|
90
169
|
const stats = {};
|
|
170
|
+
const entry = (providerId, model, modelState) => ({
|
|
171
|
+
provider: providerId,
|
|
172
|
+
rpm_now: currentRpm(modelState, now),
|
|
173
|
+
rpm_limit: model.rpm ?? null,
|
|
174
|
+
rpd_used: modelState.rpdUsed,
|
|
175
|
+
rpd_limit: model.rpd ?? null,
|
|
176
|
+
tpd_used: modelState.tpdUsed,
|
|
177
|
+
tpd_limit: model.tpd ?? null,
|
|
178
|
+
cooldown: isUntilActive(modelState.cooldownUntil, now),
|
|
179
|
+
degraded: isUntilActive(modelState.degradedUntil, now),
|
|
180
|
+
failures: modelState.failures || 0,
|
|
181
|
+
circuit: breakers ? breakers.stateOf(providerId) || 'closed' : null
|
|
182
|
+
});
|
|
91
183
|
providers.forEach(provider => {
|
|
92
184
|
(provider.models || []).forEach(model => {
|
|
93
185
|
const key = modelKey(provider, model);
|
|
94
|
-
|
|
95
|
-
stats[key] = {
|
|
96
|
-
provider: provider.id,
|
|
97
|
-
rpm_now: currentRpm(modelState, now),
|
|
98
|
-
rpm_limit: model.rpm ?? null,
|
|
99
|
-
rpd_used: modelState.rpdUsed,
|
|
100
|
-
rpd_limit: model.rpd ?? null,
|
|
101
|
-
tpd_used: modelState.tpdUsed,
|
|
102
|
-
tpd_limit: model.tpd ?? null,
|
|
103
|
-
cooldown: isUntilActive(modelState.cooldownUntil, now),
|
|
104
|
-
degraded: isUntilActive(modelState.degradedUntil, now),
|
|
105
|
-
failures: modelState.failures || 0
|
|
106
|
-
};
|
|
186
|
+
stats[key] = entry(provider.id, model, state.models[key] || createModelState());
|
|
107
187
|
});
|
|
108
188
|
});
|
|
189
|
+
/* Les modèles demandés par `request.candidates` et les voies n'existent
|
|
190
|
+
qu'une fois utilisés : ils apparaissent alors, sans limites déclarées. */
|
|
191
|
+
Object.entries(state.models).forEach(([key, modelState]) => {
|
|
192
|
+
if (!stats[key]) stats[key] = entry(key.split(':')[0], {}, modelState);
|
|
193
|
+
});
|
|
109
194
|
return stats;
|
|
110
195
|
}
|
|
111
196
|
|
|
197
|
+
/** Oublie repos, dégradations, échecs et usage (tests, remise à zéro manuelle). */
|
|
198
|
+
function reset() {
|
|
199
|
+
Object.keys(state.models).forEach((key) => { state.models[key] = createModelState(); });
|
|
200
|
+
}
|
|
201
|
+
|
|
112
202
|
function stop() {
|
|
113
203
|
if (midnightTimer) {
|
|
114
204
|
clearTimeout(midnightTimer);
|
|
@@ -117,7 +207,29 @@ function createProviderRouter(config = {}) {
|
|
|
117
207
|
redis.disconnect();
|
|
118
208
|
}
|
|
119
209
|
|
|
120
|
-
return { ask, getStats, stop };
|
|
210
|
+
return { ask, route, getStats, reset, stop, breakers };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** A pool, a factory, or nothing. */
|
|
214
|
+
function resolveBreakers(value) {
|
|
215
|
+
if (!value) return null;
|
|
216
|
+
if (typeof value === 'function') return createBreakerPool({ create: value });
|
|
217
|
+
if (typeof value.run === 'function' && typeof value.get === 'function') return value;
|
|
218
|
+
throw new Error('createProviderRouter: breakers must be a factory (providerId) => breaker or a pool from createBreakerPool().');
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/* The masker is the router's business; the provider never sees it. */
|
|
222
|
+
function withoutMasker(ctx) {
|
|
223
|
+
const rest = { ...ctx };
|
|
224
|
+
delete rest.masker;
|
|
225
|
+
return rest;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function unmaskResult(masker, value) {
|
|
229
|
+
if (value && typeof value[Symbol.asyncIterator] === 'function' && typeof masker.unmaskStream === 'function') {
|
|
230
|
+
return masker.unmaskStream(value);
|
|
231
|
+
}
|
|
232
|
+
return typeof masker.unmaskDeep === 'function' ? masker.unmaskDeep(value) : value;
|
|
121
233
|
}
|
|
122
234
|
|
|
123
235
|
function createInitialState(providers) {
|
|
@@ -148,6 +260,7 @@ function createRedisLink(config, options) {
|
|
|
148
260
|
connect: async () => {},
|
|
149
261
|
restoreState: async () => {},
|
|
150
262
|
reserveUsage: async () => true,
|
|
263
|
+
releaseUsage: () => {},
|
|
151
264
|
mirrorFailure: () => {},
|
|
152
265
|
mirrorFailures: () => {},
|
|
153
266
|
disconnect: () => {}
|
|
@@ -205,6 +318,8 @@ function createRedisLink(config, options) {
|
|
|
205
318
|
|
|
206
319
|
async function reserveUsage(key, model, request) {
|
|
207
320
|
if (!client || !client.isOpen) return true;
|
|
321
|
+
const reservationId = randomUUID();
|
|
322
|
+
const tokenCost = estimatedTokens(request);
|
|
208
323
|
try {
|
|
209
324
|
const reserved = await client.eval(RESERVE_USAGE_SCRIPT, {
|
|
210
325
|
keys: [
|
|
@@ -217,19 +332,29 @@ function createRedisLink(config, options) {
|
|
|
217
332
|
String(model.rpm || 0),
|
|
218
333
|
String(model.rpd || 0),
|
|
219
334
|
String(model.tpd || 0),
|
|
220
|
-
String(
|
|
221
|
-
|
|
335
|
+
String(tokenCost),
|
|
336
|
+
reservationId,
|
|
222
337
|
String(RPM_WINDOW_MS),
|
|
223
338
|
String(Math.ceil(RPM_WINDOW_MS / 1000)),
|
|
224
339
|
String(secondsUntilMidnight())
|
|
225
340
|
]
|
|
226
341
|
});
|
|
227
|
-
return Number(reserved) === 1;
|
|
342
|
+
return Number(reserved) === 1 ? { key, reservationId, tokenCost } : false;
|
|
228
343
|
} catch (_error) {
|
|
229
344
|
return true;
|
|
230
345
|
}
|
|
231
346
|
}
|
|
232
347
|
|
|
348
|
+
/* Best effort: a reservation for a call that never left is given back. */
|
|
349
|
+
function releaseUsage(reservation) {
|
|
350
|
+
if (!reservation || reservation === true || !client || !client.isOpen) return;
|
|
351
|
+
const { key, reservationId, tokenCost } = reservation;
|
|
352
|
+
const quietly = (fn) => { try { Promise.resolve(fn()).catch(() => {}); } catch (_error) { /* never surfaces */ } };
|
|
353
|
+
quietly(() => client.zRem(stateKey('rpm', key), reservationId));
|
|
354
|
+
quietly(() => client.decrBy(dayKey('rpd', key), 1));
|
|
355
|
+
if (tokenCost) quietly(() => client.decrBy(dayKey('tpd', key), tokenCost));
|
|
356
|
+
}
|
|
357
|
+
|
|
233
358
|
function mirrorFailures(key, modelState) {
|
|
234
359
|
if (!client || !client.isOpen) return;
|
|
235
360
|
client.del(stateKey('failures', key)).catch(() => {});
|
|
@@ -256,10 +381,33 @@ function createRedisLink(config, options) {
|
|
|
256
381
|
client = null;
|
|
257
382
|
}
|
|
258
383
|
|
|
259
|
-
return { connect, restoreState, reserveUsage, mirrorFailure, mirrorFailures, disconnect };
|
|
384
|
+
return { connect, restoreState, reserveUsage, releaseUsage, mirrorFailure, mirrorFailures, disconnect };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/*
|
|
388
|
+
* `request.candidates` : l'ordre de CETTE demande, à travers les fournisseurs
|
|
389
|
+
* ([{ provider, model, ...champs du modèle }]). Un projet a souvent plusieurs
|
|
390
|
+
* ordres pour les mêmes modèles (le plus rapide d'abord pour ce qu'on attend,
|
|
391
|
+
* le plus capable d'abord pour le reste). Un fournisseur inconnu est ignoré ;
|
|
392
|
+
* un modèle non déclaré par le fournisseur est pris tel que décrit.
|
|
393
|
+
*/
|
|
394
|
+
function explicitCandidates(providers, list) {
|
|
395
|
+
const byId = new Map(providers.map((provider) => [provider.id, provider]));
|
|
396
|
+
return list.flatMap((entry) => {
|
|
397
|
+
const provider = entry && byId.get(entry.provider);
|
|
398
|
+
if (!provider || !entry.model) return [];
|
|
399
|
+
const fields = { ...entry };
|
|
400
|
+
delete fields.provider;
|
|
401
|
+
delete fields.model;
|
|
402
|
+
const declared = (provider.models || []).find((model) => model.id === entry.model);
|
|
403
|
+
return [{ provider, model: { ...(declared || {}), ...fields, id: entry.model } }];
|
|
404
|
+
});
|
|
260
405
|
}
|
|
261
406
|
|
|
262
407
|
function selectCandidates(providers, intentRouting, request) {
|
|
408
|
+
if (Array.isArray(request.candidates)) {
|
|
409
|
+
return explicitCandidates(providers, request.candidates).filter(({ model }) => supportsComplexity(model, request.complexity));
|
|
410
|
+
}
|
|
263
411
|
const preferred = request.intent && intentRouting[request.intent]
|
|
264
412
|
? intentRouting[request.intent].preferred || []
|
|
265
413
|
: [];
|
|
@@ -287,8 +435,7 @@ function supportsComplexity(model, complexity) {
|
|
|
287
435
|
return Array.isArray(model.complexity) && model.complexity.includes(complexity);
|
|
288
436
|
}
|
|
289
437
|
|
|
290
|
-
function isModelAvailable(state, key, model, request) {
|
|
291
|
-
const now = Date.now();
|
|
438
|
+
function isModelAvailable(state, key, model, request, now) {
|
|
292
439
|
const modelState = state.models[key] || createModelState();
|
|
293
440
|
state.models[key] = modelState;
|
|
294
441
|
|
|
@@ -300,14 +447,21 @@ function isModelAvailable(state, key, model, request) {
|
|
|
300
447
|
return true;
|
|
301
448
|
}
|
|
302
449
|
|
|
303
|
-
function reserveUsage(state, key, request) {
|
|
450
|
+
function reserveUsage(state, key, request, now) {
|
|
304
451
|
const modelState = state.models[key] || createModelState();
|
|
305
|
-
const
|
|
452
|
+
const tokens = estimatedTokens(request);
|
|
306
453
|
modelState.rpmWindow = modelState.rpmWindow.filter(ts => now - ts < RPM_WINDOW_MS);
|
|
307
454
|
modelState.rpmWindow.push(now);
|
|
308
455
|
modelState.rpdUsed += 1;
|
|
309
|
-
modelState.tpdUsed +=
|
|
456
|
+
modelState.tpdUsed += tokens;
|
|
310
457
|
state.models[key] = modelState;
|
|
458
|
+
/* Undo, for a call the breaker refused before it left. */
|
|
459
|
+
return () => {
|
|
460
|
+
const index = modelState.rpmWindow.lastIndexOf(now);
|
|
461
|
+
if (index !== -1) modelState.rpmWindow.splice(index, 1);
|
|
462
|
+
modelState.rpdUsed = Math.max(0, modelState.rpdUsed - 1);
|
|
463
|
+
modelState.tpdUsed = Math.max(0, modelState.tpdUsed - tokens);
|
|
464
|
+
};
|
|
311
465
|
}
|
|
312
466
|
|
|
313
467
|
function estimatedTokens(request) {
|
|
@@ -327,12 +481,12 @@ function markFailure(state, modelId, error, options) {
|
|
|
327
481
|
const modelState = state.models[modelId] || createModelState();
|
|
328
482
|
modelState.failures = (modelState.failures || 0) + 1;
|
|
329
483
|
|
|
330
|
-
if (
|
|
331
|
-
modelState.cooldownUntil =
|
|
484
|
+
if (options.cooldownOn(error)) {
|
|
485
|
+
modelState.cooldownUntil = options.now() + options.cooldownMs + jitter(options.cooldownJitterMs);
|
|
332
486
|
}
|
|
333
487
|
|
|
334
488
|
if (modelState.failures >= options.maxFailures) {
|
|
335
|
-
modelState.degradedUntil =
|
|
489
|
+
modelState.degradedUntil = options.now() + options.degradedMs;
|
|
336
490
|
}
|
|
337
491
|
|
|
338
492
|
state.models[modelId] = modelState;
|
|
@@ -348,6 +502,32 @@ function modelKey(provider, model) {
|
|
|
348
502
|
return `${provider.id}:${model.id}`;
|
|
349
503
|
}
|
|
350
504
|
|
|
505
|
+
/*
|
|
506
|
+
* Une voie : une clé à part pour un usage (son propre quota gratuit). Elle se
|
|
507
|
+
* repose seule — une clé saturée par un travail de fond n'arrête jamais les
|
|
508
|
+
* conversations servies par la clé générale.
|
|
509
|
+
*/
|
|
510
|
+
function laneKey(provider, model, ctx) {
|
|
511
|
+
const lane = typeof provider.lane === 'function' ? provider.lane(ctx) : null;
|
|
512
|
+
return lane ? `${modelKey(provider, model)}@${lane}` : modelKey(provider, model);
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
function unavailable(code, message) {
|
|
516
|
+
const error = new AppError(message, 503);
|
|
517
|
+
error.code = code;
|
|
518
|
+
return error;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/* Une demande abandonnée par l'appelant s'arrête net, avec la raison de l'appelant. */
|
|
522
|
+
function stopIfAborted(signal) {
|
|
523
|
+
if (signal && signal.aborted) throw signal.reason;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
function unref(timer) {
|
|
527
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
528
|
+
return timer;
|
|
529
|
+
}
|
|
530
|
+
|
|
351
531
|
function isRateLimitError(error) {
|
|
352
532
|
return error && (
|
|
353
533
|
error.statusCode === 429 ||
|