@astratra/ai 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -0
- package/package.json +2 -1
- package/src/agentLoop.js +88 -11
- package/src/answerText.js +117 -0
- package/src/askLimit.js +69 -0
- package/src/breakers.js +97 -0
- package/src/index.d.ts +252 -0
- package/src/index.js +9 -1
- package/src/language.js +85 -0
- package/src/masking.js +279 -0
- package/src/openaiCompatible.js +170 -0
- package/src/passages.js +101 -0
- package/src/providerRouter.js +77 -11
- package/src/sources.js +243 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Providers that speak the OpenAI chat format (Groq, Mistral, Cloudflare
|
|
3
|
+
* Workers AI, gateways…): one request shape, one way to read the answer.
|
|
4
|
+
*
|
|
5
|
+
* Three things real traffic taught:
|
|
6
|
+
*
|
|
7
|
+
* - a model sometimes returns tool arguments that are not JSON. Parsing them
|
|
8
|
+
* blindly threw inside the turn and lost the whole answer; a malformed call
|
|
9
|
+
* is now reported as such and the loop decides;
|
|
10
|
+
* - reasoning models write their thinking between think tags inside the
|
|
11
|
+
* answer — never shown, even when the closing tag is missing;
|
|
12
|
+
* - an error carries the HTTP status (a 429 must reach the router's cooldown)
|
|
13
|
+
* and never the key.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const THINK = /<think>[\s\S]*?(?:<\/think>|$)/gi;
|
|
17
|
+
|
|
18
|
+
function toTranscript(messages) {
|
|
19
|
+
return (Array.isArray(messages) ? messages : []).map((message) => {
|
|
20
|
+
if (message.role === 'assistant') {
|
|
21
|
+
return {
|
|
22
|
+
role: 'assistant',
|
|
23
|
+
content: message.text === undefined ? message.content || '' : message.text,
|
|
24
|
+
...(message.toolCalls && message.toolCalls.length ? {
|
|
25
|
+
tool_calls: message.toolCalls.map((call) => ({
|
|
26
|
+
id: call.id,
|
|
27
|
+
type: 'function',
|
|
28
|
+
function: { name: call.name, arguments: JSON.stringify(call.args === undefined ? {} : call.args) }
|
|
29
|
+
}))
|
|
30
|
+
} : {})
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (message.role === 'tool') {
|
|
34
|
+
return { role: 'tool', tool_call_id: message.toolCallId, content: typeof message.result === 'string' ? message.result : JSON.stringify(message.result) };
|
|
35
|
+
}
|
|
36
|
+
const text = message.text === undefined ? message.content || '' : message.text;
|
|
37
|
+
/* A photo joined by the person: an image part with a data address. Only
|
|
38
|
+
send it to models that see. */
|
|
39
|
+
if (message.images && message.images.length) {
|
|
40
|
+
return {
|
|
41
|
+
role: 'user',
|
|
42
|
+
content: [
|
|
43
|
+
{ type: 'text', text },
|
|
44
|
+
...message.images.map(({ mimeType, data }) => ({ type: 'image_url', image_url: { url: `data:${mimeType};base64,${data}` } }))
|
|
45
|
+
]
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
return { role: message.role === 'system' ? 'system' : 'user', content: text };
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Tool arguments as an object, or null with `invalid: true` for anything else. */
|
|
53
|
+
function readToolArguments(raw) {
|
|
54
|
+
if (raw === undefined || raw === null || raw === '') return { args: {}, invalid: false };
|
|
55
|
+
if (typeof raw === 'object') return { args: raw, invalid: false };
|
|
56
|
+
try {
|
|
57
|
+
const parsed = JSON.parse(String(raw));
|
|
58
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? { args: parsed, invalid: false } : { args: null, invalid: true };
|
|
59
|
+
} catch (_error) {
|
|
60
|
+
return { args: null, invalid: true };
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Ask one model.
|
|
66
|
+
* @param {{ url: string, key: string, model: string, extra?: object }} target
|
|
67
|
+
* @param {{ system?: string, messages: object[], tools?: object[], maxTokens?: number, temperature?: number }} request
|
|
68
|
+
* @param {{ fetch: Function, signal?: AbortSignal, timeoutMs?: number }} io
|
|
69
|
+
* @returns {Promise<{ status: number, text?: string|null, toolCalls?: object[], cut?: boolean }>}
|
|
70
|
+
* A non-2xx answer resolves `{ status }` — nothing is thrown for it.
|
|
71
|
+
*/
|
|
72
|
+
async function askChatModel(target, request, io = {}) {
|
|
73
|
+
if (typeof io.fetch !== 'function') throw new Error('askChatModel requires io.fetch.');
|
|
74
|
+
const { url, key, model, extra = {} } = target || {};
|
|
75
|
+
const timeoutMs = io.timeoutMs || 45_000;
|
|
76
|
+
const Signal = globalThis.AbortSignal;
|
|
77
|
+
const timeout = Signal.timeout(timeoutMs);
|
|
78
|
+
const signal = io.signal && typeof Signal.any === 'function' ? Signal.any([io.signal, timeout]) : (io.signal || timeout);
|
|
79
|
+
const messages = [
|
|
80
|
+
...(request.system ? [{ role: 'system', content: request.system }] : []),
|
|
81
|
+
...toTranscript(request.messages)
|
|
82
|
+
];
|
|
83
|
+
const response = await io.fetch(url, {
|
|
84
|
+
method: 'POST',
|
|
85
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
|
|
86
|
+
body: JSON.stringify({
|
|
87
|
+
model,
|
|
88
|
+
messages,
|
|
89
|
+
temperature: request.temperature === undefined ? 0.4 : request.temperature,
|
|
90
|
+
max_tokens: request.maxTokens || 1200,
|
|
91
|
+
...(request.tools && request.tools.length ? { tools: request.tools.map((tool) => ({ type: 'function', function: tool })) } : {}),
|
|
92
|
+
...extra
|
|
93
|
+
}),
|
|
94
|
+
signal
|
|
95
|
+
});
|
|
96
|
+
if (!response.ok) return { status: response.status };
|
|
97
|
+
const answer = await response.json();
|
|
98
|
+
const choice = (answer && answer.choices && answer.choices[0]) || {};
|
|
99
|
+
const message = choice.message || {};
|
|
100
|
+
if (Array.isArray(message.tool_calls) && message.tool_calls.length) {
|
|
101
|
+
return {
|
|
102
|
+
status: response.status,
|
|
103
|
+
text: null,
|
|
104
|
+
toolCalls: message.tool_calls.map((call) => {
|
|
105
|
+
const { args, invalid } = readToolArguments(call.function && call.function.arguments);
|
|
106
|
+
return { id: call.id, name: call.function && call.function.name, args, ...(invalid ? { invalid: true } : {}) };
|
|
107
|
+
})
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
const text = String(message.content === null || message.content === undefined ? '' : message.content).replace(THINK, '').trim();
|
|
111
|
+
return { status: response.status, text, cut: choice.finish_reason === 'length' };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* A provider for createProviderRouter, over an OpenAI-compatible endpoint.
|
|
116
|
+
*
|
|
117
|
+
* @param {object} options
|
|
118
|
+
* @param {string} options.id
|
|
119
|
+
* @param {string} options.url
|
|
120
|
+
* @param {Function} options.getKey () => key, read at every call: a key changed
|
|
121
|
+
* from the settings screen takes effect without a restart.
|
|
122
|
+
* @param {object[]} options.models router model entries ({ id, rpm, rpd, tpd, complexity }).
|
|
123
|
+
* @param {Function} options.fetch
|
|
124
|
+
* @param {boolean} [options.external] false for a model on this machine: the router then does not mask.
|
|
125
|
+
* @param {number} [options.timeoutMs]
|
|
126
|
+
* @param {object} [options.extra] joined to each request.
|
|
127
|
+
* @param {Function} [options.toRequest] (prompt, ctx) => { system?, messages }. Default: the prompt as one user message.
|
|
128
|
+
*/
|
|
129
|
+
function createOpenAICompatibleProvider(options = {}) {
|
|
130
|
+
for (const field of ['id', 'url']) {
|
|
131
|
+
if (typeof options[field] !== 'string' || !options[field]) throw new Error(`createOpenAICompatibleProvider requires options.${field}.`);
|
|
132
|
+
}
|
|
133
|
+
if (typeof options.getKey !== 'function') throw new Error('createOpenAICompatibleProvider requires options.getKey.');
|
|
134
|
+
if (typeof options.fetch !== 'function') throw new Error('createOpenAICompatibleProvider requires options.fetch.');
|
|
135
|
+
const toRequest = options.toRequest || ((prompt) => ({ messages: [{ role: 'user', text: String(prompt) }] }));
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
id: options.id,
|
|
139
|
+
models: options.models || [],
|
|
140
|
+
...(options.external === false ? { external: false } : {}),
|
|
141
|
+
async call(prompt, ctx = {}, model = {}) {
|
|
142
|
+
const key = await options.getKey();
|
|
143
|
+
if (!key) {
|
|
144
|
+
const missing = new Error(`Provider "${options.id}" has no key.`);
|
|
145
|
+
missing.statusCode = 401;
|
|
146
|
+
throw missing;
|
|
147
|
+
}
|
|
148
|
+
const answer = await askChatModel(
|
|
149
|
+
{ url: options.url, key, model: model.id, extra: options.extra },
|
|
150
|
+
{ maxTokens: model.maxTokens, ...toRequest(prompt, ctx) },
|
|
151
|
+
{ fetch: options.fetch, signal: ctx.signal, timeoutMs: options.timeoutMs }
|
|
152
|
+
);
|
|
153
|
+
if (answer.status < 200 || answer.status >= 300) {
|
|
154
|
+
/* The status reaches the router (429 → cooldown); the key never does. */
|
|
155
|
+
const failure = new Error(`Provider "${options.id}" answered ${answer.status}.`);
|
|
156
|
+
failure.statusCode = answer.status;
|
|
157
|
+
throw failure;
|
|
158
|
+
}
|
|
159
|
+
if (answer.toolCalls) return { toolCalls: answer.toolCalls };
|
|
160
|
+
if (!answer.text) {
|
|
161
|
+
const empty = new Error(`Provider "${options.id}" gave no text.`);
|
|
162
|
+
empty.statusCode = 502;
|
|
163
|
+
throw empty;
|
|
164
|
+
}
|
|
165
|
+
return answer.text;
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
module.exports = { askChatModel, createOpenAICompatibleProvider, readToolArguments };
|
package/src/passages.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Handing the model passages that may not be in the reader's language.
|
|
3
|
+
*
|
|
4
|
+
* A question in English may be best answered from a text that exists only in
|
|
5
|
+
* French. Dropping it loses the answer; slipping it in unmarked gets it quoted
|
|
6
|
+
* in French, or "translated" with a reference the reader cannot find. So each
|
|
7
|
+
* passage says its language, in a tag the model cannot miss — `[fr]` — and one
|
|
8
|
+
* instruction says what to do with a tagged passage: translate what you use,
|
|
9
|
+
* keep the ORIGINAL reference, so the reader can go and check it.
|
|
10
|
+
*
|
|
11
|
+
* Language tags are compared on their primary subtag: `fr-FR` and `fr` are
|
|
12
|
+
* the same language, and marking one as foreign to the other would have the
|
|
13
|
+
* model "translate" French into French.
|
|
14
|
+
*
|
|
15
|
+
* No wording lives here: the header, footer, instruction and "nothing found"
|
|
16
|
+
* lines are the caller's, per language. `{languages}` in the instruction is
|
|
17
|
+
* replaced by the tags found.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** 'fr-FR', 'FR', ' fr_ca ' -> 'fr'. Null for nothing usable. */
|
|
21
|
+
function primaryLanguage(tag) {
|
|
22
|
+
if (typeof tag !== 'string') return null;
|
|
23
|
+
const primary = tag.trim().toLowerCase().split(/[-_]/)[0];
|
|
24
|
+
return /^[a-z]{2,3}$/.test(primary) ? primary : null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function isForeign(passageLanguage, readerLanguage) {
|
|
28
|
+
const passage = primaryLanguage(passageLanguage);
|
|
29
|
+
const reader = primaryLanguage(readerLanguage);
|
|
30
|
+
return Boolean(passage && reader && passage !== reader);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const defaultTag = (language) => `[${language}]`;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @param {Array<{title?: string, text: string, lang?: string}>} passages
|
|
37
|
+
* @param {object} options
|
|
38
|
+
* @param {string} options.lang the reader's language.
|
|
39
|
+
* @param {Function} [options.tag] (primaryLanguage) => string. Default "[fr]".
|
|
40
|
+
* @returns {{ blocks: string[], languages: string[] }} one block per passage;
|
|
41
|
+
* `languages` lists the foreign tags present, in order of appearance.
|
|
42
|
+
*/
|
|
43
|
+
function markForeignPassages(passages, { lang, tag = defaultTag } = {}) {
|
|
44
|
+
const languages = [];
|
|
45
|
+
const blocks = (Array.isArray(passages) ? passages : []).filter((passage) => passage && passage.text).map((passage) => {
|
|
46
|
+
const foreign = isForeign(passage.lang, lang);
|
|
47
|
+
const primary = foreign ? primaryLanguage(passage.lang) : null;
|
|
48
|
+
if (primary && !languages.includes(primary)) languages.push(primary);
|
|
49
|
+
const title = [passage.title, primary ? tag(primary) : null].filter(Boolean).join(' ');
|
|
50
|
+
return title ? `${title}\n${passage.text}` : String(passage.text);
|
|
51
|
+
});
|
|
52
|
+
return { blocks, languages };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The passages block of a system prompt.
|
|
57
|
+
*
|
|
58
|
+
* @param {object} options
|
|
59
|
+
* @param {Array|null} options.passages null when the SEARCH failed — then
|
|
60
|
+
* nothing is said about the library, rather than "nothing found", which
|
|
61
|
+
* would be a lie; [] when it found nothing.
|
|
62
|
+
* @param {string} options.lang
|
|
63
|
+
* @param {object} options.texts { header?, footer?, foreign (required when a
|
|
64
|
+
* passage can be foreign), empty? } — the caller's words.
|
|
65
|
+
* @param {Function} [options.tag]
|
|
66
|
+
* @returns {string[]} lines to join into the system prompt.
|
|
67
|
+
*/
|
|
68
|
+
function buildPassagesContext({ passages, lang, texts = {}, tag } = {}) {
|
|
69
|
+
if (passages === null || passages === undefined) return [];
|
|
70
|
+
const { blocks, languages } = markForeignPassages(passages, { lang, tag });
|
|
71
|
+
if (!blocks.length) return texts.empty ? [texts.empty] : [];
|
|
72
|
+
if (languages.length && !(typeof texts.foreign === 'string' && texts.foreign.trim())) {
|
|
73
|
+
/* A tagged passage without the instruction is worse than no tag: the
|
|
74
|
+
model sees "[fr]" and guesses what it means. */
|
|
75
|
+
throw new Error('buildPassagesContext: a passage is in another language but texts.foreign is missing.');
|
|
76
|
+
}
|
|
77
|
+
return [
|
|
78
|
+
...(texts.header ? [texts.header] : []),
|
|
79
|
+
...blocks,
|
|
80
|
+
...(texts.footer ? [texts.footer] : []),
|
|
81
|
+
...(languages.length ? [texts.foreign.replace(/\{languages\}/g, () => languages.map((language) => (tag || defaultTag)(language)).join(', '))] : [])
|
|
82
|
+
];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The source card of a passage: its reference untouched — the original, even
|
|
87
|
+
* when the answer is a translation — and a short excerpt.
|
|
88
|
+
*/
|
|
89
|
+
function passageSource(passage, { excerptMax = 300 } = {}) {
|
|
90
|
+
const text = String((passage && passage.text) || '');
|
|
91
|
+
return {
|
|
92
|
+
...(passage.kind ? { kind: passage.kind } : {}),
|
|
93
|
+
...(passage.title ? { title: passage.title } : {}),
|
|
94
|
+
...(passage.ref !== undefined ? { ref: passage.ref } : {}),
|
|
95
|
+
...(passage.url ? { url: passage.url } : {}),
|
|
96
|
+
...(passage.lang ? { lang: passage.lang } : {}),
|
|
97
|
+
excerpt: text.length > excerptMax ? text.slice(0, excerptMax) : text
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
module.exports = { primaryLanguage, isForeignLanguage: isForeign, markForeignPassages, buildPassagesContext, passageSource };
|
package/src/providerRouter.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const { AppError } = require('@astratra/core');
|
|
2
2
|
const { randomUUID } = require('crypto');
|
|
3
|
+
const { createBreakerPool, isCircuitOpen } = require('./breakers');
|
|
3
4
|
|
|
4
5
|
const DEFAULT_COOLDOWN_MS = 60 * 1000;
|
|
5
6
|
const DEFAULT_COOLDOWN_JITTER_MS = 8 * 1000;
|
|
@@ -46,6 +47,11 @@ function createProviderRouter(config = {}) {
|
|
|
46
47
|
intentRouting: config.intentRouting || {},
|
|
47
48
|
redisKeyPrefix: config.redisKeyPrefix || 'astratra:ai:provider'
|
|
48
49
|
};
|
|
50
|
+
/* One circuit per PROVIDER, never one for all: a provider that is down must
|
|
51
|
+
not take the others with it. Optional — the per-model cooldown and
|
|
52
|
+
degradation below work without it. Resolved BEFORE any timer or
|
|
53
|
+
connection starts: a wiring error must not leave a timer behind. */
|
|
54
|
+
const breakers = resolveBreakers(config.breakers);
|
|
49
55
|
let midnightTimer = null;
|
|
50
56
|
const resetDailyAtMidnight = () => {
|
|
51
57
|
resetDailyUsage(state);
|
|
@@ -56,26 +62,42 @@ function createProviderRouter(config = {}) {
|
|
|
56
62
|
const redis = createRedisLink(config, options);
|
|
57
63
|
const redisReady = redis.connect().then(() => redis.restoreState(state)).catch(() => {});
|
|
58
64
|
|
|
65
|
+
|
|
59
66
|
async function ask(prompt, request = {}, ctx = {}) {
|
|
60
67
|
await redisReady;
|
|
61
68
|
const candidates = selectCandidates(providers, options.intentRouting, request);
|
|
62
69
|
const errors = [];
|
|
70
|
+
const masker = ctx && ctx.masker && typeof ctx.masker.mask === 'function' ? ctx.masker : null;
|
|
71
|
+
const providerCtx = masker ? withoutMasker(ctx) : ctx;
|
|
63
72
|
|
|
64
73
|
for (const candidate of candidates) {
|
|
65
74
|
const { provider, model } = candidate;
|
|
66
75
|
const key = modelKey(provider, model);
|
|
67
76
|
if (!isModelAvailable(state, key, model, request)) continue;
|
|
68
|
-
|
|
77
|
+
const reservation = await redis.reserveUsage(key, model, request);
|
|
78
|
+
if (!reservation) continue;
|
|
79
|
+
|
|
80
|
+
const undo = reserveUsage(state, key, request);
|
|
69
81
|
|
|
70
|
-
|
|
82
|
+
/* Masked at the moment it leaves: a provider that runs on this machine
|
|
83
|
+
(`external: false`) gets the text as it is. */
|
|
84
|
+
const outbound = masker && provider.external !== false;
|
|
85
|
+
const send = async () => provider.call(outbound ? await masker.maskAsync(prompt) : prompt, providerCtx, model);
|
|
71
86
|
|
|
72
87
|
try {
|
|
73
|
-
const
|
|
88
|
+
const raw = breakers ? await breakers.run(provider.id, send) : await send();
|
|
74
89
|
clearFailures(state, key);
|
|
75
90
|
redis.mirrorFailures(key, state.models[key]);
|
|
76
|
-
return
|
|
91
|
+
return outbound ? unmaskResult(masker, raw) : raw;
|
|
77
92
|
} catch (error) {
|
|
78
93
|
errors.push(error);
|
|
94
|
+
if (isCircuitOpen(error)) {
|
|
95
|
+
/* The breaker refused before anything was sent: nothing was used,
|
|
96
|
+
and the model itself did not fail. */
|
|
97
|
+
undo();
|
|
98
|
+
redis.releaseUsage(reservation);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
79
101
|
markFailure(state, key, error, options);
|
|
80
102
|
redis.mirrorFailure(key, state.models[key]);
|
|
81
103
|
}
|
|
@@ -102,7 +124,8 @@ function createProviderRouter(config = {}) {
|
|
|
102
124
|
tpd_limit: model.tpd ?? null,
|
|
103
125
|
cooldown: isUntilActive(modelState.cooldownUntil, now),
|
|
104
126
|
degraded: isUntilActive(modelState.degradedUntil, now),
|
|
105
|
-
failures: modelState.failures || 0
|
|
127
|
+
failures: modelState.failures || 0,
|
|
128
|
+
circuit: breakers ? breakers.stateOf(provider.id) || 'closed' : null
|
|
106
129
|
};
|
|
107
130
|
});
|
|
108
131
|
});
|
|
@@ -117,7 +140,29 @@ function createProviderRouter(config = {}) {
|
|
|
117
140
|
redis.disconnect();
|
|
118
141
|
}
|
|
119
142
|
|
|
120
|
-
return { ask, getStats, stop };
|
|
143
|
+
return { ask, getStats, stop, breakers };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** A pool, a factory, or nothing. */
|
|
147
|
+
function resolveBreakers(value) {
|
|
148
|
+
if (!value) return null;
|
|
149
|
+
if (typeof value === 'function') return createBreakerPool({ create: value });
|
|
150
|
+
if (typeof value.run === 'function' && typeof value.get === 'function') return value;
|
|
151
|
+
throw new Error('createProviderRouter: breakers must be a factory (providerId) => breaker or a pool from createBreakerPool().');
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/* The masker is the router's business; the provider never sees it. */
|
|
155
|
+
function withoutMasker(ctx) {
|
|
156
|
+
const rest = { ...ctx };
|
|
157
|
+
delete rest.masker;
|
|
158
|
+
return rest;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function unmaskResult(masker, value) {
|
|
162
|
+
if (value && typeof value[Symbol.asyncIterator] === 'function' && typeof masker.unmaskStream === 'function') {
|
|
163
|
+
return masker.unmaskStream(value);
|
|
164
|
+
}
|
|
165
|
+
return typeof masker.unmaskDeep === 'function' ? masker.unmaskDeep(value) : value;
|
|
121
166
|
}
|
|
122
167
|
|
|
123
168
|
function createInitialState(providers) {
|
|
@@ -148,6 +193,7 @@ function createRedisLink(config, options) {
|
|
|
148
193
|
connect: async () => {},
|
|
149
194
|
restoreState: async () => {},
|
|
150
195
|
reserveUsage: async () => true,
|
|
196
|
+
releaseUsage: () => {},
|
|
151
197
|
mirrorFailure: () => {},
|
|
152
198
|
mirrorFailures: () => {},
|
|
153
199
|
disconnect: () => {}
|
|
@@ -205,6 +251,8 @@ function createRedisLink(config, options) {
|
|
|
205
251
|
|
|
206
252
|
async function reserveUsage(key, model, request) {
|
|
207
253
|
if (!client || !client.isOpen) return true;
|
|
254
|
+
const reservationId = randomUUID();
|
|
255
|
+
const tokenCost = estimatedTokens(request);
|
|
208
256
|
try {
|
|
209
257
|
const reserved = await client.eval(RESERVE_USAGE_SCRIPT, {
|
|
210
258
|
keys: [
|
|
@@ -217,19 +265,29 @@ function createRedisLink(config, options) {
|
|
|
217
265
|
String(model.rpm || 0),
|
|
218
266
|
String(model.rpd || 0),
|
|
219
267
|
String(model.tpd || 0),
|
|
220
|
-
String(
|
|
221
|
-
|
|
268
|
+
String(tokenCost),
|
|
269
|
+
reservationId,
|
|
222
270
|
String(RPM_WINDOW_MS),
|
|
223
271
|
String(Math.ceil(RPM_WINDOW_MS / 1000)),
|
|
224
272
|
String(secondsUntilMidnight())
|
|
225
273
|
]
|
|
226
274
|
});
|
|
227
|
-
return Number(reserved) === 1;
|
|
275
|
+
return Number(reserved) === 1 ? { key, reservationId, tokenCost } : false;
|
|
228
276
|
} catch (_error) {
|
|
229
277
|
return true;
|
|
230
278
|
}
|
|
231
279
|
}
|
|
232
280
|
|
|
281
|
+
/* Best effort: a reservation for a call that never left is given back. */
|
|
282
|
+
function releaseUsage(reservation) {
|
|
283
|
+
if (!reservation || reservation === true || !client || !client.isOpen) return;
|
|
284
|
+
const { key, reservationId, tokenCost } = reservation;
|
|
285
|
+
const quietly = (fn) => { try { Promise.resolve(fn()).catch(() => {}); } catch (_error) { /* never surfaces */ } };
|
|
286
|
+
quietly(() => client.zRem(stateKey('rpm', key), reservationId));
|
|
287
|
+
quietly(() => client.decrBy(dayKey('rpd', key), 1));
|
|
288
|
+
if (tokenCost) quietly(() => client.decrBy(dayKey('tpd', key), tokenCost));
|
|
289
|
+
}
|
|
290
|
+
|
|
233
291
|
function mirrorFailures(key, modelState) {
|
|
234
292
|
if (!client || !client.isOpen) return;
|
|
235
293
|
client.del(stateKey('failures', key)).catch(() => {});
|
|
@@ -256,7 +314,7 @@ function createRedisLink(config, options) {
|
|
|
256
314
|
client = null;
|
|
257
315
|
}
|
|
258
316
|
|
|
259
|
-
return { connect, restoreState, reserveUsage, mirrorFailure, mirrorFailures, disconnect };
|
|
317
|
+
return { connect, restoreState, reserveUsage, releaseUsage, mirrorFailure, mirrorFailures, disconnect };
|
|
260
318
|
}
|
|
261
319
|
|
|
262
320
|
function selectCandidates(providers, intentRouting, request) {
|
|
@@ -303,11 +361,19 @@ function isModelAvailable(state, key, model, request) {
|
|
|
303
361
|
function reserveUsage(state, key, request) {
|
|
304
362
|
const modelState = state.models[key] || createModelState();
|
|
305
363
|
const now = Date.now();
|
|
364
|
+
const tokens = estimatedTokens(request);
|
|
306
365
|
modelState.rpmWindow = modelState.rpmWindow.filter(ts => now - ts < RPM_WINDOW_MS);
|
|
307
366
|
modelState.rpmWindow.push(now);
|
|
308
367
|
modelState.rpdUsed += 1;
|
|
309
|
-
modelState.tpdUsed +=
|
|
368
|
+
modelState.tpdUsed += tokens;
|
|
310
369
|
state.models[key] = modelState;
|
|
370
|
+
/* Undo, for a call the breaker refused before it left. */
|
|
371
|
+
return () => {
|
|
372
|
+
const index = modelState.rpmWindow.lastIndexOf(now);
|
|
373
|
+
if (index !== -1) modelState.rpmWindow.splice(index, 1);
|
|
374
|
+
modelState.rpdUsed = Math.max(0, modelState.rpdUsed - 1);
|
|
375
|
+
modelState.tpdUsed = Math.max(0, modelState.tpdUsed - tokens);
|
|
376
|
+
};
|
|
311
377
|
}
|
|
312
378
|
|
|
313
379
|
function estimatedTokens(request) {
|