@astratra/ai 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Providers that speak the OpenAI chat format (Groq, Mistral, Cloudflare
3
+ * Workers AI, gateways…): one request shape, one way to read the answer.
4
+ *
5
+ * Three things real traffic taught:
6
+ *
7
+ * - a model sometimes returns tool arguments that are not JSON. Parsing them
8
+ * blindly threw inside the turn and lost the whole answer; a malformed call
9
+ * is now reported as such and the loop decides;
10
+ * - reasoning models write their thinking between think tags inside the
11
+ * answer — never shown, even when the closing tag is missing;
12
+ * - an error carries the HTTP status (a 429 must reach the router's cooldown)
13
+ * and never the key.
14
+ */
15
+
16
+ const THINK = /<think>[\s\S]*?(?:<\/think>|$)/gi;
17
+
18
+ function toTranscript(messages) {
19
+ return (Array.isArray(messages) ? messages : []).map((message) => {
20
+ if (message.role === 'assistant') {
21
+ return {
22
+ role: 'assistant',
23
+ content: message.text === undefined ? message.content || '' : message.text,
24
+ ...(message.toolCalls && message.toolCalls.length ? {
25
+ tool_calls: message.toolCalls.map((call) => ({
26
+ id: call.id,
27
+ type: 'function',
28
+ function: { name: call.name, arguments: JSON.stringify(call.args === undefined ? {} : call.args) }
29
+ }))
30
+ } : {})
31
+ };
32
+ }
33
+ if (message.role === 'tool') {
34
+ return { role: 'tool', tool_call_id: message.toolCallId, content: typeof message.result === 'string' ? message.result : JSON.stringify(message.result) };
35
+ }
36
+ const text = message.text === undefined ? message.content || '' : message.text;
37
+ /* A photo joined by the person: an image part with a data address. Only
38
+ send it to models that see. */
39
+ if (message.images && message.images.length) {
40
+ return {
41
+ role: 'user',
42
+ content: [
43
+ { type: 'text', text },
44
+ ...message.images.map(({ mimeType, data }) => ({ type: 'image_url', image_url: { url: `data:${mimeType};base64,${data}` } }))
45
+ ]
46
+ };
47
+ }
48
+ return { role: message.role === 'system' ? 'system' : 'user', content: text };
49
+ });
50
+ }
51
+
52
+ /** Tool arguments as an object, or null with `invalid: true` for anything else. */
53
+ function readToolArguments(raw) {
54
+ if (raw === undefined || raw === null || raw === '') return { args: {}, invalid: false };
55
+ if (typeof raw === 'object') return { args: raw, invalid: false };
56
+ try {
57
+ const parsed = JSON.parse(String(raw));
58
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? { args: parsed, invalid: false } : { args: null, invalid: true };
59
+ } catch (_error) {
60
+ return { args: null, invalid: true };
61
+ }
62
+ }
63
+
64
+ /**
65
+ * Ask one model.
66
+ * @param {{ url: string, key: string, model: string, extra?: object }} target
67
+ * @param {{ system?: string, messages: object[], tools?: object[], maxTokens?: number, temperature?: number }} request
68
+ * @param {{ fetch: Function, signal?: AbortSignal, timeoutMs?: number }} io
69
+ * @returns {Promise<{ status: number, text?: string|null, toolCalls?: object[], cut?: boolean }>}
70
+ * A non-2xx answer resolves `{ status }` — nothing is thrown for it.
71
+ */
72
+ async function askChatModel(target, request, io = {}) {
73
+ if (typeof io.fetch !== 'function') throw new Error('askChatModel requires io.fetch.');
74
+ const { url, key, model, extra = {} } = target || {};
75
+ const timeoutMs = io.timeoutMs || 45_000;
76
+ const Signal = globalThis.AbortSignal;
77
+ const timeout = Signal.timeout(timeoutMs);
78
+ const signal = io.signal && typeof Signal.any === 'function' ? Signal.any([io.signal, timeout]) : (io.signal || timeout);
79
+ const messages = [
80
+ ...(request.system ? [{ role: 'system', content: request.system }] : []),
81
+ ...toTranscript(request.messages)
82
+ ];
83
+ const response = await io.fetch(url, {
84
+ method: 'POST',
85
+ headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
86
+ body: JSON.stringify({
87
+ model,
88
+ messages,
89
+ temperature: request.temperature === undefined ? 0.4 : request.temperature,
90
+ max_tokens: request.maxTokens || 1200,
91
+ ...(request.tools && request.tools.length ? { tools: request.tools.map((tool) => ({ type: 'function', function: tool })) } : {}),
92
+ ...extra
93
+ }),
94
+ signal
95
+ });
96
+ if (!response.ok) return { status: response.status };
97
+ const answer = await response.json();
98
+ const choice = (answer && answer.choices && answer.choices[0]) || {};
99
+ const message = choice.message || {};
100
+ if (Array.isArray(message.tool_calls) && message.tool_calls.length) {
101
+ return {
102
+ status: response.status,
103
+ text: null,
104
+ toolCalls: message.tool_calls.map((call) => {
105
+ const { args, invalid } = readToolArguments(call.function && call.function.arguments);
106
+ return { id: call.id, name: call.function && call.function.name, args, ...(invalid ? { invalid: true } : {}) };
107
+ })
108
+ };
109
+ }
110
+ const text = String(message.content === null || message.content === undefined ? '' : message.content).replace(THINK, '').trim();
111
+ return { status: response.status, text, cut: choice.finish_reason === 'length' };
112
+ }
113
+
114
+ /**
115
+ * A provider for createProviderRouter, over an OpenAI-compatible endpoint.
116
+ *
117
+ * @param {object} options
118
+ * @param {string} options.id
119
+ * @param {string} options.url
120
+ * @param {Function} options.getKey () => key, read at every call: a key changed
121
+ * from the settings screen takes effect without a restart.
122
+ * @param {object[]} options.models router model entries ({ id, rpm, rpd, tpd, complexity }).
123
+ * @param {Function} options.fetch
124
+ * @param {boolean} [options.external] false for a model on this machine: the router then does not mask.
125
+ * @param {number} [options.timeoutMs]
126
+ * @param {object} [options.extra] joined to each request.
127
+ * @param {Function} [options.toRequest] (prompt, ctx) => { system?, messages }. Default: the prompt as one user message.
128
+ */
129
+ function createOpenAICompatibleProvider(options = {}) {
130
+ for (const field of ['id', 'url']) {
131
+ if (typeof options[field] !== 'string' || !options[field]) throw new Error(`createOpenAICompatibleProvider requires options.${field}.`);
132
+ }
133
+ if (typeof options.getKey !== 'function') throw new Error('createOpenAICompatibleProvider requires options.getKey.');
134
+ if (typeof options.fetch !== 'function') throw new Error('createOpenAICompatibleProvider requires options.fetch.');
135
+ const toRequest = options.toRequest || ((prompt) => ({ messages: [{ role: 'user', text: String(prompt) }] }));
136
+
137
+ return {
138
+ id: options.id,
139
+ models: options.models || [],
140
+ ...(options.external === false ? { external: false } : {}),
141
+ async call(prompt, ctx = {}, model = {}) {
142
+ const key = await options.getKey();
143
+ if (!key) {
144
+ const missing = new Error(`Provider "${options.id}" has no key.`);
145
+ missing.statusCode = 401;
146
+ throw missing;
147
+ }
148
+ const answer = await askChatModel(
149
+ { url: options.url, key, model: model.id, extra: options.extra },
150
+ { maxTokens: model.maxTokens, ...toRequest(prompt, ctx) },
151
+ { fetch: options.fetch, signal: ctx.signal, timeoutMs: options.timeoutMs }
152
+ );
153
+ if (answer.status < 200 || answer.status >= 300) {
154
+ /* The status reaches the router (429 → cooldown); the key never does. */
155
+ const failure = new Error(`Provider "${options.id}" answered ${answer.status}.`);
156
+ failure.statusCode = answer.status;
157
+ throw failure;
158
+ }
159
+ if (answer.toolCalls) return { toolCalls: answer.toolCalls };
160
+ if (!answer.text) {
161
+ const empty = new Error(`Provider "${options.id}" gave no text.`);
162
+ empty.statusCode = 502;
163
+ throw empty;
164
+ }
165
+ return answer.text;
166
+ }
167
+ };
168
+ }
169
+
170
+ module.exports = { askChatModel, createOpenAICompatibleProvider, readToolArguments };
@@ -0,0 +1,101 @@
1
+ /**
2
+ * Handing the model passages that may not be in the reader's language.
3
+ *
4
+ * A question in English may be best answered from a text that exists only in
5
+ * French. Dropping it loses the answer; slipping it in unmarked gets it quoted
6
+ * in French, or "translated" with a reference the reader cannot find. So each
7
+ * passage says its language, in a tag the model cannot miss — `[fr]` — and one
8
+ * instruction says what to do with a tagged passage: translate what you use,
9
+ * keep the ORIGINAL reference, so the reader can go and check it.
10
+ *
11
+ * Language tags are compared on their primary subtag: `fr-FR` and `fr` are
12
+ * the same language, and marking one as foreign to the other would have the
13
+ * model "translate" French into French.
14
+ *
15
+ * No wording lives here: the header, footer, instruction and "nothing found"
16
+ * lines are the caller's, per language. `{languages}` in the instruction is
17
+ * replaced by the tags found.
18
+ */
19
+
20
+ /** 'fr-FR', 'FR', ' fr_ca ' -> 'fr'. Null for nothing usable. */
21
+ function primaryLanguage(tag) {
22
+ if (typeof tag !== 'string') return null;
23
+ const primary = tag.trim().toLowerCase().split(/[-_]/)[0];
24
+ return /^[a-z]{2,3}$/.test(primary) ? primary : null;
25
+ }
26
+
27
+ function isForeign(passageLanguage, readerLanguage) {
28
+ const passage = primaryLanguage(passageLanguage);
29
+ const reader = primaryLanguage(readerLanguage);
30
+ return Boolean(passage && reader && passage !== reader);
31
+ }
32
+
33
+ const defaultTag = (language) => `[${language}]`;
34
+
35
+ /**
36
+ * @param {Array<{title?: string, text: string, lang?: string}>} passages
37
+ * @param {object} options
38
+ * @param {string} options.lang the reader's language.
39
+ * @param {Function} [options.tag] (primaryLanguage) => string. Default "[fr]".
40
+ * @returns {{ blocks: string[], languages: string[] }} one block per passage;
41
+ * `languages` lists the foreign tags present, in order of appearance.
42
+ */
43
+ function markForeignPassages(passages, { lang, tag = defaultTag } = {}) {
44
+ const languages = [];
45
+ const blocks = (Array.isArray(passages) ? passages : []).filter((passage) => passage && passage.text).map((passage) => {
46
+ const foreign = isForeign(passage.lang, lang);
47
+ const primary = foreign ? primaryLanguage(passage.lang) : null;
48
+ if (primary && !languages.includes(primary)) languages.push(primary);
49
+ const title = [passage.title, primary ? tag(primary) : null].filter(Boolean).join(' ');
50
+ return title ? `${title}\n${passage.text}` : String(passage.text);
51
+ });
52
+ return { blocks, languages };
53
+ }
54
+
55
+ /**
56
+ * The passages block of a system prompt.
57
+ *
58
+ * @param {object} options
59
+ * @param {Array|null} options.passages null when the SEARCH failed — then
60
+ * nothing is said about the library, rather than "nothing found", which
61
+ * would be a lie; [] when it found nothing.
62
+ * @param {string} options.lang
63
+ * @param {object} options.texts { header?, footer?, foreign (required when a
64
+ * passage can be foreign), empty? } — the caller's words.
65
+ * @param {Function} [options.tag]
66
+ * @returns {string[]} lines to join into the system prompt.
67
+ */
68
+ function buildPassagesContext({ passages, lang, texts = {}, tag } = {}) {
69
+ if (passages === null || passages === undefined) return [];
70
+ const { blocks, languages } = markForeignPassages(passages, { lang, tag });
71
+ if (!blocks.length) return texts.empty ? [texts.empty] : [];
72
+ if (languages.length && !(typeof texts.foreign === 'string' && texts.foreign.trim())) {
73
+ /* A tagged passage without the instruction is worse than no tag: the
74
+ model sees "[fr]" and guesses what it means. */
75
+ throw new Error('buildPassagesContext: a passage is in another language but texts.foreign is missing.');
76
+ }
77
+ return [
78
+ ...(texts.header ? [texts.header] : []),
79
+ ...blocks,
80
+ ...(texts.footer ? [texts.footer] : []),
81
+ ...(languages.length ? [texts.foreign.replace(/\{languages\}/g, () => languages.map((language) => (tag || defaultTag)(language)).join(', '))] : [])
82
+ ];
83
+ }
84
+
85
+ /**
86
+ * The source card of a passage: its reference untouched — the original, even
87
+ * when the answer is a translation — and a short excerpt.
88
+ */
89
+ function passageSource(passage, { excerptMax = 300 } = {}) {
90
+ const text = String((passage && passage.text) || '');
91
+ return {
92
+ ...(passage.kind ? { kind: passage.kind } : {}),
93
+ ...(passage.title ? { title: passage.title } : {}),
94
+ ...(passage.ref !== undefined ? { ref: passage.ref } : {}),
95
+ ...(passage.url ? { url: passage.url } : {}),
96
+ ...(passage.lang ? { lang: passage.lang } : {}),
97
+ excerpt: text.length > excerptMax ? text.slice(0, excerptMax) : text
98
+ };
99
+ }
100
+
101
+ module.exports = { primaryLanguage, isForeignLanguage: isForeign, markForeignPassages, buildPassagesContext, passageSource };
@@ -1,5 +1,6 @@
1
1
  const { AppError } = require('@astratra/core');
2
2
  const { randomUUID } = require('crypto');
3
+ const { createBreakerPool, isCircuitOpen } = require('./breakers');
3
4
 
4
5
  const DEFAULT_COOLDOWN_MS = 60 * 1000;
5
6
  const DEFAULT_COOLDOWN_JITTER_MS = 8 * 1000;
@@ -46,6 +47,11 @@ function createProviderRouter(config = {}) {
46
47
  intentRouting: config.intentRouting || {},
47
48
  redisKeyPrefix: config.redisKeyPrefix || 'astratra:ai:provider'
48
49
  };
50
+ /* One circuit per PROVIDER, never one for all: a provider that is down must
51
+ not take the others with it. Optional — the per-model cooldown and
52
+ degradation below work without it. Resolved BEFORE any timer or
53
+ connection starts: a wiring error must not leave a timer behind. */
54
+ const breakers = resolveBreakers(config.breakers);
49
55
  let midnightTimer = null;
50
56
  const resetDailyAtMidnight = () => {
51
57
  resetDailyUsage(state);
@@ -56,26 +62,42 @@ function createProviderRouter(config = {}) {
56
62
  const redis = createRedisLink(config, options);
57
63
  const redisReady = redis.connect().then(() => redis.restoreState(state)).catch(() => {});
58
64
 
65
+
59
66
  async function ask(prompt, request = {}, ctx = {}) {
60
67
  await redisReady;
61
68
  const candidates = selectCandidates(providers, options.intentRouting, request);
62
69
  const errors = [];
70
+ const masker = ctx && ctx.masker && typeof ctx.masker.mask === 'function' ? ctx.masker : null;
71
+ const providerCtx = masker ? withoutMasker(ctx) : ctx;
63
72
 
64
73
  for (const candidate of candidates) {
65
74
  const { provider, model } = candidate;
66
75
  const key = modelKey(provider, model);
67
76
  if (!isModelAvailable(state, key, model, request)) continue;
68
- if (!await redis.reserveUsage(key, model, request)) continue;
77
+ const reservation = await redis.reserveUsage(key, model, request);
78
+ if (!reservation) continue;
79
+
80
+ const undo = reserveUsage(state, key, request);
69
81
 
70
- reserveUsage(state, key, request);
82
+ /* Masked at the moment it leaves: a provider that runs on this machine
83
+ (`external: false`) gets the text as it is. */
84
+ const outbound = masker && provider.external !== false;
85
+ const send = async () => provider.call(outbound ? await masker.maskAsync(prompt) : prompt, providerCtx, model);
71
86
 
72
87
  try {
73
- const result = await provider.call(prompt, ctx, model);
88
+ const raw = breakers ? await breakers.run(provider.id, send) : await send();
74
89
  clearFailures(state, key);
75
90
  redis.mirrorFailures(key, state.models[key]);
76
- return result;
91
+ return outbound ? unmaskResult(masker, raw) : raw;
77
92
  } catch (error) {
78
93
  errors.push(error);
94
+ if (isCircuitOpen(error)) {
95
+ /* The breaker refused before anything was sent: nothing was used,
96
+ and the model itself did not fail. */
97
+ undo();
98
+ redis.releaseUsage(reservation);
99
+ continue;
100
+ }
79
101
  markFailure(state, key, error, options);
80
102
  redis.mirrorFailure(key, state.models[key]);
81
103
  }
@@ -102,7 +124,8 @@ function createProviderRouter(config = {}) {
102
124
  tpd_limit: model.tpd ?? null,
103
125
  cooldown: isUntilActive(modelState.cooldownUntil, now),
104
126
  degraded: isUntilActive(modelState.degradedUntil, now),
105
- failures: modelState.failures || 0
127
+ failures: modelState.failures || 0,
128
+ circuit: breakers ? breakers.stateOf(provider.id) || 'closed' : null
106
129
  };
107
130
  });
108
131
  });
@@ -117,7 +140,29 @@ function createProviderRouter(config = {}) {
117
140
  redis.disconnect();
118
141
  }
119
142
 
120
- return { ask, getStats, stop };
143
+ return { ask, getStats, stop, breakers };
144
+ }
145
+
146
+ /** A pool, a factory, or nothing. */
147
+ function resolveBreakers(value) {
148
+ if (!value) return null;
149
+ if (typeof value === 'function') return createBreakerPool({ create: value });
150
+ if (typeof value.run === 'function' && typeof value.get === 'function') return value;
151
+ throw new Error('createProviderRouter: breakers must be a factory (providerId) => breaker or a pool from createBreakerPool().');
152
+ }
153
+
154
+ /* The masker is the router's business; the provider never sees it. */
155
+ function withoutMasker(ctx) {
156
+ const rest = { ...ctx };
157
+ delete rest.masker;
158
+ return rest;
159
+ }
160
+
161
+ function unmaskResult(masker, value) {
162
+ if (value && typeof value[Symbol.asyncIterator] === 'function' && typeof masker.unmaskStream === 'function') {
163
+ return masker.unmaskStream(value);
164
+ }
165
+ return typeof masker.unmaskDeep === 'function' ? masker.unmaskDeep(value) : value;
121
166
  }
122
167
 
123
168
  function createInitialState(providers) {
@@ -148,6 +193,7 @@ function createRedisLink(config, options) {
148
193
  connect: async () => {},
149
194
  restoreState: async () => {},
150
195
  reserveUsage: async () => true,
196
+ releaseUsage: () => {},
151
197
  mirrorFailure: () => {},
152
198
  mirrorFailures: () => {},
153
199
  disconnect: () => {}
@@ -205,6 +251,8 @@ function createRedisLink(config, options) {
205
251
 
206
252
  async function reserveUsage(key, model, request) {
207
253
  if (!client || !client.isOpen) return true;
254
+ const reservationId = randomUUID();
255
+ const tokenCost = estimatedTokens(request);
208
256
  try {
209
257
  const reserved = await client.eval(RESERVE_USAGE_SCRIPT, {
210
258
  keys: [
@@ -217,19 +265,29 @@ function createRedisLink(config, options) {
217
265
  String(model.rpm || 0),
218
266
  String(model.rpd || 0),
219
267
  String(model.tpd || 0),
220
- String(estimatedTokens(request)),
221
- randomUUID(),
268
+ String(tokenCost),
269
+ reservationId,
222
270
  String(RPM_WINDOW_MS),
223
271
  String(Math.ceil(RPM_WINDOW_MS / 1000)),
224
272
  String(secondsUntilMidnight())
225
273
  ]
226
274
  });
227
- return Number(reserved) === 1;
275
+ return Number(reserved) === 1 ? { key, reservationId, tokenCost } : false;
228
276
  } catch (_error) {
229
277
  return true;
230
278
  }
231
279
  }
232
280
 
281
+ /* Best effort: a reservation for a call that never left is given back. */
282
+ function releaseUsage(reservation) {
283
+ if (!reservation || reservation === true || !client || !client.isOpen) return;
284
+ const { key, reservationId, tokenCost } = reservation;
285
+ const quietly = (fn) => { try { Promise.resolve(fn()).catch(() => {}); } catch (_error) { /* never surfaces */ } };
286
+ quietly(() => client.zRem(stateKey('rpm', key), reservationId));
287
+ quietly(() => client.decrBy(dayKey('rpd', key), 1));
288
+ if (tokenCost) quietly(() => client.decrBy(dayKey('tpd', key), tokenCost));
289
+ }
290
+
233
291
  function mirrorFailures(key, modelState) {
234
292
  if (!client || !client.isOpen) return;
235
293
  client.del(stateKey('failures', key)).catch(() => {});
@@ -256,7 +314,7 @@ function createRedisLink(config, options) {
256
314
  client = null;
257
315
  }
258
316
 
259
- return { connect, restoreState, reserveUsage, mirrorFailure, mirrorFailures, disconnect };
317
+ return { connect, restoreState, reserveUsage, releaseUsage, mirrorFailure, mirrorFailures, disconnect };
260
318
  }
261
319
 
262
320
  function selectCandidates(providers, intentRouting, request) {
@@ -303,11 +361,19 @@ function isModelAvailable(state, key, model, request) {
303
361
  function reserveUsage(state, key, request) {
304
362
  const modelState = state.models[key] || createModelState();
305
363
  const now = Date.now();
364
+ const tokens = estimatedTokens(request);
306
365
  modelState.rpmWindow = modelState.rpmWindow.filter(ts => now - ts < RPM_WINDOW_MS);
307
366
  modelState.rpmWindow.push(now);
308
367
  modelState.rpdUsed += 1;
309
- modelState.tpdUsed += estimatedTokens(request);
368
+ modelState.tpdUsed += tokens;
310
369
  state.models[key] = modelState;
370
+ /* Undo, for a call the breaker refused before it left. */
371
+ return () => {
372
+ const index = modelState.rpmWindow.lastIndexOf(now);
373
+ if (index !== -1) modelState.rpmWindow.splice(index, 1);
374
+ modelState.rpdUsed = Math.max(0, modelState.rpdUsed - 1);
375
+ modelState.tpdUsed = Math.max(0, modelState.tpdUsed - tokens);
376
+ };
311
377
  }
312
378
 
313
379
  function estimatedTokens(request) {