@astratra/ai 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +458 -6
- package/deploy/llama-cpp/.env.example +10 -0
- package/deploy/llama-cpp/docker-compose.yml +47 -0
- package/package.json +7 -4
- package/src/agentLoop.js +101 -5
- package/src/aiConfig.js +72 -0
- package/src/gemini.js +9 -3
- package/src/index.d.ts +374 -5
- package/src/index.js +10 -1
- package/src/llamaCpp.js +192 -0
- package/src/localModels.js +99 -0
- package/src/managedRouter.js +258 -0
- package/src/modelAnswers.js +1 -0
- package/src/nativeTools.js +44 -5
- package/src/openaiCompatible.js +5 -1
- package/src/providerLimits.js +45 -0
- package/src/routingState.js +87 -0
- package/src/serviceHttp.js +56 -0
- package/src/serviceLanguage.js +23 -0
- package/src/toolRegistry.js +13 -2
- package/src/voice.js +181 -0
package/README.md
CHANGED
|
@@ -4,7 +4,9 @@ Routing IA multi-provider générique, registre d'outils et une boucle
|
|
|
4
4
|
d'agent minimale à tool-calling. Dépend de `@astratra/core`.
|
|
5
5
|
|
|
6
6
|
Ce package ne fournit volontairement aucun catalogue de modèles, aucun SDK
|
|
7
|
-
provider, aucun outil métier — tout ça vient du projet consommateur.
|
|
7
|
+
provider, aucun outil métier — tout ça vient du projet consommateur. Le routeur
|
|
8
|
+
configuré, les clients de modèles locaux et les services vocaux ci-dessous
|
|
9
|
+
partagent désormais les mécanismes utilisés par Oracle et Tertius. Ce
|
|
8
10
|
qu'il fournit, c'est le mécanisme durement acquis : suivi de quota, ordre de
|
|
9
11
|
fallback, cooldown/dégradation, deux adaptateurs HTTP sans dépendance (le
|
|
10
12
|
format OpenAI et Gemini), et deux boucles d'agent (protocole écrit, ou
|
|
@@ -177,6 +179,7 @@ registry.register({
|
|
|
177
179
|
name: 'get_patient_record',
|
|
178
180
|
description: "Récupère le dossier d'un patient par son id",
|
|
179
181
|
type: 'read',
|
|
182
|
+
risk: 'LOW', // LOW, MEDIUM ou HIGH ; absent = UNKNOWN
|
|
180
183
|
roles: ['doctor', 'admin'],
|
|
181
184
|
params: { patientId: 'string' },
|
|
182
185
|
handler: async ({ patientId }, ctx) => patientStore.findById(patientId)
|
|
@@ -222,10 +225,46 @@ comme résultat d'outil) et continue, il peut réagir (expliquer, proposer
|
|
|
222
225
|
autre chose, s'arrêter). Omis, chaque outil autorisé s'exécute automatiquement,
|
|
223
226
|
comme avant.
|
|
224
227
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
228
|
+
### Garde-fous de la boucle, résultats volumineux et risque
|
|
229
|
+
|
|
230
|
+
Les appels répétés avec le même nom et les mêmes paramètres JSON (l'ordre des
|
|
231
|
+
clés d'objet ne compte pas) reçoivent un rappel aux répétitions 3 et 5, puis
|
|
232
|
+
la boucle s'arrête à 8. Ces seuils se règlent avec `loopGuard`; `onEvent`
|
|
233
|
+
reçoit `tool_loop_warning` ou `tool_loop_stopped`. Le nombre d'étapes par
|
|
234
|
+
défaut reste 5 : l'arrêt à 8 ne joue que si `maxSteps` est réglé à 8 ou plus.
|
|
235
|
+
|
|
236
|
+
```js
|
|
237
|
+
await runAgentLoop({
|
|
238
|
+
prompt, registry, router, userRole: 'doctor',
|
|
239
|
+
loopGuard: { reminder: 3, firmReminder: 5, stop: 8 },
|
|
240
|
+
spill: { threshold: 12_000 },
|
|
241
|
+
confirmationPolicy: { threshold: 'HIGH' },
|
|
242
|
+
riskAnalyzers: [async (call, ctx, tool) => classifier(call, ctx, tool)],
|
|
243
|
+
pendingActions
|
|
244
|
+
});
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
Au-dessus du seuil (`spill.threshold`, 12 000 caractères par défaut), le
|
|
248
|
+
résultat complet est écrit dans `spill.store` (stockage mémoire par défaut).
|
|
249
|
+
Le modèle reçoit un extrait et un identifiant, puis peut appeler
|
|
250
|
+
`read_spilled_result` avec `{ id, start, length }` pour lire une tranche. Si
|
|
251
|
+
l'écriture échoue, le contenu est supprimé de la réponse et le modèle reçoit
|
|
252
|
+
`result_storage_failed`. Un stockage injecté expose `set(id, value)` et
|
|
253
|
+
`get(id)`.
|
|
254
|
+
|
|
255
|
+
Les risques possibles sont `LOW`, `MEDIUM`, `HIGH` et `UNKNOWN` (valeur des
|
|
256
|
+
outils sans déclaration). La politique de confirmation est inactive par
|
|
257
|
+
défaut, pour préserver l'exécution existante. Elle accepte `'always'`,
|
|
258
|
+
`'never'` ou `{ threshold: 'MEDIUM' }`; dans ce dernier cas, les risques au
|
|
259
|
+
moins aussi élevés que le seuil, dont `UNKNOWN`, demandent une confirmation.
|
|
260
|
+
Les `riskAnalyzers` reçoivent l'appel, le contexte et l'outil ; le niveau le
|
|
261
|
+
plus élevé entre l'outil et les analyseurs l'emporte. Les appels à confirmer
|
|
262
|
+
sont enregistrés par `pendingActions.propose()` et attendent l'approbation
|
|
263
|
+
via le cycle de vie déjà fourni par `createPendingActions`.
|
|
264
|
+
|
|
265
|
+
**Vision :** les adaptateurs HTTP et `runToolLoop` acceptent les images.
|
|
266
|
+
`createConfiguredRouter` les filtre selon les capacités déclarées. La boucle
|
|
267
|
+
textuelle historique `runAgentLoop` garde son protocole de texte.
|
|
229
268
|
|
|
230
269
|
## Un disjoncteur par fournisseur, jamais un pour tous
|
|
231
270
|
|
|
@@ -270,7 +309,7 @@ const { createReversibleMasker } = require('@astratra/ai');
|
|
|
270
309
|
const masker = createReversibleMasker({
|
|
271
310
|
names: registreDeLEcole, // sensible à la casse
|
|
272
311
|
patterns: [{ type: 'EMAIL', pattern: /[\w.+-]+@[\w-]+\.[\w.]+/g }],
|
|
273
|
-
detect: (texte) => serviceLocal.
|
|
312
|
+
detect: (texte) => serviceLocal.entities(texte, ['person']), // NER local, jamais distant
|
|
274
313
|
});
|
|
275
314
|
|
|
276
315
|
// Routeur : un fournisseur externe reçoit le texte masqué, la réponse revient démasquée
|
|
@@ -363,6 +402,77 @@ Le modèle lit un code, jamais le message d'erreur. À court de tours ou de
|
|
|
363
402
|
temps, un dernier tour sans outils répond avec ce qui a été lu, plutôt qu'une
|
|
364
403
|
erreur.
|
|
365
404
|
|
|
405
|
+
## llama.cpp : un modèle sur CPU, sur ton VPS
|
|
406
|
+
|
|
407
|
+
Fournisseur « **llama.cpp server** » : un modèle qui tourne sur **CPU** (un VPS
|
|
408
|
+
sans GPU) derrière l'API compatible OpenAI de `llama-server`. Aucune dépendance
|
|
409
|
+
(`src/llamaCpp.js`). C'est un client ; il ne lance rien. Le serveur se lance avec
|
|
410
|
+
le `docker-compose` de `deploy/llama-cpp/`. Ce n'est pas un doublon de
|
|
411
|
+
`createOpenAICompatibleProvider` : celui-ci branche un modèle dans le routeur et
|
|
412
|
+
rend `{ status }` pour une erreur HTTP, alors qu'un serveur hébergé par toi a
|
|
413
|
+
besoin de sa santé, du chargement du modèle (503) et d'erreurs typées.
|
|
414
|
+
|
|
415
|
+
### Client
|
|
416
|
+
|
|
417
|
+
```js
|
|
418
|
+
const { createLlamaCppProvider } = require('@astratra/ai');
|
|
419
|
+
|
|
420
|
+
const llm = createLlamaCppProvider({
|
|
421
|
+
baseUrl: 'http://127.0.0.1:8080', // avec ou sans /v1
|
|
422
|
+
apiKey: process.env.LLAMA_API_KEY,
|
|
423
|
+
model: 'local', // l'alias donné au serveur (facultatif)
|
|
424
|
+
timeoutMs: 120_000,
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
await llm.waitUntilReady({ timeoutMs: 120_000 }); // le chargement du modèle prend du temps
|
|
428
|
+
console.log(await llm.health()); // { ok, status: 'ok'|'loading'|'unauthorized'|'error'|'unreachable'|'timeout', latencyMs }
|
|
429
|
+
console.log(await llm.listModels()); // [{ id: 'local', ownedBy: 'llamacpp' }]
|
|
430
|
+
|
|
431
|
+
const { text, usage } = await llm.chat([{ role: 'user', content: 'Bonjour' }], { temperature: 0.2 });
|
|
432
|
+
for await (const piece of llm.chatStream([{ role: 'user', content: 'Raconte-moi une histoire' }])) process.stdout.write(piece);
|
|
433
|
+
|
|
434
|
+
llm.toOpenAICompatible(); // { baseURL: '…/v1', apiKey, model } pour tout autre client compatible OpenAI
|
|
435
|
+
```
|
|
436
|
+
|
|
437
|
+
`health()` ne lance jamais d'exception (c'est une sonde). Les autres appels
|
|
438
|
+
lancent `LocalLlmError` avec `code` : `UNREACHABLE`, `TIMEOUT`, `UNAUTHORIZED`,
|
|
439
|
+
`LOADING` (503, modèle pas chargé), `HTTP_ERROR` (avec `status`), `INVALID_RESPONSE`.
|
|
440
|
+
|
|
441
|
+
### Lancer `llama-server` sur un VPS CPU
|
|
442
|
+
|
|
443
|
+
Image officielle : `ghcr.io/ggml-org/llama.cpp:server` (CPU ; épingle un tag
|
|
444
|
+
`server-b<numéro>` en production).
|
|
445
|
+
|
|
446
|
+
```bash
|
|
447
|
+
cd packages/ai/deploy/llama-cpp
|
|
448
|
+
cp .env.example .env # remplis LLAMA_API_KEY (longue valeur aléatoire) et LLAMA_MODEL_FILE
|
|
449
|
+
mkdir -p models # y déposer le fichier .gguf (quantisation Q4_K_M : bon compromis)
|
|
450
|
+
docker compose up -d
|
|
451
|
+
docker compose logs -f llama # attendre « server is listening »
|
|
452
|
+
curl -H "Authorization: Bearer $LLAMA_API_KEY" http://127.0.0.1:8080/v1/models
|
|
453
|
+
```
|
|
454
|
+
|
|
455
|
+
Points à régler :
|
|
456
|
+
|
|
457
|
+
- **`LLAMA_THREADS`** : cœurs *physiques*, pas vCPU. Trop de fils ralentit.
|
|
458
|
+
- **`LLAMA_PARALLEL`** : requêtes simultanées. Le contexte (`LLAMA_CTX`) est
|
|
459
|
+
**partagé** : chacune dispose de `LLAMA_CTX / LLAMA_PARALLEL` jetons.
|
|
460
|
+
- **Mémoire** : prévoir la taille du fichier .gguf + ~1 à 2 Go pour le contexte ;
|
|
461
|
+
`LLAMA_MEMORY_LIMIT` plafonne le conteneur.
|
|
462
|
+
- **Débit** : sur CPU, un modèle de 3 à 4 milliards de paramètres en Q4 donne de
|
|
463
|
+
l'ordre de quelques à une dizaine de jetons par seconde selon le VPS ; à
|
|
464
|
+
mesurer sur le tien avant de promettre quoi que ce soit.
|
|
465
|
+
- Le port n'est publié que sur `127.0.0.1` : l'ouverture au public passe par un
|
|
466
|
+
proxy avec TLS, et la clé API reste obligatoire.
|
|
467
|
+
- La clé est lue dans l'environnement (`LLAMA_API_KEY`), jamais en argument.
|
|
468
|
+
|
|
469
|
+
### Limites
|
|
470
|
+
|
|
471
|
+
Pas de répartition de charge ni de repli intégrés au client : pour router ce
|
|
472
|
+
serveur avec les autres fournisseurs, donne `toOpenAICompatible()` (adresse et clé)
|
|
473
|
+
à `createOpenAICompatibleProvider`. Testé avec un faux
|
|
474
|
+
serveur HTTP ; le vrai `llama-server` n'a pas été lancé dans ce dépôt.
|
|
475
|
+
|
|
366
476
|
## Tests
|
|
367
477
|
|
|
368
478
|
```bash
|
|
@@ -520,3 +630,345 @@ absente ou inconnue vaut **téléphone** : se tromper vers le petit coûte une l
|
|
|
520
630
|
là où un tableau tenait, se tromper vers le grand coûte un tableau illisible.
|
|
521
631
|
La règle `paragraphs` est **obligatoire** : un pack de langue qui ne l'a pas est
|
|
522
632
|
refusé à la création, pas découvert sur un écran.
|
|
633
|
+
|
|
634
|
+
## Configured routing (Oracle and Tertius)
|
|
635
|
+
|
|
636
|
+
`createConfiguredRouter` builds HTTP providers and routes by usage on top of
|
|
637
|
+
`createProviderRouter`. Existing exports and their defaults remain compatible.
|
|
638
|
+
The model catalogue belongs to your application: names, quotas and capabilities
|
|
639
|
+
are explicit configuration, not historical assumptions baked into the package.
|
|
640
|
+
|
|
641
|
+
```js
|
|
642
|
+
const { createConfiguredRouter } = require('@astratra/ai');
|
|
643
|
+
const router = createConfiguredRouter({
|
|
644
|
+
providers: [
|
|
645
|
+
{ id: 'groq', format: 'openai',
|
|
646
|
+
url: 'https://api.groq.com/openai/v1/chat/completions',
|
|
647
|
+
keys: () => [process.env.GROQ_API_KEY, process.env.GROQ_API_KEY_2],
|
|
648
|
+
models: [{ id: 'your-fast-model', rpm: 30, rpd: 1000, tpd: 200000,
|
|
649
|
+
complexity: ['simple', 'medium'], extra: { include_reasoning: false } }] },
|
|
650
|
+
{ id: 'gemini', format: 'gemini',
|
|
651
|
+
url: 'https://generativelanguage.googleapis.com/v1beta/models',
|
|
652
|
+
keys: () => [process.env.GEMINI_API_KEY, process.env.GEMINI_API_KEY_2],
|
|
653
|
+
keysByUsage: { news: () => [process.env.GEMINI_NEWS_KEY] },
|
|
654
|
+
vision: true,
|
|
655
|
+
models: [{ id: 'your-vision-model', rpm: 10, rpd: 200 }] },
|
|
656
|
+
{ id: 'ollama', format: 'openai', local: true,
|
|
657
|
+
url: 'http://127.0.0.1:11434/v1/chat/completions',
|
|
658
|
+
models: [{ id: 'your-local-model' }] }
|
|
659
|
+
],
|
|
660
|
+
usages: {
|
|
661
|
+
quick: [{ provider: 'groq', model: 'your-fast-model' }, { provider: 'ollama', model: 'your-local-model' }],
|
|
662
|
+
chat: [{ provider: 'gemini', model: 'your-vision-model' }, { provider: 'groq', model: 'your-fast-model' }],
|
|
663
|
+
tools: [{ provider: 'groq', model: 'your-fast-model' }], // pilot is an alias list
|
|
664
|
+
json: [{ provider: 'groq', model: 'your-fast-model', extra: { response_format: { type: 'json_object' } } }],
|
|
665
|
+
vision: [{ provider: 'gemini', model: 'your-vision-model' }]
|
|
666
|
+
},
|
|
667
|
+
intentRouting: { report: { preferred: ['gemini:your-vision-model'] } },
|
|
668
|
+
quotaMargin: { rpm: 0.75, rpd: 0.90, tpd: 0.85 },
|
|
669
|
+
tokenBuffer: 1.2, cooldownMs: 120000, maxFailures: 3, degradedMs: 300000,
|
|
670
|
+
waitedTimeoutMs: 12000, backgroundTimeoutMs: 45000,
|
|
671
|
+
statePrefix: 'my-app:', whenAllCooling: 'skip'
|
|
672
|
+
});
|
|
673
|
+
|
|
674
|
+
const result = await router.route({
|
|
675
|
+
system: 'Answer from the supplied information.',
|
|
676
|
+
messages: [{ role: 'user', text: 'Compare these proposals.' }], maxTokens: 1200
|
|
677
|
+
}, { usage: 'chat', waitedOn: true, intent: 'report', signal });
|
|
678
|
+
// { value: { text, toolCalls, cut }, provider, model, key, stateKey,
|
|
679
|
+
// usage, complexity, partial, fallback }
|
|
680
|
+
await router.ask('Hello'); // detailed answer only
|
|
681
|
+
await router.getStats(); // async, counters and rests; never credentials
|
|
682
|
+
await router.reset(); // only state keys used by this router
|
|
683
|
+
router.stop(); // rejects subsequent calls; no owned DB connection
|
|
684
|
+
```
|
|
685
|
+
|
|
686
|
+
Without `usage`, tools select `tools`, images select `vision`, explicit `json:
|
|
687
|
+
true` selects `json`, simple requests select `quick`, and the rest selects
|
|
688
|
+
`chat`. `estimateRequestComplexity` retains Oracle's word/keyword rules (with
|
|
689
|
+
English equivalents); you can override `complexity`. Within a list, intent
|
|
690
|
+
preferences come first, then compatible complexity, then other models; preview
|
|
691
|
+
models come last. Without a usage list, provider/model declaration order is the
|
|
692
|
+
fallback order. Thus Oracle's Groq → Mistral → Gemini → Vercel → Ollama and
|
|
693
|
+
Tertius's distinct quick/chat/pilot orders are both expressible.
|
|
694
|
+
|
|
695
|
+
Model fields: `vision`, `visionOnly`, `complexity`, `extra`, `maxTokens`,
|
|
696
|
+
`maxInputTokens`, `noAnalytics`, `languages`, `preview`, `disabled`, `rpm`, `rpd`,
|
|
697
|
+
`tpd`. `exclude` uses `provider:model`. Declared models are required; unknown
|
|
698
|
+
candidates are ignored. Provider and candidate `extra` objects merge, with
|
|
699
|
+
candidate values taking precedence (nested values replace whole fields).
|
|
700
|
+
Gemini `extra` fields merge at the top of its JSON body. JSON usage is an
|
|
701
|
+
explicit selection, not a keyword guess or automatic validation: configure its
|
|
702
|
+
format through `extra` and/or supply `accepts`.
|
|
703
|
+
|
|
704
|
+
Images are filtered for every usage, including tool fallback. Vision capability
|
|
705
|
+
must be declared on the provider or model; `vision: false` on a model overrides
|
|
706
|
+
a provider's capability. With `reserveVision: true`, `visionOnly` models are
|
|
707
|
+
kept for images. `local: true` works without credentials and preserves the
|
|
708
|
+
existing masking boundary: only external providers receive masked requests.
|
|
709
|
+
Ollama uses `/v1/chat/completions`; `createLlamaCppProvider` remains available
|
|
710
|
+
for health, loading, streaming and direct llama.cpp calls.
|
|
711
|
+
|
|
712
|
+
Reservations happen atomically before sending. RPM uses a rolling 60-second
|
|
713
|
+
window; RPD/TPD reset on the UTC date. Quotas with no field are unlimited;
|
|
714
|
+
zero or a margin-rounded limit of zero forbids calls. Reservations include the
|
|
715
|
+
system prompt, transcript, tool schemas and output ceiling; explicit
|
|
716
|
+
`estimatedTokens` must already include the full call. Failed calls conservatively
|
|
717
|
+
keep their reservation. A 429 or 503 or timeout rests the credential/model;
|
|
718
|
+
three consecutive actual failures exclude it for five minutes. A 429 does not
|
|
719
|
+
count as an actual failure. Request errors 400/413/414/422 do not penalize health;
|
|
720
|
+
404 excludes the model until `reset()`. Caller cancellation stops immediately
|
|
721
|
+
and does not mark a failure. `whenAllCooling: 'try'` can retry resting models
|
|
722
|
+
when all rest; it never bypasses quotas, exclusion or degradation.
|
|
723
|
+
|
|
724
|
+
Keys are deduplicated, re-read per request and tried in rotating order, with a
|
|
725
|
+
fallback to the next credential. State identifies credentials by ordinal
|
|
726
|
+
(`provider:model@key-1`), never by any part of a key. Replacing a credential in
|
|
727
|
+
one slot keeps that slot's existing accounting until it expires or is reset.
|
|
728
|
+
`keysByUsage` isolates dedicated pools (for example news from chat);
|
|
729
|
+
`keyLane(ctx)` identifies custom dynamic pools. Keys default to independent
|
|
730
|
+
quotas; set `sharedKeyQuotas: true` when they share one account's model quota.
|
|
731
|
+
All instances sharing a store must use the same pool order and state prefix.
|
|
732
|
+
|
|
733
|
+
`route` rejects empty answers and unknown tool names, keeps partial text as a
|
|
734
|
+
last resort, and trims it to whole sentences. If no tool-capable answer is
|
|
735
|
+
accepted, it retries the chat/vision list with `tools: []` and keeps the read
|
|
736
|
+
transcript. `fallbackWithoutTools: false` disables that retry. Tools are never
|
|
737
|
+
executed by the router. An agent can use `runToolLoop({ turn: async request => {
|
|
738
|
+
const r = await router.route(request); return { ...r.value, model: r.key }; },
|
|
739
|
+
... })`. Business retrieval and source collection remain injected.
|
|
740
|
+
|
|
741
|
+
Errors have clean codes: `AI_NO_PROVIDER`, `AI_NO_MATCH`, `AI_UNAVAILABLE`,
|
|
742
|
+
`AI_STATE_UNAVAILABLE`, `AI_ROUTER_STOPPED`. State outages fail closed instead of
|
|
743
|
+
silently losing a distributed quota. Transport messages and credentials are
|
|
744
|
+
not included in these errors.
|
|
745
|
+
|
|
746
|
+
### Injectable state: memory, Redis, PostgreSQL
|
|
747
|
+
|
|
748
|
+
```js
|
|
749
|
+
const { createMemoryRoutingStore, createRedisRoutingStore,
|
|
750
|
+
createPostgresRoutingStore } = require('@astratra/ai');
|
|
751
|
+
const memory = createMemoryRoutingStore();
|
|
752
|
+
const redisStore = createRedisRoutingStore({ client: connectedRedis, prefix: 'app:ai:' });
|
|
753
|
+
const postgresStore = createPostgresRoutingStore({ pool, table: 'ai_routing_state' });
|
|
754
|
+
await postgresStore.initialize(); // explicit table creation, once during setup
|
|
755
|
+
const router = createConfiguredRouter({ ...config, store: postgresStore });
|
|
756
|
+
```
|
|
757
|
+
|
|
758
|
+
Memory is local to a process unless you share the same object. Redis uses
|
|
759
|
+
optimistic CAS in Lua with a two-day TTL (configurable, at least one day), and
|
|
760
|
+
retries contention. PostgreSQL locks even first insertion using a transaction
|
|
761
|
+
advisory lock and stores JSONB. Connections and database drivers belong to the
|
|
762
|
+
consumer; the package does not connect, install a driver, or close their pool.
|
|
763
|
+
A custom Redis/PostgreSQL/other adapter implements:
|
|
764
|
+
|
|
765
|
+
```ts
|
|
766
|
+
interface RoutingStateStore {
|
|
767
|
+
read(key: string): Promise<RoutingState | null>;
|
|
768
|
+
update<T>(key: string, change: (old: RoutingState | null) => {
|
|
769
|
+
state: RoutingState | null; value?: T;
|
|
770
|
+
}): Promise<T | undefined>;
|
|
771
|
+
}
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
`update` must serialize each key across processes. The synchronous callback
|
|
775
|
+
must be side-effect free and may run multiple times. `state: null` deletes the
|
|
776
|
+
key. `read` returns an independent snapshot.
|
|
777
|
+
|
|
778
|
+
### Environment configuration
|
|
779
|
+
|
|
780
|
+
`readAiConfigFromEnv({ env: process.env, prefix: 'IA_' })` reads only the supplied
|
|
781
|
+
object, never an `.env` file. `ARIA_` or another prefix works identically.
|
|
782
|
+
|
|
783
|
+
| Variable (default prefix) | Value |
|
|
784
|
+
| --- | --- |
|
|
785
|
+
| `IA_PROVIDERS` | JSON array of `{ id, format, url, local?, vision?, models, extra?, sharedKeyQuotas?, envName? }` |
|
|
786
|
+
| `IA_<ID>_URL`, `IA_<ID>_LOCAL` | Optional provider overrides; local accepts true/false/1/0 |
|
|
787
|
+
| `IA_<ID>_KEY`, `IA_<ID>_KEY_2` … | Keys, read at every request; sparse suffixes are supported; ID uppercased with non-alphanumeric characters replaced by `_` (or `envName`) |
|
|
788
|
+
| `IA_MODELS_QUICK`, `IA_MODELS_CHAT`, `IA_MODELS_TOOLS`, `IA_MODELS_PILOT`, `IA_MODELS_JSON`, `IA_MODELS_VISION` | JSON arrays of `{ provider, model, extra? }` |
|
|
789
|
+
| `IA_INTENT_ROUTING` | JSON object `{ intent: { preferred: ['provider:model'] } }` |
|
|
790
|
+
| `IA_QUOTA_MARGIN` | JSON number or object `{ rpm, rpd, tpd }`; defaults .75/.90/.85 |
|
|
791
|
+
| `IA_TOKEN_BUFFER` | Number; default 1.2 |
|
|
792
|
+
| `IA_WAITED_TIMEOUT_MS`, `IA_BACKGROUND_TIMEOUT_MS` | Defaults 12000/45000 |
|
|
793
|
+
| `IA_COOLDOWN_MS`, `IA_DEGRADED_MS`, `IA_MAX_FAILURES` | Defaults 120000/300000/3 |
|
|
794
|
+
| `IA_STATE_PREFIX`, `IA_WHEN_ALL_COOLING`, `IA_RESERVE_VISION` | Namespace, skip/try, true/false |
|
|
795
|
+
| `IA_MODELS_URL`, `IA_MODELS_TOKEN_FILE` | Optional Scolaris model server and its Bearer file |
|
|
796
|
+
| `IA_TTS_URL`, `IA_TTS_TOKEN_FILE` | Optional Piper server and Bearer file |
|
|
797
|
+
| `IA_LANGUAGE_URL`, `IA_LANGUAGE_TIMEOUT_MS` | Optional fastText server; default timeout 800 |
|
|
798
|
+
| `IA_GROQ_WHISPER_URL`, `IA_GROQ_WHISPER_MODELS` | Full endpoint, JSON model IDs; defaults Groq transcription endpoint and turbo/large-v3 |
|
|
799
|
+
| `IA_CLOUDFLARE_ACCOUNT_ID`, `IA_CLOUDFLARE_WHISPER_URL` | Account for the default Whisper run URL, or explicit full URL |
|
|
800
|
+
| `IA_GEMINI_TTS_ENDPOINT`, `IA_GEMINI_TTS_MODELS`, `IA_GEMINI_TTS_VOICE` | Models base URL, JSON model IDs, voice; defaults Gemini endpoint, gemini-2.5-flash-preview-tts, Aoede |
|
|
801
|
+
|
|
802
|
+
Voice uses `IA_GROQ_KEY[_N]`, `IA_CLOUDFLARE_KEY[_N]` and
|
|
803
|
+
`IA_GEMINI_KEY[_N]`. No provider catalogue is auto-created: `IA_PROVIDERS`
|
|
804
|
+
is required to configure text routing. Voice/local sections are returned as
|
|
805
|
+
`config.voice`, `config.localModels`, `config.language`. Inject the store in
|
|
806
|
+
code after reading the configuration. Invalid JSON is reported by variable
|
|
807
|
+
name only, never by its value. Endpoint/model defaults are configurable
|
|
808
|
+
protocol examples, not a claim that a model or free tier remains available.
|
|
809
|
+
|
|
810
|
+
```js
|
|
811
|
+
const { readAiConfigFromEnv, createConfiguredRouter } = require('@astratra/ai');
|
|
812
|
+
const config = readAiConfigFromEnv({ prefix: 'ARIA_' });
|
|
813
|
+
const router = createConfiguredRouter({ ...config, store });
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
## Local models (Scolaris HTTP protocol)
|
|
817
|
+
|
|
818
|
+
```js
|
|
819
|
+
const { createLocalModelsClient } = require('@astratra/ai');
|
|
820
|
+
const models = createLocalModelsClient({
|
|
821
|
+
baseUrl: 'http://127.0.0.1:5007', tokenFile: '/etc/my-app/models-token',
|
|
822
|
+
onError: (error, route) => metrics.count(route, error.code)
|
|
823
|
+
});
|
|
824
|
+
await models.reclasser('question', ['passage']); // number[] | null
|
|
825
|
+
await models.nli([{ premisse: 'premise', hypothese: 'claim' }]); // comparer alias
|
|
826
|
+
await models.entites('Ada', ['person']); // UTF-16 spans
|
|
827
|
+
const vectors = await models.vecteurs(['text']); // vectors.modele/dimensions
|
|
828
|
+
await models.transcrire(pcm16Base64, { langue: 'fr', vocabulaire: '', timeoutMs: 15000 });
|
|
829
|
+
models.disponible('/vecteurs'); // available alias
|
|
830
|
+
```
|
|
831
|
+
|
|
832
|
+
Without `baseUrl`, each operation returns `null` and availability is false.
|
|
833
|
+
Invalid input, malformed results, downtime or deadlines also return `null`;
|
|
834
|
+
`onError` receives a clean `AiServiceError` and route. Caller abort throws its
|
|
835
|
+
reason. Limits match the source server: 50 passages, 20 NLI pairs, entity text
|
|
836
|
+
8000 UTF-16 units, 10 labels of 50 characters, 32 vector texts of 2000 characters,
|
|
837
|
+
and PCM16 mono 16kHz for .2–30 seconds, vocabulary 600 characters. Whisper accepts
|
|
838
|
+
fr/en/es/ln. Vector results must come from `bge-m3-onnx-int8`, 1024 dimensions
|
|
839
|
+
(configurable when the server's embedding space changes); mismatches are refused.
|
|
840
|
+
|
|
841
|
+
The Bearer file is cached for one minute, including concurrent reads. If a file
|
|
842
|
+
is explicitly configured but unreadable, calls fail closed; without a file no
|
|
843
|
+
authentication header is sent. The server receives `X-Delai-Ms`. Defaults per
|
|
844
|
+
route: rerank 1000ms, NLI/vector 1500ms, entities 800ms, transcription 15000ms.
|
|
845
|
+
Each route has its own three-failure/60-second circuit, with one recovery probe;
|
|
846
|
+
4xx refusals do not open it. Python engines, ONNX runtimes and weights are not
|
|
847
|
+
bundled or downloaded by this client. Tertius's generic in-process embedding
|
|
848
|
+
and native voice primitives remain in `@astratra/rag` and `@astratra/voice`.
|
|
849
|
+
|
|
850
|
+
## Transcription, speech and language
|
|
851
|
+
|
|
852
|
+
```js
|
|
853
|
+
const { readAiConfigFromEnv, createTranscriptionService,
|
|
854
|
+
createSpeechService, createServiceLanguageDetector } = require('@astratra/ai');
|
|
855
|
+
const config = readAiConfigFromEnv({ prefix: 'IA_' });
|
|
856
|
+
const transcription = createTranscriptionService({ ...config.voice, localModels: config.localModels });
|
|
857
|
+
const heard = await transcription.transcribe({
|
|
858
|
+
audio: pcm16Bytes, format: 'pcm16', language: 'fr', vocabulary: '', signal,
|
|
859
|
+
allowRemote: false // optional: confidential calls remain on this machine
|
|
860
|
+
});
|
|
861
|
+
const speech = createSpeechService(config.voice);
|
|
862
|
+
const spoken = await speech.synthesize({ text: '**Hello**', language: 'en', signal });
|
|
863
|
+
// { audio: Buffer, format: 'wav', mimeType: 'audio/wav', sampleRate, channels,
|
|
864
|
+
// provider: 'piper' | 'gemini', model, fallback }
|
|
865
|
+
const language = createServiceLanguageDetector(config.language);
|
|
866
|
+
await language.reply('hello', 'fr'); // fastText first, then small-word heuristic
|
|
867
|
+
```
|
|
868
|
+
|
|
869
|
+
Transcription tries local Whisper, then Groq (multipart with a WAV file for PCM),
|
|
870
|
+
then Cloudflare (raw file base64 in `{ audio }`, including `success:false` checks).
|
|
871
|
+
`format: 'pcm16'` is little-endian 16kHz mono; `'wav'` can use the local server
|
|
872
|
+
when it contains that same PCM format. `'file'` forwards compressed audio to
|
|
873
|
+
remote services; this client does not transcode it. Base64 or byte input works.
|
|
874
|
+
Local silence is a valid empty transcription and stops the chain. Low logprob,
|
|
875
|
+
stock subtitle hallucinations or repeated words are rejected; `stockPhrases`,
|
|
876
|
+
`noSpeechThreshold`, `logprobThreshold` are configurable. Remote language remains
|
|
877
|
+
`null` if the service did not identify it. Limits, timeouts, key rotation and
|
|
878
|
+
rests are enforced, and an aborted request never proceeds to the next provider.
|
|
879
|
+
|
|
880
|
+
Speech tries Gemini TTS first (online, rotating keys/models), then Piper `/tts`
|
|
881
|
+
(`{ text, lang }` → validated WAV) as the local fallback. `order: ['piper', 'gemini']`
|
|
882
|
+
reverses it; `allowRemote: false` keeps everything local. Gemini PCM is wrapped in a WAV header with the supplied
|
|
883
|
+
sample rate (24kHz by default), never mislabeled as an existing WAV file.
|
|
884
|
+
`speakableText` removes markup/emoji and limits text to 6000 characters;
|
|
885
|
+
`prepareText` can provide your app's pronunciation rules. A missing Piper
|
|
886
|
+
language rests only that language. Local timeout is 30s, Gemini timeout 45s;
|
|
887
|
+
keys/models rest for 120s after failure, configurable per provider. A whole
|
|
888
|
+
synthesis uses one provider. Native segmentation, prosody, AAC finishing and
|
|
889
|
+
voice caching are separate existing `@astratra/voice` primitives.
|
|
890
|
+
|
|
891
|
+
`AI_TRANSCRIPTION_UNAVAILABLE` and `AI_SPEECH_UNAVAILABLE` identify exhausted
|
|
892
|
+
chains; input errors use `AI_AUDIO_INPUT`/`AI_SPEECH_INPUT`. Unconfigured optional
|
|
893
|
+
services never initiate an implicit local request. Language supports fr/en/es
|
|
894
|
+
by default; custom word lists can extend it. Statistical confidence defaults to
|
|
895
|
+
0.5, or 0.9 for fewer than three words; unsupported/uncertain results keep the
|
|
896
|
+
app language or a clear heuristic result.
|
|
897
|
+
|
|
898
|
+
## Native tool risk and confirmation
|
|
899
|
+
|
|
900
|
+
```js
|
|
901
|
+
validateNativeTools(tools, { requireSummary: true, requireRisk: true });
|
|
902
|
+
const ledger = createSourceLedger();
|
|
903
|
+
const exposure = { untrusted: false };
|
|
904
|
+
const callTool = createToolCaller({
|
|
905
|
+
tools, context: { userId }, timeoutMs: 20000, signal,
|
|
906
|
+
confirmationPolicy: { threshold: 'MEDIUM' }, exposure,
|
|
907
|
+
readsUntrusted: (tool, result) => tool.name === 'read_shared_page'
|
|
908
|
+
|| result?.sources?.some((source) => source.kind === 'web'),
|
|
909
|
+
keep: ledger.keep,
|
|
910
|
+
record: async ({ tool, args, found }) => saveAction(tool.kind, args, found)
|
|
911
|
+
});
|
|
912
|
+
await runToolLoop({ system, messages, tools: toolSpecs(tools), turn, callTool,
|
|
913
|
+
maxTurns: 6, maxMs: 60000, signal,
|
|
914
|
+
finalInstruction: 'Answer now with what you have read: no more tools.' });
|
|
915
|
+
ledger.sources(); ledger.evidence();
|
|
916
|
+
```
|
|
917
|
+
|
|
918
|
+
All new options are opt-in. `requireRisk` enforces LOW/MEDIUM/HIGH, LOW-only
|
|
919
|
+
standalone writes, and undo for LOW confirmation tools. Under a confirmation
|
|
920
|
+
policy, a `confirm` tool runs its preparation; MEDIUM/HIGH (or UNKNOWN),
|
|
921
|
+
untrusted exposure or a result's `confirm: true` wait and never run `perform`.
|
|
922
|
+
A LOW tool below the threshold can perform immediately, only with an undo
|
|
923
|
+
handler; `record` then receives its effective `kind: 'write'` and undo data.
|
|
924
|
+
The preparation must be read-only. Without a policy, the original behavior
|
|
925
|
+
remains: all `confirm` tools prepare and wait. `toolRiskOf` and
|
|
926
|
+
`waitsForToolConfirmation` expose the same decisions. Unknown risk always waits,
|
|
927
|
+
even under `'never'`; untrusted exposure also forces waiting.
|
|
928
|
+
|
|
929
|
+
`readsUntrusted` is application policy, not a fixed list of business tools. All
|
|
930
|
+
calls share `exposure`; subsequent tools see the reads already completed.
|
|
931
|
+
Tools in a parallel batch are concurrent: do not schedule a write alongside
|
|
932
|
+
an untrusted read when its decision depends on that read. Tool handlers must
|
|
933
|
+
honor their injected AbortSignal; timed-out preparation cannot subsequently
|
|
934
|
+
start `perform`. Existing source ledger, legacy registry risk policy/pending
|
|
935
|
+
actions, and loop budgets are reused. Loop `maxMs` is checked between turns;
|
|
936
|
+
model and tool deadlines bound individual operations, and the final no-tool
|
|
937
|
+
turn remains allowed after the budget boundary.
|
|
938
|
+
|
|
939
|
+
### Verification commands
|
|
940
|
+
|
|
941
|
+
```sh
|
|
942
|
+
cd packages/ai
|
|
943
|
+
npm test -- --runInBand
|
|
944
|
+
node scripts/verify-mutations.cjs
|
|
945
|
+
cd ../..
|
|
946
|
+
npm run typecheck
|
|
947
|
+
```
|
|
948
|
+
|
|
949
|
+
HTTP contract tests start only loopback servers. If the execution environment
|
|
950
|
+
forbids sockets, `AI_TEST_TRANSPORT=memory` explicitly uses simulated HTTP
|
|
951
|
+
responses for the new suites. That mode proves behavior and serialization,
|
|
952
|
+
not actual socket transport; the existing socket suites still require loopback
|
|
953
|
+
permission. No test makes a remote provider call.
|
|
954
|
+
|
|
955
|
+
Local configured providers also accept `maxConcurrent` (default 8) and
|
|
956
|
+
`maxQueue` (default 20): inference is bounded and queued cancellation consumes
|
|
957
|
+
no quota. The queue wait is bounded by the model deadline. `Retry-After`
|
|
958
|
+
(seconds or HTTP date) takes precedence over the configured 429/503 rest;
|
|
959
|
+
`cooldownJitterMs` (environment: `IA_COOLDOWN_JITTER_MS`, default 0) adds optional
|
|
960
|
+
jitter only when no provider delay was announced. `readRetryAfter` and
|
|
961
|
+
`createInferenceQueue` are available separately.
|
|
962
|
+
|
|
963
|
+
The local client also exposes English adapters: `rerank`/`embed` are aliases;
|
|
964
|
+
`compare` maps `{ premise, hypothesis }` to `{ entailment, neutral,
|
|
965
|
+
contradiction }`; `entities` maps UTF-16 spans to `{ text, type, start, end,
|
|
966
|
+
score }`. They connect directly to existing package primitives:
|
|
967
|
+
|
|
968
|
+
```js
|
|
969
|
+
await rerankResults(query, found, { score: models.rerank });
|
|
970
|
+
await findContradiction(answer, excerpts, { compare: models.compare });
|
|
971
|
+
const masker = createReversibleMasker({
|
|
972
|
+
detect: (text) => models.entities(text, ['person'])
|
|
973
|
+
});
|
|
974
|
+
```
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Copie en .env (jamais commité) puis remplis.
|
|
2
|
+
LLAMA_API_KEY=remplace-par-une-longue-valeur-aleatoire
|
|
3
|
+
LLAMA_MODEL_FILE=nom-du-modele-Q4_K_M.gguf
|
|
4
|
+
LLAMA_ALIAS=local
|
|
5
|
+
LLAMA_CTX=8192
|
|
6
|
+
LLAMA_THREADS=4
|
|
7
|
+
LLAMA_PARALLEL=2
|
|
8
|
+
LLAMA_MEMORY_LIMIT=6g
|
|
9
|
+
LLAMA_MODELS_DIR=./models
|
|
10
|
+
LLAMA_PORT=8080
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# llama.cpp en inférence CPU sur un VPS — API compatible OpenAI sur le port 8080.
|
|
2
|
+
# NE démarre rien tout seul : `docker compose up -d` après avoir lu le README du paquet.
|
|
3
|
+
#
|
|
4
|
+
# Image officielle : ghcr.io/ggml-org/llama.cpp:server (CPU). Épingle un tag
|
|
5
|
+
# `server-b<numéro>` en production plutôt que le tag flottant.
|
|
6
|
+
services:
|
|
7
|
+
llama:
|
|
8
|
+
image: ${LLAMA_IMAGE:-ghcr.io/ggml-org/llama.cpp:server}
|
|
9
|
+
restart: unless-stopped
|
|
10
|
+
# Écoute en local seulement : la sortie publique passe par un proxy avec TLS (nginx, Caddy).
|
|
11
|
+
ports:
|
|
12
|
+
- "127.0.0.1:${LLAMA_PORT:-8080}:8080"
|
|
13
|
+
volumes:
|
|
14
|
+
# Les fichiers .gguf, à télécharger une fois (voir README).
|
|
15
|
+
- ${LLAMA_MODELS_DIR:-./models}:/models:ro
|
|
16
|
+
environment:
|
|
17
|
+
# La clé est lue dans l'environnement, jamais passée en argument (visible dans `ps`).
|
|
18
|
+
LLAMA_API_KEY: ${LLAMA_API_KEY:?définis LLAMA_API_KEY dans .env}
|
|
19
|
+
command:
|
|
20
|
+
- --host
|
|
21
|
+
- 0.0.0.0
|
|
22
|
+
- --port
|
|
23
|
+
- "8080"
|
|
24
|
+
- --model
|
|
25
|
+
- /models/${LLAMA_MODEL_FILE:?définis LLAMA_MODEL_FILE dans .env}
|
|
26
|
+
- --alias
|
|
27
|
+
- ${LLAMA_ALIAS:-local}
|
|
28
|
+
- --ctx-size
|
|
29
|
+
- "${LLAMA_CTX:-8192}"
|
|
30
|
+
# Fils de calcul : le nombre de cœurs PHYSIQUES, pas de vCPU.
|
|
31
|
+
- --threads
|
|
32
|
+
- "${LLAMA_THREADS:-4}"
|
|
33
|
+
# Requêtes simultanées ; le contexte est partagé entre elles (ctx-size / parallel chacune).
|
|
34
|
+
- --parallel
|
|
35
|
+
- "${LLAMA_PARALLEL:-2}"
|
|
36
|
+
- --metrics
|
|
37
|
+
healthcheck:
|
|
38
|
+
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
|
|
39
|
+
interval: 30s
|
|
40
|
+
timeout: 5s
|
|
41
|
+
retries: 5
|
|
42
|
+
# Le chargement d'un modèle de plusieurs Go prend du temps.
|
|
43
|
+
start_period: 120s
|
|
44
|
+
deploy:
|
|
45
|
+
resources:
|
|
46
|
+
limits:
|
|
47
|
+
memory: ${LLAMA_MEMORY_LIMIT:-6g}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@astratra/ai",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"description": "Generic AI provider routing, tool registry, and agent loop utilities for Astratra.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20"
|
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
"types": "src/index.d.ts",
|
|
10
10
|
"type": "commonjs",
|
|
11
11
|
"files": [
|
|
12
|
-
"src"
|
|
12
|
+
"src",
|
|
13
|
+
"deploy"
|
|
13
14
|
],
|
|
14
15
|
"scripts": {
|
|
15
16
|
"test": "jest"
|
|
@@ -22,7 +23,9 @@
|
|
|
22
23
|
"keywords": [
|
|
23
24
|
"astratra",
|
|
24
25
|
"ai",
|
|
25
|
-
"agents"
|
|
26
|
+
"agents",
|
|
27
|
+
"llama.cpp",
|
|
28
|
+
"cpu-inference"
|
|
26
29
|
],
|
|
27
30
|
"license": "MIT",
|
|
28
31
|
"publishConfig": {
|
|
@@ -40,7 +43,7 @@
|
|
|
40
43
|
}
|
|
41
44
|
},
|
|
42
45
|
"devDependencies": {
|
|
43
|
-
"@astratra/resilience": "^0.
|
|
46
|
+
"@astratra/resilience": "^1.0.0",
|
|
44
47
|
"jest": "30.4.2",
|
|
45
48
|
"redis": "^4.7.0"
|
|
46
49
|
}
|