@astratra/ai 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.d.ts CHANGED
@@ -12,6 +12,8 @@ export interface ProviderModel {
12
12
  export interface Provider {
13
13
  id: string;
14
14
  models: ProviderModel[];
15
+ /** False for a model running on this machine: the router does not mask for it. */
16
+ external?: boolean;
15
17
  call(prompt: string, ctx: Record<string, unknown>, model: ProviderModel): Awaitable<unknown>;
16
18
  [key: string]: unknown;
17
19
  }
@@ -33,6 +35,8 @@ export interface ProviderRouterConfig {
33
35
  intentRouting?: Record<string, { preferred?: string[] }>;
34
36
  redisKeyPrefix?: string;
35
37
  redisUrl?: string;
38
+ /** One circuit per provider: a factory (providerId) => breaker, or a pool. */
39
+ breakers?: ((providerId: string) => BreakerLike) | BreakerPool;
36
40
  }
37
41
 
38
42
  export interface ProviderStats {
@@ -46,6 +50,8 @@ export interface ProviderStats {
46
50
  cooldown: boolean;
47
51
  degraded: boolean;
48
52
  failures: number;
53
+ /** The provider's circuit state, or null without breakers. */
54
+ circuit: string | null;
49
55
  }
50
56
 
51
57
  export interface ProviderRouter {
@@ -53,6 +59,7 @@ export interface ProviderRouter {
53
59
  /** Les entrées sont indexées par "providerId:modelId". */
54
60
  getStats(): Record<string, ProviderStats>;
55
61
  stop(): void;
62
+ breakers: BreakerPool | null;
56
63
  }
57
64
 
58
65
  export function createProviderRouter(config?: ProviderRouterConfig): ProviderRouter;
@@ -62,6 +69,8 @@ export interface ToolDefinition<TParams = Record<string, unknown>, TResult = unk
62
69
  description: string;
63
70
  type: string;
64
71
  roles: string[];
72
+ /** Reaches outside (web search, third-party API): its parameters are masked. */
73
+ external?: boolean;
65
74
  params?: Record<string, unknown>;
66
75
  handler(params: TParams, ctx: Record<string, unknown>): Awaitable<TResult>;
67
76
  }
@@ -115,6 +124,19 @@ export interface AgentLoopOptions {
115
124
  * every allowed tool call, unchanged from before.
116
125
  */
117
126
  confirmTool?: (toolCall: ToolCall, ctx: Record<string, unknown>) => Awaitable<boolean>;
127
+ /**
128
+ * Masks what leaves for the model, unmasks what comes back. Tools marked
129
+ * `external: true` receive masked parameters.
130
+ */
131
+ masker?: ReversibleMasker;
132
+ /** A throwing tool becomes { error: 'tool_failed' | 'tool_timeout' } for the model. Default false (throws). */
133
+ reportToolErrors?: boolean;
134
+ toolTimeoutMs?: number;
135
+ /** Time budget for the whole loop. */
136
+ maxMs?: number;
137
+ /** When steps or time run out: one last turn without tools, with this instruction. */
138
+ finalInstruction?: string;
139
+ now?: () => number;
118
140
  }
119
141
 
120
142
  export function runAgentLoop(options: AgentLoopOptions): Promise<string>;
@@ -285,3 +307,233 @@ export function createFormatInstructions(options: {
285
307
  defaultSurface?: string;
286
308
  fallbackLanguage?: string;
287
309
  }): FormatInstructions;
310
+
311
+ /* ─────────────────── Circuits ─────────────────── */
312
+
313
+ export interface BreakerLike {
314
+ call<T>(fn: () => Awaitable<T>): Promise<T>;
315
+ status?(): { state?: string; [key: string]: unknown };
316
+ reset?(): void;
317
+ }
318
+
319
+ export interface BreakerPool {
320
+ get(key: string): BreakerLike;
321
+ run<T>(key: string, fn: () => Awaitable<T>): Promise<T>;
322
+ status(): Record<string, unknown>;
323
+ stateOf(key: string): string | null;
324
+ reset(key?: string): void;
325
+ keys(): string[];
326
+ }
327
+
328
+ /** Refuses a factory that returns the same breaker for two keys. */
329
+ export function createBreakerPool(options: { create: (key: string) => BreakerLike }): BreakerPool;
330
+ /** Timeouts, network errors, 408 and 5xx are outages; 429 and other 4xx are not. */
331
+ export function isProviderOutage(error: unknown): boolean;
332
+ export function isCircuitOpen(error: unknown): boolean;
333
+
334
+ /* ─────────────────── Outbound masking ─────────────────── */
335
+
336
+ export interface DetectedEntity {
337
+ text: string;
338
+ type: string;
339
+ score: number;
340
+ }
341
+
342
+ export interface ReversibleMasker {
343
+ mask(text: string, options?: { names?: string[] }): string;
344
+ /** Runs the detector first (keep it local), learns, then masks. */
345
+ maskAsync(text: string, options?: { names?: string[] }): Promise<string>;
346
+ maskDeep<T>(value: T): T;
347
+ unmask(text: string): string;
348
+ unmaskDeep<T>(value: T): T;
349
+ unmaskStream(chunks: AsyncIterable<unknown>): AsyncGenerator<string>;
350
+ detectNames(text: string): Promise<string[]>;
351
+ size(): number;
352
+ }
353
+
354
+ export function createReversibleMasker(options?: {
355
+ names?: string[];
356
+ patterns?: Array<{ type: string; pattern: RegExp }>;
357
+ detect?: (text: string) => Awaitable<DetectedEntity[] | null | undefined>;
358
+ detectTypes?: string[];
359
+ minScore?: number;
360
+ detectMaxChars?: number;
361
+ minLength?: number;
362
+ token?: (type: string, sequence: number) => string;
363
+ typeOf?: (value: string) => string;
364
+ keep?: (value: string) => boolean;
365
+ }): ReversibleMasker;
366
+
367
+ export function withOutboundMasking<Rest extends unknown[], R>(
368
+ masker: ReversibleMasker,
369
+ call: (input: any, ...rest: Rest) => Awaitable<R>
370
+ ): (input: unknown, ...rest: Rest) => Promise<R>;
371
+
372
+ export function headWithoutCuttingWords(text: string, max: number): string;
373
+
374
+ /* ─────────────────── Passages in another language ─────────────────── */
375
+
376
+ export interface Passage {
377
+ title?: string;
378
+ text: string;
379
+ lang?: string;
380
+ kind?: string;
381
+ ref?: unknown;
382
+ url?: string;
383
+ }
384
+
385
+ export interface PassageTexts {
386
+ header?: string;
387
+ footer?: string;
388
+ /** Required when a passage can be foreign. `{languages}` becomes the tags found. */
389
+ foreign?: string;
390
+ empty?: string;
391
+ }
392
+
393
+ export function primaryLanguage(tag: unknown): string | null;
394
+ export function isForeignLanguage(passageLanguage: unknown, readerLanguage: unknown): boolean;
395
+ export function markForeignPassages(passages: Passage[], options: { lang: string; tag?: (language: string) => string }): { blocks: string[]; languages: string[] };
396
+ /** `passages: null` means the search failed: nothing is said. */
397
+ export function buildPassagesContext(options: { passages: Passage[] | null | undefined; lang: string; texts?: PassageTexts; tag?: (language: string) => string }): string[];
398
+ export function passageSource(passage: Passage, options?: { excerptMax?: number }): Omit<Passage, 'text'> & { excerpt: string };
399
+
400
+ /* ─────────────────── Sources ─────────────────── */
401
+
402
+ export interface SourceLedger<S = Record<string, unknown>> {
403
+ keep(found: S[], data?: unknown): void;
404
+ sources(): S[];
405
+ evidence(): string[];
406
+ size(): number;
407
+ }
408
+
409
+ export function createSourceLedger<S = Record<string, unknown>>(options?: { keyOf?: (source: S) => string }): SourceLedger<S>;
410
+
411
+ export function usedSources<S extends { url?: string; title?: string }, Place = unknown>(
412
+ answer: string,
413
+ sources: S[],
414
+ evidence?: string[],
415
+ options?: {
416
+ references?: (text: string) => Place[];
417
+ sameReference?: (a: Place, b: Place) => boolean;
418
+ ownReference?: (source: S) => Place | null | undefined;
419
+ commonWords?: Iterable<string>;
420
+ minWordLength?: number;
421
+ sharedMin?: number;
422
+ shortEvidence?: number;
423
+ }
424
+ ): S[];
425
+
426
+ export interface NliScores { entailment: number; neutral?: number; contradiction: number }
427
+
428
+ export function findContradiction(
429
+ answer: string,
430
+ sources: Array<string | { content?: string; snippet?: string; url?: string; source?: string }>,
431
+ options: {
432
+ compare: (pairs: Array<{ premise: string; hypothesis: string }>) => Awaitable<NliScores[] | null>;
433
+ timeoutMs?: number;
434
+ minWords?: number;
435
+ sharedMin?: number;
436
+ maxPairs?: number;
437
+ contradictionMin?: number;
438
+ entailmentMax?: number;
439
+ commonWords?: Iterable<string>;
440
+ }
441
+ ): Promise<{ contradicts: boolean; sentence: string | null; source: string | null } | null>;
442
+
443
+ export function factualSentences(answer: string, minWords: number): string[];
444
+
445
+ export function rerankResults<R, S, F extends { results: R[]; sources?: S[] }>(
446
+ query: string,
447
+ found: F,
448
+ options: {
449
+ score: (query: string, passages: string[]) => Awaitable<number[] | null>;
450
+ passageOf?: (result: R) => string;
451
+ sourceMatches?: (source: S, result: R) => boolean;
452
+ }
453
+ ): Promise<F>;
454
+
455
+ /* ─────────────────── Answer text ─────────────────── */
456
+
457
+ export function plainText(text: unknown): string;
458
+ export function tidyMarkdown(text: unknown, options?: { removeEmoji?: boolean }): string;
459
+ export function isFaithfulQuotation(quoted: string, source: string): boolean;
460
+ export function verifyQuotations<Refs = unknown>(text: string, options: {
461
+ findReferences: (text: string) => Awaitable<Array<{ start: number; end: number; refs: Refs }>>;
462
+ resolve: (refs: Refs) => Awaitable<string | null | undefined>;
463
+ }): Promise<string>;
464
+
465
+ /* ─────────────────── Reply language ─────────────────── */
466
+
467
+ export interface LanguageDetector {
468
+ byWords(text: string): string | null;
469
+ reply(text: string, appLanguage: string): Promise<string>;
470
+ languages: string[];
471
+ }
472
+
473
+ export function createLanguageDetector(options: {
474
+ words: Record<string, string[]>;
475
+ identify?: (text: string) => Awaitable<{ language: string; confidence: number }>;
476
+ fallback?: string;
477
+ confidenceMin?: number;
478
+ shortConfidenceMin?: number;
479
+ shortWords?: number;
480
+ lead?: number;
481
+ maxChars?: number;
482
+ }): LanguageDetector;
483
+
484
+ /* ─────────────────── Ask limit ─────────────────── */
485
+
486
+ export interface AskLimitVerdict { allowed: boolean; remaining: number; retryInMs: number }
487
+
488
+ export interface AskLimit {
489
+ /** Throws a 429 AppError with `code` and `retryInMs` past the limit. */
490
+ take(subject: string | number, now?: number): AskLimitVerdict;
491
+ peek(subject: string | number, now?: number): AskLimitVerdict;
492
+ reset(subject?: string | number): void;
493
+ size(): number;
494
+ sweep(now: number): void;
495
+ }
496
+
497
+ export function createAskLimit(options: { max: number; windowMs: number; code?: string; sweepEvery?: number }): AskLimit;
498
+
499
+ /* ─────────────────── OpenAI-compatible endpoints ─────────────────── */
500
+
501
+ export interface ChatMessage {
502
+ role: 'user' | 'assistant' | 'tool' | 'system';
503
+ text?: string;
504
+ content?: string;
505
+ images?: Array<{ mimeType: string; data: string }>;
506
+ toolCalls?: Array<{ id: string; name: string; args: unknown }>;
507
+ toolCallId?: string;
508
+ result?: unknown;
509
+ }
510
+
511
+ export interface ChatToolCall {
512
+ id: string;
513
+ name: string;
514
+ args: Record<string, unknown> | null;
515
+ /** The model sent arguments that are not a JSON object. */
516
+ invalid?: true;
517
+ }
518
+
519
+ export type FetchLike = (url: string, init: { method: string; headers: Record<string, string>; body: string; signal?: AbortSignal }) => Promise<{ ok: boolean; status: number; json(): Promise<any> }>;
520
+
521
+ export function askChatModel(
522
+ target: { url: string; key: string; model: string; extra?: Record<string, unknown> },
523
+ request: { system?: string; messages: ChatMessage[]; tools?: Array<Record<string, unknown>>; maxTokens?: number; temperature?: number },
524
+ io: { fetch: FetchLike; signal?: AbortSignal; timeoutMs?: number }
525
+ ): Promise<{ status: number; text?: string | null; toolCalls?: ChatToolCall[]; cut?: boolean }>;
526
+
527
+ export function readToolArguments(raw: unknown): { args: Record<string, unknown> | null; invalid: boolean };
528
+
529
+ export function createOpenAICompatibleProvider(options: {
530
+ id: string;
531
+ url: string;
532
+ getKey: () => Awaitable<string | null | undefined>;
533
+ models: ProviderModel[];
534
+ fetch: FetchLike;
535
+ external?: boolean;
536
+ timeoutMs?: number;
537
+ extra?: Record<string, unknown>;
538
+ toRequest?: (prompt: string, ctx: Record<string, unknown>) => { system?: string; messages: ChatMessage[] };
539
+ }): Provider;
package/src/index.js CHANGED
@@ -5,5 +5,13 @@ module.exports = {
5
5
  ...require('./pendingActions'),
6
6
  ...require('./fallback'),
7
7
  ...require('./responseCleaner'),
8
- ...require('./formatInstructions')
8
+ ...require('./formatInstructions'),
9
+ ...require('./breakers'),
10
+ ...require('./masking'),
11
+ ...require('./passages'),
12
+ ...require('./sources'),
13
+ ...require('./answerText'),
14
+ ...require('./language'),
15
+ ...require('./askLimit'),
16
+ ...require('./openaiCompatible')
9
17
  };
@@ -0,0 +1,85 @@
1
+ /**
2
+ * The language to answer in.
3
+ *
4
+ * The language set in the app stays the rule; a message CLEARLY written in
5
+ * another supported language is answered in that one. Two readers:
6
+ *
7
+ * 1. the small words — articles, pronouns, the few verbs of every sentence
8
+ * — which decide a short message ("merci", "ok") that no statistical
9
+ * identifier can;
10
+ * 2. an identifier (fastText's lid.176 or anything alike), injected, trusted
11
+ * only above a confidence — a far higher one for a message of two words.
12
+ *
13
+ * Anything else — an unsupported language, a message too short to tell, the
14
+ * identifier out of reach — keeps the app's language. Guessing wrong switches
15
+ * the whole answer, the voice and the sources into a language the person did
16
+ * not ask for.
17
+ */
18
+
19
+ /**
20
+ * @param {object} options
21
+ * @param {Record<string, string[]>} options.words language -> its small words (lower case).
22
+ * @param {Function} [options.identify] async (text) => { language, confidence }.
23
+ * @param {string} [options.fallback] when the app language is not supported. Default: the first language.
24
+ * @param {number} [options.confidenceMin] Default 0.5.
25
+ * @param {number} [options.shortConfidenceMin] for fewer than `shortWords` words. Default 0.9.
26
+ * @param {number} [options.shortWords] Default 3.
27
+ * @param {number} [options.lead] how far the small words must lean to decide alone. Default 2.
28
+ * @param {number} [options.maxChars] text handed to the identifier. Default 2000.
29
+ */
30
+ function createLanguageDetector(options = {}) {
31
+ const entries = Object.entries(options.words || {});
32
+ if (entries.length < 2) throw new Error('createLanguageDetector requires small words for at least two languages.');
33
+ const sets = new Map(entries.map(([language, list]) => [language, new Set(list.map((word) => String(word).toLowerCase().replace(/’/g, "'")))]));
34
+ const languages = [...sets.keys()];
35
+ const identify = typeof options.identify === 'function' ? options.identify : null;
36
+ const fallbackLanguage = languages.includes(options.fallback) ? options.fallback : languages[0];
37
+ const confidenceMin = options.confidenceMin === undefined ? 0.5 : options.confidenceMin;
38
+ const shortConfidenceMin = options.shortConfidenceMin === undefined ? 0.9 : options.shortConfidenceMin;
39
+ const shortWords = options.shortWords || 3;
40
+ const lead = options.lead || 2;
41
+ const maxChars = options.maxChars || 2000;
42
+
43
+ /** The small words' vote: a language, or null when they do not lean clearly. */
44
+ function byWords(text) {
45
+ const words = (String(text === null || text === undefined ? '' : text).toLowerCase().match(/[\p{L}'’-]+/gu) || [])
46
+ .map((word) => word.replace(/’/g, "'"));
47
+ const counts = languages.map((language) => ({ language, count: words.filter((word) => sets.get(language).has(word)).length }))
48
+ .sort((a, b) => b.count - a.count);
49
+ const [first, second] = counts;
50
+ if (!first.count) return null;
51
+ if (first.count >= second.count + lead) return first.language;
52
+ /* One or two words, all from one language: "merci", "thank you". */
53
+ if (words.length <= 2 && second.count === 0) return first.language;
54
+ return null;
55
+ }
56
+
57
+ /**
58
+ * @param {string} text what the person wrote.
59
+ * @param {string} appLanguage the language set in the app.
60
+ */
61
+ async function reply(text, appLanguage) {
62
+ const fallback = languages.includes(appLanguage) ? appLanguage : fallbackLanguage;
63
+ const written = String(text === null || text === undefined ? '' : text).trim();
64
+ if (!written) return fallback;
65
+ const words = byWords(written);
66
+ if (identify) {
67
+ try {
68
+ const found = await identify(written.slice(0, maxChars));
69
+ const language = found && typeof found.language === 'string' ? found.language.toLowerCase() : null;
70
+ const confidence = Number(found && found.confidence) || 0;
71
+ const short = (written.match(/\p{L}+/gu) || []).length < shortWords;
72
+ if (confidence >= (short ? shortConfidenceMin : confidenceMin)) {
73
+ return languages.includes(language) ? language : (words || fallback);
74
+ }
75
+ } catch (_error) {
76
+ /* The identifier out of reach: the small words alone. */
77
+ }
78
+ }
79
+ return words || fallback;
80
+ }
81
+
82
+ return { byWords, reply, languages };
83
+ }
84
+
85
+ module.exports = { createLanguageDetector };
package/src/masking.js ADDED
@@ -0,0 +1,279 @@
1
+ /**
2
+ * Masking what identifies people before a text leaves for an outside provider,
3
+ * and putting it back in the answer.
4
+ *
5
+ * The prompt was masked; the web search was not. The model wrote the search
6
+ * query from the masked question, the loop unmasked the tool parameters for
7
+ * the tools that need real names — and the child's name went to the search
8
+ * engine in clear. So the rule here is about DIRECTION, not about which tool:
9
+ * whatever leaves the machine is masked at the moment it leaves, and only what
10
+ * comes back is unmasked.
11
+ *
12
+ * One masker per conversation. It LEARNS: a name found once (in the school's
13
+ * register, by a pattern, by an entity detector) is masked everywhere after,
14
+ * with the same token, so the model can still tell two people apart.
15
+ *
16
+ * Choices kept from real use:
17
+ *
18
+ * - register names match CASE-SENSITIVELY and on whole words. "Grace",
19
+ * "Chance" and "Merveille" are real first names; "grâce à" and "la chance"
20
+ * must survive. A common word damaged costs less than a child's name in
21
+ * clear, but the case keeps most sentences intact;
22
+ * - a name shorter than three letters identifies no one and breaks sentences;
23
+ * - a register name absent from the text creates NO token — a register of a
24
+ * thousand names must not pollute the unmasking table;
25
+ * - models drop the leading "#" of a token; unmasking tolerates it;
26
+ * - tokens are replaced longest first, on token boundaries, so #PERSON_0001
27
+ * never eats the start of #PERSON_00012.
28
+ */
29
+
30
+ const DEFAULT_MIN_LENGTH = 3;
31
+ const DEFAULT_MIN_SCORE = 0.6;
32
+ const DEFAULT_DETECT_MAX_CHARS = 1500;
33
+
34
+ const ESCAPE = /[.*+?^${}()|[\]\\]/g;
35
+ const escape = (text) => String(text).replace(ESCAPE, '\\$&');
36
+ const LEXICAL = '\\p{L}\\p{N}\\p{M}_';
37
+
38
+ const defaultToken = (type, sequence) => `#${String(type || 'ENTITY').toUpperCase()}_${String(sequence).padStart(4, '0')}`;
39
+
40
+ /** At most `max` characters, cut on a blank: a name is never sliced in two. */
41
+ function headWithoutCuttingWords(text, max) {
42
+ if (text.length <= max) return text;
43
+ const cut = text.slice(0, max);
44
+ const blank = cut.search(/\s\S*$/);
45
+ return blank > 0 ? cut.slice(0, blank) : cut;
46
+ }
47
+
48
+ /**
49
+ * @param {object} [options]
50
+ * @param {string[]} [options.names] register names, masked case-sensitively.
51
+ * @param {Array<{type: string, pattern: RegExp}>} [options.patterns] global
52
+ * regexes for structured data (e-mail, phone…), masked as found.
53
+ * @param {Function} [options.detect] async (text) => [{ text, type, score }] — an
54
+ * entity detector (a local NER model). Its absence, or a null answer, leaves
55
+ * the register and the patterns alone.
56
+ * @param {string[]} [options.detectTypes] which detected types to mask. Default ['person'].
57
+ * @param {number} [options.minScore] detections under this are ignored. Default 0.6 —
58
+ * a false name masked damages the sentence sent to the model.
59
+ * @param {number} [options.detectMaxChars] text handed to the detector, cut on a blank. Default 1500.
60
+ * @param {number} [options.minLength] shortest value masked. Default 3.
61
+ * @param {Function} [options.token] (type, sequence) => string. Default "#TYPE_0001".
62
+ * @param {Function} [options.typeOf] (value) => type for register names. Default 'PERSON'.
63
+ * @param {Function} [options.keep] (value) => true to never mask it (a product name…).
64
+ */
65
+ function createReversibleMasker(options = {}) {
66
+ const minLength = options.minLength || DEFAULT_MIN_LENGTH;
67
+ const minScore = options.minScore === undefined ? DEFAULT_MIN_SCORE : options.minScore;
68
+ const detectMaxChars = options.detectMaxChars || DEFAULT_DETECT_MAX_CHARS;
69
+ const detect = typeof options.detect === 'function' ? options.detect : null;
70
+ const detectTypes = new Set(options.detectTypes || ['person']);
71
+ const tokenFor = options.token || defaultToken;
72
+ const typeOf = options.typeOf || (() => 'PERSON');
73
+ const keep = options.keep || (() => false);
74
+ const patterns = (options.patterns || []).map(({ type, pattern }) => {
75
+ if (!(pattern instanceof RegExp) || !pattern.global) {
76
+ throw new Error('createReversibleMasker: every pattern must be a global RegExp.');
77
+ }
78
+ return { type, pattern };
79
+ });
80
+
81
+ /* token -> real value ; real value -> token */
82
+ const byToken = new Map();
83
+ const byValue = new Map();
84
+ /* values matched case-insensitively (patterns: an e-mail is an e-mail in any case) */
85
+ const loose = new Set();
86
+ const register = new Set();
87
+ let sequence = 0;
88
+
89
+ const looksLikeToken = (value) => byToken.has(value) || byToken.has(`#${value}`);
90
+ const alreadyMasked = /#[A-Z]+_\d+/;
91
+
92
+ function tokenOf(value, type) {
93
+ if (byValue.has(value)) return byValue.get(value);
94
+ let token;
95
+ do {
96
+ sequence += 1;
97
+ token = tokenFor(type, sequence);
98
+ } while (byToken.has(token));
99
+ byToken.set(token, value);
100
+ byValue.set(value, token);
101
+ return token;
102
+ }
103
+
104
+ function acceptable(value) {
105
+ const text = String(value === null || value === undefined ? '' : value).trim();
106
+ if (text.length < minLength) return null;
107
+ if (alreadyMasked.test(text) || looksLikeToken(text)) return null;
108
+ if (keep(text)) return null;
109
+ return text;
110
+ }
111
+
112
+ function replaceWhole(text, value, token, caseSensitive) {
113
+ const regex = new RegExp(`(^|[^${LEXICAL}])(${escape(value)})(?=$|[^${LEXICAL}])`, caseSensitive ? 'gu' : 'giu');
114
+ return text.replace(regex, (_whole, prefix) => `${prefix}${token}`);
115
+ }
116
+
117
+ function learn(names, type) {
118
+ for (const raw of names || []) {
119
+ const value = acceptable(raw);
120
+ if (value) register.add(value);
121
+ if (value && type) typeOverride.set(value, type);
122
+ }
123
+ }
124
+ const typeOverride = new Map();
125
+ learn(options.names);
126
+
127
+ /** Mask a string with everything known so far, plus `names` for this call. */
128
+ function mask(text, { names } = {}) {
129
+ if (typeof text !== 'string' || !text) return text;
130
+ if (names) learn(names);
131
+ let out = text;
132
+
133
+ /* Structured data first, on the intact text. */
134
+ for (const { type, pattern } of patterns) {
135
+ pattern.lastIndex = 0;
136
+ for (const match of out.matchAll(pattern)) {
137
+ const value = acceptable(match[0]);
138
+ if (!value) continue;
139
+ tokenOf(value, type);
140
+ loose.add(value);
141
+ }
142
+ }
143
+
144
+ /* Longest first: "Marie Kabongo" before "Marie". Case-sensitive for the
145
+ register; a value absent from the text creates no token. */
146
+ const candidates = [...new Set([...register, ...byValue.keys()])].sort((a, b) => b.length - a.length);
147
+ for (const value of candidates) {
148
+ const caseSensitive = !loose.has(value);
149
+ const present = caseSensitive ? out.includes(value) : out.toLowerCase().includes(value.toLowerCase());
150
+ if (!present) continue;
151
+ const token = tokenOf(value, typeOverride.get(value) || typeOf(value));
152
+ out = replaceWhole(out, value, token, caseSensitive);
153
+ }
154
+ return out;
155
+ }
156
+
157
+ /**
158
+ * Names the detector finds, for the register — deduplicated, filtered by
159
+ * type, score and length; tokens already placed are not names.
160
+ */
161
+ async function detectNames(text) {
162
+ if (!detect || typeof text !== 'string' || !text.trim()) return [];
163
+ let found;
164
+ try {
165
+ found = await detect(headWithoutCuttingWords(text, detectMaxChars));
166
+ } catch (_error) {
167
+ return [];
168
+ }
169
+ if (!Array.isArray(found)) return [];
170
+ const names = found
171
+ .filter((entity) => entity && detectTypes.has(entity.type) && typeof entity.score === 'number' && entity.score >= minScore)
172
+ .map((entity) => acceptable(entity.text))
173
+ .filter(Boolean);
174
+ return [...new Set(names)];
175
+ }
176
+
177
+ /** Detect, learn, then mask. The detector sees the text in clear — keep it local. */
178
+ async function maskAsync(text, opts = {}) {
179
+ if (typeof text !== 'string' || !text) return text;
180
+ const detected = await detectNames(text);
181
+ return mask(text, { names: [...(opts.names || []), ...detected] });
182
+ }
183
+
184
+ /** Every string of a structure, masked. Keys are left alone. */
185
+ function maskDeep(value) {
186
+ if (typeof value === 'string') return mask(value);
187
+ if (Array.isArray(value)) return value.map(maskDeep);
188
+ if (value instanceof Date) return new Date(value.getTime());
189
+ if (value && typeof value === 'object') {
190
+ return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, maskDeep(child)]));
191
+ }
192
+ return value;
193
+ }
194
+
195
+ function unmask(text) {
196
+ if (typeof text !== 'string' || !text || !byToken.size) return text;
197
+ let out = text;
198
+ const tokens = [...byToken.keys()].sort((a, b) => b.length - a.length);
199
+ for (const token of tokens) {
200
+ const value = byToken.get(token);
201
+ const variants = token.startsWith('#') ? [token, token.slice(1)] : [token];
202
+ for (const variant of variants) {
203
+ if (!out.includes(variant)) continue;
204
+ const regex = new RegExp(`(^|[^${LEXICAL}#])(${escape(variant)})(?![${LEXICAL}])`, 'gu');
205
+ out = out.replace(regex, (_whole, prefix) => `${prefix}${value}`);
206
+ }
207
+ }
208
+ return out;
209
+ }
210
+
211
+ function unmaskDeep(value) {
212
+ if (typeof value === 'string') return unmask(value);
213
+ if (Array.isArray(value)) return value.map(unmaskDeep);
214
+ if (value instanceof Date) return new Date(value.getTime());
215
+ if (value && typeof value === 'object') {
216
+ return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, unmaskDeep(child)]));
217
+ }
218
+ return value;
219
+ }
220
+
221
+ /**
222
+ * Unmask a stream of chunks. A token can straddle two chunks ("#PERS" then
223
+ * "ON_0001"); the tail that could still be the start of a token is held
224
+ * back until the next chunk decides.
225
+ */
226
+ async function* unmaskStream(chunks) {
227
+ let carry = '';
228
+ const longest = () => Math.max(0, ...[...byToken.keys()].map((token) => token.length));
229
+ for await (const chunk of chunks) {
230
+ const text = carry + String(chunk);
231
+ const hold = holdBack(text, longest());
232
+ carry = text.slice(text.length - hold);
233
+ const ready = text.slice(0, text.length - hold);
234
+ if (ready) yield unmask(ready);
235
+ }
236
+ if (carry) yield unmask(carry);
237
+ }
238
+
239
+ function holdBack(text, maxLength) {
240
+ if (!maxLength) return 0;
241
+ const tokens = [...byToken.keys()].flatMap((token) => (token.startsWith('#') ? [token, token.slice(1)] : [token]));
242
+ for (let size = Math.min(maxLength, text.length); size > 0; size -= 1) {
243
+ const tail = text.slice(text.length - size);
244
+ /* A whole token at the very end may still be followed by a digit that
245
+ makes it another token: hold it too. */
246
+ if (tokens.some((token) => token.startsWith(tail))) return size;
247
+ }
248
+ return 0;
249
+ }
250
+
251
+ return {
252
+ mask,
253
+ maskAsync,
254
+ maskDeep,
255
+ unmask,
256
+ unmaskDeep,
257
+ unmaskStream,
258
+ detectNames,
259
+ /** How many values are masked so far — never the values. */
260
+ size: () => byToken.size
261
+ };
262
+ }
263
+
264
+ /**
265
+ * Wrap an outside call so its input is masked on the way out and its answer
266
+ * unmasked on the way back.
267
+ */
268
+ function withOutboundMasking(masker, call) {
269
+ if (!masker || typeof masker.maskAsync !== 'function' || typeof masker.unmaskDeep !== 'function') {
270
+ throw new Error('withOutboundMasking requires a masker from createReversibleMasker().');
271
+ }
272
+ if (typeof call !== 'function') throw new Error('withOutboundMasking requires a function to wrap.');
273
+ return async (input, ...rest) => {
274
+ const outbound = typeof input === 'string' ? await masker.maskAsync(input) : masker.maskDeep(input);
275
+ return masker.unmaskDeep(await call(outbound, ...rest));
276
+ };
277
+ }
278
+
279
+ module.exports = { createReversibleMasker, withOutboundMasking, headWithoutCuttingWords };