@astratra/ai 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -0
- package/package.json +1 -1
- package/src/formatInstructions.js +99 -0
- package/src/index.d.ts +72 -0
- package/src/index.js +3 -1
- package/src/responseCleaner.js +337 -0
package/README.md
CHANGED
|
@@ -185,3 +185,78 @@ rien n'était apprend aux utilisateurs à se méfier des bonnes.
|
|
|
185
185
|
Et l'erreur du fournisseur est **transportée**, pas avalée : la gober en
|
|
186
186
|
silence cacherait la panne à ta propre supervision. Une question sans réponse
|
|
187
187
|
déterministe est déclinée, pas inventée.
|
|
188
|
+
|
|
189
|
+
## Nettoyer une réponse avant qu'un humain la lise
|
|
190
|
+
|
|
191
|
+
Une règle écrite dans l'invite n'est qu'un vœu : le modèle la suit quand ça
|
|
192
|
+
l'arrange. Trois fuites arrivaient quand même à l'écran — du JSON brut, les
|
|
193
|
+
étiquettes d'une structure que le modèle s'est inventée (`**introduction** :`),
|
|
194
|
+
et des brouillons de raisonnement ou des relances robotiques en fin de réponse.
|
|
195
|
+
`createResponseCleaner` les retire de façon déterministe.
|
|
196
|
+
|
|
197
|
+
```js
|
|
198
|
+
const { createResponseCleaner } = require('@astratra/ai');
|
|
199
|
+
|
|
200
|
+
const nettoyeur = createResponseCleaner({
|
|
201
|
+
shared: { // appliqué quelle que soit la langue
|
|
202
|
+
payloadKeys: ['response', 'message', 'answer'],
|
|
203
|
+
titleKeys: ['title', 'name'],
|
|
204
|
+
lineLabels: ['title', 'introduction', 'features', /key[ _]features/],
|
|
205
|
+
reasoningStarters: ['Wait', 'Actually'],
|
|
206
|
+
finalMarkers: [/Final\s*answer/],
|
|
207
|
+
},
|
|
208
|
+
languages: {
|
|
209
|
+
fr: {
|
|
210
|
+
payloadKeys: ['réponse'],
|
|
211
|
+
closingPhrases: [/Besoin d'autre chose[^.?!]*[.?!]?/],
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
fallbackLanguage: 'fr',
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
nettoyeur.clean(texteDuModele, { language: 'fr' });
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
**Aucune clé JSON à l'écran, jamais.** Un ancien convertisseur écrivait
|
|
221
|
+
« title : … », « features : … » : les clés restaient visibles. Ici un titre
|
|
222
|
+
devient une ligne en gras, un texte un paragraphe, une liste des puces, un
|
|
223
|
+
élément nommé « **Nom** — description ». Quand le modèle enveloppe sa réponse
|
|
224
|
+
(`{"role": "...", "response": "..."}`), le champ utile EST la réponse.
|
|
225
|
+
|
|
226
|
+
Les étiquettes de ligne ne sont retirées **qu'en début de ligne** : le même mot
|
|
227
|
+
au milieu d'une phrase n'est jamais touché. Pour un brouillon, seul ce qui suit
|
|
228
|
+
la **dernière** ligne de raisonnement est gardé. Les relances empilées en fin de
|
|
229
|
+
réponse sont retirées en plusieurs passes, bornées (`maxClosingPasses`, 3 par
|
|
230
|
+
défaut).
|
|
231
|
+
|
|
232
|
+
Le paquet ne contient **aucun mot** : chaque entrée est un mot littéral
|
|
233
|
+
(échappé) ou une `RegExp`, fournie par l'appelant, par langue. Sans vocabulaire,
|
|
234
|
+
seul le nettoyage structurel tourne (blocs `<think>`, JSON, espaces).
|
|
235
|
+
|
|
236
|
+
## Dire au modèle où part sa réponse
|
|
237
|
+
|
|
238
|
+
```js
|
|
239
|
+
const { createFormatInstructions } = require('@astratra/ai');
|
|
240
|
+
|
|
241
|
+
const forme = createFormatInstructions({
|
|
242
|
+
languages: {
|
|
243
|
+
fr: {
|
|
244
|
+
heading: '## Où part ta réponse',
|
|
245
|
+
intro: "Ta réponse s'affiche sur {surface}.",
|
|
246
|
+
surfaceNames: { phone: 'un téléphone', tablet: 'une tablette', desktop: 'un navigateur' },
|
|
247
|
+
table: 'Un tableau tient au maximum {columns} colonnes courtes.',
|
|
248
|
+
narrow: "Sur un téléphone, préfère la liste au moindre doute.",
|
|
249
|
+
wide: 'Le tableau reste réservé aux données réellement tabulaires.',
|
|
250
|
+
paragraphs: 'Écris en paragraphes : une idée chacun, des phrases complètes, jamais de trait (---).',
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
forme.build(req.headers['x-surface'], 'fr');
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
Surfaces par défaut : téléphone 3 colonnes, tablette 4, bureau 6. Une surface
|
|
259
|
+
absente ou inconnue vaut **téléphone** : se tromper vers le petit coûte une liste
|
|
260
|
+
là où un tableau tenait, se tromper vers le grand coûte un tableau illisible.
|
|
261
|
+
La règle `paragraphs` est **obligatoire** : un pack de langue qui ne l'a pas est
|
|
262
|
+
refusé à la création, pas découvert sur un écran.
|
package/package.json
CHANGED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telling the model where its reply will be displayed.
|
|
3
|
+
*
|
|
4
|
+
* The same answer goes to a 1,400-pixel desktop console and to a 390-point
|
|
5
|
+
* phone. A model that is not told writes for the screen it imagines, and ships
|
|
6
|
+
* five-column tables that a phone renders as crushed columns or a horizontal
|
|
7
|
+
* scroll nobody finds.
|
|
8
|
+
*
|
|
9
|
+
* Why the default surface is the PHONE: guessing small wrongly costs a list
|
|
10
|
+
* where a table would have fit — still readable. Guessing large wrongly costs an
|
|
11
|
+
* unreadable table on the phone. The risk is not symmetric.
|
|
12
|
+
*
|
|
13
|
+
* Why the paragraph rule is mandatory: without it models start a sentence and
|
|
14
|
+
* leave it hanging, or pack a compact block with no breathing room, and put
|
|
15
|
+
* horizontal rules (---) between ideas. The rule must reach EVERY surface — so a
|
|
16
|
+
* language pack without it is refused at creation, not discovered on a screen.
|
|
17
|
+
*
|
|
18
|
+
* All wording is supplied by the caller, per language. Templates may use
|
|
19
|
+
* {surface} (that language's name for the surface) and {columns}.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const DEFAULT_SURFACES = {
|
|
23
|
+
phone: { columns: 3, narrow: true, aliases: ['mobile', 'ios', 'android'] },
|
|
24
|
+
tablet: { columns: 4, narrow: false, aliases: ['ipad'] },
|
|
25
|
+
desktop: { columns: 6, narrow: false, aliases: ['web', 'browser'] }
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
function fill(template, values) {
|
|
29
|
+
return String(template).replace(/\{(\w+)\}/g, (whole, name) => (
|
|
30
|
+
Object.prototype.hasOwnProperty.call(values, name) ? String(values[name]) : whole
|
|
31
|
+
));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {object} options
|
|
36
|
+
* @param {Record<string, object>} options.languages language -> pack:
|
|
37
|
+
* { paragraphs (required), heading?, intro?, table?, narrow?, wide?,
|
|
38
|
+
* rules?: string[], surfaceNames?: Record<surfaceId, string> }
|
|
39
|
+
* @param {Record<string, {columns: number, narrow?: boolean, aliases?: string[]}>} [options.surfaces]
|
|
40
|
+
* @param {string} [options.defaultSurface='phone']
|
|
41
|
+
* @param {string} [options.fallbackLanguage]
|
|
42
|
+
*/
|
|
43
|
+
function createFormatInstructions(options = {}) {
|
|
44
|
+
const surfaces = options.surfaces || DEFAULT_SURFACES;
|
|
45
|
+
const defaultSurface = options.defaultSurface || 'phone';
|
|
46
|
+
if (!surfaces[defaultSurface]) {
|
|
47
|
+
throw new Error(`createFormatInstructions: unknown defaultSurface "${defaultSurface}".`);
|
|
48
|
+
}
|
|
49
|
+
const languages = options.languages || {};
|
|
50
|
+
const names = Object.keys(languages);
|
|
51
|
+
if (!names.length) throw new Error('createFormatInstructions requires at least one language pack.');
|
|
52
|
+
for (const name of names) {
|
|
53
|
+
const pack = languages[name];
|
|
54
|
+
if (!pack || typeof pack.paragraphs !== 'string' || !pack.paragraphs.trim()) {
|
|
55
|
+
throw new Error(`createFormatInstructions: language "${name}" has no paragraphs rule.`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
const fallbackLanguage = options.fallbackLanguage && languages[options.fallbackLanguage]
|
|
59
|
+
? options.fallbackLanguage
|
|
60
|
+
: names[0];
|
|
61
|
+
|
|
62
|
+
const aliasIndex = new Map();
|
|
63
|
+
for (const [id, surface] of Object.entries(surfaces)) {
|
|
64
|
+
aliasIndex.set(id.toLowerCase(), id);
|
|
65
|
+
for (const alias of surface.aliases || []) aliasIndex.set(String(alias).toLowerCase(), id);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Any client-sent value maps to a known surface; missing or unknown means the default. */
|
|
69
|
+
function normalizeSurface(value) {
|
|
70
|
+
if (typeof value !== 'string') return defaultSurface;
|
|
71
|
+
return aliasIndex.get(value.trim().toLowerCase()) || defaultSurface;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function build(surfaceValue, language) {
|
|
75
|
+
const id = normalizeSurface(surfaceValue);
|
|
76
|
+
const surface = surfaces[id];
|
|
77
|
+
const pack = languages[language] || languages[fallbackLanguage];
|
|
78
|
+
const values = {
|
|
79
|
+
surface: (pack.surfaceNames && pack.surfaceNames[id]) || id,
|
|
80
|
+
columns: surface.columns
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const lines = [];
|
|
84
|
+
if (pack.intro) lines.push(fill(pack.intro, values));
|
|
85
|
+
const bullets = [];
|
|
86
|
+
if (pack.table) bullets.push(pack.table);
|
|
87
|
+
const sizeRule = surface.narrow ? pack.narrow : pack.wide;
|
|
88
|
+
if (sizeRule) bullets.push(sizeRule);
|
|
89
|
+
bullets.push(pack.paragraphs);
|
|
90
|
+
for (const rule of pack.rules || []) bullets.push(rule);
|
|
91
|
+
lines.push(bullets.map((b) => `- ${fill(b, values)}`).join('\n'));
|
|
92
|
+
|
|
93
|
+
return (pack.heading ? `${fill(pack.heading, values)}\n` : '') + lines.join('\n\n');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return { build, normalizeSurface, surfaces: Object.keys(surfaces), languages: names };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
module.exports = { createFormatInstructions, DEFAULT_SURFACES };
|
package/src/index.d.ts
CHANGED
|
@@ -213,3 +213,75 @@ export function createDeterministicFallback<Input = Record<string, unknown>>(opt
|
|
|
213
213
|
classify?: (input: Input) => string | null | undefined;
|
|
214
214
|
markDegraded?: (answer: Record<string, unknown>) => Record<string, unknown>;
|
|
215
215
|
}): DeterministicFallback<Input>;
|
|
216
|
+
|
|
217
|
+
/** A literal word (escaped) or a RegExp whose source is used as-is. */
|
|
218
|
+
export type VocabularyEntry = string | RegExp;
|
|
219
|
+
|
|
220
|
+
export interface CleanerVocabulary {
|
|
221
|
+
/** Wrapper keys whose string value IS the answer ({"response": "..."}). */
|
|
222
|
+
payloadKeys?: VocabularyEntry[];
|
|
223
|
+
/** Object keys rendered as a bold heading when JSON becomes prose. */
|
|
224
|
+
titleKeys?: string[];
|
|
225
|
+
/** Labels removed at the start of a line only ("**introduction** : ..."). */
|
|
226
|
+
lineLabels?: VocabularyEntry[];
|
|
227
|
+
/** Form-like labels removed anywhere they are followed by ":", "-" or "—". */
|
|
228
|
+
inlineLabels?: VocabularyEntry[];
|
|
229
|
+
/** Labels removed when alone on a line or leading one, bold or not, colon or not. */
|
|
230
|
+
headingLabels?: VocabularyEntry[];
|
|
231
|
+
reasoningLabels?: VocabularyEntry[];
|
|
232
|
+
reasoningStarters?: VocabularyEntry[];
|
|
233
|
+
finalMarkers?: VocabularyEntry[];
|
|
234
|
+
planningStarters?: VocabularyEntry[];
|
|
235
|
+
leadingFillers?: VocabularyEntry[];
|
|
236
|
+
annotationMarkers?: VocabularyEntry[];
|
|
237
|
+
/** Robotic closings, removed only at the very end of the reply. */
|
|
238
|
+
closingPhrases?: VocabularyEntry[];
|
|
239
|
+
openers?: VocabularyEntry[];
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export interface ResponseCleaner {
|
|
243
|
+
clean(text: string | null | undefined, options?: { language?: string }): string;
|
|
244
|
+
jsonToProse(value: unknown, language?: string): string;
|
|
245
|
+
languages: string[];
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function createResponseCleaner(options?: {
|
|
249
|
+
shared?: CleanerVocabulary;
|
|
250
|
+
languages?: Record<string, CleanerVocabulary>;
|
|
251
|
+
fallbackLanguage?: string;
|
|
252
|
+
maxClosingPasses?: number;
|
|
253
|
+
}): ResponseCleaner;
|
|
254
|
+
|
|
255
|
+
export interface FormatSurface {
|
|
256
|
+
columns: number;
|
|
257
|
+
narrow?: boolean;
|
|
258
|
+
aliases?: string[];
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export interface FormatLanguagePack {
|
|
262
|
+
/** Mandatory: the paragraph rule reaches every surface. */
|
|
263
|
+
paragraphs: string;
|
|
264
|
+
heading?: string;
|
|
265
|
+
intro?: string;
|
|
266
|
+
table?: string;
|
|
267
|
+
narrow?: string;
|
|
268
|
+
wide?: string;
|
|
269
|
+
rules?: string[];
|
|
270
|
+
surfaceNames?: Record<string, string>;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export interface FormatInstructions {
|
|
274
|
+
build(surface: unknown, language?: string): string;
|
|
275
|
+
normalizeSurface(surface: unknown): string;
|
|
276
|
+
surfaces: string[];
|
|
277
|
+
languages: string[];
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export const DEFAULT_SURFACES: Record<'phone' | 'tablet' | 'desktop', FormatSurface>;
|
|
281
|
+
|
|
282
|
+
export function createFormatInstructions(options: {
|
|
283
|
+
languages: Record<string, FormatLanguagePack>;
|
|
284
|
+
surfaces?: Record<string, FormatSurface>;
|
|
285
|
+
defaultSurface?: string;
|
|
286
|
+
fallbackLanguage?: string;
|
|
287
|
+
}): FormatInstructions;
|
package/src/index.js
CHANGED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cleaning a model's reply before a human reads it.
|
|
3
|
+
*
|
|
4
|
+
* A rule written in the system prompt is a wish: the model follows it when it
|
|
5
|
+
* suits it. Three leaks kept reaching real screens despite explicit
|
|
6
|
+
* instructions, so this cleaner removes them deterministically:
|
|
7
|
+
*
|
|
8
|
+
* 1. Raw JSON. The model answers `{"title": "...", "features": [...]}` or wraps
|
|
9
|
+
* its real answer as `{"role": "...", "response": "..."}`. An earlier
|
|
10
|
+
* converter turned JSON into "title: ...", "features: ..." — the KEYS were
|
|
11
|
+
* still on screen. A JSON key is not a word for the user, even translated,
|
|
12
|
+
* so none is ever shown: a title becomes a bold line, text a paragraph, a
|
|
13
|
+
* list bullets, a named item "**Name** — description".
|
|
14
|
+
* 2. Labels copied from a structure the model invented for itself
|
|
15
|
+
* ("**introduction** : ..."). Removed at the start of a line only — the same
|
|
16
|
+
* word inside a sentence is never touched.
|
|
17
|
+
* 3. Reasoning drafts and robotic closings ("Wait, ...", "Final answer:",
|
|
18
|
+
* "Let me know if you need anything else.").
|
|
19
|
+
*
|
|
20
|
+
* The structural part (think blocks, JSON detection, prose conversion,
|
|
21
|
+
* whitespace) is language-neutral and built in. Every WORD — which keys carry
|
|
22
|
+
* the answer, which labels to drop, which closings are robotic — belongs to a
|
|
23
|
+
* language and is supplied by the caller, per language.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
const ESCAPE = /[.*+?^${}()|[\]\\]/g;
|
|
27
|
+
|
|
28
|
+
/** A vocabulary entry is a literal word (escaped) or a RegExp (its source is used as-is). */
|
|
29
|
+
function toSource(entry) {
|
|
30
|
+
if (entry instanceof RegExp) return entry.source;
|
|
31
|
+
return String(entry).replace(ESCAPE, '\\$&');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function alternation(entries) {
|
|
35
|
+
return entries.map(toSource).join('|');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const VOCABULARY_FIELDS = [
|
|
39
|
+
'payloadKeys', 'titleKeys', 'lineLabels', 'inlineLabels', 'headingLabels',
|
|
40
|
+
'reasoningLabels', 'reasoningStarters', 'finalMarkers', 'planningStarters',
|
|
41
|
+
'leadingFillers', 'annotationMarkers', 'closingPhrases', 'openers'
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
function mergeVocabularies(parts) {
|
|
45
|
+
const merged = {};
|
|
46
|
+
for (const field of VOCABULARY_FIELDS) {
|
|
47
|
+
const seen = new Set();
|
|
48
|
+
merged[field] = [];
|
|
49
|
+
for (const part of parts) {
|
|
50
|
+
for (const entry of (part && part[field]) || []) {
|
|
51
|
+
const key = entry instanceof RegExp ? `re:${entry.source}` : `s:${entry}`;
|
|
52
|
+
if (seen.has(key)) continue;
|
|
53
|
+
seen.add(key);
|
|
54
|
+
merged[field].push(entry);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return merged;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/* Lenient parse: models emit trailing commas, single quotes and unquoted keys.
|
|
62
|
+
A strict JSON.parse alone let those through to the screen untouched. */
|
|
63
|
+
function lenientParse(str) {
|
|
64
|
+
try { return JSON.parse(str); } catch (_e) { /* try the repaired form */ }
|
|
65
|
+
try {
|
|
66
|
+
const fixed = str
|
|
67
|
+
.replace(/,\s*([}\]])/g, '$1')
|
|
68
|
+
.replace(/'/g, '"')
|
|
69
|
+
// Unquoted keys, only right after { or , so text values are not rewritten.
|
|
70
|
+
.replace(/([{,]\s*)(\w+)\s*:/g, '$1"$2":')
|
|
71
|
+
.replace(/""+/g, '"');
|
|
72
|
+
return JSON.parse(fixed);
|
|
73
|
+
} catch (_e) {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function inlineValue(value) {
|
|
79
|
+
if (value === null || value === undefined) return '';
|
|
80
|
+
if (typeof value !== 'object') return String(value).trim();
|
|
81
|
+
if (Array.isArray(value)) return value.map(inlineValue).filter(Boolean).join(', ');
|
|
82
|
+
return Object.values(value).map(inlineValue).filter(Boolean).join(' — ');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function makeJsonToProse(titleKeys) {
|
|
86
|
+
const titles = titleKeys.map((k) => String(k).toLowerCase());
|
|
87
|
+
|
|
88
|
+
function jsonToProse(value, depth = 0) {
|
|
89
|
+
const indent = ' '.repeat(depth);
|
|
90
|
+
if (value === null || value === undefined) return '';
|
|
91
|
+
if (typeof value !== 'object') return String(value).trim();
|
|
92
|
+
|
|
93
|
+
if (Array.isArray(value)) {
|
|
94
|
+
return value.map((item) => {
|
|
95
|
+
if (item && typeof item === 'object' && !Array.isArray(item)) {
|
|
96
|
+
const nameKey = Object.keys(item).find((k) => titles.includes(k.toLowerCase())
|
|
97
|
+
&& typeof item[k] === 'string' && item[k].trim());
|
|
98
|
+
const rest = Object.entries(item)
|
|
99
|
+
.filter(([k]) => k !== nameKey)
|
|
100
|
+
.map(([, v]) => inlineValue(v))
|
|
101
|
+
.filter(Boolean)
|
|
102
|
+
.join(' — ');
|
|
103
|
+
const head = nameKey ? `**${item[nameKey].trim()}**` : '';
|
|
104
|
+
return `${indent}- ${head && rest ? `${head} — ${rest}` : head || rest}`;
|
|
105
|
+
}
|
|
106
|
+
return `${indent}- ${inlineValue(item)}`;
|
|
107
|
+
}).filter((line) => line.trim() !== '-').join('\n');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// An object: its values in order, WITHOUT their keys. A title opens in bold.
|
|
111
|
+
const blocks = [];
|
|
112
|
+
for (const [key, v] of Object.entries(value)) {
|
|
113
|
+
if (v === null || v === undefined || v === '') continue;
|
|
114
|
+
if (typeof v !== 'object') {
|
|
115
|
+
const text = String(v).trim();
|
|
116
|
+
if (!text) continue;
|
|
117
|
+
blocks.push(titles.includes(key.toLowerCase()) ? `**${text}**` : text);
|
|
118
|
+
} else {
|
|
119
|
+
const prose = jsonToProse(v, Array.isArray(v) ? depth : depth + 1);
|
|
120
|
+
if (prose) blocks.push(prose);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return blocks.join('\n\n');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return jsonToProse;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function makeJsonConverter(vocabulary) {
|
|
130
|
+
const jsonToProse = makeJsonToProse(vocabulary.titleKeys);
|
|
131
|
+
|
|
132
|
+
/* When the model wraps its answer ({"role": "...", "response": "..."}), the
|
|
133
|
+
payload field IS the answer; listing role/name alongside it is noise. */
|
|
134
|
+
function payloadOf(parsed) {
|
|
135
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null;
|
|
136
|
+
const key = vocabulary.payloadKeys.find((k) => typeof parsed[k] === 'string' && parsed[k].trim());
|
|
137
|
+
return key ? parsed[key] : null;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function render(parsed) {
|
|
141
|
+
return payloadOf(parsed) || jsonToProse(parsed, 0);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function convertRawJson(text) {
|
|
145
|
+
let trimmed = text.trim();
|
|
146
|
+
const fence = trimmed.match(/^```(?:json)?\s*\n([\s\S]*?)\n\s*```\s*$/);
|
|
147
|
+
if (fence) trimmed = fence[1].trim();
|
|
148
|
+
|
|
149
|
+
// The whole reply is JSON — only when it has at least one quoted key, so a
|
|
150
|
+
// reply that merely starts with "[" is left alone.
|
|
151
|
+
if ((trimmed.startsWith('{') || trimmed.startsWith('[')) && /"[\wÀ-ÿ_]+"\s*:/.test(trimmed)) {
|
|
152
|
+
const parsed = lenientParse(trimmed);
|
|
153
|
+
if (parsed) return render(parsed) || text;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// A short preamble, then a JSON block.
|
|
157
|
+
const jsonStart = trimmed.search(/\n\s*[{[]/);
|
|
158
|
+
if (jsonStart > 0 && jsonStart < 200) {
|
|
159
|
+
const preamble = trimmed.slice(0, jsonStart).trim();
|
|
160
|
+
const parsed = lenientParse(trimmed.slice(jsonStart).trim());
|
|
161
|
+
if (parsed) {
|
|
162
|
+
const body = render(parsed);
|
|
163
|
+
if (body) return (preamble ? `${preamble}\n\n` : '') + body;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Unparseable but clearly structured (3+ quoted keys): keep the values only.
|
|
168
|
+
// Keys are erased here too — this path used to be the last place they leaked.
|
|
169
|
+
const quotedKeys = (trimmed.match(/"[\wÀ-ÿ_]+"\s*:/g) || []).length;
|
|
170
|
+
if (quotedKeys >= 3 && /^\s*\{/.test(trimmed)) {
|
|
171
|
+
return trimmed
|
|
172
|
+
.replace(/^\s*\{/, '').replace(/\}\s*$/, '')
|
|
173
|
+
.replace(/"([\wÀ-ÿ_ ]+)"\s*:\s*/g, '')
|
|
174
|
+
.replace(/[[\]{}]/g, '')
|
|
175
|
+
.replace(/",?\s*/g, '\n')
|
|
176
|
+
.replace(/"/g, '')
|
|
177
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
178
|
+
.trim();
|
|
179
|
+
}
|
|
180
|
+
return text;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/* JSON fragments pasted in the middle of prose (a tool result copied back). */
|
|
184
|
+
function convertInlineJson(text) {
|
|
185
|
+
return text.replace(
|
|
186
|
+
/\{(?:\s*"[\wÀ-ÿ_]+"\s*:\s*(?:"[^"]*"|[\d.]+|true|false|null|\[.*?\]|\{[^}]*\})\s*,?\s*){2,}\}/g,
|
|
187
|
+
(match) => {
|
|
188
|
+
const parsed = lenientParse(match);
|
|
189
|
+
if (!parsed) return match;
|
|
190
|
+
return render(parsed) || match;
|
|
191
|
+
}
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return { convertRawJson, convertInlineJson, jsonToProse };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function compile(vocabulary) {
|
|
199
|
+
const v = vocabulary;
|
|
200
|
+
const any = (list) => list.length > 0;
|
|
201
|
+
const rx = {};
|
|
202
|
+
|
|
203
|
+
if (any(v.payloadKeys)) {
|
|
204
|
+
rx.payloadLabel = new RegExp(`^\\s*(?:\\*\\*)?(?:${alternation(v.payloadKeys)})(?:\\*\\*)?\\s*[:\\-—]\\s*`, 'i');
|
|
205
|
+
}
|
|
206
|
+
if (any(v.inlineLabels)) {
|
|
207
|
+
rx.inlineLabels = new RegExp(`(?:^|\\s)(?:\\*\\*)?(?:${alternation(v.inlineLabels)})(?:\\*\\*)?\\s*[:\\-—]\\s*`, 'gi');
|
|
208
|
+
}
|
|
209
|
+
if (any(v.lineLabels)) {
|
|
210
|
+
// Replaced by a paragraph break: the content stays, split as it was.
|
|
211
|
+
rx.lineLabels = new RegExp(`(^|\\n)[ \\t]*(?:\\*\\*)?(?:${alternation(v.lineLabels)})(?:\\*\\*)?[ \\t]*:[ \\t]*`, 'gi');
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/* Drafts chain corrections with no fixed marker ("Wait, ...", "Actually, ...",
|
|
215
|
+
"Final answer:"). Patching word by word never ends, so the LAST line that
|
|
216
|
+
looks like reasoning is found and only what follows it is kept: the model
|
|
217
|
+
writes its last meta comment right before the real answer. */
|
|
218
|
+
const boundary = [];
|
|
219
|
+
if (any(v.finalMarkers)) {
|
|
220
|
+
// The answer often follows on the same line after the colon: consume up to ':' only.
|
|
221
|
+
boundary.push(`^[ \\t]*(?:${alternation(v.finalMarkers)})\\s*(?:\\([^)]*\\))?\\s*[:\\-—]`);
|
|
222
|
+
}
|
|
223
|
+
const labelBlock = any(v.reasoningLabels)
|
|
224
|
+
? `^[ \\t]*\\(?(?:${alternation(v.reasoningLabels)})(?:[\\s-]+\\w+){0,6}(?:\\s*\\([^)]*\\))?\\)?\\s*[:\\-—]`
|
|
225
|
+
: null;
|
|
226
|
+
if (labelBlock) boundary.push(`${labelBlock}.*$`);
|
|
227
|
+
if (any(v.reasoningStarters)) boundary.push(`^[ \\t]*(?:${alternation(v.reasoningStarters)})\\b.*$`);
|
|
228
|
+
if (boundary.length) rx.boundary = new RegExp(boundary.join('|'), 'gim');
|
|
229
|
+
// Safety net: a labelled block still left (label + paragraph up to a blank line).
|
|
230
|
+
if (labelBlock) rx.labelledBlock = new RegExp(`${labelBlock}.*(?:\\n(?!\\s*\\n).*)*`, 'gim');
|
|
231
|
+
|
|
232
|
+
if (any(v.planningStarters)) {
|
|
233
|
+
rx.planning = new RegExp(`^[\\s]*[*\\-•]\\s*(?:${alternation(v.planningStarters)}).*$`, 'gim');
|
|
234
|
+
}
|
|
235
|
+
if (any(v.leadingFillers)) {
|
|
236
|
+
// Punctuation is required after the filler: "Bon" must not eat "Bonjour".
|
|
237
|
+
rx.filler = new RegExp(`^(?:${alternation(v.leadingFillers)})\\s*[.,!]+\\s*`, 'i');
|
|
238
|
+
}
|
|
239
|
+
if (any(v.annotationMarkers)) {
|
|
240
|
+
rx.annotation = new RegExp(`^.*\\((?:${alternation(v.annotationMarkers)}).*\\).*$`, 'gim');
|
|
241
|
+
}
|
|
242
|
+
if (any(v.headingLabels)) {
|
|
243
|
+
/* Bold may wrap the label, and the colon is absent when the label holds its
|
|
244
|
+
own line: an expression requiring "^label:" saw neither case. */
|
|
245
|
+
const h = alternation(v.headingLabels);
|
|
246
|
+
rx.headingAlone = new RegExp(`^\\s*(?:\\*\\*)?\\s*(?:${h})\\s*(?:\\*\\*)?\\s*:?\\s*(?:\\*\\*)?\\s*$`, 'gim');
|
|
247
|
+
rx.headingLead = new RegExp(`^\\s*(?:\\*\\*)?\\s*(?:${h})\\s*(?:\\*\\*)?\\s*:\\s*`, 'gim');
|
|
248
|
+
}
|
|
249
|
+
if (any(v.closingPhrases)) {
|
|
250
|
+
rx.closing = new RegExp(`\\s*(?:${alternation(v.closingPhrases)})\\s*$`, 'i');
|
|
251
|
+
}
|
|
252
|
+
if (any(v.openers)) {
|
|
253
|
+
rx.openers = new RegExp(`(?:^|\\n)\\s*(?:${alternation(v.openers)})\\s*[.!?]?\\s*`, 'gi');
|
|
254
|
+
}
|
|
255
|
+
return rx;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* @param {object} [options]
|
|
260
|
+
* @param {object} [options.shared] vocabulary applied whatever the language
|
|
261
|
+
* (JSON keys are usually English whatever language the user speaks).
|
|
262
|
+
* @param {Record<string, object>} [options.languages] vocabulary per language.
|
|
263
|
+
* @param {string} [options.fallbackLanguage] used when clean() gets an unknown language.
|
|
264
|
+
* @param {number} [options.maxClosingPasses=3] successive robotic closings removed.
|
|
265
|
+
*/
|
|
266
|
+
function createResponseCleaner(options = {}) {
|
|
267
|
+
const shared = options.shared || {};
|
|
268
|
+
const languages = options.languages || {};
|
|
269
|
+
const fallbackLanguage = options.fallbackLanguage || null;
|
|
270
|
+
const maxClosingPasses = options.maxClosingPasses ?? 3;
|
|
271
|
+
const cache = new Map();
|
|
272
|
+
|
|
273
|
+
function forLanguage(language) {
|
|
274
|
+
const lang = language && languages[language] ? language : fallbackLanguage;
|
|
275
|
+
const key = lang || '';
|
|
276
|
+
if (!cache.has(key)) {
|
|
277
|
+
const vocabulary = mergeVocabularies([shared, lang ? languages[lang] : null]);
|
|
278
|
+
cache.set(key, { vocabulary, rx: compile(vocabulary), json: makeJsonConverter(vocabulary) });
|
|
279
|
+
}
|
|
280
|
+
return cache.get(key);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function clean(text, cleanOptions = {}) {
|
|
284
|
+
if (text === null || text === undefined) return '';
|
|
285
|
+
const { rx, json } = forLanguage(cleanOptions.language);
|
|
286
|
+
let out = String(text);
|
|
287
|
+
|
|
288
|
+
// Reasoning blocks, closed or cut off mid-stream.
|
|
289
|
+
out = out.replace(/<think>[\s\S]*?<\/think>/gi, '');
|
|
290
|
+
out = out.replace(/<think>[\s\S]*/gi, '');
|
|
291
|
+
|
|
292
|
+
out = json.convertRawJson(out);
|
|
293
|
+
if (rx.payloadLabel) out = out.replace(rx.payloadLabel, '');
|
|
294
|
+
if (rx.inlineLabels) out = out.replace(rx.inlineLabels, ' ');
|
|
295
|
+
if (rx.lineLabels) out = out.replace(rx.lineLabels, '$1\n');
|
|
296
|
+
out = json.convertInlineJson(out);
|
|
297
|
+
|
|
298
|
+
if (rx.boundary) {
|
|
299
|
+
const matches = [...out.matchAll(rx.boundary)];
|
|
300
|
+
if (matches.length) {
|
|
301
|
+
const last = matches[matches.length - 1];
|
|
302
|
+
out = out.slice(last.index + last[0].length);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
if (rx.labelledBlock) out = out.replace(rx.labelledBlock, '');
|
|
306
|
+
if (rx.planning) out = out.replace(rx.planning, '');
|
|
307
|
+
if (rx.filler) out = out.replace(rx.filler, '');
|
|
308
|
+
if (rx.annotation) out = out.replace(rx.annotation, '');
|
|
309
|
+
if (rx.headingAlone) out = out.replace(rx.headingAlone, '');
|
|
310
|
+
if (rx.headingLead) out = out.replace(rx.headingLead, '');
|
|
311
|
+
|
|
312
|
+
/* Closings stack ("Need anything else? Don't hesitate to ask."): one pass
|
|
313
|
+
left the first one standing. Bounded, so a reply made only of phrases
|
|
314
|
+
that look robotic cannot be eaten whole by a loop. */
|
|
315
|
+
if (rx.closing) {
|
|
316
|
+
for (let i = 0; i < maxClosingPasses; i += 1) {
|
|
317
|
+
const next = out.replace(rx.closing, '');
|
|
318
|
+
if (next === out) break;
|
|
319
|
+
out = next;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (rx.openers) out = out.replace(rx.openers, '\n');
|
|
323
|
+
|
|
324
|
+
out = out.replace(/\n{3,}/g, '\n\n');
|
|
325
|
+
out = out.replace(/[ \t]{2,}/g, ' ');
|
|
326
|
+
return out.trim();
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
return {
|
|
330
|
+
clean,
|
|
331
|
+
/** Converts a parsed JSON value to prose without any key (for tool results shown as text). */
|
|
332
|
+
jsonToProse: (value, language) => forLanguage(language).json.jsonToProse(value, 0),
|
|
333
|
+
languages: Object.keys(languages)
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
module.exports = { createResponseCleaner };
|