@seanhogg/builderforce-memory 2026.9.30 → 2026.9.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridges/FetchBridge.d.ts +12 -5
- package/dist/bridges/FetchBridge.d.ts.map +1 -1
- package/dist/bridges/FetchBridge.js +6 -4
- package/dist/bridges/FetchBridge.js.map +1 -1
- package/dist/bridges/OpenAIBridge.d.ts +17 -4
- package/dist/bridges/OpenAIBridge.d.ts.map +1 -1
- package/dist/bridges/OpenAIBridge.js +68 -57
- package/dist/bridges/OpenAIBridge.js.map +1 -1
- package/dist/bridges/TransformerBridge.d.ts +6 -0
- package/dist/bridges/TransformerBridge.d.ts.map +1 -1
- package/dist/evermind/index.d.ts +6 -0
- package/dist/evermind/index.d.ts.map +1 -1
- package/dist/evermind/index.js +5 -0
- package/dist/evermind/index.js.map +1 -1
- package/dist/evermind/runtime.d.ts +158 -0
- package/dist/evermind/runtime.d.ts.map +1 -0
- package/dist/evermind/runtime.js +236 -0
- package/dist/evermind/runtime.js.map +1 -0
- package/dist/evermind/textCoherence.d.ts +55 -0
- package/dist/evermind/textCoherence.d.ts.map +1 -0
- package/dist/evermind/textCoherence.js +218 -0
- package/dist/evermind/textCoherence.js.map +1 -0
- package/dist/evermind/toolCall.d.ts +175 -0
- package/dist/evermind/toolCall.d.ts.map +1 -0
- package/dist/evermind/toolCall.js +398 -0
- package/dist/evermind/toolCall.js.map +1 -0
- package/dist/evermind/wordLexicon.d.ts +103 -0
- package/dist/evermind/wordLexicon.d.ts.map +1 -0
- package/dist/evermind/wordLexicon.js +361 -0
- package/dist/evermind/wordLexicon.js.map +1 -0
- package/dist/wire/chatClient.d.ts +100 -0
- package/dist/wire/chatClient.d.ts.map +1 -0
- package/dist/wire/chatClient.js +123 -0
- package/dist/wire/chatClient.js.map +1 -0
- package/dist/wire/index.d.ts +3 -1
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -1
- package/dist/wire/index.js.map +1 -1
- package/dist/wire/sse.d.ts +17 -9
- package/dist/wire/sse.d.ts.map +1 -1
- package/dist/wire/sse.js +45 -22
- package/dist/wire/sse.js.map +1 -1
- package/package.json +3 -3
- package/src/bridges/FetchBridge.ts +15 -6
- package/src/bridges/OpenAIBridge.ts +82 -72
- package/src/bridges/TransformerBridge.ts +6 -0
- package/src/evermind/index.ts +48 -0
- package/src/evermind/runtime.ts +364 -0
- package/src/evermind/textCoherence.ts +248 -0
- package/src/evermind/toolCall.ts +523 -0
- package/src/evermind/wordLexicon.ts +395 -0
- package/src/wire/chatClient.ts +196 -0
- package/src/wire/index.ts +3 -1
- package/src/wire/sse.ts +43 -21
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-driving half of Evermind serving: flatten a chat into a continuation
|
|
3
|
+
* prompt, generate under a wall-clock budget, grade a head's fitness to serve, and
|
|
4
|
+
* bind a head to the tool planner's decoder port.
|
|
5
|
+
*
|
|
6
|
+
* Written against STRUCTURAL model/tokenizer shapes ({@link EvermindTextModel},
|
|
7
|
+
* {@link EvermindTextTokenizer}) that the engine's `EvermindLM` and `BPETokenizer`
|
|
8
|
+
* satisfy, so this module imports nothing and the `/evermind` subpath stays
|
|
9
|
+
* dependency-free. Where the head comes FROM (an R2 artifact, a file, a freshly merged
|
|
10
|
+
* in-memory model) is the host's business; everything here takes a loaded head.
|
|
11
|
+
*/
|
|
12
|
+
import { isServableText } from './textCoherence.js';
|
|
13
|
+
import { planEvermindToolCall, } from './toolCall.js';
|
|
14
|
+
/** Flatten chat messages into a single continuation prompt for the LM. */
|
|
15
|
+
export function messagesToPrompt(messages) {
|
|
16
|
+
const lines = messages
|
|
17
|
+
.map((m) => {
|
|
18
|
+
const role = typeof m.role === 'string' ? m.role : 'user';
|
|
19
|
+
const content = typeof m.content === 'string' ? m.content : '';
|
|
20
|
+
return content ? `${role}: ${content}` : '';
|
|
21
|
+
})
|
|
22
|
+
.filter(Boolean);
|
|
23
|
+
return `${lines.join('\n')}\nassistant:`;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Tokens generated between wall-clock checks. The deadline is enforced BETWEEN slices
|
|
27
|
+
* because the engine's generate is one synchronous loop with no per-token hook.
|
|
28
|
+
* Slicing costs nothing: the forward pass already recomputes the whole sequence per
|
|
29
|
+
* token, so re-entering with `prompt + producedSoFar` is the work the next token was
|
|
30
|
+
* going to do anyway. Small enough that the overshoot is one slice.
|
|
31
|
+
*/
|
|
32
|
+
const GENERATE_SLICE_TOKENS = 16;
|
|
33
|
+
/** Default wall-clock budget for one generation — inside a Worker's CPU allowance, so
|
|
34
|
+
* the caller gets a partial answer plus `truncated` rather than a killed request. */
|
|
35
|
+
const DEFAULT_GENERATE_DEADLINE_MS = 8000;
|
|
36
|
+
/**
|
|
37
|
+
* Generate a reply from a loaded head. Deterministic for a given (prompt, seed,
|
|
38
|
+
* maxTokens): each slice derives its seed from the base seed and its index, so two
|
|
39
|
+
* operators running the same probe see the same text.
|
|
40
|
+
*/
|
|
41
|
+
export function generateEvermindText(lm, tok, messages, opts = {}) {
|
|
42
|
+
const prompt = messagesToPrompt(messages);
|
|
43
|
+
const maxTokens = opts.maxTokens ?? 256;
|
|
44
|
+
const temperature = opts.temperature ?? 0.7;
|
|
45
|
+
const baseSeed = opts.seed;
|
|
46
|
+
const deadlineMs = opts.deadlineMs ?? DEFAULT_GENERATE_DEADLINE_MS;
|
|
47
|
+
const started = Date.now();
|
|
48
|
+
let content = '';
|
|
49
|
+
let truncated = false;
|
|
50
|
+
let produced = 0;
|
|
51
|
+
let slice = 0;
|
|
52
|
+
while (produced < maxTokens) {
|
|
53
|
+
const want = Math.min(GENERATE_SLICE_TOKENS, maxTokens - produced);
|
|
54
|
+
const chunk = lm.generateText(`${prompt}${content}`, tok, {
|
|
55
|
+
maxNewTokens: want,
|
|
56
|
+
temperature,
|
|
57
|
+
...(baseSeed != null ? { seed: baseSeed + slice } : {}),
|
|
58
|
+
});
|
|
59
|
+
slice++;
|
|
60
|
+
// An empty slice means the model stopped producing; continuing would spin.
|
|
61
|
+
if (!chunk)
|
|
62
|
+
break;
|
|
63
|
+
content += chunk;
|
|
64
|
+
produced += want;
|
|
65
|
+
if (Date.now() - started >= deadlineMs) {
|
|
66
|
+
truncated = produced < maxTokens;
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const prompt_tokens = tok.encode(prompt).length;
|
|
71
|
+
const completion_tokens = content ? tok.encode(content).length : 0;
|
|
72
|
+
return {
|
|
73
|
+
content,
|
|
74
|
+
usage: { prompt_tokens, completion_tokens, total_tokens: prompt_tokens + completion_tokens },
|
|
75
|
+
truncated,
|
|
76
|
+
elapsedMs: Date.now() - started,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
// ── Fitness to serve ────────────────────────────────────────────────────────
|
|
80
|
+
/** Neutral probe prompts a project chat head should answer coherently. Fixed and
|
|
81
|
+
* generic (not project-specific) so the probe measures GENERATION QUALITY, not
|
|
82
|
+
* recall. */
|
|
83
|
+
export const COHERENCE_PROBE_PROMPTS = [
|
|
84
|
+
'Summarize the current status of the project.',
|
|
85
|
+
'What has the team been working on recently?',
|
|
86
|
+
'List the main things left to do.',
|
|
87
|
+
];
|
|
88
|
+
/**
|
|
89
|
+
* Score a loaded head's fitness to serve chat: generate from the neutral probe
|
|
90
|
+
* prompts and grade each with the serve-time bar ({@link isServableText}).
|
|
91
|
+
* Synchronous and storage-free, so the promote-to-inference gate and a learning
|
|
92
|
+
* coordinator holding a freshly merged model grade it the same way. Deterministic
|
|
93
|
+
* seeds keep the verdict reproducible; a majority must pass by default.
|
|
94
|
+
*/
|
|
95
|
+
export function assessLMCoherence(lm, tok, opts = {}) {
|
|
96
|
+
const samples = COHERENCE_PROBE_PROMPTS.map((prompt, i) => {
|
|
97
|
+
const text = lm.generateText(messagesToPrompt([{ role: 'user', content: prompt }]), tok, {
|
|
98
|
+
maxNewTokens: 80,
|
|
99
|
+
temperature: 0.7,
|
|
100
|
+
seed: 1234 + i,
|
|
101
|
+
});
|
|
102
|
+
const verdict = isServableText(text, { context: prompt });
|
|
103
|
+
return { prompt, text, coherent: verdict.coherent, failure: verdict.failure, detail: verdict.detail };
|
|
104
|
+
});
|
|
105
|
+
const passRate = samples.length ? samples.filter((s) => s.coherent).length / samples.length : 0;
|
|
106
|
+
return { ready: passRate >= (opts.minPassRate ?? 0.5), passRate, samples };
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Test bench: run ONE prompt through a head and grade the output with the SAME bar
|
|
110
|
+
* the serve path uses, so "what will this model actually produce?" is answerable
|
|
111
|
+
* before inference is switched on. Deterministic by default (`seed`).
|
|
112
|
+
*/
|
|
113
|
+
export function probeEvermindText(lm, tok, prompt, opts = {}) {
|
|
114
|
+
const gen = generateEvermindText(lm, tok, [{ role: 'user', content: prompt }], {
|
|
115
|
+
maxTokens: opts.maxTokens ?? 120,
|
|
116
|
+
temperature: opts.temperature ?? 0.7,
|
|
117
|
+
seed: opts.seed ?? 1234,
|
|
118
|
+
...(opts.deadlineMs != null ? { deadlineMs: opts.deadlineMs } : {}),
|
|
119
|
+
});
|
|
120
|
+
const verdict = isServableText(gen.content, { context: prompt });
|
|
121
|
+
return {
|
|
122
|
+
prompt,
|
|
123
|
+
text: gen.content,
|
|
124
|
+
coherent: verdict.coherent,
|
|
125
|
+
failure: verdict.failure,
|
|
126
|
+
detail: verdict.detail,
|
|
127
|
+
usage: gen.usage,
|
|
128
|
+
truncated: gen.truncated,
|
|
129
|
+
elapsedMs: gen.elapsedMs,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
// ── Tool calling ─────────────────────────────────────────────────────────────
|
|
133
|
+
/**
|
|
134
|
+
* A tool-decision prompt is rebuilt for every candidate and every argument, and each
|
|
135
|
+
* costs a full forward pass, so the conversation is capped rather than replayed whole.
|
|
136
|
+
* Truncated from the LEFT: the most recent turns are what a tool choice depends on.
|
|
137
|
+
*/
|
|
138
|
+
const MAX_TOOL_PROMPT_CHARS = 6000;
|
|
139
|
+
/** Log-probability the model assigned to `id` at a position, from that position's raw
|
|
140
|
+
* logits (log-softmax, max-shifted so long-tail logits don't overflow). */
|
|
141
|
+
function logProbOf(row, id) {
|
|
142
|
+
let max = -Infinity;
|
|
143
|
+
for (let i = 0; i < row.length; i++) {
|
|
144
|
+
const v = row[i];
|
|
145
|
+
if (v > max)
|
|
146
|
+
max = v;
|
|
147
|
+
}
|
|
148
|
+
if (!Number.isFinite(max))
|
|
149
|
+
return -Infinity;
|
|
150
|
+
let sum = 0;
|
|
151
|
+
for (let i = 0; i < row.length; i++)
|
|
152
|
+
sum += Math.exp(row[i] - max);
|
|
153
|
+
const logit = id >= 0 && id < row.length ? row[id] : -Infinity;
|
|
154
|
+
return logit - (max + Math.log(sum));
|
|
155
|
+
}
|
|
156
|
+
/** Keep the tail of an over-long prompt (see {@link MAX_TOOL_PROMPT_CHARS}). */
|
|
157
|
+
function clampPromptText(prompt) {
|
|
158
|
+
return prompt.length <= MAX_TOOL_PROMPT_CHARS ? prompt : prompt.slice(prompt.length - MAX_TOOL_PROMPT_CHARS);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Bind a loaded head to the {@link EvermindToolDecoder} port.
|
|
162
|
+
*
|
|
163
|
+
* `score`: teacher forcing lets one forward pass over `prompt + continuation` yield
|
|
164
|
+
* the log-prob of EVERY continuation token at once (position `t` predicts `t+1`), so
|
|
165
|
+
* ranking a candidate costs one pass. The mean is returned — not the sum — so a long
|
|
166
|
+
* tool name is not out-voted by a short one for its length alone.
|
|
167
|
+
*
|
|
168
|
+
* `generate` is greedy by default: a tool ARGUMENT is a value to get right, not prose
|
|
169
|
+
* to vary, and determinism keeps a replayed run reproducible.
|
|
170
|
+
*/
|
|
171
|
+
export function createEvermindToolDecoder(lm, tok, opts = {}) {
|
|
172
|
+
let promptTokens = 0;
|
|
173
|
+
let completionTokens = 0;
|
|
174
|
+
return {
|
|
175
|
+
score(prompt, continuation) {
|
|
176
|
+
const contIds = tok.encode(continuation);
|
|
177
|
+
if (contIds.length === 0)
|
|
178
|
+
return -Infinity;
|
|
179
|
+
// A leading token is required for the first continuation token to have a
|
|
180
|
+
// position to be predicted FROM; an empty prompt gets the same id-0 prefix the
|
|
181
|
+
// engine's own sampler uses.
|
|
182
|
+
const promptIds = tok.encode(clampPromptText(prompt));
|
|
183
|
+
const prefix = promptIds.length > 0 ? promptIds : [0];
|
|
184
|
+
const { logits } = lm.forward([...prefix, ...contIds]);
|
|
185
|
+
let total = 0;
|
|
186
|
+
for (let i = 0; i < contIds.length; i++) {
|
|
187
|
+
total += logProbOf(logits[prefix.length + i - 1], contIds[i]);
|
|
188
|
+
}
|
|
189
|
+
promptTokens += prefix.length;
|
|
190
|
+
completionTokens += contIds.length;
|
|
191
|
+
return total / contIds.length;
|
|
192
|
+
},
|
|
193
|
+
generate(prompt, maxTokens) {
|
|
194
|
+
const text = lm.generateText(clampPromptText(prompt), tok, {
|
|
195
|
+
maxNewTokens: maxTokens,
|
|
196
|
+
temperature: opts.temperature ?? 0,
|
|
197
|
+
...(opts.seed != null ? { seed: opts.seed } : {}),
|
|
198
|
+
});
|
|
199
|
+
promptTokens += tok.encode(clampPromptText(prompt)).length;
|
|
200
|
+
completionTokens += text ? tok.encode(text).length : 0;
|
|
201
|
+
return text;
|
|
202
|
+
},
|
|
203
|
+
usage: () => ({ prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens }),
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Run a TOOL-BEARING request against a loaded head: plan a call by constrained
|
|
208
|
+
* decoding, or fall through to ordinary prose when the head elected to answer directly.
|
|
209
|
+
*/
|
|
210
|
+
export function generateEvermindWithTools(lm, tok, messages, tools, choice, opts = {}) {
|
|
211
|
+
const decoder = createEvermindToolDecoder(lm, tok, {
|
|
212
|
+
...(opts.temperature != null ? { temperature: opts.temperature } : {}),
|
|
213
|
+
...(opts.seed != null ? { seed: opts.seed } : {}),
|
|
214
|
+
});
|
|
215
|
+
// The planner is given the conversation WITHOUT the `assistant:` primer — it is
|
|
216
|
+
// choosing an action, not continuing a reply.
|
|
217
|
+
const conversation = messagesToPrompt(messages).replace(/\nassistant:$/, '');
|
|
218
|
+
const plan = planEvermindToolCall(decoder, conversation, tools, choice);
|
|
219
|
+
if (plan.call) {
|
|
220
|
+
return { call: plan.call, calls: plan.calls, content: '', usage: decoder.usage(), margin: plan.margin };
|
|
221
|
+
}
|
|
222
|
+
const gen = generateEvermindText(lm, tok, messages, opts);
|
|
223
|
+
const usage = decoder.usage();
|
|
224
|
+
return {
|
|
225
|
+
call: null,
|
|
226
|
+
calls: [],
|
|
227
|
+
content: gen.content,
|
|
228
|
+
usage: {
|
|
229
|
+
prompt_tokens: usage.prompt_tokens + gen.usage.prompt_tokens,
|
|
230
|
+
completion_tokens: usage.completion_tokens + gen.usage.completion_tokens,
|
|
231
|
+
total_tokens: usage.total_tokens + gen.usage.total_tokens,
|
|
232
|
+
},
|
|
233
|
+
margin: plan.margin,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
//# sourceMappingURL=runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/evermind/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,cAAc,EAAyB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EACL,oBAAoB,GAKrB,MAAM,eAAe,CAAC;AA0CvB,0EAA0E;AAC1E,MAAM,UAAU,gBAAgB,CAAC,QAAsD;IACrF,MAAM,KAAK,GAAG,QAAQ;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC;sFACsF;AACtF,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAAqB,EACrB,GAA0B,EAC1B,QAAsD,EACtD,OAAgC,EAAE;IAElC,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,4BAA4B,CAAC;IAEnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,QAAQ,GAAG,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,SAAS,GAAG,QAAQ,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,MAAM,GAAG,OAAO,EAAE,EAAE,GAAG,EAAE;YACxD,YAAY,EAAE,IAAI;YAClB,WAAW;YACX,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC,CAAC;QACH,KAAK,EAAE,CAAC;QACR,2EAA2E;QAC3E,IAAI,CAAC,KAAK;YAAE,MAAM;QAClB,OAAO,IAAI,KAAK,CAAC;QACjB,QAAQ,IAAI,IAAI,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC;YACvC,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAC;YACjC,MAAM;QACR,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChD,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,OAAO;QACL,OAAO;QACP,KAAK,EAAE,EAAE,aAAa,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,GAAG,iBAAiB,EAAE;QAC5F,SAAS;QACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KAChC,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;cAEc;AACd,MAAM,CAAC,MAAM,uBAAuB,GAAsB;IACxD,8CAA8C;IAC9C,6CAA6C;IAC7C,kCAAkC;CACnC,CAAC;AAsBF;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,GAA0B,EAC1B,OAAiC,EAAE;IAEnC,MAAM,OAAO,GAA8B,uBAAuB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACnF,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE;YACvF,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,GAAG;YAChB,IAAI,EAAE,IAAI,GAAG,CAAC;SACf,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACxG,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAChG,OAAO,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC7E,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,GAA0B,EAC1B,MAAc,EACd,OAAgC,EAAE;IAElC,MAAM,GAAG,GAAG,oBAAoB,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE;QAC7E,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,GAAG;QAChC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;QACpC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;QACvB,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,OAAO;QACL,MAAM;QACN,IAAI,EAAE,GAAG,CAAC,OAAO;QACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;KACzB,CAAC;AACJ,CAAC;AAED,gFAAgF;AAEhF;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAEnC;4EAC4E;AAC5E,SAAS,SAAS,CAAC,GAAsB,EAAE,EAAU;IACnD,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;QAAC,IAAI,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,CAAC,CAAC;IAAC,CAAC;IACjF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,QAAQ,CAAC;IAC5C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChE,OAAO,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,CAAC,MAAM,IAAI,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC;AAC/G,CAAC;AAQD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,yBAAyB,CACvC,EAAqB,EACrB,GAA0B,EAC1B,OAAgD,EAAE;IAElD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,OAAO;QACL,KAAK,CAAC,MAAc,EAAE,YAAoB;YACxC,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,CAAC,QAAQ,CAAC;YAC3C,yEAAyE;YACzE,+EAA+E;YAC/E,6BAA6B;YAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;YACtD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;YACvD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,KAAK,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAE,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YAC9B,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;YACnC,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAChC,CAAC;QACD,QAAQ,CAAC,MAAc,EAAE,SAAiB;YACxC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE;gBACzD,YAAY,EAAE,SAAS;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAClD,CAAC,CAAC;YACH,YAAY,IAAI,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3D,gBAAgB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,YAAY,EAAE,YAAY,GAAG,gBAAgB,EAAE,CAAC;KACnI,CAAC;AACJ,CAAC;AAkBD;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CACvC,EAAqB,EACrB,GAA0B,EAC1B,QAAsD,EACtD,KAAuB,EACvB,MAAsB,EACtB,OAAgC,EAAE;IAElC,MAAM,OAAO,GAAG,yBAAyB,CAAC,EAAE,EAAE,GAAG,EAAE;QACjD,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;IACH,gFAAgF;IAChF,8CAA8C;IAC9C,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACxE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC1G,CAAC;IACD,MAAM,GAAG,GAAG,oBAAoB,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAC9B,OAAO;QACL,IAAI,EAAE,IAAI;QACV,KAAK,EAAE,EAAE;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,KAAK,EAAE;YACL,aAAa,EAAE,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa;YAC5D,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,GAAG,GAAG,CAAC,KAAK,CAAC,iBAAiB;YACxE,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY;SAC1D;QACD,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* textCoherence — pure, zero-dependency text-quality primitives shared by every
|
|
3
|
+
* surface that must decide whether text a project's OWN SSM (Evermind) produced is
|
|
4
|
+
* fit to show a user. Kept dependency-free (no db/env/engine imports) so it can be
|
|
5
|
+
* imported by the memory-first resolver, the model probe and the gateway vendor —
|
|
6
|
+
* and by a browser client — without an import cycle.
|
|
7
|
+
*/
|
|
8
|
+
/** A reply shorter than this isn't a real answer — fall through to the LLM. Mirrors
|
|
9
|
+
* the cloud/BrainService threshold so every surface adopts Evermind identically. */
|
|
10
|
+
export declare const EVERMIND_ANSWER_MIN_CHARS = 20;
|
|
11
|
+
/** Why the gate rejected a text. Surfaced to operators (console test bench, quarantine
|
|
12
|
+
* reason, vendor error) so "it produced gibberish" is never an unexplained verdict. */
|
|
13
|
+
export type CoherenceFailure = 'empty' | 'replacement-chars' | 'repetition' | 'dominant-token' | 'unbalanced-delimiters' | 'no-function-words' | 'non-words';
|
|
14
|
+
/** The gate's verdict + the specific signal that produced it. */
|
|
15
|
+
export interface CoherenceVerdict {
|
|
16
|
+
coherent: boolean;
|
|
17
|
+
/** The failing signal, or null when the text passed. */
|
|
18
|
+
failure: CoherenceFailure | null;
|
|
19
|
+
/** Short operator-facing explanation of {@link failure} (empty when coherent). */
|
|
20
|
+
detail: string;
|
|
21
|
+
}
|
|
22
|
+
export interface CoherenceOptions {
|
|
23
|
+
/**
|
|
24
|
+
* The prompt/question the text answers, when the caller has it. Domain jargon
|
|
25
|
+
* ("Evermind", "neocortex", a repo name) reliably echoes the ask, so treating
|
|
26
|
+
* context words as known is what keeps the non-word check from mis-accusing a
|
|
27
|
+
* legitimate, jargon-dense answer. Optional — the gate works without it.
|
|
28
|
+
*/
|
|
29
|
+
context?: string;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Cheap coherence gate for text a project's OWN SSM (Evermind) produced. An
|
|
33
|
+
* under-trained head emits fluent-LOOKING garbage — broken UTF-8 (Unicode
|
|
34
|
+
* replacement chars from byte-level BPE), degenerate word repetition ("commit …
|
|
35
|
+
* commit … commit", "in the in the"), and fluent-shaped prose made entirely of
|
|
36
|
+
* INVENTED words — that trivially clears a length check. Serving it makes the
|
|
37
|
+
* assistant reply in gibberish, so this gate rejects it and the caller falls through
|
|
38
|
+
* to a real LLM (a garbled reply IS a memory miss).
|
|
39
|
+
*
|
|
40
|
+
* Signals 1–4 are structural and language-agnostic. Signals 5–6 consult a lexicon and
|
|
41
|
+
* are therefore GUARDED: they run only on predominantly-Latin, non-code text, and the
|
|
42
|
+
* dictionary check only when the text is confidently English — so an es/fr/de/pt/it
|
|
43
|
+
* reply, a CJK reply, or a code block is never judged by an English word list.
|
|
44
|
+
*/
|
|
45
|
+
export declare function assessTextCoherence(text: string, opts?: CoherenceOptions): CoherenceVerdict;
|
|
46
|
+
/**
|
|
47
|
+
* Boolean form of {@link assessTextCoherence} — the shape every existing caller uses.
|
|
48
|
+
* Prefer the assessment when you have somewhere to SHOW the reason.
|
|
49
|
+
*/
|
|
50
|
+
export declare function looksLikeCoherentText(text: string, opts?: CoherenceOptions): boolean;
|
|
51
|
+
/** Is this text both substantive (long enough) AND coherent — the ONE bar every serve
|
|
52
|
+
* path applies before showing SSM output to a user. Shared so the length threshold and
|
|
53
|
+
* the gate can never drift apart between the memory resolver, the vendor and the probe. */
|
|
54
|
+
export declare function isServableText(text: string | null | undefined, opts?: CoherenceOptions): CoherenceVerdict;
|
|
55
|
+
//# sourceMappingURL=textCoherence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"textCoherence.d.ts","sourceRoot":"","sources":["../../src/evermind/textCoherence.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH;qFACqF;AACrF,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAE5C;wFACwF;AACxF,MAAM,MAAM,gBAAgB,GACxB,OAAO,GACP,mBAAmB,GACnB,YAAY,GACZ,gBAAgB,GAChB,uBAAuB,GACvB,mBAAmB,GACnB,WAAW,CAAC;AAEhB,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,wDAAwD;IACxD,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,kFAAkF;IAClF,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAaD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,gBAAgB,CAyC/F;AA+DD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,OAAO,CAExF;AAED;;4FAE4F;AAC5F,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,IAAI,GAAE,gBAAqB,GAAG,gBAAgB,CAM7G"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* textCoherence — pure, zero-dependency text-quality primitives shared by every
|
|
3
|
+
* surface that must decide whether text a project's OWN SSM (Evermind) produced is
|
|
4
|
+
* fit to show a user. Kept dependency-free (no db/env/engine imports) so it can be
|
|
5
|
+
* imported by the memory-first resolver, the model probe and the gateway vendor —
|
|
6
|
+
* and by a browser client — without an import cycle.
|
|
7
|
+
*/
|
|
8
|
+
import { detectLatinLanguage, isCodeishToken, isFunctionWord, scoreEnglishWordiness, } from './wordLexicon.js';
|
|
9
|
+
/** A reply shorter than this isn't a real answer — fall through to the LLM. Mirrors
|
|
10
|
+
* the cloud/BrainService threshold so every surface adopts Evermind identically. */
|
|
11
|
+
export const EVERMIND_ANSWER_MIN_CHARS = 20;
|
|
12
|
+
/** Human-readable explanations, one per failure signal. */
|
|
13
|
+
const FAILURE_DETAIL = {
|
|
14
|
+
'empty': 'the model returned nothing',
|
|
15
|
+
'replacement-chars': 'the output contains Unicode replacement characters (broken token decoding)',
|
|
16
|
+
'repetition': 'the decoder is stuck repeating the same word or phrase',
|
|
17
|
+
'dominant-token': 'the output recycles a handful of words — the head has collapsed onto part of its vocabulary',
|
|
18
|
+
'unbalanced-delimiters': 'the output closes brackets it never opened',
|
|
19
|
+
'no-function-words': 'the output has no recognisable function words in any supported language',
|
|
20
|
+
'non-words': 'most of the content words are not real words — the head is emitting invented tokens',
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Cheap coherence gate for text a project's OWN SSM (Evermind) produced. An
|
|
24
|
+
* under-trained head emits fluent-LOOKING garbage — broken UTF-8 (Unicode
|
|
25
|
+
* replacement chars from byte-level BPE), degenerate word repetition ("commit …
|
|
26
|
+
* commit … commit", "in the in the"), and fluent-shaped prose made entirely of
|
|
27
|
+
* INVENTED words — that trivially clears a length check. Serving it makes the
|
|
28
|
+
* assistant reply in gibberish, so this gate rejects it and the caller falls through
|
|
29
|
+
* to a real LLM (a garbled reply IS a memory miss).
|
|
30
|
+
*
|
|
31
|
+
* Signals 1–4 are structural and language-agnostic. Signals 5–6 consult a lexicon and
|
|
32
|
+
* are therefore GUARDED: they run only on predominantly-Latin, non-code text, and the
|
|
33
|
+
* dictionary check only when the text is confidently English — so an es/fr/de/pt/it
|
|
34
|
+
* reply, a CJK reply, or a code block is never judged by an English word list.
|
|
35
|
+
*/
|
|
36
|
+
export function assessTextCoherence(text, opts = {}) {
|
|
37
|
+
const t = (text ?? '').trim();
|
|
38
|
+
if (!t)
|
|
39
|
+
return fail('empty');
|
|
40
|
+
// 1) Broken token decoding. A byte-level BPE emitting a low-probability token
|
|
41
|
+
// sequence frequently yields invalid UTF-8 → the replacement char. Real
|
|
42
|
+
// answers effectively never contain it → near-certain garbage.
|
|
43
|
+
if (t.includes('�'))
|
|
44
|
+
return fail('replacement-chars');
|
|
45
|
+
const words = t.toLowerCase().match(/\p{L}+/gu) ?? [];
|
|
46
|
+
// Too few alphabetic tokens to score structurally (CJK collapses to one run); it
|
|
47
|
+
// already cleared the length + replacement-char gates, so accept (a handful of
|
|
48
|
+
// clean words is a real answer).
|
|
49
|
+
if (words.length < 6)
|
|
50
|
+
return pass();
|
|
51
|
+
// 2) Adjacent runaway repetition: a stuck decoder repeats a token ("commit
|
|
52
|
+
// commit") or a bigram ("in the in the"). A little is normal prose.
|
|
53
|
+
let rep = 0;
|
|
54
|
+
for (let i = 1; i < words.length; i++)
|
|
55
|
+
if (words[i] === words[i - 1])
|
|
56
|
+
rep++;
|
|
57
|
+
for (let i = 3; i < words.length; i++) {
|
|
58
|
+
if (words[i] === words[i - 2] && words[i - 1] === words[i - 3])
|
|
59
|
+
rep++;
|
|
60
|
+
}
|
|
61
|
+
if (rep >= 3 && rep / words.length > 0.06)
|
|
62
|
+
return fail('repetition');
|
|
63
|
+
// 3) Content-word collapse: an under-trained head that overfit its corpus stops
|
|
64
|
+
// reaching for new words and recycles a handful ("update update the file update
|
|
65
|
+
// the file update"). Judged on CONTENT words only — grammar is supposed to repeat,
|
|
66
|
+
// and the length proxy this replaces (any token of 3+ letters) counted `the`,
|
|
67
|
+
// `and`, `was` and `not` as content, which is why its floors had to be set so high
|
|
68
|
+
// that they stopped meaning anything on a short reply.
|
|
69
|
+
if (collapsedVocabulary(words))
|
|
70
|
+
return fail('dominant-token');
|
|
71
|
+
// 4) Orphaned closing delimiters. The checks above key on REPETITION, so they miss
|
|
72
|
+
// the other failure mode of an under-trained byte-BPE head: fluent-shaped text made
|
|
73
|
+
// of invented words, each different. What that text does emit is punctuation it
|
|
74
|
+
// never opened — it learned that `)` and `.` follow tokens, not that a bracket has
|
|
75
|
+
// to be opened first. Structural, language-agnostic, and cheap: real prose and real
|
|
76
|
+
// code balance their delimiters. The floor of TWO spares a legitimate stray paren.
|
|
77
|
+
if (unmatchedClosers(t) >= 2)
|
|
78
|
+
return fail('unbalanced-delimiters');
|
|
79
|
+
return assessVocabulary(t, words, opts);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Signals 5–6 — the LEXICAL half of the gate, split out so the guards that make it
|
|
83
|
+
* safe live in one readable place. This is what closes the gap the structural checks
|
|
84
|
+
* cannot see: invented words with balanced punctuation and no repetition.
|
|
85
|
+
*
|
|
86
|
+
* It is skipped entirely unless the text is predominantly Latin-script and mostly
|
|
87
|
+
* prose (not code), because outside that envelope a word list proves nothing.
|
|
88
|
+
*/
|
|
89
|
+
function assessVocabulary(t, words, opts) {
|
|
90
|
+
// Raw whitespace tokens paired with their alphabetic core, so the scorer can tell an
|
|
91
|
+
// identifier (`projectEvermindRef`, `api/src/x.ts`, `v2.1`) from a prose word.
|
|
92
|
+
const rawTokens = [];
|
|
93
|
+
const coreWords = [];
|
|
94
|
+
for (const raw of t.split(/\s+/u)) {
|
|
95
|
+
const core = raw
|
|
96
|
+
.replace(/^[^\p{L}]+/u, '')
|
|
97
|
+
.replace(/[^\p{L}]+$/u, '')
|
|
98
|
+
.toLowerCase()
|
|
99
|
+
.replace(/[''’-]/gu, '');
|
|
100
|
+
if (!core || !/^\p{L}+$/u.test(core)) {
|
|
101
|
+
rawTokens.push(raw);
|
|
102
|
+
coreWords.push('');
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
rawTokens.push(raw);
|
|
106
|
+
coreWords.push(core);
|
|
107
|
+
}
|
|
108
|
+
// Code-dominant text (a diff, a config dump, a stack trace) is legitimately made of
|
|
109
|
+
// non-words — never judge it lexically.
|
|
110
|
+
const codeish = rawTokens.filter((r) => r && isCodeishToken(r)).length;
|
|
111
|
+
if (rawTokens.length > 0 && codeish / rawTokens.length > 0.3)
|
|
112
|
+
return pass();
|
|
113
|
+
const lang = detectLatinLanguage(words, t);
|
|
114
|
+
if (!lang.latin)
|
|
115
|
+
return pass(); // CJK / Cyrillic / Arabic — out of scope by design.
|
|
116
|
+
// 5) No function words at all. Prose in EVERY supported language is ≥ ~10% function
|
|
117
|
+
// words; a long Latin-script, non-code passage with essentially none is not a
|
|
118
|
+
// language. This is the check that catches gibberish too degenerate to even
|
|
119
|
+
// identify a language for — the case the dictionary check below can't reach.
|
|
120
|
+
if (words.length >= 25 && lang.anyShare < 0.04)
|
|
121
|
+
return fail('no-function-words');
|
|
122
|
+
// 6) Invented-word share. English-only: a reply that votes decisively for Spanish,
|
|
123
|
+
// French, German, Portuguese or Italian is never measured against an English
|
|
124
|
+
// lexicon. Everything else — English wins, or nothing wins and English still
|
|
125
|
+
// leads — IS measured, because "no language won" describes gibberish at least as
|
|
126
|
+
// often as it describes prose, and passing it unjudged was the hole that let
|
|
127
|
+
// invented-word output through. The scorer additionally forgives any token that
|
|
128
|
+
// echoes the prompt or recurs in the text, so jargon-dense real answers pass while
|
|
129
|
+
// each-word-different gibberish does not.
|
|
130
|
+
if (!lang.englishLeads)
|
|
131
|
+
return pass();
|
|
132
|
+
const score = scoreEnglishWordiness(coreWords.filter(Boolean), rawTokens.filter((_, i) => !!coreWords[i]), opts.context);
|
|
133
|
+
if (score.scored && score.unknown >= 5 && score.unknownShare >= 0.5) {
|
|
134
|
+
return {
|
|
135
|
+
coherent: false,
|
|
136
|
+
failure: 'non-words',
|
|
137
|
+
detail: `${FAILURE_DETAIL['non-words']} (${score.unknown}/${score.eligible} unrecognised)`,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
return pass();
|
|
141
|
+
}
|
|
142
|
+
const pass = () => ({ coherent: true, failure: null, detail: '' });
|
|
143
|
+
const fail = (failure) => ({ coherent: false, failure, detail: FAILURE_DETAIL[failure] });
|
|
144
|
+
/**
|
|
145
|
+
* Boolean form of {@link assessTextCoherence} — the shape every existing caller uses.
|
|
146
|
+
* Prefer the assessment when you have somewhere to SHOW the reason.
|
|
147
|
+
*/
|
|
148
|
+
export function looksLikeCoherentText(text, opts = {}) {
|
|
149
|
+
return assessTextCoherence(text, opts).coherent;
|
|
150
|
+
}
|
|
151
|
+
/** Is this text both substantive (long enough) AND coherent — the ONE bar every serve
|
|
152
|
+
* path applies before showing SSM output to a user. Shared so the length threshold and
|
|
153
|
+
* the gate can never drift apart between the memory resolver, the vendor and the probe. */
|
|
154
|
+
export function isServableText(text, opts = {}) {
|
|
155
|
+
const t = (text ?? '').trim();
|
|
156
|
+
if (t.length < EVERMIND_ANSWER_MIN_CHARS) {
|
|
157
|
+
return { coherent: false, failure: 'empty', detail: `the model returned less than ${EVERMIND_ANSWER_MIN_CHARS} characters` };
|
|
158
|
+
}
|
|
159
|
+
return assessTextCoherence(t, opts);
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Has the head stopped reaching for new words? Two independent shapes of the same
|
|
163
|
+
* failure, measured over CONTENT words (see {@link isFunctionWord}) so that ordinary
|
|
164
|
+
* grammar cannot trip either one:
|
|
165
|
+
*
|
|
166
|
+
* - **Too few distinct words for the length.** A reply whose content vocabulary is at
|
|
167
|
+
* most half its content tokens is recycling, not composing. This is what works on a
|
|
168
|
+
* SHORT reply, where any absolute occurrence floor is out of reach: `update update
|
|
169
|
+
* the file update the file update` is 7 content tokens drawn from 2 words.
|
|
170
|
+
* - **One word taking most of the reply.** The long-form shape, where enough distinct
|
|
171
|
+
* words survive to clear the diversity bar but one of them still dominates — the
|
|
172
|
+
* observed `commit … commit … commit` sample sprayed it ~15 times.
|
|
173
|
+
*
|
|
174
|
+
* The dominance arm is measured against CONTENT tokens and needs a body of them, which
|
|
175
|
+
* is what spares a legitimately repetitive real answer: "every commit on that branch is
|
|
176
|
+
* a merge commit, so the commit history reads as one commit per pull request" says
|
|
177
|
+
* `commit` five times and is a perfectly good sentence — the previous rule, which
|
|
178
|
+
* measured against ALL tokens with a fixed floor of 5 occurrences, rejected it.
|
|
179
|
+
*/
|
|
180
|
+
function collapsedVocabulary(words) {
|
|
181
|
+
const freq = new Map();
|
|
182
|
+
let tokens = 0;
|
|
183
|
+
for (const w of words) {
|
|
184
|
+
if (w.length < 3 || isFunctionWord(w))
|
|
185
|
+
continue;
|
|
186
|
+
tokens++;
|
|
187
|
+
freq.set(w, (freq.get(w) ?? 0) + 1);
|
|
188
|
+
}
|
|
189
|
+
// Below this there is no vocabulary to have collapsed — a handful of content words is
|
|
190
|
+
// a terse answer, and every ratio over it is noise.
|
|
191
|
+
if (tokens < 5)
|
|
192
|
+
return false;
|
|
193
|
+
if (freq.size / tokens <= 0.5)
|
|
194
|
+
return true;
|
|
195
|
+
let maxCount = 0;
|
|
196
|
+
for (const c of freq.values())
|
|
197
|
+
if (c > maxCount)
|
|
198
|
+
maxCount = c;
|
|
199
|
+
return tokens >= 12 && maxCount / tokens > 0.4;
|
|
200
|
+
}
|
|
201
|
+
/** How many closing brackets appear with no opener before them. Pure counting — a
|
|
202
|
+
* balanced or over-opened text scores 0. Shared by {@link assessTextCoherence}. */
|
|
203
|
+
function unmatchedClosers(text) {
|
|
204
|
+
let depth = 0;
|
|
205
|
+
let orphans = 0;
|
|
206
|
+
for (const ch of text) {
|
|
207
|
+
if (ch === '(' || ch === '[' || ch === '{')
|
|
208
|
+
depth++;
|
|
209
|
+
else if (ch === ')' || ch === ']' || ch === '}') {
|
|
210
|
+
if (depth > 0)
|
|
211
|
+
depth--;
|
|
212
|
+
else
|
|
213
|
+
orphans++;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return orphans;
|
|
217
|
+
}
|
|
218
|
+
//# sourceMappingURL=textCoherence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"textCoherence.js","sourceRoot":"","sources":["../../src/evermind/textCoherence.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAE1B;qFACqF;AACrF,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAgC5C,2DAA2D;AAC3D,MAAM,cAAc,GAAqC;IACvD,OAAO,EAAE,4BAA4B;IACrC,mBAAmB,EAAE,4EAA4E;IACjG,YAAY,EAAE,wDAAwD;IACtE,gBAAgB,EAAE,6FAA6F;IAC/G,uBAAuB,EAAE,4CAA4C;IACrE,mBAAmB,EAAE,yEAAyE;IAC9F,WAAW,EAAE,qFAAqF;CACnG,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,OAAyB,EAAE;IAC3E,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC;IAE7B,8EAA8E;IAC9E,2EAA2E;IAC3E,kEAAkE;IAClE,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IACtD,iFAAiF;IACjF,+EAA+E;IAC/E,iCAAiC;IACjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,EAAE,CAAC;IAEpC,2EAA2E;IAC3E,uEAAuE;IACvE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;IAC5E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC;IAErE,gFAAgF;IAChF,mFAAmF;IACnF,sFAAsF;IACtF,iFAAiF;IACjF,sFAAsF;IACtF,0DAA0D;IAC1D,IAAI,mBAAmB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAE9D,mFAAmF;IACnF,uFAAuF;IACvF,mFAAmF;IACnF,sFAAsF;IACtF,uFAAuF;IACvF,sFAAsF;IACtF,IAAI,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEnE,OAAO,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,CAAS,EAAE,KAAe,EAAE,IAAsB;IAC1E,qFAAqF;IACrF,+EAA+E;IAC/E,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,GAAG;aACb,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,WAAW,EAAE;aACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC5F,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IACD,oFAAoF;IACpF,wCAAwC;IACxC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,EAAE,CAAC;IAE5E,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,EAAE,CAAC,CAAC,oDAAoD;IAEpF,oFAAoF;IACpF,iFAAiF;IACjF,+EAA+E;IAC/E,gFAAgF;IAChF,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,QAAQ,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAEjF,mFAAmF;IACnF,gFAAgF;IAChF,gFAAgF;IAChF,oFAAoF;IACpF,gFAAgF;IAChF,mFAAmF;IACnF,sFAAsF;IACtF,6CAA6C;IAC7C,IAAI,CAAC,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACzH,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;QACpE,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,GAAG,cAAc,CAAC,WAAW,CAAC,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,QAAQ,gBAAgB;SAC3F,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,EAAE,CAAC;AAChB,CAAC;AAED,MAAM,IAAI,GAAG,GAAqB,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;AACrF,MAAM,IAAI,GAAG,CAAC,OAAyB,EAAoB,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAE9H;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAyB,EAAE;IAC7E,OAAO,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC;AAClD,CAAC;AAED;;4FAE4F;AAC5F,MAAM,UAAU,cAAc,CAAC,IAA+B,EAAE,OAAyB,EAAE;IACzF,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,CAAC,MAAM,GAAG,yBAAyB,EAAE,CAAC;QACzC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gCAAgC,yBAAyB,aAAa,EAAE,CAAC;IAC/H,CAAC;IACD,OAAO,mBAAmB,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,mBAAmB,CAAC,KAAwB;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC;YAAE,SAAS;QAChD,MAAM,EAAE,CAAC;QACT,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,sFAAsF;IACtF,oDAAoD;IACpD,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7B,IAAI,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IAE3C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE;QAAE,IAAI,CAAC,GAAG,QAAQ;YAAE,QAAQ,GAAG,CAAC,CAAC;IAC9D,OAAO,MAAM,IAAI,EAAE,IAAI,QAAQ,GAAG,MAAM,GAAG,GAAG,CAAC;AACjD,CAAC;AAED;oFACoF;AACpF,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,EAAE,CAAC;aAC/C,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAChD,IAAI,KAAK,GAAG,CAAC;gBAAE,KAAK,EAAE,CAAC;;gBAClB,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|