@seanhogg/builderforce-memory 2026.9.30 → 2026.9.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/bridges/FetchBridge.d.ts +12 -5
  2. package/dist/bridges/FetchBridge.d.ts.map +1 -1
  3. package/dist/bridges/FetchBridge.js +6 -4
  4. package/dist/bridges/FetchBridge.js.map +1 -1
  5. package/dist/bridges/OpenAIBridge.d.ts +17 -4
  6. package/dist/bridges/OpenAIBridge.d.ts.map +1 -1
  7. package/dist/bridges/OpenAIBridge.js +68 -57
  8. package/dist/bridges/OpenAIBridge.js.map +1 -1
  9. package/dist/bridges/TransformerBridge.d.ts +6 -0
  10. package/dist/bridges/TransformerBridge.d.ts.map +1 -1
  11. package/dist/evermind/index.d.ts +6 -0
  12. package/dist/evermind/index.d.ts.map +1 -1
  13. package/dist/evermind/index.js +5 -0
  14. package/dist/evermind/index.js.map +1 -1
  15. package/dist/evermind/runtime.d.ts +158 -0
  16. package/dist/evermind/runtime.d.ts.map +1 -0
  17. package/dist/evermind/runtime.js +236 -0
  18. package/dist/evermind/runtime.js.map +1 -0
  19. package/dist/evermind/textCoherence.d.ts +55 -0
  20. package/dist/evermind/textCoherence.d.ts.map +1 -0
  21. package/dist/evermind/textCoherence.js +218 -0
  22. package/dist/evermind/textCoherence.js.map +1 -0
  23. package/dist/evermind/toolCall.d.ts +175 -0
  24. package/dist/evermind/toolCall.d.ts.map +1 -0
  25. package/dist/evermind/toolCall.js +398 -0
  26. package/dist/evermind/toolCall.js.map +1 -0
  27. package/dist/evermind/wordLexicon.d.ts +103 -0
  28. package/dist/evermind/wordLexicon.d.ts.map +1 -0
  29. package/dist/evermind/wordLexicon.js +361 -0
  30. package/dist/evermind/wordLexicon.js.map +1 -0
  31. package/dist/wire/chatClient.d.ts +100 -0
  32. package/dist/wire/chatClient.d.ts.map +1 -0
  33. package/dist/wire/chatClient.js +123 -0
  34. package/dist/wire/chatClient.js.map +1 -0
  35. package/dist/wire/index.d.ts +3 -1
  36. package/dist/wire/index.d.ts.map +1 -1
  37. package/dist/wire/index.js +2 -1
  38. package/dist/wire/index.js.map +1 -1
  39. package/dist/wire/sse.d.ts +17 -9
  40. package/dist/wire/sse.d.ts.map +1 -1
  41. package/dist/wire/sse.js +45 -22
  42. package/dist/wire/sse.js.map +1 -1
  43. package/package.json +3 -3
  44. package/src/bridges/FetchBridge.ts +15 -6
  45. package/src/bridges/OpenAIBridge.ts +82 -72
  46. package/src/bridges/TransformerBridge.ts +6 -0
  47. package/src/evermind/index.ts +48 -0
  48. package/src/evermind/runtime.ts +364 -0
  49. package/src/evermind/textCoherence.ts +248 -0
  50. package/src/evermind/toolCall.ts +523 -0
  51. package/src/evermind/wordLexicon.ts +395 -0
  52. package/src/wire/chatClient.ts +196 -0
  53. package/src/wire/index.ts +3 -1
  54. package/src/wire/sse.ts +43 -21
@@ -0,0 +1,236 @@
1
+ /**
2
+ * The model-driving half of Evermind serving: flatten a chat into a continuation
3
+ * prompt, generate under a wall-clock budget, grade a head's fitness to serve, and
4
+ * bind a head to the tool planner's decoder port.
5
+ *
6
+ * Written against STRUCTURAL model/tokenizer shapes ({@link EvermindTextModel},
7
+ * {@link EvermindTextTokenizer}) that the engine's `EvermindLM` and `BPETokenizer`
8
+ * satisfy, so this module imports nothing and the `/evermind` subpath stays
9
+ * dependency-free. Where the head comes FROM (an R2 artifact, a file, a freshly merged
10
+ * in-memory model) is the host's business; everything here takes a loaded head.
11
+ */
12
+ import { isServableText } from './textCoherence.js';
13
+ import { planEvermindToolCall, } from './toolCall.js';
14
+ /** Flatten chat messages into a single continuation prompt for the LM. */
15
+ export function messagesToPrompt(messages) {
16
+ const lines = messages
17
+ .map((m) => {
18
+ const role = typeof m.role === 'string' ? m.role : 'user';
19
+ const content = typeof m.content === 'string' ? m.content : '';
20
+ return content ? `${role}: ${content}` : '';
21
+ })
22
+ .filter(Boolean);
23
+ return `${lines.join('\n')}\nassistant:`;
24
+ }
25
+ /**
26
+ * Tokens generated between wall-clock checks. The deadline is enforced BETWEEN slices
27
+ * because the engine's generate is one synchronous loop with no per-token hook.
28
+ * Slicing costs nothing: the forward pass already recomputes the whole sequence per
29
+ * token, so re-entering with `prompt + producedSoFar` is the work the next token was
30
+ * going to do anyway. Small enough that the overshoot is one slice.
31
+ */
32
+ const GENERATE_SLICE_TOKENS = 16;
33
+ /** Default wall-clock budget for one generation — inside a Worker's CPU allowance, so
34
+ * the caller gets a partial answer plus `truncated` rather than a killed request. */
35
+ const DEFAULT_GENERATE_DEADLINE_MS = 8000;
36
+ /**
37
+ * Generate a reply from a loaded head. Deterministic for a given (prompt, seed,
38
+ * maxTokens): each slice derives its seed from the base seed and its index, so two
39
+ * operators running the same probe see the same text.
40
+ */
41
+ export function generateEvermindText(lm, tok, messages, opts = {}) {
42
+ const prompt = messagesToPrompt(messages);
43
+ const maxTokens = opts.maxTokens ?? 256;
44
+ const temperature = opts.temperature ?? 0.7;
45
+ const baseSeed = opts.seed;
46
+ const deadlineMs = opts.deadlineMs ?? DEFAULT_GENERATE_DEADLINE_MS;
47
+ const started = Date.now();
48
+ let content = '';
49
+ let truncated = false;
50
+ let produced = 0;
51
+ let slice = 0;
52
+ while (produced < maxTokens) {
53
+ const want = Math.min(GENERATE_SLICE_TOKENS, maxTokens - produced);
54
+ const chunk = lm.generateText(`${prompt}${content}`, tok, {
55
+ maxNewTokens: want,
56
+ temperature,
57
+ ...(baseSeed != null ? { seed: baseSeed + slice } : {}),
58
+ });
59
+ slice++;
60
+ // An empty slice means the model stopped producing; continuing would spin.
61
+ if (!chunk)
62
+ break;
63
+ content += chunk;
64
+ produced += want;
65
+ if (Date.now() - started >= deadlineMs) {
66
+ truncated = produced < maxTokens;
67
+ break;
68
+ }
69
+ }
70
+ const prompt_tokens = tok.encode(prompt).length;
71
+ const completion_tokens = content ? tok.encode(content).length : 0;
72
+ return {
73
+ content,
74
+ usage: { prompt_tokens, completion_tokens, total_tokens: prompt_tokens + completion_tokens },
75
+ truncated,
76
+ elapsedMs: Date.now() - started,
77
+ };
78
+ }
79
+ // ── Fitness to serve ────────────────────────────────────────────────────────
80
+ /** Neutral probe prompts a project chat head should answer coherently. Fixed and
81
+ * generic (not project-specific) so the probe measures GENERATION QUALITY, not
82
+ * recall. */
83
+ export const COHERENCE_PROBE_PROMPTS = [
84
+ 'Summarize the current status of the project.',
85
+ 'What has the team been working on recently?',
86
+ 'List the main things left to do.',
87
+ ];
88
+ /**
89
+ * Score a loaded head's fitness to serve chat: generate from the neutral probe
90
+ * prompts and grade each with the serve-time bar ({@link isServableText}).
91
+ * Synchronous and storage-free, so the promote-to-inference gate and a learning
92
+ * coordinator holding a freshly merged model grade it the same way. Deterministic
93
+ * seeds keep the verdict reproducible; a majority must pass by default.
94
+ */
95
+ export function assessLMCoherence(lm, tok, opts = {}) {
96
+ const samples = COHERENCE_PROBE_PROMPTS.map((prompt, i) => {
97
+ const text = lm.generateText(messagesToPrompt([{ role: 'user', content: prompt }]), tok, {
98
+ maxNewTokens: 80,
99
+ temperature: 0.7,
100
+ seed: 1234 + i,
101
+ });
102
+ const verdict = isServableText(text, { context: prompt });
103
+ return { prompt, text, coherent: verdict.coherent, failure: verdict.failure, detail: verdict.detail };
104
+ });
105
+ const passRate = samples.length ? samples.filter((s) => s.coherent).length / samples.length : 0;
106
+ return { ready: passRate >= (opts.minPassRate ?? 0.5), passRate, samples };
107
+ }
108
+ /**
109
+ * Test bench: run ONE prompt through a head and grade the output with the SAME bar
110
+ * the serve path uses, so "what will this model actually produce?" is answerable
111
+ * before inference is switched on. Deterministic by default (`seed`).
112
+ */
113
+ export function probeEvermindText(lm, tok, prompt, opts = {}) {
114
+ const gen = generateEvermindText(lm, tok, [{ role: 'user', content: prompt }], {
115
+ maxTokens: opts.maxTokens ?? 120,
116
+ temperature: opts.temperature ?? 0.7,
117
+ seed: opts.seed ?? 1234,
118
+ ...(opts.deadlineMs != null ? { deadlineMs: opts.deadlineMs } : {}),
119
+ });
120
+ const verdict = isServableText(gen.content, { context: prompt });
121
+ return {
122
+ prompt,
123
+ text: gen.content,
124
+ coherent: verdict.coherent,
125
+ failure: verdict.failure,
126
+ detail: verdict.detail,
127
+ usage: gen.usage,
128
+ truncated: gen.truncated,
129
+ elapsedMs: gen.elapsedMs,
130
+ };
131
+ }
132
+ // ── Tool calling ─────────────────────────────────────────────────────────────
133
+ /**
134
+ * A tool-decision prompt is rebuilt for every candidate and every argument, and each
135
+ * costs a full forward pass, so the conversation is capped rather than replayed whole.
136
+ * Truncated from the LEFT: the most recent turns are what a tool choice depends on.
137
+ */
138
+ const MAX_TOOL_PROMPT_CHARS = 6000;
139
+ /** Log-probability the model assigned to `id` at a position, from that position's raw
140
+ * logits (log-softmax, max-shifted so long-tail logits don't overflow). */
141
+ function logProbOf(row, id) {
142
+ let max = -Infinity;
143
+ for (let i = 0; i < row.length; i++) {
144
+ const v = row[i];
145
+ if (v > max)
146
+ max = v;
147
+ }
148
+ if (!Number.isFinite(max))
149
+ return -Infinity;
150
+ let sum = 0;
151
+ for (let i = 0; i < row.length; i++)
152
+ sum += Math.exp(row[i] - max);
153
+ const logit = id >= 0 && id < row.length ? row[id] : -Infinity;
154
+ return logit - (max + Math.log(sum));
155
+ }
156
+ /** Keep the tail of an over-long prompt (see {@link MAX_TOOL_PROMPT_CHARS}). */
157
+ function clampPromptText(prompt) {
158
+ return prompt.length <= MAX_TOOL_PROMPT_CHARS ? prompt : prompt.slice(prompt.length - MAX_TOOL_PROMPT_CHARS);
159
+ }
160
+ /**
161
+ * Bind a loaded head to the {@link EvermindToolDecoder} port.
162
+ *
163
+ * `score`: teacher forcing lets one forward pass over `prompt + continuation` yield
164
+ * the log-prob of EVERY continuation token at once (position `t` predicts `t+1`), so
165
+ * ranking a candidate costs one pass. The mean is returned — not the sum — so a long
166
+ * tool name is not out-voted by a short one for its length alone.
167
+ *
168
+ * `generate` is greedy by default: a tool ARGUMENT is a value to get right, not prose
169
+ * to vary, and determinism keeps a replayed run reproducible.
170
+ */
171
+ export function createEvermindToolDecoder(lm, tok, opts = {}) {
172
+ let promptTokens = 0;
173
+ let completionTokens = 0;
174
+ return {
175
+ score(prompt, continuation) {
176
+ const contIds = tok.encode(continuation);
177
+ if (contIds.length === 0)
178
+ return -Infinity;
179
+ // A leading token is required for the first continuation token to have a
180
+ // position to be predicted FROM; an empty prompt gets the same id-0 prefix the
181
+ // engine's own sampler uses.
182
+ const promptIds = tok.encode(clampPromptText(prompt));
183
+ const prefix = promptIds.length > 0 ? promptIds : [0];
184
+ const { logits } = lm.forward([...prefix, ...contIds]);
185
+ let total = 0;
186
+ for (let i = 0; i < contIds.length; i++) {
187
+ total += logProbOf(logits[prefix.length + i - 1], contIds[i]);
188
+ }
189
+ promptTokens += prefix.length;
190
+ completionTokens += contIds.length;
191
+ return total / contIds.length;
192
+ },
193
+ generate(prompt, maxTokens) {
194
+ const text = lm.generateText(clampPromptText(prompt), tok, {
195
+ maxNewTokens: maxTokens,
196
+ temperature: opts.temperature ?? 0,
197
+ ...(opts.seed != null ? { seed: opts.seed } : {}),
198
+ });
199
+ promptTokens += tok.encode(clampPromptText(prompt)).length;
200
+ completionTokens += text ? tok.encode(text).length : 0;
201
+ return text;
202
+ },
203
+ usage: () => ({ prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens }),
204
+ };
205
+ }
206
+ /**
207
+ * Run a TOOL-BEARING request against a loaded head: plan a call by constrained
208
+ * decoding, or fall through to ordinary prose when the head elected to answer directly.
209
+ */
210
+ export function generateEvermindWithTools(lm, tok, messages, tools, choice, opts = {}) {
211
+ const decoder = createEvermindToolDecoder(lm, tok, {
212
+ ...(opts.temperature != null ? { temperature: opts.temperature } : {}),
213
+ ...(opts.seed != null ? { seed: opts.seed } : {}),
214
+ });
215
+ // The planner is given the conversation WITHOUT the `assistant:` primer — it is
216
+ // choosing an action, not continuing a reply.
217
+ const conversation = messagesToPrompt(messages).replace(/\nassistant:$/, '');
218
+ const plan = planEvermindToolCall(decoder, conversation, tools, choice);
219
+ if (plan.call) {
220
+ return { call: plan.call, calls: plan.calls, content: '', usage: decoder.usage(), margin: plan.margin };
221
+ }
222
+ const gen = generateEvermindText(lm, tok, messages, opts);
223
+ const usage = decoder.usage();
224
+ return {
225
+ call: null,
226
+ calls: [],
227
+ content: gen.content,
228
+ usage: {
229
+ prompt_tokens: usage.prompt_tokens + gen.usage.prompt_tokens,
230
+ completion_tokens: usage.completion_tokens + gen.usage.completion_tokens,
231
+ total_tokens: usage.total_tokens + gen.usage.total_tokens,
232
+ },
233
+ margin: plan.margin,
234
+ };
235
+ }
236
+ //# sourceMappingURL=runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/evermind/runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,cAAc,EAAyB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EACL,oBAAoB,GAKrB,MAAM,eAAe,CAAC;AA0CvB,0EAA0E;AAC1E,MAAM,UAAU,gBAAgB,CAAC,QAAsD;IACrF,MAAM,KAAK,GAAG,QAAQ;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC;sFACsF;AACtF,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAAqB,EACrB,GAA0B,EAC1B,QAAsD,EACtD,OAAgC,EAAE;IAElC,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,4BAA4B,CAAC;IAEnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,QAAQ,GAAG,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,SAAS,GAAG,QAAQ,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,MAAM,GAAG,OAAO,EAAE,EAAE,GAAG,EAAE;YACxD,YAAY,EAAE,IAAI;YAClB,WAAW;YACX,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC,CAAC;QACH,KAAK,EAAE,CAAC;QACR,2EAA2E;QAC3E,IAAI,CAAC,KAAK;YAAE,MAAM;QAClB,OAAO,IAAI,KAAK,CAAC;QACjB,QAAQ,IAAI,IAAI,CAAC;QACjB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC;YACvC,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAC;YACjC,MAAM;QACR,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChD,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,OAAO;QACL,OAAO;QACP,KAAK,EAAE,EAAE,aAAa,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,GAAG,iBAAiB,EAAE;QAC5F,SAAS;QACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KAChC,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;cAEc;AACd,MAAM,CAAC,MAAM,uBAAuB,GAAsB;IACxD,8CAA8C;IAC9C,6CAA6C;IAC7C,kCAAkC;CACnC,CAAC;AAsBF;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,GAA0B,EAC1B,OAAiC,EAAE;IAEnC,MAAM,OAAO,GAA8B,uBAAuB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACnF,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE;YACvF,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,GAAG;YAChB,IAAI,EAAE,IAAI,GAAG,CAAC;SACf,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACxG,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAChG,OAAO,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC7E,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,GAA0B,EAC1B,MAAc,EACd,OAAgC,EAAE;IAElC,MAAM,GAAG,GAAG,oBAAoB,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE;QAC7E,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,GAAG;QAChC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;QACpC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;QACvB,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,OAAO;QACL,MAAM;QACN,IAAI,EAAE,GAAG,CAAC,OAAO;QACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;KACzB,CAAC;AACJ,CAAC;AAED,gFAAgF;AAEhF;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAEnC;4EAC4E;AAC5E,SAAS,SAAS,CAAC,GAAsB,EAAE,EAAU;IACnD,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;QAAC,IAAI,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,CAAC,CAAC;IAAC,CAAC;IACjF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,QAAQ,CAAC;IAC5C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChE,OAAO,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,gFAAgF;AAChF,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,CAAC,MAAM,IAAI,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC;AAC/G,CAAC;AAQD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,yBAAyB,CACvC,EAAqB,EACrB,GAA0B,EAC1B,OAAgD,EAAE;IAElD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,OAAO;QACL,KAAK,CAAC,MAAc,EAAE,YAAoB;YACxC,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,CAAC,QAAQ,CAAC;YAC3C,yEAAyE;YACzE,+EAA+E;YAC/E,6BAA6B;YAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;YACtD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;YACvD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,KAAK,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAE,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAClE,CAAC;YACD,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YAC9B,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;YACnC,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAChC,CAAC;QACD,QAAQ,CAAC,MAAc,EAAE,SAAiB;YACxC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE;gBACzD,YAAY,EAAE,SAAS;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAClD,CAAC,CAAC;YACH,YAAY,IAAI,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3D,gBAAgB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,YAAY,EAAE,YAAY,GAAG,gBAAgB,EAAE,CAAC;KACnI,CAAC;AACJ,CAAC;AAkBD;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CACvC,EAAqB,EACrB,GAA0B,EAC1B,QAAsD,EACtD,KAAuB,EACvB,MAAsB,EACtB,OAAgC,EAAE;IAElC,MAAM,OAAO,GAAG,yBAAyB,CAAC,EAAE,EAAE,GAAG,EAAE;QACjD,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;IACH,gFAAgF;IAChF,8CAA8C;IAC9C,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,oBAAoB,CAAC,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACxE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC1G,CAAC;IACD,MAAM,GAAG,GAAG,oBAAoB,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAC9B,OAAO;QACL,IAAI,EAAE,IAAI;QACV,KAAK,EAAE,EAAE;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,KAAK,EAAE;YACL,aAAa,EAAE,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa;YAC5D,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,GAAG,GAAG,CAAC,KAAK,CAAC,iBAAiB;YACxE,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY;SAC1D;QACD,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC;AACJ,CAAC"}
@@ -0,0 +1,55 @@
1
+ /**
2
+ * textCoherence — pure, zero-dependency text-quality primitives shared by every
3
+ * surface that must decide whether text a project's OWN SSM (Evermind) produced is
4
+ * fit to show a user. Kept dependency-free (no db/env/engine imports) so it can be
5
+ * imported by the memory-first resolver, the model probe and the gateway vendor —
6
+ * and by a browser client — without an import cycle.
7
+ */
8
+ /** A reply shorter than this isn't a real answer — fall through to the LLM. Mirrors
9
+ * the cloud/BrainService threshold so every surface adopts Evermind identically. */
10
+ export declare const EVERMIND_ANSWER_MIN_CHARS = 20;
11
+ /** Why the gate rejected a text. Surfaced to operators (console test bench, quarantine
12
+ * reason, vendor error) so "it produced gibberish" is never an unexplained verdict. */
13
+ export type CoherenceFailure = 'empty' | 'replacement-chars' | 'repetition' | 'dominant-token' | 'unbalanced-delimiters' | 'no-function-words' | 'non-words';
14
+ /** The gate's verdict + the specific signal that produced it. */
15
+ export interface CoherenceVerdict {
16
+ coherent: boolean;
17
+ /** The failing signal, or null when the text passed. */
18
+ failure: CoherenceFailure | null;
19
+ /** Short operator-facing explanation of {@link failure} (empty when coherent). */
20
+ detail: string;
21
+ }
22
+ export interface CoherenceOptions {
23
+ /**
24
+ * The prompt/question the text answers, when the caller has it. Domain jargon
25
+ * ("Evermind", "neocortex", a repo name) reliably echoes the ask, so treating
26
+ * context words as known is what keeps the non-word check from mis-accusing a
27
+ * legitimate, jargon-dense answer. Optional — the gate works without it.
28
+ */
29
+ context?: string;
30
+ }
31
+ /**
32
+ * Cheap coherence gate for text a project's OWN SSM (Evermind) produced. An
33
+ * under-trained head emits fluent-LOOKING garbage — broken UTF-8 (Unicode
34
+ * replacement chars from byte-level BPE), degenerate word repetition ("commit …
35
+ * commit … commit", "in the in the"), and fluent-shaped prose made entirely of
36
+ * INVENTED words — that trivially clears a length check. Serving it makes the
37
+ * assistant reply in gibberish, so this gate rejects it and the caller falls through
38
+ * to a real LLM (a garbled reply IS a memory miss).
39
+ *
40
+ * Signals 1–4 are structural and language-agnostic. Signals 5–6 consult a lexicon and
41
+ * are therefore GUARDED: they run only on predominantly-Latin, non-code text, and the
42
+ * dictionary check only when the text is confidently English — so an es/fr/de/pt/it
43
+ * reply, a CJK reply, or a code block is never judged by an English word list.
44
+ */
45
+ export declare function assessTextCoherence(text: string, opts?: CoherenceOptions): CoherenceVerdict;
46
+ /**
47
+ * Boolean form of {@link assessTextCoherence} — the shape every existing caller uses.
48
+ * Prefer the assessment when you have somewhere to SHOW the reason.
49
+ */
50
+ export declare function looksLikeCoherentText(text: string, opts?: CoherenceOptions): boolean;
51
+ /** Is this text both substantive (long enough) AND coherent — the ONE bar every serve
52
+ * path applies before showing SSM output to a user. Shared so the length threshold and
53
+ * the gate can never drift apart between the memory resolver, the vendor and the probe. */
54
+ export declare function isServableText(text: string | null | undefined, opts?: CoherenceOptions): CoherenceVerdict;
55
+ //# sourceMappingURL=textCoherence.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"textCoherence.d.ts","sourceRoot":"","sources":["../../src/evermind/textCoherence.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH;qFACqF;AACrF,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAE5C;wFACwF;AACxF,MAAM,MAAM,gBAAgB,GACxB,OAAO,GACP,mBAAmB,GACnB,YAAY,GACZ,gBAAgB,GAChB,uBAAuB,GACvB,mBAAmB,GACnB,WAAW,CAAC;AAEhB,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,wDAAwD;IACxD,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,kFAAkF;IAClF,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAaD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,gBAAgB,CAyC/F;AA+DD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,OAAO,CAExF;AAED;;4FAE4F;AAC5F,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,IAAI,GAAE,gBAAqB,GAAG,gBAAgB,CAM7G"}
@@ -0,0 +1,218 @@
1
+ /**
2
+ * textCoherence — pure, zero-dependency text-quality primitives shared by every
3
+ * surface that must decide whether text a project's OWN SSM (Evermind) produced is
4
+ * fit to show a user. Kept dependency-free (no db/env/engine imports) so it can be
5
+ * imported by the memory-first resolver, the model probe and the gateway vendor —
6
+ * and by a browser client — without an import cycle.
7
+ */
8
+ import { detectLatinLanguage, isCodeishToken, isFunctionWord, scoreEnglishWordiness, } from './wordLexicon.js';
9
+ /** A reply shorter than this isn't a real answer — fall through to the LLM. Mirrors
10
+ * the cloud/BrainService threshold so every surface adopts Evermind identically. */
11
+ export const EVERMIND_ANSWER_MIN_CHARS = 20;
12
+ /** Human-readable explanations, one per failure signal. */
13
+ const FAILURE_DETAIL = {
14
+ 'empty': 'the model returned nothing',
15
+ 'replacement-chars': 'the output contains Unicode replacement characters (broken token decoding)',
16
+ 'repetition': 'the decoder is stuck repeating the same word or phrase',
17
+ 'dominant-token': 'the output recycles a handful of words — the head has collapsed onto part of its vocabulary',
18
+ 'unbalanced-delimiters': 'the output closes brackets it never opened',
19
+ 'no-function-words': 'the output has no recognisable function words in any supported language',
20
+ 'non-words': 'most of the content words are not real words — the head is emitting invented tokens',
21
+ };
22
+ /**
23
+ * Cheap coherence gate for text a project's OWN SSM (Evermind) produced. An
24
+ * under-trained head emits fluent-LOOKING garbage — broken UTF-8 (Unicode
25
+ * replacement chars from byte-level BPE), degenerate word repetition ("commit …
26
+ * commit … commit", "in the in the"), and fluent-shaped prose made entirely of
27
+ * INVENTED words — that trivially clears a length check. Serving it makes the
28
+ * assistant reply in gibberish, so this gate rejects it and the caller falls through
29
+ * to a real LLM (a garbled reply IS a memory miss).
30
+ *
31
+ * Signals 1–4 are structural and language-agnostic. Signals 5–6 consult a lexicon and
32
+ * are therefore GUARDED: they run only on predominantly-Latin, non-code text, and the
33
+ * dictionary check only when the text is confidently English — so an es/fr/de/pt/it
34
+ * reply, a CJK reply, or a code block is never judged by an English word list.
35
+ */
36
+ export function assessTextCoherence(text, opts = {}) {
37
+ const t = (text ?? '').trim();
38
+ if (!t)
39
+ return fail('empty');
40
+ // 1) Broken token decoding. A byte-level BPE emitting a low-probability token
41
+ // sequence frequently yields invalid UTF-8 → the replacement char. Real
42
+ // answers effectively never contain it → near-certain garbage.
43
+ if (t.includes('�'))
44
+ return fail('replacement-chars');
45
+ const words = t.toLowerCase().match(/\p{L}+/gu) ?? [];
46
+ // Too few alphabetic tokens to score structurally (CJK collapses to one run); it
47
+ // already cleared the length + replacement-char gates, so accept (a handful of
48
+ // clean words is a real answer).
49
+ if (words.length < 6)
50
+ return pass();
51
+ // 2) Adjacent runaway repetition: a stuck decoder repeats a token ("commit
52
+ // commit") or a bigram ("in the in the"). A little is normal prose.
53
+ let rep = 0;
54
+ for (let i = 1; i < words.length; i++)
55
+ if (words[i] === words[i - 1])
56
+ rep++;
57
+ for (let i = 3; i < words.length; i++) {
58
+ if (words[i] === words[i - 2] && words[i - 1] === words[i - 3])
59
+ rep++;
60
+ }
61
+ if (rep >= 3 && rep / words.length > 0.06)
62
+ return fail('repetition');
63
+ // 3) Content-word collapse: an under-trained head that overfit its corpus stops
64
+ // reaching for new words and recycles a handful ("update update the file update
65
+ // the file update"). Judged on CONTENT words only — grammar is supposed to repeat,
66
+ // and the length proxy this replaces (any token of 3+ letters) counted `the`,
67
+ // `and`, `was` and `not` as content, which is why its floors had to be set so high
68
+ // that they stopped meaning anything on a short reply.
69
+ if (collapsedVocabulary(words))
70
+ return fail('dominant-token');
71
+ // 4) Orphaned closing delimiters. The checks above key on REPETITION, so they miss
72
+ // the other failure mode of an under-trained byte-BPE head: fluent-shaped text made
73
+ // of invented words, each different. What that text does emit is punctuation it
74
+ // never opened — it learned that `)` and `.` follow tokens, not that a bracket has
75
+ // to be opened first. Structural, language-agnostic, and cheap: real prose and real
76
+ // code balance their delimiters. The floor of TWO spares a legitimate stray paren.
77
+ if (unmatchedClosers(t) >= 2)
78
+ return fail('unbalanced-delimiters');
79
+ return assessVocabulary(t, words, opts);
80
+ }
81
+ /**
82
+ * Signals 5–6 — the LEXICAL half of the gate, split out so the guards that make it
83
+ * safe live in one readable place. This is what closes the gap the structural checks
84
+ * cannot see: invented words with balanced punctuation and no repetition.
85
+ *
86
+ * It is skipped entirely unless the text is predominantly Latin-script and mostly
87
+ * prose (not code), because outside that envelope a word list proves nothing.
88
+ */
89
+ function assessVocabulary(t, words, opts) {
90
+ // Raw whitespace tokens paired with their alphabetic core, so the scorer can tell an
91
+ // identifier (`projectEvermindRef`, `api/src/x.ts`, `v2.1`) from a prose word.
92
+ const rawTokens = [];
93
+ const coreWords = [];
94
+ for (const raw of t.split(/\s+/u)) {
95
+ const core = raw
96
+ .replace(/^[^\p{L}]+/u, '')
97
+ .replace(/[^\p{L}]+$/u, '')
98
+ .toLowerCase()
99
+ .replace(/[''’-]/gu, '');
100
+ if (!core || !/^\p{L}+$/u.test(core)) {
101
+ rawTokens.push(raw);
102
+ coreWords.push('');
103
+ continue;
104
+ }
105
+ rawTokens.push(raw);
106
+ coreWords.push(core);
107
+ }
108
+ // Code-dominant text (a diff, a config dump, a stack trace) is legitimately made of
109
+ // non-words — never judge it lexically.
110
+ const codeish = rawTokens.filter((r) => r && isCodeishToken(r)).length;
111
+ if (rawTokens.length > 0 && codeish / rawTokens.length > 0.3)
112
+ return pass();
113
+ const lang = detectLatinLanguage(words, t);
114
+ if (!lang.latin)
115
+ return pass(); // CJK / Cyrillic / Arabic — out of scope by design.
116
+ // 5) No function words at all. Prose in EVERY supported language is ≥ ~10% function
117
+ // words; a long Latin-script, non-code passage with essentially none is not a
118
+ // language. This is the check that catches gibberish too degenerate to even
119
+ // identify a language for — the case the dictionary check below can't reach.
120
+ if (words.length >= 25 && lang.anyShare < 0.04)
121
+ return fail('no-function-words');
122
+ // 6) Invented-word share. English-only: a reply that votes decisively for Spanish,
123
+ // French, German, Portuguese or Italian is never measured against an English
124
+ // lexicon. Everything else — English wins, or nothing wins and English still
125
+ // leads — IS measured, because "no language won" describes gibberish at least as
126
+ // often as it describes prose, and passing it unjudged was the hole that let
127
+ // invented-word output through. The scorer additionally forgives any token that
128
+ // echoes the prompt or recurs in the text, so jargon-dense real answers pass while
129
+ // each-word-different gibberish does not.
130
+ if (!lang.englishLeads)
131
+ return pass();
132
+ const score = scoreEnglishWordiness(coreWords.filter(Boolean), rawTokens.filter((_, i) => !!coreWords[i]), opts.context);
133
+ if (score.scored && score.unknown >= 5 && score.unknownShare >= 0.5) {
134
+ return {
135
+ coherent: false,
136
+ failure: 'non-words',
137
+ detail: `${FAILURE_DETAIL['non-words']} (${score.unknown}/${score.eligible} unrecognised)`,
138
+ };
139
+ }
140
+ return pass();
141
+ }
142
+ const pass = () => ({ coherent: true, failure: null, detail: '' });
143
+ const fail = (failure) => ({ coherent: false, failure, detail: FAILURE_DETAIL[failure] });
144
+ /**
145
+ * Boolean form of {@link assessTextCoherence} — the shape every existing caller uses.
146
+ * Prefer the assessment when you have somewhere to SHOW the reason.
147
+ */
148
+ export function looksLikeCoherentText(text, opts = {}) {
149
+ return assessTextCoherence(text, opts).coherent;
150
+ }
151
+ /** Is this text both substantive (long enough) AND coherent — the ONE bar every serve
152
+ * path applies before showing SSM output to a user. Shared so the length threshold and
153
+ * the gate can never drift apart between the memory resolver, the vendor and the probe. */
154
+ export function isServableText(text, opts = {}) {
155
+ const t = (text ?? '').trim();
156
+ if (t.length < EVERMIND_ANSWER_MIN_CHARS) {
157
+ return { coherent: false, failure: 'empty', detail: `the model returned less than ${EVERMIND_ANSWER_MIN_CHARS} characters` };
158
+ }
159
+ return assessTextCoherence(t, opts);
160
+ }
161
+ /**
162
+ * Has the head stopped reaching for new words? Two independent shapes of the same
163
+ * failure, measured over CONTENT words (see {@link isFunctionWord}) so that ordinary
164
+ * grammar cannot trip either one:
165
+ *
166
+ * - **Too few distinct words for the length.** A reply whose content vocabulary is at
167
+ * most half its content tokens is recycling, not composing. This is what works on a
168
+ * SHORT reply, where any absolute occurrence floor is out of reach: `update update
169
+ * the file update the file update` is 7 content tokens drawn from 2 words.
170
+ * - **One word taking most of the reply.** The long-form shape, where enough distinct
171
+ * words survive to clear the diversity bar but one of them still dominates — the
172
+ * observed `commit … commit … commit` sample sprayed it ~15 times.
173
+ *
174
+ * The dominance arm is measured against CONTENT tokens and needs a body of them, which
175
+ * is what spares a legitimately repetitive real answer: "every commit on that branch is
176
+ * a merge commit, so the commit history reads as one commit per pull request" says
177
+ * `commit` five times and is a perfectly good sentence — the previous rule, which
178
+ * measured against ALL tokens with a fixed floor of 5 occurrences, rejected it.
179
+ */
180
+ function collapsedVocabulary(words) {
181
+ const freq = new Map();
182
+ let tokens = 0;
183
+ for (const w of words) {
184
+ if (w.length < 3 || isFunctionWord(w))
185
+ continue;
186
+ tokens++;
187
+ freq.set(w, (freq.get(w) ?? 0) + 1);
188
+ }
189
+ // Below this there is no vocabulary to have collapsed — a handful of content words is
190
+ // a terse answer, and every ratio over it is noise.
191
+ if (tokens < 5)
192
+ return false;
193
+ if (freq.size / tokens <= 0.5)
194
+ return true;
195
+ let maxCount = 0;
196
+ for (const c of freq.values())
197
+ if (c > maxCount)
198
+ maxCount = c;
199
+ return tokens >= 12 && maxCount / tokens > 0.4;
200
+ }
201
+ /** How many closing brackets appear with no opener before them. Pure counting — a
202
+ * balanced or over-opened text scores 0. Shared by {@link assessTextCoherence}. */
203
+ function unmatchedClosers(text) {
204
+ let depth = 0;
205
+ let orphans = 0;
206
+ for (const ch of text) {
207
+ if (ch === '(' || ch === '[' || ch === '{')
208
+ depth++;
209
+ else if (ch === ')' || ch === ']' || ch === '}') {
210
+ if (depth > 0)
211
+ depth--;
212
+ else
213
+ orphans++;
214
+ }
215
+ }
216
+ return orphans;
217
+ }
218
+ //# sourceMappingURL=textCoherence.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"textCoherence.js","sourceRoot":"","sources":["../../src/evermind/textCoherence.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAE1B;qFACqF;AACrF,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,CAAC;AAgC5C,2DAA2D;AAC3D,MAAM,cAAc,GAAqC;IACvD,OAAO,EAAE,4BAA4B;IACrC,mBAAmB,EAAE,4EAA4E;IACjG,YAAY,EAAE,wDAAwD;IACtE,gBAAgB,EAAE,6FAA6F;IAC/G,uBAAuB,EAAE,4CAA4C;IACrE,mBAAmB,EAAE,yEAAyE;IAC9F,WAAW,EAAE,qFAAqF;CACnG,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,OAAyB,EAAE;IAC3E,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC;IAE7B,8EAA8E;IAC9E,2EAA2E;IAC3E,kEAAkE;IAClE,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IACtD,iFAAiF;IACjF,+EAA+E;IAC/E,iCAAiC;IACjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,EAAE,CAAC;IAEpC,2EAA2E;IAC3E,uEAAuE;IACvE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;IAC5E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC;IAErE,gFAAgF;IAChF,mFAAmF;IACnF,sFAAsF;IACtF,iFAAiF;IACjF,sFAAsF;IACtF,0DAA0D;IAC1D,IAAI,mBAAmB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAE9D,mFAAmF;IACnF,uFAAuF;IACvF,mFAAmF;IACnF,sFAAsF;IACtF,uFAAuF;IACvF,sFAAsF;IACtF,IAAI,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEnE,OAAO,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,CAAS,EAAE,KAAe,EAAE,IAAsB;IAC1E,qFAAqF;IACrF,+EAA+E;IAC/E,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,GAAG;aACb,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;aAC1B,WAAW,EAAE;aACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC5F,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IACD,oFAAoF;IACpF,wCAAwC;IACxC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,EAAE,CAAC;IAE5E,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,EAAE,CAAC,CAAC,oDAAoD;IAEpF,oFAAoF;IACpF,iFAAiF;IACjF,+EAA+E;IAC/E,gFAAgF;IAChF,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,QAAQ,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAEjF,mFAAmF;IACnF,gFAAgF;IAChF,gFAAgF;IAChF,oFAAoF;IACpF,gFAAgF;IAChF,mFAAmF;IACnF,sFAAsF;IACtF,6CAA6C;IAC7C,IAAI,CAAC,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACzH,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;QACpE,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,GAAG,cAAc,CAAC,WAAW,CAAC,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,QAAQ,gBAAgB;SAC3F,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,EAAE,CAAC;AAChB,CAAC;AAED,MAAM,IAAI,GAAG,GAAqB,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;AACrF,MAAM,IAAI,GAAG,CAAC,OAAyB,EAAoB,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAE9H;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAyB,EAAE;IAC7E,OAAO,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC;AAClD,CAAC;AAED;;4FAE4F;AAC5F,MAAM,UAAU,cAAc,CAAC,IAA+B,EAAE,OAAyB,EAAE;IACzF,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,CAAC,MAAM,GAAG,yBAAyB,EAAE,CAAC;QACzC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gCAAgC,yBAAyB,aAAa,EAAE,CAAC;IAC/H,CAAC;IACD,OAAO,mBAAmB,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,mBAAmB,CAAC,KAAwB;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC;YAAE,SAAS;QAChD,MAAM,EAAE,CAAC;QACT,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,sFAAsF;IACtF,oDAAoD;IACpD,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7B,IAAI,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IAE3C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE;QAAE,IAAI,CAAC,GAAG,QAAQ;YAAE,QAAQ,GAAG,CAAC,CAAC;IAC9D,OAAO,MAAM,IAAI,EAAE,IAAI,QAAQ,GAAG,MAAM,GAAG,GAAG,CAAC;AACjD,CAAC;AAED;oFACoF;AACpF,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,EAAE,CAAC;aAC/C,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAChD,IAAI,KAAK,GAAG,CAAC;gBAAE,KAAK,EAAE,CAAC;;gBAClB,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}