@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +108 -0
  2. package/dist/cjs/analytics/ufo.js +2 -1
  3. package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
  4. package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
  5. package/dist/cjs/pm-plugins/autocomplete-plugin.js +498 -160
  6. package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
  7. package/dist/cjs/pm-plugins/debug-mode.js +105 -7
  8. package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
  9. package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
  10. package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
  11. package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
  12. package/dist/cjs/pm-plugins/text-predictor.js +2099 -309
  13. package/dist/es2019/analytics/ufo.js +2 -0
  14. package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
  15. package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
  16. package/dist/es2019/pm-plugins/autocomplete-plugin.js +489 -152
  17. package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
  18. package/dist/es2019/pm-plugins/debug-mode.js +102 -7
  19. package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
  20. package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
  21. package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
  22. package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
  23. package/dist/es2019/pm-plugins/text-predictor.js +1692 -200
  24. package/dist/esm/analytics/ufo.js +2 -1
  25. package/dist/esm/pm-plugins/artifact-loader.js +128 -0
  26. package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
  27. package/dist/esm/pm-plugins/autocomplete-plugin.js +501 -163
  28. package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
  29. package/dist/esm/pm-plugins/debug-mode.js +104 -6
  30. package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
  31. package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
  32. package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
  33. package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
  34. package/dist/esm/pm-plugins/text-predictor.js +2101 -313
  35. package/dist/types/analytics/ufo.d.ts +1 -0
  36. package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
  37. package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
  38. package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
  39. package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
  40. package/dist/types/pm-plugins/debug-mode.d.ts +55 -1
  41. package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
  42. package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
  43. package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
  44. package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
  45. package/dist/types/pm-plugins/text-predictor.d.ts +156 -3
  46. package/package.json +2 -2
  47. package/src/analytics/ufo.ts +5 -0
  48. package/src/pm-plugins/artifact-loader.ts +92 -0
  49. package/src/pm-plugins/artifacts-manifest.ts +3 -0
  50. package/src/pm-plugins/autocomplete-plugin.ts +584 -154
  51. package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
  52. package/src/pm-plugins/debug-mode.ts +134 -7
  53. package/src/pm-plugins/ghost-text-decoration.ts +13 -4
  54. package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
  55. package/src/pm-plugins/scoring-pipeline.ts +167 -17
  56. package/src/pm-plugins/slow-lane-client.ts +147 -62
  57. package/src/pm-plugins/text-predictor.ts +2119 -254
  58. package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
  59. package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
  60. package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
  61. package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
  62. package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
  63. package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
  64. package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  65. package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  66. package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
  67. package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  68. package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  69. package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
  70. package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
  71. package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
  72. package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
  73. package/scripts/gen_first_token_to_words.py +0 -170
  74. package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
  75. package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
  76. package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
  77. package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
  78. package/src/pm-plugins/slow-lane-client/package.json +0 -8
  79. package/src/pm-plugins/text-predictor/package.json +0 -8
@@ -4,7 +4,7 @@ import _defineProperty from "@babel/runtime/helpers/defineProperty";
4
4
  *
5
5
  * Two prediction modes:
6
6
  * 1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
7
- * 2. Mid-word (≥3 chars) → trie prefix search → scoring pipeline → top result
7
+ * 2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
8
8
  *
9
9
  * Scoring is delegated to scoring-pipeline.ts which handles:
10
10
  * Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
@@ -13,18 +13,19 @@ import _defineProperty from "@babel/runtime/helpers/defineProperty";
13
13
  * Falls back to cold mode (freq-only) when vectors not yet loaded.
14
14
  *
15
15
  * Session personalization (L1): words the user types are incrementally boosted
16
- * via incrementSessionFreq(), called on word boundaries from the plugin.
16
+ * via incrementSessionFreq(), called on word boundaries from the plugin, and
17
+ * words in ingested context text via ingestDocumentPage(). What the session has
18
+ * boosted is visible at any time from the console: `__atlCtcDebug__.session()`,
19
+ * or `__atlCtcDebug__.session('poll')` for one family — see inspectSessionBoosts.
17
20
  */
18
21
 
19
22
  import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
20
-
21
- // The vocabulary, L3 and word-index payloads are fetched from the CDN in
22
- // loadDefaultVocabulary / loadVectorsAsync so their (large) contents never enter
23
- // the editor's bundle and only load when autocomplete is initialised.
24
- import { ARTIFACT_NAME, fetchArtifactJson, getArtifactUrl } from './artifacts-manifest';
25
- import { isAutocompleteDebugEnabled } from './debug-mode';
26
- import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
27
- import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
23
+ import { fetchAutocompleteArtifactBinary, fetchAutocompleteArtifactJson } from './artifact-loader';
24
+ import { ARTIFACT_NAME } from './artifacts-manifest';
25
+ import { createCanonicalContextPositionCache, deriveCanonicalCandidateContext, deriveWhitespaceBoundaryContext, logSoftmaxAt, logSumExp, selectBoundaryPrimeRequests } from './canonical-lm-scoring';
26
+ import { CTC_STYLES, ctcSection, ctcTag, isAutocompleteDebugEnabled, isAutocompleteDebugVerbose, registerCtcSessionInspector } from './debug-mode';
27
+ import { loadGrammarDataAsync, rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE, MIN_WINNER_MARGIN } from './scoring-pipeline';
28
+ import { getBoundaryLmState, getCanonicalSurfaceCount, getCanonicalSurfaceTokenIds, getDefaultSlowLaneClientStatus, getProgressiveSurfaceEvidence, getStoredContextInput, getStoredContextVector, getStoredLmLogits, getSurfaceScore, isCanonicalSurfaceScoringSupported, primeBoundaryLm, requestProgressiveSurfaceScores } from './slow-lane-client';
28
29
 
29
30
  // ─── Constants ───────────────────────────────────────────────────────────────
30
31
 
@@ -36,14 +37,153 @@ const WHITESPACE_SPLIT_REGEX = /\s+/;
36
37
  const SENTENCE_BOUNDARY_REGEX = /[\n.?!]+/;
37
38
  // eslint-disable-next-line require-unicode-regexp
38
39
  const TRAILING_WHITESPACE_REGEX = /\s$/;
39
- const MIN_PREFIX_LENGTH = 3;
40
+ const TRAILING_SURFACE_TOKEN_REGEX = /[\p{L}\p{N}_'-]+$/u;
41
+ const SURFACE_TOKEN_REGEX = /[\p{L}\p{N}_'-]+/gu;
42
+ const ONLY_WHITESPACE_REGEX = /^\s+$/u;
43
+ const UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH = 1;
44
+ const MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH = 1;
45
+ const DISPLAY_MIN_PREFIX_LENGTH = 3;
40
46
  const MAX_CANDIDATES = 200;
41
47
  const CONTEXT_WORDS = 10;
42
- const MIN_SCORE_THRESHOLD = 0.35;
43
48
  const L3_BASELINE_FREQ = 0.001;
49
+ // Max tokens in a completion term (bigram=2, phrase=3). Mirrors the producer's
50
+ // `phrase_max_words` default so the FE window never exceeds what was mined.
51
+ const PHRASE_MAX_WORDS = 3;
52
+ // Per-window cap on phrase-trie subtree collection (keeps the hot path bounded).
53
+ const MAX_PHRASE_CANDIDATES = 50;
54
+ /**
55
+ * Share of its context shortlist's probability mass the model must put on a
56
+ * surface before that surface may be shown.
57
+ *
58
+ * Candidates competing at the same boundary are normalised into a posterior
59
+ * over that shortlist, so this reads directly as "the model is at least this
60
+ * sure". Measuring distance from the best candidate instead cannot express
61
+ * being unsure: the leader scores full marks whether it won by a nose or a
62
+ * mile, which pushes the entire question of how contested a context is onto
63
+ * `MIN_WINNER_MARGIN` — and there the blended score's only remaining spread is
64
+ * the corpus prior, the weakest ranking signal we have.
65
+ *
66
+ * Each extra token raises the bar. A wrong multi-word ghost costs the reader
67
+ * more to notice and undo than a wrong single word, and it is offered from the
68
+ * same keystroke, so precision has to be bought per token of commitment.
69
+ */
70
+ const MIN_LM_POSTERIOR = {
71
+ word: 0.5,
72
+ bigram: 0.6,
73
+ phrase: 0.7
74
+ };
75
+ /**
76
+ * How many scored candidates a context must hold before its leader may show.
77
+ *
78
+ * A posterior is a share of a shortlist, so a shortlist of one hands its only
79
+ * member 1.0 whatever the model thinks of it — `logSumExp` of a single value
80
+ * returns that value. Such a ghost sits in the top bucket without having beaten
81
+ * anything, and it is the gate's blind spot rather than a case the gate ruled
82
+ * on: 1 in 14 of them were accepted against 34% for word ghosts overall, and
83
+ * every phrase shown through this hole was rejected.
84
+ *
85
+ * Counted over pool membership rather than over the normaliser, because those
86
+ * two deliberately differ. Where a pool holds a surface and its own extension
87
+ * the chain rule leaves one term in the denominator, and the resulting 1.0 is
88
+ * earned — whichever the user meant, the shorter form is a correct ghost.
89
+ *
90
+ * Waived where the context never held a second candidate at all. The 1-in-14
91
+ * acceptance above was measured over a population dominated by pools that read
92
+ * as one because rivals had not come back, and it is that absence of evidence
93
+ * the floor is for. A vocabulary offering a single continuation of a long
94
+ * distinctive prefix is the opposite situation, and it is also where a ghost
95
+ * saves the most keystrokes, so refusing it spends the most to learn the least.
96
+ */
97
+ const MIN_SCORED_POOL_SIZE = 2;
98
+ /**
99
+ * Mean per-token log-probability a surface must hold, in nats, independently of
100
+ * anything it was competing against.
101
+ *
102
+ * Every other display test is relative: a posterior is a share of a shortlist,
103
+ * and a winner margin is a distance from a rival. So a surface reaches the
104
+ * screen by being the best of what happened to be in its pool, and nothing ever
105
+ * asks whether the model finds it plausible at all. That gap was tolerable while
106
+ * little was being read, and stopped being tolerable once the scheduler started
107
+ * reading roughly twice as much per decision: shown-per-decision went from 3.2%
108
+ * to 7.4% with no threshold moved, and acceptance of what showed fell from 32%
109
+ * to 24.5%. More reading does not weaken a relative test — it hands it far more
110
+ * pools to be the winner of, and the marginal winner is worse than the average.
111
+ *
112
+ * Set at the point where a uniform distribution over the model's vocabulary
113
+ * sits: 49,152 tokens, so `ln(1/49152)` is about -10.8 nats. Below it the model
114
+ * assigns the surface less mass than a token drawn at random, which is the only
115
+ * statement about a surface that needs no reference to what it competed with.
116
+ *
117
+ * It was first tried at -8, chosen from the bucket where acceptance was measured
118
+ * to fall away, and that reasoning does not survive more data. Acceptance against
119
+ * this quantity turned out to be a smooth slope — about 19% below -8, 26% from -8
120
+ * to -6, then flat near 35% above - with no discontinuity to place a threshold
121
+ * at. A floor on a smooth slope is a coverage dial rather than a test: -8 removed
122
+ * a band converting at 26% and cost 3.7 points of show rate, for an acceptance
123
+ * movement no sample of this size can resolve. Anchoring to uniform instead makes
124
+ * the value an argument about what is indefensible rather than a point picked off
125
+ * a histogram, which is what stops it from being retuned every batch.
126
+ *
127
+ * Applied to the mean over the verified prefix, the same quantity
128
+ * `ghostsByMeanLogProb` buckets, and deliberately not to
129
+ * `meanTokenLogProbabilityUpperBound`. The bound is the right instrument for
130
+ * keeping a candidate in the race, where dropping something that could still
131
+ * clear the bar would be an error, but it is systematically higher than the
132
+ * quantity acceptance is measured against. Judging a surface on the prefix that
133
+ * was actually read is also what every other test here does.
134
+ *
135
+ * One value across all three shapes. Splitting it per shape needs an acceptance
136
+ * curve per shape, and 19 bigrams and 1 phrase across five sessions cannot
137
+ * support one.
138
+ */
139
+ const MIN_MEAN_TOKEN_LOG_PROBABILITY = -10.8;
140
+ // Tab replaces the whole remaining suffix, so an N-character completion saves
141
+ // N-1 keystrokes: at one character it saves nothing at all and at two it saves
142
+ // one, which does not repay noticing the ghost and reaching for Tab. Offering
143
+ // them anyway spends the acceptance-rate denominator on completions nobody
144
+ // wants and teaches the user to stop reading ghosts. `DISPLAY_MIN_PREFIX_LENGTH`
145
+ // is the same guard on the side the user has already typed.
146
+ const MIN_SUGGESTION_LENGTH = 3;
147
+ /**
148
+ * How many leading tokens of a surface must be scored before it may be shown.
149
+ *
150
+ * Scoring a token costs one model round trip, and a round trip costs about the
151
+ * same no matter how much is in it, so the price of a candidate is its token
152
+ * count. Demanding every token therefore caps what can ever reach the screen at
153
+ * roughly two tokens inside the decision budget — which is why no phrase has
154
+ * ever been displayed. Verifying a fixed prefix makes the cost of a candidate
155
+ * independent of its length.
156
+ */
157
+ const REQUIRED_VERIFIED_TOKENS = 2;
158
+ /**
159
+ * Extra winner margin demanded of a surface judged on a verified prefix rather
160
+ * than on every one of its tokens, to buy back the precision given up by
161
+ * leaving the tail unread.
162
+ */
163
+ const PARTIAL_EVIDENCE_MARGIN_PREMIUM = 0.08;
164
+ const ARBITRATION_MODE = 'confidence-v2';
165
+ const DEBUG_TEXT_TAIL_CHARS = 120;
44
166
 
45
167
  // ─── Types ───────────────────────────────────────────────────────────────────
46
168
 
169
+ /**
170
+ * Which constraint stopped an evaluation from putting a ghost on screen.
171
+ *
172
+ * Each one implies different work: `below-posterior-gate` is a threshold to
173
+ * calibrate, `winner-margin` is two candidates the model cannot separate,
174
+ * `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
175
+ * is vocabulary coverage.
176
+ */
177
+
178
+ /**
179
+ * A candidate paired with the length of the already-typed prefix it completes.
180
+ * For a single word this is the current partial token length; for a phrase it
181
+ * is the length of the matched multi-word window (previous words + partial),
182
+ * so the ghost tail is `word.slice(matchedPrefixLen)`.
183
+ */
184
+
185
+ /** Optional per-term metadata carried by bigram/phrase completion terms. */
186
+
47
187
  class TrieNode {
48
188
  constructor() {
49
189
  _defineProperty(this, "children", new Map());
@@ -52,6 +192,12 @@ class TrieNode {
52
192
  _defineProperty(this, "docFreq", 0);
53
193
  _defineProperty(this, "authorFreq", 0);
54
194
  _defineProperty(this, "sessionFreq", 0);
195
+ // Term-type + phrase metadata. Plain words keep `termType='word'` and null
196
+ // metadata; bigram/phrase nodes carry the producer-shipped POS + association.
197
+ _defineProperty(this, "termType", 'word');
198
+ _defineProperty(this, "posSeq", null);
199
+ _defineProperty(this, "headPos", null);
200
+ _defineProperty(this, "association", null);
55
201
  }
56
202
  }
57
203
  class WeightedWordTrie {
@@ -60,7 +206,8 @@ class WeightedWordTrie {
60
206
  /** Highest tenantFreq seen — used to normalize freq scores at query time */
61
207
  _defineProperty(this, "maxTenantFreq", 1);
62
208
  }
63
- insert(word, tenantFreq, docFreq, authorFreq) {
209
+ insert(word, tenantFreq, docFreq, authorFreq, meta) {
210
+ var _meta$termType, _meta$posSeq, _meta$headPos, _meta$association;
64
211
  let node = this.root;
65
212
  for (const char of word.toLowerCase()) {
66
213
  let next = node.children.get(char);
@@ -74,6 +221,10 @@ class WeightedWordTrie {
74
221
  node.tenantFreq = tenantFreq;
75
222
  node.docFreq = docFreq;
76
223
  node.authorFreq = authorFreq;
224
+ node.termType = (_meta$termType = meta === null || meta === void 0 ? void 0 : meta.termType) !== null && _meta$termType !== void 0 ? _meta$termType : 'word';
225
+ node.posSeq = (_meta$posSeq = meta === null || meta === void 0 ? void 0 : meta.posSeq) !== null && _meta$posSeq !== void 0 ? _meta$posSeq : null;
226
+ node.headPos = (_meta$headPos = meta === null || meta === void 0 ? void 0 : meta.headPos) !== null && _meta$headPos !== void 0 ? _meta$headPos : null;
227
+ node.association = (_meta$association = meta === null || meta === void 0 ? void 0 : meta.association) !== null && _meta$association !== void 0 ? _meta$association : null;
77
228
  if (tenantFreq > this.maxTenantFreq) {
78
229
  this.maxTenantFreq = tenantFreq;
79
230
  }
@@ -150,6 +301,44 @@ class WeightedWordTrie {
150
301
  node.sessionFreq += 1;
151
302
  return true;
152
303
  }
304
+
305
+ /**
306
+ * Every word carrying a session boost, optionally limited to one prefix's
307
+ * subtree. Unlike `getCandidates` a word equal to the prefix is included,
308
+ * since the question here is what the session holds rather than what could
309
+ * still be typed.
310
+ *
311
+ * Walks the trie instead of reading an index, so nothing has to be kept in
312
+ * step on the write path for the sake of being able to ask.
313
+ */
314
+ collectSessionBoosted(prefix = '') {
315
+ let node = this.root;
316
+ for (const char of prefix.toLowerCase()) {
317
+ const next = node.children.get(char);
318
+ if (!next) {
319
+ return [];
320
+ }
321
+ node = next;
322
+ }
323
+ const boosted = [];
324
+ const stack = [node];
325
+ while (stack.length > 0) {
326
+ const current = stack.pop();
327
+ if (!current) {
328
+ continue;
329
+ }
330
+ if (current.word !== null && current.sessionFreq > 0) {
331
+ boosted.push({
332
+ word: current.word,
333
+ node: current
334
+ });
335
+ }
336
+ for (const child of current.children.values()) {
337
+ stack.push(child);
338
+ }
339
+ }
340
+ return boosted;
341
+ }
153
342
  }
154
343
 
155
344
  // L1/L2 Trie (Session + Atlassian Domain)
@@ -158,6 +347,15 @@ const wordTrie = new WeightedWordTrie();
158
347
  // L3 Trie (General English Fallback)
159
348
  const l3Trie = new WeightedWordTrie();
160
349
 
350
+ /**
351
+ * L2 phrase/bigram trie. Kept SEPARATE from `wordTrie` so single-word
352
+ * completion + L3 gap-fill stay untouched; multi-word units are only surfaced
353
+ * by the (12b) multiword prefix-window matcher. Keys are the full normalized
354
+ * surface string (spaces included), e.g. `"return on investment"`, so a lookup
355
+ * is a plain char-trie walk over the whole phrase.
356
+ */
357
+ const phraseTrie = new WeightedWordTrie();
358
+
161
359
  /**
162
360
  * Loads the General English vocabulary.
163
361
  * expects a simple array of strings: ["about", "above", "actually", ...]
@@ -168,10 +366,8 @@ export const initL3Vocabulary = l3Words => {
168
366
  // loses to any domain word in Stage 1, but still scores above 0.
169
367
  l3Trie.insert(word, L3_BASELINE_FREQ, 0, 0);
170
368
  }
171
- if (isAutocompleteDebugEnabled()) {
172
- // eslint-disable-next-line no-console
173
- console.log(`[text-predictor] L3 General English loaded: ${l3Words.length} words`);
174
- }
369
+ recallGeneration++;
370
+ ctcTag('init', `L3 general English loaded: ${l3Words.length} words`);
175
371
  };
176
372
 
177
373
  // const bigramMap: Map<string, Record<string, number>> = new Map(
@@ -181,8 +377,50 @@ export const initL3Vocabulary = l3Words => {
181
377
  let isInitialized = false;
182
378
  let vectorStore = null;
183
379
  let vectorsLoadStarted = false;
380
+
381
+ // Phrase/bigram artifacts load independently of the word vocabulary so a
382
+ // missing/late `bigrams.json`/`phrases.json` never breaks word completion.
383
+ let phrasesLoadStarted = false;
384
+ let phrasesLoaded = false;
385
+ let phraseTermCount = 0;
386
+ // Per-type frequency maxima, tracked separately because bigrams and phrases
387
+ // share one trie but have very different absolute frequency ranges (bigrams are
388
+ // far more frequent than 3-word phrases). Normalizing each type by its OWN max
389
+ // stops phrase freq scores from being crushed against the top bigram's count.
390
+ let maxBigramFreq = 1;
391
+ let maxPhraseFreq = 1;
184
392
  let lastPredictionDebug = null;
185
393
 
394
+ // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
395
+ // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
396
+ // generally keep a just-accepted unit from being re-offered:
397
+ // • repetition: never surface a candidate whose full surface already equals the
398
+ // run of tokens immediately before the cursor (handled inline in predict()).
399
+ // • cooldown: after an accept, briefly suppress re-offering that same unit. The
400
+ // cooldown lifts only once BOTH the keystroke budget and the time window have
401
+ // elapsed (either one still counting keeps it active).
402
+ const COOLDOWN_KEYSTROKES = 5;
403
+ const COOLDOWN_MS = 2000;
404
+
405
+ // Active post-accept cooldown, or null when none.
406
+ let acceptCooldown = null;
407
+
408
+ /**
409
+ * Start a short cooldown for the exact surface the editor inserted. The caller
410
+ * passes the committed snapshot's surface so background re-ranking can never
411
+ * move cooldown bookkeeping away from what the user actually accepted.
412
+ */
413
+ export const noteSuggestionAccepted = surface => {
414
+ const normalizedSurface = surface.trim().toLowerCase();
415
+ if (normalizedSurface) {
416
+ acceptCooldown = {
417
+ predictionsSince: 0,
418
+ surface: normalizedSurface,
419
+ ts: performance.now()
420
+ };
421
+ }
422
+ };
423
+
186
424
  /** Get vector for a word from the store. */
187
425
  const getWordVector = word => {
188
426
  if (!vectorStore) {
@@ -260,6 +498,111 @@ const extractPreviousWord = text => {
260
498
  return words.length >= 2 ? words[words.length - 2] : '';
261
499
  };
262
500
 
501
+ /**
502
+ * Collect phrase/bigram completion candidates for the multi-word window ending
503
+ * at the current partial token.
504
+ *
505
+ * Builds windows of the last `2..PHRASE_MAX_WORDS` tokens of the current
506
+ * sentence (widest first) and prefix-searches the phrase trie for each, so
507
+ * `quarterly pla` matches `quarterly planning meeting`. Windows never cross a
508
+ * sentence boundary (mirrors `extractPreviousWord`). Each result records the
509
+ * matched window length so the caller can compute the ghost tail.
510
+ *
511
+ * :params:
512
+ * trimmed: Text before the cursor, trailing whitespace already removed
513
+ * :returns:
514
+ * Matched phrase candidates (deduped, widest-window match wins per term)
515
+ */
516
+ const getPhraseCandidates = trimmed => {
517
+ // Records every prefix window we probe and its match count so the (verbose)
518
+ // debug line makes it obvious whether the phrase path found nothing because the
519
+ // trie is empty, or because no phrase begins with the typed window.
520
+ const windowsTried = [];
521
+ const logPhrasePath = skipReason => {
522
+ if (!isAutocompleteDebugVerbose()) {
523
+ return;
524
+ }
525
+ const detail = skipReason ? skipReason : `windows: ${windowsTried.map(w => `"${w.window}"→${w.matches}`).join(', ') || '(none)'}`;
526
+ ctcTag('phrase', `loaded=${phraseTermCount} (${phrasesLoaded ? 'ready' : 'not loaded'}) · ${detail}`, CTC_STYLES.section);
527
+ };
528
+ if (phraseTermCount === 0) {
529
+ logPhrasePath('phrase trie empty — bigrams.json/phrases.json not loaded (or 0 terms)');
530
+ return [];
531
+ }
532
+ const tokenMatches = Array.from(trimmed.matchAll(SURFACE_TOKEN_REGEX));
533
+ const lastMatch = tokenMatches[tokenMatches.length - 1];
534
+ const lastStart = lastMatch === null || lastMatch === void 0 ? void 0 : lastMatch.index;
535
+ if (!lastMatch || lastStart === undefined || lastStart + lastMatch[0].length !== trimmed.length) {
536
+ logPhrasePath('no tokens in current sentence');
537
+ return [];
538
+ }
539
+ const trailingTokens = [{
540
+ start: lastStart,
541
+ value: lastMatch[0]
542
+ }];
543
+ for (let i = tokenMatches.length - 2; i >= 0; i--) {
544
+ const current = trailingTokens[0];
545
+ const match = tokenMatches[i];
546
+ const start = match.index;
547
+ if (start === undefined) {
548
+ break;
549
+ }
550
+ const end = start + match[0].length;
551
+ const gap = trimmed.slice(end, current.start);
552
+ if (!ONLY_WHITESPACE_REGEX.test(gap)) {
553
+ break;
554
+ }
555
+ trailingTokens.unshift({
556
+ start,
557
+ value: match[0]
558
+ });
559
+ if (trailingTokens.length >= PHRASE_MAX_WORDS) {
560
+ break;
561
+ }
562
+ }
563
+ const partial = trailingTokens[trailingTokens.length - 1].value;
564
+ const results = [];
565
+ const seen = new Set();
566
+ const maxPrev = trailingTokens.length - 1;
567
+
568
+ // Widest window first so the most specific (longest-context) match wins the
569
+ // dedup for a given phrase term. `prev` is the number of *preceding* complete
570
+ // words included ahead of the current partial:
571
+ // prev ≥ 1 → "continue a phrase I've started" (e.g. "root c" → root cause).
572
+ // Preceding-word context is strong, so a short partial is fine.
573
+ // prev = 0 → "the partial is a phrase's FIRST word" (e.g. "root" → root
574
+ // cause). No preceding context, so require the partial to be a
575
+ // one character so grouped LM work can begin early. UI remains
576
+ // gated at DISPLAY_MIN_PREFIX_LENGTH.
577
+ for (let prev = maxPrev; prev >= 0; prev--) {
578
+ if (prev === 0 && partial.length < MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH) {
579
+ continue;
580
+ }
581
+ const windowTokens = trailingTokens.slice(-(prev + 1));
582
+ const windowPrefix = windowTokens.map(token => token.value).join(' ').toLowerCase();
583
+ const windowLen = windowPrefix.length;
584
+ const matches = phraseTrie.getCandidates(windowPrefix, MAX_PHRASE_CANDIDATES);
585
+ windowsTried.push({
586
+ window: windowPrefix,
587
+ matches: matches.length
588
+ });
589
+ for (const match of matches) {
590
+ if (seen.has(match.word)) {
591
+ continue;
592
+ }
593
+ seen.add(match.word);
594
+ results.push({
595
+ node: match.node,
596
+ word: match.word,
597
+ matchedPrefixLen: windowLen,
598
+ surfaceStart: windowTokens[0].start
599
+ });
600
+ }
601
+ }
602
+ logPhrasePath();
603
+ return results;
604
+ };
605
+
263
606
  // ─── Debug Helpers ───────────────────────────────────────────────────────────
264
607
 
265
608
  /**
@@ -268,11 +611,25 @@ const extractPreviousWord = text => {
268
611
  * wordCount: number of words in vector store (0 if not loaded)
269
612
  */
270
613
  export const getPredictorStatus = () => {
614
+ const slowLaneStatus = getDefaultSlowLaneClientStatus();
271
615
  return {
616
+ canonicalScoringSupported: slowLaneStatus.canonicalScoringSupported,
617
+ canonicalSurfaceCount: slowLaneStatus.canonicalSurfaceCount,
618
+ clientId: slowLaneStatus.clientId,
619
+ clientKind: slowLaneStatus.clientKind,
620
+ localModelReady: slowLaneStatus.localModelReady,
621
+ slowLaneRegistered: slowLaneStatus.registered,
272
622
  vectorsLoaded: vectorStore !== null,
273
623
  wordCount: vectorStore ? Object.keys(vectorStore.wordIndex).length : 0,
274
624
  vectorsLoadStarted,
275
- isInitialized
625
+ isInitialized,
626
+ phrasesLoaded,
627
+ phraseCount: phraseTermCount,
628
+ maxFreqByType: {
629
+ word: wordTrie.maxTenantFreq,
630
+ bigram: maxBigramFreq,
631
+ phrase: maxPhraseFreq
632
+ }
276
633
  };
277
634
  };
278
635
 
@@ -288,6 +645,41 @@ export const initVocabulary = vocabulary => {
288
645
  wordTrie.insert(term.word, term.freq, term.docFreq, term.authorFreq);
289
646
  }
290
647
  isInitialized = true;
648
+ recallGeneration++;
649
+ };
650
+
651
+ /**
652
+ * Insert a producer-shipped bigram/phrase artifact into the phrase trie.
653
+ *
654
+ * :params:
655
+ * artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
656
+ * termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
657
+ * :returns:
658
+ * The number of terms inserted
659
+ */
660
+ export const initPhrases = (artifact, termType) => {
661
+ let count = 0;
662
+ for (const [term, stats] of Object.entries(artifact)) {
663
+ var _stats$pos_seq, _stats$head_pos, _stats$association;
664
+ phraseTrie.insert(term, stats.freq, stats.doc_freq, stats.author_freq, {
665
+ termType,
666
+ posSeq: (_stats$pos_seq = stats.pos_seq) !== null && _stats$pos_seq !== void 0 ? _stats$pos_seq : null,
667
+ headPos: (_stats$head_pos = stats.head_pos) !== null && _stats$head_pos !== void 0 ? _stats$head_pos : null,
668
+ association: (_stats$association = stats.association) !== null && _stats$association !== void 0 ? _stats$association : null
669
+ });
670
+ // Track each type's own frequency ceiling for per-type normalization.
671
+ if (termType === 'bigram') {
672
+ if (stats.freq > maxBigramFreq) {
673
+ maxBigramFreq = stats.freq;
674
+ }
675
+ } else if (stats.freq > maxPhraseFreq) {
676
+ maxPhraseFreq = stats.freq;
677
+ }
678
+ count++;
679
+ }
680
+ phraseTermCount += count;
681
+ recallGeneration++;
682
+ return count;
291
683
  };
292
684
 
293
685
  /**
@@ -321,107 +713,222 @@ export const ingestDocumentPage = pageContent => {
321
713
  }
322
714
  }
323
715
  if (isAutocompleteDebugEnabled() && validBoostedWords.size > 0) {
324
- // eslint-disable-next-line no-console
325
- console.groupCollapsed(`%c[L1 Session] %cPrimed ${validBoostedWords.size} valid dictionary words from page`, 'color: #00b8d9; font-weight: bold;', 'color: inherit; font-style: italic;');
326
- // eslint-disable-next-line no-console
327
- console.dir(Array.from(validBoostedWords).sort());
328
- // eslint-disable-next-line no-console
329
- console.groupEnd();
716
+ ctcTag('init', `L1 session primed ${validBoostedWords.size} words from page · __atlCtcDebug__.session() to inspect`, CTC_STYLES.brand);
717
+ if (isAutocompleteDebugVerbose()) {
718
+ // eslint-disable-next-line no-console
719
+ console.dir(Array.from(validBoostedWords).sort());
720
+ }
330
721
  }
331
722
  };
723
+
724
+ /**
725
+ * How many boosted words `inspectSessionBoosts` lists.
726
+ *
727
+ * A page ingest can boost thousands, and a list that long is not read. The
728
+ * strongest boosts are the ones that change an ordering, and `boosted` still
729
+ * reports the full size, so the cap loses nothing but volume.
730
+ */
731
+ const MAX_LISTED_SESSION_WORDS = 50;
732
+ /**
733
+ * Read the session's L1 boosts, optionally narrowed to a prefix.
734
+ *
735
+ * Installed as `__atlCtcDebug__.session()`, with `__atlCtcDebug__.session('poll')`
736
+ * to ask about one family. Returned rather than logged, so the console renders it
737
+ * as an inspectable object and a caller can assert on it.
738
+ *
739
+ * Only words the vocabulary already holds can carry a boost, because both writers
740
+ * go through `incrementSessionFreq` and it only finds existing nodes. An ingested
741
+ * word absent from the vocabulary is therefore missing from here and always will
742
+ * be.
743
+ */
744
+ export const inspectSessionBoosts = prefix => {
745
+ const boosted = wordTrie.collectSessionBoosted(prefix !== null && prefix !== void 0 ? prefix : '');
746
+ return {
747
+ boosted: boosted.length,
748
+ limit: MAX_LISTED_SESSION_WORDS,
749
+ ...(prefix === undefined ? {} : {
750
+ prefix
751
+ }),
752
+ words: boosted.sort((a, b) => b.node.sessionFreq - a.node.sessionFreq || a.word.localeCompare(b.word, 'en', {
753
+ numeric: true,
754
+ sensitivity: 'base'
755
+ })).slice(0, MAX_LISTED_SESSION_WORDS).map(({
756
+ node,
757
+ word
758
+ }) => ({
759
+ sessionFreq: node.sessionFreq,
760
+ sessionOnly: node.tenantFreq === 0,
761
+ surface: word,
762
+ tenantFreq: node.tenantFreq
763
+ }))
764
+ };
765
+ };
766
+
767
+ // At module scope so the console answers before the first keystroke, which is
768
+ // when someone reaching for it usually asks.
769
+ registerCtcSessionInspector(inspectSessionBoosts);
770
+
771
+ /**
772
+ * Result of a prediction: the ghost tail to insert plus an immutable record of
773
+ * the evidence that authorized the UI commitment.
774
+ */
775
+
776
+ // ─── Recall memo ─────────────────────────────────────────────────────────────
777
+
778
+ /** Trie recall plus the canonical context derived for each matched term. */
779
+
780
+ /**
781
+ * Bumped whenever an artifact load changes what the tries can return, which is
782
+ * the only way the candidate *set* for a given prefix can change.
783
+ *
784
+ * Session-frequency boosts are deliberately not counted: they mutate trie nodes
785
+ * in place, and the memo holds those nodes by reference, so a boost is already
786
+ * visible through a cached entry.
787
+ */
788
+ let recallGeneration = 0;
789
+ let recallMemo = null;
790
+ const computeCanonicalRecall = (trimmed, currentWord, currentWordStart) => {
791
+ // Start recall at one character so grouped causal work can overlap later
792
+ // keystrokes. Nothing is displayed until three characters.
793
+ const wordCandidates = currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH ? wordTrie.getCandidates(currentWord, MAX_CANDIDATES) : [];
794
+
795
+ // Gap-fill from the L3 general-English trie, requesting a full buffer so
796
+ // enough survive de-duplication against the L2 results.
797
+ if (currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH && wordCandidates.length < MAX_CANDIDATES) {
798
+ const l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
799
+ const existingWords = new Set(wordCandidates.map(c => c.word));
800
+ for (const l3c of l3Candidates) {
801
+ if (wordCandidates.length >= MAX_CANDIDATES) break;
802
+ if (!existingWords.has(l3c.word)) {
803
+ wordCandidates.push(l3c);
804
+ }
805
+ }
806
+ }
807
+
808
+ // Multi-word completion terms (bigrams/phrases) matched on the trailing
809
+ // window ending at the current partial token. Runs even when the unigram path
810
+ // was skipped for a short partial, so first-word/early phrase matches surface.
811
+ const phraseCandidates = getPhraseCandidates(trimmed);
812
+
813
+ // Unify: a word completes the current partial token; a phrase completes its
814
+ // matched multi-word window. Track the prefix length per term so the ghost
815
+ // tail is sliced correctly regardless of term type.
816
+ const matched = [...wordCandidates.map(({
817
+ word,
818
+ node
819
+ }) => ({
820
+ word,
821
+ node,
822
+ matchedPrefixLen: currentWord.length,
823
+ surfaceStart: currentWordStart
824
+ })), ...phraseCandidates];
825
+ const positionCache = createCanonicalContextPositionCache();
826
+ const canonicalMatched = matched.map(candidate => ({
827
+ ...candidate,
828
+ ...deriveCanonicalCandidateContext(trimmed, candidate.matchedPrefixLen, candidate.word, getCanonicalSurfaceTokenIds, candidate.surfaceStart, positionCache)
829
+ }));
830
+ const prefixLenByWord = new Map();
831
+ for (const m of canonicalMatched) {
832
+ if (!prefixLenByWord.has(m.word)) {
833
+ prefixLenByWord.set(m.word, m.matchedPrefixLen);
834
+ }
835
+ }
836
+ return {
837
+ canonicalMatched,
838
+ prefixLenByWord
839
+ };
840
+ };
841
+
842
+ /**
843
+ * Recall and canonical derivation for `trimmed`, reusing the previous result
844
+ * when nothing they depend on has changed.
845
+ *
846
+ * `predict()` runs several times for one keystroke — once when the decision
847
+ * opens and again for each async evidence signal that lands inside the budget —
848
+ * and only the evidence differs between those runs. Without this, every run
849
+ * re-walks the tries and re-derives a canonical context per candidate.
850
+ */
851
+ const getCanonicalRecall = (trimmed, currentWord, currentWordStart) => {
852
+ const surfaceCount = getCanonicalSurfaceCount();
853
+ if (recallMemo !== null && recallMemo.trimmed === trimmed && recallMemo.generation === recallGeneration && recallMemo.surfaceCount === surfaceCount) {
854
+ return recallMemo;
855
+ }
856
+ const recall = computeCanonicalRecall(trimmed, currentWord, currentWordStart);
857
+ recallMemo = {
858
+ ...recall,
859
+ generation: recallGeneration,
860
+ surfaceCount,
861
+ trimmed
862
+ };
863
+ return recall;
864
+ };
332
865
  export const predict = textBefore => {
866
+ var _trimmed$match$, _trimmed$match, _canonicalMatched$fin, _eligible$, _eligible$0$score, _eligible$2, _judgedEvidenceFor3, _prefixLenByWord$get2, _runtimeBySurface$get1, _judgedEvidenceFor$me3, _judgedEvidenceFor6, _contextTotals$get$le4, _contextTotals$get4, _selectedEvidence$ver, _selectedEvidence$tot, _selectedEvidence$ver2;
333
867
  if (!isInitialized) {
334
- // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
335
- // and skip this keystroke; the plugin also primes it on focus, so the tries
336
- // are usually ready before the user types.
337
- void loadDefaultVocabulary().catch(() => {});
868
+ // Vocabulary artifacts load asynchronously over the autocomplete gateway.
869
+ // Kick off the load and skip this keystroke; the plugin also primes it on
870
+ // focus, so the tries are usually ready before the user types.
871
+ void loadDefaultVocabulary({
872
+ source: 'predict'
873
+ }).catch(() => {});
338
874
  return null;
339
875
  }
340
876
  const t0 = performance.now();
341
877
 
342
- // ── Step 1: Bigram-based next-word suggestion at word boundary ───────────
343
- // if (textBefore.length > 0 && /\s$/u.test(textBefore)) {
344
- // const words = textBefore.toLowerCase().trimEnd().split(/\s+/u);
345
- // const prevWord = words[words.length - 1];
346
- // const nextWords = bigramMap.get(prevWord);
347
- // if (nextWords) {
348
- // const sorted = Object.entries(nextWords).sort((a, b) => b[1] - a[1]);
349
- // let bestWord = '';
350
- // for (const [word] of sorted) {
351
- // if (!isGrammarAllowed(prevWord, word)) {
352
- // continue;
353
- // }
354
- // bestWord = word;
355
- // break;
356
- // }
357
- // if (bestWord) {
358
- // if (debugMode) {
359
- // const latencyMs = performance.now() - t0;
360
- // console.log(
361
- // '%c[autocomplete] BIGRAM',
362
- // 'color:cyan',
363
- // '| "' + prevWord + '" -> "' + bestWord + '" | ' + latencyMs.toFixed(1) + 'ms',
364
- // );
365
- // }
366
- // return bestWord;
367
- // }
368
- // }
369
- // if (debugMode) {
370
- // console.log(
371
- // '%c[autocomplete] BIGRAM-MISS',
372
- // 'color:gray',
373
- // '| no bigram for "' + words[words.length - 1] + '", skipping prefix completion',
374
- // );
375
- // }
376
- // return null;
377
- // }
378
-
379
- // ── Step 2: Prefix completion (≥3 chars typed) ──────────────────────────
878
+ // Advance the post-accept cooldown once per prediction. It stays active until
879
+ // BOTH the keystroke budget and the time window have elapsed; while active it
880
+ // suppresses re-offering the just-accepted surface (see arbitration below).
881
+ let cooledSurface = null;
882
+ if (acceptCooldown) {
883
+ acceptCooldown.predictionsSince += 1;
884
+ const stillByKeys = acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES;
885
+ const stillByTime = performance.now() - acceptCooldown.ts < COOLDOWN_MS;
886
+ if (stillByKeys || stillByTime) {
887
+ cooledSurface = acceptCooldown.surface;
888
+ } else {
889
+ acceptCooldown = null;
890
+ }
891
+ }
892
+
893
+ // ── Boundary/early-prefix prefetch; display remains gated at ≥3 ─────────
380
894
  if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
895
+ const boundaryContext = deriveWhitespaceBoundaryContext(textBefore);
896
+ if (boundaryContext) {
897
+ primeBoundaryLm({
898
+ ...boundaryContext,
899
+ familyKey: boundaryContext.contextKey,
900
+ priority: 0
901
+ });
902
+ }
381
903
  return null;
382
904
  }
383
905
  const trimmed = textBefore.trimEnd();
384
- const lastSpaceIdx = trimmed.lastIndexOf(' ');
385
- const currentWord = lastSpaceIdx === -1 ? trimmed : trimmed.slice(lastSpaceIdx + 1);
386
- if (currentWord.length < MIN_PREFIX_LENGTH) {
906
+ const trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
907
+ if (trailingSurfaceToken.length === 0) {
387
908
  return null;
388
909
  }
389
- const candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
910
+ const currentWord = trailingSurfaceToken;
911
+ const currentWordStart = trimmed.length - currentWord.length;
912
+ const {
913
+ canonicalMatched,
914
+ prefixLenByWord
915
+ } = getCanonicalRecall(trimmed, currentWord, currentWordStart);
390
916
 
391
- // Gap-fill from the L3 general-English trie, requesting a full buffer so
392
- // enough survive de-duplication against the L2 results.
393
- if (candidates.length < MAX_CANDIDATES) {
394
- const l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
395
- const existingWords = new Set(candidates.map(c => c.word));
396
- for (const l3c of l3Candidates) {
397
- if (candidates.length >= MAX_CANDIDATES) break;
398
- if (!existingWords.has(l3c.word)) {
399
- candidates.push(l3c);
400
- }
917
+ // If every trie was empty for this prefix
918
+ if (canonicalMatched.length === 0) {
919
+ if (isAutocompleteDebugEnabled()) {
920
+ // eslint-disable-next-line no-console
921
+ console.log(`%c[CTC]%c — abstain: no matches for "${currentWord}"`, CTC_STYLES.brand, CTC_STYLES.body);
401
922
  }
402
- }
403
-
404
- // If both Tries are completely empty for this prefix
405
- if (candidates.length === 0) {
406
923
  return null;
407
924
  }
408
925
  const previousWord = extractPreviousWord(trimmed);
409
926
  const contextVector = getContextVectorForScoring(trimmed);
410
927
  const lmLogits = getStoredLmLogits();
411
-
412
- // Raw LM Output Logger
413
- if (isAutocompleteDebugEnabled() && lmLogits && Object.keys(lmLogits).length > 0) {
414
- const rawLmTop = Object.entries(lmLogits).sort((a, b) => b[1] - a[1]).slice(0, 5).map(([word, score]) => ({
415
- Word: word,
416
- Prob: Number(score.toFixed(5))
417
- }));
418
- // eslint-disable-next-line no-console
419
- console.log('%c[Raw LM Prediction]🧠', 'color: #e83e8c; font-weight: bold;', rawLmTop);
420
- }
421
928
  const mode = contextVector ? 'warm' : 'cold';
422
929
 
423
- // Build ScoringCandidate array from TrieNodes
424
- const scoringCandidates = candidates.map(({
930
+ // Build ScoringCandidate array from matched terms (words + phrases)
931
+ const scoringCandidates = canonicalMatched.map(({
425
932
  word,
426
933
  node
427
934
  }) => ({
@@ -429,29 +936,730 @@ export const predict = textBefore => {
429
936
  tenantFreq: node.tenantFreq,
430
937
  docFreq: node.docFreq,
431
938
  authorFreq: node.authorFreq,
432
- sessionFreq: node.sessionFreq
939
+ sessionFreq: node.sessionFreq,
940
+ termType: node.termType,
941
+ posSeq: node.posSeq,
942
+ headPos: node.headPos,
943
+ association: node.association
433
944
  }));
434
945
 
435
946
  // Filter the LM payload to only words matching the current prefix so that
436
947
  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
437
948
  const prefix = currentWord.toLowerCase();
438
- const prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(([word]) => word.startsWith(prefix))) : null;
949
+ const currentWordSeparator = (_canonicalMatched$fin = canonicalMatched.find(candidate => candidate.node.termType === 'word')) === null || _canonicalMatched$fin === void 0 ? void 0 : _canonicalMatched$fin.separatorKind;
950
+ const prefixLmLogits = lmLogits && currentWordSeparator === 'whitespace' ? Object.fromEntries(Object.entries(lmLogits).filter(([word]) => word.startsWith(prefix))) : null;
951
+ const canonicalScoringSupported = isCanonicalSurfaceScoringSupported();
952
+ const eligibleContextKeys = Array.from(new Set(canonicalMatched.filter(candidate => candidate.canonicalTokenIds !== null).map(candidate => candidate.contextKey))).sort();
953
+ const familyKey = eligibleContextKeys.join('\u0001');
954
+ const primeRequests = selectBoundaryPrimeRequests(familyKey, canonicalMatched, PHRASE_MAX_WORDS);
955
+ for (const request of primeRequests) {
956
+ primeBoundaryLm(request);
957
+ }
958
+ const runtimeBySurface = new Map(canonicalMatched.map(candidate => [candidate.word, candidate]));
959
+ const canonicalEvidence = new Map();
960
+ const firstTokenGroups = new Map();
961
+ for (const candidate of canonicalMatched) {
962
+ var _firstTokenGroups$get;
963
+ if (candidate.canonicalTokenIds === null) {
964
+ continue;
965
+ }
966
+ // This accessor also materializes an exact cache entry when all tokens
967
+ // have already been covered by shared-prefix work.
968
+ getProgressiveSurfaceEvidence(candidate.contextKey, candidate.word);
969
+ const exact = getSurfaceScore(candidate.contextKey, candidate.word);
970
+ if (exact) {
971
+ canonicalEvidence.set(candidate.word, {
972
+ contextKey: candidate.contextKey,
973
+ meanTokenLogProbability: exact.meanTokenLogProbability,
974
+ score: 0,
975
+ separatorKind: 'whitespace',
976
+ source: 'canonical-full-surface',
977
+ totalLogProbability: exact.totalLogProbability,
978
+ totalSurfaceCharCount: candidate.word.length,
979
+ totalTokenCount: exact.tokenCount,
980
+ verifiedCharCount: candidate.word.length,
981
+ verifiedTokenCount: exact.tokenCount
982
+ });
983
+ continue;
984
+ }
985
+ const group = (_firstTokenGroups$get = firstTokenGroups.get(candidate.contextKey)) !== null && _firstTokenGroups$get !== void 0 ? _firstTokenGroups$get : [];
986
+ group.push(candidate);
987
+ firstTokenGroups.set(candidate.contextKey, group);
988
+ }
989
+ for (const [contextKey, group] of firstTokenGroups) {
990
+ const boundary = getBoundaryLmState(contextKey);
991
+ if (!boundary) {
992
+ continue;
993
+ }
994
+ let maxLogit = -Infinity;
995
+ for (const candidate of group) {
996
+ var _candidate$canonicalT;
997
+ const tokenId = (_candidate$canonicalT = candidate.canonicalTokenIds) === null || _candidate$canonicalT === void 0 ? void 0 : _candidate$canonicalT[0];
998
+ const rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
999
+ if (rawLogit !== undefined && Number.isFinite(rawLogit) && rawLogit > maxLogit) {
1000
+ maxLogit = rawLogit;
1001
+ }
1002
+ }
1003
+ if (!Number.isFinite(maxLogit)) {
1004
+ continue;
1005
+ }
1006
+ for (const candidate of group) {
1007
+ var _candidate$canonicalT2, _candidate$canonicalT3, _candidate$canonicalT4;
1008
+ const tokenId = (_candidate$canonicalT2 = candidate.canonicalTokenIds) === null || _candidate$canonicalT2 === void 0 ? void 0 : _candidate$canonicalT2[0];
1009
+ const rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
1010
+ if (rawLogit === undefined || !Number.isFinite(rawLogit)) {
1011
+ continue;
1012
+ }
1013
+ const tokenCount = (_candidate$canonicalT3 = (_candidate$canonicalT4 = candidate.canonicalTokenIds) === null || _candidate$canonicalT4 === void 0 ? void 0 : _candidate$canonicalT4.length) !== null && _candidate$canonicalT3 !== void 0 ? _candidate$canonicalT3 : 0;
1014
+ if (tokenId !== undefined && tokenCount === 1) {
1015
+ const totalLogProbability = logSoftmaxAt(boundary.rawLogits, tokenId);
1016
+ if (!Number.isFinite(totalLogProbability)) {
1017
+ continue;
1018
+ }
1019
+ canonicalEvidence.set(candidate.word, {
1020
+ contextKey,
1021
+ meanTokenLogProbability: totalLogProbability,
1022
+ score: 0,
1023
+ separatorKind: 'whitespace',
1024
+ source: 'canonical-full-surface',
1025
+ totalLogProbability,
1026
+ totalSurfaceCharCount: candidate.word.length,
1027
+ totalTokenCount: 1,
1028
+ verifiedCharCount: candidate.word.length,
1029
+ verifiedTokenCount: 1
1030
+ });
1031
+ continue;
1032
+ }
1033
+ canonicalEvidence.set(candidate.word, {
1034
+ contextKey,
1035
+ rawLogit,
1036
+ score: 0,
1037
+ separatorKind: 'whitespace',
1038
+ source: 'canonical-first-token',
1039
+ totalSurfaceCharCount: candidate.word.length,
1040
+ totalTokenCount: tokenCount,
1041
+ verifiedCharCount: 0,
1042
+ verifiedTokenCount: 1
1043
+ });
1044
+ }
1045
+ }
439
1046
  const {
440
1047
  candidates: ranked,
441
1048
  grammarMeta,
442
1049
  pipelineDebug
443
- } = rankCandidates(scoringCandidates, contextVector, w => getWordVector(w), prefixLmLogits, wordTrie.maxTenantFreq, previousWord);
444
- const best = ranked[0];
445
- const suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
1050
+ } = rankCandidates(scoringCandidates, contextVector, w => getWordVector(w), prefixLmLogits, {
1051
+ word: wordTrie.maxTenantFreq,
1052
+ bigram: maxBigramFreq,
1053
+ phrase: maxPhraseFreq
1054
+ }, previousWord, surface => {
1055
+ var _canonicalEvidence$ge;
1056
+ return (_canonicalEvidence$ge = canonicalEvidence.get(surface)) !== null && _canonicalEvidence$ge !== void 0 ? _canonicalEvidence$ge : null;
1057
+ });
1058
+ const progressiveEligible = ranked.filter(candidate => {
1059
+ const runtime = runtimeBySurface.get(candidate.word);
1060
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined;
1061
+ });
1062
+ if (canonicalScoringSupported && progressiveEligible.length > 0) {
1063
+ const byContext = new Map();
1064
+ for (const candidate of progressiveEligible) {
1065
+ var _byContext$get;
1066
+ const runtime = runtimeBySurface.get(candidate.word);
1067
+ if (!runtime || runtime.canonicalTokenIds === null) {
1068
+ continue;
1069
+ }
1070
+ const group = (_byContext$get = byContext.get(runtime.contextKey)) !== null && _byContext$get !== void 0 ? _byContext$get : [];
1071
+ group.push({
1072
+ runtime,
1073
+ rankHint: candidate.stage1Score
1074
+ });
1075
+ byContext.set(runtime.contextKey, group);
1076
+ }
1077
+ for (const [contextKey, group] of byContext) {
1078
+ requestProgressiveSurfaceScores({
1079
+ familyKey,
1080
+ contextKey,
1081
+ prompt: group[0].runtime.contextBeforeSurface,
1082
+ candidates: group.map(({
1083
+ runtime,
1084
+ rankHint
1085
+ }) => {
1086
+ var _runtime$canonicalTok;
1087
+ return {
1088
+ surface: runtime.word,
1089
+ tokenIds: (_runtime$canonicalTok = runtime.canonicalTokenIds) !== null && _runtime$canonicalTok !== void 0 ? _runtime$canonicalTok : [],
1090
+ rankHint
1091
+ };
1092
+ })
1093
+ });
1094
+ }
1095
+ }
1096
+
1097
+ // The one/two-character passes exist only to hide model latency. They may
1098
+ // prime and expand token groups above, but cannot commit UI.
1099
+ const displayPrefixReady = currentWord.length >= DISPLAY_MIN_PREFIX_LENGTH;
1100
+
1101
+ // ── Arbitration: full-surface evidence → plausibility → winner margin ──
1102
+ const canonicalLmSupported = canonicalScoringSupported;
1103
+ const hasExactEvidence = candidate => {
1104
+ var _candidate$lmEvidence;
1105
+ return ((_candidate$lmEvidence = candidate.lmEvidence) === null || _candidate$lmEvidence === void 0 ? void 0 : _candidate$lmEvidence.source) === 'canonical-full-surface' && candidate.lmEvidence.meanTokenLogProbability !== undefined && Number.isFinite(candidate.lmEvidence.meanTokenLogProbability);
1106
+ };
1107
+
1108
+ /**
1109
+ * The evidence this candidate is judged on: the summed log-probability of
1110
+ * the tokens that have been read, their per-token mean, and whether they
1111
+ * cover the whole surface.
1112
+ *
1113
+ * `total` is the quantity the posterior is built from — it is a sequence
1114
+ * log-likelihood, so it is comparable across surfaces of different lengths
1115
+ * in a way the mean is not. `mean` is retained for reporting and for the
1116
+ * oracle arms, which compare aggregation rules against each other.
1117
+ *
1118
+ * Every token read costs a round trip, so a surface longer than
1119
+ * `REQUIRED_VERIFIED_TOKENS` is judged on its leading tokens and its tail is
1120
+ * left unread. Callers compensate with a wider winner margin rather than
1121
+ * treating the two kinds of evidence as equivalent.
1122
+ *
1123
+ * This is the depth-unaware read, and it is what the pool below is built
1124
+ * from. Arbitration goes through `judgedEvidenceFor`, which adds the one
1125
+ * requirement that can only be stated once the pool is known: a surface
1126
+ * cannot be judged against members it has not yet been read level with.
1127
+ */
1128
+ const readJudgedEvidence = candidate => {
1129
+ const evidence = candidate.lmEvidence;
1130
+ const exactMean = evidence === null || evidence === void 0 ? void 0 : evidence.meanTokenLogProbability;
1131
+ if (hasExactEvidence(candidate) && evidence && exactMean !== undefined) {
1132
+ var _evidence$totalLogPro;
1133
+ const total = (_evidence$totalLogPro = evidence.totalLogProbability) !== null && _evidence$totalLogPro !== void 0 ? _evidence$totalLogPro : exactMean * evidence.totalTokenCount;
1134
+ return Number.isFinite(total) ? {
1135
+ complete: true,
1136
+ mean: exactMean,
1137
+ total
1138
+ } : null;
1139
+ }
1140
+ const runtime = runtimeBySurface.get(candidate.word);
1141
+ if (!runtime || runtime.canonicalTokenIds === null) {
1142
+ return null;
1143
+ }
1144
+ const progressive = getProgressiveSurfaceEvidence(runtime.contextKey, candidate.word);
1145
+ if (!progressive || progressive.scoredTokenCount < REQUIRED_VERIFIED_TOKENS) {
1146
+ return null;
1147
+ }
1148
+ const total = progressive.totalLogProbability;
1149
+ const mean = total / progressive.scoredTokenCount;
1150
+ return Number.isFinite(mean) && Number.isFinite(total) ? {
1151
+ complete: false,
1152
+ mean,
1153
+ total
1154
+ } : null;
1155
+ };
1156
+
1157
+ // Normalise each context's shortlist into a posterior. `total` is a sequence
1158
+ // log-likelihood, so `exp(total - logSumExp(totals))` is the share of the
1159
+ // shortlist's probability mass the model puts on that surface.
1160
+ //
1161
+ // Unknown future token log-probabilities cannot exceed zero, so a partially
1162
+ // read candidate's total is an upper bound on its eventual total, and its
1163
+ // posterior an upper bound on its eventual posterior. That is what lets the
1164
+ // plausibility floor and the display gate be the same threshold — one
1165
+ // applied to the bound, one to the verified value — instead of two numbers
1166
+ // that have to be kept consistent by hand.
1167
+ //
1168
+ // Candidates with no evidence at all are left out of the sum rather than
1169
+ // defaulted. Any default would have to stand in for a log-probability, and a
1170
+ // value low enough not to distort the normaliser is indistinguishable from
1171
+ // omitting the candidate.
1172
+ const optimisticTotalByWord = new Map();
1173
+ const contextTotals = new Map();
1174
+ const contextSurfaces = new Map();
1175
+ // Counted before the total is looked up, so this is every candidate the
1176
+ // context could have scored rather than every candidate it did. The gap
1177
+ // between the two is what separates a context holding one match from a
1178
+ // context whose rivals have not come back yet.
1179
+ const contextRequestedCount = new Map();
1180
+ const bestTotalByContext = new Map();
1181
+ const bestCandidateByContext = new Map();
1182
+ for (const candidate of ranked) {
1183
+ var _contextRequestedCoun, _judged$total;
1184
+ const runtime = runtimeBySurface.get(candidate.word);
1185
+ if (!runtime || runtime.canonicalTokenIds === null) {
1186
+ continue;
1187
+ }
1188
+ contextRequestedCount.set(runtime.contextKey, ((_contextRequestedCoun = contextRequestedCount.get(runtime.contextKey)) !== null && _contextRequestedCoun !== void 0 ? _contextRequestedCoun : 0) + 1);
1189
+ const judged = readJudgedEvidence(candidate);
1190
+ const progressive = getProgressiveSurfaceEvidence(runtime.contextKey, candidate.word);
1191
+ // Once a candidate has been read far enough to judge, its verified total
1192
+ // replaces the bound.
1193
+ const optimisticTotal = (_judged$total = judged === null || judged === void 0 ? void 0 : judged.total) !== null && _judged$total !== void 0 ? _judged$total : progressive === null || progressive === void 0 ? void 0 : progressive.totalLogProbability;
1194
+ if (optimisticTotal === undefined || !Number.isFinite(optimisticTotal)) {
1195
+ continue;
1196
+ }
1197
+ optimisticTotalByWord.set(candidate.word, optimisticTotal);
1198
+ const totals = contextTotals.get(runtime.contextKey);
1199
+ const surfaces = contextSurfaces.get(runtime.contextKey);
1200
+ if (totals && surfaces) {
1201
+ totals.push(optimisticTotal);
1202
+ surfaces.push(candidate.word);
1203
+ } else {
1204
+ contextTotals.set(runtime.contextKey, [optimisticTotal]);
1205
+ contextSurfaces.set(runtime.contextKey, [candidate.word]);
1206
+ }
1207
+ const best = bestTotalByContext.get(runtime.contextKey);
1208
+ if (best === undefined || optimisticTotal > best) {
1209
+ bestTotalByContext.set(runtime.contextKey, optimisticTotal);
1210
+ bestCandidateByContext.set(runtime.contextKey, candidate);
1211
+ }
1212
+ }
1213
+
1214
+ /**
1215
+ * Whether a shorter member of the same pool is this surface's word-boundary
1216
+ * prefix — `happy path` where `happy` is also present.
1217
+ *
1218
+ * Only a multi-word surface can extend another, and a pool is a set, so this
1219
+ * costs one lookup per space rather than a comparison against every member.
1220
+ */
1221
+ const extendsAPoolMember = (surface, pool) => {
1222
+ for (let space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1223
+ if (pool.has(surface.slice(0, space))) {
1224
+ return true;
1225
+ }
1226
+ }
1227
+ return false;
1228
+ };
1229
+
1230
+ // A sequence total only falls as more of the sequence is read, so a partial
1231
+ // sum over `k` tokens sits above the same surface's total over `m > k`. An
1232
+ // extension read less far than the member it continues therefore carries a
1233
+ // total above that member's — and because the chain rule narrows the
1234
+ // denominator to the pool's minimal members, `exp(total - normaliser)` comes
1235
+ // out above 1 for a surface whose own continuation has not been looked at.
1236
+ // It needs a three-token first word to happen, which 2.6% of served bigrams
1237
+ // have, and it inflates both the posterior the gate reads and the
1238
+ // `confidenceScore` every other candidate's margin is measured against.
1239
+ //
1240
+ // A surface's canonical tokenization begins with the tokenization of its own
1241
+ // leading words — the producer encodes `" " + surface` in one call and the
1242
+ // ByteLevel pre-tokenizer splits on word boundaries, so no merge crosses a
1243
+ // space — which is what makes both corrections below exact rather than
1244
+ // approximate.
1245
+ //
1246
+ // The total is capped at what it continues: `P(happy path)` cannot exceed
1247
+ // `P(happy)`. Shortest-first, so a three-word phrase is capped against a
1248
+ // two-word prefix that has already been capped itself.
1249
+ //
1250
+ // And the surface is held unjudgeable until it has been read level with what
1251
+ // it continues, because the cap alone would leave it looking exactly as
1252
+ // certain as its prefix while saying nothing about its own tail.
1253
+ const underReadExtensions = new Set();
1254
+ for (const [contextKey, surfaces] of contextSurfaces) {
1255
+ const pool = new Set(surfaces);
1256
+ for (const surface of [...surfaces].sort((a, b) => a.length - b.length)) {
1257
+ var _getProgressiveSurfac, _getProgressiveSurfac2, _runtimeBySurface$get4, _runtimeBySurface$get5, _runtimeBySurface$get6;
1258
+ const own = optimisticTotalByWord.get(surface);
1259
+ if (own === undefined) {
1260
+ continue;
1261
+ }
1262
+ let cap = Infinity;
1263
+ let continuedDepth = 0;
1264
+ for (let space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1265
+ var _runtimeBySurface$get, _runtimeBySurface$get2, _runtimeBySurface$get3;
1266
+ const prefix = surface.slice(0, space);
1267
+ if (!pool.has(prefix)) {
1268
+ continue;
1269
+ }
1270
+ const prefixTotal = optimisticTotalByWord.get(prefix);
1271
+ if (prefixTotal !== undefined && prefixTotal < cap) {
1272
+ cap = prefixTotal;
1273
+ }
1274
+ const prefixDepth = (_runtimeBySurface$get = (_runtimeBySurface$get2 = runtimeBySurface.get(prefix)) === null || _runtimeBySurface$get2 === void 0 ? void 0 : (_runtimeBySurface$get3 = _runtimeBySurface$get2.canonicalTokenIds) === null || _runtimeBySurface$get3 === void 0 ? void 0 : _runtimeBySurface$get3.length) !== null && _runtimeBySurface$get !== void 0 ? _runtimeBySurface$get : 0;
1275
+ if (prefixDepth > continuedDepth) {
1276
+ continuedDepth = prefixDepth;
1277
+ }
1278
+ }
1279
+ if (own > cap) {
1280
+ optimisticTotalByWord.set(surface, cap);
1281
+ }
1282
+ const scoredDepth = (_getProgressiveSurfac = (_getProgressiveSurfac2 = getProgressiveSurfaceEvidence(contextKey, surface)) === null || _getProgressiveSurfac2 === void 0 ? void 0 : _getProgressiveSurfac2.scoredTokenCount) !== null && _getProgressiveSurfac !== void 0 ? _getProgressiveSurfac : 0;
1283
+ const surfaceDepth = (_runtimeBySurface$get4 = (_runtimeBySurface$get5 = runtimeBySurface.get(surface)) === null || _runtimeBySurface$get5 === void 0 ? void 0 : (_runtimeBySurface$get6 = _runtimeBySurface$get5.canonicalTokenIds) === null || _runtimeBySurface$get6 === void 0 ? void 0 : _runtimeBySurface$get6.length) !== null && _runtimeBySurface$get4 !== void 0 ? _runtimeBySurface$get4 : 0;
1284
+ if (scoredDepth < continuedDepth && scoredDepth < surfaceDepth) {
1285
+ underReadExtensions.add(surface);
1286
+ }
1287
+ }
1288
+ }
1289
+
1290
+ /**
1291
+ * The evidence arbitration judges a candidate on.
1292
+ *
1293
+ * Wraps the depth-unaware read with the pool-relative requirement: a surface
1294
+ * that continues another pool member is not judgeable until it has been read
1295
+ * level with it, so the two totals being compared are taken at comparable
1296
+ * depths.
1297
+ */
1298
+ const judgedEvidenceFor = candidate => underReadExtensions.has(candidate.word) ? null : readJudgedEvidence(candidate);
1299
+ const hasJudgeableEvidence = candidate => judgedEvidenceFor(candidate) !== null;
1300
+
1301
+ // Normalise over the pool's minimal members only, and score everyone —
1302
+ // extensions included — against that.
1303
+ //
1304
+ // A sequence total is its prefix's total plus a log-probability that cannot
1305
+ // be positive, so summing a surface and its own extension double-counts:
1306
+ // `exp(total)` for `happy` already contains every continuation of `happy`,
1307
+ // `happy path` among them. Dividing one pool's mass between the two capped
1308
+ // the shorter form at half however certain it was, and the longer form below
1309
+ // that — both under gates neither could then clear, for a reason that was
1310
+ // arithmetic rather than evidence.
1311
+ //
1312
+ // Excluding extensions from the denominator is the chain rule: the minimal
1313
+ // members partition the mass, and an extension's share of that same
1314
+ // denominator is `P(prefix) * P(continuation | prefix)`, which is exactly
1315
+ // what a multi-word suggestion should be held to. Extensions stay in
1316
+ // `contextTotals`, so how contested a context is still counts every scored
1317
+ // candidate.
1318
+ const logSumExpByContext = new Map();
1319
+ for (const [contextKey, surfaces] of contextSurfaces) {
1320
+ const pool = new Set(surfaces);
1321
+ const minimalTotals = [];
1322
+ for (const surface of surfaces) {
1323
+ const total = optimisticTotalByWord.get(surface);
1324
+ if (total !== undefined && !extendsAPoolMember(surface, pool)) {
1325
+ minimalTotals.push(total);
1326
+ }
1327
+ }
1328
+ // An extension is strictly longer than what it extends, so the shortest
1329
+ // member of any non-empty pool is always minimal and this is never empty.
1330
+ logSumExpByContext.set(contextKey, logSumExp(minimalTotals));
1331
+ }
1332
+
1333
+ /**
1334
+ * A candidate's share of its context shortlist's probability mass.
1335
+ *
1336
+ * Pass an optimistic (partially read) total for an upper bound on the
1337
+ * eventual posterior, or a verified total for the posterior itself.
1338
+ */
1339
+ const posteriorFor = (candidate, total) => {
1340
+ const runtime = runtimeBySurface.get(candidate.word);
1341
+ const normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
1342
+ if (total === undefined || normalizer === undefined || !Number.isFinite(normalizer)) {
1343
+ return 0;
1344
+ }
1345
+ return Math.exp(total - normalizer);
1346
+ };
1347
+ const optimisticPosterior = candidate => posteriorFor(candidate, optimisticTotalByWord.get(candidate.word));
1348
+ const judgedPosterior = candidate => {
1349
+ var _judgedEvidenceFor;
1350
+ return posteriorFor(candidate, (_judgedEvidenceFor = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor === void 0 ? void 0 : _judgedEvidenceFor.total);
1351
+ };
1352
+
1353
+ // A candidate stays in the race for as long as its optimistic posterior
1354
+ // could still clear the gate it will be held to. The bound only falls as
1355
+ // more tokens are read, so nothing dropped here could have gone on to win.
1356
+ const couldClearGate = candidate => optimisticPosterior(candidate) >= MIN_LM_POSTERIOR[candidate.termType];
1357
+ const passesLmFloor = candidate => judgedEvidenceFor(candidate) !== null && couldClearGate(candidate);
1358
+ const hasRequiredEvidence = candidate => {
1359
+ var _candidate$lmEvidence2;
1360
+ if (canonicalLmSupported) {
1361
+ return passesLmFloor(candidate);
1362
+ }
1363
+ return candidate.termType === 'word' && ((_candidate$lmEvidence2 = candidate.lmEvidence) === null || _candidate$lmEvidence2 === void 0 ? void 0 : _candidate$lmEvidence2.source) === 'network-logit';
1364
+ };
1365
+
1366
+ // ── Stabilization guards (QI-2) ──────────────────────────────────────────
1367
+ // Whole-surface repetition: drop a candidate whose full surface already equals
1368
+ // the run of tokens immediately before the cursor — accepting it would
1369
+ // duplicate what was just typed (`end to end` → `end to end to end`).
1370
+ const beforeTokens = tokenize(trimmed);
1371
+ const duplicatesPreceding = c => {
1372
+ const surfaceTokens = tokenize(c.word);
1373
+ const n = surfaceTokens.length;
1374
+ if (n === 0 || beforeTokens.length < n) {
1375
+ return false;
1376
+ }
1377
+ for (let i = 0; i < n; i++) {
1378
+ if (beforeTokens[beforeTokens.length - n + i] !== surfaceTokens[i]) {
1379
+ return false;
1380
+ }
1381
+ }
1382
+ return true;
1383
+ };
1384
+ // Post-accept cooldown: don't re-offer the surface the user just accepted while
1385
+ // its cooldown is still active (see cooldown advance near t0).
1386
+ const isCooledDown = c => cooledSurface !== null && c.word.toLowerCase() === cooledSurface;
1387
+ const isStabilized = candidate => !duplicatesPreceding(candidate) && !isCooledDown(candidate);
1388
+ // A candidate the user has all but finished typing is not worth a ghost, and
1389
+ // a candidate we would never show is not worth waiting for either.
1390
+ const suggestionLengthFor = candidate => {
1391
+ var _prefixLenByWord$get;
1392
+ return candidate.word.length - ((_prefixLenByWord$get = prefixLenByWord.get(candidate.word)) !== null && _prefixLenByWord$get !== void 0 ? _prefixLenByWord$get : currentWord.length);
1393
+ };
1394
+ const isWorthShowing = candidate => suggestionLengthFor(candidate) >= MIN_SUGGESTION_LENGTH;
1395
+ const evidenceBacked = ranked.filter(candidate => hasRequiredEvidence(candidate) && isStabilized(candidate));
1396
+ const confidenceScore = candidate => {
1397
+ if (!canonicalLmSupported) {
1398
+ return candidate.finalScore;
1399
+ }
1400
+ const runtime = runtimeBySurface.get(candidate.word);
1401
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) === null || !optimisticTotalByWord.has(candidate.word)) {
1402
+ return candidate.stage1Score;
1403
+ }
1404
+ return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * optimisticPosterior(candidate);
1405
+ };
1406
+ const exactConfidenceScore = candidate => {
1407
+ if (!canonicalLmSupported || judgedEvidenceFor(candidate) === null) {
1408
+ return candidate.finalScore;
1409
+ }
1410
+ return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * judgedPosterior(candidate);
1411
+ };
1412
+
1413
+ /**
1414
+ * How many candidates in this one's context were scored at all.
1415
+ *
1416
+ * Read off pool membership, which counts every candidate that contributed a
1417
+ * total, rather than off the normaliser, which the chain rule narrows to the
1418
+ * pool's minimal members.
1419
+ */
1420
+ const scoredPoolSize = candidate => {
1421
+ var _runtimeBySurface$get7, _contextTotals$get$le, _contextTotals$get;
1422
+ const contextKey = (_runtimeBySurface$get7 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get7 === void 0 ? void 0 : _runtimeBySurface$get7.contextKey;
1423
+ return contextKey === undefined ? 0 : (_contextTotals$get$le = (_contextTotals$get = contextTotals.get(contextKey)) === null || _contextTotals$get === void 0 ? void 0 : _contextTotals$get.length) !== null && _contextTotals$get$le !== void 0 ? _contextTotals$get$le : 0;
1424
+ };
1425
+
1426
+ /**
1427
+ * How many candidates the context held before any of them were scored.
1428
+ *
1429
+ * A pool of one means two opposite things. Either the vocabulary offered a
1430
+ * single continuation of what was typed, which is the least contested case
1431
+ * there is; or rivals were offered and have not been read yet, which is the
1432
+ * least informed. Only the second deserves refusing, and the two are
1433
+ * distinguishable exactly here.
1434
+ */
1435
+ const requestedPoolSize = candidate => {
1436
+ var _runtimeBySurface$get8, _contextRequestedCoun2;
1437
+ const contextKey = (_runtimeBySurface$get8 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get8 === void 0 ? void 0 : _runtimeBySurface$get8.contextKey;
1438
+ return contextKey === undefined ? 0 : (_contextRequestedCoun2 = contextRequestedCount.get(contextKey)) !== null && _contextRequestedCoun2 !== void 0 ? _contextRequestedCoun2 : 0;
1439
+ };
1440
+
1441
+ // A suppressed short completion still counts as competition below, so losing
1442
+ // to one abstains rather than promoting the runner-up in its place.
1443
+ const gateCleared = evidenceBacked.filter(isWorthShowing).map(candidate => ({
1444
+ candidate,
1445
+ posterior: judgedPosterior(candidate),
1446
+ score: exactConfidenceScore(candidate)
1447
+ }))
1448
+ // The gate is the model's own confidence in the surface; the blended score
1449
+ // only orders what has already cleared it, so a strong corpus prior can no
1450
+ // longer carry a surface the model is unsure of onto the screen.
1451
+ .filter(({
1452
+ candidate,
1453
+ posterior
1454
+ }) => posterior >= MIN_LM_POSTERIOR[candidate.termType]);
1455
+ /**
1456
+ * Whether the model finds the surface plausible on its own terms.
1457
+ *
1458
+ * The gate above asks which candidate won its pool; this asks whether
1459
+ * winning it meant anything. The two come apart exactly where a pool is
1460
+ * thin — a shortlist of one hands its only member 1.0 by construction — and
1461
+ * a relative test can never see through that however it is tuned.
1462
+ *
1463
+ * Waived without canonical scoring, where there is no judged mean to test
1464
+ * and the network-logit path would otherwise refuse everything.
1465
+ */
1466
+ const isPlausibleSurface = candidate => {
1467
+ var _judgedEvidenceFor2;
1468
+ if (!canonicalLmSupported) {
1469
+ return true;
1470
+ }
1471
+ const mean = (_judgedEvidenceFor2 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor2 === void 0 ? void 0 : _judgedEvidenceFor2.mean;
1472
+ return mean !== undefined && mean >= MIN_MEAN_TOKEN_LOG_PROBABILITY;
1473
+ };
1474
+ // Kept as its own stage rather than folded into the gate so the three
1475
+ // populations stay separable in the abstain cascade: cleared its pool and was
1476
+ // implausible, was plausible and had nothing to clear the pool against, or
1477
+ // cleared both. A candidate refused here stays in `ranked` and so still
1478
+ // counts as competition below — promoting the runner-up in place of an
1479
+ // implausible leader would show something worse, not something better.
1480
+ const plausible = gateCleared.filter(({
1481
+ candidate
1482
+ }) => isPlausibleSurface(candidate));
1483
+ // Applied after the gate rather than folded into it, so the two populations
1484
+ // stay separable: a surface refused here cleared its threshold and was
1485
+ // refused for having had nothing to clear it against.
1486
+ const eligible = plausible.filter(({
1487
+ candidate
1488
+ }) => !canonicalLmSupported || scoredPoolSize(candidate) >= MIN_SCORED_POOL_SIZE || requestedPoolSize(candidate) <= 1).sort((a, b) => b.score - a.score);
1489
+ const selected = (_eligible$ = eligible[0]) === null || _eligible$ === void 0 ? void 0 : _eligible$.candidate;
1490
+ const selectedDisplayScore = (_eligible$0$score = (_eligible$2 = eligible[0]) === null || _eligible$2 === void 0 ? void 0 : _eligible$2.score) !== null && _eligible$0$score !== void 0 ? _eligible$0$score : 0;
1491
+ const selectedEvidence = selected && hasRequiredEvidence(selected) ? selected.lmEvidence : null;
1492
+ /**
1493
+ * Whether `longer` is `shorter` continued past a word boundary, in the same
1494
+ * normaliser.
1495
+ *
1496
+ * Shared by the two places that have to agree on what an extension is: the
1497
+ * chain rule, which stops dividing a pool's mass between a sequence and its
1498
+ * own prefix, and the winner margin below, which stops treating one as the
1499
+ * other's rival. They were the same category error and are now the same test.
1500
+ */
1501
+ const extendsInSamePool = (shorter, longer) => {
1502
+ var _runtimeBySurface$get9, _runtimeBySurface$get0;
1503
+ if (longer.word === shorter.word || !optimisticTotalByWord.has(longer.word)) {
1504
+ return false;
1505
+ }
1506
+ const pool = (_runtimeBySurface$get9 = runtimeBySurface.get(shorter.word)) === null || _runtimeBySurface$get9 === void 0 ? void 0 : _runtimeBySurface$get9.contextKey;
1507
+ if (pool === undefined || ((_runtimeBySurface$get0 = runtimeBySurface.get(longer.word)) === null || _runtimeBySurface$get0 === void 0 ? void 0 : _runtimeBySurface$get0.contextKey) !== pool) {
1508
+ return false;
1509
+ }
1510
+ return longer.word.length > shorter.word.length && longer.word.startsWith(shorter.word) && longer.word[shorter.word.length] === ' ';
1511
+ };
1512
+ // A candidate that extends the selection is not competing with it. Whichever
1513
+ // continuation the user meant, the shorter surface is a correct ghost — it is
1514
+ // the prefix of both — so the margin has nothing to arbitrate and charging the
1515
+ // selection for its own extension abstains on a case that could not be wrong.
1516
+ // The chain rule already removed this from the denominator; leaving it in the
1517
+ // margin meant a nested leader cleared its gate and then lost to the candidate
1518
+ // that had just been freed from dividing its mass. It penalised bigrams and
1519
+ // phrases specifically, because a nested pool is where they mostly appear.
1520
+ const runnerUpScore = selected ? Math.max(...ranked.filter(candidate => candidate.word !== selected.word && isStabilized(candidate) && !extendsInSamePool(selected, candidate)).map(confidenceScore), 0) : 0;
1521
+ const winnerMargin = selected ? selectedDisplayScore - runnerUpScore : 0;
1522
+ const hasMissingArtifactCompetitor = canonicalLmSupported && ranked.some(candidate => {
1523
+ const runtime = runtimeBySurface.get(candidate.word);
1524
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) === 'whitespace' && runtime.canonicalTokenIds === null;
1525
+ });
1526
+ // A candidate with canonical tokens but no evidence at all cannot join the
1527
+ // posterior: it has no likelihood to contribute, and any stand-in value would
1528
+ // distort the normaliser for every other candidate in its context. Absent
1529
+ // evidence is not evidence of absence though — the context's boundary may
1530
+ // simply not be primed yet — so it holds the ghost back instead of being
1531
+ // silently ignored. In a primed context every candidate has at least
1532
+ // first-token evidence, so this only fires while a context is still cold.
1533
+ const hasUnscoredCompetitor = canonicalLmSupported && ranked.some(candidate => {
1534
+ const runtime = runtimeBySurface.get(candidate.word);
1535
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !optimisticTotalByWord.has(candidate.word) && isStabilized(candidate) && isWorthShowing(candidate);
1536
+ });
1537
+ // A rival only holds the ghost back while it is still too unread to judge.
1538
+ // Under full-surface evidence that meant any long surface blocked everything
1539
+ // until every one of its tokens had been paid for, which abstained far more
1540
+ // often than it ever changed the winner. A rival now becomes judgeable after
1541
+ // `REQUIRED_VERIFIED_TOKENS`, so this settles within the budget.
1542
+ const hasUnresolvedPotential = canonicalLmSupported && ranked.some(candidate => {
1543
+ const runtime = runtimeBySurface.get(candidate.word);
1544
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !hasJudgeableEvidence(candidate) && couldClearGate(candidate) && isStabilized(candidate) && isWorthShowing(candidate);
1545
+ });
1546
+ const requiredWinnerMargin = MIN_WINNER_MARGIN + (selected && ((_judgedEvidenceFor3 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor3 === void 0 ? void 0 : _judgedEvidenceFor3.complete) === false ? PARTIAL_EVIDENCE_MARGIN_PREMIUM : 0);
1547
+ // Everything the display decision needs except the veto itself. Splitting it
1548
+ // out is what lets the veto be priced: the value here at the moment the veto
1549
+ // fires is exactly the ghost the veto cost us.
1550
+ const clearsMarginBeforeColdVeto = selected !== undefined && !hasMissingArtifactCompetitor && winnerMargin >= requiredWinnerMargin;
1551
+ const clearsWinnerMargin = clearsMarginBeforeColdVeto && !hasUnscoredCompetitor;
1552
+ const suggestion = displayPrefixReady && selected && clearsWinnerMargin ? selected.word.slice((_prefixLenByWord$get2 = prefixLenByWord.get(selected.word)) !== null && _prefixLenByWord$get2 !== void 0 ? _prefixLenByWord$get2 : currentWord.length) : null;
1553
+
1554
+ // Which constraint bound. Computed on every evaluation rather than only under
1555
+ // debug, because it is counted for the session as well as printed, and
1556
+ // deriving the debug header from it keeps the two from disagreeing.
1557
+ const resolveAbstainReason = () => {
1558
+ if (suggestion !== null && suggestion.length > 0) {
1559
+ return null;
1560
+ }
1561
+ if (!displayPrefixReady) {
1562
+ return 'prefetch';
1563
+ }
1564
+ if (ranked.length === 0) {
1565
+ return 'no-candidate';
1566
+ }
1567
+ if (hasMissingArtifactCompetitor) {
1568
+ return 'missing-artifact';
1569
+ }
1570
+ if (hasUnscoredCompetitor) {
1571
+ return 'cold-competitor';
1572
+ }
1573
+ if (hasUnresolvedPotential && !clearsWinnerMargin) {
1574
+ return 'unresolved-rival';
1575
+ }
1576
+ if (eligible.length > 0) {
1577
+ return selected && !clearsWinnerMargin ? 'winner-margin' : 'empty-completion';
1578
+ }
1579
+ if (plausible.length > 0) {
1580
+ // Cleared its gate, was plausible on its own terms, and was refused for
1581
+ // the shortlist it cleared that gate against holding nobody else.
1582
+ return 'lone-candidate';
1583
+ }
1584
+ if (gateCleared.length > 0) {
1585
+ // Held the largest share of its shortlist and still was not a surface
1586
+ // the model found plausible — which is only possible because a share is
1587
+ // relative and this is not.
1588
+ return 'implausible-surface';
1589
+ }
1590
+ if (evidenceBacked.length > 0 && evidenceBacked.every(candidate => !isWorthShowing(candidate))) {
1591
+ return 'short-completion';
1592
+ }
1593
+ // The floor and the gate are one threshold, applied to the bound and to
1594
+ // the verified value, so a candidate short of it is dropped before
1595
+ // `evidenceBacked` is built. Asking whether anything was judgeable
1596
+ // separates a gate that rejected read candidates from a context where
1597
+ // nothing was read at all — the first is a threshold to calibrate, the
1598
+ // second is scheduling.
1599
+ return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
1600
+ };
1601
+ const abstainReason = resolveAbstainReason();
1602
+
1603
+ // The leader is the best-supported candidate, not the selected one: an
1604
+ // evaluation that showed nothing is exactly the one whose posterior needs
1605
+ // recording, and it has no selection to report.
1606
+ const posteriorLeader = ranked.reduce((best, candidate) => {
1607
+ const posterior = optimisticPosterior(candidate);
1608
+ return posterior > 0 && (best === null || posterior > best.posterior) ? {
1609
+ candidate,
1610
+ posterior
1611
+ } : best;
1612
+ }, null);
1613
+ /**
1614
+ * The longer surface in the leader's normaliser that extends the given
1615
+ * candidate, if the pool holds one.
1616
+ *
1617
+ * `logSumExp` divides one pool's mass between a sequence and its own prefix,
1618
+ * and the longer total is the shorter one plus a log-probability that cannot
1619
+ * be positive — so the longer form holds at most half, or at most a third
1620
+ * where both shorter forms are present. Both sit under the gate it is held
1621
+ * to, so it cannot display however certain its continuation is.
1622
+ *
1623
+ * Asked from the shorter form because that is the one that leads: the
1624
+ * extending candidate is strictly lower in the same pool and so can never be
1625
+ * the leader.
1626
+ */
1627
+ const poolExtensionOf = shorter => ranked.find(candidate => extendsInSamePool(shorter, candidate));
446
1628
  if (isAutocompleteDebugEnabled()) {
1629
+ var _posteriorLeader$cand, _posteriorLeader$post, _selectedEvidence$mea, _selectedEvidence$mea2, _acceptCooldown$predi, _acceptCooldown;
1630
+ const verbose = isAutocompleteDebugVerbose();
447
1631
  const latencyMs = performance.now() - t0;
448
1632
  const tokens = tokenize(trimmed);
449
1633
  const contextWords = tokens.slice(-CONTEXT_WORDS);
450
- const belowThreshold = best && best.finalScore < MIN_SCORE_THRESHOLD;
451
- const suggestionLabel = belowThreshold ? '🚫 (below threshold)' : suggestion && suggestion.length > 0 ? `✨ "${suggestion}"` : '🚫 (no match)';
1634
+
1635
+ // Decision drives the (collapsed) group header. Abstains carry the reason
1636
+ // the session counters recorded, plus the numbers behind it, so it is clear
1637
+ // *why* nothing showed without expanding the group.
1638
+ const abstainDetail = {
1639
+ 'below-posterior-gate': `posterior below ${MIN_LM_POSTERIOR[((_posteriorLeader$cand = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.candidate) !== null && _posteriorLeader$cand !== void 0 ? _posteriorLeader$cand : ranked[0]).termType]} (best ${((_posteriorLeader$post = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.posterior) !== null && _posteriorLeader$post !== void 0 ? _posteriorLeader$post : 0).toFixed(2)})`,
1640
+ 'cold-competitor': 'competitor has no LM evidence yet',
1641
+ 'empty-completion': 'empty completion',
1642
+ 'implausible-surface': `mean per-token log-probability below ${MIN_MEAN_TOKEN_LOG_PROBABILITY} (best ${gateCleared.map(({
1643
+ candidate
1644
+ }) => {
1645
+ var _judgedEvidenceFor$me, _judgedEvidenceFor4;
1646
+ return (_judgedEvidenceFor$me = (_judgedEvidenceFor4 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor4 === void 0 ? void 0 : _judgedEvidenceFor4.mean) !== null && _judgedEvidenceFor$me !== void 0 ? _judgedEvidenceFor$me : -Infinity;
1647
+ }).reduce((best, mean) => Math.max(best, mean), -Infinity).toFixed(2)})`,
1648
+ 'lone-candidate': `cleared its gate against fewer than ${MIN_SCORED_POOL_SIZE} scored candidates, rivals unread`,
1649
+ 'missing-artifact': 'canonical artifact coverage missing',
1650
+ 'no-candidate': 'nothing cleared scoring',
1651
+ 'no-evidence': 'full-surface evidence absent',
1652
+ prefetch: `prefetch: ${currentWord.length}/${DISPLAY_MIN_PREFIX_LENGTH} chars`,
1653
+ 'short-completion': `completion shorter than ${MIN_SUGGESTION_LENGTH} chars`,
1654
+ 'unresolved-rival': 'expanding plausible token-prefix groups',
1655
+ 'winner-margin': `winner margin ${winnerMargin.toFixed(2)} < ${requiredWinnerMargin.toFixed(2)}`
1656
+ };
1657
+ const decision = abstainReason === null ? `✨ "${suggestion !== null && suggestion !== void 0 ? suggestion : ''}"` : abstainReason === 'prefetch' ? `⏳ ${abstainDetail.prefetch}` : `— abstain: ${abstainDetail[abstainReason]}`;
452
1658
  lastPredictionDebug = {
1659
+ awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
453
1660
  textBefore: trimmed,
454
1661
  currentWord,
1662
+ decision,
455
1663
  mode,
456
1664
  contextWords,
457
1665
  topCandidates: ranked.slice(0, 5).map(r => ({
@@ -464,62 +1672,179 @@ export const predict = textBefore => {
464
1672
  suggestion: suggestion && suggestion.length > 0 ? suggestion : null
465
1673
  };
466
1674
 
467
- // ── Mode label: COLD / WARM(local) / WARM(BE) ───────────────────────
1675
+ // Mode describes the semantic-vector source without implying that a
1676
+ // network backend ran; the snapshot may come from the local MLC embedder.
468
1677
  const slowLaneVec = getStoredContextVector();
469
1678
  const isUsingSlowLaneVector = slowLaneVec !== null && vectorStore !== null && slowLaneVec.length === vectorStore.dim;
470
- const modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(BE)' : 'WARM(local)';
471
- const modeColor = modeLabel === 'WARM(BE)' ? 'color: #ff9800; font-weight: bold;' : modeLabel === 'WARM(local)' ? 'color: #4caf50; font-weight: bold;' : 'color: #9e9e9e; font-weight: bold;';
1679
+ const modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(semantic)' : 'WARM(local-avg)';
1680
+ const modeColor = modeLabel === 'WARM(semantic)' ? CTC_STYLES.good : modeLabel === 'WARM(local-avg)' ? CTC_STYLES.warn : CTC_STYLES.cold;
472
1681
 
473
- // 1. Collapsible group header
1682
+ // Group header: [CTC:<mode>] <decision> · <mode> · <type> · <latency>
1683
+ // arbitrationMode ('v1') tags the line so this deploy's logs diff cleanly
1684
+ // against the baseline deploy's.
474
1685
  // eslint-disable-next-line no-console
475
- console.groupCollapsed(`%c[Autocomplete] %c${modeLabel} %c| "${currentWord}" ➔ ${suggestionLabel} | ⏱ ${latencyMs.toFixed(1)}ms`, 'color: #00b8d9; font-weight: bold;', modeColor, 'color: inherit; font-weight: normal;');
1686
+ console.groupCollapsed(`%c[CTC:${ARBITRATION_MODE}]%c ${decision} %c· ${modeLabel}${selected ? ` · ${selected.termType}` : ''} · ⏱ ${latencyMs.toFixed(1)}ms`, CTC_STYLES.brand, CTC_STYLES.body, modeColor);
1687
+ const semanticInput = getStoredContextInput();
1688
+ const causalPrompts = primeRequests.map((request, index) => {
1689
+ const tail = request.prompt.slice(-DEBUG_TEXT_TAIL_CHARS);
1690
+ return `#${index + 1}:${JSON.stringify(tail)}`;
1691
+ });
476
1692
 
477
- // 2. Context window (what local vector averaging sees)
478
- // eslint-disable-next-line no-console
479
- console.log('%cContext Window:', 'color: #888; font-style: italic;', contextWords.length ? contextWords.join(' ') : '(none)');
1693
+ // Keep the exact model inputs separate from normalized lookup tokens.
1694
+ ctcSection('INPUT', `raw: ${JSON.stringify(trimmed.slice(-DEBUG_TEXT_TAIL_CHARS))} · prev: ${previousWord ? JSON.stringify(previousWord) : '—'}`);
1695
+ ctcSection('LOOKUP', `normalized(last ${CONTEXT_WORDS}): ${contextWords.length ? JSON.stringify(contextWords.join(' ')) : '(none)'}`);
1696
+ ctcSection('SEMANTIC', `snapshot: ${semanticInput ? JSON.stringify(semanticInput.slice(-DEBUG_TEXT_TAIL_CHARS)) : '(unavailable)'}`);
1697
+ ctcSection('CAUSAL', `prompt${causalPrompts.length === 1 ? '' : 's'}: ${causalPrompts.join(' · ') || '(not ready)'}`);
480
1698
 
481
- // 3. Slow-lane status
482
- const vectorStatus = isUsingSlowLaneVector ? `✅ BE semantic vector (dim=${slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length})` : vectorStore ? '⚠️ Local vector average (slow-lane not yet returned)' : '❌ No vectors (cold)';
483
- const logitsStatus = lmLogits && Object.keys(lmLogits).length > 0 ? `✅ LM logits active (${Object.keys(lmLogits).length} tokens)` : '⏳ No LM logits (slow-lane pending or failed)';
484
- // eslint-disable-next-line no-console
485
- console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
1699
+ // SIGNALS — semantic state plus context-keyed canonical LM coverage.
1700
+ const logitCount = lmLogits ? Object.keys(lmLogits).length : 0;
1701
+ const semanticStatus = isUsingSlowLaneVector ? `semantic ✅ snapshot(${slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length}d)` : vectorStore ? 'semantic ⚠️ local-avg' : 'semantic ❌ cold';
1702
+ const tierACount = ranked.filter(candidate => {
1703
+ var _candidate$lmEvidence3;
1704
+ return ((_candidate$lmEvidence3 = candidate.lmEvidence) === null || _candidate$lmEvidence3 === void 0 ? void 0 : _candidate$lmEvidence3.source) === 'canonical-first-token';
1705
+ }).length;
1706
+ const exactCount = ranked.filter(candidate => {
1707
+ var _candidate$lmEvidence4;
1708
+ return ((_candidate$lmEvidence4 = candidate.lmEvidence) === null || _candidate$lmEvidence4 === void 0 ? void 0 : _candidate$lmEvidence4.source) === 'canonical-full-surface';
1709
+ }).length;
1710
+ const boundaryReady = primeRequests.filter(request => getBoundaryLmState(request.contextKey) !== null).length;
1711
+ const artifactEligible = canonicalMatched.filter(candidate => candidate.canonicalTokenIds !== null).length;
1712
+ const documentStartFallbacks = canonicalMatched.filter(candidate => candidate.separatorKind === 'document-start').length;
1713
+ const punctuationFallbacks = canonicalMatched.filter(candidate => candidate.separatorKind === 'non-space').length;
1714
+ const logitsStatus = tierACount + exactCount > 0 ? `canonical ✅ first-token:${tierACount} exact:${exactCount}` : logitCount > 0 ? `network logits ✅ ${logitCount}` : 'canonical ⏳/absent';
1715
+ const formula = tierACount + exactCount + logitCount > 0 ? `Stage1×${STAGE1_WEIGHT}+LM×${STAGE2_WEIGHT}` : 'Stage1-only';
1716
+ ctcSection('SIGNALS', `${semanticStatus} · ${logitsStatus} · ${formula}`);
1717
+ ctcSection('CANONICAL', `eligible ${artifactEligible}/${canonicalMatched.length} · separator fallback doc:${documentStartFallbacks} punct:${punctuationFallbacks} · contexts ${boundaryReady}/${primeRequests.length} ready (≤${PHRASE_MAX_WORDS}) · family=${familyKey.slice(0, 72) || 'none'}`);
1718
+ if (verbose && logitCount > 0 && lmLogits) {
1719
+ const rawLmTop = Object.entries(lmLogits).sort((a, b) => b[1] - a[1]).slice(0, 5).map(([word, score]) => `${word}:${score.toFixed(3)}`).join(', ');
1720
+ ctcSection(' rawLM', `🧠 ${rawLmTop}`);
1721
+ }
486
1722
 
487
- // 4. Scoring formula active this prediction
488
- const formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? `Stage1(×${STAGE1_WEIGHT}) + LM(×${STAGE2_WEIGHT})` : 'Stage1 only (no LM logits)';
489
- // eslint-disable-next-line no-console
490
- console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
1723
+ // GENERATE — how many candidates matched, split by term type
1724
+ const genByType = {
1725
+ word: 0,
1726
+ bigram: 0,
1727
+ phrase: 0
1728
+ };
1729
+ for (const m of canonicalMatched) {
1730
+ genByType[m.node.termType] += 1;
1731
+ }
1732
+ ctcSection('GENERATE', `matched ${canonicalMatched.length} → word:${genByType.word} bigram:${genByType.bigram} phrase:${genByType.phrase}${canonicalLmSupported ? ' · display needs exact surface evidence' : ''}`);
491
1733
 
492
- // 5. Grammar filter result (collected inside rankCandidates, logged here)
1734
+ // SCORE — Stage-1 + grammar funnel and canonical evidence depth.
1735
+ const lmCoverage = ranked.slice(0, 10).filter(r => r.lmScore > 0).length;
1736
+ ctcSection('SCORE', `in ${pipelineDebug.initial} → stage1(<${MIN_STAGE1_SCORE}) −${pipelineDebug.stage1Rejected.length} → grammar −${pipelineDebug.grammarRejected.length} → final ${pipelineDebug.final} · LM cov ${lmCoverage}/${Math.min(ranked.length, 10)} · first-token ${tierACount} · exact ${exactCount}`);
493
1737
  if (grammarMeta) {
494
- // eslint-disable-next-line no-console
495
- console.log(`%c[Grammar] "${grammarMeta.prevWord}" [${grammarMeta.prevTags.join('|')}] → ${grammarMeta.before} candidates → ${grammarMeta.after} after filter`, 'color: #4caf50; font-weight: bold;');
496
- if (grammarMeta.dropped.length > 0) {
497
- // eslint-disable-next-line no-console
498
- console.log(`%c🚫 Dropped: ${grammarMeta.dropped.join(', ')}`, 'color: #f44336; font-style: italic;');
499
- }
1738
+ ctcSection(' grammar', `"${grammarMeta.prevWord}" [${grammarMeta.prevTags.join('|')}] ${grammarMeta.before}→${grammarMeta.after}${verbose && grammarMeta.dropped.length > 0 ? ` · dropped: ${grammarMeta.dropped.join(', ')}` : ''}`);
500
1739
  }
1740
+ const gateReasonFor = candidate => {
1741
+ const runtime = runtimeBySurface.get(candidate.word);
1742
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace') {
1743
+ var _runtime$separatorKin;
1744
+ return `separator:${(_runtime$separatorKin = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin !== void 0 ? _runtime$separatorKin : 'unknown'}`;
1745
+ }
1746
+ if (runtime.canonicalTokenIds === null) {
1747
+ return 'missing-artifact';
1748
+ }
1749
+ if (!hasJudgeableEvidence(candidate)) {
1750
+ return `awaiting-tokens:${REQUIRED_VERIFIED_TOKENS}`;
1751
+ }
1752
+ if (!passesLmFloor(candidate)) {
1753
+ return `bound:${optimisticPosterior(candidate).toFixed(2)}<${MIN_LM_POSTERIOR[candidate.termType]}`;
1754
+ }
1755
+ if (duplicatesPreceding(candidate)) {
1756
+ return 'repetition';
1757
+ }
1758
+ if (isCooledDown(candidate)) {
1759
+ return 'cooldown';
1760
+ }
1761
+ if (!isWorthShowing(candidate)) {
1762
+ return `suffix:${suggestionLengthFor(candidate)}<${MIN_SUGGESTION_LENGTH}`;
1763
+ }
1764
+ const floor = MIN_LM_POSTERIOR[candidate.termType];
1765
+ const posterior = judgedPosterior(candidate);
1766
+ if (posterior < floor) {
1767
+ return `posterior:${posterior.toFixed(2)}<${floor}`;
1768
+ }
1769
+ if (!isPlausibleSurface(candidate)) {
1770
+ var _judgedEvidenceFor$me2, _judgedEvidenceFor5;
1771
+ return `implausible:${((_judgedEvidenceFor$me2 = (_judgedEvidenceFor5 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor5 === void 0 ? void 0 : _judgedEvidenceFor5.mean) !== null && _judgedEvidenceFor$me2 !== void 0 ? _judgedEvidenceFor$me2 : NaN).toFixed(2)}<${MIN_MEAN_TOKEN_LOG_PROBABILITY}`;
1772
+ }
1773
+ return 'eligible';
1774
+ };
1775
+ const gateFunnel = termType => {
1776
+ const typeRanked = ranked.filter(candidate => candidate.termType === termType);
1777
+ const exact = typeRanked.filter(hasJudgeableEvidence);
1778
+ const absolute = exact.filter(passesLmFloor);
1779
+ const stabilized = absolute.filter(isStabilized);
1780
+ const longEnough = stabilized.filter(isWorthShowing);
1781
+ const floorPassed = longEnough.filter(candidate => judgedPosterior(candidate) >= MIN_LM_POSTERIOR[termType]);
1782
+ const plausiblePassed = floorPassed.filter(isPlausibleSurface);
1783
+ return `${termType} m:${genByType[termType]} r:${typeRanked.length} exact:${exact.length} abs:${absolute.length} stable:${stabilized.length} suffix:${longEnough.length} floor:${floorPassed.length} plausible:${plausiblePassed.length} eligible:${eligible.filter(({
1784
+ candidate
1785
+ }) => candidate.termType === termType).length}`;
1786
+ };
1787
+ ctcSection('GATES', `${gateFunnel('word')} | ${gateFunnel('bigram')} | ${gateFunnel('phrase')}`);
501
1788
 
502
- // 6. Pipeline funnel
503
- // eslint-disable-next-line no-console
504
- console.log(`%c[Pipeline Funnel] %c📥 In: ${pipelineDebug.initial} | ❌ Stage 1 (< ${MIN_STAGE1_SCORE}): -${pipelineDebug.stage1Rejected.length} | ❌ Grammar: -${pipelineDebug.grammarRejected.length} | ✅ Final: ${pipelineDebug.final}`, 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
1789
+ // One line per context: how many candidates share the normaliser, and how
1790
+ // much of the mass the leader holds. A leader well under its threshold
1791
+ // means the context is contested, which is the abstention we want.
1792
+ const contextLeaders = Array.from(bestCandidateByContext.entries()).slice(0, PHRASE_MAX_WORDS).map(([contextKey, leader]) => {
1793
+ var _contextTotals$get$le2, _contextTotals$get2;
1794
+ const shortlistSize = (_contextTotals$get$le2 = (_contextTotals$get2 = contextTotals.get(contextKey)) === null || _contextTotals$get2 === void 0 ? void 0 : _contextTotals$get2.length) !== null && _contextTotals$get$le2 !== void 0 ? _contextTotals$get$le2 : 0;
1795
+ const evidenceKind = hasExactEvidence(leader) ? 'exact' : 'upper';
1796
+ return `${contextKey.slice(0, 32)} → n=${shortlistSize} ${leader.termType}:"${leader.word}" ${evidenceKind} p=${optimisticPosterior(leader).toFixed(3)}/${MIN_LM_POSTERIOR[leader.termType]}`;
1797
+ }).join(' | ');
1798
+ ctcSection('NORMALIZE', contextLeaders.length > 0 ? contextLeaders : 'no scored candidate in any context');
1799
+ ctcSection('READINESS', `prefix ${currentWord.length}/${DISPLAY_MIN_PREFIX_LENGTH} · boundary ${boundaryReady}/${primeRequests.length} · exact word:${ranked.filter(candidate => candidate.termType === 'word' && hasExactEvidence(candidate)).length} bigram:${ranked.filter(candidate => candidate.termType === 'bigram' && hasExactEvidence(candidate)).length} phrase:${ranked.filter(candidate => candidate.termType === 'phrase' && hasExactEvidence(candidate)).length} · ${suggestion ? 'display-ready' : decision}`);
505
1800
 
506
- // 7. Candidate table
507
- if (ranked.length > 0) {
508
- const lmCoverage = ranked.slice(0, 10).filter(r => r.lmScore > 0).length;
509
- // eslint-disable-next-line no-console
510
- console.log(`%cLM coverage: ${lmCoverage}/${Math.min(ranked.length, 10)} candidates had real logit scores`, 'color: #888; font-style: italic;');
1801
+ // ARBITRATE — expose every condition that can authorize or suppress ghost.
1802
+ const topOfType = t => ranked.find(c => c.termType === t);
1803
+ const fmtType = t => {
1804
+ var _top$lmEvidence$sourc, _top$lmEvidence;
1805
+ const top = topOfType(t);
1806
+ if (!top) {
1807
+ return `${t}:—`;
1808
+ }
1809
+ const floor = MIN_LM_POSTERIOR[t];
1810
+ const bound = optimisticPosterior(top);
1811
+ const evidence = hasRequiredEvidence(top) ? 'TierB✅' : `${(_top$lmEvidence$sourc = (_top$lmEvidence = top.lmEvidence) === null || _top$lmEvidence === void 0 ? void 0 : _top$lmEvidence.source) !== null && _top$lmEvidence$sourc !== void 0 ? _top$lmEvidence$sourc : 'absent'}❌`;
1812
+ return `${t}:"${top.word}" p≤${bound.toFixed(3)}${bound >= floor ? '≥' : '<'}${floor} fs≤${confidenceScore(top).toFixed(2)} ${evidence}`;
1813
+ };
1814
+ ctcSection('ARBITRATE', `${selected ? `▸ ${selected.termType} "${selected.word}" fs=${selectedDisplayScore.toFixed(2)} mean=${(_selectedEvidence$mea = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : (_selectedEvidence$mea2 = selectedEvidence.meanTokenLogProbability) === null || _selectedEvidence$mea2 === void 0 ? void 0 : _selectedEvidence$mea2.toFixed(2)) !== null && _selectedEvidence$mea !== void 0 ? _selectedEvidence$mea : 'network'} margin=${winnerMargin.toFixed(2)}${clearsWinnerMargin ? '✅' : '❌'}` : '▸ (none)'} · prefix ${currentWord.length}/${DISPLAY_MIN_PREFIX_LENGTH}${displayPrefixReady ? '✅' : '⏳'} · grouped pending ${hasUnresolvedPotential ? 'yes' : 'no'} | ${fmtType('word')} ${fmtType('bigram')} ${fmtType('phrase')}`);
1815
+
1816
+ // STABILIZE — post-accept cooldown + whole-surface repetition (QI-2)
1817
+ const repetitionBlocked = ranked.filter(c => hasRequiredEvidence(c) && duplicatesPreceding(c)).map(c => c.word);
1818
+ const cooldownLabel = cooledSurface ? `cooldown "${cooledSurface}" (${(_acceptCooldown$predi = (_acceptCooldown = acceptCooldown) === null || _acceptCooldown === void 0 ? void 0 : _acceptCooldown.predictionsSince) !== null && _acceptCooldown$predi !== void 0 ? _acceptCooldown$predi : 0}/${COOLDOWN_KEYSTROKES} keys · ${COOLDOWN_MS}ms)` : 'cooldown —';
1819
+ ctcSection('STABILIZE', `${cooldownLabel} · repetition-blocked: ${repetitionBlocked.length > 0 ? repetitionBlocked.join(', ') : '—'}`);
1820
+
1821
+ // CANDIDATES — full scored table (verbose only)
1822
+ if (verbose && ranked.length > 0) {
511
1823
  const tableData = ranked.slice(0, 10).map(r => {
1824
+ var _r$lmEvidence, _contextTotals$get$le3, _contextTotals$get3, _r$lmEvidence$source, _r$lmEvidence2, _runtime$separatorKin2, _runtime$canonicalTok2, _runtime$canonicalTok3, _runtime$contextKey, _r$lmEvidence3, _r$lmEvidence4;
512
1825
  let rawLogit = 'Not in Payload';
513
- if (prefixLmLogits) {
1826
+ if (((_r$lmEvidence = r.lmEvidence) === null || _r$lmEvidence === void 0 ? void 0 : _r$lmEvidence.rawLogit) !== undefined) {
1827
+ rawLogit = Number(r.lmEvidence.rawLogit.toFixed(5));
1828
+ } else if (prefixLmLogits) {
514
1829
  const val = prefixLmLogits[r.word.toLowerCase()];
515
1830
  if (val !== undefined) {
516
1831
  rawLogit = Number(val.toFixed(5));
517
1832
  }
518
1833
  }
519
1834
  const original = scoringCandidates.find(sc => sc.word === r.word);
1835
+ const runtime = runtimeBySurface.get(r.word);
1836
+ const progress = runtime ? getProgressiveSurfaceEvidence(runtime.contextKey, r.word) : null;
1837
+ const optimisticTotal = optimisticTotalByWord.get(r.word);
1838
+ const normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
1839
+ const shortlistSize = runtime ? (_contextTotals$get$le3 = (_contextTotals$get3 = contextTotals.get(runtime.contextKey)) === null || _contextTotals$get3 === void 0 ? void 0 : _contextTotals$get3.length) !== null && _contextTotals$get$le3 !== void 0 ? _contextTotals$get$le3 : 0 : 0;
1840
+ const contextLeader = runtime ? bestCandidateByContext.get(runtime.contextKey) : undefined;
520
1841
  let source = 'Unknown';
521
1842
  if (original) {
522
- if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
1843
+ if (original.termType === 'bigram') {
1844
+ source = '🔗 L2 (Bigram)';
1845
+ } else if (original.termType === 'phrase') {
1846
+ source = '🧩 L2 (Phrase)';
1847
+ } else if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
523
1848
  source = '🌍 L3 (Generic)';
524
1849
  } else if (original.sessionFreq > 0 && original.tenantFreq === 0) {
525
1850
  source = '👤 L1 (Session Only)';
@@ -527,13 +1852,40 @@ export const predict = textBefore => {
527
1852
  source = '🏢 L2 (Domain)';
528
1853
  }
529
1854
  }
1855
+ const floor = MIN_LM_POSTERIOR[r.termType];
530
1856
  return {
531
1857
  Candidate: r.word,
532
1858
  Source: source,
1859
+ Type: r.termType,
533
1860
  'Final Score': Number(r.finalScore.toFixed(4)),
1861
+ 'Bounded Score': Number(confidenceScore(r).toFixed(4)),
1862
+ 'Exact Gate Score': hasExactEvidence(r) ? Number(exactConfidenceScore(r).toFixed(4)) : '—',
1863
+ Floor: floor,
1864
+ 'Posterior Floor': optimisticPosterior(r) >= floor ? '✅' : '❌',
1865
+ 'LM Floor': hasExactEvidence(r) ? passesLmFloor(r) ? '✅' : '❌' : '⏳',
1866
+ 'Gate Reason': gateReasonFor(r),
1867
+ 'Display Eligible': hasRequiredEvidence(r) && isStabilized(r) && judgedPosterior(r) >= floor ? '✅' : '❌',
534
1868
  Semantics: Number(r.semanticScore.toFixed(4)),
535
1869
  Freq: Number(r.freqScore.toFixed(4)),
536
1870
  'LM Score': Number(r.lmScore.toFixed(4)),
1871
+ 'LM Evidence': (_r$lmEvidence$source = (_r$lmEvidence2 = r.lmEvidence) === null || _r$lmEvidence2 === void 0 ? void 0 : _r$lmEvidence2.source) !== null && _r$lmEvidence$source !== void 0 ? _r$lmEvidence$source : 'absent',
1872
+ 'Verified Tokens': r.lmEvidence ? `${r.lmEvidence.verifiedTokenCount}/${r.lmEvidence.totalTokenCount}` : '0/0',
1873
+ 'Progressive Tokens': progress ? `${progress.scoredTokenCount}/${progress.totalTokenCount}` : '0/0',
1874
+ 'LM Mean Upper': progress !== null ? Number(progress.meanTokenLogProbabilityUpperBound.toFixed(5)) : '—',
1875
+ 'Optimistic Total': optimisticTotal !== undefined ? Number(optimisticTotal.toFixed(5)) : '—',
1876
+ 'Context logSumExp': normalizer !== undefined && Number.isFinite(normalizer) ? Number(normalizer.toFixed(5)) : '—',
1877
+ 'Shortlist Size': shortlistSize,
1878
+ 'Posterior (bound)': Number(optimisticPosterior(r).toFixed(5)),
1879
+ 'Posterior (judged)': hasJudgeableEvidence(r) ? Number(judgedPosterior(r).toFixed(5)) : '—',
1880
+ 'Context Leader': contextLeader ? `${contextLeader.termType}:${contextLeader.word}` : '—',
1881
+ 'Verified Chars': r.lmEvidence ? `${r.lmEvidence.verifiedCharCount}/${r.lmEvidence.totalSurfaceCharCount}` : `0/${r.word.length}`,
1882
+ Separator: (_runtime$separatorKin2 = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin2 !== void 0 ? _runtime$separatorKin2 : 'unknown',
1883
+ 'Canonical Eligibility': (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace' ? 'incompatible-separator' : runtime.canonicalTokenIds === null ? 'missing-artifact' : 'eligible',
1884
+ 'Artifact Tokens': (_runtime$canonicalTok2 = runtime === null || runtime === void 0 ? void 0 : (_runtime$canonicalTok3 = runtime.canonicalTokenIds) === null || _runtime$canonicalTok3 === void 0 ? void 0 : _runtime$canonicalTok3.join(',')) !== null && _runtime$canonicalTok2 !== void 0 ? _runtime$canonicalTok2 : 'absent',
1885
+ 'Context Key': (_runtime$contextKey = runtime === null || runtime === void 0 ? void 0 : runtime.contextKey) !== null && _runtime$contextKey !== void 0 ? _runtime$contextKey : '—',
1886
+ 'Exact Total': ((_r$lmEvidence3 = r.lmEvidence) === null || _r$lmEvidence3 === void 0 ? void 0 : _r$lmEvidence3.totalLogProbability) !== undefined ? Number(r.lmEvidence.totalLogProbability.toFixed(5)) : '—',
1887
+ 'Exact Mean': ((_r$lmEvidence4 = r.lmEvidence) === null || _r$lmEvidence4 === void 0 ? void 0 : _r$lmEvidence4.meanTokenLogProbability) !== undefined ? Number(r.lmEvidence.meanTokenLogProbability.toFixed(5)) : '—',
1888
+ Assoc: typeof (original === null || original === void 0 ? void 0 : original.association) === 'number' ? Number(original.association.toFixed(4)) : '—',
537
1889
  'Raw Logit': rawLogit,
538
1890
  'Session Freq': (original === null || original === void 0 ? void 0 : original.sessionFreq) || 0
539
1891
  };
@@ -541,19 +1893,105 @@ export const predict = textBefore => {
541
1893
 
542
1894
  // eslint-disable-next-line no-console
543
1895
  console.table(tableData);
544
- } else {
545
- // eslint-disable-next-line no-console
546
- console.log('No candidates found.');
547
1896
  }
548
1897
 
549
1898
  // eslint-disable-next-line no-console
550
1899
  console.groupEnd();
551
1900
  }
552
- return suggestion && suggestion.length > 0 ? suggestion : null;
1901
+ const selectedContextKey = selected ? (_runtimeBySurface$get1 = runtimeBySurface.get(selected.word)) === null || _runtimeBySurface$get1 === void 0 ? void 0 : _runtimeBySurface$get1.contextKey : undefined;
1902
+ return suggestion && suggestion.length > 0 && selected && selectedEvidence ? {
1903
+ text: suggestion,
1904
+ surface: selected.word,
1905
+ termType: selected.termType,
1906
+ posterior: judgedPosterior(selected),
1907
+ meanTokenLogProbability: (_judgedEvidenceFor$me3 = (_judgedEvidenceFor6 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor6 === void 0 ? void 0 : _judgedEvidenceFor6.mean) !== null && _judgedEvidenceFor$me3 !== void 0 ? _judgedEvidenceFor$me3 : NaN,
1908
+ poolHeldExtension: poolExtensionOf(selected) !== undefined,
1909
+ shortlistSize: selectedContextKey === undefined ? 1 : (_contextTotals$get$le4 = (_contextTotals$get4 = contextTotals.get(selectedContextKey)) === null || _contextTotals$get4 === void 0 ? void 0 : _contextTotals$get4.length) !== null && _contextTotals$get$le4 !== void 0 ? _contextTotals$get$le4 : 1,
1910
+ rankScore: selectedDisplayScore,
1911
+ evidenceTier: (selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.source) === 'network-logit' ? 'network-logit' : 'canonical-full-surface',
1912
+ evidenceDepth: {
1913
+ verifiedTokens: (_selectedEvidence$ver = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedTokenCount) !== null && _selectedEvidence$ver !== void 0 ? _selectedEvidence$ver : 0,
1914
+ totalTokens: (_selectedEvidence$tot = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.totalTokenCount) !== null && _selectedEvidence$tot !== void 0 ? _selectedEvidence$tot : 0,
1915
+ verifiedChars: (_selectedEvidence$ver2 = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedCharCount) !== null && _selectedEvidence$ver2 !== void 0 ? _selectedEvidence$ver2 : 0,
1916
+ totalChars: selected.word.length
1917
+ },
1918
+ winnerMargin
1919
+ } : null;
553
1920
  };
554
1921
 
555
1922
  // ─── Data Loading ────────────────────────────────────────────────────────────
556
1923
 
1924
+ const isVocabularyJson = payload => {
1925
+ if (payload == null || typeof payload !== 'object') {
1926
+ return false;
1927
+ }
1928
+ const words = payload.words;
1929
+ return words != null && typeof words === 'object';
1930
+ };
1931
+ const isStringArray = payload => Array.isArray(payload) && payload.every(entry => typeof entry === 'string');
1932
+ const isWordToOffsetMap = value => value != null && typeof value === 'object' && Object.values(value).every(entry => typeof entry === 'number');
1933
+ const isWordIndexPayload = payload => {
1934
+ if (payload == null || typeof payload !== 'object') {
1935
+ return false;
1936
+ }
1937
+ const index = payload.index;
1938
+ return index === undefined ? isWordToOffsetMap(payload) : isWordToOffsetMap(index);
1939
+ };
1940
+
1941
+ /**
1942
+ * The word index has shipped both bare and wrapped in `{ index: ... }`, so
1943
+ * accept either rather than silently building an empty vector store.
1944
+ */
1945
+ const unwrapWordIndex = payload => {
1946
+ const wrapped = payload.index;
1947
+ return wrapped !== null && wrapped !== void 0 ? wrapped : payload;
1948
+ };
1949
+
1950
+ /**
1951
+ * Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
1952
+ * confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
1953
+ * + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
1954
+ * are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
1955
+ * a number.
1956
+ */
1957
+
1958
+ /** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
1959
+
1960
+ const isPhraseStats = value => value != null && typeof value === 'object' && typeof value.freq === 'number';
1961
+ const isTermStatsMap = value => {
1962
+ if (value == null || typeof value !== 'object') {
1963
+ return false;
1964
+ }
1965
+ const values = Object.values(value);
1966
+ return values.length > 0 && values.every(isPhraseStats);
1967
+ };
1968
+
1969
+ /**
1970
+ * Normalize a `bigrams.json` / `phrases.json` payload into a flat
1971
+ * `{term: stats}` map. Tolerant of the exact wire shape (not yet finalized by
1972
+ * the producer): accepts either the flat map from spec §9 or a single wrapper
1973
+ * key (`terms`/`bigrams`/`phrases`/`words`). Returns `null` if unrecognized.
1974
+ *
1975
+ * :params:
1976
+ * payload: Raw JSON parsed from the artifact endpoint
1977
+ * :returns:
1978
+ * A `{term: stats}` map, or `null` when the shape is not a term-stats map
1979
+ */
1980
+ const normalizePhraseArtifact = payload => {
1981
+ if (payload == null || typeof payload !== 'object') {
1982
+ return null;
1983
+ }
1984
+ if (isTermStatsMap(payload)) {
1985
+ return payload;
1986
+ }
1987
+ const obj = payload;
1988
+ for (const key of ['terms', 'bigrams', 'phrases', 'words']) {
1989
+ if (isTermStatsMap(obj[key])) {
1990
+ return obj[key];
1991
+ }
1992
+ }
1993
+ return null;
1994
+ };
557
1995
  export const loadVectorsAsync = async options => {
558
1996
  var _options$isLocalLLM;
559
1997
  if (vectorStore || vectorsLoadStarted) {
@@ -568,53 +2006,17 @@ export const loadVectorsAsync = async options => {
568
2006
  surface
569
2007
  } : {})
570
2008
  });
571
- let url;
572
2009
  try {
573
- url = await getArtifactUrl(ARTIFACT_NAME.WORD_VECTORS);
574
- } catch (e) {
575
- vectorsLoadStarted = false;
576
- failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', {
577
- isLocalLLM,
578
- errorType: 'resolve_url',
579
- ...(surface ? {
580
- surface
581
- } : {})
582
- });
583
- // eslint-disable-next-line no-console
584
- console.warn('[text-predictor] Failed to resolve vectors URL:', e);
585
- return;
586
- }
587
- try {
588
- const res = await fetch(url);
589
- if (!res.ok) {
590
- vectorsLoadStarted = false;
591
- failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', {
592
- isLocalLLM,
593
- status: res.status,
594
- errorType: 'http_error',
595
- ...(surface ? {
596
- surface
597
- } : {})
598
- });
599
- // eslint-disable-next-line no-console
600
- console.warn(`[text-predictor] Failed to load vectors: ${res.status}`);
601
- return;
602
- }
603
- const buffer = await res.arrayBuffer();
2010
+ const buffer = await fetchAutocompleteArtifactBinary(ARTIFACT_NAME.WORD_VECTORS, 'word_vectors_10k');
604
2011
  const float32 = new Float32Array(buffer);
605
-
606
- // The word index is published either bare or wrapped as `{ "index": {…} }`
607
- // (the wrapper exists so no real entry, e.g. the word "default", can shadow
608
- // a bundler-synthesised ESM export). Detect the wrapper by value type, not
609
- // by key presence — "index" is itself a plausible vocabulary word, and in a
610
- // bare index every value is a number.
611
- const wordIndexPayload = await fetchArtifactJson(ARTIFACT_NAME.WORD_INDEX);
612
- const wrappedIndex = wordIndexPayload === null || wordIndexPayload === void 0 ? void 0 : wordIndexPayload.index;
613
- const wordIndex = typeof wrappedIndex === 'object' && wrappedIndex !== null ? wrappedIndex : wordIndexPayload !== null && wordIndexPayload !== void 0 ? wordIndexPayload : {};
2012
+ const wordIndexPayload = await fetchAutocompleteArtifactJson(ARTIFACT_NAME.WORD_INDEX, {
2013
+ summarize: payload => `${Object.keys(unwrapWordIndex(payload)).length} entries`,
2014
+ validate: isWordIndexPayload
2015
+ });
2016
+ const wordIndex = unwrapWordIndex(wordIndexPayload);
614
2017
  const nWords = Object.keys(wordIndex).length;
615
2018
  if (nWords === 0) {
616
- // eslint-disable-next-line no-console
617
- console.warn(`[text-predictor] ${ARTIFACT_NAME.WORD_INDEX} was empty — semantic scoring will be a no-op.`);
2019
+ ctcTag('init', '⚠️ word_index_10k.json was empty — semantic scoring will be a no-op', CTC_STYLES.warn);
618
2020
  }
619
2021
  const dim = float32.length / nWords;
620
2022
  vectorStore = {
@@ -631,14 +2033,7 @@ export const loadVectorsAsync = async options => {
631
2033
  surface
632
2034
  } : {})
633
2035
  });
634
- if (isAutocompleteDebugEnabled()) {
635
- // eslint-disable-next-line no-console
636
- console.log('[text-predictor] Vectors loaded:', {
637
- wordCount: nWords,
638
- dim,
639
- sizeBytes: float32.byteLength
640
- });
641
- }
2036
+ ctcTag('init', `vectors loaded: ${nWords} words · dim ${dim} · ${float32.byteLength}B`);
642
2037
  } catch (e) {
643
2038
  vectorsLoadStarted = false;
644
2039
  failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', {
@@ -648,23 +2043,88 @@ export const loadVectorsAsync = async options => {
648
2043
  surface
649
2044
  } : {})
650
2045
  });
651
- // eslint-disable-next-line no-console
652
- console.warn('[text-predictor] Failed to load vectors:', e);
2046
+ ctcTag('init', `⚠️ failed to load vectors: ${String(e)}`, CTC_STYLES.bad);
653
2047
  }
654
2048
  };
655
2049
  export const initVectors = store => {
656
2050
  vectorStore = store;
657
2051
  };
2052
+
2053
+ /**
2054
+ * Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
2055
+ * over the TDP-OS gateway and insert them into the phrase trie.
2056
+ *
2057
+ * Each file is fetched independently (`Promise.allSettled`) so a missing or
2058
+ * late-published artifact only skips that term type — word completion and the
2059
+ * other term type are unaffected. Phrase VECTORS are not fetched here: per the
2060
+ * producer's `extend` decision they are appended into the existing
2061
+ * `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
2062
+ * covers them.
2063
+ *
2064
+ * :params:
2065
+ * options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
2066
+ * :returns:
2067
+ * A promise that resolves once both fetches have settled
2068
+ */
2069
+ export const loadPhraseArtifacts = async options => {
2070
+ var _options$isLocalLLM2;
2071
+ if (phrasesLoadStarted) {
2072
+ return;
2073
+ }
2074
+ phrasesLoadStarted = true;
2075
+ const isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2076
+ startExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2077
+ isLocalLLM
2078
+ });
2079
+ const loadOne = async (artifactName, termType, label) => {
2080
+ const payload = await fetchAutocompleteArtifactJson(artifactName, {
2081
+ label,
2082
+ validate: p => p != null && typeof p === 'object'
2083
+ });
2084
+ const normalized = normalizePhraseArtifact(payload);
2085
+ if (!normalized) {
2086
+ throw new Error(`[text-predictor] ${label} payload was not a recognised term-stats map`);
2087
+ }
2088
+ return initPhrases(normalized, termType);
2089
+ };
2090
+ const [bigramsResult, phrasesResult] = await Promise.allSettled([loadOne(ARTIFACT_NAME.BIGRAMS, 'bigram', 'bigrams'), loadOne(ARTIFACT_NAME.PHRASES, 'phrase', 'phrases')]);
2091
+ const bigramCount = bigramsResult.status === 'fulfilled' ? bigramsResult.value : 0;
2092
+ const phraseCount = phrasesResult.status === 'fulfilled' ? phrasesResult.value : 0;
2093
+ if (bigramsResult.status === 'rejected') {
2094
+ ctcTag('init', `⚠️ failed to load bigrams.json: ${String(bigramsResult.reason)}`, CTC_STYLES.warn);
2095
+ }
2096
+ if (phrasesResult.status === 'rejected') {
2097
+ ctcTag('init', `⚠️ failed to load phrases.json: ${String(phrasesResult.reason)}`, CTC_STYLES.warn);
2098
+ }
2099
+ if (bigramsResult.status === 'fulfilled' || phrasesResult.status === 'fulfilled') {
2100
+ phrasesLoaded = true;
2101
+ succeedExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2102
+ isLocalLLM,
2103
+ bigramCount,
2104
+ phraseCount
2105
+ });
2106
+ ctcTag('init', `phrase artifacts loaded: ${bigramCount} bigrams · ${phraseCount} phrases`);
2107
+ } else {
2108
+ // Both failed — e.g. artifacts not published to the tenant manifest yet.
2109
+ // Reset the guard so a later focus/predict can retry once they land.
2110
+ phrasesLoadStarted = false;
2111
+ failExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2112
+ isLocalLLM,
2113
+ errorType: 'network'
2114
+ });
2115
+ }
2116
+ };
658
2117
  let vocabularyLoadPromise;
659
2118
  export const loadDefaultVocabulary = options => {
660
- var _options$isLocalLLM2;
2119
+ var _options$isLocalLLM3, _options$source;
661
2120
  if (isInitialized) {
662
2121
  return Promise.resolve();
663
2122
  }
664
2123
  if (vocabularyLoadPromise) {
665
2124
  return vocabularyLoadPromise;
666
2125
  }
667
- const isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2126
+ const isLocalLLM = (_options$isLocalLLM3 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM3 !== void 0 ? _options$isLocalLLM3 : false;
2127
+ const source = (_options$source = options === null || options === void 0 ? void 0 : options.source) !== null && _options$source !== void 0 ? _options$source : 'predict';
668
2128
  const surface = options === null || options === void 0 ? void 0 : options.surface;
669
2129
  vocabularyLoadPromise = (async () => {
670
2130
  startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
@@ -674,10 +2134,36 @@ export const loadDefaultVocabulary = options => {
674
2134
  } : {})
675
2135
  });
676
2136
  try {
677
- const [vocabularyData, l3VocabularyData] = await Promise.all([fetchArtifactJson(ARTIFACT_NAME.VOCABULARY), fetchArtifactJson(ARTIFACT_NAME.L3_VOCABULARY)]);
678
- if ((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData)) {
679
- throw new Error(`[text-predictor] unexpected vocabulary payload — ` + `${ARTIFACT_NAME.VOCABULARY} ${(vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null ? 'is missing its `words` wrapper' : 'is valid'}, ` + `${ARTIFACT_NAME.L3_VOCABULARY} ${Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'}`);
2137
+ ctcTag('init', `loading artifacts (trigger: ${source} · localLLM: ${isLocalLLM})`);
2138
+ if (isAutocompleteDebugVerbose()) {
2139
+ // eslint-disable-next-line no-console
2140
+ console.groupCollapsed('%c[CTC:init]%c artifact manifest', CTC_STYLES.brand, CTC_STYLES.body);
2141
+ // eslint-disable-next-line no-console
2142
+ console.log('Eager:', [ARTIFACT_NAME.VOCABULARY, ARTIFACT_NAME.L3_VOCABULARY, ARTIFACT_NAME.POS_TAGS, ARTIFACT_NAME.GHOST_POS_TAGS, ARTIFACT_NAME.GRAMMAR_TRANSITIONS]);
2143
+ // eslint-disable-next-line no-console
2144
+ console.log('Conditional:', [`${ARTIFACT_NAME.WORD_INDEX} (only when vectors load; phrase/bigram surfaces share this index)`, `${ARTIFACT_NAME.BIGRAMS} + ${ARTIFACT_NAME.PHRASES} (completion-term stats; fire-and-forget)`, `${ARTIFACT_NAME.PHRASE_CONTINUATION_TOKENS} (local model only; canonical token IDs for the full served union)`]);
2145
+ // eslint-disable-next-line no-console
2146
+ console.groupEnd();
680
2147
  }
2148
+ const [vocabularyData, l3VocabularyData] = await Promise.all([fetchAutocompleteArtifactJson(ARTIFACT_NAME.VOCABULARY, {
2149
+ summarize: payload => `${Object.keys(payload.words).length} words`,
2150
+ validate: isVocabularyJson
2151
+ }), fetchAutocompleteArtifactJson(ARTIFACT_NAME.L3_VOCABULARY, {
2152
+ summarize: payload => `${payload.length} words`,
2153
+ validate: isStringArray
2154
+ }),
2155
+ // Awaited alongside the vocabulary so the grammar filter is settled
2156
+ // before the first suggestion can be produced. Without this the
2157
+ // predictor initialises on the smaller vocabulary payload and serves
2158
+ // candidates the filter would have dropped until the grammar lands.
2159
+ // Only settlement is required: a failed load leaves the filter as a
2160
+ // pass-through, which is the same degradation as an absent artifact.
2161
+ loadGrammarDataAsync({
2162
+ isLocalLLM,
2163
+ surface
2164
+ }).catch(error => {
2165
+ ctcTag('init', `⚠️ grammar data unavailable; filtering skipped: ${String(error)}`, CTC_STYLES.warn);
2166
+ })]);
681
2167
  const terms = Object.entries(vocabularyData.words).map(([word, stats]) => ({
682
2168
  word,
683
2169
  freq: stats.freq,
@@ -692,6 +2178,12 @@ export const loadDefaultVocabulary = options => {
692
2178
  initVocabulary({
693
2179
  terms
694
2180
  });
2181
+
2182
+ // Phrase/bigram artifacts load independently and must never affect the
2183
+ // word path — fire-and-forget with its own error handling inside.
2184
+ void loadPhraseArtifacts({
2185
+ isLocalLLM
2186
+ }).catch(() => {});
695
2187
  succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
696
2188
  isLocalLLM,
697
2189
  l2WordCount: terms.length,
@@ -703,7 +2195,7 @@ export const loadDefaultVocabulary = options => {
703
2195
  } catch (e) {
704
2196
  failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
705
2197
  isLocalLLM,
706
- errorType: 'load_error',
2198
+ errorType: 'parse_error',
707
2199
  ...(surface ? {
708
2200
  surface
709
2201
  } : {})