@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +108 -0
  2. package/dist/cjs/analytics/ufo.js +2 -1
  3. package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
  4. package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
  5. package/dist/cjs/pm-plugins/autocomplete-plugin.js +498 -160
  6. package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
  7. package/dist/cjs/pm-plugins/debug-mode.js +105 -7
  8. package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
  9. package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
  10. package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
  11. package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
  12. package/dist/cjs/pm-plugins/text-predictor.js +2099 -309
  13. package/dist/es2019/analytics/ufo.js +2 -0
  14. package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
  15. package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
  16. package/dist/es2019/pm-plugins/autocomplete-plugin.js +489 -152
  17. package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
  18. package/dist/es2019/pm-plugins/debug-mode.js +102 -7
  19. package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
  20. package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
  21. package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
  22. package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
  23. package/dist/es2019/pm-plugins/text-predictor.js +1692 -200
  24. package/dist/esm/analytics/ufo.js +2 -1
  25. package/dist/esm/pm-plugins/artifact-loader.js +128 -0
  26. package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
  27. package/dist/esm/pm-plugins/autocomplete-plugin.js +501 -163
  28. package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
  29. package/dist/esm/pm-plugins/debug-mode.js +104 -6
  30. package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
  31. package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
  32. package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
  33. package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
  34. package/dist/esm/pm-plugins/text-predictor.js +2101 -313
  35. package/dist/types/analytics/ufo.d.ts +1 -0
  36. package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
  37. package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
  38. package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
  39. package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
  40. package/dist/types/pm-plugins/debug-mode.d.ts +55 -1
  41. package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
  42. package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
  43. package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
  44. package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
  45. package/dist/types/pm-plugins/text-predictor.d.ts +156 -3
  46. package/package.json +2 -2
  47. package/src/analytics/ufo.ts +5 -0
  48. package/src/pm-plugins/artifact-loader.ts +92 -0
  49. package/src/pm-plugins/artifacts-manifest.ts +3 -0
  50. package/src/pm-plugins/autocomplete-plugin.ts +584 -154
  51. package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
  52. package/src/pm-plugins/debug-mode.ts +134 -7
  53. package/src/pm-plugins/ghost-text-decoration.ts +13 -4
  54. package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
  55. package/src/pm-plugins/scoring-pipeline.ts +167 -17
  56. package/src/pm-plugins/slow-lane-client.ts +147 -62
  57. package/src/pm-plugins/text-predictor.ts +2119 -254
  58. package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
  59. package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
  60. package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
  61. package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
  62. package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
  63. package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
  64. package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  65. package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  66. package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
  67. package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  68. package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  69. package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
  70. package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
  71. package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
  72. package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
  73. package/scripts/gen_first_token_to_words.py +0 -170
  74. package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
  75. package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
  76. package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
  77. package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
  78. package/src/pm-plugins/slow-lane-client/package.json +0 -8
  79. package/src/pm-plugins/text-predictor/package.json +0 -8
@@ -1,5 +1,6 @@
1
- import _typeof from "@babel/runtime/helpers/typeof";
2
1
  import _asyncToGenerator from "@babel/runtime/helpers/asyncToGenerator";
2
+ import _typeof from "@babel/runtime/helpers/typeof";
3
+ import _toConsumableArray from "@babel/runtime/helpers/toConsumableArray";
3
4
  import _slicedToArray from "@babel/runtime/helpers/slicedToArray";
4
5
  import _createClass from "@babel/runtime/helpers/createClass";
5
6
  import _classCallCheck from "@babel/runtime/helpers/classCallCheck";
@@ -15,7 +16,7 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
15
16
  *
16
17
  * Two prediction modes:
17
18
  * 1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
18
- * 2. Mid-word (≥3 chars) → trie prefix search → scoring pipeline → top result
19
+ * 2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
19
20
  *
20
21
  * Scoring is delegated to scoring-pipeline.ts which handles:
21
22
  * Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
@@ -24,18 +25,19 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
24
25
  * Falls back to cold mode (freq-only) when vectors not yet loaded.
25
26
  *
26
27
  * Session personalization (L1): words the user types are incrementally boosted
27
- * via incrementSessionFreq(), called on word boundaries from the plugin.
28
+ * via incrementSessionFreq(), called on word boundaries from the plugin, and
29
+ * words in ingested context text via ingestDocumentPage(). What the session has
30
+ * boosted is visible at any time from the console: `__atlCtcDebug__.session()`,
31
+ * or `__atlCtcDebug__.session('poll')` for one family — see inspectSessionBoosts.
28
32
  */
29
33
 
30
34
  import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
31
-
32
- // The vocabulary, L3 and word-index payloads are fetched from the CDN in
33
- // loadDefaultVocabulary / loadVectorsAsync so their (large) contents never enter
34
- // the editor's bundle and only load when autocomplete is initialised.
35
- import { ARTIFACT_NAME, fetchArtifactJson, getArtifactUrl } from './artifacts-manifest';
36
- import { isAutocompleteDebugEnabled } from './debug-mode';
37
- import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
38
- import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
35
+ import { fetchAutocompleteArtifactBinary, fetchAutocompleteArtifactJson } from './artifact-loader';
36
+ import { ARTIFACT_NAME } from './artifacts-manifest';
37
+ import { createCanonicalContextPositionCache, deriveCanonicalCandidateContext, deriveWhitespaceBoundaryContext, logSoftmaxAt, logSumExp, selectBoundaryPrimeRequests } from './canonical-lm-scoring';
38
+ import { CTC_STYLES, ctcSection, ctcTag, isAutocompleteDebugEnabled, isAutocompleteDebugVerbose, registerCtcSessionInspector } from './debug-mode';
39
+ import { loadGrammarDataAsync, rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE, MIN_WINNER_MARGIN } from './scoring-pipeline';
40
+ import { getBoundaryLmState, getCanonicalSurfaceCount, getCanonicalSurfaceTokenIds, getDefaultSlowLaneClientStatus, getProgressiveSurfaceEvidence, getStoredContextInput, getStoredContextVector, getStoredLmLogits, getSurfaceScore, isCanonicalSurfaceScoringSupported, primeBoundaryLm, requestProgressiveSurfaceScores } from './slow-lane-client';
39
41
 
40
42
  // ─── Constants ───────────────────────────────────────────────────────────────
41
43
 
@@ -47,13 +49,152 @@ var WHITESPACE_SPLIT_REGEX = /\s+/;
47
49
  var SENTENCE_BOUNDARY_REGEX = /[\n.?!]+/;
48
50
  // eslint-disable-next-line require-unicode-regexp
49
51
  var TRAILING_WHITESPACE_REGEX = /\s$/;
50
- var MIN_PREFIX_LENGTH = 3;
52
+ var TRAILING_SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+$/;
53
+ var SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+/g;
54
+ var ONLY_WHITESPACE_REGEX = /^[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+$/;
55
+ var UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH = 1;
56
+ var MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH = 1;
57
+ var DISPLAY_MIN_PREFIX_LENGTH = 3;
51
58
  var MAX_CANDIDATES = 200;
52
59
  var CONTEXT_WORDS = 10;
53
- var MIN_SCORE_THRESHOLD = 0.35;
54
60
  var L3_BASELINE_FREQ = 0.001;
61
+ // Max tokens in a completion term (bigram=2, phrase=3). Mirrors the producer's
62
+ // `phrase_max_words` default so the FE window never exceeds what was mined.
63
+ var PHRASE_MAX_WORDS = 3;
64
+ // Per-window cap on phrase-trie subtree collection (keeps the hot path bounded).
65
+ var MAX_PHRASE_CANDIDATES = 50;
66
+ /**
67
+ * Share of its context shortlist's probability mass the model must put on a
68
+ * surface before that surface may be shown.
69
+ *
70
+ * Candidates competing at the same boundary are normalised into a posterior
71
+ * over that shortlist, so this reads directly as "the model is at least this
72
+ * sure". Measuring distance from the best candidate instead cannot express
73
+ * being unsure: the leader scores full marks whether it won by a nose or a
74
+ * mile, which pushes the entire question of how contested a context is onto
75
+ * `MIN_WINNER_MARGIN` — and there the blended score's only remaining spread is
76
+ * the corpus prior, the weakest ranking signal we have.
77
+ *
78
+ * Each extra token raises the bar. A wrong multi-word ghost costs the reader
79
+ * more to notice and undo than a wrong single word, and it is offered from the
80
+ * same keystroke, so precision has to be bought per token of commitment.
81
+ */
82
+ var MIN_LM_POSTERIOR = {
83
+ word: 0.5,
84
+ bigram: 0.6,
85
+ phrase: 0.7
86
+ };
87
+ /**
88
+ * How many scored candidates a context must hold before its leader may show.
89
+ *
90
+ * A posterior is a share of a shortlist, so a shortlist of one hands its only
91
+ * member 1.0 whatever the model thinks of it — `logSumExp` of a single value
92
+ * returns that value. Such a ghost sits in the top bucket without having beaten
93
+ * anything, and it is the gate's blind spot rather than a case the gate ruled
94
+ * on: 1 in 14 of them were accepted against 34% for word ghosts overall, and
95
+ * every phrase shown through this hole was rejected.
96
+ *
97
+ * Counted over pool membership rather than over the normaliser, because those
98
+ * two deliberately differ. Where a pool holds a surface and its own extension
99
+ * the chain rule leaves one term in the denominator, and the resulting 1.0 is
100
+ * earned — whichever the user meant, the shorter form is a correct ghost.
101
+ *
102
+ * Waived where the context never held a second candidate at all. The 1-in-14
103
+ * acceptance above was measured over a population dominated by pools that read
104
+ * as one because rivals had not come back, and it is that absence of evidence
105
+ * the floor is for. A vocabulary offering a single continuation of a long
106
+ * distinctive prefix is the opposite situation, and it is also where a ghost
107
+ * saves the most keystrokes, so refusing it spends the most to learn the least.
108
+ */
109
+ var MIN_SCORED_POOL_SIZE = 2;
110
+ /**
111
+ * Mean per-token log-probability a surface must hold, in nats, independently of
112
+ * anything it was competing against.
113
+ *
114
+ * Every other display test is relative: a posterior is a share of a shortlist,
115
+ * and a winner margin is a distance from a rival. So a surface reaches the
116
+ * screen by being the best of what happened to be in its pool, and nothing ever
117
+ * asks whether the model finds it plausible at all. That gap was tolerable while
118
+ * little was being read, and stopped being tolerable once the scheduler started
119
+ * reading roughly twice as much per decision: shown-per-decision went from 3.2%
120
+ * to 7.4% with no threshold moved, and acceptance of what showed fell from 32%
121
+ * to 24.5%. More reading does not weaken a relative test — it hands it far more
122
+ * pools to be the winner of, and the marginal winner is worse than the average.
123
+ *
124
+ * Set at the point where a uniform distribution over the model's vocabulary
125
+ * sits: 49,152 tokens, so `ln(1/49152)` is about -10.8 nats. Below it the model
126
+ * assigns the surface less mass than a token drawn at random, which is the only
127
+ * statement about a surface that needs no reference to what it competed with.
128
+ *
129
+ * It was first tried at -8, chosen from the bucket where acceptance was measured
130
+ * to fall away, and that reasoning does not survive more data. Acceptance against
131
+ * this quantity turned out to be a smooth slope — about 19% below -8, 26% from -8
132
+ * to -6, then flat near 35% above - with no discontinuity to place a threshold
133
+ * at. A floor on a smooth slope is a coverage dial rather than a test: -8 removed
134
+ * a band converting at 26% and cost 3.7 points of show rate, for an acceptance
135
+ * movement no sample of this size can resolve. Anchoring to uniform instead makes
136
+ * the value an argument about what is indefensible rather than a point picked off
137
+ * a histogram, which is what stops it from being retuned every batch.
138
+ *
139
+ * Applied to the mean over the verified prefix, the same quantity
140
+ * `ghostsByMeanLogProb` buckets, and deliberately not to
141
+ * `meanTokenLogProbabilityUpperBound`. The bound is the right instrument for
142
+ * keeping a candidate in the race, where dropping something that could still
143
+ * clear the bar would be an error, but it is systematically higher than the
144
+ * quantity acceptance is measured against. Judging a surface on the prefix that
145
+ * was actually read is also what every other test here does.
146
+ *
147
+ * One value across all three shapes. Splitting it per shape needs an acceptance
148
+ * curve per shape, and 19 bigrams and 1 phrase across five sessions cannot
149
+ * support one.
150
+ */
151
+ var MIN_MEAN_TOKEN_LOG_PROBABILITY = -10.8;
152
+ // Tab replaces the whole remaining suffix, so an N-character completion saves
153
+ // N-1 keystrokes: at one character it saves nothing at all and at two it saves
154
+ // one, which does not repay noticing the ghost and reaching for Tab. Offering
155
+ // them anyway spends the acceptance-rate denominator on completions nobody
156
+ // wants and teaches the user to stop reading ghosts. `DISPLAY_MIN_PREFIX_LENGTH`
157
+ // is the same guard on the side the user has already typed.
158
+ var MIN_SUGGESTION_LENGTH = 3;
159
+ /**
160
+ * How many leading tokens of a surface must be scored before it may be shown.
161
+ *
162
+ * Scoring a token costs one model round trip, and a round trip costs about the
163
+ * same no matter how much is in it, so the price of a candidate is its token
164
+ * count. Demanding every token therefore caps what can ever reach the screen at
165
+ * roughly two tokens inside the decision budget — which is why no phrase has
166
+ * ever been displayed. Verifying a fixed prefix makes the cost of a candidate
167
+ * independent of its length.
168
+ */
169
+ var REQUIRED_VERIFIED_TOKENS = 2;
170
+ /**
171
+ * Extra winner margin demanded of a surface judged on a verified prefix rather
172
+ * than on every one of its tokens, to buy back the precision given up by
173
+ * leaving the tail unread.
174
+ */
175
+ var PARTIAL_EVIDENCE_MARGIN_PREMIUM = 0.08;
176
+ var ARBITRATION_MODE = 'confidence-v2';
177
+ var DEBUG_TEXT_TAIL_CHARS = 120;
55
178
 
56
179
  // ─── Types ───────────────────────────────────────────────────────────────────
180
+
181
+ /**
182
+ * Which constraint stopped an evaluation from putting a ghost on screen.
183
+ *
184
+ * Each one implies different work: `below-posterior-gate` is a threshold to
185
+ * calibrate, `winner-margin` is two candidates the model cannot separate,
186
+ * `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
187
+ * is vocabulary coverage.
188
+ */
189
+
190
+ /**
191
+ * A candidate paired with the length of the already-typed prefix it completes.
192
+ * For a single word this is the current partial token length; for a phrase it
193
+ * is the length of the matched multi-word window (previous words + partial),
194
+ * so the ghost tail is `word.slice(matchedPrefixLen)`.
195
+ */
196
+
197
+ /** Optional per-term metadata carried by bigram/phrase completion terms. */
57
198
  var TrieNode = /*#__PURE__*/_createClass(function TrieNode() {
58
199
  _classCallCheck(this, TrieNode);
59
200
  _defineProperty(this, "children", new Map());
@@ -62,6 +203,12 @@ var TrieNode = /*#__PURE__*/_createClass(function TrieNode() {
62
203
  _defineProperty(this, "docFreq", 0);
63
204
  _defineProperty(this, "authorFreq", 0);
64
205
  _defineProperty(this, "sessionFreq", 0);
206
+ // Term-type + phrase metadata. Plain words keep `termType='word'` and null
207
+ // metadata; bigram/phrase nodes carry the producer-shipped POS + association.
208
+ _defineProperty(this, "termType", 'word');
209
+ _defineProperty(this, "posSeq", null);
210
+ _defineProperty(this, "headPos", null);
211
+ _defineProperty(this, "association", null);
65
212
  });
66
213
  var WeightedWordTrie = /*#__PURE__*/function () {
67
214
  function WeightedWordTrie() {
@@ -72,7 +219,8 @@ var WeightedWordTrie = /*#__PURE__*/function () {
72
219
  }
73
220
  return _createClass(WeightedWordTrie, [{
74
221
  key: "insert",
75
- value: function insert(word, tenantFreq, docFreq, authorFreq) {
222
+ value: function insert(word, tenantFreq, docFreq, authorFreq, meta) {
223
+ var _meta$termType, _meta$posSeq, _meta$headPos, _meta$association;
76
224
  var node = this.root;
77
225
  var _iterator = _createForOfIteratorHelper(word.toLowerCase()),
78
226
  _step;
@@ -95,6 +243,10 @@ var WeightedWordTrie = /*#__PURE__*/function () {
95
243
  node.tenantFreq = tenantFreq;
96
244
  node.docFreq = docFreq;
97
245
  node.authorFreq = authorFreq;
246
+ node.termType = (_meta$termType = meta === null || meta === void 0 ? void 0 : meta.termType) !== null && _meta$termType !== void 0 ? _meta$termType : 'word';
247
+ node.posSeq = (_meta$posSeq = meta === null || meta === void 0 ? void 0 : meta.posSeq) !== null && _meta$posSeq !== void 0 ? _meta$posSeq : null;
248
+ node.headPos = (_meta$headPos = meta === null || meta === void 0 ? void 0 : meta.headPos) !== null && _meta$headPos !== void 0 ? _meta$headPos : null;
249
+ node.association = (_meta$association = meta === null || meta === void 0 ? void 0 : meta.association) !== null && _meta$association !== void 0 ? _meta$association : null;
98
250
  if (tenantFreq > this.maxTenantFreq) {
99
251
  this.maxTenantFreq = tenantFreq;
100
252
  }
@@ -207,6 +359,65 @@ var WeightedWordTrie = /*#__PURE__*/function () {
207
359
  node.sessionFreq += 1;
208
360
  return true;
209
361
  }
362
+
363
+ /**
364
+ * Every word carrying a session boost, optionally limited to one prefix's
365
+ * subtree. Unlike `getCandidates` a word equal to the prefix is included,
366
+ * since the question here is what the session holds rather than what could
367
+ * still be typed.
368
+ *
369
+ * Walks the trie instead of reading an index, so nothing has to be kept in
370
+ * step on the write path for the sake of being able to ask.
371
+ */
372
+ }, {
373
+ key: "collectSessionBoosted",
374
+ value: function collectSessionBoosted() {
375
+ var prefix = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : '';
376
+ var node = this.root;
377
+ var _iterator5 = _createForOfIteratorHelper(prefix.toLowerCase()),
378
+ _step5;
379
+ try {
380
+ for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
381
+ var char = _step5.value;
382
+ var next = node.children.get(char);
383
+ if (!next) {
384
+ return [];
385
+ }
386
+ node = next;
387
+ }
388
+ } catch (err) {
389
+ _iterator5.e(err);
390
+ } finally {
391
+ _iterator5.f();
392
+ }
393
+ var boosted = [];
394
+ var stack = [node];
395
+ while (stack.length > 0) {
396
+ var current = stack.pop();
397
+ if (!current) {
398
+ continue;
399
+ }
400
+ if (current.word !== null && current.sessionFreq > 0) {
401
+ boosted.push({
402
+ word: current.word,
403
+ node: current
404
+ });
405
+ }
406
+ var _iterator6 = _createForOfIteratorHelper(current.children.values()),
407
+ _step6;
408
+ try {
409
+ for (_iterator6.s(); !(_step6 = _iterator6.n()).done;) {
410
+ var child = _step6.value;
411
+ stack.push(child);
412
+ }
413
+ } catch (err) {
414
+ _iterator6.e(err);
415
+ } finally {
416
+ _iterator6.f();
417
+ }
418
+ }
419
+ return boosted;
420
+ }
210
421
  }]);
211
422
  }(); // L1/L2 Trie (Session + Atlassian Domain)
212
423
  var wordTrie = new WeightedWordTrie();
@@ -214,29 +425,36 @@ var wordTrie = new WeightedWordTrie();
214
425
  // L3 Trie (General English Fallback)
215
426
  var l3Trie = new WeightedWordTrie();
216
427
 
428
+ /**
429
+ * L2 phrase/bigram trie. Kept SEPARATE from `wordTrie` so single-word
430
+ * completion + L3 gap-fill stay untouched; multi-word units are only surfaced
431
+ * by the (12b) multiword prefix-window matcher. Keys are the full normalized
432
+ * surface string (spaces included), e.g. `"return on investment"`, so a lookup
433
+ * is a plain char-trie walk over the whole phrase.
434
+ */
435
+ var phraseTrie = new WeightedWordTrie();
436
+
217
437
  /**
218
438
  * Loads the General English vocabulary.
219
439
  * expects a simple array of strings: ["about", "above", "actually", ...]
220
440
  */
221
441
  export var initL3Vocabulary = function initL3Vocabulary(l3Words) {
222
- var _iterator5 = _createForOfIteratorHelper(l3Words),
223
- _step5;
442
+ var _iterator7 = _createForOfIteratorHelper(l3Words),
443
+ _step7;
224
444
  try {
225
- for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
226
- var word = _step5.value;
445
+ for (_iterator7.s(); !(_step7 = _iterator7.n()).done;) {
446
+ var word = _step7.value;
227
447
  // Insert with a tiny baseline frequency so it mathematically
228
448
  // loses to any domain word in Stage 1, but still scores above 0.
229
449
  l3Trie.insert(word, L3_BASELINE_FREQ, 0, 0);
230
450
  }
231
451
  } catch (err) {
232
- _iterator5.e(err);
452
+ _iterator7.e(err);
233
453
  } finally {
234
- _iterator5.f();
235
- }
236
- if (isAutocompleteDebugEnabled()) {
237
- // eslint-disable-next-line no-console
238
- console.log("[text-predictor] L3 General English loaded: ".concat(l3Words.length, " words"));
454
+ _iterator7.f();
239
455
  }
456
+ recallGeneration++;
457
+ ctcTag('init', "L3 general English loaded: ".concat(l3Words.length, " words"));
240
458
  };
241
459
 
242
460
  // const bigramMap: Map<string, Record<string, number>> = new Map(
@@ -246,8 +464,50 @@ export var initL3Vocabulary = function initL3Vocabulary(l3Words) {
246
464
  var isInitialized = false;
247
465
  var vectorStore = null;
248
466
  var vectorsLoadStarted = false;
467
+
468
+ // Phrase/bigram artifacts load independently of the word vocabulary so a
469
+ // missing/late `bigrams.json`/`phrases.json` never breaks word completion.
470
+ var phrasesLoadStarted = false;
471
+ var phrasesLoaded = false;
472
+ var phraseTermCount = 0;
473
+ // Per-type frequency maxima, tracked separately because bigrams and phrases
474
+ // share one trie but have very different absolute frequency ranges (bigrams are
475
+ // far more frequent than 3-word phrases). Normalizing each type by its OWN max
476
+ // stops phrase freq scores from being crushed against the top bigram's count.
477
+ var maxBigramFreq = 1;
478
+ var maxPhraseFreq = 1;
249
479
  var lastPredictionDebug = null;
250
480
 
481
+ // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
482
+ // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
483
+ // generally keep a just-accepted unit from being re-offered:
484
+ // • repetition: never surface a candidate whose full surface already equals the
485
+ // run of tokens immediately before the cursor (handled inline in predict()).
486
+ // • cooldown: after an accept, briefly suppress re-offering that same unit. The
487
+ // cooldown lifts only once BOTH the keystroke budget and the time window have
488
+ // elapsed (either one still counting keeps it active).
489
+ var COOLDOWN_KEYSTROKES = 5;
490
+ var COOLDOWN_MS = 2000;
491
+
492
+ // Active post-accept cooldown, or null when none.
493
+ var acceptCooldown = null;
494
+
495
+ /**
496
+ * Start a short cooldown for the exact surface the editor inserted. The caller
497
+ * passes the committed snapshot's surface so background re-ranking can never
498
+ * move cooldown bookkeeping away from what the user actually accepted.
499
+ */
500
+ export var noteSuggestionAccepted = function noteSuggestionAccepted(surface) {
501
+ var normalizedSurface = surface.trim().toLowerCase();
502
+ if (normalizedSurface) {
503
+ acceptCooldown = {
504
+ predictionsSince: 0,
505
+ surface: normalizedSurface,
506
+ ts: performance.now()
507
+ };
508
+ }
509
+ };
510
+
251
511
  /** Get vector for a word from the store. */
252
512
  var getWordVector = function getWordVector(word) {
253
513
  if (!vectorStore) {
@@ -272,20 +532,20 @@ var computeContextVectorLocal = function computeContextVectorLocal(textBefore) {
272
532
  var tokens = tokenize(textBefore);
273
533
  var words = tokens.slice(-CONTEXT_WORDS);
274
534
  var vectors = [];
275
- var _iterator6 = _createForOfIteratorHelper(words),
276
- _step6;
535
+ var _iterator8 = _createForOfIteratorHelper(words),
536
+ _step8;
277
537
  try {
278
- for (_iterator6.s(); !(_step6 = _iterator6.n()).done;) {
279
- var word = _step6.value;
538
+ for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
539
+ var word = _step8.value;
280
540
  var _v = getWordVector(word);
281
541
  if (_v) {
282
542
  vectors.push(_v);
283
543
  }
284
544
  }
285
545
  } catch (err) {
286
- _iterator6.e(err);
546
+ _iterator8.e(err);
287
547
  } finally {
288
- _iterator6.f();
548
+ _iterator8.f();
289
549
  }
290
550
  if (vectors.length === 0) {
291
551
  return null;
@@ -318,11 +578,11 @@ var getContextVectorForScoring = function getContextVectorForScoring(textBefore)
318
578
  var tokenize = function tokenize(text) {
319
579
  var tokens = [];
320
580
  // eslint-disable-next-line @atlassian/perf-linting/no-expensive-split-replace
321
- var _iterator7 = _createForOfIteratorHelper(text.toLowerCase().split(WHITESPACE_SPLIT_REGEX)),
322
- _step7;
581
+ var _iterator9 = _createForOfIteratorHelper(text.toLowerCase().split(WHITESPACE_SPLIT_REGEX)),
582
+ _step9;
323
583
  try {
324
- for (_iterator7.s(); !(_step7 = _iterator7.n()).done;) {
325
- var raw = _step7.value;
584
+ for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
585
+ var raw = _step9.value;
326
586
  // eslint-disable-next-line @atlassian/perf-linting/no-expensive-split-replace
327
587
  var clean = raw.replace(PUNCTUATION_BOUNDARY_REGEX, '');
328
588
  if (clean.length >= 2) {
@@ -330,9 +590,9 @@ var tokenize = function tokenize(text) {
330
590
  }
331
591
  }
332
592
  } catch (err) {
333
- _iterator7.e(err);
593
+ _iterator9.e(err);
334
594
  } finally {
335
- _iterator7.f();
595
+ _iterator9.f();
336
596
  }
337
597
  return tokens;
338
598
  };
@@ -344,6 +604,124 @@ var extractPreviousWord = function extractPreviousWord(text) {
344
604
  return words.length >= 2 ? words[words.length - 2] : '';
345
605
  };
346
606
 
607
+ /**
608
+ * Collect phrase/bigram completion candidates for the multi-word window ending
609
+ * at the current partial token.
610
+ *
611
+ * Builds windows of the last `2..PHRASE_MAX_WORDS` tokens of the current
612
+ * sentence (widest first) and prefix-searches the phrase trie for each, so
613
+ * `quarterly pla` matches `quarterly planning meeting`. Windows never cross a
614
+ * sentence boundary (mirrors `extractPreviousWord`). Each result records the
615
+ * matched window length so the caller can compute the ghost tail.
616
+ *
617
+ * :params:
618
+ * trimmed: Text before the cursor, trailing whitespace already removed
619
+ * :returns:
620
+ * Matched phrase candidates (deduped, widest-window match wins per term)
621
+ */
622
+ var getPhraseCandidates = function getPhraseCandidates(trimmed) {
623
+ // Records every prefix window we probe and its match count so the (verbose)
624
+ // debug line makes it obvious whether the phrase path found nothing because the
625
+ // trie is empty, or because no phrase begins with the typed window.
626
+ var windowsTried = [];
627
+ var logPhrasePath = function logPhrasePath(skipReason) {
628
+ if (!isAutocompleteDebugVerbose()) {
629
+ return;
630
+ }
631
+ var detail = skipReason ? skipReason : "windows: ".concat(windowsTried.map(function (w) {
632
+ return "\"".concat(w.window, "\"\u2192").concat(w.matches);
633
+ }).join(', ') || '(none)');
634
+ ctcTag('phrase', "loaded=".concat(phraseTermCount, " (").concat(phrasesLoaded ? 'ready' : 'not loaded', ") \xB7 ").concat(detail), CTC_STYLES.section);
635
+ };
636
+ if (phraseTermCount === 0) {
637
+ logPhrasePath('phrase trie empty — bigrams.json/phrases.json not loaded (or 0 terms)');
638
+ return [];
639
+ }
640
+ var tokenMatches = Array.from(trimmed.matchAll(SURFACE_TOKEN_REGEX));
641
+ var lastMatch = tokenMatches[tokenMatches.length - 1];
642
+ var lastStart = lastMatch === null || lastMatch === void 0 ? void 0 : lastMatch.index;
643
+ if (!lastMatch || lastStart === undefined || lastStart + lastMatch[0].length !== trimmed.length) {
644
+ logPhrasePath('no tokens in current sentence');
645
+ return [];
646
+ }
647
+ var trailingTokens = [{
648
+ start: lastStart,
649
+ value: lastMatch[0]
650
+ }];
651
+ for (var i = tokenMatches.length - 2; i >= 0; i--) {
652
+ var current = trailingTokens[0];
653
+ var match = tokenMatches[i];
654
+ var start = match.index;
655
+ if (start === undefined) {
656
+ break;
657
+ }
658
+ var end = start + match[0].length;
659
+ var gap = trimmed.slice(end, current.start);
660
+ if (!ONLY_WHITESPACE_REGEX.test(gap)) {
661
+ break;
662
+ }
663
+ trailingTokens.unshift({
664
+ start: start,
665
+ value: match[0]
666
+ });
667
+ if (trailingTokens.length >= PHRASE_MAX_WORDS) {
668
+ break;
669
+ }
670
+ }
671
+ var partial = trailingTokens[trailingTokens.length - 1].value;
672
+ var results = [];
673
+ var seen = new Set();
674
+ var maxPrev = trailingTokens.length - 1;
675
+
676
+ // Widest window first so the most specific (longest-context) match wins the
677
+ // dedup for a given phrase term. `prev` is the number of *preceding* complete
678
+ // words included ahead of the current partial:
679
+ // prev ≥ 1 → "continue a phrase I've started" (e.g. "root c" → root cause).
680
+ // Preceding-word context is strong, so a short partial is fine.
681
+ // prev = 0 → "the partial is a phrase's FIRST word" (e.g. "root" → root
682
+ // cause). No preceding context, so require the partial to be a
683
+ // one character so grouped LM work can begin early. UI remains
684
+ // gated at DISPLAY_MIN_PREFIX_LENGTH.
685
+ for (var prev = maxPrev; prev >= 0; prev--) {
686
+ if (prev === 0 && partial.length < MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH) {
687
+ continue;
688
+ }
689
+ var windowTokens = trailingTokens.slice(-(prev + 1));
690
+ var windowPrefix = windowTokens.map(function (token) {
691
+ return token.value;
692
+ }).join(' ').toLowerCase();
693
+ var windowLen = windowPrefix.length;
694
+ var matches = phraseTrie.getCandidates(windowPrefix, MAX_PHRASE_CANDIDATES);
695
+ windowsTried.push({
696
+ window: windowPrefix,
697
+ matches: matches.length
698
+ });
699
+ var _iterator0 = _createForOfIteratorHelper(matches),
700
+ _step0;
701
+ try {
702
+ for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
703
+ var _match = _step0.value;
704
+ if (seen.has(_match.word)) {
705
+ continue;
706
+ }
707
+ seen.add(_match.word);
708
+ results.push({
709
+ node: _match.node,
710
+ word: _match.word,
711
+ matchedPrefixLen: windowLen,
712
+ surfaceStart: windowTokens[0].start
713
+ });
714
+ }
715
+ } catch (err) {
716
+ _iterator0.e(err);
717
+ } finally {
718
+ _iterator0.f();
719
+ }
720
+ }
721
+ logPhrasePath();
722
+ return results;
723
+ };
724
+
347
725
  // ─── Debug Helpers ───────────────────────────────────────────────────────────
348
726
 
349
727
  /**
@@ -352,11 +730,25 @@ var extractPreviousWord = function extractPreviousWord(text) {
352
730
  * wordCount: number of words in vector store (0 if not loaded)
353
731
  */
354
732
  export var getPredictorStatus = function getPredictorStatus() {
733
+ var slowLaneStatus = getDefaultSlowLaneClientStatus();
355
734
  return {
735
+ canonicalScoringSupported: slowLaneStatus.canonicalScoringSupported,
736
+ canonicalSurfaceCount: slowLaneStatus.canonicalSurfaceCount,
737
+ clientId: slowLaneStatus.clientId,
738
+ clientKind: slowLaneStatus.clientKind,
739
+ localModelReady: slowLaneStatus.localModelReady,
740
+ slowLaneRegistered: slowLaneStatus.registered,
356
741
  vectorsLoaded: vectorStore !== null,
357
742
  wordCount: vectorStore ? Object.keys(vectorStore.wordIndex).length : 0,
358
743
  vectorsLoadStarted: vectorsLoadStarted,
359
- isInitialized: isInitialized
744
+ isInitialized: isInitialized,
745
+ phrasesLoaded: phrasesLoaded,
746
+ phraseCount: phraseTermCount,
747
+ maxFreqByType: {
748
+ word: wordTrie.maxTenantFreq,
749
+ bigram: maxBigramFreq,
750
+ phrase: maxPhraseFreq
751
+ }
360
752
  };
361
753
  };
362
754
 
@@ -368,19 +760,57 @@ export var getLastPredictionDebug = function getLastPredictionDebug() {
368
760
  return lastPredictionDebug;
369
761
  };
370
762
  export var initVocabulary = function initVocabulary(vocabulary) {
371
- var _iterator8 = _createForOfIteratorHelper(vocabulary.terms),
372
- _step8;
763
+ var _iterator1 = _createForOfIteratorHelper(vocabulary.terms),
764
+ _step1;
373
765
  try {
374
- for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
375
- var term = _step8.value;
766
+ for (_iterator1.s(); !(_step1 = _iterator1.n()).done;) {
767
+ var term = _step1.value;
376
768
  wordTrie.insert(term.word, term.freq, term.docFreq, term.authorFreq);
377
769
  }
378
770
  } catch (err) {
379
- _iterator8.e(err);
771
+ _iterator1.e(err);
380
772
  } finally {
381
- _iterator8.f();
773
+ _iterator1.f();
382
774
  }
383
775
  isInitialized = true;
776
+ recallGeneration++;
777
+ };
778
+
779
+ /**
780
+ * Insert a producer-shipped bigram/phrase artifact into the phrase trie.
781
+ *
782
+ * :params:
783
+ * artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
784
+ * termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
785
+ * :returns:
786
+ * The number of terms inserted
787
+ */
788
+ export var initPhrases = function initPhrases(artifact, termType) {
789
+ var count = 0;
790
+ for (var _i3 = 0, _Object$entries = Object.entries(artifact); _i3 < _Object$entries.length; _i3++) {
791
+ var _stats$pos_seq, _stats$head_pos, _stats$association;
792
+ var _Object$entries$_i = _slicedToArray(_Object$entries[_i3], 2),
793
+ term = _Object$entries$_i[0],
794
+ stats = _Object$entries$_i[1];
795
+ phraseTrie.insert(term, stats.freq, stats.doc_freq, stats.author_freq, {
796
+ termType: termType,
797
+ posSeq: (_stats$pos_seq = stats.pos_seq) !== null && _stats$pos_seq !== void 0 ? _stats$pos_seq : null,
798
+ headPos: (_stats$head_pos = stats.head_pos) !== null && _stats$head_pos !== void 0 ? _stats$head_pos : null,
799
+ association: (_stats$association = stats.association) !== null && _stats$association !== void 0 ? _stats$association : null
800
+ });
801
+ // Track each type's own frequency ceiling for per-type normalization.
802
+ if (termType === 'bigram') {
803
+ if (stats.freq > maxBigramFreq) {
804
+ maxBigramFreq = stats.freq;
805
+ }
806
+ } else if (stats.freq > maxPhraseFreq) {
807
+ maxPhraseFreq = stats.freq;
808
+ }
809
+ count++;
810
+ }
811
+ phraseTermCount += count;
812
+ recallGeneration++;
813
+ return count;
384
814
  };
385
815
 
386
816
  /**
@@ -407,141 +837,258 @@ export var ingestDocumentPage = function ingestDocumentPage(pageContent) {
407
837
  }
408
838
  var words = tokenize(pageContent);
409
839
  var validBoostedWords = new Set();
410
- var _iterator9 = _createForOfIteratorHelper(words),
411
- _step9;
840
+ var _iterator10 = _createForOfIteratorHelper(words),
841
+ _step10;
412
842
  try {
413
- for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
414
- var word = _step9.value;
843
+ for (_iterator10.s(); !(_step10 = _iterator10.n()).done;) {
844
+ var word = _step10.value;
415
845
  var didBoost = wordTrie.incrementSessionFreq(word);
416
846
  if (didBoost) {
417
847
  validBoostedWords.add(word);
418
848
  }
419
849
  }
420
850
  } catch (err) {
421
- _iterator9.e(err);
851
+ _iterator10.e(err);
422
852
  } finally {
423
- _iterator9.f();
853
+ _iterator10.f();
424
854
  }
425
855
  if (isAutocompleteDebugEnabled() && validBoostedWords.size > 0) {
426
- // eslint-disable-next-line no-console
427
- console.groupCollapsed("%c[L1 Session] %cPrimed ".concat(validBoostedWords.size, " valid dictionary words from page"), 'color: #00b8d9; font-weight: bold;', 'color: inherit; font-style: italic;');
428
- // eslint-disable-next-line no-console
429
- console.dir(Array.from(validBoostedWords).sort());
430
- // eslint-disable-next-line no-console
431
- console.groupEnd();
856
+ ctcTag('init', "L1 session primed ".concat(validBoostedWords.size, " words from page \xB7 __atlCtcDebug__.session() to inspect"), CTC_STYLES.brand);
857
+ if (isAutocompleteDebugVerbose()) {
858
+ // eslint-disable-next-line no-console
859
+ console.dir(Array.from(validBoostedWords).sort());
860
+ }
432
861
  }
433
862
  };
434
- export var predict = function predict(textBefore) {
435
- if (!isInitialized) {
436
- // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
437
- // and skip this keystroke; the plugin also primes it on focus, so the tries
438
- // are usually ready before the user types.
439
- void loadDefaultVocabulary().catch(function () {});
440
- return null;
441
- }
442
- var t0 = performance.now();
443
863
 
444
- // ── Step 1: Bigram-based next-word suggestion at word boundary ───────────
445
- // if (textBefore.length > 0 && /\s$/u.test(textBefore)) {
446
- // const words = textBefore.toLowerCase().trimEnd().split(/\s+/u);
447
- // const prevWord = words[words.length - 1];
448
- // const nextWords = bigramMap.get(prevWord);
449
- // if (nextWords) {
450
- // const sorted = Object.entries(nextWords).sort((a, b) => b[1] - a[1]);
451
- // let bestWord = '';
452
- // for (const [word] of sorted) {
453
- // if (!isGrammarAllowed(prevWord, word)) {
454
- // continue;
455
- // }
456
- // bestWord = word;
457
- // break;
458
- // }
459
- // if (bestWord) {
460
- // if (debugMode) {
461
- // const latencyMs = performance.now() - t0;
462
- // console.log(
463
- // '%c[autocomplete] BIGRAM',
464
- // 'color:cyan',
465
- // '| "' + prevWord + '" -> "' + bestWord + '" | ' + latencyMs.toFixed(1) + 'ms',
466
- // );
467
- // }
468
- // return bestWord;
469
- // }
470
- // }
471
- // if (debugMode) {
472
- // console.log(
473
- // '%c[autocomplete] BIGRAM-MISS',
474
- // 'color:gray',
475
- // '| no bigram for "' + words[words.length - 1] + '", skipping prefix completion',
476
- // );
477
- // }
478
- // return null;
479
- // }
480
-
481
- // ── Step 2: Prefix completion (≥3 chars typed) ──────────────────────────
482
- if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
483
- return null;
484
- }
485
- var trimmed = textBefore.trimEnd();
486
- var lastSpaceIdx = trimmed.lastIndexOf(' ');
487
- var currentWord = lastSpaceIdx === -1 ? trimmed : trimmed.slice(lastSpaceIdx + 1);
488
- if (currentWord.length < MIN_PREFIX_LENGTH) {
489
- return null;
490
- }
491
- var candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
864
+ /**
865
+ * How many boosted words `inspectSessionBoosts` lists.
866
+ *
867
+ * A page ingest can boost thousands, and a list that long is not read. The
868
+ * strongest boosts are the ones that change an ordering, and `boosted` still
869
+ * reports the full size, so the cap loses nothing but volume.
870
+ */
871
+ var MAX_LISTED_SESSION_WORDS = 50;
872
+ /**
873
+ * Read the session's L1 boosts, optionally narrowed to a prefix.
874
+ *
875
+ * Installed as `__atlCtcDebug__.session()`, with `__atlCtcDebug__.session('poll')`
876
+ * to ask about one family. Returned rather than logged, so the console renders it
877
+ * as an inspectable object and a caller can assert on it.
878
+ *
879
+ * Only words the vocabulary already holds can carry a boost, because both writers
880
+ * go through `incrementSessionFreq` and it only finds existing nodes. An ingested
881
+ * word absent from the vocabulary is therefore missing from here and always will
882
+ * be.
883
+ */
884
+ export var inspectSessionBoosts = function inspectSessionBoosts(prefix) {
885
+ var boosted = wordTrie.collectSessionBoosted(prefix !== null && prefix !== void 0 ? prefix : '');
886
+ return _objectSpread(_objectSpread({
887
+ boosted: boosted.length,
888
+ limit: MAX_LISTED_SESSION_WORDS
889
+ }, prefix === undefined ? {} : {
890
+ prefix: prefix
891
+ }), {}, {
892
+ words: boosted.sort(function (a, b) {
893
+ return b.node.sessionFreq - a.node.sessionFreq || a.word.localeCompare(b.word, 'en', {
894
+ numeric: true,
895
+ sensitivity: 'base'
896
+ });
897
+ }).slice(0, MAX_LISTED_SESSION_WORDS).map(function (_ref) {
898
+ var node = _ref.node,
899
+ word = _ref.word;
900
+ return {
901
+ sessionFreq: node.sessionFreq,
902
+ sessionOnly: node.tenantFreq === 0,
903
+ surface: word,
904
+ tenantFreq: node.tenantFreq
905
+ };
906
+ })
907
+ });
908
+ };
909
+
910
+ // At module scope so the console answers before the first keystroke, which is
911
+ // when someone reaching for it usually asks.
912
+ registerCtcSessionInspector(inspectSessionBoosts);
913
+
914
+ /**
915
+ * Result of a prediction: the ghost tail to insert plus an immutable record of
916
+ * the evidence that authorized the UI commitment.
917
+ */
918
+
919
+ // ─── Recall memo ─────────────────────────────────────────────────────────────
920
+
921
+ /** Trie recall plus the canonical context derived for each matched term. */
922
+
923
+ /**
924
+ * Bumped whenever an artifact load changes what the tries can return, which is
925
+ * the only way the candidate *set* for a given prefix can change.
926
+ *
927
+ * Session-frequency boosts are deliberately not counted: they mutate trie nodes
928
+ * in place, and the memo holds those nodes by reference, so a boost is already
929
+ * visible through a cached entry.
930
+ */
931
+ var recallGeneration = 0;
932
+ var recallMemo = null;
933
+ var computeCanonicalRecall = function computeCanonicalRecall(trimmed, currentWord, currentWordStart) {
934
+ // Start recall at one character so grouped causal work can overlap later
935
+ // keystrokes. Nothing is displayed until three characters.
936
+ var wordCandidates = currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH ? wordTrie.getCandidates(currentWord, MAX_CANDIDATES) : [];
492
937
 
493
938
  // Gap-fill from the L3 general-English trie, requesting a full buffer so
494
939
  // enough survive de-duplication against the L2 results.
495
- if (candidates.length < MAX_CANDIDATES) {
940
+ if (currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH && wordCandidates.length < MAX_CANDIDATES) {
496
941
  var l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
497
- var existingWords = new Set(candidates.map(function (c) {
942
+ var existingWords = new Set(wordCandidates.map(function (c) {
498
943
  return c.word;
499
944
  }));
500
- var _iterator0 = _createForOfIteratorHelper(l3Candidates),
501
- _step0;
945
+ var _iterator11 = _createForOfIteratorHelper(l3Candidates),
946
+ _step11;
502
947
  try {
503
- for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
504
- var l3c = _step0.value;
505
- if (candidates.length >= MAX_CANDIDATES) break;
948
+ for (_iterator11.s(); !(_step11 = _iterator11.n()).done;) {
949
+ var l3c = _step11.value;
950
+ if (wordCandidates.length >= MAX_CANDIDATES) break;
506
951
  if (!existingWords.has(l3c.word)) {
507
- candidates.push(l3c);
952
+ wordCandidates.push(l3c);
508
953
  }
509
954
  }
510
955
  } catch (err) {
511
- _iterator0.e(err);
956
+ _iterator11.e(err);
512
957
  } finally {
513
- _iterator0.f();
958
+ _iterator11.f();
959
+ }
960
+ }
961
+
962
+ // Multi-word completion terms (bigrams/phrases) matched on the trailing
963
+ // window ending at the current partial token. Runs even when the unigram path
964
+ // was skipped for a short partial, so first-word/early phrase matches surface.
965
+ var phraseCandidates = getPhraseCandidates(trimmed);
966
+
967
+ // Unify: a word completes the current partial token; a phrase completes its
968
+ // matched multi-word window. Track the prefix length per term so the ghost
969
+ // tail is sliced correctly regardless of term type.
970
+ var matched = [].concat(_toConsumableArray(wordCandidates.map(function (_ref2) {
971
+ var word = _ref2.word,
972
+ node = _ref2.node;
973
+ return {
974
+ word: word,
975
+ node: node,
976
+ matchedPrefixLen: currentWord.length,
977
+ surfaceStart: currentWordStart
978
+ };
979
+ })), _toConsumableArray(phraseCandidates));
980
+ var positionCache = createCanonicalContextPositionCache();
981
+ var canonicalMatched = matched.map(function (candidate) {
982
+ return _objectSpread(_objectSpread({}, candidate), deriveCanonicalCandidateContext(trimmed, candidate.matchedPrefixLen, candidate.word, getCanonicalSurfaceTokenIds, candidate.surfaceStart, positionCache));
983
+ });
984
+ var prefixLenByWord = new Map();
985
+ var _iterator12 = _createForOfIteratorHelper(canonicalMatched),
986
+ _step12;
987
+ try {
988
+ for (_iterator12.s(); !(_step12 = _iterator12.n()).done;) {
989
+ var m = _step12.value;
990
+ if (!prefixLenByWord.has(m.word)) {
991
+ prefixLenByWord.set(m.word, m.matchedPrefixLen);
992
+ }
993
+ }
994
+ } catch (err) {
995
+ _iterator12.e(err);
996
+ } finally {
997
+ _iterator12.f();
998
+ }
999
+ return {
1000
+ canonicalMatched: canonicalMatched,
1001
+ prefixLenByWord: prefixLenByWord
1002
+ };
1003
+ };
1004
+
1005
+ /**
1006
+ * Recall and canonical derivation for `trimmed`, reusing the previous result
1007
+ * when nothing they depend on has changed.
1008
+ *
1009
+ * `predict()` runs several times for one keystroke — once when the decision
1010
+ * opens and again for each async evidence signal that lands inside the budget —
1011
+ * and only the evidence differs between those runs. Without this, every run
1012
+ * re-walks the tries and re-derives a canonical context per candidate.
1013
+ */
1014
+ var getCanonicalRecall = function getCanonicalRecall(trimmed, currentWord, currentWordStart) {
1015
+ var surfaceCount = getCanonicalSurfaceCount();
1016
+ if (recallMemo !== null && recallMemo.trimmed === trimmed && recallMemo.generation === recallGeneration && recallMemo.surfaceCount === surfaceCount) {
1017
+ return recallMemo;
1018
+ }
1019
+ var recall = computeCanonicalRecall(trimmed, currentWord, currentWordStart);
1020
+ recallMemo = _objectSpread(_objectSpread({}, recall), {}, {
1021
+ generation: recallGeneration,
1022
+ surfaceCount: surfaceCount,
1023
+ trimmed: trimmed
1024
+ });
1025
+ return recall;
1026
+ };
1027
+ export var predict = function predict(textBefore) {
1028
+ var _trimmed$match$, _trimmed$match, _canonicalMatched$fin, _eligible$, _eligible$0$score, _eligible$2, _judgedEvidenceFor3, _prefixLenByWord$get2, _runtimeBySurface$get5, _judgedEvidenceFor$me3, _judgedEvidenceFor6, _contextTotals$get$le4, _contextTotals$get4, _selectedEvidence$ver, _selectedEvidence$tot, _selectedEvidence$ver2;
1029
+ if (!isInitialized) {
1030
+ // Vocabulary artifacts load asynchronously over the autocomplete gateway.
1031
+ // Kick off the load and skip this keystroke; the plugin also primes it on
1032
+ // focus, so the tries are usually ready before the user types.
1033
+ void loadDefaultVocabulary({
1034
+ source: 'predict'
1035
+ }).catch(function () {});
1036
+ return null;
1037
+ }
1038
+ var t0 = performance.now();
1039
+
1040
+ // Advance the post-accept cooldown once per prediction. It stays active until
1041
+ // BOTH the keystroke budget and the time window have elapsed; while active it
1042
+ // suppresses re-offering the just-accepted surface (see arbitration below).
1043
+ var cooledSurface = null;
1044
+ if (acceptCooldown) {
1045
+ acceptCooldown.predictionsSince += 1;
1046
+ var stillByKeys = acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES;
1047
+ var stillByTime = performance.now() - acceptCooldown.ts < COOLDOWN_MS;
1048
+ if (stillByKeys || stillByTime) {
1049
+ cooledSurface = acceptCooldown.surface;
1050
+ } else {
1051
+ acceptCooldown = null;
1052
+ }
1053
+ }
1054
+
1055
+ // ── Boundary/early-prefix prefetch; display remains gated at ≥3 ─────────
1056
+ if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
1057
+ var boundaryContext = deriveWhitespaceBoundaryContext(textBefore);
1058
+ if (boundaryContext) {
1059
+ primeBoundaryLm(_objectSpread(_objectSpread({}, boundaryContext), {}, {
1060
+ familyKey: boundaryContext.contextKey,
1061
+ priority: 0
1062
+ }));
514
1063
  }
1064
+ return null;
515
1065
  }
1066
+ var trimmed = textBefore.trimEnd();
1067
+ var trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
1068
+ if (trailingSurfaceToken.length === 0) {
1069
+ return null;
1070
+ }
1071
+ var currentWord = trailingSurfaceToken;
1072
+ var currentWordStart = trimmed.length - currentWord.length;
1073
+ var _getCanonicalRecall = getCanonicalRecall(trimmed, currentWord, currentWordStart),
1074
+ canonicalMatched = _getCanonicalRecall.canonicalMatched,
1075
+ prefixLenByWord = _getCanonicalRecall.prefixLenByWord;
516
1076
 
517
- // If both Tries are completely empty for this prefix
518
- if (candidates.length === 0) {
1077
+ // If every trie was empty for this prefix
1078
+ if (canonicalMatched.length === 0) {
1079
+ if (isAutocompleteDebugEnabled()) {
1080
+ // eslint-disable-next-line no-console
1081
+ console.log("%c[CTC]%c \u2014 abstain: no matches for \"".concat(currentWord, "\""), CTC_STYLES.brand, CTC_STYLES.body);
1082
+ }
519
1083
  return null;
520
1084
  }
521
1085
  var previousWord = extractPreviousWord(trimmed);
522
1086
  var contextVector = getContextVectorForScoring(trimmed);
523
1087
  var lmLogits = getStoredLmLogits();
524
-
525
- // Raw LM Output Logger
526
- if (isAutocompleteDebugEnabled() && lmLogits && Object.keys(lmLogits).length > 0) {
527
- var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
528
- return b[1] - a[1];
529
- }).slice(0, 5).map(function (_ref) {
530
- var _ref2 = _slicedToArray(_ref, 2),
531
- word = _ref2[0],
532
- score = _ref2[1];
533
- return {
534
- Word: word,
535
- Prob: Number(score.toFixed(5))
536
- };
537
- });
538
- // eslint-disable-next-line no-console
539
- console.log('%c[Raw LM Prediction]🧠', 'color: #e83e8c; font-weight: bold;', rawLmTop);
540
- }
541
1088
  var mode = contextVector ? 'warm' : 'cold';
542
1089
 
543
- // Build ScoringCandidate array from TrieNodes
544
- var scoringCandidates = candidates.map(function (_ref3) {
1090
+ // Build ScoringCandidate array from matched terms (words + phrases)
1091
+ var scoringCandidates = canonicalMatched.map(function (_ref3) {
545
1092
  var word = _ref3.word,
546
1093
  node = _ref3.node;
547
1094
  return {
@@ -549,98 +1096,1111 @@ export var predict = function predict(textBefore) {
549
1096
  tenantFreq: node.tenantFreq,
550
1097
  docFreq: node.docFreq,
551
1098
  authorFreq: node.authorFreq,
552
- sessionFreq: node.sessionFreq
1099
+ sessionFreq: node.sessionFreq,
1100
+ termType: node.termType,
1101
+ posSeq: node.posSeq,
1102
+ headPos: node.headPos,
1103
+ association: node.association
553
1104
  };
554
1105
  });
555
1106
 
556
1107
  // Filter the LM payload to only words matching the current prefix so that
557
1108
  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
558
1109
  var prefix = currentWord.toLowerCase();
559
- var prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
1110
+ var currentWordSeparator = (_canonicalMatched$fin = canonicalMatched.find(function (candidate) {
1111
+ return candidate.node.termType === 'word';
1112
+ })) === null || _canonicalMatched$fin === void 0 ? void 0 : _canonicalMatched$fin.separatorKind;
1113
+ var prefixLmLogits = lmLogits && currentWordSeparator === 'whitespace' ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
560
1114
  var _ref5 = _slicedToArray(_ref4, 1),
561
1115
  word = _ref5[0];
562
1116
  return word.startsWith(prefix);
563
1117
  })) : null;
564
- var _rankCandidates = rankCandidates(scoringCandidates, contextVector, function (w) {
565
- return getWordVector(w);
566
- }, prefixLmLogits, wordTrie.maxTenantFreq, previousWord),
567
- ranked = _rankCandidates.candidates,
568
- grammarMeta = _rankCandidates.grammarMeta,
569
- pipelineDebug = _rankCandidates.pipelineDebug;
570
- var best = ranked[0];
571
- var suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
572
- if (isAutocompleteDebugEnabled()) {
573
- var latencyMs = performance.now() - t0;
574
- var tokens = tokenize(trimmed);
575
- var contextWords = tokens.slice(-CONTEXT_WORDS);
576
- var belowThreshold = best && best.finalScore < MIN_SCORE_THRESHOLD;
577
- var suggestionLabel = belowThreshold ? '🚫 (below threshold)' : suggestion && suggestion.length > 0 ? "\u2728 \"".concat(suggestion, "\"") : '🚫 (no match)';
578
- lastPredictionDebug = {
579
- textBefore: trimmed,
580
- currentWord: currentWord,
581
- mode: mode,
582
- contextWords: contextWords,
583
- topCandidates: ranked.slice(0, 5).map(function (r) {
584
- return {
585
- word: r.word,
586
- finalScore: r.finalScore,
587
- semanticScore: r.semanticScore,
588
- freqScore: r.freqScore,
589
- lmScore: r.lmScore
1118
+ var canonicalScoringSupported = isCanonicalSurfaceScoringSupported();
1119
+ var eligibleContextKeys = Array.from(new Set(canonicalMatched.filter(function (candidate) {
1120
+ return candidate.canonicalTokenIds !== null;
1121
+ }).map(function (candidate) {
1122
+ return candidate.contextKey;
1123
+ }))).sort();
1124
+ var familyKey = eligibleContextKeys.join("\x01");
1125
+ var primeRequests = selectBoundaryPrimeRequests(familyKey, canonicalMatched, PHRASE_MAX_WORDS);
1126
+ var _iterator13 = _createForOfIteratorHelper(primeRequests),
1127
+ _step13;
1128
+ try {
1129
+ for (_iterator13.s(); !(_step13 = _iterator13.n()).done;) {
1130
+ var request = _step13.value;
1131
+ primeBoundaryLm(request);
1132
+ }
1133
+ } catch (err) {
1134
+ _iterator13.e(err);
1135
+ } finally {
1136
+ _iterator13.f();
1137
+ }
1138
+ var runtimeBySurface = new Map(canonicalMatched.map(function (candidate) {
1139
+ return [candidate.word, candidate];
1140
+ }));
1141
+ var canonicalEvidence = new Map();
1142
+ var firstTokenGroups = new Map();
1143
+ var _iterator14 = _createForOfIteratorHelper(canonicalMatched),
1144
+ _step14;
1145
+ try {
1146
+ for (_iterator14.s(); !(_step14 = _iterator14.n()).done;) {
1147
+ var _firstTokenGroups$get;
1148
+ var _candidate = _step14.value;
1149
+ if (_candidate.canonicalTokenIds === null) {
1150
+ continue;
1151
+ }
1152
+ // This accessor also materializes an exact cache entry when all tokens
1153
+ // have already been covered by shared-prefix work.
1154
+ getProgressiveSurfaceEvidence(_candidate.contextKey, _candidate.word);
1155
+ var exact = getSurfaceScore(_candidate.contextKey, _candidate.word);
1156
+ if (exact) {
1157
+ canonicalEvidence.set(_candidate.word, {
1158
+ contextKey: _candidate.contextKey,
1159
+ meanTokenLogProbability: exact.meanTokenLogProbability,
1160
+ score: 0,
1161
+ separatorKind: 'whitespace',
1162
+ source: 'canonical-full-surface',
1163
+ totalLogProbability: exact.totalLogProbability,
1164
+ totalSurfaceCharCount: _candidate.word.length,
1165
+ totalTokenCount: exact.tokenCount,
1166
+ verifiedCharCount: _candidate.word.length,
1167
+ verifiedTokenCount: exact.tokenCount
1168
+ });
1169
+ continue;
1170
+ }
1171
+ var _group2 = (_firstTokenGroups$get = firstTokenGroups.get(_candidate.contextKey)) !== null && _firstTokenGroups$get !== void 0 ? _firstTokenGroups$get : [];
1172
+ _group2.push(_candidate);
1173
+ firstTokenGroups.set(_candidate.contextKey, _group2);
1174
+ }
1175
+ } catch (err) {
1176
+ _iterator14.e(err);
1177
+ } finally {
1178
+ _iterator14.f();
1179
+ }
1180
+ var _iterator15 = _createForOfIteratorHelper(firstTokenGroups),
1181
+ _step15;
1182
+ try {
1183
+ for (_iterator15.s(); !(_step15 = _iterator15.n()).done;) {
1184
+ var _step15$value = _slicedToArray(_step15.value, 2),
1185
+ _contextKey = _step15$value[0],
1186
+ _group3 = _step15$value[1];
1187
+ var boundary = getBoundaryLmState(_contextKey);
1188
+ if (!boundary) {
1189
+ continue;
1190
+ }
1191
+ var maxLogit = -Infinity;
1192
+ var _iterator22 = _createForOfIteratorHelper(_group3),
1193
+ _step22;
1194
+ try {
1195
+ for (_iterator22.s(); !(_step22 = _iterator22.n()).done;) {
1196
+ var _candidate2$canonical;
1197
+ var _candidate2 = _step22.value;
1198
+ var tokenId = (_candidate2$canonical = _candidate2.canonicalTokenIds) === null || _candidate2$canonical === void 0 ? void 0 : _candidate2$canonical[0];
1199
+ var rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
1200
+ if (rawLogit !== undefined && Number.isFinite(rawLogit) && rawLogit > maxLogit) {
1201
+ maxLogit = rawLogit;
1202
+ }
1203
+ }
1204
+ } catch (err) {
1205
+ _iterator22.e(err);
1206
+ } finally {
1207
+ _iterator22.f();
1208
+ }
1209
+ if (!Number.isFinite(maxLogit)) {
1210
+ continue;
1211
+ }
1212
+ var _iterator23 = _createForOfIteratorHelper(_group3),
1213
+ _step23;
1214
+ try {
1215
+ for (_iterator23.s(); !(_step23 = _iterator23.n()).done;) {
1216
+ var _candidate3$canonical, _candidate3$canonical2, _candidate3$canonical3;
1217
+ var _candidate3 = _step23.value;
1218
+ var _tokenId = (_candidate3$canonical = _candidate3.canonicalTokenIds) === null || _candidate3$canonical === void 0 ? void 0 : _candidate3$canonical[0];
1219
+ var _rawLogit = _tokenId === undefined ? undefined : boundary.rawLogits[_tokenId];
1220
+ if (_rawLogit === undefined || !Number.isFinite(_rawLogit)) {
1221
+ continue;
1222
+ }
1223
+ var tokenCount = (_candidate3$canonical2 = (_candidate3$canonical3 = _candidate3.canonicalTokenIds) === null || _candidate3$canonical3 === void 0 ? void 0 : _candidate3$canonical3.length) !== null && _candidate3$canonical2 !== void 0 ? _candidate3$canonical2 : 0;
1224
+ if (_tokenId !== undefined && tokenCount === 1) {
1225
+ var totalLogProbability = logSoftmaxAt(boundary.rawLogits, _tokenId);
1226
+ if (!Number.isFinite(totalLogProbability)) {
1227
+ continue;
1228
+ }
1229
+ canonicalEvidence.set(_candidate3.word, {
1230
+ contextKey: _contextKey,
1231
+ meanTokenLogProbability: totalLogProbability,
1232
+ score: 0,
1233
+ separatorKind: 'whitespace',
1234
+ source: 'canonical-full-surface',
1235
+ totalLogProbability: totalLogProbability,
1236
+ totalSurfaceCharCount: _candidate3.word.length,
1237
+ totalTokenCount: 1,
1238
+ verifiedCharCount: _candidate3.word.length,
1239
+ verifiedTokenCount: 1
1240
+ });
1241
+ continue;
1242
+ }
1243
+ canonicalEvidence.set(_candidate3.word, {
1244
+ contextKey: _contextKey,
1245
+ rawLogit: _rawLogit,
1246
+ score: 0,
1247
+ separatorKind: 'whitespace',
1248
+ source: 'canonical-first-token',
1249
+ totalSurfaceCharCount: _candidate3.word.length,
1250
+ totalTokenCount: tokenCount,
1251
+ verifiedCharCount: 0,
1252
+ verifiedTokenCount: 1
1253
+ });
1254
+ }
1255
+ } catch (err) {
1256
+ _iterator23.e(err);
1257
+ } finally {
1258
+ _iterator23.f();
1259
+ }
1260
+ }
1261
+ } catch (err) {
1262
+ _iterator15.e(err);
1263
+ } finally {
1264
+ _iterator15.f();
1265
+ }
1266
+ var _rankCandidates = rankCandidates(scoringCandidates, contextVector, function (w) {
1267
+ return getWordVector(w);
1268
+ }, prefixLmLogits, {
1269
+ word: wordTrie.maxTenantFreq,
1270
+ bigram: maxBigramFreq,
1271
+ phrase: maxPhraseFreq
1272
+ }, previousWord, function (surface) {
1273
+ var _canonicalEvidence$ge;
1274
+ return (_canonicalEvidence$ge = canonicalEvidence.get(surface)) !== null && _canonicalEvidence$ge !== void 0 ? _canonicalEvidence$ge : null;
1275
+ }),
1276
+ ranked = _rankCandidates.candidates,
1277
+ grammarMeta = _rankCandidates.grammarMeta,
1278
+ pipelineDebug = _rankCandidates.pipelineDebug;
1279
+ var progressiveEligible = ranked.filter(function (candidate) {
1280
+ var runtime = runtimeBySurface.get(candidate.word);
1281
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined;
1282
+ });
1283
+ if (canonicalScoringSupported && progressiveEligible.length > 0) {
1284
+ var byContext = new Map();
1285
+ var _iterator16 = _createForOfIteratorHelper(progressiveEligible),
1286
+ _step16;
1287
+ try {
1288
+ for (_iterator16.s(); !(_step16 = _iterator16.n()).done;) {
1289
+ var _byContext$get;
1290
+ var candidate = _step16.value;
1291
+ var runtime = runtimeBySurface.get(candidate.word);
1292
+ if (!runtime || runtime.canonicalTokenIds === null) {
1293
+ continue;
1294
+ }
1295
+ var group = (_byContext$get = byContext.get(runtime.contextKey)) !== null && _byContext$get !== void 0 ? _byContext$get : [];
1296
+ group.push({
1297
+ runtime: runtime,
1298
+ rankHint: candidate.stage1Score
1299
+ });
1300
+ byContext.set(runtime.contextKey, group);
1301
+ }
1302
+ } catch (err) {
1303
+ _iterator16.e(err);
1304
+ } finally {
1305
+ _iterator16.f();
1306
+ }
1307
+ var _iterator17 = _createForOfIteratorHelper(byContext),
1308
+ _step17;
1309
+ try {
1310
+ for (_iterator17.s(); !(_step17 = _iterator17.n()).done;) {
1311
+ var _step17$value = _slicedToArray(_step17.value, 2),
1312
+ contextKey = _step17$value[0],
1313
+ _group = _step17$value[1];
1314
+ requestProgressiveSurfaceScores({
1315
+ familyKey: familyKey,
1316
+ contextKey: contextKey,
1317
+ prompt: _group[0].runtime.contextBeforeSurface,
1318
+ candidates: _group.map(function (_ref6) {
1319
+ var _runtime$canonicalTok;
1320
+ var runtime = _ref6.runtime,
1321
+ rankHint = _ref6.rankHint;
1322
+ return {
1323
+ surface: runtime.word,
1324
+ tokenIds: (_runtime$canonicalTok = runtime.canonicalTokenIds) !== null && _runtime$canonicalTok !== void 0 ? _runtime$canonicalTok : [],
1325
+ rankHint: rankHint
1326
+ };
1327
+ })
1328
+ });
1329
+ }
1330
+ } catch (err) {
1331
+ _iterator17.e(err);
1332
+ } finally {
1333
+ _iterator17.f();
1334
+ }
1335
+ }
1336
+
1337
+ // The one/two-character passes exist only to hide model latency. They may
1338
+ // prime and expand token groups above, but cannot commit UI.
1339
+ var displayPrefixReady = currentWord.length >= DISPLAY_MIN_PREFIX_LENGTH;
1340
+
1341
+ // ── Arbitration: full-surface evidence → plausibility → winner margin ──
1342
+ var canonicalLmSupported = canonicalScoringSupported;
1343
+ var hasExactEvidence = function hasExactEvidence(candidate) {
1344
+ var _candidate$lmEvidence;
1345
+ return ((_candidate$lmEvidence = candidate.lmEvidence) === null || _candidate$lmEvidence === void 0 ? void 0 : _candidate$lmEvidence.source) === 'canonical-full-surface' && candidate.lmEvidence.meanTokenLogProbability !== undefined && Number.isFinite(candidate.lmEvidence.meanTokenLogProbability);
1346
+ };
1347
+
1348
+ /**
1349
+ * The evidence this candidate is judged on: the summed log-probability of
1350
+ * the tokens that have been read, their per-token mean, and whether they
1351
+ * cover the whole surface.
1352
+ *
1353
+ * `total` is the quantity the posterior is built from — it is a sequence
1354
+ * log-likelihood, so it is comparable across surfaces of different lengths
1355
+ * in a way the mean is not. `mean` is retained for reporting and for the
1356
+ * oracle arms, which compare aggregation rules against each other.
1357
+ *
1358
+ * Every token read costs a round trip, so a surface longer than
1359
+ * `REQUIRED_VERIFIED_TOKENS` is judged on its leading tokens and its tail is
1360
+ * left unread. Callers compensate with a wider winner margin rather than
1361
+ * treating the two kinds of evidence as equivalent.
1362
+ *
1363
+ * This is the depth-unaware read, and it is what the pool below is built
1364
+ * from. Arbitration goes through `judgedEvidenceFor`, which adds the one
1365
+ * requirement that can only be stated once the pool is known: a surface
1366
+ * cannot be judged against members it has not yet been read level with.
1367
+ */
1368
+ var readJudgedEvidence = function readJudgedEvidence(candidate) {
1369
+ var evidence = candidate.lmEvidence;
1370
+ var exactMean = evidence === null || evidence === void 0 ? void 0 : evidence.meanTokenLogProbability;
1371
+ if (hasExactEvidence(candidate) && evidence && exactMean !== undefined) {
1372
+ var _evidence$totalLogPro;
1373
+ var _total = (_evidence$totalLogPro = evidence.totalLogProbability) !== null && _evidence$totalLogPro !== void 0 ? _evidence$totalLogPro : exactMean * evidence.totalTokenCount;
1374
+ return Number.isFinite(_total) ? {
1375
+ complete: true,
1376
+ mean: exactMean,
1377
+ total: _total
1378
+ } : null;
1379
+ }
1380
+ var runtime = runtimeBySurface.get(candidate.word);
1381
+ if (!runtime || runtime.canonicalTokenIds === null) {
1382
+ return null;
1383
+ }
1384
+ var progressive = getProgressiveSurfaceEvidence(runtime.contextKey, candidate.word);
1385
+ if (!progressive || progressive.scoredTokenCount < REQUIRED_VERIFIED_TOKENS) {
1386
+ return null;
1387
+ }
1388
+ var total = progressive.totalLogProbability;
1389
+ var mean = total / progressive.scoredTokenCount;
1390
+ return Number.isFinite(mean) && Number.isFinite(total) ? {
1391
+ complete: false,
1392
+ mean: mean,
1393
+ total: total
1394
+ } : null;
1395
+ };
1396
+
1397
+ // Normalise each context's shortlist into a posterior. `total` is a sequence
1398
+ // log-likelihood, so `exp(total - logSumExp(totals))` is the share of the
1399
+ // shortlist's probability mass the model puts on that surface.
1400
+ //
1401
+ // Unknown future token log-probabilities cannot exceed zero, so a partially
1402
+ // read candidate's total is an upper bound on its eventual total, and its
1403
+ // posterior an upper bound on its eventual posterior. That is what lets the
1404
+ // plausibility floor and the display gate be the same threshold — one
1405
+ // applied to the bound, one to the verified value — instead of two numbers
1406
+ // that have to be kept consistent by hand.
1407
+ //
1408
+ // Candidates with no evidence at all are left out of the sum rather than
1409
+ // defaulted. Any default would have to stand in for a log-probability, and a
1410
+ // value low enough not to distort the normaliser is indistinguishable from
1411
+ // omitting the candidate.
1412
+ var optimisticTotalByWord = new Map();
1413
+ var contextTotals = new Map();
1414
+ var contextSurfaces = new Map();
1415
+ // Counted before the total is looked up, so this is every candidate the
1416
+ // context could have scored rather than every candidate it did. The gap
1417
+ // between the two is what separates a context holding one match from a
1418
+ // context whose rivals have not come back yet.
1419
+ var contextRequestedCount = new Map();
1420
+ var bestTotalByContext = new Map();
1421
+ var bestCandidateByContext = new Map();
1422
+ var _iterator18 = _createForOfIteratorHelper(ranked),
1423
+ _step18;
1424
+ try {
1425
+ for (_iterator18.s(); !(_step18 = _iterator18.n()).done;) {
1426
+ var _contextRequestedCoun2, _judged$total;
1427
+ var _candidate4 = _step18.value;
1428
+ var _runtime = runtimeBySurface.get(_candidate4.word);
1429
+ if (!_runtime || _runtime.canonicalTokenIds === null) {
1430
+ continue;
1431
+ }
1432
+ contextRequestedCount.set(_runtime.contextKey, ((_contextRequestedCoun2 = contextRequestedCount.get(_runtime.contextKey)) !== null && _contextRequestedCoun2 !== void 0 ? _contextRequestedCoun2 : 0) + 1);
1433
+ var judged = readJudgedEvidence(_candidate4);
1434
+ var progressive = getProgressiveSurfaceEvidence(_runtime.contextKey, _candidate4.word);
1435
+ // Once a candidate has been read far enough to judge, its verified total
1436
+ // replaces the bound.
1437
+ var optimisticTotal = (_judged$total = judged === null || judged === void 0 ? void 0 : judged.total) !== null && _judged$total !== void 0 ? _judged$total : progressive === null || progressive === void 0 ? void 0 : progressive.totalLogProbability;
1438
+ if (optimisticTotal === undefined || !Number.isFinite(optimisticTotal)) {
1439
+ continue;
1440
+ }
1441
+ optimisticTotalByWord.set(_candidate4.word, optimisticTotal);
1442
+ var totals = contextTotals.get(_runtime.contextKey);
1443
+ var surfaces = contextSurfaces.get(_runtime.contextKey);
1444
+ if (totals && surfaces) {
1445
+ totals.push(optimisticTotal);
1446
+ surfaces.push(_candidate4.word);
1447
+ } else {
1448
+ contextTotals.set(_runtime.contextKey, [optimisticTotal]);
1449
+ contextSurfaces.set(_runtime.contextKey, [_candidate4.word]);
1450
+ }
1451
+ var best = bestTotalByContext.get(_runtime.contextKey);
1452
+ if (best === undefined || optimisticTotal > best) {
1453
+ bestTotalByContext.set(_runtime.contextKey, optimisticTotal);
1454
+ bestCandidateByContext.set(_runtime.contextKey, _candidate4);
1455
+ }
1456
+ }
1457
+
1458
+ /**
1459
+ * Whether a shorter member of the same pool is this surface's word-boundary
1460
+ * prefix — `happy path` where `happy` is also present.
1461
+ *
1462
+ * Only a multi-word surface can extend another, and a pool is a set, so this
1463
+ * costs one lookup per space rather than a comparison against every member.
1464
+ */
1465
+ } catch (err) {
1466
+ _iterator18.e(err);
1467
+ } finally {
1468
+ _iterator18.f();
1469
+ }
1470
+ var extendsAPoolMember = function extendsAPoolMember(surface, pool) {
1471
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1472
+ if (pool.has(surface.slice(0, space))) {
1473
+ return true;
1474
+ }
1475
+ }
1476
+ return false;
1477
+ };
1478
+
1479
+ // A sequence total only falls as more of the sequence is read, so a partial
1480
+ // sum over `k` tokens sits above the same surface's total over `m > k`. An
1481
+ // extension read less far than the member it continues therefore carries a
1482
+ // total above that member's — and because the chain rule narrows the
1483
+ // denominator to the pool's minimal members, `exp(total - normaliser)` comes
1484
+ // out above 1 for a surface whose own continuation has not been looked at.
1485
+ // It needs a three-token first word to happen, which 2.6% of served bigrams
1486
+ // have, and it inflates both the posterior the gate reads and the
1487
+ // `confidenceScore` every other candidate's margin is measured against.
1488
+ //
1489
+ // A surface's canonical tokenization begins with the tokenization of its own
1490
+ // leading words — the producer encodes `" " + surface` in one call and the
1491
+ // ByteLevel pre-tokenizer splits on word boundaries, so no merge crosses a
1492
+ // space — which is what makes both corrections below exact rather than
1493
+ // approximate.
1494
+ //
1495
+ // The total is capped at what it continues: `P(happy path)` cannot exceed
1496
+ // `P(happy)`. Shortest-first, so a three-word phrase is capped against a
1497
+ // two-word prefix that has already been capped itself.
1498
+ //
1499
+ // And the surface is held unjudgeable until it has been read level with what
1500
+ // it continues, because the cap alone would leave it looking exactly as
1501
+ // certain as its prefix while saying nothing about its own tail.
1502
+ var underReadExtensions = new Set();
1503
+ var _iterator19 = _createForOfIteratorHelper(contextSurfaces),
1504
+ _step19;
1505
+ try {
1506
+ for (_iterator19.s(); !(_step19 = _iterator19.n()).done;) {
1507
+ var _step19$value = _slicedToArray(_step19.value, 2),
1508
+ _contextKey2 = _step19$value[0],
1509
+ _surfaces = _step19$value[1];
1510
+ var pool = new Set(_surfaces);
1511
+ var _iterator24 = _createForOfIteratorHelper(_toConsumableArray(_surfaces).sort(function (a, b) {
1512
+ return a.length - b.length;
1513
+ })),
1514
+ _step24;
1515
+ try {
1516
+ for (_iterator24.s(); !(_step24 = _iterator24.n()).done;) {
1517
+ var _getProgressiveSurfac, _getProgressiveSurfac2, _runtimeBySurface$get8, _runtimeBySurface$get9;
1518
+ var surface = _step24.value;
1519
+ var own = optimisticTotalByWord.get(surface);
1520
+ if (own === undefined) {
1521
+ continue;
1522
+ }
1523
+ var cap = Infinity;
1524
+ var continuedDepth = 0;
1525
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1526
+ var _runtimeBySurface$get6, _runtimeBySurface$get7;
1527
+ var _prefix = surface.slice(0, space);
1528
+ if (!pool.has(_prefix)) {
1529
+ continue;
1530
+ }
1531
+ var prefixTotal = optimisticTotalByWord.get(_prefix);
1532
+ if (prefixTotal !== undefined && prefixTotal < cap) {
1533
+ cap = prefixTotal;
1534
+ }
1535
+ var prefixDepth = (_runtimeBySurface$get6 = (_runtimeBySurface$get7 = runtimeBySurface.get(_prefix)) === null || _runtimeBySurface$get7 === void 0 || (_runtimeBySurface$get7 = _runtimeBySurface$get7.canonicalTokenIds) === null || _runtimeBySurface$get7 === void 0 ? void 0 : _runtimeBySurface$get7.length) !== null && _runtimeBySurface$get6 !== void 0 ? _runtimeBySurface$get6 : 0;
1536
+ if (prefixDepth > continuedDepth) {
1537
+ continuedDepth = prefixDepth;
1538
+ }
1539
+ }
1540
+ if (own > cap) {
1541
+ optimisticTotalByWord.set(surface, cap);
1542
+ }
1543
+ var scoredDepth = (_getProgressiveSurfac = (_getProgressiveSurfac2 = getProgressiveSurfaceEvidence(_contextKey2, surface)) === null || _getProgressiveSurfac2 === void 0 ? void 0 : _getProgressiveSurfac2.scoredTokenCount) !== null && _getProgressiveSurfac !== void 0 ? _getProgressiveSurfac : 0;
1544
+ var surfaceDepth = (_runtimeBySurface$get8 = (_runtimeBySurface$get9 = runtimeBySurface.get(surface)) === null || _runtimeBySurface$get9 === void 0 || (_runtimeBySurface$get9 = _runtimeBySurface$get9.canonicalTokenIds) === null || _runtimeBySurface$get9 === void 0 ? void 0 : _runtimeBySurface$get9.length) !== null && _runtimeBySurface$get8 !== void 0 ? _runtimeBySurface$get8 : 0;
1545
+ if (scoredDepth < continuedDepth && scoredDepth < surfaceDepth) {
1546
+ underReadExtensions.add(surface);
1547
+ }
1548
+ }
1549
+ } catch (err) {
1550
+ _iterator24.e(err);
1551
+ } finally {
1552
+ _iterator24.f();
1553
+ }
1554
+ }
1555
+
1556
+ /**
1557
+ * The evidence arbitration judges a candidate on.
1558
+ *
1559
+ * Wraps the depth-unaware read with the pool-relative requirement: a surface
1560
+ * that continues another pool member is not judgeable until it has been read
1561
+ * level with it, so the two totals being compared are taken at comparable
1562
+ * depths.
1563
+ */
1564
+ } catch (err) {
1565
+ _iterator19.e(err);
1566
+ } finally {
1567
+ _iterator19.f();
1568
+ }
1569
+ var judgedEvidenceFor = function judgedEvidenceFor(candidate) {
1570
+ return underReadExtensions.has(candidate.word) ? null : readJudgedEvidence(candidate);
1571
+ };
1572
+ var hasJudgeableEvidence = function hasJudgeableEvidence(candidate) {
1573
+ return judgedEvidenceFor(candidate) !== null;
1574
+ };
1575
+
1576
+ // Normalise over the pool's minimal members only, and score everyone —
1577
+ // extensions included — against that.
1578
+ //
1579
+ // A sequence total is its prefix's total plus a log-probability that cannot
1580
+ // be positive, so summing a surface and its own extension double-counts:
1581
+ // `exp(total)` for `happy` already contains every continuation of `happy`,
1582
+ // `happy path` among them. Dividing one pool's mass between the two capped
1583
+ // the shorter form at half however certain it was, and the longer form below
1584
+ // that — both under gates neither could then clear, for a reason that was
1585
+ // arithmetic rather than evidence.
1586
+ //
1587
+ // Excluding extensions from the denominator is the chain rule: the minimal
1588
+ // members partition the mass, and an extension's share of that same
1589
+ // denominator is `P(prefix) * P(continuation | prefix)`, which is exactly
1590
+ // what a multi-word suggestion should be held to. Extensions stay in
1591
+ // `contextTotals`, so how contested a context is still counts every scored
1592
+ // candidate.
1593
+ var logSumExpByContext = new Map();
1594
+ var _iterator20 = _createForOfIteratorHelper(contextSurfaces),
1595
+ _step20;
1596
+ try {
1597
+ for (_iterator20.s(); !(_step20 = _iterator20.n()).done;) {
1598
+ var _step20$value = _slicedToArray(_step20.value, 2),
1599
+ _contextKey3 = _step20$value[0],
1600
+ _surfaces2 = _step20$value[1];
1601
+ var _pool = new Set(_surfaces2);
1602
+ var minimalTotals = [];
1603
+ var _iterator25 = _createForOfIteratorHelper(_surfaces2),
1604
+ _step25;
1605
+ try {
1606
+ for (_iterator25.s(); !(_step25 = _iterator25.n()).done;) {
1607
+ var _surface = _step25.value;
1608
+ var total = optimisticTotalByWord.get(_surface);
1609
+ if (total !== undefined && !extendsAPoolMember(_surface, _pool)) {
1610
+ minimalTotals.push(total);
1611
+ }
1612
+ }
1613
+ // An extension is strictly longer than what it extends, so the shortest
1614
+ // member of any non-empty pool is always minimal and this is never empty.
1615
+ } catch (err) {
1616
+ _iterator25.e(err);
1617
+ } finally {
1618
+ _iterator25.f();
1619
+ }
1620
+ logSumExpByContext.set(_contextKey3, logSumExp(minimalTotals));
1621
+ }
1622
+
1623
+ /**
1624
+ * A candidate's share of its context shortlist's probability mass.
1625
+ *
1626
+ * Pass an optimistic (partially read) total for an upper bound on the
1627
+ * eventual posterior, or a verified total for the posterior itself.
1628
+ */
1629
+ } catch (err) {
1630
+ _iterator20.e(err);
1631
+ } finally {
1632
+ _iterator20.f();
1633
+ }
1634
+ var posteriorFor = function posteriorFor(candidate, total) {
1635
+ var runtime = runtimeBySurface.get(candidate.word);
1636
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
1637
+ if (total === undefined || normalizer === undefined || !Number.isFinite(normalizer)) {
1638
+ return 0;
1639
+ }
1640
+ return Math.exp(total - normalizer);
1641
+ };
1642
+ var optimisticPosterior = function optimisticPosterior(candidate) {
1643
+ return posteriorFor(candidate, optimisticTotalByWord.get(candidate.word));
1644
+ };
1645
+ var judgedPosterior = function judgedPosterior(candidate) {
1646
+ var _judgedEvidenceFor;
1647
+ return posteriorFor(candidate, (_judgedEvidenceFor = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor === void 0 ? void 0 : _judgedEvidenceFor.total);
1648
+ };
1649
+
1650
+ // A candidate stays in the race for as long as its optimistic posterior
1651
+ // could still clear the gate it will be held to. The bound only falls as
1652
+ // more tokens are read, so nothing dropped here could have gone on to win.
1653
+ var couldClearGate = function couldClearGate(candidate) {
1654
+ return optimisticPosterior(candidate) >= MIN_LM_POSTERIOR[candidate.termType];
1655
+ };
1656
+ var passesLmFloor = function passesLmFloor(candidate) {
1657
+ return judgedEvidenceFor(candidate) !== null && couldClearGate(candidate);
1658
+ };
1659
+ var hasRequiredEvidence = function hasRequiredEvidence(candidate) {
1660
+ var _candidate$lmEvidence2;
1661
+ if (canonicalLmSupported) {
1662
+ return passesLmFloor(candidate);
1663
+ }
1664
+ return candidate.termType === 'word' && ((_candidate$lmEvidence2 = candidate.lmEvidence) === null || _candidate$lmEvidence2 === void 0 ? void 0 : _candidate$lmEvidence2.source) === 'network-logit';
1665
+ };
1666
+
1667
+ // ── Stabilization guards (QI-2) ──────────────────────────────────────────
1668
+ // Whole-surface repetition: drop a candidate whose full surface already equals
1669
+ // the run of tokens immediately before the cursor — accepting it would
1670
+ // duplicate what was just typed (`end to end` → `end to end to end`).
1671
+ var beforeTokens = tokenize(trimmed);
1672
+ var duplicatesPreceding = function duplicatesPreceding(c) {
1673
+ var surfaceTokens = tokenize(c.word);
1674
+ var n = surfaceTokens.length;
1675
+ if (n === 0 || beforeTokens.length < n) {
1676
+ return false;
1677
+ }
1678
+ for (var i = 0; i < n; i++) {
1679
+ if (beforeTokens[beforeTokens.length - n + i] !== surfaceTokens[i]) {
1680
+ return false;
1681
+ }
1682
+ }
1683
+ return true;
1684
+ };
1685
+ // Post-accept cooldown: don't re-offer the surface the user just accepted while
1686
+ // its cooldown is still active (see cooldown advance near t0).
1687
+ var isCooledDown = function isCooledDown(c) {
1688
+ return cooledSurface !== null && c.word.toLowerCase() === cooledSurface;
1689
+ };
1690
+ var isStabilized = function isStabilized(candidate) {
1691
+ return !duplicatesPreceding(candidate) && !isCooledDown(candidate);
1692
+ };
1693
+ // A candidate the user has all but finished typing is not worth a ghost, and
1694
+ // a candidate we would never show is not worth waiting for either.
1695
+ var suggestionLengthFor = function suggestionLengthFor(candidate) {
1696
+ var _prefixLenByWord$get;
1697
+ return candidate.word.length - ((_prefixLenByWord$get = prefixLenByWord.get(candidate.word)) !== null && _prefixLenByWord$get !== void 0 ? _prefixLenByWord$get : currentWord.length);
1698
+ };
1699
+ var isWorthShowing = function isWorthShowing(candidate) {
1700
+ return suggestionLengthFor(candidate) >= MIN_SUGGESTION_LENGTH;
1701
+ };
1702
+ var evidenceBacked = ranked.filter(function (candidate) {
1703
+ return hasRequiredEvidence(candidate) && isStabilized(candidate);
1704
+ });
1705
+ var confidenceScore = function confidenceScore(candidate) {
1706
+ if (!canonicalLmSupported) {
1707
+ return candidate.finalScore;
1708
+ }
1709
+ var runtime = runtimeBySurface.get(candidate.word);
1710
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) === null || !optimisticTotalByWord.has(candidate.word)) {
1711
+ return candidate.stage1Score;
1712
+ }
1713
+ return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * optimisticPosterior(candidate);
1714
+ };
1715
+ var exactConfidenceScore = function exactConfidenceScore(candidate) {
1716
+ if (!canonicalLmSupported || judgedEvidenceFor(candidate) === null) {
1717
+ return candidate.finalScore;
1718
+ }
1719
+ return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * judgedPosterior(candidate);
1720
+ };
1721
+
1722
+ /**
1723
+ * How many candidates in this one's context were scored at all.
1724
+ *
1725
+ * Read off pool membership, which counts every candidate that contributed a
1726
+ * total, rather than off the normaliser, which the chain rule narrows to the
1727
+ * pool's minimal members.
1728
+ */
1729
+ var scoredPoolSize = function scoredPoolSize(candidate) {
1730
+ var _runtimeBySurface$get, _contextTotals$get$le, _contextTotals$get;
1731
+ var contextKey = (_runtimeBySurface$get = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get === void 0 ? void 0 : _runtimeBySurface$get.contextKey;
1732
+ return contextKey === undefined ? 0 : (_contextTotals$get$le = (_contextTotals$get = contextTotals.get(contextKey)) === null || _contextTotals$get === void 0 ? void 0 : _contextTotals$get.length) !== null && _contextTotals$get$le !== void 0 ? _contextTotals$get$le : 0;
1733
+ };
1734
+
1735
+ /**
1736
+ * How many candidates the context held before any of them were scored.
1737
+ *
1738
+ * A pool of one means two opposite things. Either the vocabulary offered a
1739
+ * single continuation of what was typed, which is the least contested case
1740
+ * there is; or rivals were offered and have not been read yet, which is the
1741
+ * least informed. Only the second deserves refusing, and the two are
1742
+ * distinguishable exactly here.
1743
+ */
1744
+ var requestedPoolSize = function requestedPoolSize(candidate) {
1745
+ var _runtimeBySurface$get2, _contextRequestedCoun;
1746
+ var contextKey = (_runtimeBySurface$get2 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get2 === void 0 ? void 0 : _runtimeBySurface$get2.contextKey;
1747
+ return contextKey === undefined ? 0 : (_contextRequestedCoun = contextRequestedCount.get(contextKey)) !== null && _contextRequestedCoun !== void 0 ? _contextRequestedCoun : 0;
1748
+ };
1749
+
1750
+ // A suppressed short completion still counts as competition below, so losing
1751
+ // to one abstains rather than promoting the runner-up in its place.
1752
+ var gateCleared = evidenceBacked.filter(isWorthShowing).map(function (candidate) {
1753
+ return {
1754
+ candidate: candidate,
1755
+ posterior: judgedPosterior(candidate),
1756
+ score: exactConfidenceScore(candidate)
1757
+ };
1758
+ })
1759
+ // The gate is the model's own confidence in the surface; the blended score
1760
+ // only orders what has already cleared it, so a strong corpus prior can no
1761
+ // longer carry a surface the model is unsure of onto the screen.
1762
+ .filter(function (_ref7) {
1763
+ var candidate = _ref7.candidate,
1764
+ posterior = _ref7.posterior;
1765
+ return posterior >= MIN_LM_POSTERIOR[candidate.termType];
1766
+ });
1767
+ /**
1768
+ * Whether the model finds the surface plausible on its own terms.
1769
+ *
1770
+ * The gate above asks which candidate won its pool; this asks whether
1771
+ * winning it meant anything. The two come apart exactly where a pool is
1772
+ * thin — a shortlist of one hands its only member 1.0 by construction — and
1773
+ * a relative test can never see through that however it is tuned.
1774
+ *
1775
+ * Waived without canonical scoring, where there is no judged mean to test
1776
+ * and the network-logit path would otherwise refuse everything.
1777
+ */
1778
+ var isPlausibleSurface = function isPlausibleSurface(candidate) {
1779
+ var _judgedEvidenceFor2;
1780
+ if (!canonicalLmSupported) {
1781
+ return true;
1782
+ }
1783
+ var mean = (_judgedEvidenceFor2 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor2 === void 0 ? void 0 : _judgedEvidenceFor2.mean;
1784
+ return mean !== undefined && mean >= MIN_MEAN_TOKEN_LOG_PROBABILITY;
1785
+ };
1786
+ // Kept as its own stage rather than folded into the gate so the three
1787
+ // populations stay separable in the abstain cascade: cleared its pool and was
1788
+ // implausible, was plausible and had nothing to clear the pool against, or
1789
+ // cleared both. A candidate refused here stays in `ranked` and so still
1790
+ // counts as competition below — promoting the runner-up in place of an
1791
+ // implausible leader would show something worse, not something better.
1792
+ var plausible = gateCleared.filter(function (_ref8) {
1793
+ var candidate = _ref8.candidate;
1794
+ return isPlausibleSurface(candidate);
1795
+ });
1796
+ // Applied after the gate rather than folded into it, so the two populations
1797
+ // stay separable: a surface refused here cleared its threshold and was
1798
+ // refused for having had nothing to clear it against.
1799
+ var eligible = plausible.filter(function (_ref9) {
1800
+ var candidate = _ref9.candidate;
1801
+ return !canonicalLmSupported || scoredPoolSize(candidate) >= MIN_SCORED_POOL_SIZE || requestedPoolSize(candidate) <= 1;
1802
+ }).sort(function (a, b) {
1803
+ return b.score - a.score;
1804
+ });
1805
+ var selected = (_eligible$ = eligible[0]) === null || _eligible$ === void 0 ? void 0 : _eligible$.candidate;
1806
+ var selectedDisplayScore = (_eligible$0$score = (_eligible$2 = eligible[0]) === null || _eligible$2 === void 0 ? void 0 : _eligible$2.score) !== null && _eligible$0$score !== void 0 ? _eligible$0$score : 0;
1807
+ var selectedEvidence = selected && hasRequiredEvidence(selected) ? selected.lmEvidence : null;
1808
+ /**
1809
+ * Whether `longer` is `shorter` continued past a word boundary, in the same
1810
+ * normaliser.
1811
+ *
1812
+ * Shared by the two places that have to agree on what an extension is: the
1813
+ * chain rule, which stops dividing a pool's mass between a sequence and its
1814
+ * own prefix, and the winner margin below, which stops treating one as the
1815
+ * other's rival. They were the same category error and are now the same test.
1816
+ */
1817
+ var extendsInSamePool = function extendsInSamePool(shorter, longer) {
1818
+ var _runtimeBySurface$get3, _runtimeBySurface$get4;
1819
+ if (longer.word === shorter.word || !optimisticTotalByWord.has(longer.word)) {
1820
+ return false;
1821
+ }
1822
+ var pool = (_runtimeBySurface$get3 = runtimeBySurface.get(shorter.word)) === null || _runtimeBySurface$get3 === void 0 ? void 0 : _runtimeBySurface$get3.contextKey;
1823
+ if (pool === undefined || ((_runtimeBySurface$get4 = runtimeBySurface.get(longer.word)) === null || _runtimeBySurface$get4 === void 0 ? void 0 : _runtimeBySurface$get4.contextKey) !== pool) {
1824
+ return false;
1825
+ }
1826
+ return longer.word.length > shorter.word.length && longer.word.startsWith(shorter.word) && longer.word[shorter.word.length] === ' ';
1827
+ };
1828
+ // A candidate that extends the selection is not competing with it. Whichever
1829
+ // continuation the user meant, the shorter surface is a correct ghost — it is
1830
+ // the prefix of both — so the margin has nothing to arbitrate and charging the
1831
+ // selection for its own extension abstains on a case that could not be wrong.
1832
+ // The chain rule already removed this from the denominator; leaving it in the
1833
+ // margin meant a nested leader cleared its gate and then lost to the candidate
1834
+ // that had just been freed from dividing its mass. It penalised bigrams and
1835
+ // phrases specifically, because a nested pool is where they mostly appear.
1836
+ var runnerUpScore = selected ? Math.max.apply(Math, _toConsumableArray(ranked.filter(function (candidate) {
1837
+ return candidate.word !== selected.word && isStabilized(candidate) && !extendsInSamePool(selected, candidate);
1838
+ }).map(confidenceScore)).concat([0])) : 0;
1839
+ var winnerMargin = selected ? selectedDisplayScore - runnerUpScore : 0;
1840
+ var hasMissingArtifactCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1841
+ var runtime = runtimeBySurface.get(candidate.word);
1842
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) === 'whitespace' && runtime.canonicalTokenIds === null;
1843
+ });
1844
+ // A candidate with canonical tokens but no evidence at all cannot join the
1845
+ // posterior: it has no likelihood to contribute, and any stand-in value would
1846
+ // distort the normaliser for every other candidate in its context. Absent
1847
+ // evidence is not evidence of absence though — the context's boundary may
1848
+ // simply not be primed yet — so it holds the ghost back instead of being
1849
+ // silently ignored. In a primed context every candidate has at least
1850
+ // first-token evidence, so this only fires while a context is still cold.
1851
+ var hasUnscoredCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1852
+ var runtime = runtimeBySurface.get(candidate.word);
1853
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !optimisticTotalByWord.has(candidate.word) && isStabilized(candidate) && isWorthShowing(candidate);
1854
+ });
1855
+ // A rival only holds the ghost back while it is still too unread to judge.
1856
+ // Under full-surface evidence that meant any long surface blocked everything
1857
+ // until every one of its tokens had been paid for, which abstained far more
1858
+ // often than it ever changed the winner. A rival now becomes judgeable after
1859
+ // `REQUIRED_VERIFIED_TOKENS`, so this settles within the budget.
1860
+ var hasUnresolvedPotential = canonicalLmSupported && ranked.some(function (candidate) {
1861
+ var runtime = runtimeBySurface.get(candidate.word);
1862
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !hasJudgeableEvidence(candidate) && couldClearGate(candidate) && isStabilized(candidate) && isWorthShowing(candidate);
1863
+ });
1864
+ var requiredWinnerMargin = MIN_WINNER_MARGIN + (selected && ((_judgedEvidenceFor3 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor3 === void 0 ? void 0 : _judgedEvidenceFor3.complete) === false ? PARTIAL_EVIDENCE_MARGIN_PREMIUM : 0);
1865
+ // Everything the display decision needs except the veto itself. Splitting it
1866
+ // out is what lets the veto be priced: the value here at the moment the veto
1867
+ // fires is exactly the ghost the veto cost us.
1868
+ var clearsMarginBeforeColdVeto = selected !== undefined && !hasMissingArtifactCompetitor && winnerMargin >= requiredWinnerMargin;
1869
+ var clearsWinnerMargin = clearsMarginBeforeColdVeto && !hasUnscoredCompetitor;
1870
+ var suggestion = displayPrefixReady && selected && clearsWinnerMargin ? selected.word.slice((_prefixLenByWord$get2 = prefixLenByWord.get(selected.word)) !== null && _prefixLenByWord$get2 !== void 0 ? _prefixLenByWord$get2 : currentWord.length) : null;
1871
+
1872
+ // Which constraint bound. Computed on every evaluation rather than only under
1873
+ // debug, because it is counted for the session as well as printed, and
1874
+ // deriving the debug header from it keeps the two from disagreeing.
1875
+ var resolveAbstainReason = function resolveAbstainReason() {
1876
+ if (suggestion !== null && suggestion.length > 0) {
1877
+ return null;
1878
+ }
1879
+ if (!displayPrefixReady) {
1880
+ return 'prefetch';
1881
+ }
1882
+ if (ranked.length === 0) {
1883
+ return 'no-candidate';
1884
+ }
1885
+ if (hasMissingArtifactCompetitor) {
1886
+ return 'missing-artifact';
1887
+ }
1888
+ if (hasUnscoredCompetitor) {
1889
+ return 'cold-competitor';
1890
+ }
1891
+ if (hasUnresolvedPotential && !clearsWinnerMargin) {
1892
+ return 'unresolved-rival';
1893
+ }
1894
+ if (eligible.length > 0) {
1895
+ return selected && !clearsWinnerMargin ? 'winner-margin' : 'empty-completion';
1896
+ }
1897
+ if (plausible.length > 0) {
1898
+ // Cleared its gate, was plausible on its own terms, and was refused for
1899
+ // the shortlist it cleared that gate against holding nobody else.
1900
+ return 'lone-candidate';
1901
+ }
1902
+ if (gateCleared.length > 0) {
1903
+ // Held the largest share of its shortlist and still was not a surface
1904
+ // the model found plausible — which is only possible because a share is
1905
+ // relative and this is not.
1906
+ return 'implausible-surface';
1907
+ }
1908
+ if (evidenceBacked.length > 0 && evidenceBacked.every(function (candidate) {
1909
+ return !isWorthShowing(candidate);
1910
+ })) {
1911
+ return 'short-completion';
1912
+ }
1913
+ // The floor and the gate are one threshold, applied to the bound and to
1914
+ // the verified value, so a candidate short of it is dropped before
1915
+ // `evidenceBacked` is built. Asking whether anything was judgeable
1916
+ // separates a gate that rejected read candidates from a context where
1917
+ // nothing was read at all — the first is a threshold to calibrate, the
1918
+ // second is scheduling.
1919
+ return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
1920
+ };
1921
+ var abstainReason = resolveAbstainReason();
1922
+
1923
+ // The leader is the best-supported candidate, not the selected one: an
1924
+ // evaluation that showed nothing is exactly the one whose posterior needs
1925
+ // recording, and it has no selection to report.
1926
+ var posteriorLeader = ranked.reduce(function (best, candidate) {
1927
+ var posterior = optimisticPosterior(candidate);
1928
+ return posterior > 0 && (best === null || posterior > best.posterior) ? {
1929
+ candidate: candidate,
1930
+ posterior: posterior
1931
+ } : best;
1932
+ }, null);
1933
+ /**
1934
+ * The longer surface in the leader's normaliser that extends the given
1935
+ * candidate, if the pool holds one.
1936
+ *
1937
+ * `logSumExp` divides one pool's mass between a sequence and its own prefix,
1938
+ * and the longer total is the shorter one plus a log-probability that cannot
1939
+ * be positive — so the longer form holds at most half, or at most a third
1940
+ * where both shorter forms are present. Both sit under the gate it is held
1941
+ * to, so it cannot display however certain its continuation is.
1942
+ *
1943
+ * Asked from the shorter form because that is the one that leads: the
1944
+ * extending candidate is strictly lower in the same pool and so can never be
1945
+ * the leader.
1946
+ */
1947
+ var poolExtensionOf = function poolExtensionOf(shorter) {
1948
+ return ranked.find(function (candidate) {
1949
+ return extendsInSamePool(shorter, candidate);
1950
+ });
1951
+ };
1952
+ if (isAutocompleteDebugEnabled()) {
1953
+ var _posteriorLeader$cand, _posteriorLeader$post, _selectedEvidence$mea, _selectedEvidence$mea2, _acceptCooldown$predi, _acceptCooldown;
1954
+ var verbose = isAutocompleteDebugVerbose();
1955
+ var latencyMs = performance.now() - t0;
1956
+ var tokens = tokenize(trimmed);
1957
+ var contextWords = tokens.slice(-CONTEXT_WORDS);
1958
+
1959
+ // Decision drives the (collapsed) group header. Abstains carry the reason
1960
+ // the session counters recorded, plus the numbers behind it, so it is clear
1961
+ // *why* nothing showed without expanding the group.
1962
+ var abstainDetail = {
1963
+ 'below-posterior-gate': "posterior below ".concat(MIN_LM_POSTERIOR[((_posteriorLeader$cand = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.candidate) !== null && _posteriorLeader$cand !== void 0 ? _posteriorLeader$cand : ranked[0]).termType], " (best ").concat(((_posteriorLeader$post = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.posterior) !== null && _posteriorLeader$post !== void 0 ? _posteriorLeader$post : 0).toFixed(2), ")"),
1964
+ 'cold-competitor': 'competitor has no LM evidence yet',
1965
+ 'empty-completion': 'empty completion',
1966
+ 'implausible-surface': "mean per-token log-probability below ".concat(MIN_MEAN_TOKEN_LOG_PROBABILITY, " (best ").concat(gateCleared.map(function (_ref0) {
1967
+ var _judgedEvidenceFor$me, _judgedEvidenceFor4;
1968
+ var candidate = _ref0.candidate;
1969
+ return (_judgedEvidenceFor$me = (_judgedEvidenceFor4 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor4 === void 0 ? void 0 : _judgedEvidenceFor4.mean) !== null && _judgedEvidenceFor$me !== void 0 ? _judgedEvidenceFor$me : -Infinity;
1970
+ }).reduce(function (best, mean) {
1971
+ return Math.max(best, mean);
1972
+ }, -Infinity).toFixed(2), ")"),
1973
+ 'lone-candidate': "cleared its gate against fewer than ".concat(MIN_SCORED_POOL_SIZE, " scored candidates, rivals unread"),
1974
+ 'missing-artifact': 'canonical artifact coverage missing',
1975
+ 'no-candidate': 'nothing cleared scoring',
1976
+ 'no-evidence': 'full-surface evidence absent',
1977
+ prefetch: "prefetch: ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " chars"),
1978
+ 'short-completion': "completion shorter than ".concat(MIN_SUGGESTION_LENGTH, " chars"),
1979
+ 'unresolved-rival': 'expanding plausible token-prefix groups',
1980
+ 'winner-margin': "winner margin ".concat(winnerMargin.toFixed(2), " < ").concat(requiredWinnerMargin.toFixed(2))
1981
+ };
1982
+ var decision = abstainReason === null ? "\u2728 \"".concat(suggestion !== null && suggestion !== void 0 ? suggestion : '', "\"") : abstainReason === 'prefetch' ? "\u23F3 ".concat(abstainDetail.prefetch) : "\u2014 abstain: ".concat(abstainDetail[abstainReason]);
1983
+ lastPredictionDebug = {
1984
+ awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
1985
+ textBefore: trimmed,
1986
+ currentWord: currentWord,
1987
+ decision: decision,
1988
+ mode: mode,
1989
+ contextWords: contextWords,
1990
+ topCandidates: ranked.slice(0, 5).map(function (r) {
1991
+ return {
1992
+ word: r.word,
1993
+ finalScore: r.finalScore,
1994
+ semanticScore: r.semanticScore,
1995
+ freqScore: r.freqScore,
1996
+ lmScore: r.lmScore
590
1997
  };
591
1998
  }),
592
1999
  suggestion: suggestion && suggestion.length > 0 ? suggestion : null
593
2000
  };
594
2001
 
595
- // ── Mode label: COLD / WARM(local) / WARM(BE) ───────────────────────
2002
+ // Mode describes the semantic-vector source without implying that a
2003
+ // network backend ran; the snapshot may come from the local MLC embedder.
596
2004
  var slowLaneVec = getStoredContextVector();
597
2005
  var isUsingSlowLaneVector = slowLaneVec !== null && vectorStore !== null && slowLaneVec.length === vectorStore.dim;
598
- var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(BE)' : 'WARM(local)';
599
- var modeColor = modeLabel === 'WARM(BE)' ? 'color: #ff9800; font-weight: bold;' : modeLabel === 'WARM(local)' ? 'color: #4caf50; font-weight: bold;' : 'color: #9e9e9e; font-weight: bold;';
2006
+ var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(semantic)' : 'WARM(local-avg)';
2007
+ var modeColor = modeLabel === 'WARM(semantic)' ? CTC_STYLES.good : modeLabel === 'WARM(local-avg)' ? CTC_STYLES.warn : CTC_STYLES.cold;
600
2008
 
601
- // 1. Collapsible group header
2009
+ // Group header: [CTC:<mode>] <decision> · <mode> · <type> · <latency>
2010
+ // arbitrationMode ('v1') tags the line so this deploy's logs diff cleanly
2011
+ // against the baseline deploy's.
602
2012
  // eslint-disable-next-line no-console
603
- console.groupCollapsed("%c[Autocomplete] %c".concat(modeLabel, " %c| \"").concat(currentWord, "\" \u2794 ").concat(suggestionLabel, " | \u23F1 ").concat(latencyMs.toFixed(1), "ms"), 'color: #00b8d9; font-weight: bold;', modeColor, 'color: inherit; font-weight: normal;');
2013
+ console.groupCollapsed("%c[CTC:".concat(ARBITRATION_MODE, "]%c ").concat(decision, " %c\xB7 ").concat(modeLabel).concat(selected ? " \xB7 ".concat(selected.termType) : '', " \xB7 \u23F1 ").concat(latencyMs.toFixed(1), "ms"), CTC_STYLES.brand, CTC_STYLES.body, modeColor);
2014
+ var semanticInput = getStoredContextInput();
2015
+ var causalPrompts = primeRequests.map(function (request, index) {
2016
+ var tail = request.prompt.slice(-DEBUG_TEXT_TAIL_CHARS);
2017
+ return "#".concat(index + 1, ":").concat(JSON.stringify(tail));
2018
+ });
604
2019
 
605
- // 2. Context window (what local vector averaging sees)
606
- // eslint-disable-next-line no-console
607
- console.log('%cContext Window:', 'color: #888; font-style: italic;', contextWords.length ? contextWords.join(' ') : '(none)');
2020
+ // Keep the exact model inputs separate from normalized lookup tokens.
2021
+ ctcSection('INPUT', "raw: ".concat(JSON.stringify(trimmed.slice(-DEBUG_TEXT_TAIL_CHARS)), " \xB7 prev: ").concat(previousWord ? JSON.stringify(previousWord) : '—'));
2022
+ ctcSection('LOOKUP', "normalized(last ".concat(CONTEXT_WORDS, "): ").concat(contextWords.length ? JSON.stringify(contextWords.join(' ')) : '(none)'));
2023
+ ctcSection('SEMANTIC', "snapshot: ".concat(semanticInput ? JSON.stringify(semanticInput.slice(-DEBUG_TEXT_TAIL_CHARS)) : '(unavailable)'));
2024
+ ctcSection('CAUSAL', "prompt".concat(causalPrompts.length === 1 ? '' : 's', ": ").concat(causalPrompts.join(' · ') || '(not ready)'));
608
2025
 
609
- // 3. Slow-lane status
610
- var vectorStatus = isUsingSlowLaneVector ? "\u2705 BE semantic vector (dim=".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, ")") : vectorStore ? '⚠️ Local vector average (slow-lane not yet returned)' : '❌ No vectors (cold)';
611
- var logitsStatus = lmLogits && Object.keys(lmLogits).length > 0 ? "\u2705 LM logits active (".concat(Object.keys(lmLogits).length, " tokens)") : '⏳ No LM logits (slow-lane pending or failed)';
612
- // eslint-disable-next-line no-console
613
- console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
2026
+ // SIGNALS — semantic state plus context-keyed canonical LM coverage.
2027
+ var logitCount = lmLogits ? Object.keys(lmLogits).length : 0;
2028
+ var semanticStatus = isUsingSlowLaneVector ? "semantic \u2705 snapshot(".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, "d)") : vectorStore ? 'semantic ⚠️ local-avg' : 'semantic ❌ cold';
2029
+ var tierACount = ranked.filter(function (candidate) {
2030
+ var _candidate$lmEvidence3;
2031
+ return ((_candidate$lmEvidence3 = candidate.lmEvidence) === null || _candidate$lmEvidence3 === void 0 ? void 0 : _candidate$lmEvidence3.source) === 'canonical-first-token';
2032
+ }).length;
2033
+ var exactCount = ranked.filter(function (candidate) {
2034
+ var _candidate$lmEvidence4;
2035
+ return ((_candidate$lmEvidence4 = candidate.lmEvidence) === null || _candidate$lmEvidence4 === void 0 ? void 0 : _candidate$lmEvidence4.source) === 'canonical-full-surface';
2036
+ }).length;
2037
+ var boundaryReady = primeRequests.filter(function (request) {
2038
+ return getBoundaryLmState(request.contextKey) !== null;
2039
+ }).length;
2040
+ var artifactEligible = canonicalMatched.filter(function (candidate) {
2041
+ return candidate.canonicalTokenIds !== null;
2042
+ }).length;
2043
+ var documentStartFallbacks = canonicalMatched.filter(function (candidate) {
2044
+ return candidate.separatorKind === 'document-start';
2045
+ }).length;
2046
+ var punctuationFallbacks = canonicalMatched.filter(function (candidate) {
2047
+ return candidate.separatorKind === 'non-space';
2048
+ }).length;
2049
+ var logitsStatus = tierACount + exactCount > 0 ? "canonical \u2705 first-token:".concat(tierACount, " exact:").concat(exactCount) : logitCount > 0 ? "network logits \u2705 ".concat(logitCount) : 'canonical ⏳/absent';
2050
+ var formula = tierACount + exactCount + logitCount > 0 ? "Stage1\xD7".concat(STAGE1_WEIGHT, "+LM\xD7").concat(STAGE2_WEIGHT) : 'Stage1-only';
2051
+ ctcSection('SIGNALS', "".concat(semanticStatus, " \xB7 ").concat(logitsStatus, " \xB7 ").concat(formula));
2052
+ ctcSection('CANONICAL', "eligible ".concat(artifactEligible, "/").concat(canonicalMatched.length, " \xB7 separator fallback doc:").concat(documentStartFallbacks, " punct:").concat(punctuationFallbacks, " \xB7 contexts ").concat(boundaryReady, "/").concat(primeRequests.length, " ready (\u2264").concat(PHRASE_MAX_WORDS, ") \xB7 family=").concat(familyKey.slice(0, 72) || 'none'));
2053
+ if (verbose && logitCount > 0 && lmLogits) {
2054
+ var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
2055
+ return b[1] - a[1];
2056
+ }).slice(0, 5).map(function (_ref1) {
2057
+ var _ref10 = _slicedToArray(_ref1, 2),
2058
+ word = _ref10[0],
2059
+ score = _ref10[1];
2060
+ return "".concat(word, ":").concat(score.toFixed(3));
2061
+ }).join(', ');
2062
+ ctcSection(' rawLM', "\uD83E\uDDE0 ".concat(rawLmTop));
2063
+ }
614
2064
 
615
- // 4. Scoring formula active this prediction
616
- var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? "Stage1(\xD7".concat(STAGE1_WEIGHT, ") + LM(\xD7").concat(STAGE2_WEIGHT, ")") : 'Stage1 only (no LM logits)';
617
- // eslint-disable-next-line no-console
618
- console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
2065
+ // GENERATE — how many candidates matched, split by term type
2066
+ var genByType = {
2067
+ word: 0,
2068
+ bigram: 0,
2069
+ phrase: 0
2070
+ };
2071
+ var _iterator21 = _createForOfIteratorHelper(canonicalMatched),
2072
+ _step21;
2073
+ try {
2074
+ for (_iterator21.s(); !(_step21 = _iterator21.n()).done;) {
2075
+ var m = _step21.value;
2076
+ genByType[m.node.termType] += 1;
2077
+ }
2078
+ } catch (err) {
2079
+ _iterator21.e(err);
2080
+ } finally {
2081
+ _iterator21.f();
2082
+ }
2083
+ ctcSection('GENERATE', "matched ".concat(canonicalMatched.length, " \u2192 word:").concat(genByType.word, " bigram:").concat(genByType.bigram, " phrase:").concat(genByType.phrase).concat(canonicalLmSupported ? ' · display needs exact surface evidence' : ''));
619
2084
 
620
- // 5. Grammar filter result (collected inside rankCandidates, logged here)
2085
+ // SCORE — Stage-1 + grammar funnel and canonical evidence depth.
2086
+ var lmCoverage = ranked.slice(0, 10).filter(function (r) {
2087
+ return r.lmScore > 0;
2088
+ }).length;
2089
+ ctcSection('SCORE', "in ".concat(pipelineDebug.initial, " \u2192 stage1(<").concat(MIN_STAGE1_SCORE, ") \u2212").concat(pipelineDebug.stage1Rejected.length, " \u2192 grammar \u2212").concat(pipelineDebug.grammarRejected.length, " \u2192 final ").concat(pipelineDebug.final, " \xB7 LM cov ").concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " \xB7 first-token ").concat(tierACount, " \xB7 exact ").concat(exactCount));
621
2090
  if (grammarMeta) {
622
- // eslint-disable-next-line no-console
623
- console.log("%c[Grammar] \"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] \u2192 ").concat(grammarMeta.before, " candidates \u2192 ").concat(grammarMeta.after, " after filter"), 'color: #4caf50; font-weight: bold;');
624
- if (grammarMeta.dropped.length > 0) {
625
- // eslint-disable-next-line no-console
626
- console.log("%c\uD83D\uDEAB Dropped: ".concat(grammarMeta.dropped.join(', ')), 'color: #f44336; font-style: italic;');
627
- }
2091
+ ctcSection(' grammar', "\"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] ").concat(grammarMeta.before, "\u2192").concat(grammarMeta.after).concat(verbose && grammarMeta.dropped.length > 0 ? " \xB7 dropped: ".concat(grammarMeta.dropped.join(', ')) : ''));
628
2092
  }
2093
+ var gateReasonFor = function gateReasonFor(candidate) {
2094
+ var runtime = runtimeBySurface.get(candidate.word);
2095
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace') {
2096
+ var _runtime$separatorKin;
2097
+ return "separator:".concat((_runtime$separatorKin = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin !== void 0 ? _runtime$separatorKin : 'unknown');
2098
+ }
2099
+ if (runtime.canonicalTokenIds === null) {
2100
+ return 'missing-artifact';
2101
+ }
2102
+ if (!hasJudgeableEvidence(candidate)) {
2103
+ return "awaiting-tokens:".concat(REQUIRED_VERIFIED_TOKENS);
2104
+ }
2105
+ if (!passesLmFloor(candidate)) {
2106
+ return "bound:".concat(optimisticPosterior(candidate).toFixed(2), "<").concat(MIN_LM_POSTERIOR[candidate.termType]);
2107
+ }
2108
+ if (duplicatesPreceding(candidate)) {
2109
+ return 'repetition';
2110
+ }
2111
+ if (isCooledDown(candidate)) {
2112
+ return 'cooldown';
2113
+ }
2114
+ if (!isWorthShowing(candidate)) {
2115
+ return "suffix:".concat(suggestionLengthFor(candidate), "<").concat(MIN_SUGGESTION_LENGTH);
2116
+ }
2117
+ var floor = MIN_LM_POSTERIOR[candidate.termType];
2118
+ var posterior = judgedPosterior(candidate);
2119
+ if (posterior < floor) {
2120
+ return "posterior:".concat(posterior.toFixed(2), "<").concat(floor);
2121
+ }
2122
+ if (!isPlausibleSurface(candidate)) {
2123
+ var _judgedEvidenceFor$me2, _judgedEvidenceFor5;
2124
+ return "implausible:".concat(((_judgedEvidenceFor$me2 = (_judgedEvidenceFor5 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor5 === void 0 ? void 0 : _judgedEvidenceFor5.mean) !== null && _judgedEvidenceFor$me2 !== void 0 ? _judgedEvidenceFor$me2 : NaN).toFixed(2), "<").concat(MIN_MEAN_TOKEN_LOG_PROBABILITY);
2125
+ }
2126
+ return 'eligible';
2127
+ };
2128
+ var gateFunnel = function gateFunnel(termType) {
2129
+ var typeRanked = ranked.filter(function (candidate) {
2130
+ return candidate.termType === termType;
2131
+ });
2132
+ var exact = typeRanked.filter(hasJudgeableEvidence);
2133
+ var absolute = exact.filter(passesLmFloor);
2134
+ var stabilized = absolute.filter(isStabilized);
2135
+ var longEnough = stabilized.filter(isWorthShowing);
2136
+ var floorPassed = longEnough.filter(function (candidate) {
2137
+ return judgedPosterior(candidate) >= MIN_LM_POSTERIOR[termType];
2138
+ });
2139
+ var plausiblePassed = floorPassed.filter(isPlausibleSurface);
2140
+ return "".concat(termType, " m:").concat(genByType[termType], " r:").concat(typeRanked.length, " exact:").concat(exact.length, " abs:").concat(absolute.length, " stable:").concat(stabilized.length, " suffix:").concat(longEnough.length, " floor:").concat(floorPassed.length, " plausible:").concat(plausiblePassed.length, " eligible:").concat(eligible.filter(function (_ref11) {
2141
+ var candidate = _ref11.candidate;
2142
+ return candidate.termType === termType;
2143
+ }).length);
2144
+ };
2145
+ ctcSection('GATES', "".concat(gateFunnel('word'), " | ").concat(gateFunnel('bigram'), " | ").concat(gateFunnel('phrase')));
629
2146
 
630
- // 6. Pipeline funnel
631
- // eslint-disable-next-line no-console
632
- console.log("%c[Pipeline Funnel] %c\uD83D\uDCE5 In: ".concat(pipelineDebug.initial, " | \u274C Stage 1 (< ").concat(MIN_STAGE1_SCORE, "): -").concat(pipelineDebug.stage1Rejected.length, " | \u274C Grammar: -").concat(pipelineDebug.grammarRejected.length, " | \u2705 Final: ").concat(pipelineDebug.final), 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
2147
+ // One line per context: how many candidates share the normaliser, and how
2148
+ // much of the mass the leader holds. A leader well under its threshold
2149
+ // means the context is contested, which is the abstention we want.
2150
+ var contextLeaders = Array.from(bestCandidateByContext.entries()).slice(0, PHRASE_MAX_WORDS).map(function (_ref12) {
2151
+ var _contextTotals$get$le2, _contextTotals$get2;
2152
+ var _ref13 = _slicedToArray(_ref12, 2),
2153
+ contextKey = _ref13[0],
2154
+ leader = _ref13[1];
2155
+ var shortlistSize = (_contextTotals$get$le2 = (_contextTotals$get2 = contextTotals.get(contextKey)) === null || _contextTotals$get2 === void 0 ? void 0 : _contextTotals$get2.length) !== null && _contextTotals$get$le2 !== void 0 ? _contextTotals$get$le2 : 0;
2156
+ var evidenceKind = hasExactEvidence(leader) ? 'exact' : 'upper';
2157
+ return "".concat(contextKey.slice(0, 32), " \u2192 n=").concat(shortlistSize, " ").concat(leader.termType, ":\"").concat(leader.word, "\" ").concat(evidenceKind, " p=").concat(optimisticPosterior(leader).toFixed(3), "/").concat(MIN_LM_POSTERIOR[leader.termType]);
2158
+ }).join(' | ');
2159
+ ctcSection('NORMALIZE', contextLeaders.length > 0 ? contextLeaders : 'no scored candidate in any context');
2160
+ ctcSection('READINESS', "prefix ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " \xB7 boundary ").concat(boundaryReady, "/").concat(primeRequests.length, " \xB7 exact word:").concat(ranked.filter(function (candidate) {
2161
+ return candidate.termType === 'word' && hasExactEvidence(candidate);
2162
+ }).length, " bigram:").concat(ranked.filter(function (candidate) {
2163
+ return candidate.termType === 'bigram' && hasExactEvidence(candidate);
2164
+ }).length, " phrase:").concat(ranked.filter(function (candidate) {
2165
+ return candidate.termType === 'phrase' && hasExactEvidence(candidate);
2166
+ }).length, " \xB7 ").concat(suggestion ? 'display-ready' : decision));
633
2167
 
634
- // 7. Candidate table
635
- if (ranked.length > 0) {
636
- var lmCoverage = ranked.slice(0, 10).filter(function (r) {
637
- return r.lmScore > 0;
638
- }).length;
639
- // eslint-disable-next-line no-console
640
- console.log("%cLM coverage: ".concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " candidates had real logit scores"), 'color: #888; font-style: italic;');
2168
+ // ARBITRATE — expose every condition that can authorize or suppress ghost.
2169
+ var topOfType = function topOfType(t) {
2170
+ return ranked.find(function (c) {
2171
+ return c.termType === t;
2172
+ });
2173
+ };
2174
+ var fmtType = function fmtType(t) {
2175
+ var _top$lmEvidence$sourc, _top$lmEvidence;
2176
+ var top = topOfType(t);
2177
+ if (!top) {
2178
+ return "".concat(t, ":\u2014");
2179
+ }
2180
+ var floor = MIN_LM_POSTERIOR[t];
2181
+ var bound = optimisticPosterior(top);
2182
+ var evidence = hasRequiredEvidence(top) ? 'TierB✅' : "".concat((_top$lmEvidence$sourc = (_top$lmEvidence = top.lmEvidence) === null || _top$lmEvidence === void 0 ? void 0 : _top$lmEvidence.source) !== null && _top$lmEvidence$sourc !== void 0 ? _top$lmEvidence$sourc : 'absent', "\u274C");
2183
+ return "".concat(t, ":\"").concat(top.word, "\" p\u2264").concat(bound.toFixed(3)).concat(bound >= floor ? '≥' : '<').concat(floor, " fs\u2264").concat(confidenceScore(top).toFixed(2), " ").concat(evidence);
2184
+ };
2185
+ ctcSection('ARBITRATE', "".concat(selected ? "\u25B8 ".concat(selected.termType, " \"").concat(selected.word, "\" fs=").concat(selectedDisplayScore.toFixed(2), " mean=").concat((_selectedEvidence$mea = selectedEvidence === null || selectedEvidence === void 0 || (_selectedEvidence$mea2 = selectedEvidence.meanTokenLogProbability) === null || _selectedEvidence$mea2 === void 0 ? void 0 : _selectedEvidence$mea2.toFixed(2)) !== null && _selectedEvidence$mea !== void 0 ? _selectedEvidence$mea : 'network', " margin=").concat(winnerMargin.toFixed(2)).concat(clearsWinnerMargin ? '✅' : '❌') : '▸ (none)', " \xB7 prefix ").concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH).concat(displayPrefixReady ? '✅' : '⏳', " \xB7 grouped pending ").concat(hasUnresolvedPotential ? 'yes' : 'no', " | ").concat(fmtType('word'), " ").concat(fmtType('bigram'), " ").concat(fmtType('phrase')));
2186
+
2187
+ // STABILIZE — post-accept cooldown + whole-surface repetition (QI-2)
2188
+ var repetitionBlocked = ranked.filter(function (c) {
2189
+ return hasRequiredEvidence(c) && duplicatesPreceding(c);
2190
+ }).map(function (c) {
2191
+ return c.word;
2192
+ });
2193
+ var cooldownLabel = cooledSurface ? "cooldown \"".concat(cooledSurface, "\" (").concat((_acceptCooldown$predi = (_acceptCooldown = acceptCooldown) === null || _acceptCooldown === void 0 ? void 0 : _acceptCooldown.predictionsSince) !== null && _acceptCooldown$predi !== void 0 ? _acceptCooldown$predi : 0, "/").concat(COOLDOWN_KEYSTROKES, " keys \xB7 ").concat(COOLDOWN_MS, "ms)") : 'cooldown —';
2194
+ ctcSection('STABILIZE', "".concat(cooldownLabel, " \xB7 repetition-blocked: ").concat(repetitionBlocked.length > 0 ? repetitionBlocked.join(', ') : '—'));
2195
+
2196
+ // CANDIDATES — full scored table (verbose only)
2197
+ if (verbose && ranked.length > 0) {
641
2198
  var tableData = ranked.slice(0, 10).map(function (r) {
2199
+ var _r$lmEvidence, _contextTotals$get$le3, _contextTotals$get3, _r$lmEvidence$source, _r$lmEvidence2, _runtime$separatorKin2, _runtime$canonicalTok2, _runtime$canonicalTok3, _runtime$contextKey, _r$lmEvidence3, _r$lmEvidence4;
642
2200
  var rawLogit = 'Not in Payload';
643
- if (prefixLmLogits) {
2201
+ if (((_r$lmEvidence = r.lmEvidence) === null || _r$lmEvidence === void 0 ? void 0 : _r$lmEvidence.rawLogit) !== undefined) {
2202
+ rawLogit = Number(r.lmEvidence.rawLogit.toFixed(5));
2203
+ } else if (prefixLmLogits) {
644
2204
  var val = prefixLmLogits[r.word.toLowerCase()];
645
2205
  if (val !== undefined) {
646
2206
  rawLogit = Number(val.toFixed(5));
@@ -649,9 +2209,19 @@ export var predict = function predict(textBefore) {
649
2209
  var original = scoringCandidates.find(function (sc) {
650
2210
  return sc.word === r.word;
651
2211
  });
2212
+ var runtime = runtimeBySurface.get(r.word);
2213
+ var progress = runtime ? getProgressiveSurfaceEvidence(runtime.contextKey, r.word) : null;
2214
+ var optimisticTotal = optimisticTotalByWord.get(r.word);
2215
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
2216
+ var shortlistSize = runtime ? (_contextTotals$get$le3 = (_contextTotals$get3 = contextTotals.get(runtime.contextKey)) === null || _contextTotals$get3 === void 0 ? void 0 : _contextTotals$get3.length) !== null && _contextTotals$get$le3 !== void 0 ? _contextTotals$get$le3 : 0 : 0;
2217
+ var contextLeader = runtime ? bestCandidateByContext.get(runtime.contextKey) : undefined;
652
2218
  var source = 'Unknown';
653
2219
  if (original) {
654
- if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
2220
+ if (original.termType === 'bigram') {
2221
+ source = '🔗 L2 (Bigram)';
2222
+ } else if (original.termType === 'phrase') {
2223
+ source = '🧩 L2 (Phrase)';
2224
+ } else if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
655
2225
  source = '🌍 L3 (Generic)';
656
2226
  } else if (original.sessionFreq > 0 && original.tenantFreq === 0) {
657
2227
  source = '👤 L1 (Session Only)';
@@ -659,13 +2229,40 @@ export var predict = function predict(textBefore) {
659
2229
  source = '🏢 L2 (Domain)';
660
2230
  }
661
2231
  }
2232
+ var floor = MIN_LM_POSTERIOR[r.termType];
662
2233
  return {
663
2234
  Candidate: r.word,
664
2235
  Source: source,
2236
+ Type: r.termType,
665
2237
  'Final Score': Number(r.finalScore.toFixed(4)),
2238
+ 'Bounded Score': Number(confidenceScore(r).toFixed(4)),
2239
+ 'Exact Gate Score': hasExactEvidence(r) ? Number(exactConfidenceScore(r).toFixed(4)) : '—',
2240
+ Floor: floor,
2241
+ 'Posterior Floor': optimisticPosterior(r) >= floor ? '✅' : '❌',
2242
+ 'LM Floor': hasExactEvidence(r) ? passesLmFloor(r) ? '✅' : '❌' : '⏳',
2243
+ 'Gate Reason': gateReasonFor(r),
2244
+ 'Display Eligible': hasRequiredEvidence(r) && isStabilized(r) && judgedPosterior(r) >= floor ? '✅' : '❌',
666
2245
  Semantics: Number(r.semanticScore.toFixed(4)),
667
2246
  Freq: Number(r.freqScore.toFixed(4)),
668
2247
  'LM Score': Number(r.lmScore.toFixed(4)),
2248
+ 'LM Evidence': (_r$lmEvidence$source = (_r$lmEvidence2 = r.lmEvidence) === null || _r$lmEvidence2 === void 0 ? void 0 : _r$lmEvidence2.source) !== null && _r$lmEvidence$source !== void 0 ? _r$lmEvidence$source : 'absent',
2249
+ 'Verified Tokens': r.lmEvidence ? "".concat(r.lmEvidence.verifiedTokenCount, "/").concat(r.lmEvidence.totalTokenCount) : '0/0',
2250
+ 'Progressive Tokens': progress ? "".concat(progress.scoredTokenCount, "/").concat(progress.totalTokenCount) : '0/0',
2251
+ 'LM Mean Upper': progress !== null ? Number(progress.meanTokenLogProbabilityUpperBound.toFixed(5)) : '—',
2252
+ 'Optimistic Total': optimisticTotal !== undefined ? Number(optimisticTotal.toFixed(5)) : '—',
2253
+ 'Context logSumExp': normalizer !== undefined && Number.isFinite(normalizer) ? Number(normalizer.toFixed(5)) : '—',
2254
+ 'Shortlist Size': shortlistSize,
2255
+ 'Posterior (bound)': Number(optimisticPosterior(r).toFixed(5)),
2256
+ 'Posterior (judged)': hasJudgeableEvidence(r) ? Number(judgedPosterior(r).toFixed(5)) : '—',
2257
+ 'Context Leader': contextLeader ? "".concat(contextLeader.termType, ":").concat(contextLeader.word) : '—',
2258
+ 'Verified Chars': r.lmEvidence ? "".concat(r.lmEvidence.verifiedCharCount, "/").concat(r.lmEvidence.totalSurfaceCharCount) : "0/".concat(r.word.length),
2259
+ Separator: (_runtime$separatorKin2 = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin2 !== void 0 ? _runtime$separatorKin2 : 'unknown',
2260
+ 'Canonical Eligibility': (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace' ? 'incompatible-separator' : runtime.canonicalTokenIds === null ? 'missing-artifact' : 'eligible',
2261
+ 'Artifact Tokens': (_runtime$canonicalTok2 = runtime === null || runtime === void 0 || (_runtime$canonicalTok3 = runtime.canonicalTokenIds) === null || _runtime$canonicalTok3 === void 0 ? void 0 : _runtime$canonicalTok3.join(',')) !== null && _runtime$canonicalTok2 !== void 0 ? _runtime$canonicalTok2 : 'absent',
2262
+ 'Context Key': (_runtime$contextKey = runtime === null || runtime === void 0 ? void 0 : runtime.contextKey) !== null && _runtime$contextKey !== void 0 ? _runtime$contextKey : '—',
2263
+ 'Exact Total': ((_r$lmEvidence3 = r.lmEvidence) === null || _r$lmEvidence3 === void 0 ? void 0 : _r$lmEvidence3.totalLogProbability) !== undefined ? Number(r.lmEvidence.totalLogProbability.toFixed(5)) : '—',
2264
+ 'Exact Mean': ((_r$lmEvidence4 = r.lmEvidence) === null || _r$lmEvidence4 === void 0 ? void 0 : _r$lmEvidence4.meanTokenLogProbability) !== undefined ? Number(r.lmEvidence.meanTokenLogProbability.toFixed(5)) : '—',
2265
+ Assoc: typeof (original === null || original === void 0 ? void 0 : original.association) === 'number' ? Number(original.association.toFixed(4)) : '—',
669
2266
  'Raw Logit': rawLogit,
670
2267
  'Session Freq': (original === null || original === void 0 ? void 0 : original.sessionFreq) || 0
671
2268
  };
@@ -673,23 +2270,120 @@ export var predict = function predict(textBefore) {
673
2270
 
674
2271
  // eslint-disable-next-line no-console
675
2272
  console.table(tableData);
676
- } else {
677
- // eslint-disable-next-line no-console
678
- console.log('No candidates found.');
679
2273
  }
680
2274
 
681
2275
  // eslint-disable-next-line no-console
682
2276
  console.groupEnd();
683
2277
  }
684
- return suggestion && suggestion.length > 0 ? suggestion : null;
2278
+ var selectedContextKey = selected ? (_runtimeBySurface$get5 = runtimeBySurface.get(selected.word)) === null || _runtimeBySurface$get5 === void 0 ? void 0 : _runtimeBySurface$get5.contextKey : undefined;
2279
+ return suggestion && suggestion.length > 0 && selected && selectedEvidence ? {
2280
+ text: suggestion,
2281
+ surface: selected.word,
2282
+ termType: selected.termType,
2283
+ posterior: judgedPosterior(selected),
2284
+ meanTokenLogProbability: (_judgedEvidenceFor$me3 = (_judgedEvidenceFor6 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor6 === void 0 ? void 0 : _judgedEvidenceFor6.mean) !== null && _judgedEvidenceFor$me3 !== void 0 ? _judgedEvidenceFor$me3 : NaN,
2285
+ poolHeldExtension: poolExtensionOf(selected) !== undefined,
2286
+ shortlistSize: selectedContextKey === undefined ? 1 : (_contextTotals$get$le4 = (_contextTotals$get4 = contextTotals.get(selectedContextKey)) === null || _contextTotals$get4 === void 0 ? void 0 : _contextTotals$get4.length) !== null && _contextTotals$get$le4 !== void 0 ? _contextTotals$get$le4 : 1,
2287
+ rankScore: selectedDisplayScore,
2288
+ evidenceTier: (selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.source) === 'network-logit' ? 'network-logit' : 'canonical-full-surface',
2289
+ evidenceDepth: {
2290
+ verifiedTokens: (_selectedEvidence$ver = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedTokenCount) !== null && _selectedEvidence$ver !== void 0 ? _selectedEvidence$ver : 0,
2291
+ totalTokens: (_selectedEvidence$tot = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.totalTokenCount) !== null && _selectedEvidence$tot !== void 0 ? _selectedEvidence$tot : 0,
2292
+ verifiedChars: (_selectedEvidence$ver2 = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedCharCount) !== null && _selectedEvidence$ver2 !== void 0 ? _selectedEvidence$ver2 : 0,
2293
+ totalChars: selected.word.length
2294
+ },
2295
+ winnerMargin: winnerMargin
2296
+ } : null;
685
2297
  };
686
2298
 
687
2299
  // ─── Data Loading ────────────────────────────────────────────────────────────
688
2300
 
2301
+ var isVocabularyJson = function isVocabularyJson(payload) {
2302
+ if (payload == null || _typeof(payload) !== 'object') {
2303
+ return false;
2304
+ }
2305
+ var words = payload.words;
2306
+ return words != null && _typeof(words) === 'object';
2307
+ };
2308
+ var isStringArray = function isStringArray(payload) {
2309
+ return Array.isArray(payload) && payload.every(function (entry) {
2310
+ return typeof entry === 'string';
2311
+ });
2312
+ };
2313
+ var isWordToOffsetMap = function isWordToOffsetMap(value) {
2314
+ return value != null && _typeof(value) === 'object' && Object.values(value).every(function (entry) {
2315
+ return typeof entry === 'number';
2316
+ });
2317
+ };
2318
+ var isWordIndexPayload = function isWordIndexPayload(payload) {
2319
+ if (payload == null || _typeof(payload) !== 'object') {
2320
+ return false;
2321
+ }
2322
+ var index = payload.index;
2323
+ return index === undefined ? isWordToOffsetMap(payload) : isWordToOffsetMap(index);
2324
+ };
2325
+
2326
+ /**
2327
+ * The word index has shipped both bare and wrapped in `{ index: ... }`, so
2328
+ * accept either rather than silently building an empty vector store.
2329
+ */
2330
+ var unwrapWordIndex = function unwrapWordIndex(payload) {
2331
+ var wrapped = payload.index;
2332
+ return wrapped !== null && wrapped !== void 0 ? wrapped : payload;
2333
+ };
2334
+
2335
+ /**
2336
+ * Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
2337
+ * confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
2338
+ * + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
2339
+ * are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
2340
+ * a number.
2341
+ */
2342
+
2343
+ /** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
2344
+
2345
+ var isPhraseStats = function isPhraseStats(value) {
2346
+ return value != null && _typeof(value) === 'object' && typeof value.freq === 'number';
2347
+ };
2348
+ var isTermStatsMap = function isTermStatsMap(value) {
2349
+ if (value == null || _typeof(value) !== 'object') {
2350
+ return false;
2351
+ }
2352
+ var values = Object.values(value);
2353
+ return values.length > 0 && values.every(isPhraseStats);
2354
+ };
2355
+
2356
+ /**
2357
+ * Normalize a `bigrams.json` / `phrases.json` payload into a flat
2358
+ * `{term: stats}` map. Tolerant of the exact wire shape (not yet finalized by
2359
+ * the producer): accepts either the flat map from spec §9 or a single wrapper
2360
+ * key (`terms`/`bigrams`/`phrases`/`words`). Returns `null` if unrecognized.
2361
+ *
2362
+ * :params:
2363
+ * payload: Raw JSON parsed from the artifact endpoint
2364
+ * :returns:
2365
+ * A `{term: stats}` map, or `null` when the shape is not a term-stats map
2366
+ */
2367
+ var normalizePhraseArtifact = function normalizePhraseArtifact(payload) {
2368
+ if (payload == null || _typeof(payload) !== 'object') {
2369
+ return null;
2370
+ }
2371
+ if (isTermStatsMap(payload)) {
2372
+ return payload;
2373
+ }
2374
+ var obj = payload;
2375
+ for (var _i4 = 0, _arr = ['terms', 'bigrams', 'phrases', 'words']; _i4 < _arr.length; _i4++) {
2376
+ var key = _arr[_i4];
2377
+ if (isTermStatsMap(obj[key])) {
2378
+ return obj[key];
2379
+ }
2380
+ }
2381
+ return null;
2382
+ };
689
2383
  export var loadVectorsAsync = /*#__PURE__*/function () {
690
- var _ref6 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
2384
+ var _ref14 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
691
2385
  var _options$isLocalLLM;
692
- var isLocalLLM, surface, url, res, buffer, float32, wordIndexPayload, wrappedIndex, wordIndex, nWords, dim, _t, _t2;
2386
+ var isLocalLLM, surface, buffer, float32, wordIndexPayload, wordIndex, nWords, dim, _t;
693
2387
  return _regeneratorRuntime.wrap(function (_context) {
694
2388
  while (1) switch (_context.prev = _context.next) {
695
2389
  case 0:
@@ -709,65 +2403,23 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
709
2403
  } : {}));
710
2404
  _context.prev = 2;
711
2405
  _context.next = 3;
712
- return getArtifactUrl(ARTIFACT_NAME.WORD_VECTORS);
2406
+ return fetchAutocompleteArtifactBinary(ARTIFACT_NAME.WORD_VECTORS, 'word_vectors_10k');
713
2407
  case 3:
714
- url = _context.sent;
715
- _context.next = 5;
716
- break;
717
- case 4:
718
- _context.prev = 4;
719
- _t = _context["catch"](2);
720
- vectorsLoadStarted = false;
721
- failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
722
- isLocalLLM: isLocalLLM,
723
- errorType: 'resolve_url'
724
- }, surface ? {
725
- surface: surface
726
- } : {}));
727
- // eslint-disable-next-line no-console
728
- console.warn('[text-predictor] Failed to resolve vectors URL:', _t);
729
- return _context.abrupt("return");
730
- case 5:
731
- _context.prev = 5;
732
- _context.next = 6;
733
- return fetch(url);
734
- case 6:
735
- res = _context.sent;
736
- if (res.ok) {
737
- _context.next = 7;
738
- break;
739
- }
740
- vectorsLoadStarted = false;
741
- failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
742
- isLocalLLM: isLocalLLM,
743
- status: res.status,
744
- errorType: 'http_error'
745
- }, surface ? {
746
- surface: surface
747
- } : {}));
748
- // eslint-disable-next-line no-console
749
- console.warn("[text-predictor] Failed to load vectors: ".concat(res.status));
750
- return _context.abrupt("return");
751
- case 7:
752
- _context.next = 8;
753
- return res.arrayBuffer();
754
- case 8:
755
2408
  buffer = _context.sent;
756
- float32 = new Float32Array(buffer); // The word index is published either bare or wrapped as `{ "index": {…} }`
757
- // (the wrapper exists so no real entry, e.g. the word "default", can shadow
758
- // a bundler-synthesised ESM export). Detect the wrapper by value type, not
759
- // by key presence — "index" is itself a plausible vocabulary word, and in a
760
- // bare index every value is a number.
761
- _context.next = 9;
762
- return fetchArtifactJson(ARTIFACT_NAME.WORD_INDEX);
763
- case 9:
2409
+ float32 = new Float32Array(buffer);
2410
+ _context.next = 4;
2411
+ return fetchAutocompleteArtifactJson(ARTIFACT_NAME.WORD_INDEX, {
2412
+ summarize: function summarize(payload) {
2413
+ return "".concat(Object.keys(unwrapWordIndex(payload)).length, " entries");
2414
+ },
2415
+ validate: isWordIndexPayload
2416
+ });
2417
+ case 4:
764
2418
  wordIndexPayload = _context.sent;
765
- wrappedIndex = wordIndexPayload === null || wordIndexPayload === void 0 ? void 0 : wordIndexPayload.index;
766
- wordIndex = _typeof(wrappedIndex) === 'object' && wrappedIndex !== null ? wrappedIndex : wordIndexPayload !== null && wordIndexPayload !== void 0 ? wordIndexPayload : {};
2419
+ wordIndex = unwrapWordIndex(wordIndexPayload);
767
2420
  nWords = Object.keys(wordIndex).length;
768
2421
  if (nWords === 0) {
769
- // eslint-disable-next-line no-console
770
- console.warn("[text-predictor] ".concat(ARTIFACT_NAME.WORD_INDEX, " was empty \u2014 semantic scoring will be a no-op."));
2422
+ ctcTag('init', '⚠️ word_index_10k.json was empty — semantic scoring will be a no-op', CTC_STYLES.warn);
771
2423
  }
772
2424
  dim = float32.length / nWords;
773
2425
  vectorStore = {
@@ -783,19 +2435,12 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
783
2435
  }, surface ? {
784
2436
  surface: surface
785
2437
  } : {}));
786
- if (isAutocompleteDebugEnabled()) {
787
- // eslint-disable-next-line no-console
788
- console.log('[text-predictor] Vectors loaded:', {
789
- wordCount: nWords,
790
- dim: dim,
791
- sizeBytes: float32.byteLength
792
- });
793
- }
794
- _context.next = 11;
2438
+ ctcTag('init', "vectors loaded: ".concat(nWords, " words \xB7 dim ").concat(dim, " \xB7 ").concat(float32.byteLength, "B"));
2439
+ _context.next = 6;
795
2440
  break;
796
- case 10:
797
- _context.prev = 10;
798
- _t2 = _context["catch"](5);
2441
+ case 5:
2442
+ _context.prev = 5;
2443
+ _t = _context["catch"](2);
799
2444
  vectorsLoadStarted = false;
800
2445
  failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
801
2446
  isLocalLLM: isLocalLLM,
@@ -803,60 +2448,197 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
803
2448
  }, surface ? {
804
2449
  surface: surface
805
2450
  } : {}));
806
- // eslint-disable-next-line no-console
807
- console.warn('[text-predictor] Failed to load vectors:', _t2);
808
- case 11:
2451
+ ctcTag('init', "\u26A0\uFE0F failed to load vectors: ".concat(String(_t)), CTC_STYLES.bad);
2452
+ case 6:
809
2453
  case "end":
810
2454
  return _context.stop();
811
2455
  }
812
- }, _callee, null, [[2, 4], [5, 10]]);
2456
+ }, _callee, null, [[2, 5]]);
813
2457
  }));
814
2458
  return function loadVectorsAsync(_x) {
815
- return _ref6.apply(this, arguments);
2459
+ return _ref14.apply(this, arguments);
816
2460
  };
817
2461
  }();
818
2462
  export var initVectors = function initVectors(store) {
819
2463
  vectorStore = store;
820
2464
  };
2465
+
2466
+ /**
2467
+ * Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
2468
+ * over the TDP-OS gateway and insert them into the phrase trie.
2469
+ *
2470
+ * Each file is fetched independently (`Promise.allSettled`) so a missing or
2471
+ * late-published artifact only skips that term type — word completion and the
2472
+ * other term type are unaffected. Phrase VECTORS are not fetched here: per the
2473
+ * producer's `extend` decision they are appended into the existing
2474
+ * `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
2475
+ * covers them.
2476
+ *
2477
+ * :params:
2478
+ * options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
2479
+ * :returns:
2480
+ * A promise that resolves once both fetches have settled
2481
+ */
2482
+ export var loadPhraseArtifacts = /*#__PURE__*/function () {
2483
+ var _ref15 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee3(options) {
2484
+ var _options$isLocalLLM2;
2485
+ var isLocalLLM, loadOne, _yield$Promise$allSet, _yield$Promise$allSet2, bigramsResult, phrasesResult, bigramCount, phraseCount;
2486
+ return _regeneratorRuntime.wrap(function (_context3) {
2487
+ while (1) switch (_context3.prev = _context3.next) {
2488
+ case 0:
2489
+ if (!phrasesLoadStarted) {
2490
+ _context3.next = 1;
2491
+ break;
2492
+ }
2493
+ return _context3.abrupt("return");
2494
+ case 1:
2495
+ phrasesLoadStarted = true;
2496
+ isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2497
+ startExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2498
+ isLocalLLM: isLocalLLM
2499
+ });
2500
+ loadOne = /*#__PURE__*/function () {
2501
+ var _ref16 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee2(artifactName, termType, label) {
2502
+ var payload, normalized;
2503
+ return _regeneratorRuntime.wrap(function (_context2) {
2504
+ while (1) switch (_context2.prev = _context2.next) {
2505
+ case 0:
2506
+ _context2.next = 1;
2507
+ return fetchAutocompleteArtifactJson(artifactName, {
2508
+ label: label,
2509
+ validate: function validate(p) {
2510
+ return p != null && _typeof(p) === 'object';
2511
+ }
2512
+ });
2513
+ case 1:
2514
+ payload = _context2.sent;
2515
+ normalized = normalizePhraseArtifact(payload);
2516
+ if (normalized) {
2517
+ _context2.next = 2;
2518
+ break;
2519
+ }
2520
+ throw new Error("[text-predictor] ".concat(label, " payload was not a recognised term-stats map"));
2521
+ case 2:
2522
+ return _context2.abrupt("return", initPhrases(normalized, termType));
2523
+ case 3:
2524
+ case "end":
2525
+ return _context2.stop();
2526
+ }
2527
+ }, _callee2);
2528
+ }));
2529
+ return function loadOne(_x3, _x4, _x5) {
2530
+ return _ref16.apply(this, arguments);
2531
+ };
2532
+ }();
2533
+ _context3.next = 2;
2534
+ return Promise.allSettled([loadOne(ARTIFACT_NAME.BIGRAMS, 'bigram', 'bigrams'), loadOne(ARTIFACT_NAME.PHRASES, 'phrase', 'phrases')]);
2535
+ case 2:
2536
+ _yield$Promise$allSet = _context3.sent;
2537
+ _yield$Promise$allSet2 = _slicedToArray(_yield$Promise$allSet, 2);
2538
+ bigramsResult = _yield$Promise$allSet2[0];
2539
+ phrasesResult = _yield$Promise$allSet2[1];
2540
+ bigramCount = bigramsResult.status === 'fulfilled' ? bigramsResult.value : 0;
2541
+ phraseCount = phrasesResult.status === 'fulfilled' ? phrasesResult.value : 0;
2542
+ if (bigramsResult.status === 'rejected') {
2543
+ ctcTag('init', "\u26A0\uFE0F failed to load bigrams.json: ".concat(String(bigramsResult.reason)), CTC_STYLES.warn);
2544
+ }
2545
+ if (phrasesResult.status === 'rejected') {
2546
+ ctcTag('init', "\u26A0\uFE0F failed to load phrases.json: ".concat(String(phrasesResult.reason)), CTC_STYLES.warn);
2547
+ }
2548
+ if (bigramsResult.status === 'fulfilled' || phrasesResult.status === 'fulfilled') {
2549
+ phrasesLoaded = true;
2550
+ succeedExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2551
+ isLocalLLM: isLocalLLM,
2552
+ bigramCount: bigramCount,
2553
+ phraseCount: phraseCount
2554
+ });
2555
+ ctcTag('init', "phrase artifacts loaded: ".concat(bigramCount, " bigrams \xB7 ").concat(phraseCount, " phrases"));
2556
+ } else {
2557
+ // Both failed — e.g. artifacts not published to the tenant manifest yet.
2558
+ // Reset the guard so a later focus/predict can retry once they land.
2559
+ phrasesLoadStarted = false;
2560
+ failExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2561
+ isLocalLLM: isLocalLLM,
2562
+ errorType: 'network'
2563
+ });
2564
+ }
2565
+ case 3:
2566
+ case "end":
2567
+ return _context3.stop();
2568
+ }
2569
+ }, _callee3);
2570
+ }));
2571
+ return function loadPhraseArtifacts(_x2) {
2572
+ return _ref15.apply(this, arguments);
2573
+ };
2574
+ }();
821
2575
  var vocabularyLoadPromise;
822
2576
  export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
823
- var _options$isLocalLLM2;
2577
+ var _options$isLocalLLM3, _options$source;
824
2578
  if (isInitialized) {
825
2579
  return Promise.resolve();
826
2580
  }
827
2581
  if (vocabularyLoadPromise) {
828
2582
  return vocabularyLoadPromise;
829
2583
  }
830
- var isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2584
+ var isLocalLLM = (_options$isLocalLLM3 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM3 !== void 0 ? _options$isLocalLLM3 : false;
2585
+ var source = (_options$source = options === null || options === void 0 ? void 0 : options.source) !== null && _options$source !== void 0 ? _options$source : 'predict';
831
2586
  var surface = options === null || options === void 0 ? void 0 : options.surface;
832
- vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee2() {
833
- var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t3;
834
- return _regeneratorRuntime.wrap(function (_context2) {
835
- while (1) switch (_context2.prev = _context2.next) {
2587
+ vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee4() {
2588
+ var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t2;
2589
+ return _regeneratorRuntime.wrap(function (_context4) {
2590
+ while (1) switch (_context4.prev = _context4.next) {
836
2591
  case 0:
837
2592
  startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
838
2593
  isLocalLLM: isLocalLLM
839
2594
  }, surface ? {
840
2595
  surface: surface
841
2596
  } : {}));
842
- _context2.prev = 1;
843
- _context2.next = 2;
844
- return Promise.all([fetchArtifactJson(ARTIFACT_NAME.VOCABULARY), fetchArtifactJson(ARTIFACT_NAME.L3_VOCABULARY)]);
2597
+ _context4.prev = 1;
2598
+ ctcTag('init', "loading artifacts (trigger: ".concat(source, " \xB7 localLLM: ").concat(isLocalLLM, ")"));
2599
+ if (isAutocompleteDebugVerbose()) {
2600
+ // eslint-disable-next-line no-console
2601
+ console.groupCollapsed('%c[CTC:init]%c artifact manifest', CTC_STYLES.brand, CTC_STYLES.body);
2602
+ // eslint-disable-next-line no-console
2603
+ console.log('Eager:', [ARTIFACT_NAME.VOCABULARY, ARTIFACT_NAME.L3_VOCABULARY, ARTIFACT_NAME.POS_TAGS, ARTIFACT_NAME.GHOST_POS_TAGS, ARTIFACT_NAME.GRAMMAR_TRANSITIONS]);
2604
+ // eslint-disable-next-line no-console
2605
+ console.log('Conditional:', ["".concat(ARTIFACT_NAME.WORD_INDEX, " (only when vectors load; phrase/bigram surfaces share this index)"), "".concat(ARTIFACT_NAME.BIGRAMS, " + ").concat(ARTIFACT_NAME.PHRASES, " (completion-term stats; fire-and-forget)"), "".concat(ARTIFACT_NAME.PHRASE_CONTINUATION_TOKENS, " (local model only; canonical token IDs for the full served union)")]);
2606
+ // eslint-disable-next-line no-console
2607
+ console.groupEnd();
2608
+ }
2609
+ _context4.next = 2;
2610
+ return Promise.all([fetchAutocompleteArtifactJson(ARTIFACT_NAME.VOCABULARY, {
2611
+ summarize: function summarize(payload) {
2612
+ return "".concat(Object.keys(payload.words).length, " words");
2613
+ },
2614
+ validate: isVocabularyJson
2615
+ }), fetchAutocompleteArtifactJson(ARTIFACT_NAME.L3_VOCABULARY, {
2616
+ summarize: function summarize(payload) {
2617
+ return "".concat(payload.length, " words");
2618
+ },
2619
+ validate: isStringArray
2620
+ }),
2621
+ // Awaited alongside the vocabulary so the grammar filter is settled
2622
+ // before the first suggestion can be produced. Without this the
2623
+ // predictor initialises on the smaller vocabulary payload and serves
2624
+ // candidates the filter would have dropped until the grammar lands.
2625
+ // Only settlement is required: a failed load leaves the filter as a
2626
+ // pass-through, which is the same degradation as an absent artifact.
2627
+ loadGrammarDataAsync({
2628
+ isLocalLLM: isLocalLLM,
2629
+ surface: surface
2630
+ }).catch(function (error) {
2631
+ ctcTag('init', "\u26A0\uFE0F grammar data unavailable; filtering skipped: ".concat(String(error)), CTC_STYLES.warn);
2632
+ })]);
845
2633
  case 2:
846
- _yield$Promise$all = _context2.sent;
2634
+ _yield$Promise$all = _context4.sent;
847
2635
  _yield$Promise$all2 = _slicedToArray(_yield$Promise$all, 2);
848
2636
  vocabularyData = _yield$Promise$all2[0];
849
2637
  l3VocabularyData = _yield$Promise$all2[1];
850
- if (!((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData))) {
851
- _context2.next = 3;
852
- break;
853
- }
854
- throw new Error("[text-predictor] unexpected vocabulary payload \u2014 " + "".concat(ARTIFACT_NAME.VOCABULARY, " ").concat((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null ? 'is missing its `words` wrapper' : 'is valid', ", ") + "".concat(ARTIFACT_NAME.L3_VOCABULARY, " ").concat(Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'));
855
- case 3:
856
- terms = Object.entries(vocabularyData.words).map(function (_ref8) {
857
- var _ref9 = _slicedToArray(_ref8, 2),
858
- word = _ref9[0],
859
- stats = _ref9[1];
2638
+ terms = Object.entries(vocabularyData.words).map(function (_ref18) {
2639
+ var _ref19 = _slicedToArray(_ref18, 2),
2640
+ word = _ref19[0],
2641
+ stats = _ref19[1];
860
2642
  return {
861
2643
  word: word,
862
2644
  freq: stats.freq,
@@ -870,6 +2652,12 @@ export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
870
2652
  initVocabulary({
871
2653
  terms: terms
872
2654
  });
2655
+
2656
+ // Phrase/bigram artifacts load independently and must never affect the
2657
+ // word path — fire-and-forget with its own error handling inside.
2658
+ void loadPhraseArtifacts({
2659
+ isLocalLLM: isLocalLLM
2660
+ }).catch(function () {});
873
2661
  succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
874
2662
  isLocalLLM: isLocalLLM,
875
2663
  l2WordCount: terms.length,
@@ -877,25 +2665,25 @@ export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
877
2665
  }, surface ? {
878
2666
  surface: surface
879
2667
  } : {}));
880
- _context2.next = 5;
2668
+ _context4.next = 4;
881
2669
  break;
882
- case 4:
883
- _context2.prev = 4;
884
- _t3 = _context2["catch"](1);
2670
+ case 3:
2671
+ _context4.prev = 3;
2672
+ _t2 = _context4["catch"](1);
885
2673
  failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
886
2674
  isLocalLLM: isLocalLLM,
887
- errorType: 'load_error'
2675
+ errorType: 'parse_error'
888
2676
  }, surface ? {
889
2677
  surface: surface
890
2678
  } : {}));
891
2679
  // Allow a later call to retry the load rather than caching the failure.
892
2680
  vocabularyLoadPromise = undefined;
893
- throw _t3;
894
- case 5:
2681
+ throw _t2;
2682
+ case 4:
895
2683
  case "end":
896
- return _context2.stop();
2684
+ return _context4.stop();
897
2685
  }
898
- }, _callee2, null, [[1, 4]]);
2686
+ }, _callee4, null, [[1, 3]]);
899
2687
  }))();
900
2688
  return vocabularyLoadPromise;
901
2689
  };