@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +80 -0
  2. package/dist/cjs/analytics/ufo.js +2 -1
  3. package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
  4. package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
  5. package/dist/cjs/pm-plugins/autocomplete-plugin.js +416 -129
  6. package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
  7. package/dist/cjs/pm-plugins/debug-mode.js +90 -7
  8. package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
  9. package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
  10. package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
  11. package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
  12. package/dist/cjs/pm-plugins/text-predictor.js +1951 -273
  13. package/dist/es2019/analytics/ufo.js +2 -0
  14. package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
  15. package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
  16. package/dist/es2019/pm-plugins/autocomplete-plugin.js +425 -130
  17. package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
  18. package/dist/es2019/pm-plugins/debug-mode.js +87 -7
  19. package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
  20. package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
  21. package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
  22. package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
  23. package/dist/es2019/pm-plugins/text-predictor.js +1603 -199
  24. package/dist/esm/analytics/ufo.js +2 -1
  25. package/dist/esm/pm-plugins/artifact-loader.js +128 -0
  26. package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
  27. package/dist/esm/pm-plugins/autocomplete-plugin.js +419 -132
  28. package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
  29. package/dist/esm/pm-plugins/debug-mode.js +89 -6
  30. package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
  31. package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
  32. package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
  33. package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
  34. package/dist/esm/pm-plugins/text-predictor.js +1953 -277
  35. package/dist/types/analytics/ufo.d.ts +1 -0
  36. package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
  37. package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
  38. package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
  39. package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
  40. package/dist/types/pm-plugins/debug-mode.d.ts +36 -1
  41. package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
  42. package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
  43. package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
  44. package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
  45. package/dist/types/pm-plugins/text-predictor.d.ts +120 -2
  46. package/package.json +2 -2
  47. package/src/analytics/ufo.ts +5 -0
  48. package/src/pm-plugins/artifact-loader.ts +92 -0
  49. package/src/pm-plugins/artifacts-manifest.ts +3 -0
  50. package/src/pm-plugins/autocomplete-plugin.ts +517 -133
  51. package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
  52. package/src/pm-plugins/debug-mode.ts +109 -7
  53. package/src/pm-plugins/ghost-text-decoration.ts +13 -4
  54. package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
  55. package/src/pm-plugins/scoring-pipeline.ts +167 -17
  56. package/src/pm-plugins/slow-lane-client.ts +147 -62
  57. package/src/pm-plugins/text-predictor.ts +2006 -255
  58. package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
  59. package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
  60. package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
  61. package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
  62. package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
  63. package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
  64. package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  65. package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  66. package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
  67. package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  68. package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  69. package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
  70. package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
  71. package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
  72. package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
  73. package/scripts/gen_first_token_to_words.py +0 -170
  74. package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
  75. package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
  76. package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
  77. package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
  78. package/src/pm-plugins/slow-lane-client/package.json +0 -8
  79. package/src/pm-plugins/text-predictor/package.json +0 -8
@@ -1,5 +1,6 @@
1
- import _typeof from "@babel/runtime/helpers/typeof";
2
1
  import _asyncToGenerator from "@babel/runtime/helpers/asyncToGenerator";
2
+ import _typeof from "@babel/runtime/helpers/typeof";
3
+ import _toConsumableArray from "@babel/runtime/helpers/toConsumableArray";
3
4
  import _slicedToArray from "@babel/runtime/helpers/slicedToArray";
4
5
  import _createClass from "@babel/runtime/helpers/createClass";
5
6
  import _classCallCheck from "@babel/runtime/helpers/classCallCheck";
@@ -15,7 +16,7 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
15
16
  *
16
17
  * Two prediction modes:
17
18
  * 1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
18
- * 2. Mid-word (≥3 chars) → trie prefix search → scoring pipeline → top result
19
+ * 2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
19
20
  *
20
21
  * Scoring is delegated to scoring-pipeline.ts which handles:
21
22
  * Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
@@ -28,14 +29,12 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
28
29
  */
29
30
 
30
31
  import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
31
-
32
- // The vocabulary, L3 and word-index payloads are fetched from the CDN in
33
- // loadDefaultVocabulary / loadVectorsAsync so their (large) contents never enter
34
- // the editor's bundle and only load when autocomplete is initialised.
35
- import { ARTIFACT_NAME, fetchArtifactJson, getArtifactUrl } from './artifacts-manifest';
36
- import { isAutocompleteDebugEnabled } from './debug-mode';
37
- import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
38
- import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
32
+ import { fetchAutocompleteArtifactBinary, fetchAutocompleteArtifactJson } from './artifact-loader';
33
+ import { ARTIFACT_NAME } from './artifacts-manifest';
34
+ import { createCanonicalContextPositionCache, deriveCanonicalCandidateContext, deriveWhitespaceBoundaryContext, logSoftmaxAt, logSumExp, selectBoundaryPrimeRequests } from './canonical-lm-scoring';
35
+ import { CTC_STYLES, ctcSection, ctcTag, isAutocompleteDebugEnabled, isAutocompleteDebugVerbose } from './debug-mode';
36
+ import { loadGrammarDataAsync, rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE, MIN_WINNER_MARGIN } from './scoring-pipeline';
37
+ import { getBoundaryLmState, getCanonicalSurfaceCount, getCanonicalSurfaceTokenIds, getDefaultSlowLaneClientStatus, getProgressiveSurfaceEvidence, getStoredContextInput, getStoredContextVector, getStoredLmLogits, getSurfaceScore, isCanonicalSurfaceScoringSupported, primeBoundaryLm, requestProgressiveSurfaceScores } from './slow-lane-client';
39
38
 
40
39
  // ─── Constants ───────────────────────────────────────────────────────────────
41
40
 
@@ -47,13 +46,152 @@ var WHITESPACE_SPLIT_REGEX = /\s+/;
47
46
  var SENTENCE_BOUNDARY_REGEX = /[\n.?!]+/;
48
47
  // eslint-disable-next-line require-unicode-regexp
49
48
  var TRAILING_WHITESPACE_REGEX = /\s$/;
50
- var MIN_PREFIX_LENGTH = 3;
49
+ var TRAILING_SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+$/;
50
+ var SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+/g;
51
+ var ONLY_WHITESPACE_REGEX = /^[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+$/;
52
+ var UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH = 1;
53
+ var MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH = 1;
54
+ var DISPLAY_MIN_PREFIX_LENGTH = 3;
51
55
  var MAX_CANDIDATES = 200;
52
56
  var CONTEXT_WORDS = 10;
53
- var MIN_SCORE_THRESHOLD = 0.35;
54
57
  var L3_BASELINE_FREQ = 0.001;
58
+ // Max tokens in a completion term (bigram=2, phrase=3). Mirrors the producer's
59
+ // `phrase_max_words` default so the FE window never exceeds what was mined.
60
+ var PHRASE_MAX_WORDS = 3;
61
+ // Per-window cap on phrase-trie subtree collection (keeps the hot path bounded).
62
+ var MAX_PHRASE_CANDIDATES = 50;
63
+ /**
64
+ * Share of its context shortlist's probability mass the model must put on a
65
+ * surface before that surface may be shown.
66
+ *
67
+ * Candidates competing at the same boundary are normalised into a posterior
68
+ * over that shortlist, so this reads directly as "the model is at least this
69
+ * sure". Measuring distance from the best candidate instead cannot express
70
+ * being unsure: the leader scores full marks whether it won by a nose or a
71
+ * mile, which pushes the entire question of how contested a context is onto
72
+ * `MIN_WINNER_MARGIN` — and there the blended score's only remaining spread is
73
+ * the corpus prior, the weakest ranking signal we have.
74
+ *
75
+ * Each extra token raises the bar. A wrong multi-word ghost costs the reader
76
+ * more to notice and undo than a wrong single word, and it is offered from the
77
+ * same keystroke, so precision has to be bought per token of commitment.
78
+ */
79
+ var MIN_LM_POSTERIOR = {
80
+ word: 0.5,
81
+ bigram: 0.6,
82
+ phrase: 0.7
83
+ };
84
+ /**
85
+ * How many scored candidates a context must hold before its leader may show.
86
+ *
87
+ * A posterior is a share of a shortlist, so a shortlist of one hands its only
88
+ * member 1.0 whatever the model thinks of it — `logSumExp` of a single value
89
+ * returns that value. Such a ghost sits in the top bucket without having beaten
90
+ * anything, and it is the gate's blind spot rather than a case the gate ruled
91
+ * on: 1 in 14 of them were accepted against 34% for word ghosts overall, and
92
+ * every phrase shown through this hole was rejected.
93
+ *
94
+ * Counted over pool membership rather than over the normaliser, because those
95
+ * two deliberately differ. Where a pool holds a surface and its own extension
96
+ * the chain rule leaves one term in the denominator, and the resulting 1.0 is
97
+ * earned — whichever the user meant, the shorter form is a correct ghost.
98
+ *
99
+ * Waived where the context never held a second candidate at all. The 1-in-14
100
+ * acceptance above was measured over a population dominated by pools that read
101
+ * as one because rivals had not come back, and it is that absence of evidence
102
+ * the floor is for. A vocabulary offering a single continuation of a long
103
+ * distinctive prefix is the opposite situation, and it is also where a ghost
104
+ * saves the most keystrokes, so refusing it spends the most to learn the least.
105
+ */
106
+ var MIN_SCORED_POOL_SIZE = 2;
107
+ /**
108
+ * Mean per-token log-probability a surface must hold, in nats, independently of
109
+ * anything it was competing against.
110
+ *
111
+ * Every other display test is relative: a posterior is a share of a shortlist,
112
+ * and a winner margin is a distance from a rival. So a surface reaches the
113
+ * screen by being the best of what happened to be in its pool, and nothing ever
114
+ * asks whether the model finds it plausible at all. That gap was tolerable while
115
+ * little was being read, and stopped being tolerable once the scheduler started
116
+ * reading roughly twice as much per decision: shown-per-decision went from 3.2%
117
+ * to 7.4% with no threshold moved, and acceptance of what showed fell from 32%
118
+ * to 24.5%. More reading does not weaken a relative test — it hands it far more
119
+ * pools to be the winner of, and the marginal winner is worse than the average.
120
+ *
121
+ * Set at the point where a uniform distribution over the model's vocabulary
122
+ * sits: 49,152 tokens, so `ln(1/49152)` is about -10.8 nats. Below it the model
123
+ * assigns the surface less mass than a token drawn at random, which is the only
124
+ * statement about a surface that needs no reference to what it competed with.
125
+ *
126
+ * It was first tried at -8, chosen from the bucket where acceptance was measured
127
+ * to fall away, and that reasoning does not survive more data. Acceptance against
128
+ * this quantity turned out to be a smooth slope — about 19% below -8, 26% from -8
129
+ * to -6, then flat near 35% above - with no discontinuity to place a threshold
130
+ * at. A floor on a smooth slope is a coverage dial rather than a test: -8 removed
131
+ * a band converting at 26% and cost 3.7 points of show rate, for an acceptance
132
+ * movement no sample of this size can resolve. Anchoring to uniform instead makes
133
+ * the value an argument about what is indefensible rather than a point picked off
134
+ * a histogram, which is what stops it from being retuned every batch.
135
+ *
136
+ * Applied to the mean over the verified prefix, the same quantity
137
+ * `ghostsByMeanLogProb` buckets, and deliberately not to
138
+ * `meanTokenLogProbabilityUpperBound`. The bound is the right instrument for
139
+ * keeping a candidate in the race, where dropping something that could still
140
+ * clear the bar would be an error, but it is systematically higher than the
141
+ * quantity acceptance is measured against. Judging a surface on the prefix that
142
+ * was actually read is also what every other test here does.
143
+ *
144
+ * One value across all three shapes. Splitting it per shape needs an acceptance
145
+ * curve per shape, and 19 bigrams and 1 phrase across five sessions cannot
146
+ * support one.
147
+ */
148
+ var MIN_MEAN_TOKEN_LOG_PROBABILITY = -10.8;
149
+ // Tab replaces the whole remaining suffix, so an N-character completion saves
150
+ // N-1 keystrokes: at one character it saves nothing at all and at two it saves
151
+ // one, which does not repay noticing the ghost and reaching for Tab. Offering
152
+ // them anyway spends the acceptance-rate denominator on completions nobody
153
+ // wants and teaches the user to stop reading ghosts. `DISPLAY_MIN_PREFIX_LENGTH`
154
+ // is the same guard on the side the user has already typed.
155
+ var MIN_SUGGESTION_LENGTH = 3;
156
+ /**
157
+ * How many leading tokens of a surface must be scored before it may be shown.
158
+ *
159
+ * Scoring a token costs one model round trip, and a round trip costs about the
160
+ * same no matter how much is in it, so the price of a candidate is its token
161
+ * count. Demanding every token therefore caps what can ever reach the screen at
162
+ * roughly two tokens inside the decision budget — which is why no phrase has
163
+ * ever been displayed. Verifying a fixed prefix makes the cost of a candidate
164
+ * independent of its length.
165
+ */
166
+ var REQUIRED_VERIFIED_TOKENS = 2;
167
+ /**
168
+ * Extra winner margin demanded of a surface judged on a verified prefix rather
169
+ * than on every one of its tokens, to buy back the precision given up by
170
+ * leaving the tail unread.
171
+ */
172
+ var PARTIAL_EVIDENCE_MARGIN_PREMIUM = 0.08;
173
+ var ARBITRATION_MODE = 'confidence-v2';
174
+ var DEBUG_TEXT_TAIL_CHARS = 120;
55
175
 
56
176
  // ─── Types ───────────────────────────────────────────────────────────────────
177
+
178
+ /**
179
+ * Which constraint stopped an evaluation from putting a ghost on screen.
180
+ *
181
+ * Each one implies different work: `below-posterior-gate` is a threshold to
182
+ * calibrate, `winner-margin` is two candidates the model cannot separate,
183
+ * `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
184
+ * is vocabulary coverage.
185
+ */
186
+
187
+ /**
188
+ * A candidate paired with the length of the already-typed prefix it completes.
189
+ * For a single word this is the current partial token length; for a phrase it
190
+ * is the length of the matched multi-word window (previous words + partial),
191
+ * so the ghost tail is `word.slice(matchedPrefixLen)`.
192
+ */
193
+
194
+ /** Optional per-term metadata carried by bigram/phrase completion terms. */
57
195
  var TrieNode = /*#__PURE__*/_createClass(function TrieNode() {
58
196
  _classCallCheck(this, TrieNode);
59
197
  _defineProperty(this, "children", new Map());
@@ -62,6 +200,12 @@ var TrieNode = /*#__PURE__*/_createClass(function TrieNode() {
62
200
  _defineProperty(this, "docFreq", 0);
63
201
  _defineProperty(this, "authorFreq", 0);
64
202
  _defineProperty(this, "sessionFreq", 0);
203
+ // Term-type + phrase metadata. Plain words keep `termType='word'` and null
204
+ // metadata; bigram/phrase nodes carry the producer-shipped POS + association.
205
+ _defineProperty(this, "termType", 'word');
206
+ _defineProperty(this, "posSeq", null);
207
+ _defineProperty(this, "headPos", null);
208
+ _defineProperty(this, "association", null);
65
209
  });
66
210
  var WeightedWordTrie = /*#__PURE__*/function () {
67
211
  function WeightedWordTrie() {
@@ -72,7 +216,8 @@ var WeightedWordTrie = /*#__PURE__*/function () {
72
216
  }
73
217
  return _createClass(WeightedWordTrie, [{
74
218
  key: "insert",
75
- value: function insert(word, tenantFreq, docFreq, authorFreq) {
219
+ value: function insert(word, tenantFreq, docFreq, authorFreq, meta) {
220
+ var _meta$termType, _meta$posSeq, _meta$headPos, _meta$association;
76
221
  var node = this.root;
77
222
  var _iterator = _createForOfIteratorHelper(word.toLowerCase()),
78
223
  _step;
@@ -95,6 +240,10 @@ var WeightedWordTrie = /*#__PURE__*/function () {
95
240
  node.tenantFreq = tenantFreq;
96
241
  node.docFreq = docFreq;
97
242
  node.authorFreq = authorFreq;
243
+ node.termType = (_meta$termType = meta === null || meta === void 0 ? void 0 : meta.termType) !== null && _meta$termType !== void 0 ? _meta$termType : 'word';
244
+ node.posSeq = (_meta$posSeq = meta === null || meta === void 0 ? void 0 : meta.posSeq) !== null && _meta$posSeq !== void 0 ? _meta$posSeq : null;
245
+ node.headPos = (_meta$headPos = meta === null || meta === void 0 ? void 0 : meta.headPos) !== null && _meta$headPos !== void 0 ? _meta$headPos : null;
246
+ node.association = (_meta$association = meta === null || meta === void 0 ? void 0 : meta.association) !== null && _meta$association !== void 0 ? _meta$association : null;
98
247
  if (tenantFreq > this.maxTenantFreq) {
99
248
  this.maxTenantFreq = tenantFreq;
100
249
  }
@@ -214,6 +363,15 @@ var wordTrie = new WeightedWordTrie();
214
363
  // L3 Trie (General English Fallback)
215
364
  var l3Trie = new WeightedWordTrie();
216
365
 
366
+ /**
367
+ * L2 phrase/bigram trie. Kept SEPARATE from `wordTrie` so single-word
368
+ * completion + L3 gap-fill stay untouched; multi-word units are only surfaced
369
+ * by the (12b) multiword prefix-window matcher. Keys are the full normalized
370
+ * surface string (spaces included), e.g. `"return on investment"`, so a lookup
371
+ * is a plain char-trie walk over the whole phrase.
372
+ */
373
+ var phraseTrie = new WeightedWordTrie();
374
+
217
375
  /**
218
376
  * Loads the General English vocabulary.
219
377
  * expects a simple array of strings: ["about", "above", "actually", ...]
@@ -233,10 +391,8 @@ export var initL3Vocabulary = function initL3Vocabulary(l3Words) {
233
391
  } finally {
234
392
  _iterator5.f();
235
393
  }
236
- if (isAutocompleteDebugEnabled()) {
237
- // eslint-disable-next-line no-console
238
- console.log("[text-predictor] L3 General English loaded: ".concat(l3Words.length, " words"));
239
- }
394
+ recallGeneration++;
395
+ ctcTag('init', "L3 general English loaded: ".concat(l3Words.length, " words"));
240
396
  };
241
397
 
242
398
  // const bigramMap: Map<string, Record<string, number>> = new Map(
@@ -246,8 +402,50 @@ export var initL3Vocabulary = function initL3Vocabulary(l3Words) {
246
402
  var isInitialized = false;
247
403
  var vectorStore = null;
248
404
  var vectorsLoadStarted = false;
405
+
406
+ // Phrase/bigram artifacts load independently of the word vocabulary so a
407
+ // missing/late `bigrams.json`/`phrases.json` never breaks word completion.
408
+ var phrasesLoadStarted = false;
409
+ var phrasesLoaded = false;
410
+ var phraseTermCount = 0;
411
+ // Per-type frequency maxima, tracked separately because bigrams and phrases
412
+ // share one trie but have very different absolute frequency ranges (bigrams are
413
+ // far more frequent than 3-word phrases). Normalizing each type by its OWN max
414
+ // stops phrase freq scores from being crushed against the top bigram's count.
415
+ var maxBigramFreq = 1;
416
+ var maxPhraseFreq = 1;
249
417
  var lastPredictionDebug = null;
250
418
 
419
+ // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
420
+ // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
421
+ // generally keep a just-accepted unit from being re-offered:
422
+ // • repetition: never surface a candidate whose full surface already equals the
423
+ // run of tokens immediately before the cursor (handled inline in predict()).
424
+ // • cooldown: after an accept, briefly suppress re-offering that same unit. The
425
+ // cooldown lifts only once BOTH the keystroke budget and the time window have
426
+ // elapsed (either one still counting keeps it active).
427
+ var COOLDOWN_KEYSTROKES = 5;
428
+ var COOLDOWN_MS = 2000;
429
+
430
+ // Active post-accept cooldown, or null when none.
431
+ var acceptCooldown = null;
432
+
433
+ /**
434
+ * Start a short cooldown for the exact surface the editor inserted. The caller
435
+ * passes the committed snapshot's surface so background re-ranking can never
436
+ * move cooldown bookkeeping away from what the user actually accepted.
437
+ */
438
+ export var noteSuggestionAccepted = function noteSuggestionAccepted(surface) {
439
+ var normalizedSurface = surface.trim().toLowerCase();
440
+ if (normalizedSurface) {
441
+ acceptCooldown = {
442
+ predictionsSince: 0,
443
+ surface: normalizedSurface,
444
+ ts: performance.now()
445
+ };
446
+ }
447
+ };
448
+
251
449
  /** Get vector for a word from the store. */
252
450
  var getWordVector = function getWordVector(word) {
253
451
  if (!vectorStore) {
@@ -344,6 +542,124 @@ var extractPreviousWord = function extractPreviousWord(text) {
344
542
  return words.length >= 2 ? words[words.length - 2] : '';
345
543
  };
346
544
 
545
+ /**
546
+ * Collect phrase/bigram completion candidates for the multi-word window ending
547
+ * at the current partial token.
548
+ *
549
+ * Builds windows of the last `2..PHRASE_MAX_WORDS` tokens of the current
550
+ * sentence (widest first) and prefix-searches the phrase trie for each, so
551
+ * `quarterly pla` matches `quarterly planning meeting`. Windows never cross a
552
+ * sentence boundary (mirrors `extractPreviousWord`). Each result records the
553
+ * matched window length so the caller can compute the ghost tail.
554
+ *
555
+ * :params:
556
+ * trimmed: Text before the cursor, trailing whitespace already removed
557
+ * :returns:
558
+ * Matched phrase candidates (deduped, widest-window match wins per term)
559
+ */
560
+ var getPhraseCandidates = function getPhraseCandidates(trimmed) {
561
+ // Records every prefix window we probe and its match count so the (verbose)
562
+ // debug line makes it obvious whether the phrase path found nothing because the
563
+ // trie is empty, or because no phrase begins with the typed window.
564
+ var windowsTried = [];
565
+ var logPhrasePath = function logPhrasePath(skipReason) {
566
+ if (!isAutocompleteDebugVerbose()) {
567
+ return;
568
+ }
569
+ var detail = skipReason ? skipReason : "windows: ".concat(windowsTried.map(function (w) {
570
+ return "\"".concat(w.window, "\"\u2192").concat(w.matches);
571
+ }).join(', ') || '(none)');
572
+ ctcTag('phrase', "loaded=".concat(phraseTermCount, " (").concat(phrasesLoaded ? 'ready' : 'not loaded', ") \xB7 ").concat(detail), CTC_STYLES.section);
573
+ };
574
+ if (phraseTermCount === 0) {
575
+ logPhrasePath('phrase trie empty — bigrams.json/phrases.json not loaded (or 0 terms)');
576
+ return [];
577
+ }
578
+ var tokenMatches = Array.from(trimmed.matchAll(SURFACE_TOKEN_REGEX));
579
+ var lastMatch = tokenMatches[tokenMatches.length - 1];
580
+ var lastStart = lastMatch === null || lastMatch === void 0 ? void 0 : lastMatch.index;
581
+ if (!lastMatch || lastStart === undefined || lastStart + lastMatch[0].length !== trimmed.length) {
582
+ logPhrasePath('no tokens in current sentence');
583
+ return [];
584
+ }
585
+ var trailingTokens = [{
586
+ start: lastStart,
587
+ value: lastMatch[0]
588
+ }];
589
+ for (var i = tokenMatches.length - 2; i >= 0; i--) {
590
+ var current = trailingTokens[0];
591
+ var match = tokenMatches[i];
592
+ var start = match.index;
593
+ if (start === undefined) {
594
+ break;
595
+ }
596
+ var end = start + match[0].length;
597
+ var gap = trimmed.slice(end, current.start);
598
+ if (!ONLY_WHITESPACE_REGEX.test(gap)) {
599
+ break;
600
+ }
601
+ trailingTokens.unshift({
602
+ start: start,
603
+ value: match[0]
604
+ });
605
+ if (trailingTokens.length >= PHRASE_MAX_WORDS) {
606
+ break;
607
+ }
608
+ }
609
+ var partial = trailingTokens[trailingTokens.length - 1].value;
610
+ var results = [];
611
+ var seen = new Set();
612
+ var maxPrev = trailingTokens.length - 1;
613
+
614
+ // Widest window first so the most specific (longest-context) match wins the
615
+ // dedup for a given phrase term. `prev` is the number of *preceding* complete
616
+ // words included ahead of the current partial:
617
+ // prev ≥ 1 → "continue a phrase I've started" (e.g. "root c" → root cause).
618
+ // Preceding-word context is strong, so a short partial is fine.
619
+ // prev = 0 → "the partial is a phrase's FIRST word" (e.g. "root" → root
620
+ // cause). No preceding context, so require the partial to be a
621
+ // one character so grouped LM work can begin early. UI remains
622
+ // gated at DISPLAY_MIN_PREFIX_LENGTH.
623
+ for (var prev = maxPrev; prev >= 0; prev--) {
624
+ if (prev === 0 && partial.length < MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH) {
625
+ continue;
626
+ }
627
+ var windowTokens = trailingTokens.slice(-(prev + 1));
628
+ var windowPrefix = windowTokens.map(function (token) {
629
+ return token.value;
630
+ }).join(' ').toLowerCase();
631
+ var windowLen = windowPrefix.length;
632
+ var matches = phraseTrie.getCandidates(windowPrefix, MAX_PHRASE_CANDIDATES);
633
+ windowsTried.push({
634
+ window: windowPrefix,
635
+ matches: matches.length
636
+ });
637
+ var _iterator8 = _createForOfIteratorHelper(matches),
638
+ _step8;
639
+ try {
640
+ for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
641
+ var _match = _step8.value;
642
+ if (seen.has(_match.word)) {
643
+ continue;
644
+ }
645
+ seen.add(_match.word);
646
+ results.push({
647
+ node: _match.node,
648
+ word: _match.word,
649
+ matchedPrefixLen: windowLen,
650
+ surfaceStart: windowTokens[0].start
651
+ });
652
+ }
653
+ } catch (err) {
654
+ _iterator8.e(err);
655
+ } finally {
656
+ _iterator8.f();
657
+ }
658
+ }
659
+ logPhrasePath();
660
+ return results;
661
+ };
662
+
347
663
  // ─── Debug Helpers ───────────────────────────────────────────────────────────
348
664
 
349
665
  /**
@@ -352,11 +668,25 @@ var extractPreviousWord = function extractPreviousWord(text) {
352
668
  * wordCount: number of words in vector store (0 if not loaded)
353
669
  */
354
670
  export var getPredictorStatus = function getPredictorStatus() {
671
+ var slowLaneStatus = getDefaultSlowLaneClientStatus();
355
672
  return {
673
+ canonicalScoringSupported: slowLaneStatus.canonicalScoringSupported,
674
+ canonicalSurfaceCount: slowLaneStatus.canonicalSurfaceCount,
675
+ clientId: slowLaneStatus.clientId,
676
+ clientKind: slowLaneStatus.clientKind,
677
+ localModelReady: slowLaneStatus.localModelReady,
678
+ slowLaneRegistered: slowLaneStatus.registered,
356
679
  vectorsLoaded: vectorStore !== null,
357
680
  wordCount: vectorStore ? Object.keys(vectorStore.wordIndex).length : 0,
358
681
  vectorsLoadStarted: vectorsLoadStarted,
359
- isInitialized: isInitialized
682
+ isInitialized: isInitialized,
683
+ phrasesLoaded: phrasesLoaded,
684
+ phraseCount: phraseTermCount,
685
+ maxFreqByType: {
686
+ word: wordTrie.maxTenantFreq,
687
+ bigram: maxBigramFreq,
688
+ phrase: maxPhraseFreq
689
+ }
360
690
  };
361
691
  };
362
692
 
@@ -368,19 +698,57 @@ export var getLastPredictionDebug = function getLastPredictionDebug() {
368
698
  return lastPredictionDebug;
369
699
  };
370
700
  export var initVocabulary = function initVocabulary(vocabulary) {
371
- var _iterator8 = _createForOfIteratorHelper(vocabulary.terms),
372
- _step8;
701
+ var _iterator9 = _createForOfIteratorHelper(vocabulary.terms),
702
+ _step9;
373
703
  try {
374
- for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
375
- var term = _step8.value;
704
+ for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
705
+ var term = _step9.value;
376
706
  wordTrie.insert(term.word, term.freq, term.docFreq, term.authorFreq);
377
707
  }
378
708
  } catch (err) {
379
- _iterator8.e(err);
709
+ _iterator9.e(err);
380
710
  } finally {
381
- _iterator8.f();
711
+ _iterator9.f();
382
712
  }
383
713
  isInitialized = true;
714
+ recallGeneration++;
715
+ };
716
+
717
+ /**
718
+ * Insert a producer-shipped bigram/phrase artifact into the phrase trie.
719
+ *
720
+ * :params:
721
+ * artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
722
+ * termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
723
+ * :returns:
724
+ * The number of terms inserted
725
+ */
726
+ export var initPhrases = function initPhrases(artifact, termType) {
727
+ var count = 0;
728
+ for (var _i3 = 0, _Object$entries = Object.entries(artifact); _i3 < _Object$entries.length; _i3++) {
729
+ var _stats$pos_seq, _stats$head_pos, _stats$association;
730
+ var _Object$entries$_i = _slicedToArray(_Object$entries[_i3], 2),
731
+ term = _Object$entries$_i[0],
732
+ stats = _Object$entries$_i[1];
733
+ phraseTrie.insert(term, stats.freq, stats.doc_freq, stats.author_freq, {
734
+ termType: termType,
735
+ posSeq: (_stats$pos_seq = stats.pos_seq) !== null && _stats$pos_seq !== void 0 ? _stats$pos_seq : null,
736
+ headPos: (_stats$head_pos = stats.head_pos) !== null && _stats$head_pos !== void 0 ? _stats$head_pos : null,
737
+ association: (_stats$association = stats.association) !== null && _stats$association !== void 0 ? _stats$association : null
738
+ });
739
+ // Track each type's own frequency ceiling for per-type normalization.
740
+ if (termType === 'bigram') {
741
+ if (stats.freq > maxBigramFreq) {
742
+ maxBigramFreq = stats.freq;
743
+ }
744
+ } else if (stats.freq > maxPhraseFreq) {
745
+ maxPhraseFreq = stats.freq;
746
+ }
747
+ count++;
748
+ }
749
+ phraseTermCount += count;
750
+ recallGeneration++;
751
+ return count;
384
752
  };
385
753
 
386
754
  /**
@@ -407,177 +775,1104 @@ export var ingestDocumentPage = function ingestDocumentPage(pageContent) {
407
775
  }
408
776
  var words = tokenize(pageContent);
409
777
  var validBoostedWords = new Set();
410
- var _iterator9 = _createForOfIteratorHelper(words),
411
- _step9;
778
+ var _iterator0 = _createForOfIteratorHelper(words),
779
+ _step0;
412
780
  try {
413
- for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
414
- var word = _step9.value;
781
+ for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
782
+ var word = _step0.value;
415
783
  var didBoost = wordTrie.incrementSessionFreq(word);
416
784
  if (didBoost) {
417
785
  validBoostedWords.add(word);
418
786
  }
419
787
  }
420
788
  } catch (err) {
421
- _iterator9.e(err);
789
+ _iterator0.e(err);
422
790
  } finally {
423
- _iterator9.f();
791
+ _iterator0.f();
424
792
  }
425
793
  if (isAutocompleteDebugEnabled() && validBoostedWords.size > 0) {
426
- // eslint-disable-next-line no-console
427
- console.groupCollapsed("%c[L1 Session] %cPrimed ".concat(validBoostedWords.size, " valid dictionary words from page"), 'color: #00b8d9; font-weight: bold;', 'color: inherit; font-style: italic;');
428
- // eslint-disable-next-line no-console
429
- console.dir(Array.from(validBoostedWords).sort());
430
- // eslint-disable-next-line no-console
431
- console.groupEnd();
794
+ ctcTag('init', "L1 session primed ".concat(validBoostedWords.size, " words from page"), CTC_STYLES.brand);
795
+ if (isAutocompleteDebugVerbose()) {
796
+ // eslint-disable-next-line no-console
797
+ console.dir(Array.from(validBoostedWords).sort());
798
+ }
432
799
  }
433
800
  };
434
- export var predict = function predict(textBefore) {
435
- if (!isInitialized) {
436
- // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
437
- // and skip this keystroke; the plugin also primes it on focus, so the tries
438
- // are usually ready before the user types.
439
- void loadDefaultVocabulary().catch(function () {});
440
- return null;
441
- }
442
- var t0 = performance.now();
443
801
 
444
- // ── Step 1: Bigram-based next-word suggestion at word boundary ───────────
445
- // if (textBefore.length > 0 && /\s$/u.test(textBefore)) {
446
- // const words = textBefore.toLowerCase().trimEnd().split(/\s+/u);
447
- // const prevWord = words[words.length - 1];
448
- // const nextWords = bigramMap.get(prevWord);
449
- // if (nextWords) {
450
- // const sorted = Object.entries(nextWords).sort((a, b) => b[1] - a[1]);
451
- // let bestWord = '';
452
- // for (const [word] of sorted) {
453
- // if (!isGrammarAllowed(prevWord, word)) {
454
- // continue;
455
- // }
456
- // bestWord = word;
457
- // break;
458
- // }
459
- // if (bestWord) {
460
- // if (debugMode) {
461
- // const latencyMs = performance.now() - t0;
462
- // console.log(
463
- // '%c[autocomplete] BIGRAM',
464
- // 'color:cyan',
465
- // '| "' + prevWord + '" -> "' + bestWord + '" | ' + latencyMs.toFixed(1) + 'ms',
466
- // );
467
- // }
468
- // return bestWord;
469
- // }
470
- // }
471
- // if (debugMode) {
472
- // console.log(
473
- // '%c[autocomplete] BIGRAM-MISS',
474
- // 'color:gray',
475
- // '| no bigram for "' + words[words.length - 1] + '", skipping prefix completion',
476
- // );
477
- // }
478
- // return null;
479
- // }
480
-
481
- // ── Step 2: Prefix completion (≥3 chars typed) ──────────────────────────
482
- if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
483
- return null;
484
- }
485
- var trimmed = textBefore.trimEnd();
486
- var lastSpaceIdx = trimmed.lastIndexOf(' ');
487
- var currentWord = lastSpaceIdx === -1 ? trimmed : trimmed.slice(lastSpaceIdx + 1);
488
- if (currentWord.length < MIN_PREFIX_LENGTH) {
489
- return null;
490
- }
491
- var candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
802
+ /**
803
+ * Result of a prediction: the ghost tail to insert plus an immutable record of
804
+ * the evidence that authorized the UI commitment.
805
+ */
806
+
807
+ // ─── Recall memo ─────────────────────────────────────────────────────────────
808
+
809
+ /** Trie recall plus the canonical context derived for each matched term. */
810
+
811
+ /**
812
+ * Bumped whenever an artifact load changes what the tries can return, which is
813
+ * the only way the candidate *set* for a given prefix can change.
814
+ *
815
+ * Session-frequency boosts are deliberately not counted: they mutate trie nodes
816
+ * in place, and the memo holds those nodes by reference, so a boost is already
817
+ * visible through a cached entry.
818
+ */
819
+ var recallGeneration = 0;
820
+ var recallMemo = null;
821
+ var computeCanonicalRecall = function computeCanonicalRecall(trimmed, currentWord, currentWordStart) {
822
+ // Start recall at one character so grouped causal work can overlap later
823
+ // keystrokes. Nothing is displayed until three characters.
824
+ var wordCandidates = currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH ? wordTrie.getCandidates(currentWord, MAX_CANDIDATES) : [];
492
825
 
493
826
  // Gap-fill from the L3 general-English trie, requesting a full buffer so
494
827
  // enough survive de-duplication against the L2 results.
495
- if (candidates.length < MAX_CANDIDATES) {
828
+ if (currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH && wordCandidates.length < MAX_CANDIDATES) {
496
829
  var l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
497
- var existingWords = new Set(candidates.map(function (c) {
830
+ var existingWords = new Set(wordCandidates.map(function (c) {
498
831
  return c.word;
499
832
  }));
500
- var _iterator0 = _createForOfIteratorHelper(l3Candidates),
501
- _step0;
833
+ var _iterator1 = _createForOfIteratorHelper(l3Candidates),
834
+ _step1;
502
835
  try {
503
- for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
504
- var l3c = _step0.value;
505
- if (candidates.length >= MAX_CANDIDATES) break;
836
+ for (_iterator1.s(); !(_step1 = _iterator1.n()).done;) {
837
+ var l3c = _step1.value;
838
+ if (wordCandidates.length >= MAX_CANDIDATES) break;
506
839
  if (!existingWords.has(l3c.word)) {
507
- candidates.push(l3c);
840
+ wordCandidates.push(l3c);
508
841
  }
509
842
  }
510
843
  } catch (err) {
511
- _iterator0.e(err);
844
+ _iterator1.e(err);
512
845
  } finally {
513
- _iterator0.f();
846
+ _iterator1.f();
847
+ }
848
+ }
849
+
850
+ // Multi-word completion terms (bigrams/phrases) matched on the trailing
851
+ // window ending at the current partial token. Runs even when the unigram path
852
+ // was skipped for a short partial, so first-word/early phrase matches surface.
853
+ var phraseCandidates = getPhraseCandidates(trimmed);
854
+
855
+ // Unify: a word completes the current partial token; a phrase completes its
856
+ // matched multi-word window. Track the prefix length per term so the ghost
857
+ // tail is sliced correctly regardless of term type.
858
+ var matched = [].concat(_toConsumableArray(wordCandidates.map(function (_ref) {
859
+ var word = _ref.word,
860
+ node = _ref.node;
861
+ return {
862
+ word: word,
863
+ node: node,
864
+ matchedPrefixLen: currentWord.length,
865
+ surfaceStart: currentWordStart
866
+ };
867
+ })), _toConsumableArray(phraseCandidates));
868
+ var positionCache = createCanonicalContextPositionCache();
869
+ var canonicalMatched = matched.map(function (candidate) {
870
+ return _objectSpread(_objectSpread({}, candidate), deriveCanonicalCandidateContext(trimmed, candidate.matchedPrefixLen, candidate.word, getCanonicalSurfaceTokenIds, candidate.surfaceStart, positionCache));
871
+ });
872
+ var prefixLenByWord = new Map();
873
+ var _iterator10 = _createForOfIteratorHelper(canonicalMatched),
874
+ _step10;
875
+ try {
876
+ for (_iterator10.s(); !(_step10 = _iterator10.n()).done;) {
877
+ var m = _step10.value;
878
+ if (!prefixLenByWord.has(m.word)) {
879
+ prefixLenByWord.set(m.word, m.matchedPrefixLen);
880
+ }
881
+ }
882
+ } catch (err) {
883
+ _iterator10.e(err);
884
+ } finally {
885
+ _iterator10.f();
886
+ }
887
+ return {
888
+ canonicalMatched: canonicalMatched,
889
+ prefixLenByWord: prefixLenByWord
890
+ };
891
+ };
892
+
893
+ /**
894
+ * Recall and canonical derivation for `trimmed`, reusing the previous result
895
+ * when nothing they depend on has changed.
896
+ *
897
+ * `predict()` runs several times for one keystroke — once when the decision
898
+ * opens and again for each async evidence signal that lands inside the budget —
899
+ * and only the evidence differs between those runs. Without this, every run
900
+ * re-walks the tries and re-derives a canonical context per candidate.
901
+ */
902
+ var getCanonicalRecall = function getCanonicalRecall(trimmed, currentWord, currentWordStart) {
903
+ var surfaceCount = getCanonicalSurfaceCount();
904
+ if (recallMemo !== null && recallMemo.trimmed === trimmed && recallMemo.generation === recallGeneration && recallMemo.surfaceCount === surfaceCount) {
905
+ return recallMemo;
906
+ }
907
+ var recall = computeCanonicalRecall(trimmed, currentWord, currentWordStart);
908
+ recallMemo = _objectSpread(_objectSpread({}, recall), {}, {
909
+ generation: recallGeneration,
910
+ surfaceCount: surfaceCount,
911
+ trimmed: trimmed
912
+ });
913
+ return recall;
914
+ };
915
+ export var predict = function predict(textBefore) {
916
+ var _trimmed$match$, _trimmed$match, _canonicalMatched$fin, _eligible$, _eligible$0$score, _eligible$2, _judgedEvidenceFor3, _prefixLenByWord$get2, _runtimeBySurface$get5, _judgedEvidenceFor$me3, _judgedEvidenceFor6, _contextTotals$get$le4, _contextTotals$get4, _selectedEvidence$ver, _selectedEvidence$tot, _selectedEvidence$ver2;
917
+ if (!isInitialized) {
918
+ // Vocabulary artifacts load asynchronously over the autocomplete gateway.
919
+ // Kick off the load and skip this keystroke; the plugin also primes it on
920
+ // focus, so the tries are usually ready before the user types.
921
+ void loadDefaultVocabulary({
922
+ source: 'predict'
923
+ }).catch(function () {});
924
+ return null;
925
+ }
926
+ var t0 = performance.now();
927
+
928
+ // Advance the post-accept cooldown once per prediction. It stays active until
929
+ // BOTH the keystroke budget and the time window have elapsed; while active it
930
+ // suppresses re-offering the just-accepted surface (see arbitration below).
931
+ var cooledSurface = null;
932
+ if (acceptCooldown) {
933
+ acceptCooldown.predictionsSince += 1;
934
+ var stillByKeys = acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES;
935
+ var stillByTime = performance.now() - acceptCooldown.ts < COOLDOWN_MS;
936
+ if (stillByKeys || stillByTime) {
937
+ cooledSurface = acceptCooldown.surface;
938
+ } else {
939
+ acceptCooldown = null;
940
+ }
941
+ }
942
+
943
+ // ── Boundary/early-prefix prefetch; display remains gated at ≥3 ─────────
944
+ if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
945
+ var boundaryContext = deriveWhitespaceBoundaryContext(textBefore);
946
+ if (boundaryContext) {
947
+ primeBoundaryLm(_objectSpread(_objectSpread({}, boundaryContext), {}, {
948
+ familyKey: boundaryContext.contextKey,
949
+ priority: 0
950
+ }));
514
951
  }
952
+ return null;
515
953
  }
954
+ var trimmed = textBefore.trimEnd();
955
+ var trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
956
+ if (trailingSurfaceToken.length === 0) {
957
+ return null;
958
+ }
959
+ var currentWord = trailingSurfaceToken;
960
+ var currentWordStart = trimmed.length - currentWord.length;
961
+ var _getCanonicalRecall = getCanonicalRecall(trimmed, currentWord, currentWordStart),
962
+ canonicalMatched = _getCanonicalRecall.canonicalMatched,
963
+ prefixLenByWord = _getCanonicalRecall.prefixLenByWord;
516
964
 
517
- // If both Tries are completely empty for this prefix
518
- if (candidates.length === 0) {
965
+ // If every trie was empty for this prefix
966
+ if (canonicalMatched.length === 0) {
967
+ if (isAutocompleteDebugEnabled()) {
968
+ // eslint-disable-next-line no-console
969
+ console.log("%c[CTC]%c \u2014 abstain: no matches for \"".concat(currentWord, "\""), CTC_STYLES.brand, CTC_STYLES.body);
970
+ }
519
971
  return null;
520
972
  }
521
973
  var previousWord = extractPreviousWord(trimmed);
522
974
  var contextVector = getContextVectorForScoring(trimmed);
523
975
  var lmLogits = getStoredLmLogits();
524
-
525
- // Raw LM Output Logger
526
- if (isAutocompleteDebugEnabled() && lmLogits && Object.keys(lmLogits).length > 0) {
527
- var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
528
- return b[1] - a[1];
529
- }).slice(0, 5).map(function (_ref) {
530
- var _ref2 = _slicedToArray(_ref, 2),
531
- word = _ref2[0],
532
- score = _ref2[1];
533
- return {
534
- Word: word,
535
- Prob: Number(score.toFixed(5))
536
- };
537
- });
538
- // eslint-disable-next-line no-console
539
- console.log('%c[Raw LM Prediction]🧠', 'color: #e83e8c; font-weight: bold;', rawLmTop);
540
- }
541
976
  var mode = contextVector ? 'warm' : 'cold';
542
977
 
543
- // Build ScoringCandidate array from TrieNodes
544
- var scoringCandidates = candidates.map(function (_ref3) {
545
- var word = _ref3.word,
546
- node = _ref3.node;
978
+ // Build ScoringCandidate array from matched terms (words + phrases)
979
+ var scoringCandidates = canonicalMatched.map(function (_ref2) {
980
+ var word = _ref2.word,
981
+ node = _ref2.node;
547
982
  return {
548
983
  word: word,
549
984
  tenantFreq: node.tenantFreq,
550
985
  docFreq: node.docFreq,
551
986
  authorFreq: node.authorFreq,
552
- sessionFreq: node.sessionFreq
987
+ sessionFreq: node.sessionFreq,
988
+ termType: node.termType,
989
+ posSeq: node.posSeq,
990
+ headPos: node.headPos,
991
+ association: node.association
553
992
  };
554
993
  });
555
994
 
556
995
  // Filter the LM payload to only words matching the current prefix so that
557
996
  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
558
997
  var prefix = currentWord.toLowerCase();
559
- var prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
560
- var _ref5 = _slicedToArray(_ref4, 1),
561
- word = _ref5[0];
998
+ var currentWordSeparator = (_canonicalMatched$fin = canonicalMatched.find(function (candidate) {
999
+ return candidate.node.termType === 'word';
1000
+ })) === null || _canonicalMatched$fin === void 0 ? void 0 : _canonicalMatched$fin.separatorKind;
1001
+ var prefixLmLogits = lmLogits && currentWordSeparator === 'whitespace' ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref3) {
1002
+ var _ref4 = _slicedToArray(_ref3, 1),
1003
+ word = _ref4[0];
562
1004
  return word.startsWith(prefix);
563
1005
  })) : null;
1006
+ var canonicalScoringSupported = isCanonicalSurfaceScoringSupported();
1007
+ var eligibleContextKeys = Array.from(new Set(canonicalMatched.filter(function (candidate) {
1008
+ return candidate.canonicalTokenIds !== null;
1009
+ }).map(function (candidate) {
1010
+ return candidate.contextKey;
1011
+ }))).sort();
1012
+ var familyKey = eligibleContextKeys.join("\x01");
1013
+ var primeRequests = selectBoundaryPrimeRequests(familyKey, canonicalMatched, PHRASE_MAX_WORDS);
1014
+ var _iterator11 = _createForOfIteratorHelper(primeRequests),
1015
+ _step11;
1016
+ try {
1017
+ for (_iterator11.s(); !(_step11 = _iterator11.n()).done;) {
1018
+ var request = _step11.value;
1019
+ primeBoundaryLm(request);
1020
+ }
1021
+ } catch (err) {
1022
+ _iterator11.e(err);
1023
+ } finally {
1024
+ _iterator11.f();
1025
+ }
1026
+ var runtimeBySurface = new Map(canonicalMatched.map(function (candidate) {
1027
+ return [candidate.word, candidate];
1028
+ }));
1029
+ var canonicalEvidence = new Map();
1030
+ var firstTokenGroups = new Map();
1031
+ var _iterator12 = _createForOfIteratorHelper(canonicalMatched),
1032
+ _step12;
1033
+ try {
1034
+ for (_iterator12.s(); !(_step12 = _iterator12.n()).done;) {
1035
+ var _firstTokenGroups$get;
1036
+ var _candidate = _step12.value;
1037
+ if (_candidate.canonicalTokenIds === null) {
1038
+ continue;
1039
+ }
1040
+ // This accessor also materializes an exact cache entry when all tokens
1041
+ // have already been covered by shared-prefix work.
1042
+ getProgressiveSurfaceEvidence(_candidate.contextKey, _candidate.word);
1043
+ var exact = getSurfaceScore(_candidate.contextKey, _candidate.word);
1044
+ if (exact) {
1045
+ canonicalEvidence.set(_candidate.word, {
1046
+ contextKey: _candidate.contextKey,
1047
+ meanTokenLogProbability: exact.meanTokenLogProbability,
1048
+ score: 0,
1049
+ separatorKind: 'whitespace',
1050
+ source: 'canonical-full-surface',
1051
+ totalLogProbability: exact.totalLogProbability,
1052
+ totalSurfaceCharCount: _candidate.word.length,
1053
+ totalTokenCount: exact.tokenCount,
1054
+ verifiedCharCount: _candidate.word.length,
1055
+ verifiedTokenCount: exact.tokenCount
1056
+ });
1057
+ continue;
1058
+ }
1059
+ var _group2 = (_firstTokenGroups$get = firstTokenGroups.get(_candidate.contextKey)) !== null && _firstTokenGroups$get !== void 0 ? _firstTokenGroups$get : [];
1060
+ _group2.push(_candidate);
1061
+ firstTokenGroups.set(_candidate.contextKey, _group2);
1062
+ }
1063
+ } catch (err) {
1064
+ _iterator12.e(err);
1065
+ } finally {
1066
+ _iterator12.f();
1067
+ }
1068
+ var _iterator13 = _createForOfIteratorHelper(firstTokenGroups),
1069
+ _step13;
1070
+ try {
1071
+ for (_iterator13.s(); !(_step13 = _iterator13.n()).done;) {
1072
+ var _step13$value = _slicedToArray(_step13.value, 2),
1073
+ _contextKey = _step13$value[0],
1074
+ _group3 = _step13$value[1];
1075
+ var boundary = getBoundaryLmState(_contextKey);
1076
+ if (!boundary) {
1077
+ continue;
1078
+ }
1079
+ var maxLogit = -Infinity;
1080
+ var _iterator20 = _createForOfIteratorHelper(_group3),
1081
+ _step20;
1082
+ try {
1083
+ for (_iterator20.s(); !(_step20 = _iterator20.n()).done;) {
1084
+ var _candidate2$canonical;
1085
+ var _candidate2 = _step20.value;
1086
+ var tokenId = (_candidate2$canonical = _candidate2.canonicalTokenIds) === null || _candidate2$canonical === void 0 ? void 0 : _candidate2$canonical[0];
1087
+ var rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
1088
+ if (rawLogit !== undefined && Number.isFinite(rawLogit) && rawLogit > maxLogit) {
1089
+ maxLogit = rawLogit;
1090
+ }
1091
+ }
1092
+ } catch (err) {
1093
+ _iterator20.e(err);
1094
+ } finally {
1095
+ _iterator20.f();
1096
+ }
1097
+ if (!Number.isFinite(maxLogit)) {
1098
+ continue;
1099
+ }
1100
+ var _iterator21 = _createForOfIteratorHelper(_group3),
1101
+ _step21;
1102
+ try {
1103
+ for (_iterator21.s(); !(_step21 = _iterator21.n()).done;) {
1104
+ var _candidate3$canonical, _candidate3$canonical2, _candidate3$canonical3;
1105
+ var _candidate3 = _step21.value;
1106
+ var _tokenId = (_candidate3$canonical = _candidate3.canonicalTokenIds) === null || _candidate3$canonical === void 0 ? void 0 : _candidate3$canonical[0];
1107
+ var _rawLogit = _tokenId === undefined ? undefined : boundary.rawLogits[_tokenId];
1108
+ if (_rawLogit === undefined || !Number.isFinite(_rawLogit)) {
1109
+ continue;
1110
+ }
1111
+ var tokenCount = (_candidate3$canonical2 = (_candidate3$canonical3 = _candidate3.canonicalTokenIds) === null || _candidate3$canonical3 === void 0 ? void 0 : _candidate3$canonical3.length) !== null && _candidate3$canonical2 !== void 0 ? _candidate3$canonical2 : 0;
1112
+ if (_tokenId !== undefined && tokenCount === 1) {
1113
+ var totalLogProbability = logSoftmaxAt(boundary.rawLogits, _tokenId);
1114
+ if (!Number.isFinite(totalLogProbability)) {
1115
+ continue;
1116
+ }
1117
+ canonicalEvidence.set(_candidate3.word, {
1118
+ contextKey: _contextKey,
1119
+ meanTokenLogProbability: totalLogProbability,
1120
+ score: 0,
1121
+ separatorKind: 'whitespace',
1122
+ source: 'canonical-full-surface',
1123
+ totalLogProbability: totalLogProbability,
1124
+ totalSurfaceCharCount: _candidate3.word.length,
1125
+ totalTokenCount: 1,
1126
+ verifiedCharCount: _candidate3.word.length,
1127
+ verifiedTokenCount: 1
1128
+ });
1129
+ continue;
1130
+ }
1131
+ canonicalEvidence.set(_candidate3.word, {
1132
+ contextKey: _contextKey,
1133
+ rawLogit: _rawLogit,
1134
+ score: 0,
1135
+ separatorKind: 'whitespace',
1136
+ source: 'canonical-first-token',
1137
+ totalSurfaceCharCount: _candidate3.word.length,
1138
+ totalTokenCount: tokenCount,
1139
+ verifiedCharCount: 0,
1140
+ verifiedTokenCount: 1
1141
+ });
1142
+ }
1143
+ } catch (err) {
1144
+ _iterator21.e(err);
1145
+ } finally {
1146
+ _iterator21.f();
1147
+ }
1148
+ }
1149
+ } catch (err) {
1150
+ _iterator13.e(err);
1151
+ } finally {
1152
+ _iterator13.f();
1153
+ }
564
1154
  var _rankCandidates = rankCandidates(scoringCandidates, contextVector, function (w) {
565
1155
  return getWordVector(w);
566
- }, prefixLmLogits, wordTrie.maxTenantFreq, previousWord),
1156
+ }, prefixLmLogits, {
1157
+ word: wordTrie.maxTenantFreq,
1158
+ bigram: maxBigramFreq,
1159
+ phrase: maxPhraseFreq
1160
+ }, previousWord, function (surface) {
1161
+ var _canonicalEvidence$ge;
1162
+ return (_canonicalEvidence$ge = canonicalEvidence.get(surface)) !== null && _canonicalEvidence$ge !== void 0 ? _canonicalEvidence$ge : null;
1163
+ }),
567
1164
  ranked = _rankCandidates.candidates,
568
1165
  grammarMeta = _rankCandidates.grammarMeta,
569
1166
  pipelineDebug = _rankCandidates.pipelineDebug;
570
- var best = ranked[0];
571
- var suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
1167
+ var progressiveEligible = ranked.filter(function (candidate) {
1168
+ var runtime = runtimeBySurface.get(candidate.word);
1169
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined;
1170
+ });
1171
+ if (canonicalScoringSupported && progressiveEligible.length > 0) {
1172
+ var byContext = new Map();
1173
+ var _iterator14 = _createForOfIteratorHelper(progressiveEligible),
1174
+ _step14;
1175
+ try {
1176
+ for (_iterator14.s(); !(_step14 = _iterator14.n()).done;) {
1177
+ var _byContext$get;
1178
+ var candidate = _step14.value;
1179
+ var runtime = runtimeBySurface.get(candidate.word);
1180
+ if (!runtime || runtime.canonicalTokenIds === null) {
1181
+ continue;
1182
+ }
1183
+ var group = (_byContext$get = byContext.get(runtime.contextKey)) !== null && _byContext$get !== void 0 ? _byContext$get : [];
1184
+ group.push({
1185
+ runtime: runtime,
1186
+ rankHint: candidate.stage1Score
1187
+ });
1188
+ byContext.set(runtime.contextKey, group);
1189
+ }
1190
+ } catch (err) {
1191
+ _iterator14.e(err);
1192
+ } finally {
1193
+ _iterator14.f();
1194
+ }
1195
+ var _iterator15 = _createForOfIteratorHelper(byContext),
1196
+ _step15;
1197
+ try {
1198
+ for (_iterator15.s(); !(_step15 = _iterator15.n()).done;) {
1199
+ var _step15$value = _slicedToArray(_step15.value, 2),
1200
+ contextKey = _step15$value[0],
1201
+ _group = _step15$value[1];
1202
+ requestProgressiveSurfaceScores({
1203
+ familyKey: familyKey,
1204
+ contextKey: contextKey,
1205
+ prompt: _group[0].runtime.contextBeforeSurface,
1206
+ candidates: _group.map(function (_ref5) {
1207
+ var _runtime$canonicalTok;
1208
+ var runtime = _ref5.runtime,
1209
+ rankHint = _ref5.rankHint;
1210
+ return {
1211
+ surface: runtime.word,
1212
+ tokenIds: (_runtime$canonicalTok = runtime.canonicalTokenIds) !== null && _runtime$canonicalTok !== void 0 ? _runtime$canonicalTok : [],
1213
+ rankHint: rankHint
1214
+ };
1215
+ })
1216
+ });
1217
+ }
1218
+ } catch (err) {
1219
+ _iterator15.e(err);
1220
+ } finally {
1221
+ _iterator15.f();
1222
+ }
1223
+ }
1224
+
1225
+ // The one/two-character passes exist only to hide model latency. They may
1226
+ // prime and expand token groups above, but cannot commit UI.
1227
+ var displayPrefixReady = currentWord.length >= DISPLAY_MIN_PREFIX_LENGTH;
1228
+
1229
+ // ── Arbitration: full-surface evidence → plausibility → winner margin ──
1230
+ var canonicalLmSupported = canonicalScoringSupported;
1231
+ var hasExactEvidence = function hasExactEvidence(candidate) {
1232
+ var _candidate$lmEvidence;
1233
+ return ((_candidate$lmEvidence = candidate.lmEvidence) === null || _candidate$lmEvidence === void 0 ? void 0 : _candidate$lmEvidence.source) === 'canonical-full-surface' && candidate.lmEvidence.meanTokenLogProbability !== undefined && Number.isFinite(candidate.lmEvidence.meanTokenLogProbability);
1234
+ };
1235
+
1236
+ /**
1237
+ * The evidence this candidate is judged on: the summed log-probability of
1238
+ * the tokens that have been read, their per-token mean, and whether they
1239
+ * cover the whole surface.
1240
+ *
1241
+ * `total` is the quantity the posterior is built from — it is a sequence
1242
+ * log-likelihood, so it is comparable across surfaces of different lengths
1243
+ * in a way the mean is not. `mean` is retained for reporting and for the
1244
+ * oracle arms, which compare aggregation rules against each other.
1245
+ *
1246
+ * Every token read costs a round trip, so a surface longer than
1247
+ * `REQUIRED_VERIFIED_TOKENS` is judged on its leading tokens and its tail is
1248
+ * left unread. Callers compensate with a wider winner margin rather than
1249
+ * treating the two kinds of evidence as equivalent.
1250
+ *
1251
+ * This is the depth-unaware read, and it is what the pool below is built
1252
+ * from. Arbitration goes through `judgedEvidenceFor`, which adds the one
1253
+ * requirement that can only be stated once the pool is known: a surface
1254
+ * cannot be judged against members it has not yet been read level with.
1255
+ */
1256
+ var readJudgedEvidence = function readJudgedEvidence(candidate) {
1257
+ var evidence = candidate.lmEvidence;
1258
+ var exactMean = evidence === null || evidence === void 0 ? void 0 : evidence.meanTokenLogProbability;
1259
+ if (hasExactEvidence(candidate) && evidence && exactMean !== undefined) {
1260
+ var _evidence$totalLogPro;
1261
+ var _total = (_evidence$totalLogPro = evidence.totalLogProbability) !== null && _evidence$totalLogPro !== void 0 ? _evidence$totalLogPro : exactMean * evidence.totalTokenCount;
1262
+ return Number.isFinite(_total) ? {
1263
+ complete: true,
1264
+ mean: exactMean,
1265
+ total: _total
1266
+ } : null;
1267
+ }
1268
+ var runtime = runtimeBySurface.get(candidate.word);
1269
+ if (!runtime || runtime.canonicalTokenIds === null) {
1270
+ return null;
1271
+ }
1272
+ var progressive = getProgressiveSurfaceEvidence(runtime.contextKey, candidate.word);
1273
+ if (!progressive || progressive.scoredTokenCount < REQUIRED_VERIFIED_TOKENS) {
1274
+ return null;
1275
+ }
1276
+ var total = progressive.totalLogProbability;
1277
+ var mean = total / progressive.scoredTokenCount;
1278
+ return Number.isFinite(mean) && Number.isFinite(total) ? {
1279
+ complete: false,
1280
+ mean: mean,
1281
+ total: total
1282
+ } : null;
1283
+ };
1284
+
1285
+ // Normalise each context's shortlist into a posterior. `total` is a sequence
1286
+ // log-likelihood, so `exp(total - logSumExp(totals))` is the share of the
1287
+ // shortlist's probability mass the model puts on that surface.
1288
+ //
1289
+ // Unknown future token log-probabilities cannot exceed zero, so a partially
1290
+ // read candidate's total is an upper bound on its eventual total, and its
1291
+ // posterior an upper bound on its eventual posterior. That is what lets the
1292
+ // plausibility floor and the display gate be the same threshold — one
1293
+ // applied to the bound, one to the verified value — instead of two numbers
1294
+ // that have to be kept consistent by hand.
1295
+ //
1296
+ // Candidates with no evidence at all are left out of the sum rather than
1297
+ // defaulted. Any default would have to stand in for a log-probability, and a
1298
+ // value low enough not to distort the normaliser is indistinguishable from
1299
+ // omitting the candidate.
1300
+ var optimisticTotalByWord = new Map();
1301
+ var contextTotals = new Map();
1302
+ var contextSurfaces = new Map();
1303
+ // Counted before the total is looked up, so this is every candidate the
1304
+ // context could have scored rather than every candidate it did. The gap
1305
+ // between the two is what separates a context holding one match from a
1306
+ // context whose rivals have not come back yet.
1307
+ var contextRequestedCount = new Map();
1308
+ var bestTotalByContext = new Map();
1309
+ var bestCandidateByContext = new Map();
1310
+ var _iterator16 = _createForOfIteratorHelper(ranked),
1311
+ _step16;
1312
+ try {
1313
+ for (_iterator16.s(); !(_step16 = _iterator16.n()).done;) {
1314
+ var _contextRequestedCoun2, _judged$total;
1315
+ var _candidate4 = _step16.value;
1316
+ var _runtime = runtimeBySurface.get(_candidate4.word);
1317
+ if (!_runtime || _runtime.canonicalTokenIds === null) {
1318
+ continue;
1319
+ }
1320
+ contextRequestedCount.set(_runtime.contextKey, ((_contextRequestedCoun2 = contextRequestedCount.get(_runtime.contextKey)) !== null && _contextRequestedCoun2 !== void 0 ? _contextRequestedCoun2 : 0) + 1);
1321
+ var judged = readJudgedEvidence(_candidate4);
1322
+ var progressive = getProgressiveSurfaceEvidence(_runtime.contextKey, _candidate4.word);
1323
+ // Once a candidate has been read far enough to judge, its verified total
1324
+ // replaces the bound.
1325
+ var optimisticTotal = (_judged$total = judged === null || judged === void 0 ? void 0 : judged.total) !== null && _judged$total !== void 0 ? _judged$total : progressive === null || progressive === void 0 ? void 0 : progressive.totalLogProbability;
1326
+ if (optimisticTotal === undefined || !Number.isFinite(optimisticTotal)) {
1327
+ continue;
1328
+ }
1329
+ optimisticTotalByWord.set(_candidate4.word, optimisticTotal);
1330
+ var totals = contextTotals.get(_runtime.contextKey);
1331
+ var surfaces = contextSurfaces.get(_runtime.contextKey);
1332
+ if (totals && surfaces) {
1333
+ totals.push(optimisticTotal);
1334
+ surfaces.push(_candidate4.word);
1335
+ } else {
1336
+ contextTotals.set(_runtime.contextKey, [optimisticTotal]);
1337
+ contextSurfaces.set(_runtime.contextKey, [_candidate4.word]);
1338
+ }
1339
+ var best = bestTotalByContext.get(_runtime.contextKey);
1340
+ if (best === undefined || optimisticTotal > best) {
1341
+ bestTotalByContext.set(_runtime.contextKey, optimisticTotal);
1342
+ bestCandidateByContext.set(_runtime.contextKey, _candidate4);
1343
+ }
1344
+ }
1345
+
1346
+ /**
1347
+ * Whether a shorter member of the same pool is this surface's word-boundary
1348
+ * prefix — `happy path` where `happy` is also present.
1349
+ *
1350
+ * Only a multi-word surface can extend another, and a pool is a set, so this
1351
+ * costs one lookup per space rather than a comparison against every member.
1352
+ */
1353
+ } catch (err) {
1354
+ _iterator16.e(err);
1355
+ } finally {
1356
+ _iterator16.f();
1357
+ }
1358
+ var extendsAPoolMember = function extendsAPoolMember(surface, pool) {
1359
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1360
+ if (pool.has(surface.slice(0, space))) {
1361
+ return true;
1362
+ }
1363
+ }
1364
+ return false;
1365
+ };
1366
+
1367
+ // A sequence total only falls as more of the sequence is read, so a partial
1368
+ // sum over `k` tokens sits above the same surface's total over `m > k`. An
1369
+ // extension read less far than the member it continues therefore carries a
1370
+ // total above that member's — and because the chain rule narrows the
1371
+ // denominator to the pool's minimal members, `exp(total - normaliser)` comes
1372
+ // out above 1 for a surface whose own continuation has not been looked at.
1373
+ // It needs a three-token first word to happen, which 2.6% of served bigrams
1374
+ // have, and it inflates both the posterior the gate reads and the
1375
+ // `confidenceScore` every other candidate's margin is measured against.
1376
+ //
1377
+ // A surface's canonical tokenization begins with the tokenization of its own
1378
+ // leading words — the producer encodes `" " + surface` in one call and the
1379
+ // ByteLevel pre-tokenizer splits on word boundaries, so no merge crosses a
1380
+ // space — which is what makes both corrections below exact rather than
1381
+ // approximate.
1382
+ //
1383
+ // The total is capped at what it continues: `P(happy path)` cannot exceed
1384
+ // `P(happy)`. Shortest-first, so a three-word phrase is capped against a
1385
+ // two-word prefix that has already been capped itself.
1386
+ //
1387
+ // And the surface is held unjudgeable until it has been read level with what
1388
+ // it continues, because the cap alone would leave it looking exactly as
1389
+ // certain as its prefix while saying nothing about its own tail.
1390
+ var underReadExtensions = new Set();
1391
+ var _iterator17 = _createForOfIteratorHelper(contextSurfaces),
1392
+ _step17;
1393
+ try {
1394
+ for (_iterator17.s(); !(_step17 = _iterator17.n()).done;) {
1395
+ var _step17$value = _slicedToArray(_step17.value, 2),
1396
+ _contextKey2 = _step17$value[0],
1397
+ _surfaces = _step17$value[1];
1398
+ var pool = new Set(_surfaces);
1399
+ var _iterator22 = _createForOfIteratorHelper(_toConsumableArray(_surfaces).sort(function (a, b) {
1400
+ return a.length - b.length;
1401
+ })),
1402
+ _step22;
1403
+ try {
1404
+ for (_iterator22.s(); !(_step22 = _iterator22.n()).done;) {
1405
+ var _getProgressiveSurfac, _getProgressiveSurfac2, _runtimeBySurface$get8, _runtimeBySurface$get9;
1406
+ var surface = _step22.value;
1407
+ var own = optimisticTotalByWord.get(surface);
1408
+ if (own === undefined) {
1409
+ continue;
1410
+ }
1411
+ var cap = Infinity;
1412
+ var continuedDepth = 0;
1413
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1414
+ var _runtimeBySurface$get6, _runtimeBySurface$get7;
1415
+ var _prefix = surface.slice(0, space);
1416
+ if (!pool.has(_prefix)) {
1417
+ continue;
1418
+ }
1419
+ var prefixTotal = optimisticTotalByWord.get(_prefix);
1420
+ if (prefixTotal !== undefined && prefixTotal < cap) {
1421
+ cap = prefixTotal;
1422
+ }
1423
+ var prefixDepth = (_runtimeBySurface$get6 = (_runtimeBySurface$get7 = runtimeBySurface.get(_prefix)) === null || _runtimeBySurface$get7 === void 0 || (_runtimeBySurface$get7 = _runtimeBySurface$get7.canonicalTokenIds) === null || _runtimeBySurface$get7 === void 0 ? void 0 : _runtimeBySurface$get7.length) !== null && _runtimeBySurface$get6 !== void 0 ? _runtimeBySurface$get6 : 0;
1424
+ if (prefixDepth > continuedDepth) {
1425
+ continuedDepth = prefixDepth;
1426
+ }
1427
+ }
1428
+ if (own > cap) {
1429
+ optimisticTotalByWord.set(surface, cap);
1430
+ }
1431
+ var scoredDepth = (_getProgressiveSurfac = (_getProgressiveSurfac2 = getProgressiveSurfaceEvidence(_contextKey2, surface)) === null || _getProgressiveSurfac2 === void 0 ? void 0 : _getProgressiveSurfac2.scoredTokenCount) !== null && _getProgressiveSurfac !== void 0 ? _getProgressiveSurfac : 0;
1432
+ var surfaceDepth = (_runtimeBySurface$get8 = (_runtimeBySurface$get9 = runtimeBySurface.get(surface)) === null || _runtimeBySurface$get9 === void 0 || (_runtimeBySurface$get9 = _runtimeBySurface$get9.canonicalTokenIds) === null || _runtimeBySurface$get9 === void 0 ? void 0 : _runtimeBySurface$get9.length) !== null && _runtimeBySurface$get8 !== void 0 ? _runtimeBySurface$get8 : 0;
1433
+ if (scoredDepth < continuedDepth && scoredDepth < surfaceDepth) {
1434
+ underReadExtensions.add(surface);
1435
+ }
1436
+ }
1437
+ } catch (err) {
1438
+ _iterator22.e(err);
1439
+ } finally {
1440
+ _iterator22.f();
1441
+ }
1442
+ }
1443
+
1444
+ /**
1445
+ * The evidence arbitration judges a candidate on.
1446
+ *
1447
+ * Wraps the depth-unaware read with the pool-relative requirement: a surface
1448
+ * that continues another pool member is not judgeable until it has been read
1449
+ * level with it, so the two totals being compared are taken at comparable
1450
+ * depths.
1451
+ */
1452
+ } catch (err) {
1453
+ _iterator17.e(err);
1454
+ } finally {
1455
+ _iterator17.f();
1456
+ }
1457
+ var judgedEvidenceFor = function judgedEvidenceFor(candidate) {
1458
+ return underReadExtensions.has(candidate.word) ? null : readJudgedEvidence(candidate);
1459
+ };
1460
+ var hasJudgeableEvidence = function hasJudgeableEvidence(candidate) {
1461
+ return judgedEvidenceFor(candidate) !== null;
1462
+ };
1463
+
1464
+ // Normalise over the pool's minimal members only, and score everyone —
1465
+ // extensions included — against that.
1466
+ //
1467
+ // A sequence total is its prefix's total plus a log-probability that cannot
1468
+ // be positive, so summing a surface and its own extension double-counts:
1469
+ // `exp(total)` for `happy` already contains every continuation of `happy`,
1470
+ // `happy path` among them. Dividing one pool's mass between the two capped
1471
+ // the shorter form at half however certain it was, and the longer form below
1472
+ // that — both under gates neither could then clear, for a reason that was
1473
+ // arithmetic rather than evidence.
1474
+ //
1475
+ // Excluding extensions from the denominator is the chain rule: the minimal
1476
+ // members partition the mass, and an extension's share of that same
1477
+ // denominator is `P(prefix) * P(continuation | prefix)`, which is exactly
1478
+ // what a multi-word suggestion should be held to. Extensions stay in
1479
+ // `contextTotals`, so how contested a context is still counts every scored
1480
+ // candidate.
1481
+ var logSumExpByContext = new Map();
1482
+ var _iterator18 = _createForOfIteratorHelper(contextSurfaces),
1483
+ _step18;
1484
+ try {
1485
+ for (_iterator18.s(); !(_step18 = _iterator18.n()).done;) {
1486
+ var _step18$value = _slicedToArray(_step18.value, 2),
1487
+ _contextKey3 = _step18$value[0],
1488
+ _surfaces2 = _step18$value[1];
1489
+ var _pool = new Set(_surfaces2);
1490
+ var minimalTotals = [];
1491
+ var _iterator23 = _createForOfIteratorHelper(_surfaces2),
1492
+ _step23;
1493
+ try {
1494
+ for (_iterator23.s(); !(_step23 = _iterator23.n()).done;) {
1495
+ var _surface = _step23.value;
1496
+ var total = optimisticTotalByWord.get(_surface);
1497
+ if (total !== undefined && !extendsAPoolMember(_surface, _pool)) {
1498
+ minimalTotals.push(total);
1499
+ }
1500
+ }
1501
+ // An extension is strictly longer than what it extends, so the shortest
1502
+ // member of any non-empty pool is always minimal and this is never empty.
1503
+ } catch (err) {
1504
+ _iterator23.e(err);
1505
+ } finally {
1506
+ _iterator23.f();
1507
+ }
1508
+ logSumExpByContext.set(_contextKey3, logSumExp(minimalTotals));
1509
+ }
1510
+
1511
+ /**
1512
+ * A candidate's share of its context shortlist's probability mass.
1513
+ *
1514
+ * Pass an optimistic (partially read) total for an upper bound on the
1515
+ * eventual posterior, or a verified total for the posterior itself.
1516
+ */
1517
+ } catch (err) {
1518
+ _iterator18.e(err);
1519
+ } finally {
1520
+ _iterator18.f();
1521
+ }
1522
+ var posteriorFor = function posteriorFor(candidate, total) {
1523
+ var runtime = runtimeBySurface.get(candidate.word);
1524
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
1525
+ if (total === undefined || normalizer === undefined || !Number.isFinite(normalizer)) {
1526
+ return 0;
1527
+ }
1528
+ return Math.exp(total - normalizer);
1529
+ };
1530
+ var optimisticPosterior = function optimisticPosterior(candidate) {
1531
+ return posteriorFor(candidate, optimisticTotalByWord.get(candidate.word));
1532
+ };
1533
+ var judgedPosterior = function judgedPosterior(candidate) {
1534
+ var _judgedEvidenceFor;
1535
+ return posteriorFor(candidate, (_judgedEvidenceFor = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor === void 0 ? void 0 : _judgedEvidenceFor.total);
1536
+ };
1537
+
1538
+ // A candidate stays in the race for as long as its optimistic posterior
1539
+ // could still clear the gate it will be held to. The bound only falls as
1540
+ // more tokens are read, so nothing dropped here could have gone on to win.
1541
+ var couldClearGate = function couldClearGate(candidate) {
1542
+ return optimisticPosterior(candidate) >= MIN_LM_POSTERIOR[candidate.termType];
1543
+ };
1544
+ var passesLmFloor = function passesLmFloor(candidate) {
1545
+ return judgedEvidenceFor(candidate) !== null && couldClearGate(candidate);
1546
+ };
1547
+ var hasRequiredEvidence = function hasRequiredEvidence(candidate) {
1548
+ var _candidate$lmEvidence2;
1549
+ if (canonicalLmSupported) {
1550
+ return passesLmFloor(candidate);
1551
+ }
1552
+ return candidate.termType === 'word' && ((_candidate$lmEvidence2 = candidate.lmEvidence) === null || _candidate$lmEvidence2 === void 0 ? void 0 : _candidate$lmEvidence2.source) === 'network-logit';
1553
+ };
1554
+
1555
+ // ── Stabilization guards (QI-2) ──────────────────────────────────────────
1556
+ // Whole-surface repetition: drop a candidate whose full surface already equals
1557
+ // the run of tokens immediately before the cursor — accepting it would
1558
+ // duplicate what was just typed (`end to end` → `end to end to end`).
1559
+ var beforeTokens = tokenize(trimmed);
1560
+ var duplicatesPreceding = function duplicatesPreceding(c) {
1561
+ var surfaceTokens = tokenize(c.word);
1562
+ var n = surfaceTokens.length;
1563
+ if (n === 0 || beforeTokens.length < n) {
1564
+ return false;
1565
+ }
1566
+ for (var i = 0; i < n; i++) {
1567
+ if (beforeTokens[beforeTokens.length - n + i] !== surfaceTokens[i]) {
1568
+ return false;
1569
+ }
1570
+ }
1571
+ return true;
1572
+ };
1573
+ // Post-accept cooldown: don't re-offer the surface the user just accepted while
1574
+ // its cooldown is still active (see cooldown advance near t0).
1575
+ var isCooledDown = function isCooledDown(c) {
1576
+ return cooledSurface !== null && c.word.toLowerCase() === cooledSurface;
1577
+ };
1578
+ var isStabilized = function isStabilized(candidate) {
1579
+ return !duplicatesPreceding(candidate) && !isCooledDown(candidate);
1580
+ };
1581
+ // A candidate the user has all but finished typing is not worth a ghost, and
1582
+ // a candidate we would never show is not worth waiting for either.
1583
+ var suggestionLengthFor = function suggestionLengthFor(candidate) {
1584
+ var _prefixLenByWord$get;
1585
+ return candidate.word.length - ((_prefixLenByWord$get = prefixLenByWord.get(candidate.word)) !== null && _prefixLenByWord$get !== void 0 ? _prefixLenByWord$get : currentWord.length);
1586
+ };
1587
+ var isWorthShowing = function isWorthShowing(candidate) {
1588
+ return suggestionLengthFor(candidate) >= MIN_SUGGESTION_LENGTH;
1589
+ };
1590
+ var evidenceBacked = ranked.filter(function (candidate) {
1591
+ return hasRequiredEvidence(candidate) && isStabilized(candidate);
1592
+ });
1593
+ var confidenceScore = function confidenceScore(candidate) {
1594
+ if (!canonicalLmSupported) {
1595
+ return candidate.finalScore;
1596
+ }
1597
+ var runtime = runtimeBySurface.get(candidate.word);
1598
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) === null || !optimisticTotalByWord.has(candidate.word)) {
1599
+ return candidate.stage1Score;
1600
+ }
1601
+ return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * optimisticPosterior(candidate);
1602
+ };
1603
+ var exactConfidenceScore = function exactConfidenceScore(candidate) {
1604
+ if (!canonicalLmSupported || judgedEvidenceFor(candidate) === null) {
1605
+ return candidate.finalScore;
1606
+ }
1607
+ return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * judgedPosterior(candidate);
1608
+ };
1609
+
1610
+ /**
1611
+ * How many candidates in this one's context were scored at all.
1612
+ *
1613
+ * Read off pool membership, which counts every candidate that contributed a
1614
+ * total, rather than off the normaliser, which the chain rule narrows to the
1615
+ * pool's minimal members.
1616
+ */
1617
+ var scoredPoolSize = function scoredPoolSize(candidate) {
1618
+ var _runtimeBySurface$get, _contextTotals$get$le, _contextTotals$get;
1619
+ var contextKey = (_runtimeBySurface$get = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get === void 0 ? void 0 : _runtimeBySurface$get.contextKey;
1620
+ return contextKey === undefined ? 0 : (_contextTotals$get$le = (_contextTotals$get = contextTotals.get(contextKey)) === null || _contextTotals$get === void 0 ? void 0 : _contextTotals$get.length) !== null && _contextTotals$get$le !== void 0 ? _contextTotals$get$le : 0;
1621
+ };
1622
+
1623
+ /**
1624
+ * How many candidates the context held before any of them were scored.
1625
+ *
1626
+ * A pool of one means two opposite things. Either the vocabulary offered a
1627
+ * single continuation of what was typed, which is the least contested case
1628
+ * there is; or rivals were offered and have not been read yet, which is the
1629
+ * least informed. Only the second deserves refusing, and the two are
1630
+ * distinguishable exactly here.
1631
+ */
1632
+ var requestedPoolSize = function requestedPoolSize(candidate) {
1633
+ var _runtimeBySurface$get2, _contextRequestedCoun;
1634
+ var contextKey = (_runtimeBySurface$get2 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get2 === void 0 ? void 0 : _runtimeBySurface$get2.contextKey;
1635
+ return contextKey === undefined ? 0 : (_contextRequestedCoun = contextRequestedCount.get(contextKey)) !== null && _contextRequestedCoun !== void 0 ? _contextRequestedCoun : 0;
1636
+ };
1637
+
1638
+ // A suppressed short completion still counts as competition below, so losing
1639
+ // to one abstains rather than promoting the runner-up in its place.
1640
+ var gateCleared = evidenceBacked.filter(isWorthShowing).map(function (candidate) {
1641
+ return {
1642
+ candidate: candidate,
1643
+ posterior: judgedPosterior(candidate),
1644
+ score: exactConfidenceScore(candidate)
1645
+ };
1646
+ })
1647
+ // The gate is the model's own confidence in the surface; the blended score
1648
+ // only orders what has already cleared it, so a strong corpus prior can no
1649
+ // longer carry a surface the model is unsure of onto the screen.
1650
+ .filter(function (_ref6) {
1651
+ var candidate = _ref6.candidate,
1652
+ posterior = _ref6.posterior;
1653
+ return posterior >= MIN_LM_POSTERIOR[candidate.termType];
1654
+ });
1655
+ /**
1656
+ * Whether the model finds the surface plausible on its own terms.
1657
+ *
1658
+ * The gate above asks which candidate won its pool; this asks whether
1659
+ * winning it meant anything. The two come apart exactly where a pool is
1660
+ * thin — a shortlist of one hands its only member 1.0 by construction — and
1661
+ * a relative test can never see through that however it is tuned.
1662
+ *
1663
+ * Waived without canonical scoring, where there is no judged mean to test
1664
+ * and the network-logit path would otherwise refuse everything.
1665
+ */
1666
+ var isPlausibleSurface = function isPlausibleSurface(candidate) {
1667
+ var _judgedEvidenceFor2;
1668
+ if (!canonicalLmSupported) {
1669
+ return true;
1670
+ }
1671
+ var mean = (_judgedEvidenceFor2 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor2 === void 0 ? void 0 : _judgedEvidenceFor2.mean;
1672
+ return mean !== undefined && mean >= MIN_MEAN_TOKEN_LOG_PROBABILITY;
1673
+ };
1674
+ // Kept as its own stage rather than folded into the gate so the three
1675
+ // populations stay separable in the abstain cascade: cleared its pool and was
1676
+ // implausible, was plausible and had nothing to clear the pool against, or
1677
+ // cleared both. A candidate refused here stays in `ranked` and so still
1678
+ // counts as competition below — promoting the runner-up in place of an
1679
+ // implausible leader would show something worse, not something better.
1680
+ var plausible = gateCleared.filter(function (_ref7) {
1681
+ var candidate = _ref7.candidate;
1682
+ return isPlausibleSurface(candidate);
1683
+ });
1684
+ // Applied after the gate rather than folded into it, so the two populations
1685
+ // stay separable: a surface refused here cleared its threshold and was
1686
+ // refused for having had nothing to clear it against.
1687
+ var eligible = plausible.filter(function (_ref8) {
1688
+ var candidate = _ref8.candidate;
1689
+ return !canonicalLmSupported || scoredPoolSize(candidate) >= MIN_SCORED_POOL_SIZE || requestedPoolSize(candidate) <= 1;
1690
+ }).sort(function (a, b) {
1691
+ return b.score - a.score;
1692
+ });
1693
+ var selected = (_eligible$ = eligible[0]) === null || _eligible$ === void 0 ? void 0 : _eligible$.candidate;
1694
+ var selectedDisplayScore = (_eligible$0$score = (_eligible$2 = eligible[0]) === null || _eligible$2 === void 0 ? void 0 : _eligible$2.score) !== null && _eligible$0$score !== void 0 ? _eligible$0$score : 0;
1695
+ var selectedEvidence = selected && hasRequiredEvidence(selected) ? selected.lmEvidence : null;
1696
+ /**
1697
+ * Whether `longer` is `shorter` continued past a word boundary, in the same
1698
+ * normaliser.
1699
+ *
1700
+ * Shared by the two places that have to agree on what an extension is: the
1701
+ * chain rule, which stops dividing a pool's mass between a sequence and its
1702
+ * own prefix, and the winner margin below, which stops treating one as the
1703
+ * other's rival. They were the same category error and are now the same test.
1704
+ */
1705
+ var extendsInSamePool = function extendsInSamePool(shorter, longer) {
1706
+ var _runtimeBySurface$get3, _runtimeBySurface$get4;
1707
+ if (longer.word === shorter.word || !optimisticTotalByWord.has(longer.word)) {
1708
+ return false;
1709
+ }
1710
+ var pool = (_runtimeBySurface$get3 = runtimeBySurface.get(shorter.word)) === null || _runtimeBySurface$get3 === void 0 ? void 0 : _runtimeBySurface$get3.contextKey;
1711
+ if (pool === undefined || ((_runtimeBySurface$get4 = runtimeBySurface.get(longer.word)) === null || _runtimeBySurface$get4 === void 0 ? void 0 : _runtimeBySurface$get4.contextKey) !== pool) {
1712
+ return false;
1713
+ }
1714
+ return longer.word.length > shorter.word.length && longer.word.startsWith(shorter.word) && longer.word[shorter.word.length] === ' ';
1715
+ };
1716
+ // A candidate that extends the selection is not competing with it. Whichever
1717
+ // continuation the user meant, the shorter surface is a correct ghost — it is
1718
+ // the prefix of both — so the margin has nothing to arbitrate and charging the
1719
+ // selection for its own extension abstains on a case that could not be wrong.
1720
+ // The chain rule already removed this from the denominator; leaving it in the
1721
+ // margin meant a nested leader cleared its gate and then lost to the candidate
1722
+ // that had just been freed from dividing its mass. It penalised bigrams and
1723
+ // phrases specifically, because a nested pool is where they mostly appear.
1724
+ var runnerUpScore = selected ? Math.max.apply(Math, _toConsumableArray(ranked.filter(function (candidate) {
1725
+ return candidate.word !== selected.word && isStabilized(candidate) && !extendsInSamePool(selected, candidate);
1726
+ }).map(confidenceScore)).concat([0])) : 0;
1727
+ var winnerMargin = selected ? selectedDisplayScore - runnerUpScore : 0;
1728
+ var hasMissingArtifactCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1729
+ var runtime = runtimeBySurface.get(candidate.word);
1730
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) === 'whitespace' && runtime.canonicalTokenIds === null;
1731
+ });
1732
+ // A candidate with canonical tokens but no evidence at all cannot join the
1733
+ // posterior: it has no likelihood to contribute, and any stand-in value would
1734
+ // distort the normaliser for every other candidate in its context. Absent
1735
+ // evidence is not evidence of absence though — the context's boundary may
1736
+ // simply not be primed yet — so it holds the ghost back instead of being
1737
+ // silently ignored. In a primed context every candidate has at least
1738
+ // first-token evidence, so this only fires while a context is still cold.
1739
+ var hasUnscoredCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1740
+ var runtime = runtimeBySurface.get(candidate.word);
1741
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !optimisticTotalByWord.has(candidate.word) && isStabilized(candidate) && isWorthShowing(candidate);
1742
+ });
1743
+ // A rival only holds the ghost back while it is still too unread to judge.
1744
+ // Under full-surface evidence that meant any long surface blocked everything
1745
+ // until every one of its tokens had been paid for, which abstained far more
1746
+ // often than it ever changed the winner. A rival now becomes judgeable after
1747
+ // `REQUIRED_VERIFIED_TOKENS`, so this settles within the budget.
1748
+ var hasUnresolvedPotential = canonicalLmSupported && ranked.some(function (candidate) {
1749
+ var runtime = runtimeBySurface.get(candidate.word);
1750
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !hasJudgeableEvidence(candidate) && couldClearGate(candidate) && isStabilized(candidate) && isWorthShowing(candidate);
1751
+ });
1752
+ var requiredWinnerMargin = MIN_WINNER_MARGIN + (selected && ((_judgedEvidenceFor3 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor3 === void 0 ? void 0 : _judgedEvidenceFor3.complete) === false ? PARTIAL_EVIDENCE_MARGIN_PREMIUM : 0);
1753
+ // Everything the display decision needs except the veto itself. Splitting it
1754
+ // out is what lets the veto be priced: the value here at the moment the veto
1755
+ // fires is exactly the ghost the veto cost us.
1756
+ var clearsMarginBeforeColdVeto = selected !== undefined && !hasMissingArtifactCompetitor && winnerMargin >= requiredWinnerMargin;
1757
+ var clearsWinnerMargin = clearsMarginBeforeColdVeto && !hasUnscoredCompetitor;
1758
+ var suggestion = displayPrefixReady && selected && clearsWinnerMargin ? selected.word.slice((_prefixLenByWord$get2 = prefixLenByWord.get(selected.word)) !== null && _prefixLenByWord$get2 !== void 0 ? _prefixLenByWord$get2 : currentWord.length) : null;
1759
+
1760
+ // Which constraint bound. Computed on every evaluation rather than only under
1761
+ // debug, because it is counted for the session as well as printed, and
1762
+ // deriving the debug header from it keeps the two from disagreeing.
1763
+ var resolveAbstainReason = function resolveAbstainReason() {
1764
+ if (suggestion !== null && suggestion.length > 0) {
1765
+ return null;
1766
+ }
1767
+ if (!displayPrefixReady) {
1768
+ return 'prefetch';
1769
+ }
1770
+ if (ranked.length === 0) {
1771
+ return 'no-candidate';
1772
+ }
1773
+ if (hasMissingArtifactCompetitor) {
1774
+ return 'missing-artifact';
1775
+ }
1776
+ if (hasUnscoredCompetitor) {
1777
+ return 'cold-competitor';
1778
+ }
1779
+ if (hasUnresolvedPotential && !clearsWinnerMargin) {
1780
+ return 'unresolved-rival';
1781
+ }
1782
+ if (eligible.length > 0) {
1783
+ return selected && !clearsWinnerMargin ? 'winner-margin' : 'empty-completion';
1784
+ }
1785
+ if (plausible.length > 0) {
1786
+ // Cleared its gate, was plausible on its own terms, and was refused for
1787
+ // the shortlist it cleared that gate against holding nobody else.
1788
+ return 'lone-candidate';
1789
+ }
1790
+ if (gateCleared.length > 0) {
1791
+ // Held the largest share of its shortlist and still was not a surface
1792
+ // the model found plausible — which is only possible because a share is
1793
+ // relative and this is not.
1794
+ return 'implausible-surface';
1795
+ }
1796
+ if (evidenceBacked.length > 0 && evidenceBacked.every(function (candidate) {
1797
+ return !isWorthShowing(candidate);
1798
+ })) {
1799
+ return 'short-completion';
1800
+ }
1801
+ // The floor and the gate are one threshold, applied to the bound and to
1802
+ // the verified value, so a candidate short of it is dropped before
1803
+ // `evidenceBacked` is built. Asking whether anything was judgeable
1804
+ // separates a gate that rejected read candidates from a context where
1805
+ // nothing was read at all — the first is a threshold to calibrate, the
1806
+ // second is scheduling.
1807
+ return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
1808
+ };
1809
+ var abstainReason = resolveAbstainReason();
1810
+
1811
+ // The leader is the best-supported candidate, not the selected one: an
1812
+ // evaluation that showed nothing is exactly the one whose posterior needs
1813
+ // recording, and it has no selection to report.
1814
+ var posteriorLeader = ranked.reduce(function (best, candidate) {
1815
+ var posterior = optimisticPosterior(candidate);
1816
+ return posterior > 0 && (best === null || posterior > best.posterior) ? {
1817
+ candidate: candidate,
1818
+ posterior: posterior
1819
+ } : best;
1820
+ }, null);
1821
+ /**
1822
+ * The longer surface in the leader's normaliser that extends the given
1823
+ * candidate, if the pool holds one.
1824
+ *
1825
+ * `logSumExp` divides one pool's mass between a sequence and its own prefix,
1826
+ * and the longer total is the shorter one plus a log-probability that cannot
1827
+ * be positive — so the longer form holds at most half, or at most a third
1828
+ * where both shorter forms are present. Both sit under the gate it is held
1829
+ * to, so it cannot display however certain its continuation is.
1830
+ *
1831
+ * Asked from the shorter form because that is the one that leads: the
1832
+ * extending candidate is strictly lower in the same pool and so can never be
1833
+ * the leader.
1834
+ */
1835
+ var poolExtensionOf = function poolExtensionOf(shorter) {
1836
+ return ranked.find(function (candidate) {
1837
+ return extendsInSamePool(shorter, candidate);
1838
+ });
1839
+ };
572
1840
  if (isAutocompleteDebugEnabled()) {
1841
+ var _posteriorLeader$cand, _posteriorLeader$post, _selectedEvidence$mea, _selectedEvidence$mea2, _acceptCooldown$predi, _acceptCooldown;
1842
+ var verbose = isAutocompleteDebugVerbose();
573
1843
  var latencyMs = performance.now() - t0;
574
1844
  var tokens = tokenize(trimmed);
575
1845
  var contextWords = tokens.slice(-CONTEXT_WORDS);
576
- var belowThreshold = best && best.finalScore < MIN_SCORE_THRESHOLD;
577
- var suggestionLabel = belowThreshold ? '🚫 (below threshold)' : suggestion && suggestion.length > 0 ? "\u2728 \"".concat(suggestion, "\"") : '🚫 (no match)';
1846
+
1847
+ // Decision drives the (collapsed) group header. Abstains carry the reason
1848
+ // the session counters recorded, plus the numbers behind it, so it is clear
1849
+ // *why* nothing showed without expanding the group.
1850
+ var abstainDetail = {
1851
+ 'below-posterior-gate': "posterior below ".concat(MIN_LM_POSTERIOR[((_posteriorLeader$cand = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.candidate) !== null && _posteriorLeader$cand !== void 0 ? _posteriorLeader$cand : ranked[0]).termType], " (best ").concat(((_posteriorLeader$post = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.posterior) !== null && _posteriorLeader$post !== void 0 ? _posteriorLeader$post : 0).toFixed(2), ")"),
1852
+ 'cold-competitor': 'competitor has no LM evidence yet',
1853
+ 'empty-completion': 'empty completion',
1854
+ 'implausible-surface': "mean per-token log-probability below ".concat(MIN_MEAN_TOKEN_LOG_PROBABILITY, " (best ").concat(gateCleared.map(function (_ref9) {
1855
+ var _judgedEvidenceFor$me, _judgedEvidenceFor4;
1856
+ var candidate = _ref9.candidate;
1857
+ return (_judgedEvidenceFor$me = (_judgedEvidenceFor4 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor4 === void 0 ? void 0 : _judgedEvidenceFor4.mean) !== null && _judgedEvidenceFor$me !== void 0 ? _judgedEvidenceFor$me : -Infinity;
1858
+ }).reduce(function (best, mean) {
1859
+ return Math.max(best, mean);
1860
+ }, -Infinity).toFixed(2), ")"),
1861
+ 'lone-candidate': "cleared its gate against fewer than ".concat(MIN_SCORED_POOL_SIZE, " scored candidates, rivals unread"),
1862
+ 'missing-artifact': 'canonical artifact coverage missing',
1863
+ 'no-candidate': 'nothing cleared scoring',
1864
+ 'no-evidence': 'full-surface evidence absent',
1865
+ prefetch: "prefetch: ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " chars"),
1866
+ 'short-completion': "completion shorter than ".concat(MIN_SUGGESTION_LENGTH, " chars"),
1867
+ 'unresolved-rival': 'expanding plausible token-prefix groups',
1868
+ 'winner-margin': "winner margin ".concat(winnerMargin.toFixed(2), " < ").concat(requiredWinnerMargin.toFixed(2))
1869
+ };
1870
+ var decision = abstainReason === null ? "\u2728 \"".concat(suggestion !== null && suggestion !== void 0 ? suggestion : '', "\"") : abstainReason === 'prefetch' ? "\u23F3 ".concat(abstainDetail.prefetch) : "\u2014 abstain: ".concat(abstainDetail[abstainReason]);
578
1871
  lastPredictionDebug = {
1872
+ awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
579
1873
  textBefore: trimmed,
580
1874
  currentWord: currentWord,
1875
+ decision: decision,
581
1876
  mode: mode,
582
1877
  contextWords: contextWords,
583
1878
  topCandidates: ranked.slice(0, 5).map(function (r) {
@@ -592,55 +1887,208 @@ export var predict = function predict(textBefore) {
592
1887
  suggestion: suggestion && suggestion.length > 0 ? suggestion : null
593
1888
  };
594
1889
 
595
- // ── Mode label: COLD / WARM(local) / WARM(BE) ───────────────────────
1890
+ // Mode describes the semantic-vector source without implying that a
1891
+ // network backend ran; the snapshot may come from the local MLC embedder.
596
1892
  var slowLaneVec = getStoredContextVector();
597
1893
  var isUsingSlowLaneVector = slowLaneVec !== null && vectorStore !== null && slowLaneVec.length === vectorStore.dim;
598
- var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(BE)' : 'WARM(local)';
599
- var modeColor = modeLabel === 'WARM(BE)' ? 'color: #ff9800; font-weight: bold;' : modeLabel === 'WARM(local)' ? 'color: #4caf50; font-weight: bold;' : 'color: #9e9e9e; font-weight: bold;';
1894
+ var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(semantic)' : 'WARM(local-avg)';
1895
+ var modeColor = modeLabel === 'WARM(semantic)' ? CTC_STYLES.good : modeLabel === 'WARM(local-avg)' ? CTC_STYLES.warn : CTC_STYLES.cold;
600
1896
 
601
- // 1. Collapsible group header
1897
+ // Group header: [CTC:<mode>] <decision> · <mode> · <type> · <latency>
1898
+ // arbitrationMode ('v1') tags the line so this deploy's logs diff cleanly
1899
+ // against the baseline deploy's.
602
1900
  // eslint-disable-next-line no-console
603
- console.groupCollapsed("%c[Autocomplete] %c".concat(modeLabel, " %c| \"").concat(currentWord, "\" \u2794 ").concat(suggestionLabel, " | \u23F1 ").concat(latencyMs.toFixed(1), "ms"), 'color: #00b8d9; font-weight: bold;', modeColor, 'color: inherit; font-weight: normal;');
1901
+ console.groupCollapsed("%c[CTC:".concat(ARBITRATION_MODE, "]%c ").concat(decision, " %c\xB7 ").concat(modeLabel).concat(selected ? " \xB7 ".concat(selected.termType) : '', " \xB7 \u23F1 ").concat(latencyMs.toFixed(1), "ms"), CTC_STYLES.brand, CTC_STYLES.body, modeColor);
1902
+ var semanticInput = getStoredContextInput();
1903
+ var causalPrompts = primeRequests.map(function (request, index) {
1904
+ var tail = request.prompt.slice(-DEBUG_TEXT_TAIL_CHARS);
1905
+ return "#".concat(index + 1, ":").concat(JSON.stringify(tail));
1906
+ });
604
1907
 
605
- // 2. Context window (what local vector averaging sees)
606
- // eslint-disable-next-line no-console
607
- console.log('%cContext Window:', 'color: #888; font-style: italic;', contextWords.length ? contextWords.join(' ') : '(none)');
1908
+ // Keep the exact model inputs separate from normalized lookup tokens.
1909
+ ctcSection('INPUT', "raw: ".concat(JSON.stringify(trimmed.slice(-DEBUG_TEXT_TAIL_CHARS)), " \xB7 prev: ").concat(previousWord ? JSON.stringify(previousWord) : '—'));
1910
+ ctcSection('LOOKUP', "normalized(last ".concat(CONTEXT_WORDS, "): ").concat(contextWords.length ? JSON.stringify(contextWords.join(' ')) : '(none)'));
1911
+ ctcSection('SEMANTIC', "snapshot: ".concat(semanticInput ? JSON.stringify(semanticInput.slice(-DEBUG_TEXT_TAIL_CHARS)) : '(unavailable)'));
1912
+ ctcSection('CAUSAL', "prompt".concat(causalPrompts.length === 1 ? '' : 's', ": ").concat(causalPrompts.join(' · ') || '(not ready)'));
608
1913
 
609
- // 3. Slow-lane status
610
- var vectorStatus = isUsingSlowLaneVector ? "\u2705 BE semantic vector (dim=".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, ")") : vectorStore ? '⚠️ Local vector average (slow-lane not yet returned)' : '❌ No vectors (cold)';
611
- var logitsStatus = lmLogits && Object.keys(lmLogits).length > 0 ? "\u2705 LM logits active (".concat(Object.keys(lmLogits).length, " tokens)") : '⏳ No LM logits (slow-lane pending or failed)';
612
- // eslint-disable-next-line no-console
613
- console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
1914
+ // SIGNALS — semantic state plus context-keyed canonical LM coverage.
1915
+ var logitCount = lmLogits ? Object.keys(lmLogits).length : 0;
1916
+ var semanticStatus = isUsingSlowLaneVector ? "semantic \u2705 snapshot(".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, "d)") : vectorStore ? 'semantic ⚠️ local-avg' : 'semantic ❌ cold';
1917
+ var tierACount = ranked.filter(function (candidate) {
1918
+ var _candidate$lmEvidence3;
1919
+ return ((_candidate$lmEvidence3 = candidate.lmEvidence) === null || _candidate$lmEvidence3 === void 0 ? void 0 : _candidate$lmEvidence3.source) === 'canonical-first-token';
1920
+ }).length;
1921
+ var exactCount = ranked.filter(function (candidate) {
1922
+ var _candidate$lmEvidence4;
1923
+ return ((_candidate$lmEvidence4 = candidate.lmEvidence) === null || _candidate$lmEvidence4 === void 0 ? void 0 : _candidate$lmEvidence4.source) === 'canonical-full-surface';
1924
+ }).length;
1925
+ var boundaryReady = primeRequests.filter(function (request) {
1926
+ return getBoundaryLmState(request.contextKey) !== null;
1927
+ }).length;
1928
+ var artifactEligible = canonicalMatched.filter(function (candidate) {
1929
+ return candidate.canonicalTokenIds !== null;
1930
+ }).length;
1931
+ var documentStartFallbacks = canonicalMatched.filter(function (candidate) {
1932
+ return candidate.separatorKind === 'document-start';
1933
+ }).length;
1934
+ var punctuationFallbacks = canonicalMatched.filter(function (candidate) {
1935
+ return candidate.separatorKind === 'non-space';
1936
+ }).length;
1937
+ var logitsStatus = tierACount + exactCount > 0 ? "canonical \u2705 first-token:".concat(tierACount, " exact:").concat(exactCount) : logitCount > 0 ? "network logits \u2705 ".concat(logitCount) : 'canonical ⏳/absent';
1938
+ var formula = tierACount + exactCount + logitCount > 0 ? "Stage1\xD7".concat(STAGE1_WEIGHT, "+LM\xD7").concat(STAGE2_WEIGHT) : 'Stage1-only';
1939
+ ctcSection('SIGNALS', "".concat(semanticStatus, " \xB7 ").concat(logitsStatus, " \xB7 ").concat(formula));
1940
+ ctcSection('CANONICAL', "eligible ".concat(artifactEligible, "/").concat(canonicalMatched.length, " \xB7 separator fallback doc:").concat(documentStartFallbacks, " punct:").concat(punctuationFallbacks, " \xB7 contexts ").concat(boundaryReady, "/").concat(primeRequests.length, " ready (\u2264").concat(PHRASE_MAX_WORDS, ") \xB7 family=").concat(familyKey.slice(0, 72) || 'none'));
1941
+ if (verbose && logitCount > 0 && lmLogits) {
1942
+ var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
1943
+ return b[1] - a[1];
1944
+ }).slice(0, 5).map(function (_ref0) {
1945
+ var _ref1 = _slicedToArray(_ref0, 2),
1946
+ word = _ref1[0],
1947
+ score = _ref1[1];
1948
+ return "".concat(word, ":").concat(score.toFixed(3));
1949
+ }).join(', ');
1950
+ ctcSection(' rawLM', "\uD83E\uDDE0 ".concat(rawLmTop));
1951
+ }
614
1952
 
615
- // 4. Scoring formula active this prediction
616
- var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? "Stage1(\xD7".concat(STAGE1_WEIGHT, ") + LM(\xD7").concat(STAGE2_WEIGHT, ")") : 'Stage1 only (no LM logits)';
617
- // eslint-disable-next-line no-console
618
- console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
1953
+ // GENERATE — how many candidates matched, split by term type
1954
+ var genByType = {
1955
+ word: 0,
1956
+ bigram: 0,
1957
+ phrase: 0
1958
+ };
1959
+ var _iterator19 = _createForOfIteratorHelper(canonicalMatched),
1960
+ _step19;
1961
+ try {
1962
+ for (_iterator19.s(); !(_step19 = _iterator19.n()).done;) {
1963
+ var m = _step19.value;
1964
+ genByType[m.node.termType] += 1;
1965
+ }
1966
+ } catch (err) {
1967
+ _iterator19.e(err);
1968
+ } finally {
1969
+ _iterator19.f();
1970
+ }
1971
+ ctcSection('GENERATE', "matched ".concat(canonicalMatched.length, " \u2192 word:").concat(genByType.word, " bigram:").concat(genByType.bigram, " phrase:").concat(genByType.phrase).concat(canonicalLmSupported ? ' · display needs exact surface evidence' : ''));
619
1972
 
620
- // 5. Grammar filter result (collected inside rankCandidates, logged here)
1973
+ // SCORE — Stage-1 + grammar funnel and canonical evidence depth.
1974
+ var lmCoverage = ranked.slice(0, 10).filter(function (r) {
1975
+ return r.lmScore > 0;
1976
+ }).length;
1977
+ ctcSection('SCORE', "in ".concat(pipelineDebug.initial, " \u2192 stage1(<").concat(MIN_STAGE1_SCORE, ") \u2212").concat(pipelineDebug.stage1Rejected.length, " \u2192 grammar \u2212").concat(pipelineDebug.grammarRejected.length, " \u2192 final ").concat(pipelineDebug.final, " \xB7 LM cov ").concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " \xB7 first-token ").concat(tierACount, " \xB7 exact ").concat(exactCount));
621
1978
  if (grammarMeta) {
622
- // eslint-disable-next-line no-console
623
- console.log("%c[Grammar] \"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] \u2192 ").concat(grammarMeta.before, " candidates \u2192 ").concat(grammarMeta.after, " after filter"), 'color: #4caf50; font-weight: bold;');
624
- if (grammarMeta.dropped.length > 0) {
625
- // eslint-disable-next-line no-console
626
- console.log("%c\uD83D\uDEAB Dropped: ".concat(grammarMeta.dropped.join(', ')), 'color: #f44336; font-style: italic;');
627
- }
1979
+ ctcSection(' grammar', "\"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] ").concat(grammarMeta.before, "\u2192").concat(grammarMeta.after).concat(verbose && grammarMeta.dropped.length > 0 ? " \xB7 dropped: ".concat(grammarMeta.dropped.join(', ')) : ''));
628
1980
  }
1981
+ var gateReasonFor = function gateReasonFor(candidate) {
1982
+ var runtime = runtimeBySurface.get(candidate.word);
1983
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace') {
1984
+ var _runtime$separatorKin;
1985
+ return "separator:".concat((_runtime$separatorKin = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin !== void 0 ? _runtime$separatorKin : 'unknown');
1986
+ }
1987
+ if (runtime.canonicalTokenIds === null) {
1988
+ return 'missing-artifact';
1989
+ }
1990
+ if (!hasJudgeableEvidence(candidate)) {
1991
+ return "awaiting-tokens:".concat(REQUIRED_VERIFIED_TOKENS);
1992
+ }
1993
+ if (!passesLmFloor(candidate)) {
1994
+ return "bound:".concat(optimisticPosterior(candidate).toFixed(2), "<").concat(MIN_LM_POSTERIOR[candidate.termType]);
1995
+ }
1996
+ if (duplicatesPreceding(candidate)) {
1997
+ return 'repetition';
1998
+ }
1999
+ if (isCooledDown(candidate)) {
2000
+ return 'cooldown';
2001
+ }
2002
+ if (!isWorthShowing(candidate)) {
2003
+ return "suffix:".concat(suggestionLengthFor(candidate), "<").concat(MIN_SUGGESTION_LENGTH);
2004
+ }
2005
+ var floor = MIN_LM_POSTERIOR[candidate.termType];
2006
+ var posterior = judgedPosterior(candidate);
2007
+ if (posterior < floor) {
2008
+ return "posterior:".concat(posterior.toFixed(2), "<").concat(floor);
2009
+ }
2010
+ if (!isPlausibleSurface(candidate)) {
2011
+ var _judgedEvidenceFor$me2, _judgedEvidenceFor5;
2012
+ return "implausible:".concat(((_judgedEvidenceFor$me2 = (_judgedEvidenceFor5 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor5 === void 0 ? void 0 : _judgedEvidenceFor5.mean) !== null && _judgedEvidenceFor$me2 !== void 0 ? _judgedEvidenceFor$me2 : NaN).toFixed(2), "<").concat(MIN_MEAN_TOKEN_LOG_PROBABILITY);
2013
+ }
2014
+ return 'eligible';
2015
+ };
2016
+ var gateFunnel = function gateFunnel(termType) {
2017
+ var typeRanked = ranked.filter(function (candidate) {
2018
+ return candidate.termType === termType;
2019
+ });
2020
+ var exact = typeRanked.filter(hasJudgeableEvidence);
2021
+ var absolute = exact.filter(passesLmFloor);
2022
+ var stabilized = absolute.filter(isStabilized);
2023
+ var longEnough = stabilized.filter(isWorthShowing);
2024
+ var floorPassed = longEnough.filter(function (candidate) {
2025
+ return judgedPosterior(candidate) >= MIN_LM_POSTERIOR[termType];
2026
+ });
2027
+ var plausiblePassed = floorPassed.filter(isPlausibleSurface);
2028
+ return "".concat(termType, " m:").concat(genByType[termType], " r:").concat(typeRanked.length, " exact:").concat(exact.length, " abs:").concat(absolute.length, " stable:").concat(stabilized.length, " suffix:").concat(longEnough.length, " floor:").concat(floorPassed.length, " plausible:").concat(plausiblePassed.length, " eligible:").concat(eligible.filter(function (_ref10) {
2029
+ var candidate = _ref10.candidate;
2030
+ return candidate.termType === termType;
2031
+ }).length);
2032
+ };
2033
+ ctcSection('GATES', "".concat(gateFunnel('word'), " | ").concat(gateFunnel('bigram'), " | ").concat(gateFunnel('phrase')));
629
2034
 
630
- // 6. Pipeline funnel
631
- // eslint-disable-next-line no-console
632
- console.log("%c[Pipeline Funnel] %c\uD83D\uDCE5 In: ".concat(pipelineDebug.initial, " | \u274C Stage 1 (< ").concat(MIN_STAGE1_SCORE, "): -").concat(pipelineDebug.stage1Rejected.length, " | \u274C Grammar: -").concat(pipelineDebug.grammarRejected.length, " | \u2705 Final: ").concat(pipelineDebug.final), 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
2035
+ // One line per context: how many candidates share the normaliser, and how
2036
+ // much of the mass the leader holds. A leader well under its threshold
2037
+ // means the context is contested, which is the abstention we want.
2038
+ var contextLeaders = Array.from(bestCandidateByContext.entries()).slice(0, PHRASE_MAX_WORDS).map(function (_ref11) {
2039
+ var _contextTotals$get$le2, _contextTotals$get2;
2040
+ var _ref12 = _slicedToArray(_ref11, 2),
2041
+ contextKey = _ref12[0],
2042
+ leader = _ref12[1];
2043
+ var shortlistSize = (_contextTotals$get$le2 = (_contextTotals$get2 = contextTotals.get(contextKey)) === null || _contextTotals$get2 === void 0 ? void 0 : _contextTotals$get2.length) !== null && _contextTotals$get$le2 !== void 0 ? _contextTotals$get$le2 : 0;
2044
+ var evidenceKind = hasExactEvidence(leader) ? 'exact' : 'upper';
2045
+ return "".concat(contextKey.slice(0, 32), " \u2192 n=").concat(shortlistSize, " ").concat(leader.termType, ":\"").concat(leader.word, "\" ").concat(evidenceKind, " p=").concat(optimisticPosterior(leader).toFixed(3), "/").concat(MIN_LM_POSTERIOR[leader.termType]);
2046
+ }).join(' | ');
2047
+ ctcSection('NORMALIZE', contextLeaders.length > 0 ? contextLeaders : 'no scored candidate in any context');
2048
+ ctcSection('READINESS', "prefix ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " \xB7 boundary ").concat(boundaryReady, "/").concat(primeRequests.length, " \xB7 exact word:").concat(ranked.filter(function (candidate) {
2049
+ return candidate.termType === 'word' && hasExactEvidence(candidate);
2050
+ }).length, " bigram:").concat(ranked.filter(function (candidate) {
2051
+ return candidate.termType === 'bigram' && hasExactEvidence(candidate);
2052
+ }).length, " phrase:").concat(ranked.filter(function (candidate) {
2053
+ return candidate.termType === 'phrase' && hasExactEvidence(candidate);
2054
+ }).length, " \xB7 ").concat(suggestion ? 'display-ready' : decision));
633
2055
 
634
- // 7. Candidate table
635
- if (ranked.length > 0) {
636
- var lmCoverage = ranked.slice(0, 10).filter(function (r) {
637
- return r.lmScore > 0;
638
- }).length;
639
- // eslint-disable-next-line no-console
640
- console.log("%cLM coverage: ".concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " candidates had real logit scores"), 'color: #888; font-style: italic;');
2056
+ // ARBITRATE — expose every condition that can authorize or suppress ghost.
2057
+ var topOfType = function topOfType(t) {
2058
+ return ranked.find(function (c) {
2059
+ return c.termType === t;
2060
+ });
2061
+ };
2062
+ var fmtType = function fmtType(t) {
2063
+ var _top$lmEvidence$sourc, _top$lmEvidence;
2064
+ var top = topOfType(t);
2065
+ if (!top) {
2066
+ return "".concat(t, ":\u2014");
2067
+ }
2068
+ var floor = MIN_LM_POSTERIOR[t];
2069
+ var bound = optimisticPosterior(top);
2070
+ var evidence = hasRequiredEvidence(top) ? 'TierB✅' : "".concat((_top$lmEvidence$sourc = (_top$lmEvidence = top.lmEvidence) === null || _top$lmEvidence === void 0 ? void 0 : _top$lmEvidence.source) !== null && _top$lmEvidence$sourc !== void 0 ? _top$lmEvidence$sourc : 'absent', "\u274C");
2071
+ return "".concat(t, ":\"").concat(top.word, "\" p\u2264").concat(bound.toFixed(3)).concat(bound >= floor ? '≥' : '<').concat(floor, " fs\u2264").concat(confidenceScore(top).toFixed(2), " ").concat(evidence);
2072
+ };
2073
+ ctcSection('ARBITRATE', "".concat(selected ? "\u25B8 ".concat(selected.termType, " \"").concat(selected.word, "\" fs=").concat(selectedDisplayScore.toFixed(2), " mean=").concat((_selectedEvidence$mea = selectedEvidence === null || selectedEvidence === void 0 || (_selectedEvidence$mea2 = selectedEvidence.meanTokenLogProbability) === null || _selectedEvidence$mea2 === void 0 ? void 0 : _selectedEvidence$mea2.toFixed(2)) !== null && _selectedEvidence$mea !== void 0 ? _selectedEvidence$mea : 'network', " margin=").concat(winnerMargin.toFixed(2)).concat(clearsWinnerMargin ? '✅' : '❌') : '▸ (none)', " \xB7 prefix ").concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH).concat(displayPrefixReady ? '✅' : '⏳', " \xB7 grouped pending ").concat(hasUnresolvedPotential ? 'yes' : 'no', " | ").concat(fmtType('word'), " ").concat(fmtType('bigram'), " ").concat(fmtType('phrase')));
2074
+
2075
+ // STABILIZE — post-accept cooldown + whole-surface repetition (QI-2)
2076
+ var repetitionBlocked = ranked.filter(function (c) {
2077
+ return hasRequiredEvidence(c) && duplicatesPreceding(c);
2078
+ }).map(function (c) {
2079
+ return c.word;
2080
+ });
2081
+ var cooldownLabel = cooledSurface ? "cooldown \"".concat(cooledSurface, "\" (").concat((_acceptCooldown$predi = (_acceptCooldown = acceptCooldown) === null || _acceptCooldown === void 0 ? void 0 : _acceptCooldown.predictionsSince) !== null && _acceptCooldown$predi !== void 0 ? _acceptCooldown$predi : 0, "/").concat(COOLDOWN_KEYSTROKES, " keys \xB7 ").concat(COOLDOWN_MS, "ms)") : 'cooldown —';
2082
+ ctcSection('STABILIZE', "".concat(cooldownLabel, " \xB7 repetition-blocked: ").concat(repetitionBlocked.length > 0 ? repetitionBlocked.join(', ') : '—'));
2083
+
2084
+ // CANDIDATES — full scored table (verbose only)
2085
+ if (verbose && ranked.length > 0) {
641
2086
  var tableData = ranked.slice(0, 10).map(function (r) {
2087
+ var _r$lmEvidence, _contextTotals$get$le3, _contextTotals$get3, _r$lmEvidence$source, _r$lmEvidence2, _runtime$separatorKin2, _runtime$canonicalTok2, _runtime$canonicalTok3, _runtime$contextKey, _r$lmEvidence3, _r$lmEvidence4;
642
2088
  var rawLogit = 'Not in Payload';
643
- if (prefixLmLogits) {
2089
+ if (((_r$lmEvidence = r.lmEvidence) === null || _r$lmEvidence === void 0 ? void 0 : _r$lmEvidence.rawLogit) !== undefined) {
2090
+ rawLogit = Number(r.lmEvidence.rawLogit.toFixed(5));
2091
+ } else if (prefixLmLogits) {
644
2092
  var val = prefixLmLogits[r.word.toLowerCase()];
645
2093
  if (val !== undefined) {
646
2094
  rawLogit = Number(val.toFixed(5));
@@ -649,9 +2097,19 @@ export var predict = function predict(textBefore) {
649
2097
  var original = scoringCandidates.find(function (sc) {
650
2098
  return sc.word === r.word;
651
2099
  });
2100
+ var runtime = runtimeBySurface.get(r.word);
2101
+ var progress = runtime ? getProgressiveSurfaceEvidence(runtime.contextKey, r.word) : null;
2102
+ var optimisticTotal = optimisticTotalByWord.get(r.word);
2103
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
2104
+ var shortlistSize = runtime ? (_contextTotals$get$le3 = (_contextTotals$get3 = contextTotals.get(runtime.contextKey)) === null || _contextTotals$get3 === void 0 ? void 0 : _contextTotals$get3.length) !== null && _contextTotals$get$le3 !== void 0 ? _contextTotals$get$le3 : 0 : 0;
2105
+ var contextLeader = runtime ? bestCandidateByContext.get(runtime.contextKey) : undefined;
652
2106
  var source = 'Unknown';
653
2107
  if (original) {
654
- if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
2108
+ if (original.termType === 'bigram') {
2109
+ source = '🔗 L2 (Bigram)';
2110
+ } else if (original.termType === 'phrase') {
2111
+ source = '🧩 L2 (Phrase)';
2112
+ } else if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
655
2113
  source = '🌍 L3 (Generic)';
656
2114
  } else if (original.sessionFreq > 0 && original.tenantFreq === 0) {
657
2115
  source = '👤 L1 (Session Only)';
@@ -659,13 +2117,40 @@ export var predict = function predict(textBefore) {
659
2117
  source = '🏢 L2 (Domain)';
660
2118
  }
661
2119
  }
2120
+ var floor = MIN_LM_POSTERIOR[r.termType];
662
2121
  return {
663
2122
  Candidate: r.word,
664
2123
  Source: source,
2124
+ Type: r.termType,
665
2125
  'Final Score': Number(r.finalScore.toFixed(4)),
2126
+ 'Bounded Score': Number(confidenceScore(r).toFixed(4)),
2127
+ 'Exact Gate Score': hasExactEvidence(r) ? Number(exactConfidenceScore(r).toFixed(4)) : '—',
2128
+ Floor: floor,
2129
+ 'Posterior Floor': optimisticPosterior(r) >= floor ? '✅' : '❌',
2130
+ 'LM Floor': hasExactEvidence(r) ? passesLmFloor(r) ? '✅' : '❌' : '⏳',
2131
+ 'Gate Reason': gateReasonFor(r),
2132
+ 'Display Eligible': hasRequiredEvidence(r) && isStabilized(r) && judgedPosterior(r) >= floor ? '✅' : '❌',
666
2133
  Semantics: Number(r.semanticScore.toFixed(4)),
667
2134
  Freq: Number(r.freqScore.toFixed(4)),
668
2135
  'LM Score': Number(r.lmScore.toFixed(4)),
2136
+ 'LM Evidence': (_r$lmEvidence$source = (_r$lmEvidence2 = r.lmEvidence) === null || _r$lmEvidence2 === void 0 ? void 0 : _r$lmEvidence2.source) !== null && _r$lmEvidence$source !== void 0 ? _r$lmEvidence$source : 'absent',
2137
+ 'Verified Tokens': r.lmEvidence ? "".concat(r.lmEvidence.verifiedTokenCount, "/").concat(r.lmEvidence.totalTokenCount) : '0/0',
2138
+ 'Progressive Tokens': progress ? "".concat(progress.scoredTokenCount, "/").concat(progress.totalTokenCount) : '0/0',
2139
+ 'LM Mean Upper': progress !== null ? Number(progress.meanTokenLogProbabilityUpperBound.toFixed(5)) : '—',
2140
+ 'Optimistic Total': optimisticTotal !== undefined ? Number(optimisticTotal.toFixed(5)) : '—',
2141
+ 'Context logSumExp': normalizer !== undefined && Number.isFinite(normalizer) ? Number(normalizer.toFixed(5)) : '—',
2142
+ 'Shortlist Size': shortlistSize,
2143
+ 'Posterior (bound)': Number(optimisticPosterior(r).toFixed(5)),
2144
+ 'Posterior (judged)': hasJudgeableEvidence(r) ? Number(judgedPosterior(r).toFixed(5)) : '—',
2145
+ 'Context Leader': contextLeader ? "".concat(contextLeader.termType, ":").concat(contextLeader.word) : '—',
2146
+ 'Verified Chars': r.lmEvidence ? "".concat(r.lmEvidence.verifiedCharCount, "/").concat(r.lmEvidence.totalSurfaceCharCount) : "0/".concat(r.word.length),
2147
+ Separator: (_runtime$separatorKin2 = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin2 !== void 0 ? _runtime$separatorKin2 : 'unknown',
2148
+ 'Canonical Eligibility': (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace' ? 'incompatible-separator' : runtime.canonicalTokenIds === null ? 'missing-artifact' : 'eligible',
2149
+ 'Artifact Tokens': (_runtime$canonicalTok2 = runtime === null || runtime === void 0 || (_runtime$canonicalTok3 = runtime.canonicalTokenIds) === null || _runtime$canonicalTok3 === void 0 ? void 0 : _runtime$canonicalTok3.join(',')) !== null && _runtime$canonicalTok2 !== void 0 ? _runtime$canonicalTok2 : 'absent',
2150
+ 'Context Key': (_runtime$contextKey = runtime === null || runtime === void 0 ? void 0 : runtime.contextKey) !== null && _runtime$contextKey !== void 0 ? _runtime$contextKey : '—',
2151
+ 'Exact Total': ((_r$lmEvidence3 = r.lmEvidence) === null || _r$lmEvidence3 === void 0 ? void 0 : _r$lmEvidence3.totalLogProbability) !== undefined ? Number(r.lmEvidence.totalLogProbability.toFixed(5)) : '—',
2152
+ 'Exact Mean': ((_r$lmEvidence4 = r.lmEvidence) === null || _r$lmEvidence4 === void 0 ? void 0 : _r$lmEvidence4.meanTokenLogProbability) !== undefined ? Number(r.lmEvidence.meanTokenLogProbability.toFixed(5)) : '—',
2153
+ Assoc: typeof (original === null || original === void 0 ? void 0 : original.association) === 'number' ? Number(original.association.toFixed(4)) : '—',
669
2154
  'Raw Logit': rawLogit,
670
2155
  'Session Freq': (original === null || original === void 0 ? void 0 : original.sessionFreq) || 0
671
2156
  };
@@ -673,23 +2158,120 @@ export var predict = function predict(textBefore) {
673
2158
 
674
2159
  // eslint-disable-next-line no-console
675
2160
  console.table(tableData);
676
- } else {
677
- // eslint-disable-next-line no-console
678
- console.log('No candidates found.');
679
2161
  }
680
2162
 
681
2163
  // eslint-disable-next-line no-console
682
2164
  console.groupEnd();
683
2165
  }
684
- return suggestion && suggestion.length > 0 ? suggestion : null;
2166
+ var selectedContextKey = selected ? (_runtimeBySurface$get5 = runtimeBySurface.get(selected.word)) === null || _runtimeBySurface$get5 === void 0 ? void 0 : _runtimeBySurface$get5.contextKey : undefined;
2167
+ return suggestion && suggestion.length > 0 && selected && selectedEvidence ? {
2168
+ text: suggestion,
2169
+ surface: selected.word,
2170
+ termType: selected.termType,
2171
+ posterior: judgedPosterior(selected),
2172
+ meanTokenLogProbability: (_judgedEvidenceFor$me3 = (_judgedEvidenceFor6 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor6 === void 0 ? void 0 : _judgedEvidenceFor6.mean) !== null && _judgedEvidenceFor$me3 !== void 0 ? _judgedEvidenceFor$me3 : NaN,
2173
+ poolHeldExtension: poolExtensionOf(selected) !== undefined,
2174
+ shortlistSize: selectedContextKey === undefined ? 1 : (_contextTotals$get$le4 = (_contextTotals$get4 = contextTotals.get(selectedContextKey)) === null || _contextTotals$get4 === void 0 ? void 0 : _contextTotals$get4.length) !== null && _contextTotals$get$le4 !== void 0 ? _contextTotals$get$le4 : 1,
2175
+ rankScore: selectedDisplayScore,
2176
+ evidenceTier: (selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.source) === 'network-logit' ? 'network-logit' : 'canonical-full-surface',
2177
+ evidenceDepth: {
2178
+ verifiedTokens: (_selectedEvidence$ver = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedTokenCount) !== null && _selectedEvidence$ver !== void 0 ? _selectedEvidence$ver : 0,
2179
+ totalTokens: (_selectedEvidence$tot = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.totalTokenCount) !== null && _selectedEvidence$tot !== void 0 ? _selectedEvidence$tot : 0,
2180
+ verifiedChars: (_selectedEvidence$ver2 = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedCharCount) !== null && _selectedEvidence$ver2 !== void 0 ? _selectedEvidence$ver2 : 0,
2181
+ totalChars: selected.word.length
2182
+ },
2183
+ winnerMargin: winnerMargin
2184
+ } : null;
685
2185
  };
686
2186
 
687
2187
  // ─── Data Loading ────────────────────────────────────────────────────────────
688
2188
 
2189
+ var isVocabularyJson = function isVocabularyJson(payload) {
2190
+ if (payload == null || _typeof(payload) !== 'object') {
2191
+ return false;
2192
+ }
2193
+ var words = payload.words;
2194
+ return words != null && _typeof(words) === 'object';
2195
+ };
2196
+ var isStringArray = function isStringArray(payload) {
2197
+ return Array.isArray(payload) && payload.every(function (entry) {
2198
+ return typeof entry === 'string';
2199
+ });
2200
+ };
2201
+ var isWordToOffsetMap = function isWordToOffsetMap(value) {
2202
+ return value != null && _typeof(value) === 'object' && Object.values(value).every(function (entry) {
2203
+ return typeof entry === 'number';
2204
+ });
2205
+ };
2206
+ var isWordIndexPayload = function isWordIndexPayload(payload) {
2207
+ if (payload == null || _typeof(payload) !== 'object') {
2208
+ return false;
2209
+ }
2210
+ var index = payload.index;
2211
+ return index === undefined ? isWordToOffsetMap(payload) : isWordToOffsetMap(index);
2212
+ };
2213
+
2214
+ /**
2215
+ * The word index has shipped both bare and wrapped in `{ index: ... }`, so
2216
+ * accept either rather than silently building an empty vector store.
2217
+ */
2218
+ var unwrapWordIndex = function unwrapWordIndex(payload) {
2219
+ var wrapped = payload.index;
2220
+ return wrapped !== null && wrapped !== void 0 ? wrapped : payload;
2221
+ };
2222
+
2223
+ /**
2224
+ * Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
2225
+ * confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
2226
+ * + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
2227
+ * are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
2228
+ * a number.
2229
+ */
2230
+
2231
+ /** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
2232
+
2233
+ var isPhraseStats = function isPhraseStats(value) {
2234
+ return value != null && _typeof(value) === 'object' && typeof value.freq === 'number';
2235
+ };
2236
+ var isTermStatsMap = function isTermStatsMap(value) {
2237
+ if (value == null || _typeof(value) !== 'object') {
2238
+ return false;
2239
+ }
2240
+ var values = Object.values(value);
2241
+ return values.length > 0 && values.every(isPhraseStats);
2242
+ };
2243
+
2244
+ /**
2245
+ * Normalize a `bigrams.json` / `phrases.json` payload into a flat
2246
+ * `{term: stats}` map. Tolerant of the exact wire shape (not yet finalized by
2247
+ * the producer): accepts either the flat map from spec §9 or a single wrapper
2248
+ * key (`terms`/`bigrams`/`phrases`/`words`). Returns `null` if unrecognized.
2249
+ *
2250
+ * :params:
2251
+ * payload: Raw JSON parsed from the artifact endpoint
2252
+ * :returns:
2253
+ * A `{term: stats}` map, or `null` when the shape is not a term-stats map
2254
+ */
2255
+ var normalizePhraseArtifact = function normalizePhraseArtifact(payload) {
2256
+ if (payload == null || _typeof(payload) !== 'object') {
2257
+ return null;
2258
+ }
2259
+ if (isTermStatsMap(payload)) {
2260
+ return payload;
2261
+ }
2262
+ var obj = payload;
2263
+ for (var _i4 = 0, _arr = ['terms', 'bigrams', 'phrases', 'words']; _i4 < _arr.length; _i4++) {
2264
+ var key = _arr[_i4];
2265
+ if (isTermStatsMap(obj[key])) {
2266
+ return obj[key];
2267
+ }
2268
+ }
2269
+ return null;
2270
+ };
689
2271
  export var loadVectorsAsync = /*#__PURE__*/function () {
690
- var _ref6 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
2272
+ var _ref13 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
691
2273
  var _options$isLocalLLM;
692
- var isLocalLLM, surface, url, res, buffer, float32, wordIndexPayload, wrappedIndex, wordIndex, nWords, dim, _t, _t2;
2274
+ var isLocalLLM, surface, buffer, float32, wordIndexPayload, wordIndex, nWords, dim, _t;
693
2275
  return _regeneratorRuntime.wrap(function (_context) {
694
2276
  while (1) switch (_context.prev = _context.next) {
695
2277
  case 0:
@@ -709,65 +2291,23 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
709
2291
  } : {}));
710
2292
  _context.prev = 2;
711
2293
  _context.next = 3;
712
- return getArtifactUrl(ARTIFACT_NAME.WORD_VECTORS);
2294
+ return fetchAutocompleteArtifactBinary(ARTIFACT_NAME.WORD_VECTORS, 'word_vectors_10k');
713
2295
  case 3:
714
- url = _context.sent;
715
- _context.next = 5;
716
- break;
717
- case 4:
718
- _context.prev = 4;
719
- _t = _context["catch"](2);
720
- vectorsLoadStarted = false;
721
- failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
722
- isLocalLLM: isLocalLLM,
723
- errorType: 'resolve_url'
724
- }, surface ? {
725
- surface: surface
726
- } : {}));
727
- // eslint-disable-next-line no-console
728
- console.warn('[text-predictor] Failed to resolve vectors URL:', _t);
729
- return _context.abrupt("return");
730
- case 5:
731
- _context.prev = 5;
732
- _context.next = 6;
733
- return fetch(url);
734
- case 6:
735
- res = _context.sent;
736
- if (res.ok) {
737
- _context.next = 7;
738
- break;
739
- }
740
- vectorsLoadStarted = false;
741
- failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
742
- isLocalLLM: isLocalLLM,
743
- status: res.status,
744
- errorType: 'http_error'
745
- }, surface ? {
746
- surface: surface
747
- } : {}));
748
- // eslint-disable-next-line no-console
749
- console.warn("[text-predictor] Failed to load vectors: ".concat(res.status));
750
- return _context.abrupt("return");
751
- case 7:
752
- _context.next = 8;
753
- return res.arrayBuffer();
754
- case 8:
755
2296
  buffer = _context.sent;
756
- float32 = new Float32Array(buffer); // The word index is published either bare or wrapped as `{ "index": {…} }`
757
- // (the wrapper exists so no real entry, e.g. the word "default", can shadow
758
- // a bundler-synthesised ESM export). Detect the wrapper by value type, not
759
- // by key presence — "index" is itself a plausible vocabulary word, and in a
760
- // bare index every value is a number.
761
- _context.next = 9;
762
- return fetchArtifactJson(ARTIFACT_NAME.WORD_INDEX);
763
- case 9:
2297
+ float32 = new Float32Array(buffer);
2298
+ _context.next = 4;
2299
+ return fetchAutocompleteArtifactJson(ARTIFACT_NAME.WORD_INDEX, {
2300
+ summarize: function summarize(payload) {
2301
+ return "".concat(Object.keys(unwrapWordIndex(payload)).length, " entries");
2302
+ },
2303
+ validate: isWordIndexPayload
2304
+ });
2305
+ case 4:
764
2306
  wordIndexPayload = _context.sent;
765
- wrappedIndex = wordIndexPayload === null || wordIndexPayload === void 0 ? void 0 : wordIndexPayload.index;
766
- wordIndex = _typeof(wrappedIndex) === 'object' && wrappedIndex !== null ? wrappedIndex : wordIndexPayload !== null && wordIndexPayload !== void 0 ? wordIndexPayload : {};
2307
+ wordIndex = unwrapWordIndex(wordIndexPayload);
767
2308
  nWords = Object.keys(wordIndex).length;
768
2309
  if (nWords === 0) {
769
- // eslint-disable-next-line no-console
770
- console.warn("[text-predictor] ".concat(ARTIFACT_NAME.WORD_INDEX, " was empty \u2014 semantic scoring will be a no-op."));
2310
+ ctcTag('init', '⚠️ word_index_10k.json was empty — semantic scoring will be a no-op', CTC_STYLES.warn);
771
2311
  }
772
2312
  dim = float32.length / nWords;
773
2313
  vectorStore = {
@@ -783,19 +2323,12 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
783
2323
  }, surface ? {
784
2324
  surface: surface
785
2325
  } : {}));
786
- if (isAutocompleteDebugEnabled()) {
787
- // eslint-disable-next-line no-console
788
- console.log('[text-predictor] Vectors loaded:', {
789
- wordCount: nWords,
790
- dim: dim,
791
- sizeBytes: float32.byteLength
792
- });
793
- }
794
- _context.next = 11;
2326
+ ctcTag('init', "vectors loaded: ".concat(nWords, " words \xB7 dim ").concat(dim, " \xB7 ").concat(float32.byteLength, "B"));
2327
+ _context.next = 6;
795
2328
  break;
796
- case 10:
797
- _context.prev = 10;
798
- _t2 = _context["catch"](5);
2329
+ case 5:
2330
+ _context.prev = 5;
2331
+ _t = _context["catch"](2);
799
2332
  vectorsLoadStarted = false;
800
2333
  failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
801
2334
  isLocalLLM: isLocalLLM,
@@ -803,60 +2336,197 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
803
2336
  }, surface ? {
804
2337
  surface: surface
805
2338
  } : {}));
806
- // eslint-disable-next-line no-console
807
- console.warn('[text-predictor] Failed to load vectors:', _t2);
808
- case 11:
2339
+ ctcTag('init', "\u26A0\uFE0F failed to load vectors: ".concat(String(_t)), CTC_STYLES.bad);
2340
+ case 6:
809
2341
  case "end":
810
2342
  return _context.stop();
811
2343
  }
812
- }, _callee, null, [[2, 4], [5, 10]]);
2344
+ }, _callee, null, [[2, 5]]);
813
2345
  }));
814
2346
  return function loadVectorsAsync(_x) {
815
- return _ref6.apply(this, arguments);
2347
+ return _ref13.apply(this, arguments);
816
2348
  };
817
2349
  }();
818
2350
  export var initVectors = function initVectors(store) {
819
2351
  vectorStore = store;
820
2352
  };
2353
+
2354
+ /**
2355
+ * Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
2356
+ * over the TDP-OS gateway and insert them into the phrase trie.
2357
+ *
2358
+ * Each file is fetched independently (`Promise.allSettled`) so a missing or
2359
+ * late-published artifact only skips that term type — word completion and the
2360
+ * other term type are unaffected. Phrase VECTORS are not fetched here: per the
2361
+ * producer's `extend` decision they are appended into the existing
2362
+ * `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
2363
+ * covers them.
2364
+ *
2365
+ * :params:
2366
+ * options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
2367
+ * :returns:
2368
+ * A promise that resolves once both fetches have settled
2369
+ */
2370
+ export var loadPhraseArtifacts = /*#__PURE__*/function () {
2371
+ var _ref14 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee3(options) {
2372
+ var _options$isLocalLLM2;
2373
+ var isLocalLLM, loadOne, _yield$Promise$allSet, _yield$Promise$allSet2, bigramsResult, phrasesResult, bigramCount, phraseCount;
2374
+ return _regeneratorRuntime.wrap(function (_context3) {
2375
+ while (1) switch (_context3.prev = _context3.next) {
2376
+ case 0:
2377
+ if (!phrasesLoadStarted) {
2378
+ _context3.next = 1;
2379
+ break;
2380
+ }
2381
+ return _context3.abrupt("return");
2382
+ case 1:
2383
+ phrasesLoadStarted = true;
2384
+ isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2385
+ startExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2386
+ isLocalLLM: isLocalLLM
2387
+ });
2388
+ loadOne = /*#__PURE__*/function () {
2389
+ var _ref15 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee2(artifactName, termType, label) {
2390
+ var payload, normalized;
2391
+ return _regeneratorRuntime.wrap(function (_context2) {
2392
+ while (1) switch (_context2.prev = _context2.next) {
2393
+ case 0:
2394
+ _context2.next = 1;
2395
+ return fetchAutocompleteArtifactJson(artifactName, {
2396
+ label: label,
2397
+ validate: function validate(p) {
2398
+ return p != null && _typeof(p) === 'object';
2399
+ }
2400
+ });
2401
+ case 1:
2402
+ payload = _context2.sent;
2403
+ normalized = normalizePhraseArtifact(payload);
2404
+ if (normalized) {
2405
+ _context2.next = 2;
2406
+ break;
2407
+ }
2408
+ throw new Error("[text-predictor] ".concat(label, " payload was not a recognised term-stats map"));
2409
+ case 2:
2410
+ return _context2.abrupt("return", initPhrases(normalized, termType));
2411
+ case 3:
2412
+ case "end":
2413
+ return _context2.stop();
2414
+ }
2415
+ }, _callee2);
2416
+ }));
2417
+ return function loadOne(_x3, _x4, _x5) {
2418
+ return _ref15.apply(this, arguments);
2419
+ };
2420
+ }();
2421
+ _context3.next = 2;
2422
+ return Promise.allSettled([loadOne(ARTIFACT_NAME.BIGRAMS, 'bigram', 'bigrams'), loadOne(ARTIFACT_NAME.PHRASES, 'phrase', 'phrases')]);
2423
+ case 2:
2424
+ _yield$Promise$allSet = _context3.sent;
2425
+ _yield$Promise$allSet2 = _slicedToArray(_yield$Promise$allSet, 2);
2426
+ bigramsResult = _yield$Promise$allSet2[0];
2427
+ phrasesResult = _yield$Promise$allSet2[1];
2428
+ bigramCount = bigramsResult.status === 'fulfilled' ? bigramsResult.value : 0;
2429
+ phraseCount = phrasesResult.status === 'fulfilled' ? phrasesResult.value : 0;
2430
+ if (bigramsResult.status === 'rejected') {
2431
+ ctcTag('init', "\u26A0\uFE0F failed to load bigrams.json: ".concat(String(bigramsResult.reason)), CTC_STYLES.warn);
2432
+ }
2433
+ if (phrasesResult.status === 'rejected') {
2434
+ ctcTag('init', "\u26A0\uFE0F failed to load phrases.json: ".concat(String(phrasesResult.reason)), CTC_STYLES.warn);
2435
+ }
2436
+ if (bigramsResult.status === 'fulfilled' || phrasesResult.status === 'fulfilled') {
2437
+ phrasesLoaded = true;
2438
+ succeedExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2439
+ isLocalLLM: isLocalLLM,
2440
+ bigramCount: bigramCount,
2441
+ phraseCount: phraseCount
2442
+ });
2443
+ ctcTag('init', "phrase artifacts loaded: ".concat(bigramCount, " bigrams \xB7 ").concat(phraseCount, " phrases"));
2444
+ } else {
2445
+ // Both failed — e.g. artifacts not published to the tenant manifest yet.
2446
+ // Reset the guard so a later focus/predict can retry once they land.
2447
+ phrasesLoadStarted = false;
2448
+ failExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2449
+ isLocalLLM: isLocalLLM,
2450
+ errorType: 'network'
2451
+ });
2452
+ }
2453
+ case 3:
2454
+ case "end":
2455
+ return _context3.stop();
2456
+ }
2457
+ }, _callee3);
2458
+ }));
2459
+ return function loadPhraseArtifacts(_x2) {
2460
+ return _ref14.apply(this, arguments);
2461
+ };
2462
+ }();
821
2463
  var vocabularyLoadPromise;
822
2464
  export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
823
- var _options$isLocalLLM2;
2465
+ var _options$isLocalLLM3, _options$source;
824
2466
  if (isInitialized) {
825
2467
  return Promise.resolve();
826
2468
  }
827
2469
  if (vocabularyLoadPromise) {
828
2470
  return vocabularyLoadPromise;
829
2471
  }
830
- var isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2472
+ var isLocalLLM = (_options$isLocalLLM3 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM3 !== void 0 ? _options$isLocalLLM3 : false;
2473
+ var source = (_options$source = options === null || options === void 0 ? void 0 : options.source) !== null && _options$source !== void 0 ? _options$source : 'predict';
831
2474
  var surface = options === null || options === void 0 ? void 0 : options.surface;
832
- vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee2() {
833
- var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t3;
834
- return _regeneratorRuntime.wrap(function (_context2) {
835
- while (1) switch (_context2.prev = _context2.next) {
2475
+ vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee4() {
2476
+ var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t2;
2477
+ return _regeneratorRuntime.wrap(function (_context4) {
2478
+ while (1) switch (_context4.prev = _context4.next) {
836
2479
  case 0:
837
2480
  startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
838
2481
  isLocalLLM: isLocalLLM
839
2482
  }, surface ? {
840
2483
  surface: surface
841
2484
  } : {}));
842
- _context2.prev = 1;
843
- _context2.next = 2;
844
- return Promise.all([fetchArtifactJson(ARTIFACT_NAME.VOCABULARY), fetchArtifactJson(ARTIFACT_NAME.L3_VOCABULARY)]);
2485
+ _context4.prev = 1;
2486
+ ctcTag('init', "loading artifacts (trigger: ".concat(source, " \xB7 localLLM: ").concat(isLocalLLM, ")"));
2487
+ if (isAutocompleteDebugVerbose()) {
2488
+ // eslint-disable-next-line no-console
2489
+ console.groupCollapsed('%c[CTC:init]%c artifact manifest', CTC_STYLES.brand, CTC_STYLES.body);
2490
+ // eslint-disable-next-line no-console
2491
+ console.log('Eager:', [ARTIFACT_NAME.VOCABULARY, ARTIFACT_NAME.L3_VOCABULARY, ARTIFACT_NAME.POS_TAGS, ARTIFACT_NAME.GHOST_POS_TAGS, ARTIFACT_NAME.GRAMMAR_TRANSITIONS]);
2492
+ // eslint-disable-next-line no-console
2493
+ console.log('Conditional:', ["".concat(ARTIFACT_NAME.WORD_INDEX, " (only when vectors load; phrase/bigram surfaces share this index)"), "".concat(ARTIFACT_NAME.BIGRAMS, " + ").concat(ARTIFACT_NAME.PHRASES, " (completion-term stats; fire-and-forget)"), "".concat(ARTIFACT_NAME.PHRASE_CONTINUATION_TOKENS, " (local model only; canonical token IDs for the full served union)")]);
2494
+ // eslint-disable-next-line no-console
2495
+ console.groupEnd();
2496
+ }
2497
+ _context4.next = 2;
2498
+ return Promise.all([fetchAutocompleteArtifactJson(ARTIFACT_NAME.VOCABULARY, {
2499
+ summarize: function summarize(payload) {
2500
+ return "".concat(Object.keys(payload.words).length, " words");
2501
+ },
2502
+ validate: isVocabularyJson
2503
+ }), fetchAutocompleteArtifactJson(ARTIFACT_NAME.L3_VOCABULARY, {
2504
+ summarize: function summarize(payload) {
2505
+ return "".concat(payload.length, " words");
2506
+ },
2507
+ validate: isStringArray
2508
+ }),
2509
+ // Awaited alongside the vocabulary so the grammar filter is settled
2510
+ // before the first suggestion can be produced. Without this the
2511
+ // predictor initialises on the smaller vocabulary payload and serves
2512
+ // candidates the filter would have dropped until the grammar lands.
2513
+ // Only settlement is required: a failed load leaves the filter as a
2514
+ // pass-through, which is the same degradation as an absent artifact.
2515
+ loadGrammarDataAsync({
2516
+ isLocalLLM: isLocalLLM,
2517
+ surface: surface
2518
+ }).catch(function (error) {
2519
+ ctcTag('init', "\u26A0\uFE0F grammar data unavailable; filtering skipped: ".concat(String(error)), CTC_STYLES.warn);
2520
+ })]);
845
2521
  case 2:
846
- _yield$Promise$all = _context2.sent;
2522
+ _yield$Promise$all = _context4.sent;
847
2523
  _yield$Promise$all2 = _slicedToArray(_yield$Promise$all, 2);
848
2524
  vocabularyData = _yield$Promise$all2[0];
849
2525
  l3VocabularyData = _yield$Promise$all2[1];
850
- if (!((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData))) {
851
- _context2.next = 3;
852
- break;
853
- }
854
- throw new Error("[text-predictor] unexpected vocabulary payload \u2014 " + "".concat(ARTIFACT_NAME.VOCABULARY, " ").concat((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null ? 'is missing its `words` wrapper' : 'is valid', ", ") + "".concat(ARTIFACT_NAME.L3_VOCABULARY, " ").concat(Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'));
855
- case 3:
856
- terms = Object.entries(vocabularyData.words).map(function (_ref8) {
857
- var _ref9 = _slicedToArray(_ref8, 2),
858
- word = _ref9[0],
859
- stats = _ref9[1];
2526
+ terms = Object.entries(vocabularyData.words).map(function (_ref17) {
2527
+ var _ref18 = _slicedToArray(_ref17, 2),
2528
+ word = _ref18[0],
2529
+ stats = _ref18[1];
860
2530
  return {
861
2531
  word: word,
862
2532
  freq: stats.freq,
@@ -870,6 +2540,12 @@ export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
870
2540
  initVocabulary({
871
2541
  terms: terms
872
2542
  });
2543
+
2544
+ // Phrase/bigram artifacts load independently and must never affect the
2545
+ // word path — fire-and-forget with its own error handling inside.
2546
+ void loadPhraseArtifacts({
2547
+ isLocalLLM: isLocalLLM
2548
+ }).catch(function () {});
873
2549
  succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
874
2550
  isLocalLLM: isLocalLLM,
875
2551
  l2WordCount: terms.length,
@@ -877,25 +2553,25 @@ export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
877
2553
  }, surface ? {
878
2554
  surface: surface
879
2555
  } : {}));
880
- _context2.next = 5;
2556
+ _context4.next = 4;
881
2557
  break;
882
- case 4:
883
- _context2.prev = 4;
884
- _t3 = _context2["catch"](1);
2558
+ case 3:
2559
+ _context4.prev = 3;
2560
+ _t2 = _context4["catch"](1);
885
2561
  failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
886
2562
  isLocalLLM: isLocalLLM,
887
- errorType: 'load_error'
2563
+ errorType: 'parse_error'
888
2564
  }, surface ? {
889
2565
  surface: surface
890
2566
  } : {}));
891
2567
  // Allow a later call to retry the load rather than caching the failure.
892
2568
  vocabularyLoadPromise = undefined;
893
- throw _t3;
894
- case 5:
2569
+ throw _t2;
2570
+ case 4:
895
2571
  case "end":
896
- return _context2.stop();
2572
+ return _context4.stop();
897
2573
  }
898
- }, _callee2, null, [[1, 4]]);
2574
+ }, _callee4, null, [[1, 3]]);
899
2575
  }))();
900
2576
  return vocabularyLoadPromise;
901
2577
  };