@atlaskit/editor-plugin-autocomplete 7.0.0 → 9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/CHANGELOG.md +86 -0
  2. package/compass.yml +2 -2
  3. package/dist/cjs/analytics/ufo.js +2 -1
  4. package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
  5. package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
  6. package/dist/cjs/pm-plugins/autocomplete-plugin.js +416 -129
  7. package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
  8. package/dist/cjs/pm-plugins/debug-mode.js +90 -7
  9. package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
  10. package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
  11. package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
  12. package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
  13. package/dist/cjs/pm-plugins/text-predictor.js +1951 -273
  14. package/dist/es2019/analytics/ufo.js +2 -0
  15. package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
  16. package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
  17. package/dist/es2019/pm-plugins/autocomplete-plugin.js +425 -130
  18. package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
  19. package/dist/es2019/pm-plugins/debug-mode.js +87 -7
  20. package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
  21. package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
  22. package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
  23. package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
  24. package/dist/es2019/pm-plugins/text-predictor.js +1603 -199
  25. package/dist/esm/analytics/ufo.js +2 -1
  26. package/dist/esm/pm-plugins/artifact-loader.js +128 -0
  27. package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
  28. package/dist/esm/pm-plugins/autocomplete-plugin.js +419 -132
  29. package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
  30. package/dist/esm/pm-plugins/debug-mode.js +89 -6
  31. package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
  32. package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
  33. package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
  34. package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
  35. package/dist/esm/pm-plugins/text-predictor.js +1953 -277
  36. package/dist/types/analytics/ufo.d.ts +1 -0
  37. package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
  38. package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
  39. package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
  40. package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
  41. package/dist/types/pm-plugins/debug-mode.d.ts +36 -1
  42. package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
  43. package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
  44. package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
  45. package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
  46. package/dist/types/pm-plugins/text-predictor.d.ts +120 -2
  47. package/package.json +3 -3
  48. package/src/analytics/ufo.ts +5 -0
  49. package/src/pm-plugins/artifact-loader.ts +92 -0
  50. package/src/pm-plugins/artifacts-manifest.ts +3 -0
  51. package/src/pm-plugins/autocomplete-plugin.ts +517 -133
  52. package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
  53. package/src/pm-plugins/debug-mode.ts +109 -7
  54. package/src/pm-plugins/ghost-text-decoration.ts +13 -4
  55. package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
  56. package/src/pm-plugins/scoring-pipeline.ts +167 -17
  57. package/src/pm-plugins/slow-lane-client.ts +147 -62
  58. package/src/pm-plugins/text-predictor.ts +2006 -253
  59. package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
  60. package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
  61. package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
  62. package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
  63. package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
  64. package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
  65. package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  66. package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  67. package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
  68. package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  69. package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  70. package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
  71. package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
  72. package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
  73. package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
  74. package/scripts/gen_first_token_to_words.py +0 -170
  75. package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
  76. package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
  77. package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
  78. package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
  79. package/src/pm-plugins/slow-lane-client/package.json +0 -8
  80. package/src/pm-plugins/text-predictor/package.json +0 -8
@@ -4,16 +4,19 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau
4
4
  Object.defineProperty(exports, "__esModule", {
5
5
  value: true
6
6
  });
7
- exports.predict = exports.loadVectorsAsync = exports.loadDefaultVocabulary = exports.initVocabulary = exports.initVectors = exports.initL3Vocabulary = exports.ingestDocumentPage = exports.incrementSessionFreq = exports.getPredictorStatus = exports.getLastPredictionDebug = void 0;
7
+ exports.predict = exports.noteSuggestionAccepted = exports.loadVectorsAsync = exports.loadPhraseArtifacts = exports.loadDefaultVocabulary = exports.initVocabulary = exports.initVectors = exports.initPhrases = exports.initL3Vocabulary = exports.ingestDocumentPage = exports.incrementSessionFreq = exports.getPredictorStatus = exports.getLastPredictionDebug = void 0;
8
8
  var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
9
- var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
10
9
  var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
10
+ var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
11
+ var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
11
12
  var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
12
13
  var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
13
14
  var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
14
15
  var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
15
16
  var _ufo = require("../analytics/ufo");
17
+ var _artifactLoader = require("./artifact-loader");
16
18
  var _artifactsManifest = require("./artifacts-manifest");
19
+ var _canonicalLmScoring = require("./canonical-lm-scoring");
17
20
  var _debugMode = require("./debug-mode");
18
21
  var _scoringPipeline = require("./scoring-pipeline");
19
22
  var _slowLaneClient = require("./slow-lane-client");
@@ -26,7 +29,7 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
26
29
  *
27
30
  * Two prediction modes:
28
31
  * 1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
29
- * 2. Mid-word (≥3 chars) → trie prefix search → scoring pipeline → top result
32
+ * 2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
30
33
  *
31
34
  * Scoring is delegated to scoring-pipeline.ts which handles:
32
35
  * Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
@@ -36,9 +39,7 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
36
39
  *
37
40
  * Session personalization (L1): words the user types are incrementally boosted
38
41
  * via incrementSessionFreq(), called on word boundaries from the plugin.
39
- */ // The vocabulary, L3 and word-index payloads are fetched from the CDN in
40
- // loadDefaultVocabulary / loadVectorsAsync so their (large) contents never enter
41
- // the editor's bundle and only load when autocomplete is initialised.
42
+ */
42
43
  // ─── Constants ───────────────────────────────────────────────────────────────
43
44
 
44
45
  // eslint-disable-next-line require-unicode-regexp
@@ -49,13 +50,152 @@ var WHITESPACE_SPLIT_REGEX = /\s+/;
49
50
  var SENTENCE_BOUNDARY_REGEX = /[\n.?!]+/;
50
51
  // eslint-disable-next-line require-unicode-regexp
51
52
  var TRAILING_WHITESPACE_REGEX = /\s$/;
52
- var MIN_PREFIX_LENGTH = 3;
53
+ var TRAILING_SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+$/;
54
+ var SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+/g;
55
+ var ONLY_WHITESPACE_REGEX = /^[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+$/;
56
+ var UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH = 1;
57
+ var MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH = 1;
58
+ var DISPLAY_MIN_PREFIX_LENGTH = 3;
53
59
  var MAX_CANDIDATES = 200;
54
60
  var CONTEXT_WORDS = 10;
55
- var MIN_SCORE_THRESHOLD = 0.35;
56
61
  var L3_BASELINE_FREQ = 0.001;
62
+ // Max tokens in a completion term (bigram=2, phrase=3). Mirrors the producer's
63
+ // `phrase_max_words` default so the FE window never exceeds what was mined.
64
+ var PHRASE_MAX_WORDS = 3;
65
+ // Per-window cap on phrase-trie subtree collection (keeps the hot path bounded).
66
+ var MAX_PHRASE_CANDIDATES = 50;
67
+ /**
68
+ * Share of its context shortlist's probability mass the model must put on a
69
+ * surface before that surface may be shown.
70
+ *
71
+ * Candidates competing at the same boundary are normalised into a posterior
72
+ * over that shortlist, so this reads directly as "the model is at least this
73
+ * sure". Measuring distance from the best candidate instead cannot express
74
+ * being unsure: the leader scores full marks whether it won by a nose or a
75
+ * mile, which pushes the entire question of how contested a context is onto
76
+ * `MIN_WINNER_MARGIN` — and there the blended score's only remaining spread is
77
+ * the corpus prior, the weakest ranking signal we have.
78
+ *
79
+ * Each extra token raises the bar. A wrong multi-word ghost costs the reader
80
+ * more to notice and undo than a wrong single word, and it is offered from the
81
+ * same keystroke, so precision has to be bought per token of commitment.
82
+ */
83
+ var MIN_LM_POSTERIOR = {
84
+ word: 0.5,
85
+ bigram: 0.6,
86
+ phrase: 0.7
87
+ };
88
+ /**
89
+ * How many scored candidates a context must hold before its leader may show.
90
+ *
91
+ * A posterior is a share of a shortlist, so a shortlist of one hands its only
92
+ * member 1.0 whatever the model thinks of it — `logSumExp` of a single value
93
+ * returns that value. Such a ghost sits in the top bucket without having beaten
94
+ * anything, and it is the gate's blind spot rather than a case the gate ruled
95
+ * on: 1 in 14 of them were accepted against 34% for word ghosts overall, and
96
+ * every phrase shown through this hole was rejected.
97
+ *
98
+ * Counted over pool membership rather than over the normaliser, because those
99
+ * two deliberately differ. Where a pool holds a surface and its own extension
100
+ * the chain rule leaves one term in the denominator, and the resulting 1.0 is
101
+ * earned — whichever the user meant, the shorter form is a correct ghost.
102
+ *
103
+ * Waived where the context never held a second candidate at all. The 1-in-14
104
+ * acceptance above was measured over a population dominated by pools that read
105
+ * as one because rivals had not come back, and it is that absence of evidence
106
+ * the floor is for. A vocabulary offering a single continuation of a long
107
+ * distinctive prefix is the opposite situation, and it is also where a ghost
108
+ * saves the most keystrokes, so refusing it spends the most to learn the least.
109
+ */
110
+ var MIN_SCORED_POOL_SIZE = 2;
111
+ /**
112
+ * Mean per-token log-probability a surface must hold, in nats, independently of
113
+ * anything it was competing against.
114
+ *
115
+ * Every other display test is relative: a posterior is a share of a shortlist,
116
+ * and a winner margin is a distance from a rival. So a surface reaches the
117
+ * screen by being the best of what happened to be in its pool, and nothing ever
118
+ * asks whether the model finds it plausible at all. That gap was tolerable while
119
+ * little was being read, and stopped being tolerable once the scheduler started
120
+ * reading roughly twice as much per decision: shown-per-decision went from 3.2%
121
+ * to 7.4% with no threshold moved, and acceptance of what showed fell from 32%
122
+ * to 24.5%. More reading does not weaken a relative test — it hands it far more
123
+ * pools to be the winner of, and the marginal winner is worse than the average.
124
+ *
125
+ * Set at the point where a uniform distribution over the model's vocabulary
126
+ * sits: 49,152 tokens, so `ln(1/49152)` is about -10.8 nats. Below it the model
127
+ * assigns the surface less mass than a token drawn at random, which is the only
128
+ * statement about a surface that needs no reference to what it competed with.
129
+ *
130
+ * It was first tried at -8, chosen from the bucket where acceptance was measured
131
+ * to fall away, and that reasoning does not survive more data. Acceptance against
132
+ * this quantity turned out to be a smooth slope — about 19% below -8, 26% from -8
133
+ * to -6, then flat near 35% above - with no discontinuity to place a threshold
134
+ * at. A floor on a smooth slope is a coverage dial rather than a test: -8 removed
135
+ * a band converting at 26% and cost 3.7 points of show rate, for an acceptance
136
+ * movement no sample of this size can resolve. Anchoring to uniform instead makes
137
+ * the value an argument about what is indefensible rather than a point picked off
138
+ * a histogram, which is what stops it from being retuned every batch.
139
+ *
140
+ * Applied to the mean over the verified prefix, the same quantity
141
+ * `ghostsByMeanLogProb` buckets, and deliberately not to
142
+ * `meanTokenLogProbabilityUpperBound`. The bound is the right instrument for
143
+ * keeping a candidate in the race, where dropping something that could still
144
+ * clear the bar would be an error, but it is systematically higher than the
145
+ * quantity acceptance is measured against. Judging a surface on the prefix that
146
+ * was actually read is also what every other test here does.
147
+ *
148
+ * One value across all three shapes. Splitting it per shape needs an acceptance
149
+ * curve per shape, and 19 bigrams and 1 phrase across five sessions cannot
150
+ * support one.
151
+ */
152
+ var MIN_MEAN_TOKEN_LOG_PROBABILITY = -10.8;
153
+ // Tab replaces the whole remaining suffix, so an N-character completion saves
154
+ // N-1 keystrokes: at one character it saves nothing at all and at two it saves
155
+ // one, which does not repay noticing the ghost and reaching for Tab. Offering
156
+ // them anyway spends the acceptance-rate denominator on completions nobody
157
+ // wants and teaches the user to stop reading ghosts. `DISPLAY_MIN_PREFIX_LENGTH`
158
+ // is the same guard on the side the user has already typed.
159
+ var MIN_SUGGESTION_LENGTH = 3;
160
+ /**
161
+ * How many leading tokens of a surface must be scored before it may be shown.
162
+ *
163
+ * Scoring a token costs one model round trip, and a round trip costs about the
164
+ * same no matter how much is in it, so the price of a candidate is its token
165
+ * count. Demanding every token therefore caps what can ever reach the screen at
166
+ * roughly two tokens inside the decision budget — which is why no phrase has
167
+ * ever been displayed. Verifying a fixed prefix makes the cost of a candidate
168
+ * independent of its length.
169
+ */
170
+ var REQUIRED_VERIFIED_TOKENS = 2;
171
+ /**
172
+ * Extra winner margin demanded of a surface judged on a verified prefix rather
173
+ * than on every one of its tokens, to buy back the precision given up by
174
+ * leaving the tail unread.
175
+ */
176
+ var PARTIAL_EVIDENCE_MARGIN_PREMIUM = 0.08;
177
+ var ARBITRATION_MODE = 'confidence-v2';
178
+ var DEBUG_TEXT_TAIL_CHARS = 120;
57
179
 
58
180
  // ─── Types ───────────────────────────────────────────────────────────────────
181
+
182
+ /**
183
+ * Which constraint stopped an evaluation from putting a ghost on screen.
184
+ *
185
+ * Each one implies different work: `below-posterior-gate` is a threshold to
186
+ * calibrate, `winner-margin` is two candidates the model cannot separate,
187
+ * `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
188
+ * is vocabulary coverage.
189
+ */
190
+
191
+ /**
192
+ * A candidate paired with the length of the already-typed prefix it completes.
193
+ * For a single word this is the current partial token length; for a phrase it
194
+ * is the length of the matched multi-word window (previous words + partial),
195
+ * so the ghost tail is `word.slice(matchedPrefixLen)`.
196
+ */
197
+
198
+ /** Optional per-term metadata carried by bigram/phrase completion terms. */
59
199
  var TrieNode = /*#__PURE__*/(0, _createClass2.default)(function TrieNode() {
60
200
  (0, _classCallCheck2.default)(this, TrieNode);
61
201
  (0, _defineProperty2.default)(this, "children", new Map());
@@ -64,6 +204,12 @@ var TrieNode = /*#__PURE__*/(0, _createClass2.default)(function TrieNode() {
64
204
  (0, _defineProperty2.default)(this, "docFreq", 0);
65
205
  (0, _defineProperty2.default)(this, "authorFreq", 0);
66
206
  (0, _defineProperty2.default)(this, "sessionFreq", 0);
207
+ // Term-type + phrase metadata. Plain words keep `termType='word'` and null
208
+ // metadata; bigram/phrase nodes carry the producer-shipped POS + association.
209
+ (0, _defineProperty2.default)(this, "termType", 'word');
210
+ (0, _defineProperty2.default)(this, "posSeq", null);
211
+ (0, _defineProperty2.default)(this, "headPos", null);
212
+ (0, _defineProperty2.default)(this, "association", null);
67
213
  });
68
214
  var WeightedWordTrie = /*#__PURE__*/function () {
69
215
  function WeightedWordTrie() {
@@ -74,7 +220,8 @@ var WeightedWordTrie = /*#__PURE__*/function () {
74
220
  }
75
221
  return (0, _createClass2.default)(WeightedWordTrie, [{
76
222
  key: "insert",
77
- value: function insert(word, tenantFreq, docFreq, authorFreq) {
223
+ value: function insert(word, tenantFreq, docFreq, authorFreq, meta) {
224
+ var _meta$termType, _meta$posSeq, _meta$headPos, _meta$association;
78
225
  var node = this.root;
79
226
  var _iterator = _createForOfIteratorHelper(word.toLowerCase()),
80
227
  _step;
@@ -97,6 +244,10 @@ var WeightedWordTrie = /*#__PURE__*/function () {
97
244
  node.tenantFreq = tenantFreq;
98
245
  node.docFreq = docFreq;
99
246
  node.authorFreq = authorFreq;
247
+ node.termType = (_meta$termType = meta === null || meta === void 0 ? void 0 : meta.termType) !== null && _meta$termType !== void 0 ? _meta$termType : 'word';
248
+ node.posSeq = (_meta$posSeq = meta === null || meta === void 0 ? void 0 : meta.posSeq) !== null && _meta$posSeq !== void 0 ? _meta$posSeq : null;
249
+ node.headPos = (_meta$headPos = meta === null || meta === void 0 ? void 0 : meta.headPos) !== null && _meta$headPos !== void 0 ? _meta$headPos : null;
250
+ node.association = (_meta$association = meta === null || meta === void 0 ? void 0 : meta.association) !== null && _meta$association !== void 0 ? _meta$association : null;
100
251
  if (tenantFreq > this.maxTenantFreq) {
101
252
  this.maxTenantFreq = tenantFreq;
102
253
  }
@@ -216,6 +367,15 @@ var wordTrie = new WeightedWordTrie();
216
367
  // L3 Trie (General English Fallback)
217
368
  var l3Trie = new WeightedWordTrie();
218
369
 
370
+ /**
371
+ * L2 phrase/bigram trie. Kept SEPARATE from `wordTrie` so single-word
372
+ * completion + L3 gap-fill stay untouched; multi-word units are only surfaced
373
+ * by the (12b) multiword prefix-window matcher. Keys are the full normalized
374
+ * surface string (spaces included), e.g. `"return on investment"`, so a lookup
375
+ * is a plain char-trie walk over the whole phrase.
376
+ */
377
+ var phraseTrie = new WeightedWordTrie();
378
+
219
379
  /**
220
380
  * Loads the General English vocabulary.
221
381
  * expects a simple array of strings: ["about", "above", "actually", ...]
@@ -235,10 +395,8 @@ var initL3Vocabulary = exports.initL3Vocabulary = function initL3Vocabulary(l3Wo
235
395
  } finally {
236
396
  _iterator5.f();
237
397
  }
238
- if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
239
- // eslint-disable-next-line no-console
240
- console.log("[text-predictor] L3 General English loaded: ".concat(l3Words.length, " words"));
241
- }
398
+ recallGeneration++;
399
+ (0, _debugMode.ctcTag)('init', "L3 general English loaded: ".concat(l3Words.length, " words"));
242
400
  };
243
401
 
244
402
  // const bigramMap: Map<string, Record<string, number>> = new Map(
@@ -248,8 +406,50 @@ var initL3Vocabulary = exports.initL3Vocabulary = function initL3Vocabulary(l3Wo
248
406
  var isInitialized = false;
249
407
  var vectorStore = null;
250
408
  var vectorsLoadStarted = false;
409
+
410
+ // Phrase/bigram artifacts load independently of the word vocabulary so a
411
+ // missing/late `bigrams.json`/`phrases.json` never breaks word completion.
412
+ var phrasesLoadStarted = false;
413
+ var phrasesLoaded = false;
414
+ var phraseTermCount = 0;
415
+ // Per-type frequency maxima, tracked separately because bigrams and phrases
416
+ // share one trie but have very different absolute frequency ranges (bigrams are
417
+ // far more frequent than 3-word phrases). Normalizing each type by its OWN max
418
+ // stops phrase freq scores from being crushed against the top bigram's count.
419
+ var maxBigramFreq = 1;
420
+ var maxPhraseFreq = 1;
251
421
  var lastPredictionDebug = null;
252
422
 
423
+ // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
424
+ // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
425
+ // generally keep a just-accepted unit from being re-offered:
426
+ // • repetition: never surface a candidate whose full surface already equals the
427
+ // run of tokens immediately before the cursor (handled inline in predict()).
428
+ // • cooldown: after an accept, briefly suppress re-offering that same unit. The
429
+ // cooldown lifts only once BOTH the keystroke budget and the time window have
430
+ // elapsed (either one still counting keeps it active).
431
+ var COOLDOWN_KEYSTROKES = 5;
432
+ var COOLDOWN_MS = 2000;
433
+
434
+ // Active post-accept cooldown, or null when none.
435
+ var acceptCooldown = null;
436
+
437
+ /**
438
+ * Start a short cooldown for the exact surface the editor inserted. The caller
439
+ * passes the committed snapshot's surface so background re-ranking can never
440
+ * move cooldown bookkeeping away from what the user actually accepted.
441
+ */
442
+ var noteSuggestionAccepted = exports.noteSuggestionAccepted = function noteSuggestionAccepted(surface) {
443
+ var normalizedSurface = surface.trim().toLowerCase();
444
+ if (normalizedSurface) {
445
+ acceptCooldown = {
446
+ predictionsSince: 0,
447
+ surface: normalizedSurface,
448
+ ts: performance.now()
449
+ };
450
+ }
451
+ };
452
+
253
453
  /** Get vector for a word from the store. */
254
454
  var getWordVector = function getWordVector(word) {
255
455
  if (!vectorStore) {
@@ -346,6 +546,124 @@ var extractPreviousWord = function extractPreviousWord(text) {
346
546
  return words.length >= 2 ? words[words.length - 2] : '';
347
547
  };
348
548
 
549
+ /**
550
+ * Collect phrase/bigram completion candidates for the multi-word window ending
551
+ * at the current partial token.
552
+ *
553
+ * Builds windows of the last `2..PHRASE_MAX_WORDS` tokens of the current
554
+ * sentence (widest first) and prefix-searches the phrase trie for each, so
555
+ * `quarterly pla` matches `quarterly planning meeting`. Windows never cross a
556
+ * sentence boundary (mirrors `extractPreviousWord`). Each result records the
557
+ * matched window length so the caller can compute the ghost tail.
558
+ *
559
+ * :params:
560
+ * trimmed: Text before the cursor, trailing whitespace already removed
561
+ * :returns:
562
+ * Matched phrase candidates (deduped, widest-window match wins per term)
563
+ */
564
+ var getPhraseCandidates = function getPhraseCandidates(trimmed) {
565
+ // Records every prefix window we probe and its match count so the (verbose)
566
+ // debug line makes it obvious whether the phrase path found nothing because the
567
+ // trie is empty, or because no phrase begins with the typed window.
568
+ var windowsTried = [];
569
+ var logPhrasePath = function logPhrasePath(skipReason) {
570
+ if (!(0, _debugMode.isAutocompleteDebugVerbose)()) {
571
+ return;
572
+ }
573
+ var detail = skipReason ? skipReason : "windows: ".concat(windowsTried.map(function (w) {
574
+ return "\"".concat(w.window, "\"\u2192").concat(w.matches);
575
+ }).join(', ') || '(none)');
576
+ (0, _debugMode.ctcTag)('phrase', "loaded=".concat(phraseTermCount, " (").concat(phrasesLoaded ? 'ready' : 'not loaded', ") \xB7 ").concat(detail), _debugMode.CTC_STYLES.section);
577
+ };
578
+ if (phraseTermCount === 0) {
579
+ logPhrasePath('phrase trie empty — bigrams.json/phrases.json not loaded (or 0 terms)');
580
+ return [];
581
+ }
582
+ var tokenMatches = Array.from(trimmed.matchAll(SURFACE_TOKEN_REGEX));
583
+ var lastMatch = tokenMatches[tokenMatches.length - 1];
584
+ var lastStart = lastMatch === null || lastMatch === void 0 ? void 0 : lastMatch.index;
585
+ if (!lastMatch || lastStart === undefined || lastStart + lastMatch[0].length !== trimmed.length) {
586
+ logPhrasePath('no tokens in current sentence');
587
+ return [];
588
+ }
589
+ var trailingTokens = [{
590
+ start: lastStart,
591
+ value: lastMatch[0]
592
+ }];
593
+ for (var i = tokenMatches.length - 2; i >= 0; i--) {
594
+ var current = trailingTokens[0];
595
+ var match = tokenMatches[i];
596
+ var start = match.index;
597
+ if (start === undefined) {
598
+ break;
599
+ }
600
+ var end = start + match[0].length;
601
+ var gap = trimmed.slice(end, current.start);
602
+ if (!ONLY_WHITESPACE_REGEX.test(gap)) {
603
+ break;
604
+ }
605
+ trailingTokens.unshift({
606
+ start: start,
607
+ value: match[0]
608
+ });
609
+ if (trailingTokens.length >= PHRASE_MAX_WORDS) {
610
+ break;
611
+ }
612
+ }
613
+ var partial = trailingTokens[trailingTokens.length - 1].value;
614
+ var results = [];
615
+ var seen = new Set();
616
+ var maxPrev = trailingTokens.length - 1;
617
+
618
+ // Widest window first so the most specific (longest-context) match wins the
619
+ // dedup for a given phrase term. `prev` is the number of *preceding* complete
620
+ // words included ahead of the current partial:
621
+ // prev ≥ 1 → "continue a phrase I've started" (e.g. "root c" → root cause).
622
+ // Preceding-word context is strong, so a short partial is fine.
623
+ // prev = 0 → "the partial is a phrase's FIRST word" (e.g. "root" → root
624
+ // cause). No preceding context, so require the partial to be a
625
+ // one character so grouped LM work can begin early. UI remains
626
+ // gated at DISPLAY_MIN_PREFIX_LENGTH.
627
+ for (var prev = maxPrev; prev >= 0; prev--) {
628
+ if (prev === 0 && partial.length < MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH) {
629
+ continue;
630
+ }
631
+ var windowTokens = trailingTokens.slice(-(prev + 1));
632
+ var windowPrefix = windowTokens.map(function (token) {
633
+ return token.value;
634
+ }).join(' ').toLowerCase();
635
+ var windowLen = windowPrefix.length;
636
+ var matches = phraseTrie.getCandidates(windowPrefix, MAX_PHRASE_CANDIDATES);
637
+ windowsTried.push({
638
+ window: windowPrefix,
639
+ matches: matches.length
640
+ });
641
+ var _iterator8 = _createForOfIteratorHelper(matches),
642
+ _step8;
643
+ try {
644
+ for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
645
+ var _match = _step8.value;
646
+ if (seen.has(_match.word)) {
647
+ continue;
648
+ }
649
+ seen.add(_match.word);
650
+ results.push({
651
+ node: _match.node,
652
+ word: _match.word,
653
+ matchedPrefixLen: windowLen,
654
+ surfaceStart: windowTokens[0].start
655
+ });
656
+ }
657
+ } catch (err) {
658
+ _iterator8.e(err);
659
+ } finally {
660
+ _iterator8.f();
661
+ }
662
+ }
663
+ logPhrasePath();
664
+ return results;
665
+ };
666
+
349
667
  // ─── Debug Helpers ───────────────────────────────────────────────────────────
350
668
 
351
669
  /**
@@ -354,11 +672,25 @@ var extractPreviousWord = function extractPreviousWord(text) {
354
672
  * wordCount: number of words in vector store (0 if not loaded)
355
673
  */
356
674
  var getPredictorStatus = exports.getPredictorStatus = function getPredictorStatus() {
675
+ var slowLaneStatus = (0, _slowLaneClient.getDefaultSlowLaneClientStatus)();
357
676
  return {
677
+ canonicalScoringSupported: slowLaneStatus.canonicalScoringSupported,
678
+ canonicalSurfaceCount: slowLaneStatus.canonicalSurfaceCount,
679
+ clientId: slowLaneStatus.clientId,
680
+ clientKind: slowLaneStatus.clientKind,
681
+ localModelReady: slowLaneStatus.localModelReady,
682
+ slowLaneRegistered: slowLaneStatus.registered,
358
683
  vectorsLoaded: vectorStore !== null,
359
684
  wordCount: vectorStore ? Object.keys(vectorStore.wordIndex).length : 0,
360
685
  vectorsLoadStarted: vectorsLoadStarted,
361
- isInitialized: isInitialized
686
+ isInitialized: isInitialized,
687
+ phrasesLoaded: phrasesLoaded,
688
+ phraseCount: phraseTermCount,
689
+ maxFreqByType: {
690
+ word: wordTrie.maxTenantFreq,
691
+ bigram: maxBigramFreq,
692
+ phrase: maxPhraseFreq
693
+ }
362
694
  };
363
695
  };
364
696
 
@@ -370,19 +702,57 @@ var getLastPredictionDebug = exports.getLastPredictionDebug = function getLastPr
370
702
  return lastPredictionDebug;
371
703
  };
372
704
  var initVocabulary = exports.initVocabulary = function initVocabulary(vocabulary) {
373
- var _iterator8 = _createForOfIteratorHelper(vocabulary.terms),
374
- _step8;
705
+ var _iterator9 = _createForOfIteratorHelper(vocabulary.terms),
706
+ _step9;
375
707
  try {
376
- for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
377
- var term = _step8.value;
708
+ for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
709
+ var term = _step9.value;
378
710
  wordTrie.insert(term.word, term.freq, term.docFreq, term.authorFreq);
379
711
  }
380
712
  } catch (err) {
381
- _iterator8.e(err);
713
+ _iterator9.e(err);
382
714
  } finally {
383
- _iterator8.f();
715
+ _iterator9.f();
384
716
  }
385
717
  isInitialized = true;
718
+ recallGeneration++;
719
+ };
720
+
721
+ /**
722
+ * Insert a producer-shipped bigram/phrase artifact into the phrase trie.
723
+ *
724
+ * :params:
725
+ * artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
726
+ * termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
727
+ * :returns:
728
+ * The number of terms inserted
729
+ */
730
+ var initPhrases = exports.initPhrases = function initPhrases(artifact, termType) {
731
+ var count = 0;
732
+ for (var _i3 = 0, _Object$entries = Object.entries(artifact); _i3 < _Object$entries.length; _i3++) {
733
+ var _stats$pos_seq, _stats$head_pos, _stats$association;
734
+ var _Object$entries$_i = (0, _slicedToArray2.default)(_Object$entries[_i3], 2),
735
+ term = _Object$entries$_i[0],
736
+ stats = _Object$entries$_i[1];
737
+ phraseTrie.insert(term, stats.freq, stats.doc_freq, stats.author_freq, {
738
+ termType: termType,
739
+ posSeq: (_stats$pos_seq = stats.pos_seq) !== null && _stats$pos_seq !== void 0 ? _stats$pos_seq : null,
740
+ headPos: (_stats$head_pos = stats.head_pos) !== null && _stats$head_pos !== void 0 ? _stats$head_pos : null,
741
+ association: (_stats$association = stats.association) !== null && _stats$association !== void 0 ? _stats$association : null
742
+ });
743
+ // Track each type's own frequency ceiling for per-type normalization.
744
+ if (termType === 'bigram') {
745
+ if (stats.freq > maxBigramFreq) {
746
+ maxBigramFreq = stats.freq;
747
+ }
748
+ } else if (stats.freq > maxPhraseFreq) {
749
+ maxPhraseFreq = stats.freq;
750
+ }
751
+ count++;
752
+ }
753
+ phraseTermCount += count;
754
+ recallGeneration++;
755
+ return count;
386
756
  };
387
757
 
388
758
  /**
@@ -409,177 +779,1104 @@ var ingestDocumentPage = exports.ingestDocumentPage = function ingestDocumentPag
409
779
  }
410
780
  var words = tokenize(pageContent);
411
781
  var validBoostedWords = new Set();
412
- var _iterator9 = _createForOfIteratorHelper(words),
413
- _step9;
782
+ var _iterator0 = _createForOfIteratorHelper(words),
783
+ _step0;
414
784
  try {
415
- for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
416
- var word = _step9.value;
785
+ for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
786
+ var word = _step0.value;
417
787
  var didBoost = wordTrie.incrementSessionFreq(word);
418
788
  if (didBoost) {
419
789
  validBoostedWords.add(word);
420
790
  }
421
791
  }
422
792
  } catch (err) {
423
- _iterator9.e(err);
793
+ _iterator0.e(err);
424
794
  } finally {
425
- _iterator9.f();
795
+ _iterator0.f();
426
796
  }
427
797
  if ((0, _debugMode.isAutocompleteDebugEnabled)() && validBoostedWords.size > 0) {
428
- // eslint-disable-next-line no-console
429
- console.groupCollapsed("%c[L1 Session] %cPrimed ".concat(validBoostedWords.size, " valid dictionary words from page"), 'color: #00b8d9; font-weight: bold;', 'color: inherit; font-style: italic;');
430
- // eslint-disable-next-line no-console
431
- console.dir(Array.from(validBoostedWords).sort());
432
- // eslint-disable-next-line no-console
433
- console.groupEnd();
798
+ (0, _debugMode.ctcTag)('init', "L1 session primed ".concat(validBoostedWords.size, " words from page"), _debugMode.CTC_STYLES.brand);
799
+ if ((0, _debugMode.isAutocompleteDebugVerbose)()) {
800
+ // eslint-disable-next-line no-console
801
+ console.dir(Array.from(validBoostedWords).sort());
802
+ }
434
803
  }
435
804
  };
436
- var predict = exports.predict = function predict(textBefore) {
437
- if (!isInitialized) {
438
- // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
439
- // and skip this keystroke; the plugin also primes it on focus, so the tries
440
- // are usually ready before the user types.
441
- void loadDefaultVocabulary().catch(function () {});
442
- return null;
443
- }
444
- var t0 = performance.now();
445
805
 
446
- // ── Step 1: Bigram-based next-word suggestion at word boundary ───────────
447
- // if (textBefore.length > 0 && /\s$/u.test(textBefore)) {
448
- // const words = textBefore.toLowerCase().trimEnd().split(/\s+/u);
449
- // const prevWord = words[words.length - 1];
450
- // const nextWords = bigramMap.get(prevWord);
451
- // if (nextWords) {
452
- // const sorted = Object.entries(nextWords).sort((a, b) => b[1] - a[1]);
453
- // let bestWord = '';
454
- // for (const [word] of sorted) {
455
- // if (!isGrammarAllowed(prevWord, word)) {
456
- // continue;
457
- // }
458
- // bestWord = word;
459
- // break;
460
- // }
461
- // if (bestWord) {
462
- // if (debugMode) {
463
- // const latencyMs = performance.now() - t0;
464
- // console.log(
465
- // '%c[autocomplete] BIGRAM',
466
- // 'color:cyan',
467
- // '| "' + prevWord + '" -> "' + bestWord + '" | ' + latencyMs.toFixed(1) + 'ms',
468
- // );
469
- // }
470
- // return bestWord;
471
- // }
472
- // }
473
- // if (debugMode) {
474
- // console.log(
475
- // '%c[autocomplete] BIGRAM-MISS',
476
- // 'color:gray',
477
- // '| no bigram for "' + words[words.length - 1] + '", skipping prefix completion',
478
- // );
479
- // }
480
- // return null;
481
- // }
482
-
483
- // ── Step 2: Prefix completion (≥3 chars typed) ──────────────────────────
484
- if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
485
- return null;
486
- }
487
- var trimmed = textBefore.trimEnd();
488
- var lastSpaceIdx = trimmed.lastIndexOf(' ');
489
- var currentWord = lastSpaceIdx === -1 ? trimmed : trimmed.slice(lastSpaceIdx + 1);
490
- if (currentWord.length < MIN_PREFIX_LENGTH) {
491
- return null;
492
- }
493
- var candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
806
+ /**
807
+ * Result of a prediction: the ghost tail to insert plus an immutable record of
808
+ * the evidence that authorized the UI commitment.
809
+ */
810
+
811
+ // ─── Recall memo ─────────────────────────────────────────────────────────────
812
+
813
+ /** Trie recall plus the canonical context derived for each matched term. */
814
+
815
+ /**
816
+ * Bumped whenever an artifact load changes what the tries can return, which is
817
+ * the only way the candidate *set* for a given prefix can change.
818
+ *
819
+ * Session-frequency boosts are deliberately not counted: they mutate trie nodes
820
+ * in place, and the memo holds those nodes by reference, so a boost is already
821
+ * visible through a cached entry.
822
+ */
823
+ var recallGeneration = 0;
824
+ var recallMemo = null;
825
+ var computeCanonicalRecall = function computeCanonicalRecall(trimmed, currentWord, currentWordStart) {
826
+ // Start recall at one character so grouped causal work can overlap later
827
+ // keystrokes. Nothing is displayed until three characters.
828
+ var wordCandidates = currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH ? wordTrie.getCandidates(currentWord, MAX_CANDIDATES) : [];
494
829
 
495
830
  // Gap-fill from the L3 general-English trie, requesting a full buffer so
496
831
  // enough survive de-duplication against the L2 results.
497
- if (candidates.length < MAX_CANDIDATES) {
832
+ if (currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH && wordCandidates.length < MAX_CANDIDATES) {
498
833
  var l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
499
- var existingWords = new Set(candidates.map(function (c) {
834
+ var existingWords = new Set(wordCandidates.map(function (c) {
500
835
  return c.word;
501
836
  }));
502
- var _iterator0 = _createForOfIteratorHelper(l3Candidates),
503
- _step0;
837
+ var _iterator1 = _createForOfIteratorHelper(l3Candidates),
838
+ _step1;
504
839
  try {
505
- for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
506
- var l3c = _step0.value;
507
- if (candidates.length >= MAX_CANDIDATES) break;
840
+ for (_iterator1.s(); !(_step1 = _iterator1.n()).done;) {
841
+ var l3c = _step1.value;
842
+ if (wordCandidates.length >= MAX_CANDIDATES) break;
508
843
  if (!existingWords.has(l3c.word)) {
509
- candidates.push(l3c);
844
+ wordCandidates.push(l3c);
510
845
  }
511
846
  }
512
847
  } catch (err) {
513
- _iterator0.e(err);
848
+ _iterator1.e(err);
514
849
  } finally {
515
- _iterator0.f();
850
+ _iterator1.f();
851
+ }
852
+ }
853
+
854
+ // Multi-word completion terms (bigrams/phrases) matched on the trailing
855
+ // window ending at the current partial token. Runs even when the unigram path
856
+ // was skipped for a short partial, so first-word/early phrase matches surface.
857
+ var phraseCandidates = getPhraseCandidates(trimmed);
858
+
859
+ // Unify: a word completes the current partial token; a phrase completes its
860
+ // matched multi-word window. Track the prefix length per term so the ghost
861
+ // tail is sliced correctly regardless of term type.
862
+ var matched = [].concat((0, _toConsumableArray2.default)(wordCandidates.map(function (_ref) {
863
+ var word = _ref.word,
864
+ node = _ref.node;
865
+ return {
866
+ word: word,
867
+ node: node,
868
+ matchedPrefixLen: currentWord.length,
869
+ surfaceStart: currentWordStart
870
+ };
871
+ })), (0, _toConsumableArray2.default)(phraseCandidates));
872
+ var positionCache = (0, _canonicalLmScoring.createCanonicalContextPositionCache)();
873
+ var canonicalMatched = matched.map(function (candidate) {
874
+ return _objectSpread(_objectSpread({}, candidate), (0, _canonicalLmScoring.deriveCanonicalCandidateContext)(trimmed, candidate.matchedPrefixLen, candidate.word, _slowLaneClient.getCanonicalSurfaceTokenIds, candidate.surfaceStart, positionCache));
875
+ });
876
+ var prefixLenByWord = new Map();
877
+ var _iterator10 = _createForOfIteratorHelper(canonicalMatched),
878
+ _step10;
879
+ try {
880
+ for (_iterator10.s(); !(_step10 = _iterator10.n()).done;) {
881
+ var m = _step10.value;
882
+ if (!prefixLenByWord.has(m.word)) {
883
+ prefixLenByWord.set(m.word, m.matchedPrefixLen);
884
+ }
885
+ }
886
+ } catch (err) {
887
+ _iterator10.e(err);
888
+ } finally {
889
+ _iterator10.f();
890
+ }
891
+ return {
892
+ canonicalMatched: canonicalMatched,
893
+ prefixLenByWord: prefixLenByWord
894
+ };
895
+ };
896
+
897
+ /**
898
+ * Recall and canonical derivation for `trimmed`, reusing the previous result
899
+ * when nothing they depend on has changed.
900
+ *
901
+ * `predict()` runs several times for one keystroke — once when the decision
902
+ * opens and again for each async evidence signal that lands inside the budget —
903
+ * and only the evidence differs between those runs. Without this, every run
904
+ * re-walks the tries and re-derives a canonical context per candidate.
905
+ */
906
+ var getCanonicalRecall = function getCanonicalRecall(trimmed, currentWord, currentWordStart) {
907
+ var surfaceCount = (0, _slowLaneClient.getCanonicalSurfaceCount)();
908
+ if (recallMemo !== null && recallMemo.trimmed === trimmed && recallMemo.generation === recallGeneration && recallMemo.surfaceCount === surfaceCount) {
909
+ return recallMemo;
910
+ }
911
+ var recall = computeCanonicalRecall(trimmed, currentWord, currentWordStart);
912
+ recallMemo = _objectSpread(_objectSpread({}, recall), {}, {
913
+ generation: recallGeneration,
914
+ surfaceCount: surfaceCount,
915
+ trimmed: trimmed
916
+ });
917
+ return recall;
918
+ };
919
+ var predict = exports.predict = function predict(textBefore) {
920
+ var _trimmed$match$, _trimmed$match, _canonicalMatched$fin, _eligible$, _eligible$0$score, _eligible$2, _judgedEvidenceFor3, _prefixLenByWord$get2, _runtimeBySurface$get5, _judgedEvidenceFor$me3, _judgedEvidenceFor6, _contextTotals$get$le4, _contextTotals$get4, _selectedEvidence$ver, _selectedEvidence$tot, _selectedEvidence$ver2;
921
+ if (!isInitialized) {
922
+ // Vocabulary artifacts load asynchronously over the autocomplete gateway.
923
+ // Kick off the load and skip this keystroke; the plugin also primes it on
924
+ // focus, so the tries are usually ready before the user types.
925
+ void loadDefaultVocabulary({
926
+ source: 'predict'
927
+ }).catch(function () {});
928
+ return null;
929
+ }
930
+ var t0 = performance.now();
931
+
932
+ // Advance the post-accept cooldown once per prediction. It stays active until
933
+ // BOTH the keystroke budget and the time window have elapsed; while active it
934
+ // suppresses re-offering the just-accepted surface (see arbitration below).
935
+ var cooledSurface = null;
936
+ if (acceptCooldown) {
937
+ acceptCooldown.predictionsSince += 1;
938
+ var stillByKeys = acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES;
939
+ var stillByTime = performance.now() - acceptCooldown.ts < COOLDOWN_MS;
940
+ if (stillByKeys || stillByTime) {
941
+ cooledSurface = acceptCooldown.surface;
942
+ } else {
943
+ acceptCooldown = null;
944
+ }
945
+ }
946
+
947
+ // ── Boundary/early-prefix prefetch; display remains gated at ≥3 ─────────
948
+ if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
949
+ var boundaryContext = (0, _canonicalLmScoring.deriveWhitespaceBoundaryContext)(textBefore);
950
+ if (boundaryContext) {
951
+ (0, _slowLaneClient.primeBoundaryLm)(_objectSpread(_objectSpread({}, boundaryContext), {}, {
952
+ familyKey: boundaryContext.contextKey,
953
+ priority: 0
954
+ }));
516
955
  }
956
+ return null;
517
957
  }
958
+ var trimmed = textBefore.trimEnd();
959
+ var trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
960
+ if (trailingSurfaceToken.length === 0) {
961
+ return null;
962
+ }
963
+ var currentWord = trailingSurfaceToken;
964
+ var currentWordStart = trimmed.length - currentWord.length;
965
+ var _getCanonicalRecall = getCanonicalRecall(trimmed, currentWord, currentWordStart),
966
+ canonicalMatched = _getCanonicalRecall.canonicalMatched,
967
+ prefixLenByWord = _getCanonicalRecall.prefixLenByWord;
518
968
 
519
- // If both Tries are completely empty for this prefix
520
- if (candidates.length === 0) {
969
+ // If every trie was empty for this prefix
970
+ if (canonicalMatched.length === 0) {
971
+ if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
972
+ // eslint-disable-next-line no-console
973
+ console.log("%c[CTC]%c \u2014 abstain: no matches for \"".concat(currentWord, "\""), _debugMode.CTC_STYLES.brand, _debugMode.CTC_STYLES.body);
974
+ }
521
975
  return null;
522
976
  }
523
977
  var previousWord = extractPreviousWord(trimmed);
524
978
  var contextVector = getContextVectorForScoring(trimmed);
525
979
  var lmLogits = (0, _slowLaneClient.getStoredLmLogits)();
526
-
527
- // Raw LM Output Logger
528
- if ((0, _debugMode.isAutocompleteDebugEnabled)() && lmLogits && Object.keys(lmLogits).length > 0) {
529
- var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
530
- return b[1] - a[1];
531
- }).slice(0, 5).map(function (_ref) {
532
- var _ref2 = (0, _slicedToArray2.default)(_ref, 2),
533
- word = _ref2[0],
534
- score = _ref2[1];
535
- return {
536
- Word: word,
537
- Prob: Number(score.toFixed(5))
538
- };
539
- });
540
- // eslint-disable-next-line no-console
541
- console.log('%c[Raw LM Prediction]🧠', 'color: #e83e8c; font-weight: bold;', rawLmTop);
542
- }
543
980
  var mode = contextVector ? 'warm' : 'cold';
544
981
 
545
- // Build ScoringCandidate array from TrieNodes
546
- var scoringCandidates = candidates.map(function (_ref3) {
547
- var word = _ref3.word,
548
- node = _ref3.node;
982
+ // Build ScoringCandidate array from matched terms (words + phrases)
983
+ var scoringCandidates = canonicalMatched.map(function (_ref2) {
984
+ var word = _ref2.word,
985
+ node = _ref2.node;
549
986
  return {
550
987
  word: word,
551
988
  tenantFreq: node.tenantFreq,
552
989
  docFreq: node.docFreq,
553
990
  authorFreq: node.authorFreq,
554
- sessionFreq: node.sessionFreq
991
+ sessionFreq: node.sessionFreq,
992
+ termType: node.termType,
993
+ posSeq: node.posSeq,
994
+ headPos: node.headPos,
995
+ association: node.association
555
996
  };
556
997
  });
557
998
 
558
999
  // Filter the LM payload to only words matching the current prefix so that
559
1000
  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
560
1001
  var prefix = currentWord.toLowerCase();
561
- var prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
562
- var _ref5 = (0, _slicedToArray2.default)(_ref4, 1),
563
- word = _ref5[0];
1002
+ var currentWordSeparator = (_canonicalMatched$fin = canonicalMatched.find(function (candidate) {
1003
+ return candidate.node.termType === 'word';
1004
+ })) === null || _canonicalMatched$fin === void 0 ? void 0 : _canonicalMatched$fin.separatorKind;
1005
+ var prefixLmLogits = lmLogits && currentWordSeparator === 'whitespace' ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref3) {
1006
+ var _ref4 = (0, _slicedToArray2.default)(_ref3, 1),
1007
+ word = _ref4[0];
564
1008
  return word.startsWith(prefix);
565
1009
  })) : null;
1010
+ var canonicalScoringSupported = (0, _slowLaneClient.isCanonicalSurfaceScoringSupported)();
1011
+ var eligibleContextKeys = Array.from(new Set(canonicalMatched.filter(function (candidate) {
1012
+ return candidate.canonicalTokenIds !== null;
1013
+ }).map(function (candidate) {
1014
+ return candidate.contextKey;
1015
+ }))).sort();
1016
+ var familyKey = eligibleContextKeys.join("\x01");
1017
+ var primeRequests = (0, _canonicalLmScoring.selectBoundaryPrimeRequests)(familyKey, canonicalMatched, PHRASE_MAX_WORDS);
1018
+ var _iterator11 = _createForOfIteratorHelper(primeRequests),
1019
+ _step11;
1020
+ try {
1021
+ for (_iterator11.s(); !(_step11 = _iterator11.n()).done;) {
1022
+ var request = _step11.value;
1023
+ (0, _slowLaneClient.primeBoundaryLm)(request);
1024
+ }
1025
+ } catch (err) {
1026
+ _iterator11.e(err);
1027
+ } finally {
1028
+ _iterator11.f();
1029
+ }
1030
+ var runtimeBySurface = new Map(canonicalMatched.map(function (candidate) {
1031
+ return [candidate.word, candidate];
1032
+ }));
1033
+ var canonicalEvidence = new Map();
1034
+ var firstTokenGroups = new Map();
1035
+ var _iterator12 = _createForOfIteratorHelper(canonicalMatched),
1036
+ _step12;
1037
+ try {
1038
+ for (_iterator12.s(); !(_step12 = _iterator12.n()).done;) {
1039
+ var _firstTokenGroups$get;
1040
+ var _candidate = _step12.value;
1041
+ if (_candidate.canonicalTokenIds === null) {
1042
+ continue;
1043
+ }
1044
+ // This accessor also materializes an exact cache entry when all tokens
1045
+ // have already been covered by shared-prefix work.
1046
+ (0, _slowLaneClient.getProgressiveSurfaceEvidence)(_candidate.contextKey, _candidate.word);
1047
+ var exact = (0, _slowLaneClient.getSurfaceScore)(_candidate.contextKey, _candidate.word);
1048
+ if (exact) {
1049
+ canonicalEvidence.set(_candidate.word, {
1050
+ contextKey: _candidate.contextKey,
1051
+ meanTokenLogProbability: exact.meanTokenLogProbability,
1052
+ score: 0,
1053
+ separatorKind: 'whitespace',
1054
+ source: 'canonical-full-surface',
1055
+ totalLogProbability: exact.totalLogProbability,
1056
+ totalSurfaceCharCount: _candidate.word.length,
1057
+ totalTokenCount: exact.tokenCount,
1058
+ verifiedCharCount: _candidate.word.length,
1059
+ verifiedTokenCount: exact.tokenCount
1060
+ });
1061
+ continue;
1062
+ }
1063
+ var _group2 = (_firstTokenGroups$get = firstTokenGroups.get(_candidate.contextKey)) !== null && _firstTokenGroups$get !== void 0 ? _firstTokenGroups$get : [];
1064
+ _group2.push(_candidate);
1065
+ firstTokenGroups.set(_candidate.contextKey, _group2);
1066
+ }
1067
+ } catch (err) {
1068
+ _iterator12.e(err);
1069
+ } finally {
1070
+ _iterator12.f();
1071
+ }
1072
+ var _iterator13 = _createForOfIteratorHelper(firstTokenGroups),
1073
+ _step13;
1074
+ try {
1075
+ for (_iterator13.s(); !(_step13 = _iterator13.n()).done;) {
1076
+ var _step13$value = (0, _slicedToArray2.default)(_step13.value, 2),
1077
+ _contextKey = _step13$value[0],
1078
+ _group3 = _step13$value[1];
1079
+ var boundary = (0, _slowLaneClient.getBoundaryLmState)(_contextKey);
1080
+ if (!boundary) {
1081
+ continue;
1082
+ }
1083
+ var maxLogit = -Infinity;
1084
+ var _iterator20 = _createForOfIteratorHelper(_group3),
1085
+ _step20;
1086
+ try {
1087
+ for (_iterator20.s(); !(_step20 = _iterator20.n()).done;) {
1088
+ var _candidate2$canonical;
1089
+ var _candidate2 = _step20.value;
1090
+ var tokenId = (_candidate2$canonical = _candidate2.canonicalTokenIds) === null || _candidate2$canonical === void 0 ? void 0 : _candidate2$canonical[0];
1091
+ var rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
1092
+ if (rawLogit !== undefined && Number.isFinite(rawLogit) && rawLogit > maxLogit) {
1093
+ maxLogit = rawLogit;
1094
+ }
1095
+ }
1096
+ } catch (err) {
1097
+ _iterator20.e(err);
1098
+ } finally {
1099
+ _iterator20.f();
1100
+ }
1101
+ if (!Number.isFinite(maxLogit)) {
1102
+ continue;
1103
+ }
1104
+ var _iterator21 = _createForOfIteratorHelper(_group3),
1105
+ _step21;
1106
+ try {
1107
+ for (_iterator21.s(); !(_step21 = _iterator21.n()).done;) {
1108
+ var _candidate3$canonical, _candidate3$canonical2, _candidate3$canonical3;
1109
+ var _candidate3 = _step21.value;
1110
+ var _tokenId = (_candidate3$canonical = _candidate3.canonicalTokenIds) === null || _candidate3$canonical === void 0 ? void 0 : _candidate3$canonical[0];
1111
+ var _rawLogit = _tokenId === undefined ? undefined : boundary.rawLogits[_tokenId];
1112
+ if (_rawLogit === undefined || !Number.isFinite(_rawLogit)) {
1113
+ continue;
1114
+ }
1115
+ var tokenCount = (_candidate3$canonical2 = (_candidate3$canonical3 = _candidate3.canonicalTokenIds) === null || _candidate3$canonical3 === void 0 ? void 0 : _candidate3$canonical3.length) !== null && _candidate3$canonical2 !== void 0 ? _candidate3$canonical2 : 0;
1116
+ if (_tokenId !== undefined && tokenCount === 1) {
1117
+ var totalLogProbability = (0, _canonicalLmScoring.logSoftmaxAt)(boundary.rawLogits, _tokenId);
1118
+ if (!Number.isFinite(totalLogProbability)) {
1119
+ continue;
1120
+ }
1121
+ canonicalEvidence.set(_candidate3.word, {
1122
+ contextKey: _contextKey,
1123
+ meanTokenLogProbability: totalLogProbability,
1124
+ score: 0,
1125
+ separatorKind: 'whitespace',
1126
+ source: 'canonical-full-surface',
1127
+ totalLogProbability: totalLogProbability,
1128
+ totalSurfaceCharCount: _candidate3.word.length,
1129
+ totalTokenCount: 1,
1130
+ verifiedCharCount: _candidate3.word.length,
1131
+ verifiedTokenCount: 1
1132
+ });
1133
+ continue;
1134
+ }
1135
+ canonicalEvidence.set(_candidate3.word, {
1136
+ contextKey: _contextKey,
1137
+ rawLogit: _rawLogit,
1138
+ score: 0,
1139
+ separatorKind: 'whitespace',
1140
+ source: 'canonical-first-token',
1141
+ totalSurfaceCharCount: _candidate3.word.length,
1142
+ totalTokenCount: tokenCount,
1143
+ verifiedCharCount: 0,
1144
+ verifiedTokenCount: 1
1145
+ });
1146
+ }
1147
+ } catch (err) {
1148
+ _iterator21.e(err);
1149
+ } finally {
1150
+ _iterator21.f();
1151
+ }
1152
+ }
1153
+ } catch (err) {
1154
+ _iterator13.e(err);
1155
+ } finally {
1156
+ _iterator13.f();
1157
+ }
566
1158
  var _rankCandidates = (0, _scoringPipeline.rankCandidates)(scoringCandidates, contextVector, function (w) {
567
1159
  return getWordVector(w);
568
- }, prefixLmLogits, wordTrie.maxTenantFreq, previousWord),
1160
+ }, prefixLmLogits, {
1161
+ word: wordTrie.maxTenantFreq,
1162
+ bigram: maxBigramFreq,
1163
+ phrase: maxPhraseFreq
1164
+ }, previousWord, function (surface) {
1165
+ var _canonicalEvidence$ge;
1166
+ return (_canonicalEvidence$ge = canonicalEvidence.get(surface)) !== null && _canonicalEvidence$ge !== void 0 ? _canonicalEvidence$ge : null;
1167
+ }),
569
1168
  ranked = _rankCandidates.candidates,
570
1169
  grammarMeta = _rankCandidates.grammarMeta,
571
1170
  pipelineDebug = _rankCandidates.pipelineDebug;
572
- var best = ranked[0];
573
- var suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
1171
+ var progressiveEligible = ranked.filter(function (candidate) {
1172
+ var runtime = runtimeBySurface.get(candidate.word);
1173
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined;
1174
+ });
1175
+ if (canonicalScoringSupported && progressiveEligible.length > 0) {
1176
+ var byContext = new Map();
1177
+ var _iterator14 = _createForOfIteratorHelper(progressiveEligible),
1178
+ _step14;
1179
+ try {
1180
+ for (_iterator14.s(); !(_step14 = _iterator14.n()).done;) {
1181
+ var _byContext$get;
1182
+ var candidate = _step14.value;
1183
+ var runtime = runtimeBySurface.get(candidate.word);
1184
+ if (!runtime || runtime.canonicalTokenIds === null) {
1185
+ continue;
1186
+ }
1187
+ var group = (_byContext$get = byContext.get(runtime.contextKey)) !== null && _byContext$get !== void 0 ? _byContext$get : [];
1188
+ group.push({
1189
+ runtime: runtime,
1190
+ rankHint: candidate.stage1Score
1191
+ });
1192
+ byContext.set(runtime.contextKey, group);
1193
+ }
1194
+ } catch (err) {
1195
+ _iterator14.e(err);
1196
+ } finally {
1197
+ _iterator14.f();
1198
+ }
1199
+ var _iterator15 = _createForOfIteratorHelper(byContext),
1200
+ _step15;
1201
+ try {
1202
+ for (_iterator15.s(); !(_step15 = _iterator15.n()).done;) {
1203
+ var _step15$value = (0, _slicedToArray2.default)(_step15.value, 2),
1204
+ contextKey = _step15$value[0],
1205
+ _group = _step15$value[1];
1206
+ (0, _slowLaneClient.requestProgressiveSurfaceScores)({
1207
+ familyKey: familyKey,
1208
+ contextKey: contextKey,
1209
+ prompt: _group[0].runtime.contextBeforeSurface,
1210
+ candidates: _group.map(function (_ref5) {
1211
+ var _runtime$canonicalTok;
1212
+ var runtime = _ref5.runtime,
1213
+ rankHint = _ref5.rankHint;
1214
+ return {
1215
+ surface: runtime.word,
1216
+ tokenIds: (_runtime$canonicalTok = runtime.canonicalTokenIds) !== null && _runtime$canonicalTok !== void 0 ? _runtime$canonicalTok : [],
1217
+ rankHint: rankHint
1218
+ };
1219
+ })
1220
+ });
1221
+ }
1222
+ } catch (err) {
1223
+ _iterator15.e(err);
1224
+ } finally {
1225
+ _iterator15.f();
1226
+ }
1227
+ }
1228
+
1229
+ // The one/two-character passes exist only to hide model latency. They may
1230
+ // prime and expand token groups above, but cannot commit UI.
1231
+ var displayPrefixReady = currentWord.length >= DISPLAY_MIN_PREFIX_LENGTH;
1232
+
1233
+ // ── Arbitration: full-surface evidence → plausibility → winner margin ──
1234
+ var canonicalLmSupported = canonicalScoringSupported;
1235
+ var hasExactEvidence = function hasExactEvidence(candidate) {
1236
+ var _candidate$lmEvidence;
1237
+ return ((_candidate$lmEvidence = candidate.lmEvidence) === null || _candidate$lmEvidence === void 0 ? void 0 : _candidate$lmEvidence.source) === 'canonical-full-surface' && candidate.lmEvidence.meanTokenLogProbability !== undefined && Number.isFinite(candidate.lmEvidence.meanTokenLogProbability);
1238
+ };
1239
+
1240
+ /**
1241
+ * The evidence this candidate is judged on: the summed log-probability of
1242
+ * the tokens that have been read, their per-token mean, and whether they
1243
+ * cover the whole surface.
1244
+ *
1245
+ * `total` is the quantity the posterior is built from — it is a sequence
1246
+ * log-likelihood, so it is comparable across surfaces of different lengths
1247
+ * in a way the mean is not. `mean` is retained for reporting and for the
1248
+ * oracle arms, which compare aggregation rules against each other.
1249
+ *
1250
+ * Every token read costs a round trip, so a surface longer than
1251
+ * `REQUIRED_VERIFIED_TOKENS` is judged on its leading tokens and its tail is
1252
+ * left unread. Callers compensate with a wider winner margin rather than
1253
+ * treating the two kinds of evidence as equivalent.
1254
+ *
1255
+ * This is the depth-unaware read, and it is what the pool below is built
1256
+ * from. Arbitration goes through `judgedEvidenceFor`, which adds the one
1257
+ * requirement that can only be stated once the pool is known: a surface
1258
+ * cannot be judged against members it has not yet been read level with.
1259
+ */
1260
+ var readJudgedEvidence = function readJudgedEvidence(candidate) {
1261
+ var evidence = candidate.lmEvidence;
1262
+ var exactMean = evidence === null || evidence === void 0 ? void 0 : evidence.meanTokenLogProbability;
1263
+ if (hasExactEvidence(candidate) && evidence && exactMean !== undefined) {
1264
+ var _evidence$totalLogPro;
1265
+ var _total = (_evidence$totalLogPro = evidence.totalLogProbability) !== null && _evidence$totalLogPro !== void 0 ? _evidence$totalLogPro : exactMean * evidence.totalTokenCount;
1266
+ return Number.isFinite(_total) ? {
1267
+ complete: true,
1268
+ mean: exactMean,
1269
+ total: _total
1270
+ } : null;
1271
+ }
1272
+ var runtime = runtimeBySurface.get(candidate.word);
1273
+ if (!runtime || runtime.canonicalTokenIds === null) {
1274
+ return null;
1275
+ }
1276
+ var progressive = (0, _slowLaneClient.getProgressiveSurfaceEvidence)(runtime.contextKey, candidate.word);
1277
+ if (!progressive || progressive.scoredTokenCount < REQUIRED_VERIFIED_TOKENS) {
1278
+ return null;
1279
+ }
1280
+ var total = progressive.totalLogProbability;
1281
+ var mean = total / progressive.scoredTokenCount;
1282
+ return Number.isFinite(mean) && Number.isFinite(total) ? {
1283
+ complete: false,
1284
+ mean: mean,
1285
+ total: total
1286
+ } : null;
1287
+ };
1288
+
1289
+ // Normalise each context's shortlist into a posterior. `total` is a sequence
1290
+ // log-likelihood, so `exp(total - logSumExp(totals))` is the share of the
1291
+ // shortlist's probability mass the model puts on that surface.
1292
+ //
1293
+ // Unknown future token log-probabilities cannot exceed zero, so a partially
1294
+ // read candidate's total is an upper bound on its eventual total, and its
1295
+ // posterior an upper bound on its eventual posterior. That is what lets the
1296
+ // plausibility floor and the display gate be the same threshold — one
1297
+ // applied to the bound, one to the verified value — instead of two numbers
1298
+ // that have to be kept consistent by hand.
1299
+ //
1300
+ // Candidates with no evidence at all are left out of the sum rather than
1301
+ // defaulted. Any default would have to stand in for a log-probability, and a
1302
+ // value low enough not to distort the normaliser is indistinguishable from
1303
+ // omitting the candidate.
1304
+ var optimisticTotalByWord = new Map();
1305
+ var contextTotals = new Map();
1306
+ var contextSurfaces = new Map();
1307
+ // Counted before the total is looked up, so this is every candidate the
1308
+ // context could have scored rather than every candidate it did. The gap
1309
+ // between the two is what separates a context holding one match from a
1310
+ // context whose rivals have not come back yet.
1311
+ var contextRequestedCount = new Map();
1312
+ var bestTotalByContext = new Map();
1313
+ var bestCandidateByContext = new Map();
1314
+ var _iterator16 = _createForOfIteratorHelper(ranked),
1315
+ _step16;
1316
+ try {
1317
+ for (_iterator16.s(); !(_step16 = _iterator16.n()).done;) {
1318
+ var _contextRequestedCoun2, _judged$total;
1319
+ var _candidate4 = _step16.value;
1320
+ var _runtime = runtimeBySurface.get(_candidate4.word);
1321
+ if (!_runtime || _runtime.canonicalTokenIds === null) {
1322
+ continue;
1323
+ }
1324
+ contextRequestedCount.set(_runtime.contextKey, ((_contextRequestedCoun2 = contextRequestedCount.get(_runtime.contextKey)) !== null && _contextRequestedCoun2 !== void 0 ? _contextRequestedCoun2 : 0) + 1);
1325
+ var judged = readJudgedEvidence(_candidate4);
1326
+ var progressive = (0, _slowLaneClient.getProgressiveSurfaceEvidence)(_runtime.contextKey, _candidate4.word);
1327
+ // Once a candidate has been read far enough to judge, its verified total
1328
+ // replaces the bound.
1329
+ var optimisticTotal = (_judged$total = judged === null || judged === void 0 ? void 0 : judged.total) !== null && _judged$total !== void 0 ? _judged$total : progressive === null || progressive === void 0 ? void 0 : progressive.totalLogProbability;
1330
+ if (optimisticTotal === undefined || !Number.isFinite(optimisticTotal)) {
1331
+ continue;
1332
+ }
1333
+ optimisticTotalByWord.set(_candidate4.word, optimisticTotal);
1334
+ var totals = contextTotals.get(_runtime.contextKey);
1335
+ var surfaces = contextSurfaces.get(_runtime.contextKey);
1336
+ if (totals && surfaces) {
1337
+ totals.push(optimisticTotal);
1338
+ surfaces.push(_candidate4.word);
1339
+ } else {
1340
+ contextTotals.set(_runtime.contextKey, [optimisticTotal]);
1341
+ contextSurfaces.set(_runtime.contextKey, [_candidate4.word]);
1342
+ }
1343
+ var best = bestTotalByContext.get(_runtime.contextKey);
1344
+ if (best === undefined || optimisticTotal > best) {
1345
+ bestTotalByContext.set(_runtime.contextKey, optimisticTotal);
1346
+ bestCandidateByContext.set(_runtime.contextKey, _candidate4);
1347
+ }
1348
+ }
1349
+
1350
+ /**
1351
+ * Whether a shorter member of the same pool is this surface's word-boundary
1352
+ * prefix — `happy path` where `happy` is also present.
1353
+ *
1354
+ * Only a multi-word surface can extend another, and a pool is a set, so this
1355
+ * costs one lookup per space rather than a comparison against every member.
1356
+ */
1357
+ } catch (err) {
1358
+ _iterator16.e(err);
1359
+ } finally {
1360
+ _iterator16.f();
1361
+ }
1362
+ var extendsAPoolMember = function extendsAPoolMember(surface, pool) {
1363
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1364
+ if (pool.has(surface.slice(0, space))) {
1365
+ return true;
1366
+ }
1367
+ }
1368
+ return false;
1369
+ };
1370
+
1371
+ // A sequence total only falls as more of the sequence is read, so a partial
1372
+ // sum over `k` tokens sits above the same surface's total over `m > k`. An
1373
+ // extension read less far than the member it continues therefore carries a
1374
+ // total above that member's — and because the chain rule narrows the
1375
+ // denominator to the pool's minimal members, `exp(total - normaliser)` comes
1376
+ // out above 1 for a surface whose own continuation has not been looked at.
1377
+ // It needs a three-token first word to happen, which 2.6% of served bigrams
1378
+ // have, and it inflates both the posterior the gate reads and the
1379
+ // `confidenceScore` every other candidate's margin is measured against.
1380
+ //
1381
+ // A surface's canonical tokenization begins with the tokenization of its own
1382
+ // leading words — the producer encodes `" " + surface` in one call and the
1383
+ // ByteLevel pre-tokenizer splits on word boundaries, so no merge crosses a
1384
+ // space — which is what makes both corrections below exact rather than
1385
+ // approximate.
1386
+ //
1387
+ // The total is capped at what it continues: `P(happy path)` cannot exceed
1388
+ // `P(happy)`. Shortest-first, so a three-word phrase is capped against a
1389
+ // two-word prefix that has already been capped itself.
1390
+ //
1391
+ // And the surface is held unjudgeable until it has been read level with what
1392
+ // it continues, because the cap alone would leave it looking exactly as
1393
+ // certain as its prefix while saying nothing about its own tail.
1394
+ var underReadExtensions = new Set();
1395
+ var _iterator17 = _createForOfIteratorHelper(contextSurfaces),
1396
+ _step17;
1397
+ try {
1398
+ for (_iterator17.s(); !(_step17 = _iterator17.n()).done;) {
1399
+ var _step17$value = (0, _slicedToArray2.default)(_step17.value, 2),
1400
+ _contextKey2 = _step17$value[0],
1401
+ _surfaces = _step17$value[1];
1402
+ var pool = new Set(_surfaces);
1403
+ var _iterator22 = _createForOfIteratorHelper((0, _toConsumableArray2.default)(_surfaces).sort(function (a, b) {
1404
+ return a.length - b.length;
1405
+ })),
1406
+ _step22;
1407
+ try {
1408
+ for (_iterator22.s(); !(_step22 = _iterator22.n()).done;) {
1409
+ var _getProgressiveSurfac, _getProgressiveSurfac2, _runtimeBySurface$get8, _runtimeBySurface$get9;
1410
+ var surface = _step22.value;
1411
+ var own = optimisticTotalByWord.get(surface);
1412
+ if (own === undefined) {
1413
+ continue;
1414
+ }
1415
+ var cap = Infinity;
1416
+ var continuedDepth = 0;
1417
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1418
+ var _runtimeBySurface$get6, _runtimeBySurface$get7;
1419
+ var _prefix = surface.slice(0, space);
1420
+ if (!pool.has(_prefix)) {
1421
+ continue;
1422
+ }
1423
+ var prefixTotal = optimisticTotalByWord.get(_prefix);
1424
+ if (prefixTotal !== undefined && prefixTotal < cap) {
1425
+ cap = prefixTotal;
1426
+ }
1427
+ var prefixDepth = (_runtimeBySurface$get6 = (_runtimeBySurface$get7 = runtimeBySurface.get(_prefix)) === null || _runtimeBySurface$get7 === void 0 || (_runtimeBySurface$get7 = _runtimeBySurface$get7.canonicalTokenIds) === null || _runtimeBySurface$get7 === void 0 ? void 0 : _runtimeBySurface$get7.length) !== null && _runtimeBySurface$get6 !== void 0 ? _runtimeBySurface$get6 : 0;
1428
+ if (prefixDepth > continuedDepth) {
1429
+ continuedDepth = prefixDepth;
1430
+ }
1431
+ }
1432
+ if (own > cap) {
1433
+ optimisticTotalByWord.set(surface, cap);
1434
+ }
1435
+ var scoredDepth = (_getProgressiveSurfac = (_getProgressiveSurfac2 = (0, _slowLaneClient.getProgressiveSurfaceEvidence)(_contextKey2, surface)) === null || _getProgressiveSurfac2 === void 0 ? void 0 : _getProgressiveSurfac2.scoredTokenCount) !== null && _getProgressiveSurfac !== void 0 ? _getProgressiveSurfac : 0;
1436
+ var surfaceDepth = (_runtimeBySurface$get8 = (_runtimeBySurface$get9 = runtimeBySurface.get(surface)) === null || _runtimeBySurface$get9 === void 0 || (_runtimeBySurface$get9 = _runtimeBySurface$get9.canonicalTokenIds) === null || _runtimeBySurface$get9 === void 0 ? void 0 : _runtimeBySurface$get9.length) !== null && _runtimeBySurface$get8 !== void 0 ? _runtimeBySurface$get8 : 0;
1437
+ if (scoredDepth < continuedDepth && scoredDepth < surfaceDepth) {
1438
+ underReadExtensions.add(surface);
1439
+ }
1440
+ }
1441
+ } catch (err) {
1442
+ _iterator22.e(err);
1443
+ } finally {
1444
+ _iterator22.f();
1445
+ }
1446
+ }
1447
+
1448
+ /**
1449
+ * The evidence arbitration judges a candidate on.
1450
+ *
1451
+ * Wraps the depth-unaware read with the pool-relative requirement: a surface
1452
+ * that continues another pool member is not judgeable until it has been read
1453
+ * level with it, so the two totals being compared are taken at comparable
1454
+ * depths.
1455
+ */
1456
+ } catch (err) {
1457
+ _iterator17.e(err);
1458
+ } finally {
1459
+ _iterator17.f();
1460
+ }
1461
+ var judgedEvidenceFor = function judgedEvidenceFor(candidate) {
1462
+ return underReadExtensions.has(candidate.word) ? null : readJudgedEvidence(candidate);
1463
+ };
1464
+ var hasJudgeableEvidence = function hasJudgeableEvidence(candidate) {
1465
+ return judgedEvidenceFor(candidate) !== null;
1466
+ };
1467
+
1468
+ // Normalise over the pool's minimal members only, and score everyone —
1469
+ // extensions included — against that.
1470
+ //
1471
+ // A sequence total is its prefix's total plus a log-probability that cannot
1472
+ // be positive, so summing a surface and its own extension double-counts:
1473
+ // `exp(total)` for `happy` already contains every continuation of `happy`,
1474
+ // `happy path` among them. Dividing one pool's mass between the two capped
1475
+ // the shorter form at half however certain it was, and the longer form below
1476
+ // that — both under gates neither could then clear, for a reason that was
1477
+ // arithmetic rather than evidence.
1478
+ //
1479
+ // Excluding extensions from the denominator is the chain rule: the minimal
1480
+ // members partition the mass, and an extension's share of that same
1481
+ // denominator is `P(prefix) * P(continuation | prefix)`, which is exactly
1482
+ // what a multi-word suggestion should be held to. Extensions stay in
1483
+ // `contextTotals`, so how contested a context is still counts every scored
1484
+ // candidate.
1485
+ var logSumExpByContext = new Map();
1486
+ var _iterator18 = _createForOfIteratorHelper(contextSurfaces),
1487
+ _step18;
1488
+ try {
1489
+ for (_iterator18.s(); !(_step18 = _iterator18.n()).done;) {
1490
+ var _step18$value = (0, _slicedToArray2.default)(_step18.value, 2),
1491
+ _contextKey3 = _step18$value[0],
1492
+ _surfaces2 = _step18$value[1];
1493
+ var _pool = new Set(_surfaces2);
1494
+ var minimalTotals = [];
1495
+ var _iterator23 = _createForOfIteratorHelper(_surfaces2),
1496
+ _step23;
1497
+ try {
1498
+ for (_iterator23.s(); !(_step23 = _iterator23.n()).done;) {
1499
+ var _surface = _step23.value;
1500
+ var total = optimisticTotalByWord.get(_surface);
1501
+ if (total !== undefined && !extendsAPoolMember(_surface, _pool)) {
1502
+ minimalTotals.push(total);
1503
+ }
1504
+ }
1505
+ // An extension is strictly longer than what it extends, so the shortest
1506
+ // member of any non-empty pool is always minimal and this is never empty.
1507
+ } catch (err) {
1508
+ _iterator23.e(err);
1509
+ } finally {
1510
+ _iterator23.f();
1511
+ }
1512
+ logSumExpByContext.set(_contextKey3, (0, _canonicalLmScoring.logSumExp)(minimalTotals));
1513
+ }
1514
+
1515
+ /**
1516
+ * A candidate's share of its context shortlist's probability mass.
1517
+ *
1518
+ * Pass an optimistic (partially read) total for an upper bound on the
1519
+ * eventual posterior, or a verified total for the posterior itself.
1520
+ */
1521
+ } catch (err) {
1522
+ _iterator18.e(err);
1523
+ } finally {
1524
+ _iterator18.f();
1525
+ }
1526
+ var posteriorFor = function posteriorFor(candidate, total) {
1527
+ var runtime = runtimeBySurface.get(candidate.word);
1528
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
1529
+ if (total === undefined || normalizer === undefined || !Number.isFinite(normalizer)) {
1530
+ return 0;
1531
+ }
1532
+ return Math.exp(total - normalizer);
1533
+ };
1534
+ var optimisticPosterior = function optimisticPosterior(candidate) {
1535
+ return posteriorFor(candidate, optimisticTotalByWord.get(candidate.word));
1536
+ };
1537
+ var judgedPosterior = function judgedPosterior(candidate) {
1538
+ var _judgedEvidenceFor;
1539
+ return posteriorFor(candidate, (_judgedEvidenceFor = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor === void 0 ? void 0 : _judgedEvidenceFor.total);
1540
+ };
1541
+
1542
+ // A candidate stays in the race for as long as its optimistic posterior
1543
+ // could still clear the gate it will be held to. The bound only falls as
1544
+ // more tokens are read, so nothing dropped here could have gone on to win.
1545
+ var couldClearGate = function couldClearGate(candidate) {
1546
+ return optimisticPosterior(candidate) >= MIN_LM_POSTERIOR[candidate.termType];
1547
+ };
1548
+ var passesLmFloor = function passesLmFloor(candidate) {
1549
+ return judgedEvidenceFor(candidate) !== null && couldClearGate(candidate);
1550
+ };
1551
+ var hasRequiredEvidence = function hasRequiredEvidence(candidate) {
1552
+ var _candidate$lmEvidence2;
1553
+ if (canonicalLmSupported) {
1554
+ return passesLmFloor(candidate);
1555
+ }
1556
+ return candidate.termType === 'word' && ((_candidate$lmEvidence2 = candidate.lmEvidence) === null || _candidate$lmEvidence2 === void 0 ? void 0 : _candidate$lmEvidence2.source) === 'network-logit';
1557
+ };
1558
+
1559
+ // ── Stabilization guards (QI-2) ──────────────────────────────────────────
1560
+ // Whole-surface repetition: drop a candidate whose full surface already equals
1561
+ // the run of tokens immediately before the cursor — accepting it would
1562
+ // duplicate what was just typed (`end to end` → `end to end to end`).
1563
+ var beforeTokens = tokenize(trimmed);
1564
+ var duplicatesPreceding = function duplicatesPreceding(c) {
1565
+ var surfaceTokens = tokenize(c.word);
1566
+ var n = surfaceTokens.length;
1567
+ if (n === 0 || beforeTokens.length < n) {
1568
+ return false;
1569
+ }
1570
+ for (var i = 0; i < n; i++) {
1571
+ if (beforeTokens[beforeTokens.length - n + i] !== surfaceTokens[i]) {
1572
+ return false;
1573
+ }
1574
+ }
1575
+ return true;
1576
+ };
1577
+ // Post-accept cooldown: don't re-offer the surface the user just accepted while
1578
+ // its cooldown is still active (see cooldown advance near t0).
1579
+ var isCooledDown = function isCooledDown(c) {
1580
+ return cooledSurface !== null && c.word.toLowerCase() === cooledSurface;
1581
+ };
1582
+ var isStabilized = function isStabilized(candidate) {
1583
+ return !duplicatesPreceding(candidate) && !isCooledDown(candidate);
1584
+ };
1585
+ // A candidate the user has all but finished typing is not worth a ghost, and
1586
+ // a candidate we would never show is not worth waiting for either.
1587
+ var suggestionLengthFor = function suggestionLengthFor(candidate) {
1588
+ var _prefixLenByWord$get;
1589
+ return candidate.word.length - ((_prefixLenByWord$get = prefixLenByWord.get(candidate.word)) !== null && _prefixLenByWord$get !== void 0 ? _prefixLenByWord$get : currentWord.length);
1590
+ };
1591
+ var isWorthShowing = function isWorthShowing(candidate) {
1592
+ return suggestionLengthFor(candidate) >= MIN_SUGGESTION_LENGTH;
1593
+ };
1594
+ var evidenceBacked = ranked.filter(function (candidate) {
1595
+ return hasRequiredEvidence(candidate) && isStabilized(candidate);
1596
+ });
1597
+ var confidenceScore = function confidenceScore(candidate) {
1598
+ if (!canonicalLmSupported) {
1599
+ return candidate.finalScore;
1600
+ }
1601
+ var runtime = runtimeBySurface.get(candidate.word);
1602
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) === null || !optimisticTotalByWord.has(candidate.word)) {
1603
+ return candidate.stage1Score;
1604
+ }
1605
+ return _scoringPipeline.STAGE1_WEIGHT * candidate.stage1Score + _scoringPipeline.STAGE2_WEIGHT * optimisticPosterior(candidate);
1606
+ };
1607
+ var exactConfidenceScore = function exactConfidenceScore(candidate) {
1608
+ if (!canonicalLmSupported || judgedEvidenceFor(candidate) === null) {
1609
+ return candidate.finalScore;
1610
+ }
1611
+ return _scoringPipeline.STAGE1_WEIGHT * candidate.stage1Score + _scoringPipeline.STAGE2_WEIGHT * judgedPosterior(candidate);
1612
+ };
1613
+
1614
+ /**
1615
+ * How many candidates in this one's context were scored at all.
1616
+ *
1617
+ * Read off pool membership, which counts every candidate that contributed a
1618
+ * total, rather than off the normaliser, which the chain rule narrows to the
1619
+ * pool's minimal members.
1620
+ */
1621
+ var scoredPoolSize = function scoredPoolSize(candidate) {
1622
+ var _runtimeBySurface$get, _contextTotals$get$le, _contextTotals$get;
1623
+ var contextKey = (_runtimeBySurface$get = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get === void 0 ? void 0 : _runtimeBySurface$get.contextKey;
1624
+ return contextKey === undefined ? 0 : (_contextTotals$get$le = (_contextTotals$get = contextTotals.get(contextKey)) === null || _contextTotals$get === void 0 ? void 0 : _contextTotals$get.length) !== null && _contextTotals$get$le !== void 0 ? _contextTotals$get$le : 0;
1625
+ };
1626
+
1627
+ /**
1628
+ * How many candidates the context held before any of them were scored.
1629
+ *
1630
+ * A pool of one means two opposite things. Either the vocabulary offered a
1631
+ * single continuation of what was typed, which is the least contested case
1632
+ * there is; or rivals were offered and have not been read yet, which is the
1633
+ * least informed. Only the second deserves refusing, and the two are
1634
+ * distinguishable exactly here.
1635
+ */
1636
+ var requestedPoolSize = function requestedPoolSize(candidate) {
1637
+ var _runtimeBySurface$get2, _contextRequestedCoun;
1638
+ var contextKey = (_runtimeBySurface$get2 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get2 === void 0 ? void 0 : _runtimeBySurface$get2.contextKey;
1639
+ return contextKey === undefined ? 0 : (_contextRequestedCoun = contextRequestedCount.get(contextKey)) !== null && _contextRequestedCoun !== void 0 ? _contextRequestedCoun : 0;
1640
+ };
1641
+
1642
+ // A suppressed short completion still counts as competition below, so losing
1643
+ // to one abstains rather than promoting the runner-up in its place.
1644
+ var gateCleared = evidenceBacked.filter(isWorthShowing).map(function (candidate) {
1645
+ return {
1646
+ candidate: candidate,
1647
+ posterior: judgedPosterior(candidate),
1648
+ score: exactConfidenceScore(candidate)
1649
+ };
1650
+ })
1651
+ // The gate is the model's own confidence in the surface; the blended score
1652
+ // only orders what has already cleared it, so a strong corpus prior can no
1653
+ // longer carry a surface the model is unsure of onto the screen.
1654
+ .filter(function (_ref6) {
1655
+ var candidate = _ref6.candidate,
1656
+ posterior = _ref6.posterior;
1657
+ return posterior >= MIN_LM_POSTERIOR[candidate.termType];
1658
+ });
1659
+ /**
1660
+ * Whether the model finds the surface plausible on its own terms.
1661
+ *
1662
+ * The gate above asks which candidate won its pool; this asks whether
1663
+ * winning it meant anything. The two come apart exactly where a pool is
1664
+ * thin — a shortlist of one hands its only member 1.0 by construction — and
1665
+ * a relative test can never see through that however it is tuned.
1666
+ *
1667
+ * Waived without canonical scoring, where there is no judged mean to test
1668
+ * and the network-logit path would otherwise refuse everything.
1669
+ */
1670
+ var isPlausibleSurface = function isPlausibleSurface(candidate) {
1671
+ var _judgedEvidenceFor2;
1672
+ if (!canonicalLmSupported) {
1673
+ return true;
1674
+ }
1675
+ var mean = (_judgedEvidenceFor2 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor2 === void 0 ? void 0 : _judgedEvidenceFor2.mean;
1676
+ return mean !== undefined && mean >= MIN_MEAN_TOKEN_LOG_PROBABILITY;
1677
+ };
1678
+ // Kept as its own stage rather than folded into the gate so the three
1679
+ // populations stay separable in the abstain cascade: cleared its pool and was
1680
+ // implausible, was plausible and had nothing to clear the pool against, or
1681
+ // cleared both. A candidate refused here stays in `ranked` and so still
1682
+ // counts as competition below — promoting the runner-up in place of an
1683
+ // implausible leader would show something worse, not something better.
1684
+ var plausible = gateCleared.filter(function (_ref7) {
1685
+ var candidate = _ref7.candidate;
1686
+ return isPlausibleSurface(candidate);
1687
+ });
1688
+ // Applied after the gate rather than folded into it, so the two populations
1689
+ // stay separable: a surface refused here cleared its threshold and was
1690
+ // refused for having had nothing to clear it against.
1691
+ var eligible = plausible.filter(function (_ref8) {
1692
+ var candidate = _ref8.candidate;
1693
+ return !canonicalLmSupported || scoredPoolSize(candidate) >= MIN_SCORED_POOL_SIZE || requestedPoolSize(candidate) <= 1;
1694
+ }).sort(function (a, b) {
1695
+ return b.score - a.score;
1696
+ });
1697
+ var selected = (_eligible$ = eligible[0]) === null || _eligible$ === void 0 ? void 0 : _eligible$.candidate;
1698
+ var selectedDisplayScore = (_eligible$0$score = (_eligible$2 = eligible[0]) === null || _eligible$2 === void 0 ? void 0 : _eligible$2.score) !== null && _eligible$0$score !== void 0 ? _eligible$0$score : 0;
1699
+ var selectedEvidence = selected && hasRequiredEvidence(selected) ? selected.lmEvidence : null;
1700
+ /**
1701
+ * Whether `longer` is `shorter` continued past a word boundary, in the same
1702
+ * normaliser.
1703
+ *
1704
+ * Shared by the two places that have to agree on what an extension is: the
1705
+ * chain rule, which stops dividing a pool's mass between a sequence and its
1706
+ * own prefix, and the winner margin below, which stops treating one as the
1707
+ * other's rival. They were the same category error and are now the same test.
1708
+ */
1709
+ var extendsInSamePool = function extendsInSamePool(shorter, longer) {
1710
+ var _runtimeBySurface$get3, _runtimeBySurface$get4;
1711
+ if (longer.word === shorter.word || !optimisticTotalByWord.has(longer.word)) {
1712
+ return false;
1713
+ }
1714
+ var pool = (_runtimeBySurface$get3 = runtimeBySurface.get(shorter.word)) === null || _runtimeBySurface$get3 === void 0 ? void 0 : _runtimeBySurface$get3.contextKey;
1715
+ if (pool === undefined || ((_runtimeBySurface$get4 = runtimeBySurface.get(longer.word)) === null || _runtimeBySurface$get4 === void 0 ? void 0 : _runtimeBySurface$get4.contextKey) !== pool) {
1716
+ return false;
1717
+ }
1718
+ return longer.word.length > shorter.word.length && longer.word.startsWith(shorter.word) && longer.word[shorter.word.length] === ' ';
1719
+ };
1720
+ // A candidate that extends the selection is not competing with it. Whichever
1721
+ // continuation the user meant, the shorter surface is a correct ghost — it is
1722
+ // the prefix of both — so the margin has nothing to arbitrate and charging the
1723
+ // selection for its own extension abstains on a case that could not be wrong.
1724
+ // The chain rule already removed this from the denominator; leaving it in the
1725
+ // margin meant a nested leader cleared its gate and then lost to the candidate
1726
+ // that had just been freed from dividing its mass. It penalised bigrams and
1727
+ // phrases specifically, because a nested pool is where they mostly appear.
1728
+ var runnerUpScore = selected ? Math.max.apply(Math, (0, _toConsumableArray2.default)(ranked.filter(function (candidate) {
1729
+ return candidate.word !== selected.word && isStabilized(candidate) && !extendsInSamePool(selected, candidate);
1730
+ }).map(confidenceScore)).concat([0])) : 0;
1731
+ var winnerMargin = selected ? selectedDisplayScore - runnerUpScore : 0;
1732
+ var hasMissingArtifactCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1733
+ var runtime = runtimeBySurface.get(candidate.word);
1734
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) === 'whitespace' && runtime.canonicalTokenIds === null;
1735
+ });
1736
+ // A candidate with canonical tokens but no evidence at all cannot join the
1737
+ // posterior: it has no likelihood to contribute, and any stand-in value would
1738
+ // distort the normaliser for every other candidate in its context. Absent
1739
+ // evidence is not evidence of absence though — the context's boundary may
1740
+ // simply not be primed yet — so it holds the ghost back instead of being
1741
+ // silently ignored. In a primed context every candidate has at least
1742
+ // first-token evidence, so this only fires while a context is still cold.
1743
+ var hasUnscoredCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1744
+ var runtime = runtimeBySurface.get(candidate.word);
1745
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !optimisticTotalByWord.has(candidate.word) && isStabilized(candidate) && isWorthShowing(candidate);
1746
+ });
1747
+ // A rival only holds the ghost back while it is still too unread to judge.
1748
+ // Under full-surface evidence that meant any long surface blocked everything
1749
+ // until every one of its tokens had been paid for, which abstained far more
1750
+ // often than it ever changed the winner. A rival now becomes judgeable after
1751
+ // `REQUIRED_VERIFIED_TOKENS`, so this settles within the budget.
1752
+ var hasUnresolvedPotential = canonicalLmSupported && ranked.some(function (candidate) {
1753
+ var runtime = runtimeBySurface.get(candidate.word);
1754
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !hasJudgeableEvidence(candidate) && couldClearGate(candidate) && isStabilized(candidate) && isWorthShowing(candidate);
1755
+ });
1756
+ var requiredWinnerMargin = _scoringPipeline.MIN_WINNER_MARGIN + (selected && ((_judgedEvidenceFor3 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor3 === void 0 ? void 0 : _judgedEvidenceFor3.complete) === false ? PARTIAL_EVIDENCE_MARGIN_PREMIUM : 0);
1757
+ // Everything the display decision needs except the veto itself. Splitting it
1758
+ // out is what lets the veto be priced: the value here at the moment the veto
1759
+ // fires is exactly the ghost the veto cost us.
1760
+ var clearsMarginBeforeColdVeto = selected !== undefined && !hasMissingArtifactCompetitor && winnerMargin >= requiredWinnerMargin;
1761
+ var clearsWinnerMargin = clearsMarginBeforeColdVeto && !hasUnscoredCompetitor;
1762
+ var suggestion = displayPrefixReady && selected && clearsWinnerMargin ? selected.word.slice((_prefixLenByWord$get2 = prefixLenByWord.get(selected.word)) !== null && _prefixLenByWord$get2 !== void 0 ? _prefixLenByWord$get2 : currentWord.length) : null;
1763
+
1764
+ // Which constraint bound. Computed on every evaluation rather than only under
1765
+ // debug, because it is counted for the session as well as printed, and
1766
+ // deriving the debug header from it keeps the two from disagreeing.
1767
+ var resolveAbstainReason = function resolveAbstainReason() {
1768
+ if (suggestion !== null && suggestion.length > 0) {
1769
+ return null;
1770
+ }
1771
+ if (!displayPrefixReady) {
1772
+ return 'prefetch';
1773
+ }
1774
+ if (ranked.length === 0) {
1775
+ return 'no-candidate';
1776
+ }
1777
+ if (hasMissingArtifactCompetitor) {
1778
+ return 'missing-artifact';
1779
+ }
1780
+ if (hasUnscoredCompetitor) {
1781
+ return 'cold-competitor';
1782
+ }
1783
+ if (hasUnresolvedPotential && !clearsWinnerMargin) {
1784
+ return 'unresolved-rival';
1785
+ }
1786
+ if (eligible.length > 0) {
1787
+ return selected && !clearsWinnerMargin ? 'winner-margin' : 'empty-completion';
1788
+ }
1789
+ if (plausible.length > 0) {
1790
+ // Cleared its gate, was plausible on its own terms, and was refused for
1791
+ // the shortlist it cleared that gate against holding nobody else.
1792
+ return 'lone-candidate';
1793
+ }
1794
+ if (gateCleared.length > 0) {
1795
+ // Held the largest share of its shortlist and still was not a surface
1796
+ // the model found plausible — which is only possible because a share is
1797
+ // relative and this is not.
1798
+ return 'implausible-surface';
1799
+ }
1800
+ if (evidenceBacked.length > 0 && evidenceBacked.every(function (candidate) {
1801
+ return !isWorthShowing(candidate);
1802
+ })) {
1803
+ return 'short-completion';
1804
+ }
1805
+ // The floor and the gate are one threshold, applied to the bound and to
1806
+ // the verified value, so a candidate short of it is dropped before
1807
+ // `evidenceBacked` is built. Asking whether anything was judgeable
1808
+ // separates a gate that rejected read candidates from a context where
1809
+ // nothing was read at all — the first is a threshold to calibrate, the
1810
+ // second is scheduling.
1811
+ return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
1812
+ };
1813
+ var abstainReason = resolveAbstainReason();
1814
+
1815
+ // The leader is the best-supported candidate, not the selected one: an
1816
+ // evaluation that showed nothing is exactly the one whose posterior needs
1817
+ // recording, and it has no selection to report.
1818
+ var posteriorLeader = ranked.reduce(function (best, candidate) {
1819
+ var posterior = optimisticPosterior(candidate);
1820
+ return posterior > 0 && (best === null || posterior > best.posterior) ? {
1821
+ candidate: candidate,
1822
+ posterior: posterior
1823
+ } : best;
1824
+ }, null);
1825
+ /**
1826
+ * The longer surface in the leader's normaliser that extends the given
1827
+ * candidate, if the pool holds one.
1828
+ *
1829
+ * `logSumExp` divides one pool's mass between a sequence and its own prefix,
1830
+ * and the longer total is the shorter one plus a log-probability that cannot
1831
+ * be positive — so the longer form holds at most half, or at most a third
1832
+ * where both shorter forms are present. Both sit under the gate it is held
1833
+ * to, so it cannot display however certain its continuation is.
1834
+ *
1835
+ * Asked from the shorter form because that is the one that leads: the
1836
+ * extending candidate is strictly lower in the same pool and so can never be
1837
+ * the leader.
1838
+ */
1839
+ var poolExtensionOf = function poolExtensionOf(shorter) {
1840
+ return ranked.find(function (candidate) {
1841
+ return extendsInSamePool(shorter, candidate);
1842
+ });
1843
+ };
574
1844
  if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
1845
+ var _posteriorLeader$cand, _posteriorLeader$post, _selectedEvidence$mea, _selectedEvidence$mea2, _acceptCooldown$predi, _acceptCooldown;
1846
+ var verbose = (0, _debugMode.isAutocompleteDebugVerbose)();
575
1847
  var latencyMs = performance.now() - t0;
576
1848
  var tokens = tokenize(trimmed);
577
1849
  var contextWords = tokens.slice(-CONTEXT_WORDS);
578
- var belowThreshold = best && best.finalScore < MIN_SCORE_THRESHOLD;
579
- var suggestionLabel = belowThreshold ? '🚫 (below threshold)' : suggestion && suggestion.length > 0 ? "\u2728 \"".concat(suggestion, "\"") : '🚫 (no match)';
1850
+
1851
+ // Decision drives the (collapsed) group header. Abstains carry the reason
1852
+ // the session counters recorded, plus the numbers behind it, so it is clear
1853
+ // *why* nothing showed without expanding the group.
1854
+ var abstainDetail = {
1855
+ 'below-posterior-gate': "posterior below ".concat(MIN_LM_POSTERIOR[((_posteriorLeader$cand = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.candidate) !== null && _posteriorLeader$cand !== void 0 ? _posteriorLeader$cand : ranked[0]).termType], " (best ").concat(((_posteriorLeader$post = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.posterior) !== null && _posteriorLeader$post !== void 0 ? _posteriorLeader$post : 0).toFixed(2), ")"),
1856
+ 'cold-competitor': 'competitor has no LM evidence yet',
1857
+ 'empty-completion': 'empty completion',
1858
+ 'implausible-surface': "mean per-token log-probability below ".concat(MIN_MEAN_TOKEN_LOG_PROBABILITY, " (best ").concat(gateCleared.map(function (_ref9) {
1859
+ var _judgedEvidenceFor$me, _judgedEvidenceFor4;
1860
+ var candidate = _ref9.candidate;
1861
+ return (_judgedEvidenceFor$me = (_judgedEvidenceFor4 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor4 === void 0 ? void 0 : _judgedEvidenceFor4.mean) !== null && _judgedEvidenceFor$me !== void 0 ? _judgedEvidenceFor$me : -Infinity;
1862
+ }).reduce(function (best, mean) {
1863
+ return Math.max(best, mean);
1864
+ }, -Infinity).toFixed(2), ")"),
1865
+ 'lone-candidate': "cleared its gate against fewer than ".concat(MIN_SCORED_POOL_SIZE, " scored candidates, rivals unread"),
1866
+ 'missing-artifact': 'canonical artifact coverage missing',
1867
+ 'no-candidate': 'nothing cleared scoring',
1868
+ 'no-evidence': 'full-surface evidence absent',
1869
+ prefetch: "prefetch: ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " chars"),
1870
+ 'short-completion': "completion shorter than ".concat(MIN_SUGGESTION_LENGTH, " chars"),
1871
+ 'unresolved-rival': 'expanding plausible token-prefix groups',
1872
+ 'winner-margin': "winner margin ".concat(winnerMargin.toFixed(2), " < ").concat(requiredWinnerMargin.toFixed(2))
1873
+ };
1874
+ var decision = abstainReason === null ? "\u2728 \"".concat(suggestion !== null && suggestion !== void 0 ? suggestion : '', "\"") : abstainReason === 'prefetch' ? "\u23F3 ".concat(abstainDetail.prefetch) : "\u2014 abstain: ".concat(abstainDetail[abstainReason]);
580
1875
  lastPredictionDebug = {
1876
+ awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
581
1877
  textBefore: trimmed,
582
1878
  currentWord: currentWord,
1879
+ decision: decision,
583
1880
  mode: mode,
584
1881
  contextWords: contextWords,
585
1882
  topCandidates: ranked.slice(0, 5).map(function (r) {
@@ -594,55 +1891,208 @@ var predict = exports.predict = function predict(textBefore) {
594
1891
  suggestion: suggestion && suggestion.length > 0 ? suggestion : null
595
1892
  };
596
1893
 
597
- // ── Mode label: COLD / WARM(local) / WARM(BE) ───────────────────────
1894
+ // Mode describes the semantic-vector source without implying that a
1895
+ // network backend ran; the snapshot may come from the local MLC embedder.
598
1896
  var slowLaneVec = (0, _slowLaneClient.getStoredContextVector)();
599
1897
  var isUsingSlowLaneVector = slowLaneVec !== null && vectorStore !== null && slowLaneVec.length === vectorStore.dim;
600
- var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(BE)' : 'WARM(local)';
601
- var modeColor = modeLabel === 'WARM(BE)' ? 'color: #ff9800; font-weight: bold;' : modeLabel === 'WARM(local)' ? 'color: #4caf50; font-weight: bold;' : 'color: #9e9e9e; font-weight: bold;';
1898
+ var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(semantic)' : 'WARM(local-avg)';
1899
+ var modeColor = modeLabel === 'WARM(semantic)' ? _debugMode.CTC_STYLES.good : modeLabel === 'WARM(local-avg)' ? _debugMode.CTC_STYLES.warn : _debugMode.CTC_STYLES.cold;
602
1900
 
603
- // 1. Collapsible group header
1901
+ // Group header: [CTC:<mode>] <decision> · <mode> · <type> · <latency>
1902
+ // arbitrationMode ('v1') tags the line so this deploy's logs diff cleanly
1903
+ // against the baseline deploy's.
604
1904
  // eslint-disable-next-line no-console
605
- console.groupCollapsed("%c[Autocomplete] %c".concat(modeLabel, " %c| \"").concat(currentWord, "\" \u2794 ").concat(suggestionLabel, " | \u23F1 ").concat(latencyMs.toFixed(1), "ms"), 'color: #00b8d9; font-weight: bold;', modeColor, 'color: inherit; font-weight: normal;');
1905
+ console.groupCollapsed("%c[CTC:".concat(ARBITRATION_MODE, "]%c ").concat(decision, " %c\xB7 ").concat(modeLabel).concat(selected ? " \xB7 ".concat(selected.termType) : '', " \xB7 \u23F1 ").concat(latencyMs.toFixed(1), "ms"), _debugMode.CTC_STYLES.brand, _debugMode.CTC_STYLES.body, modeColor);
1906
+ var semanticInput = (0, _slowLaneClient.getStoredContextInput)();
1907
+ var causalPrompts = primeRequests.map(function (request, index) {
1908
+ var tail = request.prompt.slice(-DEBUG_TEXT_TAIL_CHARS);
1909
+ return "#".concat(index + 1, ":").concat(JSON.stringify(tail));
1910
+ });
606
1911
 
607
- // 2. Context window (what local vector averaging sees)
608
- // eslint-disable-next-line no-console
609
- console.log('%cContext Window:', 'color: #888; font-style: italic;', contextWords.length ? contextWords.join(' ') : '(none)');
1912
+ // Keep the exact model inputs separate from normalized lookup tokens.
1913
+ (0, _debugMode.ctcSection)('INPUT', "raw: ".concat(JSON.stringify(trimmed.slice(-DEBUG_TEXT_TAIL_CHARS)), " \xB7 prev: ").concat(previousWord ? JSON.stringify(previousWord) : '—'));
1914
+ (0, _debugMode.ctcSection)('LOOKUP', "normalized(last ".concat(CONTEXT_WORDS, "): ").concat(contextWords.length ? JSON.stringify(contextWords.join(' ')) : '(none)'));
1915
+ (0, _debugMode.ctcSection)('SEMANTIC', "snapshot: ".concat(semanticInput ? JSON.stringify(semanticInput.slice(-DEBUG_TEXT_TAIL_CHARS)) : '(unavailable)'));
1916
+ (0, _debugMode.ctcSection)('CAUSAL', "prompt".concat(causalPrompts.length === 1 ? '' : 's', ": ").concat(causalPrompts.join(' · ') || '(not ready)'));
610
1917
 
611
- // 3. Slow-lane status
612
- var vectorStatus = isUsingSlowLaneVector ? "\u2705 BE semantic vector (dim=".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, ")") : vectorStore ? '⚠️ Local vector average (slow-lane not yet returned)' : '❌ No vectors (cold)';
613
- var logitsStatus = lmLogits && Object.keys(lmLogits).length > 0 ? "\u2705 LM logits active (".concat(Object.keys(lmLogits).length, " tokens)") : '⏳ No LM logits (slow-lane pending or failed)';
614
- // eslint-disable-next-line no-console
615
- console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
1918
+ // SIGNALS — semantic state plus context-keyed canonical LM coverage.
1919
+ var logitCount = lmLogits ? Object.keys(lmLogits).length : 0;
1920
+ var semanticStatus = isUsingSlowLaneVector ? "semantic \u2705 snapshot(".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, "d)") : vectorStore ? 'semantic ⚠️ local-avg' : 'semantic ❌ cold';
1921
+ var tierACount = ranked.filter(function (candidate) {
1922
+ var _candidate$lmEvidence3;
1923
+ return ((_candidate$lmEvidence3 = candidate.lmEvidence) === null || _candidate$lmEvidence3 === void 0 ? void 0 : _candidate$lmEvidence3.source) === 'canonical-first-token';
1924
+ }).length;
1925
+ var exactCount = ranked.filter(function (candidate) {
1926
+ var _candidate$lmEvidence4;
1927
+ return ((_candidate$lmEvidence4 = candidate.lmEvidence) === null || _candidate$lmEvidence4 === void 0 ? void 0 : _candidate$lmEvidence4.source) === 'canonical-full-surface';
1928
+ }).length;
1929
+ var boundaryReady = primeRequests.filter(function (request) {
1930
+ return (0, _slowLaneClient.getBoundaryLmState)(request.contextKey) !== null;
1931
+ }).length;
1932
+ var artifactEligible = canonicalMatched.filter(function (candidate) {
1933
+ return candidate.canonicalTokenIds !== null;
1934
+ }).length;
1935
+ var documentStartFallbacks = canonicalMatched.filter(function (candidate) {
1936
+ return candidate.separatorKind === 'document-start';
1937
+ }).length;
1938
+ var punctuationFallbacks = canonicalMatched.filter(function (candidate) {
1939
+ return candidate.separatorKind === 'non-space';
1940
+ }).length;
1941
+ var logitsStatus = tierACount + exactCount > 0 ? "canonical \u2705 first-token:".concat(tierACount, " exact:").concat(exactCount) : logitCount > 0 ? "network logits \u2705 ".concat(logitCount) : 'canonical ⏳/absent';
1942
+ var formula = tierACount + exactCount + logitCount > 0 ? "Stage1\xD7".concat(_scoringPipeline.STAGE1_WEIGHT, "+LM\xD7").concat(_scoringPipeline.STAGE2_WEIGHT) : 'Stage1-only';
1943
+ (0, _debugMode.ctcSection)('SIGNALS', "".concat(semanticStatus, " \xB7 ").concat(logitsStatus, " \xB7 ").concat(formula));
1944
+ (0, _debugMode.ctcSection)('CANONICAL', "eligible ".concat(artifactEligible, "/").concat(canonicalMatched.length, " \xB7 separator fallback doc:").concat(documentStartFallbacks, " punct:").concat(punctuationFallbacks, " \xB7 contexts ").concat(boundaryReady, "/").concat(primeRequests.length, " ready (\u2264").concat(PHRASE_MAX_WORDS, ") \xB7 family=").concat(familyKey.slice(0, 72) || 'none'));
1945
+ if (verbose && logitCount > 0 && lmLogits) {
1946
+ var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
1947
+ return b[1] - a[1];
1948
+ }).slice(0, 5).map(function (_ref0) {
1949
+ var _ref1 = (0, _slicedToArray2.default)(_ref0, 2),
1950
+ word = _ref1[0],
1951
+ score = _ref1[1];
1952
+ return "".concat(word, ":").concat(score.toFixed(3));
1953
+ }).join(', ');
1954
+ (0, _debugMode.ctcSection)(' rawLM', "\uD83E\uDDE0 ".concat(rawLmTop));
1955
+ }
616
1956
 
617
- // 4. Scoring formula active this prediction
618
- var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? "Stage1(\xD7".concat(_scoringPipeline.STAGE1_WEIGHT, ") + LM(\xD7").concat(_scoringPipeline.STAGE2_WEIGHT, ")") : 'Stage1 only (no LM logits)';
619
- // eslint-disable-next-line no-console
620
- console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
1957
+ // GENERATE — how many candidates matched, split by term type
1958
+ var genByType = {
1959
+ word: 0,
1960
+ bigram: 0,
1961
+ phrase: 0
1962
+ };
1963
+ var _iterator19 = _createForOfIteratorHelper(canonicalMatched),
1964
+ _step19;
1965
+ try {
1966
+ for (_iterator19.s(); !(_step19 = _iterator19.n()).done;) {
1967
+ var m = _step19.value;
1968
+ genByType[m.node.termType] += 1;
1969
+ }
1970
+ } catch (err) {
1971
+ _iterator19.e(err);
1972
+ } finally {
1973
+ _iterator19.f();
1974
+ }
1975
+ (0, _debugMode.ctcSection)('GENERATE', "matched ".concat(canonicalMatched.length, " \u2192 word:").concat(genByType.word, " bigram:").concat(genByType.bigram, " phrase:").concat(genByType.phrase).concat(canonicalLmSupported ? ' · display needs exact surface evidence' : ''));
621
1976
 
622
- // 5. Grammar filter result (collected inside rankCandidates, logged here)
1977
+ // SCORE — Stage-1 + grammar funnel and canonical evidence depth.
1978
+ var lmCoverage = ranked.slice(0, 10).filter(function (r) {
1979
+ return r.lmScore > 0;
1980
+ }).length;
1981
+ (0, _debugMode.ctcSection)('SCORE', "in ".concat(pipelineDebug.initial, " \u2192 stage1(<").concat(_scoringPipeline.MIN_STAGE1_SCORE, ") \u2212").concat(pipelineDebug.stage1Rejected.length, " \u2192 grammar \u2212").concat(pipelineDebug.grammarRejected.length, " \u2192 final ").concat(pipelineDebug.final, " \xB7 LM cov ").concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " \xB7 first-token ").concat(tierACount, " \xB7 exact ").concat(exactCount));
623
1982
  if (grammarMeta) {
624
- // eslint-disable-next-line no-console
625
- console.log("%c[Grammar] \"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] \u2192 ").concat(grammarMeta.before, " candidates \u2192 ").concat(grammarMeta.after, " after filter"), 'color: #4caf50; font-weight: bold;');
626
- if (grammarMeta.dropped.length > 0) {
627
- // eslint-disable-next-line no-console
628
- console.log("%c\uD83D\uDEAB Dropped: ".concat(grammarMeta.dropped.join(', ')), 'color: #f44336; font-style: italic;');
629
- }
1983
+ (0, _debugMode.ctcSection)(' grammar', "\"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] ").concat(grammarMeta.before, "\u2192").concat(grammarMeta.after).concat(verbose && grammarMeta.dropped.length > 0 ? " \xB7 dropped: ".concat(grammarMeta.dropped.join(', ')) : ''));
630
1984
  }
1985
+ var gateReasonFor = function gateReasonFor(candidate) {
1986
+ var runtime = runtimeBySurface.get(candidate.word);
1987
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace') {
1988
+ var _runtime$separatorKin;
1989
+ return "separator:".concat((_runtime$separatorKin = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin !== void 0 ? _runtime$separatorKin : 'unknown');
1990
+ }
1991
+ if (runtime.canonicalTokenIds === null) {
1992
+ return 'missing-artifact';
1993
+ }
1994
+ if (!hasJudgeableEvidence(candidate)) {
1995
+ return "awaiting-tokens:".concat(REQUIRED_VERIFIED_TOKENS);
1996
+ }
1997
+ if (!passesLmFloor(candidate)) {
1998
+ return "bound:".concat(optimisticPosterior(candidate).toFixed(2), "<").concat(MIN_LM_POSTERIOR[candidate.termType]);
1999
+ }
2000
+ if (duplicatesPreceding(candidate)) {
2001
+ return 'repetition';
2002
+ }
2003
+ if (isCooledDown(candidate)) {
2004
+ return 'cooldown';
2005
+ }
2006
+ if (!isWorthShowing(candidate)) {
2007
+ return "suffix:".concat(suggestionLengthFor(candidate), "<").concat(MIN_SUGGESTION_LENGTH);
2008
+ }
2009
+ var floor = MIN_LM_POSTERIOR[candidate.termType];
2010
+ var posterior = judgedPosterior(candidate);
2011
+ if (posterior < floor) {
2012
+ return "posterior:".concat(posterior.toFixed(2), "<").concat(floor);
2013
+ }
2014
+ if (!isPlausibleSurface(candidate)) {
2015
+ var _judgedEvidenceFor$me2, _judgedEvidenceFor5;
2016
+ return "implausible:".concat(((_judgedEvidenceFor$me2 = (_judgedEvidenceFor5 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor5 === void 0 ? void 0 : _judgedEvidenceFor5.mean) !== null && _judgedEvidenceFor$me2 !== void 0 ? _judgedEvidenceFor$me2 : NaN).toFixed(2), "<").concat(MIN_MEAN_TOKEN_LOG_PROBABILITY);
2017
+ }
2018
+ return 'eligible';
2019
+ };
2020
+ var gateFunnel = function gateFunnel(termType) {
2021
+ var typeRanked = ranked.filter(function (candidate) {
2022
+ return candidate.termType === termType;
2023
+ });
2024
+ var exact = typeRanked.filter(hasJudgeableEvidence);
2025
+ var absolute = exact.filter(passesLmFloor);
2026
+ var stabilized = absolute.filter(isStabilized);
2027
+ var longEnough = stabilized.filter(isWorthShowing);
2028
+ var floorPassed = longEnough.filter(function (candidate) {
2029
+ return judgedPosterior(candidate) >= MIN_LM_POSTERIOR[termType];
2030
+ });
2031
+ var plausiblePassed = floorPassed.filter(isPlausibleSurface);
2032
+ return "".concat(termType, " m:").concat(genByType[termType], " r:").concat(typeRanked.length, " exact:").concat(exact.length, " abs:").concat(absolute.length, " stable:").concat(stabilized.length, " suffix:").concat(longEnough.length, " floor:").concat(floorPassed.length, " plausible:").concat(plausiblePassed.length, " eligible:").concat(eligible.filter(function (_ref10) {
2033
+ var candidate = _ref10.candidate;
2034
+ return candidate.termType === termType;
2035
+ }).length);
2036
+ };
2037
+ (0, _debugMode.ctcSection)('GATES', "".concat(gateFunnel('word'), " | ").concat(gateFunnel('bigram'), " | ").concat(gateFunnel('phrase')));
631
2038
 
632
- // 6. Pipeline funnel
633
- // eslint-disable-next-line no-console
634
- console.log("%c[Pipeline Funnel] %c\uD83D\uDCE5 In: ".concat(pipelineDebug.initial, " | \u274C Stage 1 (< ").concat(_scoringPipeline.MIN_STAGE1_SCORE, "): -").concat(pipelineDebug.stage1Rejected.length, " | \u274C Grammar: -").concat(pipelineDebug.grammarRejected.length, " | \u2705 Final: ").concat(pipelineDebug.final), 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
2039
+ // One line per context: how many candidates share the normaliser, and how
2040
+ // much of the mass the leader holds. A leader well under its threshold
2041
+ // means the context is contested, which is the abstention we want.
2042
+ var contextLeaders = Array.from(bestCandidateByContext.entries()).slice(0, PHRASE_MAX_WORDS).map(function (_ref11) {
2043
+ var _contextTotals$get$le2, _contextTotals$get2;
2044
+ var _ref12 = (0, _slicedToArray2.default)(_ref11, 2),
2045
+ contextKey = _ref12[0],
2046
+ leader = _ref12[1];
2047
+ var shortlistSize = (_contextTotals$get$le2 = (_contextTotals$get2 = contextTotals.get(contextKey)) === null || _contextTotals$get2 === void 0 ? void 0 : _contextTotals$get2.length) !== null && _contextTotals$get$le2 !== void 0 ? _contextTotals$get$le2 : 0;
2048
+ var evidenceKind = hasExactEvidence(leader) ? 'exact' : 'upper';
2049
+ return "".concat(contextKey.slice(0, 32), " \u2192 n=").concat(shortlistSize, " ").concat(leader.termType, ":\"").concat(leader.word, "\" ").concat(evidenceKind, " p=").concat(optimisticPosterior(leader).toFixed(3), "/").concat(MIN_LM_POSTERIOR[leader.termType]);
2050
+ }).join(' | ');
2051
+ (0, _debugMode.ctcSection)('NORMALIZE', contextLeaders.length > 0 ? contextLeaders : 'no scored candidate in any context');
2052
+ (0, _debugMode.ctcSection)('READINESS', "prefix ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " \xB7 boundary ").concat(boundaryReady, "/").concat(primeRequests.length, " \xB7 exact word:").concat(ranked.filter(function (candidate) {
2053
+ return candidate.termType === 'word' && hasExactEvidence(candidate);
2054
+ }).length, " bigram:").concat(ranked.filter(function (candidate) {
2055
+ return candidate.termType === 'bigram' && hasExactEvidence(candidate);
2056
+ }).length, " phrase:").concat(ranked.filter(function (candidate) {
2057
+ return candidate.termType === 'phrase' && hasExactEvidence(candidate);
2058
+ }).length, " \xB7 ").concat(suggestion ? 'display-ready' : decision));
635
2059
 
636
- // 7. Candidate table
637
- if (ranked.length > 0) {
638
- var lmCoverage = ranked.slice(0, 10).filter(function (r) {
639
- return r.lmScore > 0;
640
- }).length;
641
- // eslint-disable-next-line no-console
642
- console.log("%cLM coverage: ".concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " candidates had real logit scores"), 'color: #888; font-style: italic;');
2060
+ // ARBITRATE — expose every condition that can authorize or suppress ghost.
2061
+ var topOfType = function topOfType(t) {
2062
+ return ranked.find(function (c) {
2063
+ return c.termType === t;
2064
+ });
2065
+ };
2066
+ var fmtType = function fmtType(t) {
2067
+ var _top$lmEvidence$sourc, _top$lmEvidence;
2068
+ var top = topOfType(t);
2069
+ if (!top) {
2070
+ return "".concat(t, ":\u2014");
2071
+ }
2072
+ var floor = MIN_LM_POSTERIOR[t];
2073
+ var bound = optimisticPosterior(top);
2074
+ var evidence = hasRequiredEvidence(top) ? 'TierB✅' : "".concat((_top$lmEvidence$sourc = (_top$lmEvidence = top.lmEvidence) === null || _top$lmEvidence === void 0 ? void 0 : _top$lmEvidence.source) !== null && _top$lmEvidence$sourc !== void 0 ? _top$lmEvidence$sourc : 'absent', "\u274C");
2075
+ return "".concat(t, ":\"").concat(top.word, "\" p\u2264").concat(bound.toFixed(3)).concat(bound >= floor ? '≥' : '<').concat(floor, " fs\u2264").concat(confidenceScore(top).toFixed(2), " ").concat(evidence);
2076
+ };
2077
+ (0, _debugMode.ctcSection)('ARBITRATE', "".concat(selected ? "\u25B8 ".concat(selected.termType, " \"").concat(selected.word, "\" fs=").concat(selectedDisplayScore.toFixed(2), " mean=").concat((_selectedEvidence$mea = selectedEvidence === null || selectedEvidence === void 0 || (_selectedEvidence$mea2 = selectedEvidence.meanTokenLogProbability) === null || _selectedEvidence$mea2 === void 0 ? void 0 : _selectedEvidence$mea2.toFixed(2)) !== null && _selectedEvidence$mea !== void 0 ? _selectedEvidence$mea : 'network', " margin=").concat(winnerMargin.toFixed(2)).concat(clearsWinnerMargin ? '✅' : '❌') : '▸ (none)', " \xB7 prefix ").concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH).concat(displayPrefixReady ? '✅' : '⏳', " \xB7 grouped pending ").concat(hasUnresolvedPotential ? 'yes' : 'no', " | ").concat(fmtType('word'), " ").concat(fmtType('bigram'), " ").concat(fmtType('phrase')));
2078
+
2079
+ // STABILIZE — post-accept cooldown + whole-surface repetition (QI-2)
2080
+ var repetitionBlocked = ranked.filter(function (c) {
2081
+ return hasRequiredEvidence(c) && duplicatesPreceding(c);
2082
+ }).map(function (c) {
2083
+ return c.word;
2084
+ });
2085
+ var cooldownLabel = cooledSurface ? "cooldown \"".concat(cooledSurface, "\" (").concat((_acceptCooldown$predi = (_acceptCooldown = acceptCooldown) === null || _acceptCooldown === void 0 ? void 0 : _acceptCooldown.predictionsSince) !== null && _acceptCooldown$predi !== void 0 ? _acceptCooldown$predi : 0, "/").concat(COOLDOWN_KEYSTROKES, " keys \xB7 ").concat(COOLDOWN_MS, "ms)") : 'cooldown —';
2086
+ (0, _debugMode.ctcSection)('STABILIZE', "".concat(cooldownLabel, " \xB7 repetition-blocked: ").concat(repetitionBlocked.length > 0 ? repetitionBlocked.join(', ') : '—'));
2087
+
2088
+ // CANDIDATES — full scored table (verbose only)
2089
+ if (verbose && ranked.length > 0) {
643
2090
  var tableData = ranked.slice(0, 10).map(function (r) {
2091
+ var _r$lmEvidence, _contextTotals$get$le3, _contextTotals$get3, _r$lmEvidence$source, _r$lmEvidence2, _runtime$separatorKin2, _runtime$canonicalTok2, _runtime$canonicalTok3, _runtime$contextKey, _r$lmEvidence3, _r$lmEvidence4;
644
2092
  var rawLogit = 'Not in Payload';
645
- if (prefixLmLogits) {
2093
+ if (((_r$lmEvidence = r.lmEvidence) === null || _r$lmEvidence === void 0 ? void 0 : _r$lmEvidence.rawLogit) !== undefined) {
2094
+ rawLogit = Number(r.lmEvidence.rawLogit.toFixed(5));
2095
+ } else if (prefixLmLogits) {
646
2096
  var val = prefixLmLogits[r.word.toLowerCase()];
647
2097
  if (val !== undefined) {
648
2098
  rawLogit = Number(val.toFixed(5));
@@ -651,9 +2101,19 @@ var predict = exports.predict = function predict(textBefore) {
651
2101
  var original = scoringCandidates.find(function (sc) {
652
2102
  return sc.word === r.word;
653
2103
  });
2104
+ var runtime = runtimeBySurface.get(r.word);
2105
+ var progress = runtime ? (0, _slowLaneClient.getProgressiveSurfaceEvidence)(runtime.contextKey, r.word) : null;
2106
+ var optimisticTotal = optimisticTotalByWord.get(r.word);
2107
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
2108
+ var shortlistSize = runtime ? (_contextTotals$get$le3 = (_contextTotals$get3 = contextTotals.get(runtime.contextKey)) === null || _contextTotals$get3 === void 0 ? void 0 : _contextTotals$get3.length) !== null && _contextTotals$get$le3 !== void 0 ? _contextTotals$get$le3 : 0 : 0;
2109
+ var contextLeader = runtime ? bestCandidateByContext.get(runtime.contextKey) : undefined;
654
2110
  var source = 'Unknown';
655
2111
  if (original) {
656
- if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
2112
+ if (original.termType === 'bigram') {
2113
+ source = '🔗 L2 (Bigram)';
2114
+ } else if (original.termType === 'phrase') {
2115
+ source = '🧩 L2 (Phrase)';
2116
+ } else if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
657
2117
  source = '🌍 L3 (Generic)';
658
2118
  } else if (original.sessionFreq > 0 && original.tenantFreq === 0) {
659
2119
  source = '👤 L1 (Session Only)';
@@ -661,13 +2121,40 @@ var predict = exports.predict = function predict(textBefore) {
661
2121
  source = '🏢 L2 (Domain)';
662
2122
  }
663
2123
  }
2124
+ var floor = MIN_LM_POSTERIOR[r.termType];
664
2125
  return {
665
2126
  Candidate: r.word,
666
2127
  Source: source,
2128
+ Type: r.termType,
667
2129
  'Final Score': Number(r.finalScore.toFixed(4)),
2130
+ 'Bounded Score': Number(confidenceScore(r).toFixed(4)),
2131
+ 'Exact Gate Score': hasExactEvidence(r) ? Number(exactConfidenceScore(r).toFixed(4)) : '—',
2132
+ Floor: floor,
2133
+ 'Posterior Floor': optimisticPosterior(r) >= floor ? '✅' : '❌',
2134
+ 'LM Floor': hasExactEvidence(r) ? passesLmFloor(r) ? '✅' : '❌' : '⏳',
2135
+ 'Gate Reason': gateReasonFor(r),
2136
+ 'Display Eligible': hasRequiredEvidence(r) && isStabilized(r) && judgedPosterior(r) >= floor ? '✅' : '❌',
668
2137
  Semantics: Number(r.semanticScore.toFixed(4)),
669
2138
  Freq: Number(r.freqScore.toFixed(4)),
670
2139
  'LM Score': Number(r.lmScore.toFixed(4)),
2140
+ 'LM Evidence': (_r$lmEvidence$source = (_r$lmEvidence2 = r.lmEvidence) === null || _r$lmEvidence2 === void 0 ? void 0 : _r$lmEvidence2.source) !== null && _r$lmEvidence$source !== void 0 ? _r$lmEvidence$source : 'absent',
2141
+ 'Verified Tokens': r.lmEvidence ? "".concat(r.lmEvidence.verifiedTokenCount, "/").concat(r.lmEvidence.totalTokenCount) : '0/0',
2142
+ 'Progressive Tokens': progress ? "".concat(progress.scoredTokenCount, "/").concat(progress.totalTokenCount) : '0/0',
2143
+ 'LM Mean Upper': progress !== null ? Number(progress.meanTokenLogProbabilityUpperBound.toFixed(5)) : '—',
2144
+ 'Optimistic Total': optimisticTotal !== undefined ? Number(optimisticTotal.toFixed(5)) : '—',
2145
+ 'Context logSumExp': normalizer !== undefined && Number.isFinite(normalizer) ? Number(normalizer.toFixed(5)) : '—',
2146
+ 'Shortlist Size': shortlistSize,
2147
+ 'Posterior (bound)': Number(optimisticPosterior(r).toFixed(5)),
2148
+ 'Posterior (judged)': hasJudgeableEvidence(r) ? Number(judgedPosterior(r).toFixed(5)) : '—',
2149
+ 'Context Leader': contextLeader ? "".concat(contextLeader.termType, ":").concat(contextLeader.word) : '—',
2150
+ 'Verified Chars': r.lmEvidence ? "".concat(r.lmEvidence.verifiedCharCount, "/").concat(r.lmEvidence.totalSurfaceCharCount) : "0/".concat(r.word.length),
2151
+ Separator: (_runtime$separatorKin2 = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin2 !== void 0 ? _runtime$separatorKin2 : 'unknown',
2152
+ 'Canonical Eligibility': (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace' ? 'incompatible-separator' : runtime.canonicalTokenIds === null ? 'missing-artifact' : 'eligible',
2153
+ 'Artifact Tokens': (_runtime$canonicalTok2 = runtime === null || runtime === void 0 || (_runtime$canonicalTok3 = runtime.canonicalTokenIds) === null || _runtime$canonicalTok3 === void 0 ? void 0 : _runtime$canonicalTok3.join(',')) !== null && _runtime$canonicalTok2 !== void 0 ? _runtime$canonicalTok2 : 'absent',
2154
+ 'Context Key': (_runtime$contextKey = runtime === null || runtime === void 0 ? void 0 : runtime.contextKey) !== null && _runtime$contextKey !== void 0 ? _runtime$contextKey : '—',
2155
+ 'Exact Total': ((_r$lmEvidence3 = r.lmEvidence) === null || _r$lmEvidence3 === void 0 ? void 0 : _r$lmEvidence3.totalLogProbability) !== undefined ? Number(r.lmEvidence.totalLogProbability.toFixed(5)) : '—',
2156
+ 'Exact Mean': ((_r$lmEvidence4 = r.lmEvidence) === null || _r$lmEvidence4 === void 0 ? void 0 : _r$lmEvidence4.meanTokenLogProbability) !== undefined ? Number(r.lmEvidence.meanTokenLogProbability.toFixed(5)) : '—',
2157
+ Assoc: typeof (original === null || original === void 0 ? void 0 : original.association) === 'number' ? Number(original.association.toFixed(4)) : '—',
671
2158
  'Raw Logit': rawLogit,
672
2159
  'Session Freq': (original === null || original === void 0 ? void 0 : original.sessionFreq) || 0
673
2160
  };
@@ -675,23 +2162,120 @@ var predict = exports.predict = function predict(textBefore) {
675
2162
 
676
2163
  // eslint-disable-next-line no-console
677
2164
  console.table(tableData);
678
- } else {
679
- // eslint-disable-next-line no-console
680
- console.log('No candidates found.');
681
2165
  }
682
2166
 
683
2167
  // eslint-disable-next-line no-console
684
2168
  console.groupEnd();
685
2169
  }
686
- return suggestion && suggestion.length > 0 ? suggestion : null;
2170
+ var selectedContextKey = selected ? (_runtimeBySurface$get5 = runtimeBySurface.get(selected.word)) === null || _runtimeBySurface$get5 === void 0 ? void 0 : _runtimeBySurface$get5.contextKey : undefined;
2171
+ return suggestion && suggestion.length > 0 && selected && selectedEvidence ? {
2172
+ text: suggestion,
2173
+ surface: selected.word,
2174
+ termType: selected.termType,
2175
+ posterior: judgedPosterior(selected),
2176
+ meanTokenLogProbability: (_judgedEvidenceFor$me3 = (_judgedEvidenceFor6 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor6 === void 0 ? void 0 : _judgedEvidenceFor6.mean) !== null && _judgedEvidenceFor$me3 !== void 0 ? _judgedEvidenceFor$me3 : NaN,
2177
+ poolHeldExtension: poolExtensionOf(selected) !== undefined,
2178
+ shortlistSize: selectedContextKey === undefined ? 1 : (_contextTotals$get$le4 = (_contextTotals$get4 = contextTotals.get(selectedContextKey)) === null || _contextTotals$get4 === void 0 ? void 0 : _contextTotals$get4.length) !== null && _contextTotals$get$le4 !== void 0 ? _contextTotals$get$le4 : 1,
2179
+ rankScore: selectedDisplayScore,
2180
+ evidenceTier: (selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.source) === 'network-logit' ? 'network-logit' : 'canonical-full-surface',
2181
+ evidenceDepth: {
2182
+ verifiedTokens: (_selectedEvidence$ver = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedTokenCount) !== null && _selectedEvidence$ver !== void 0 ? _selectedEvidence$ver : 0,
2183
+ totalTokens: (_selectedEvidence$tot = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.totalTokenCount) !== null && _selectedEvidence$tot !== void 0 ? _selectedEvidence$tot : 0,
2184
+ verifiedChars: (_selectedEvidence$ver2 = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedCharCount) !== null && _selectedEvidence$ver2 !== void 0 ? _selectedEvidence$ver2 : 0,
2185
+ totalChars: selected.word.length
2186
+ },
2187
+ winnerMargin: winnerMargin
2188
+ } : null;
687
2189
  };
688
2190
 
689
2191
  // ─── Data Loading ────────────────────────────────────────────────────────────
690
2192
 
2193
+ var isVocabularyJson = function isVocabularyJson(payload) {
2194
+ if (payload == null || (0, _typeof2.default)(payload) !== 'object') {
2195
+ return false;
2196
+ }
2197
+ var words = payload.words;
2198
+ return words != null && (0, _typeof2.default)(words) === 'object';
2199
+ };
2200
+ var isStringArray = function isStringArray(payload) {
2201
+ return Array.isArray(payload) && payload.every(function (entry) {
2202
+ return typeof entry === 'string';
2203
+ });
2204
+ };
2205
+ var isWordToOffsetMap = function isWordToOffsetMap(value) {
2206
+ return value != null && (0, _typeof2.default)(value) === 'object' && Object.values(value).every(function (entry) {
2207
+ return typeof entry === 'number';
2208
+ });
2209
+ };
2210
+ var isWordIndexPayload = function isWordIndexPayload(payload) {
2211
+ if (payload == null || (0, _typeof2.default)(payload) !== 'object') {
2212
+ return false;
2213
+ }
2214
+ var index = payload.index;
2215
+ return index === undefined ? isWordToOffsetMap(payload) : isWordToOffsetMap(index);
2216
+ };
2217
+
2218
+ /**
2219
+ * The word index has shipped both bare and wrapped in `{ index: ... }`, so
2220
+ * accept either rather than silently building an empty vector store.
2221
+ */
2222
+ var unwrapWordIndex = function unwrapWordIndex(payload) {
2223
+ var wrapped = payload.index;
2224
+ return wrapped !== null && wrapped !== void 0 ? wrapped : payload;
2225
+ };
2226
+
2227
+ /**
2228
+ * Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
2229
+ * confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
2230
+ * + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
2231
+ * are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
2232
+ * a number.
2233
+ */
2234
+
2235
+ /** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
2236
+
2237
+ var isPhraseStats = function isPhraseStats(value) {
2238
+ return value != null && (0, _typeof2.default)(value) === 'object' && typeof value.freq === 'number';
2239
+ };
2240
+ var isTermStatsMap = function isTermStatsMap(value) {
2241
+ if (value == null || (0, _typeof2.default)(value) !== 'object') {
2242
+ return false;
2243
+ }
2244
+ var values = Object.values(value);
2245
+ return values.length > 0 && values.every(isPhraseStats);
2246
+ };
2247
+
2248
+ /**
2249
+ * Normalize a `bigrams.json` / `phrases.json` payload into a flat
2250
+ * `{term: stats}` map. Tolerant of the exact wire shape (not yet finalized by
2251
+ * the producer): accepts either the flat map from spec §9 or a single wrapper
2252
+ * key (`terms`/`bigrams`/`phrases`/`words`). Returns `null` if unrecognized.
2253
+ *
2254
+ * :params:
2255
+ * payload: Raw JSON parsed from the artifact endpoint
2256
+ * :returns:
2257
+ * A `{term: stats}` map, or `null` when the shape is not a term-stats map
2258
+ */
2259
+ var normalizePhraseArtifact = function normalizePhraseArtifact(payload) {
2260
+ if (payload == null || (0, _typeof2.default)(payload) !== 'object') {
2261
+ return null;
2262
+ }
2263
+ if (isTermStatsMap(payload)) {
2264
+ return payload;
2265
+ }
2266
+ var obj = payload;
2267
+ for (var _i4 = 0, _arr = ['terms', 'bigrams', 'phrases', 'words']; _i4 < _arr.length; _i4++) {
2268
+ var key = _arr[_i4];
2269
+ if (isTermStatsMap(obj[key])) {
2270
+ return obj[key];
2271
+ }
2272
+ }
2273
+ return null;
2274
+ };
691
2275
  var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
692
- var _ref6 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(options) {
2276
+ var _ref13 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(options) {
693
2277
  var _options$isLocalLLM;
694
- var isLocalLLM, surface, url, res, buffer, float32, wordIndexPayload, wrappedIndex, wordIndex, nWords, dim, _t, _t2;
2278
+ var isLocalLLM, surface, buffer, float32, wordIndexPayload, wordIndex, nWords, dim, _t;
695
2279
  return _regenerator.default.wrap(function (_context) {
696
2280
  while (1) switch (_context.prev = _context.next) {
697
2281
  case 0:
@@ -711,65 +2295,23 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
711
2295
  } : {}));
712
2296
  _context.prev = 2;
713
2297
  _context.next = 3;
714
- return (0, _artifactsManifest.getArtifactUrl)(_artifactsManifest.ARTIFACT_NAME.WORD_VECTORS);
2298
+ return (0, _artifactLoader.fetchAutocompleteArtifactBinary)(_artifactsManifest.ARTIFACT_NAME.WORD_VECTORS, 'word_vectors_10k');
715
2299
  case 3:
716
- url = _context.sent;
717
- _context.next = 5;
718
- break;
719
- case 4:
720
- _context.prev = 4;
721
- _t = _context["catch"](2);
722
- vectorsLoadStarted = false;
723
- (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
724
- isLocalLLM: isLocalLLM,
725
- errorType: 'resolve_url'
726
- }, surface ? {
727
- surface: surface
728
- } : {}));
729
- // eslint-disable-next-line no-console
730
- console.warn('[text-predictor] Failed to resolve vectors URL:', _t);
731
- return _context.abrupt("return");
732
- case 5:
733
- _context.prev = 5;
734
- _context.next = 6;
735
- return fetch(url);
736
- case 6:
737
- res = _context.sent;
738
- if (res.ok) {
739
- _context.next = 7;
740
- break;
741
- }
742
- vectorsLoadStarted = false;
743
- (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
744
- isLocalLLM: isLocalLLM,
745
- status: res.status,
746
- errorType: 'http_error'
747
- }, surface ? {
748
- surface: surface
749
- } : {}));
750
- // eslint-disable-next-line no-console
751
- console.warn("[text-predictor] Failed to load vectors: ".concat(res.status));
752
- return _context.abrupt("return");
753
- case 7:
754
- _context.next = 8;
755
- return res.arrayBuffer();
756
- case 8:
757
2300
  buffer = _context.sent;
758
- float32 = new Float32Array(buffer); // The word index is published either bare or wrapped as `{ "index": {…} }`
759
- // (the wrapper exists so no real entry, e.g. the word "default", can shadow
760
- // a bundler-synthesised ESM export). Detect the wrapper by value type, not
761
- // by key presence — "index" is itself a plausible vocabulary word, and in a
762
- // bare index every value is a number.
763
- _context.next = 9;
764
- return (0, _artifactsManifest.fetchArtifactJson)(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX);
765
- case 9:
2301
+ float32 = new Float32Array(buffer);
2302
+ _context.next = 4;
2303
+ return (0, _artifactLoader.fetchAutocompleteArtifactJson)(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX, {
2304
+ summarize: function summarize(payload) {
2305
+ return "".concat(Object.keys(unwrapWordIndex(payload)).length, " entries");
2306
+ },
2307
+ validate: isWordIndexPayload
2308
+ });
2309
+ case 4:
766
2310
  wordIndexPayload = _context.sent;
767
- wrappedIndex = wordIndexPayload === null || wordIndexPayload === void 0 ? void 0 : wordIndexPayload.index;
768
- wordIndex = (0, _typeof2.default)(wrappedIndex) === 'object' && wrappedIndex !== null ? wrappedIndex : wordIndexPayload !== null && wordIndexPayload !== void 0 ? wordIndexPayload : {};
2311
+ wordIndex = unwrapWordIndex(wordIndexPayload);
769
2312
  nWords = Object.keys(wordIndex).length;
770
2313
  if (nWords === 0) {
771
- // eslint-disable-next-line no-console
772
- console.warn("[text-predictor] ".concat(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX, " was empty \u2014 semantic scoring will be a no-op."));
2314
+ (0, _debugMode.ctcTag)('init', '⚠️ word_index_10k.json was empty — semantic scoring will be a no-op', _debugMode.CTC_STYLES.warn);
773
2315
  }
774
2316
  dim = float32.length / nWords;
775
2317
  vectorStore = {
@@ -785,19 +2327,12 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
785
2327
  }, surface ? {
786
2328
  surface: surface
787
2329
  } : {}));
788
- if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
789
- // eslint-disable-next-line no-console
790
- console.log('[text-predictor] Vectors loaded:', {
791
- wordCount: nWords,
792
- dim: dim,
793
- sizeBytes: float32.byteLength
794
- });
795
- }
796
- _context.next = 11;
2330
+ (0, _debugMode.ctcTag)('init', "vectors loaded: ".concat(nWords, " words \xB7 dim ").concat(dim, " \xB7 ").concat(float32.byteLength, "B"));
2331
+ _context.next = 6;
797
2332
  break;
798
- case 10:
799
- _context.prev = 10;
800
- _t2 = _context["catch"](5);
2333
+ case 5:
2334
+ _context.prev = 5;
2335
+ _t = _context["catch"](2);
801
2336
  vectorsLoadStarted = false;
802
2337
  (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
803
2338
  isLocalLLM: isLocalLLM,
@@ -805,60 +2340,197 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
805
2340
  }, surface ? {
806
2341
  surface: surface
807
2342
  } : {}));
808
- // eslint-disable-next-line no-console
809
- console.warn('[text-predictor] Failed to load vectors:', _t2);
810
- case 11:
2343
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F failed to load vectors: ".concat(String(_t)), _debugMode.CTC_STYLES.bad);
2344
+ case 6:
811
2345
  case "end":
812
2346
  return _context.stop();
813
2347
  }
814
- }, _callee, null, [[2, 4], [5, 10]]);
2348
+ }, _callee, null, [[2, 5]]);
815
2349
  }));
816
2350
  return function loadVectorsAsync(_x) {
817
- return _ref6.apply(this, arguments);
2351
+ return _ref13.apply(this, arguments);
818
2352
  };
819
2353
  }();
820
2354
  var initVectors = exports.initVectors = function initVectors(store) {
821
2355
  vectorStore = store;
822
2356
  };
2357
+
2358
+ /**
2359
+ * Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
2360
+ * over the TDP-OS gateway and insert them into the phrase trie.
2361
+ *
2362
+ * Each file is fetched independently (`Promise.allSettled`) so a missing or
2363
+ * late-published artifact only skips that term type — word completion and the
2364
+ * other term type are unaffected. Phrase VECTORS are not fetched here: per the
2365
+ * producer's `extend` decision they are appended into the existing
2366
+ * `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
2367
+ * covers them.
2368
+ *
2369
+ * :params:
2370
+ * options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
2371
+ * :returns:
2372
+ * A promise that resolves once both fetches have settled
2373
+ */
2374
+ var loadPhraseArtifacts = exports.loadPhraseArtifacts = /*#__PURE__*/function () {
2375
+ var _ref14 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee3(options) {
2376
+ var _options$isLocalLLM2;
2377
+ var isLocalLLM, loadOne, _yield$Promise$allSet, _yield$Promise$allSet2, bigramsResult, phrasesResult, bigramCount, phraseCount;
2378
+ return _regenerator.default.wrap(function (_context3) {
2379
+ while (1) switch (_context3.prev = _context3.next) {
2380
+ case 0:
2381
+ if (!phrasesLoadStarted) {
2382
+ _context3.next = 1;
2383
+ break;
2384
+ }
2385
+ return _context3.abrupt("return");
2386
+ case 1:
2387
+ phrasesLoadStarted = true;
2388
+ isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2389
+ (0, _ufo.startExp)(_ufo.EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2390
+ isLocalLLM: isLocalLLM
2391
+ });
2392
+ loadOne = /*#__PURE__*/function () {
2393
+ var _ref15 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee2(artifactName, termType, label) {
2394
+ var payload, normalized;
2395
+ return _regenerator.default.wrap(function (_context2) {
2396
+ while (1) switch (_context2.prev = _context2.next) {
2397
+ case 0:
2398
+ _context2.next = 1;
2399
+ return (0, _artifactLoader.fetchAutocompleteArtifactJson)(artifactName, {
2400
+ label: label,
2401
+ validate: function validate(p) {
2402
+ return p != null && (0, _typeof2.default)(p) === 'object';
2403
+ }
2404
+ });
2405
+ case 1:
2406
+ payload = _context2.sent;
2407
+ normalized = normalizePhraseArtifact(payload);
2408
+ if (normalized) {
2409
+ _context2.next = 2;
2410
+ break;
2411
+ }
2412
+ throw new Error("[text-predictor] ".concat(label, " payload was not a recognised term-stats map"));
2413
+ case 2:
2414
+ return _context2.abrupt("return", initPhrases(normalized, termType));
2415
+ case 3:
2416
+ case "end":
2417
+ return _context2.stop();
2418
+ }
2419
+ }, _callee2);
2420
+ }));
2421
+ return function loadOne(_x3, _x4, _x5) {
2422
+ return _ref15.apply(this, arguments);
2423
+ };
2424
+ }();
2425
+ _context3.next = 2;
2426
+ return Promise.allSettled([loadOne(_artifactsManifest.ARTIFACT_NAME.BIGRAMS, 'bigram', 'bigrams'), loadOne(_artifactsManifest.ARTIFACT_NAME.PHRASES, 'phrase', 'phrases')]);
2427
+ case 2:
2428
+ _yield$Promise$allSet = _context3.sent;
2429
+ _yield$Promise$allSet2 = (0, _slicedToArray2.default)(_yield$Promise$allSet, 2);
2430
+ bigramsResult = _yield$Promise$allSet2[0];
2431
+ phrasesResult = _yield$Promise$allSet2[1];
2432
+ bigramCount = bigramsResult.status === 'fulfilled' ? bigramsResult.value : 0;
2433
+ phraseCount = phrasesResult.status === 'fulfilled' ? phrasesResult.value : 0;
2434
+ if (bigramsResult.status === 'rejected') {
2435
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F failed to load bigrams.json: ".concat(String(bigramsResult.reason)), _debugMode.CTC_STYLES.warn);
2436
+ }
2437
+ if (phrasesResult.status === 'rejected') {
2438
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F failed to load phrases.json: ".concat(String(phrasesResult.reason)), _debugMode.CTC_STYLES.warn);
2439
+ }
2440
+ if (bigramsResult.status === 'fulfilled' || phrasesResult.status === 'fulfilled') {
2441
+ phrasesLoaded = true;
2442
+ (0, _ufo.succeedExp)(_ufo.EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2443
+ isLocalLLM: isLocalLLM,
2444
+ bigramCount: bigramCount,
2445
+ phraseCount: phraseCount
2446
+ });
2447
+ (0, _debugMode.ctcTag)('init', "phrase artifacts loaded: ".concat(bigramCount, " bigrams \xB7 ").concat(phraseCount, " phrases"));
2448
+ } else {
2449
+ // Both failed — e.g. artifacts not published to the tenant manifest yet.
2450
+ // Reset the guard so a later focus/predict can retry once they land.
2451
+ phrasesLoadStarted = false;
2452
+ (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2453
+ isLocalLLM: isLocalLLM,
2454
+ errorType: 'network'
2455
+ });
2456
+ }
2457
+ case 3:
2458
+ case "end":
2459
+ return _context3.stop();
2460
+ }
2461
+ }, _callee3);
2462
+ }));
2463
+ return function loadPhraseArtifacts(_x2) {
2464
+ return _ref14.apply(this, arguments);
2465
+ };
2466
+ }();
823
2467
  var vocabularyLoadPromise;
824
2468
  var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefaultVocabulary(options) {
825
- var _options$isLocalLLM2;
2469
+ var _options$isLocalLLM3, _options$source;
826
2470
  if (isInitialized) {
827
2471
  return Promise.resolve();
828
2472
  }
829
2473
  if (vocabularyLoadPromise) {
830
2474
  return vocabularyLoadPromise;
831
2475
  }
832
- var isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2476
+ var isLocalLLM = (_options$isLocalLLM3 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM3 !== void 0 ? _options$isLocalLLM3 : false;
2477
+ var source = (_options$source = options === null || options === void 0 ? void 0 : options.source) !== null && _options$source !== void 0 ? _options$source : 'predict';
833
2478
  var surface = options === null || options === void 0 ? void 0 : options.surface;
834
- vocabularyLoadPromise = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee2() {
835
- var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t3;
836
- return _regenerator.default.wrap(function (_context2) {
837
- while (1) switch (_context2.prev = _context2.next) {
2479
+ vocabularyLoadPromise = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee4() {
2480
+ var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t2;
2481
+ return _regenerator.default.wrap(function (_context4) {
2482
+ while (1) switch (_context4.prev = _context4.next) {
838
2483
  case 0:
839
2484
  (0, _ufo.startExp)(_ufo.EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
840
2485
  isLocalLLM: isLocalLLM
841
2486
  }, surface ? {
842
2487
  surface: surface
843
2488
  } : {}));
844
- _context2.prev = 1;
845
- _context2.next = 2;
846
- return Promise.all([(0, _artifactsManifest.fetchArtifactJson)(_artifactsManifest.ARTIFACT_NAME.VOCABULARY), (0, _artifactsManifest.fetchArtifactJson)(_artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY)]);
2489
+ _context4.prev = 1;
2490
+ (0, _debugMode.ctcTag)('init', "loading artifacts (trigger: ".concat(source, " \xB7 localLLM: ").concat(isLocalLLM, ")"));
2491
+ if ((0, _debugMode.isAutocompleteDebugVerbose)()) {
2492
+ // eslint-disable-next-line no-console
2493
+ console.groupCollapsed('%c[CTC:init]%c artifact manifest', _debugMode.CTC_STYLES.brand, _debugMode.CTC_STYLES.body);
2494
+ // eslint-disable-next-line no-console
2495
+ console.log('Eager:', [_artifactsManifest.ARTIFACT_NAME.VOCABULARY, _artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY, _artifactsManifest.ARTIFACT_NAME.POS_TAGS, _artifactsManifest.ARTIFACT_NAME.GHOST_POS_TAGS, _artifactsManifest.ARTIFACT_NAME.GRAMMAR_TRANSITIONS]);
2496
+ // eslint-disable-next-line no-console
2497
+ console.log('Conditional:', ["".concat(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX, " (only when vectors load; phrase/bigram surfaces share this index)"), "".concat(_artifactsManifest.ARTIFACT_NAME.BIGRAMS, " + ").concat(_artifactsManifest.ARTIFACT_NAME.PHRASES, " (completion-term stats; fire-and-forget)"), "".concat(_artifactsManifest.ARTIFACT_NAME.PHRASE_CONTINUATION_TOKENS, " (local model only; canonical token IDs for the full served union)")]);
2498
+ // eslint-disable-next-line no-console
2499
+ console.groupEnd();
2500
+ }
2501
+ _context4.next = 2;
2502
+ return Promise.all([(0, _artifactLoader.fetchAutocompleteArtifactJson)(_artifactsManifest.ARTIFACT_NAME.VOCABULARY, {
2503
+ summarize: function summarize(payload) {
2504
+ return "".concat(Object.keys(payload.words).length, " words");
2505
+ },
2506
+ validate: isVocabularyJson
2507
+ }), (0, _artifactLoader.fetchAutocompleteArtifactJson)(_artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY, {
2508
+ summarize: function summarize(payload) {
2509
+ return "".concat(payload.length, " words");
2510
+ },
2511
+ validate: isStringArray
2512
+ }),
2513
+ // Awaited alongside the vocabulary so the grammar filter is settled
2514
+ // before the first suggestion can be produced. Without this the
2515
+ // predictor initialises on the smaller vocabulary payload and serves
2516
+ // candidates the filter would have dropped until the grammar lands.
2517
+ // Only settlement is required: a failed load leaves the filter as a
2518
+ // pass-through, which is the same degradation as an absent artifact.
2519
+ (0, _scoringPipeline.loadGrammarDataAsync)({
2520
+ isLocalLLM: isLocalLLM,
2521
+ surface: surface
2522
+ }).catch(function (error) {
2523
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F grammar data unavailable; filtering skipped: ".concat(String(error)), _debugMode.CTC_STYLES.warn);
2524
+ })]);
847
2525
  case 2:
848
- _yield$Promise$all = _context2.sent;
2526
+ _yield$Promise$all = _context4.sent;
849
2527
  _yield$Promise$all2 = (0, _slicedToArray2.default)(_yield$Promise$all, 2);
850
2528
  vocabularyData = _yield$Promise$all2[0];
851
2529
  l3VocabularyData = _yield$Promise$all2[1];
852
- if (!((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData))) {
853
- _context2.next = 3;
854
- break;
855
- }
856
- throw new Error("[text-predictor] unexpected vocabulary payload \u2014 " + "".concat(_artifactsManifest.ARTIFACT_NAME.VOCABULARY, " ").concat((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null ? 'is missing its `words` wrapper' : 'is valid', ", ") + "".concat(_artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY, " ").concat(Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'));
857
- case 3:
858
- terms = Object.entries(vocabularyData.words).map(function (_ref8) {
859
- var _ref9 = (0, _slicedToArray2.default)(_ref8, 2),
860
- word = _ref9[0],
861
- stats = _ref9[1];
2530
+ terms = Object.entries(vocabularyData.words).map(function (_ref17) {
2531
+ var _ref18 = (0, _slicedToArray2.default)(_ref17, 2),
2532
+ word = _ref18[0],
2533
+ stats = _ref18[1];
862
2534
  return {
863
2535
  word: word,
864
2536
  freq: stats.freq,
@@ -872,6 +2544,12 @@ var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefault
872
2544
  initVocabulary({
873
2545
  terms: terms
874
2546
  });
2547
+
2548
+ // Phrase/bigram artifacts load independently and must never affect the
2549
+ // word path — fire-and-forget with its own error handling inside.
2550
+ void loadPhraseArtifacts({
2551
+ isLocalLLM: isLocalLLM
2552
+ }).catch(function () {});
875
2553
  (0, _ufo.succeedExp)(_ufo.EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
876
2554
  isLocalLLM: isLocalLLM,
877
2555
  l2WordCount: terms.length,
@@ -879,25 +2557,25 @@ var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefault
879
2557
  }, surface ? {
880
2558
  surface: surface
881
2559
  } : {}));
882
- _context2.next = 5;
2560
+ _context4.next = 4;
883
2561
  break;
884
- case 4:
885
- _context2.prev = 4;
886
- _t3 = _context2["catch"](1);
2562
+ case 3:
2563
+ _context4.prev = 3;
2564
+ _t2 = _context4["catch"](1);
887
2565
  (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
888
2566
  isLocalLLM: isLocalLLM,
889
- errorType: 'load_error'
2567
+ errorType: 'parse_error'
890
2568
  }, surface ? {
891
2569
  surface: surface
892
2570
  } : {}));
893
2571
  // Allow a later call to retry the load rather than caching the failure.
894
2572
  vocabularyLoadPromise = undefined;
895
- throw _t3;
896
- case 5:
2573
+ throw _t2;
2574
+ case 4:
897
2575
  case "end":
898
- return _context2.stop();
2576
+ return _context4.stop();
899
2577
  }
900
- }, _callee2, null, [[1, 4]]);
2578
+ }, _callee4, null, [[1, 3]]);
901
2579
  }))();
902
2580
  return vocabularyLoadPromise;
903
2581
  };