@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +108 -0
  2. package/dist/cjs/analytics/ufo.js +2 -1
  3. package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
  4. package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
  5. package/dist/cjs/pm-plugins/autocomplete-plugin.js +498 -160
  6. package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
  7. package/dist/cjs/pm-plugins/debug-mode.js +105 -7
  8. package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
  9. package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
  10. package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
  11. package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
  12. package/dist/cjs/pm-plugins/text-predictor.js +2099 -309
  13. package/dist/es2019/analytics/ufo.js +2 -0
  14. package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
  15. package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
  16. package/dist/es2019/pm-plugins/autocomplete-plugin.js +489 -152
  17. package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
  18. package/dist/es2019/pm-plugins/debug-mode.js +102 -7
  19. package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
  20. package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
  21. package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
  22. package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
  23. package/dist/es2019/pm-plugins/text-predictor.js +1692 -200
  24. package/dist/esm/analytics/ufo.js +2 -1
  25. package/dist/esm/pm-plugins/artifact-loader.js +128 -0
  26. package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
  27. package/dist/esm/pm-plugins/autocomplete-plugin.js +501 -163
  28. package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
  29. package/dist/esm/pm-plugins/debug-mode.js +104 -6
  30. package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
  31. package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
  32. package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
  33. package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
  34. package/dist/esm/pm-plugins/text-predictor.js +2101 -313
  35. package/dist/types/analytics/ufo.d.ts +1 -0
  36. package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
  37. package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
  38. package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
  39. package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
  40. package/dist/types/pm-plugins/debug-mode.d.ts +55 -1
  41. package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
  42. package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
  43. package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
  44. package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
  45. package/dist/types/pm-plugins/text-predictor.d.ts +156 -3
  46. package/package.json +2 -2
  47. package/src/analytics/ufo.ts +5 -0
  48. package/src/pm-plugins/artifact-loader.ts +92 -0
  49. package/src/pm-plugins/artifacts-manifest.ts +3 -0
  50. package/src/pm-plugins/autocomplete-plugin.ts +584 -154
  51. package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
  52. package/src/pm-plugins/debug-mode.ts +134 -7
  53. package/src/pm-plugins/ghost-text-decoration.ts +13 -4
  54. package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
  55. package/src/pm-plugins/scoring-pipeline.ts +167 -17
  56. package/src/pm-plugins/slow-lane-client.ts +147 -62
  57. package/src/pm-plugins/text-predictor.ts +2119 -254
  58. package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
  59. package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
  60. package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
  61. package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
  62. package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
  63. package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
  64. package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  65. package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  66. package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
  67. package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
  68. package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
  69. package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
  70. package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
  71. package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
  72. package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
  73. package/scripts/gen_first_token_to_words.py +0 -170
  74. package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
  75. package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
  76. package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
  77. package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
  78. package/src/pm-plugins/slow-lane-client/package.json +0 -8
  79. package/src/pm-plugins/text-predictor/package.json +0 -8
@@ -4,16 +4,19 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau
4
4
  Object.defineProperty(exports, "__esModule", {
5
5
  value: true
6
6
  });
7
- exports.predict = exports.loadVectorsAsync = exports.loadDefaultVocabulary = exports.initVocabulary = exports.initVectors = exports.initL3Vocabulary = exports.ingestDocumentPage = exports.incrementSessionFreq = exports.getPredictorStatus = exports.getLastPredictionDebug = void 0;
7
+ exports.predict = exports.noteSuggestionAccepted = exports.loadVectorsAsync = exports.loadPhraseArtifacts = exports.loadDefaultVocabulary = exports.inspectSessionBoosts = exports.initVocabulary = exports.initVectors = exports.initPhrases = exports.initL3Vocabulary = exports.ingestDocumentPage = exports.incrementSessionFreq = exports.getPredictorStatus = exports.getLastPredictionDebug = void 0;
8
8
  var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
9
- var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
10
9
  var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
10
+ var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
11
+ var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
11
12
  var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
12
13
  var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
13
14
  var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
14
15
  var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
15
16
  var _ufo = require("../analytics/ufo");
17
+ var _artifactLoader = require("./artifact-loader");
16
18
  var _artifactsManifest = require("./artifacts-manifest");
19
+ var _canonicalLmScoring = require("./canonical-lm-scoring");
17
20
  var _debugMode = require("./debug-mode");
18
21
  var _scoringPipeline = require("./scoring-pipeline");
19
22
  var _slowLaneClient = require("./slow-lane-client");
@@ -26,7 +29,7 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
26
29
  *
27
30
  * Two prediction modes:
28
31
  * 1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
29
- * 2. Mid-word (≥3 chars) → trie prefix search → scoring pipeline → top result
32
+ * 2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
30
33
  *
31
34
  * Scoring is delegated to scoring-pipeline.ts which handles:
32
35
  * Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
@@ -35,10 +38,11 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
35
38
  * Falls back to cold mode (freq-only) when vectors not yet loaded.
36
39
  *
37
40
  * Session personalization (L1): words the user types are incrementally boosted
38
- * via incrementSessionFreq(), called on word boundaries from the plugin.
39
- */ // The vocabulary, L3 and word-index payloads are fetched from the CDN in
40
- // loadDefaultVocabulary / loadVectorsAsync so their (large) contents never enter
41
- // the editor's bundle and only load when autocomplete is initialised.
41
+ * via incrementSessionFreq(), called on word boundaries from the plugin, and
42
+ * words in ingested context text via ingestDocumentPage(). What the session has
43
+ * boosted is visible at any time from the console: `__atlCtcDebug__.session()`,
44
+ * or `__atlCtcDebug__.session('poll')` for one family — see inspectSessionBoosts.
45
+ */
42
46
  // ─── Constants ───────────────────────────────────────────────────────────────
43
47
 
44
48
  // eslint-disable-next-line require-unicode-regexp
@@ -49,13 +53,152 @@ var WHITESPACE_SPLIT_REGEX = /\s+/;
49
53
  var SENTENCE_BOUNDARY_REGEX = /[\n.?!]+/;
50
54
  // eslint-disable-next-line require-unicode-regexp
51
55
  var TRAILING_WHITESPACE_REGEX = /\s$/;
52
- var MIN_PREFIX_LENGTH = 3;
56
+ var TRAILING_SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+$/;
57
+ var SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+/g;
58
+ var ONLY_WHITESPACE_REGEX = /^[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+$/;
59
+ var UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH = 1;
60
+ var MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH = 1;
61
+ var DISPLAY_MIN_PREFIX_LENGTH = 3;
53
62
  var MAX_CANDIDATES = 200;
54
63
  var CONTEXT_WORDS = 10;
55
- var MIN_SCORE_THRESHOLD = 0.35;
56
64
  var L3_BASELINE_FREQ = 0.001;
65
+ // Max tokens in a completion term (bigram=2, phrase=3). Mirrors the producer's
66
+ // `phrase_max_words` default so the FE window never exceeds what was mined.
67
+ var PHRASE_MAX_WORDS = 3;
68
+ // Per-window cap on phrase-trie subtree collection (keeps the hot path bounded).
69
+ var MAX_PHRASE_CANDIDATES = 50;
70
+ /**
71
+ * Share of its context shortlist's probability mass the model must put on a
72
+ * surface before that surface may be shown.
73
+ *
74
+ * Candidates competing at the same boundary are normalised into a posterior
75
+ * over that shortlist, so this reads directly as "the model is at least this
76
+ * sure". Measuring distance from the best candidate instead cannot express
77
+ * being unsure: the leader scores full marks whether it won by a nose or a
78
+ * mile, which pushes the entire question of how contested a context is onto
79
+ * `MIN_WINNER_MARGIN` — and there the blended score's only remaining spread is
80
+ * the corpus prior, the weakest ranking signal we have.
81
+ *
82
+ * Each extra token raises the bar. A wrong multi-word ghost costs the reader
83
+ * more to notice and undo than a wrong single word, and it is offered from the
84
+ * same keystroke, so precision has to be bought per token of commitment.
85
+ */
86
+ var MIN_LM_POSTERIOR = {
87
+ word: 0.5,
88
+ bigram: 0.6,
89
+ phrase: 0.7
90
+ };
91
+ /**
92
+ * How many scored candidates a context must hold before its leader may show.
93
+ *
94
+ * A posterior is a share of a shortlist, so a shortlist of one hands its only
95
+ * member 1.0 whatever the model thinks of it — `logSumExp` of a single value
96
+ * returns that value. Such a ghost sits in the top bucket without having beaten
97
+ * anything, and it is the gate's blind spot rather than a case the gate ruled
98
+ * on: 1 in 14 of them were accepted against 34% for word ghosts overall, and
99
+ * every phrase shown through this hole was rejected.
100
+ *
101
+ * Counted over pool membership rather than over the normaliser, because those
102
+ * two deliberately differ. Where a pool holds a surface and its own extension
103
+ * the chain rule leaves one term in the denominator, and the resulting 1.0 is
104
+ * earned — whichever the user meant, the shorter form is a correct ghost.
105
+ *
106
+ * Waived where the context never held a second candidate at all. The 1-in-14
107
+ * acceptance above was measured over a population dominated by pools that read
108
+ * as one because rivals had not come back, and it is that absence of evidence
109
+ * the floor is for. A vocabulary offering a single continuation of a long
110
+ * distinctive prefix is the opposite situation, and it is also where a ghost
111
+ * saves the most keystrokes, so refusing it spends the most to learn the least.
112
+ */
113
+ var MIN_SCORED_POOL_SIZE = 2;
114
+ /**
115
+ * Mean per-token log-probability a surface must hold, in nats, independently of
116
+ * anything it was competing against.
117
+ *
118
+ * Every other display test is relative: a posterior is a share of a shortlist,
119
+ * and a winner margin is a distance from a rival. So a surface reaches the
120
+ * screen by being the best of what happened to be in its pool, and nothing ever
121
+ * asks whether the model finds it plausible at all. That gap was tolerable while
122
+ * little was being read, and stopped being tolerable once the scheduler started
123
+ * reading roughly twice as much per decision: shown-per-decision went from 3.2%
124
+ * to 7.4% with no threshold moved, and acceptance of what showed fell from 32%
125
+ * to 24.5%. More reading does not weaken a relative test — it hands it far more
126
+ * pools to be the winner of, and the marginal winner is worse than the average.
127
+ *
128
+ * Set at the point where a uniform distribution over the model's vocabulary
129
+ * sits: 49,152 tokens, so `ln(1/49152)` is about -10.8 nats. Below it the model
130
+ * assigns the surface less mass than a token drawn at random, which is the only
131
+ * statement about a surface that needs no reference to what it competed with.
132
+ *
133
+ * It was first tried at -8, chosen from the bucket where acceptance was measured
134
+ * to fall away, and that reasoning does not survive more data. Acceptance against
135
+ * this quantity turned out to be a smooth slope — about 19% below -8, 26% from -8
136
+ * to -6, then flat near 35% above - with no discontinuity to place a threshold
137
+ * at. A floor on a smooth slope is a coverage dial rather than a test: -8 removed
138
+ * a band converting at 26% and cost 3.7 points of show rate, for an acceptance
139
+ * movement no sample of this size can resolve. Anchoring to uniform instead makes
140
+ * the value an argument about what is indefensible rather than a point picked off
141
+ * a histogram, which is what stops it from being retuned every batch.
142
+ *
143
+ * Applied to the mean over the verified prefix, the same quantity
144
+ * `ghostsByMeanLogProb` buckets, and deliberately not to
145
+ * `meanTokenLogProbabilityUpperBound`. The bound is the right instrument for
146
+ * keeping a candidate in the race, where dropping something that could still
147
+ * clear the bar would be an error, but it is systematically higher than the
148
+ * quantity acceptance is measured against. Judging a surface on the prefix that
149
+ * was actually read is also what every other test here does.
150
+ *
151
+ * One value across all three shapes. Splitting it per shape needs an acceptance
152
+ * curve per shape, and 19 bigrams and 1 phrase across five sessions cannot
153
+ * support one.
154
+ */
155
+ var MIN_MEAN_TOKEN_LOG_PROBABILITY = -10.8;
156
+ // Tab replaces the whole remaining suffix, so an N-character completion saves
157
+ // N-1 keystrokes: at one character it saves nothing at all and at two it saves
158
+ // one, which does not repay noticing the ghost and reaching for Tab. Offering
159
+ // them anyway spends the acceptance-rate denominator on completions nobody
160
+ // wants and teaches the user to stop reading ghosts. `DISPLAY_MIN_PREFIX_LENGTH`
161
+ // is the same guard on the side the user has already typed.
162
+ var MIN_SUGGESTION_LENGTH = 3;
163
+ /**
164
+ * How many leading tokens of a surface must be scored before it may be shown.
165
+ *
166
+ * Scoring a token costs one model round trip, and a round trip costs about the
167
+ * same no matter how much is in it, so the price of a candidate is its token
168
+ * count. Demanding every token therefore caps what can ever reach the screen at
169
+ * roughly two tokens inside the decision budget — which is why no phrase has
170
+ * ever been displayed. Verifying a fixed prefix makes the cost of a candidate
171
+ * independent of its length.
172
+ */
173
+ var REQUIRED_VERIFIED_TOKENS = 2;
174
+ /**
175
+ * Extra winner margin demanded of a surface judged on a verified prefix rather
176
+ * than on every one of its tokens, to buy back the precision given up by
177
+ * leaving the tail unread.
178
+ */
179
+ var PARTIAL_EVIDENCE_MARGIN_PREMIUM = 0.08;
180
+ var ARBITRATION_MODE = 'confidence-v2';
181
+ var DEBUG_TEXT_TAIL_CHARS = 120;
57
182
 
58
183
  // ─── Types ───────────────────────────────────────────────────────────────────
184
+
185
+ /**
186
+ * Which constraint stopped an evaluation from putting a ghost on screen.
187
+ *
188
+ * Each one implies different work: `below-posterior-gate` is a threshold to
189
+ * calibrate, `winner-margin` is two candidates the model cannot separate,
190
+ * `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
191
+ * is vocabulary coverage.
192
+ */
193
+
194
+ /**
195
+ * A candidate paired with the length of the already-typed prefix it completes.
196
+ * For a single word this is the current partial token length; for a phrase it
197
+ * is the length of the matched multi-word window (previous words + partial),
198
+ * so the ghost tail is `word.slice(matchedPrefixLen)`.
199
+ */
200
+
201
+ /** Optional per-term metadata carried by bigram/phrase completion terms. */
59
202
  var TrieNode = /*#__PURE__*/(0, _createClass2.default)(function TrieNode() {
60
203
  (0, _classCallCheck2.default)(this, TrieNode);
61
204
  (0, _defineProperty2.default)(this, "children", new Map());
@@ -64,6 +207,12 @@ var TrieNode = /*#__PURE__*/(0, _createClass2.default)(function TrieNode() {
64
207
  (0, _defineProperty2.default)(this, "docFreq", 0);
65
208
  (0, _defineProperty2.default)(this, "authorFreq", 0);
66
209
  (0, _defineProperty2.default)(this, "sessionFreq", 0);
210
+ // Term-type + phrase metadata. Plain words keep `termType='word'` and null
211
+ // metadata; bigram/phrase nodes carry the producer-shipped POS + association.
212
+ (0, _defineProperty2.default)(this, "termType", 'word');
213
+ (0, _defineProperty2.default)(this, "posSeq", null);
214
+ (0, _defineProperty2.default)(this, "headPos", null);
215
+ (0, _defineProperty2.default)(this, "association", null);
67
216
  });
68
217
  var WeightedWordTrie = /*#__PURE__*/function () {
69
218
  function WeightedWordTrie() {
@@ -74,7 +223,8 @@ var WeightedWordTrie = /*#__PURE__*/function () {
74
223
  }
75
224
  return (0, _createClass2.default)(WeightedWordTrie, [{
76
225
  key: "insert",
77
- value: function insert(word, tenantFreq, docFreq, authorFreq) {
226
+ value: function insert(word, tenantFreq, docFreq, authorFreq, meta) {
227
+ var _meta$termType, _meta$posSeq, _meta$headPos, _meta$association;
78
228
  var node = this.root;
79
229
  var _iterator = _createForOfIteratorHelper(word.toLowerCase()),
80
230
  _step;
@@ -97,6 +247,10 @@ var WeightedWordTrie = /*#__PURE__*/function () {
97
247
  node.tenantFreq = tenantFreq;
98
248
  node.docFreq = docFreq;
99
249
  node.authorFreq = authorFreq;
250
+ node.termType = (_meta$termType = meta === null || meta === void 0 ? void 0 : meta.termType) !== null && _meta$termType !== void 0 ? _meta$termType : 'word';
251
+ node.posSeq = (_meta$posSeq = meta === null || meta === void 0 ? void 0 : meta.posSeq) !== null && _meta$posSeq !== void 0 ? _meta$posSeq : null;
252
+ node.headPos = (_meta$headPos = meta === null || meta === void 0 ? void 0 : meta.headPos) !== null && _meta$headPos !== void 0 ? _meta$headPos : null;
253
+ node.association = (_meta$association = meta === null || meta === void 0 ? void 0 : meta.association) !== null && _meta$association !== void 0 ? _meta$association : null;
100
254
  if (tenantFreq > this.maxTenantFreq) {
101
255
  this.maxTenantFreq = tenantFreq;
102
256
  }
@@ -209,6 +363,65 @@ var WeightedWordTrie = /*#__PURE__*/function () {
209
363
  node.sessionFreq += 1;
210
364
  return true;
211
365
  }
366
+
367
+ /**
368
+ * Every word carrying a session boost, optionally limited to one prefix's
369
+ * subtree. Unlike `getCandidates` a word equal to the prefix is included,
370
+ * since the question here is what the session holds rather than what could
371
+ * still be typed.
372
+ *
373
+ * Walks the trie instead of reading an index, so nothing has to be kept in
374
+ * step on the write path for the sake of being able to ask.
375
+ */
376
+ }, {
377
+ key: "collectSessionBoosted",
378
+ value: function collectSessionBoosted() {
379
+ var prefix = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : '';
380
+ var node = this.root;
381
+ var _iterator5 = _createForOfIteratorHelper(prefix.toLowerCase()),
382
+ _step5;
383
+ try {
384
+ for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
385
+ var char = _step5.value;
386
+ var next = node.children.get(char);
387
+ if (!next) {
388
+ return [];
389
+ }
390
+ node = next;
391
+ }
392
+ } catch (err) {
393
+ _iterator5.e(err);
394
+ } finally {
395
+ _iterator5.f();
396
+ }
397
+ var boosted = [];
398
+ var stack = [node];
399
+ while (stack.length > 0) {
400
+ var current = stack.pop();
401
+ if (!current) {
402
+ continue;
403
+ }
404
+ if (current.word !== null && current.sessionFreq > 0) {
405
+ boosted.push({
406
+ word: current.word,
407
+ node: current
408
+ });
409
+ }
410
+ var _iterator6 = _createForOfIteratorHelper(current.children.values()),
411
+ _step6;
412
+ try {
413
+ for (_iterator6.s(); !(_step6 = _iterator6.n()).done;) {
414
+ var child = _step6.value;
415
+ stack.push(child);
416
+ }
417
+ } catch (err) {
418
+ _iterator6.e(err);
419
+ } finally {
420
+ _iterator6.f();
421
+ }
422
+ }
423
+ return boosted;
424
+ }
212
425
  }]);
213
426
  }(); // L1/L2 Trie (Session + Atlassian Domain)
214
427
  var wordTrie = new WeightedWordTrie();
@@ -216,29 +429,36 @@ var wordTrie = new WeightedWordTrie();
216
429
  // L3 Trie (General English Fallback)
217
430
  var l3Trie = new WeightedWordTrie();
218
431
 
432
+ /**
433
+ * L2 phrase/bigram trie. Kept SEPARATE from `wordTrie` so single-word
434
+ * completion + L3 gap-fill stay untouched; multi-word units are only surfaced
435
+ * by the (12b) multiword prefix-window matcher. Keys are the full normalized
436
+ * surface string (spaces included), e.g. `"return on investment"`, so a lookup
437
+ * is a plain char-trie walk over the whole phrase.
438
+ */
439
+ var phraseTrie = new WeightedWordTrie();
440
+
219
441
  /**
220
442
  * Loads the General English vocabulary.
221
443
  * expects a simple array of strings: ["about", "above", "actually", ...]
222
444
  */
223
445
  var initL3Vocabulary = exports.initL3Vocabulary = function initL3Vocabulary(l3Words) {
224
- var _iterator5 = _createForOfIteratorHelper(l3Words),
225
- _step5;
446
+ var _iterator7 = _createForOfIteratorHelper(l3Words),
447
+ _step7;
226
448
  try {
227
- for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
228
- var word = _step5.value;
449
+ for (_iterator7.s(); !(_step7 = _iterator7.n()).done;) {
450
+ var word = _step7.value;
229
451
  // Insert with a tiny baseline frequency so it mathematically
230
452
  // loses to any domain word in Stage 1, but still scores above 0.
231
453
  l3Trie.insert(word, L3_BASELINE_FREQ, 0, 0);
232
454
  }
233
455
  } catch (err) {
234
- _iterator5.e(err);
456
+ _iterator7.e(err);
235
457
  } finally {
236
- _iterator5.f();
237
- }
238
- if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
239
- // eslint-disable-next-line no-console
240
- console.log("[text-predictor] L3 General English loaded: ".concat(l3Words.length, " words"));
458
+ _iterator7.f();
241
459
  }
460
+ recallGeneration++;
461
+ (0, _debugMode.ctcTag)('init', "L3 general English loaded: ".concat(l3Words.length, " words"));
242
462
  };
243
463
 
244
464
  // const bigramMap: Map<string, Record<string, number>> = new Map(
@@ -248,8 +468,50 @@ var initL3Vocabulary = exports.initL3Vocabulary = function initL3Vocabulary(l3Wo
248
468
  var isInitialized = false;
249
469
  var vectorStore = null;
250
470
  var vectorsLoadStarted = false;
471
+
472
+ // Phrase/bigram artifacts load independently of the word vocabulary so a
473
+ // missing/late `bigrams.json`/`phrases.json` never breaks word completion.
474
+ var phrasesLoadStarted = false;
475
+ var phrasesLoaded = false;
476
+ var phraseTermCount = 0;
477
+ // Per-type frequency maxima, tracked separately because bigrams and phrases
478
+ // share one trie but have very different absolute frequency ranges (bigrams are
479
+ // far more frequent than 3-word phrases). Normalizing each type by its OWN max
480
+ // stops phrase freq scores from being crushed against the top bigram's count.
481
+ var maxBigramFreq = 1;
482
+ var maxPhraseFreq = 1;
251
483
  var lastPredictionDebug = null;
252
484
 
485
+ // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
486
+ // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
487
+ // generally keep a just-accepted unit from being re-offered:
488
+ // • repetition: never surface a candidate whose full surface already equals the
489
+ // run of tokens immediately before the cursor (handled inline in predict()).
490
+ // • cooldown: after an accept, briefly suppress re-offering that same unit. The
491
+ // cooldown lifts only once BOTH the keystroke budget and the time window have
492
+ // elapsed (either one still counting keeps it active).
493
+ var COOLDOWN_KEYSTROKES = 5;
494
+ var COOLDOWN_MS = 2000;
495
+
496
+ // Active post-accept cooldown, or null when none.
497
+ var acceptCooldown = null;
498
+
499
+ /**
500
+ * Start a short cooldown for the exact surface the editor inserted. The caller
501
+ * passes the committed snapshot's surface so background re-ranking can never
502
+ * move cooldown bookkeeping away from what the user actually accepted.
503
+ */
504
+ var noteSuggestionAccepted = exports.noteSuggestionAccepted = function noteSuggestionAccepted(surface) {
505
+ var normalizedSurface = surface.trim().toLowerCase();
506
+ if (normalizedSurface) {
507
+ acceptCooldown = {
508
+ predictionsSince: 0,
509
+ surface: normalizedSurface,
510
+ ts: performance.now()
511
+ };
512
+ }
513
+ };
514
+
253
515
  /** Get vector for a word from the store. */
254
516
  var getWordVector = function getWordVector(word) {
255
517
  if (!vectorStore) {
@@ -274,20 +536,20 @@ var computeContextVectorLocal = function computeContextVectorLocal(textBefore) {
274
536
  var tokens = tokenize(textBefore);
275
537
  var words = tokens.slice(-CONTEXT_WORDS);
276
538
  var vectors = [];
277
- var _iterator6 = _createForOfIteratorHelper(words),
278
- _step6;
539
+ var _iterator8 = _createForOfIteratorHelper(words),
540
+ _step8;
279
541
  try {
280
- for (_iterator6.s(); !(_step6 = _iterator6.n()).done;) {
281
- var word = _step6.value;
542
+ for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
543
+ var word = _step8.value;
282
544
  var _v = getWordVector(word);
283
545
  if (_v) {
284
546
  vectors.push(_v);
285
547
  }
286
548
  }
287
549
  } catch (err) {
288
- _iterator6.e(err);
550
+ _iterator8.e(err);
289
551
  } finally {
290
- _iterator6.f();
552
+ _iterator8.f();
291
553
  }
292
554
  if (vectors.length === 0) {
293
555
  return null;
@@ -320,11 +582,11 @@ var getContextVectorForScoring = function getContextVectorForScoring(textBefore)
320
582
  var tokenize = function tokenize(text) {
321
583
  var tokens = [];
322
584
  // eslint-disable-next-line @atlassian/perf-linting/no-expensive-split-replace
323
- var _iterator7 = _createForOfIteratorHelper(text.toLowerCase().split(WHITESPACE_SPLIT_REGEX)),
324
- _step7;
585
+ var _iterator9 = _createForOfIteratorHelper(text.toLowerCase().split(WHITESPACE_SPLIT_REGEX)),
586
+ _step9;
325
587
  try {
326
- for (_iterator7.s(); !(_step7 = _iterator7.n()).done;) {
327
- var raw = _step7.value;
588
+ for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
589
+ var raw = _step9.value;
328
590
  // eslint-disable-next-line @atlassian/perf-linting/no-expensive-split-replace
329
591
  var clean = raw.replace(PUNCTUATION_BOUNDARY_REGEX, '');
330
592
  if (clean.length >= 2) {
@@ -332,9 +594,9 @@ var tokenize = function tokenize(text) {
332
594
  }
333
595
  }
334
596
  } catch (err) {
335
- _iterator7.e(err);
597
+ _iterator9.e(err);
336
598
  } finally {
337
- _iterator7.f();
599
+ _iterator9.f();
338
600
  }
339
601
  return tokens;
340
602
  };
@@ -346,6 +608,124 @@ var extractPreviousWord = function extractPreviousWord(text) {
346
608
  return words.length >= 2 ? words[words.length - 2] : '';
347
609
  };
348
610
 
611
+ /**
612
+ * Collect phrase/bigram completion candidates for the multi-word window ending
613
+ * at the current partial token.
614
+ *
615
+ * Builds windows of the last `2..PHRASE_MAX_WORDS` tokens of the current
616
+ * sentence (widest first) and prefix-searches the phrase trie for each, so
617
+ * `quarterly pla` matches `quarterly planning meeting`. Windows never cross a
618
+ * sentence boundary (mirrors `extractPreviousWord`). Each result records the
619
+ * matched window length so the caller can compute the ghost tail.
620
+ *
621
+ * :params:
622
+ * trimmed: Text before the cursor, trailing whitespace already removed
623
+ * :returns:
624
+ * Matched phrase candidates (deduped, widest-window match wins per term)
625
+ */
626
+ var getPhraseCandidates = function getPhraseCandidates(trimmed) {
627
+ // Records every prefix window we probe and its match count so the (verbose)
628
+ // debug line makes it obvious whether the phrase path found nothing because the
629
+ // trie is empty, or because no phrase begins with the typed window.
630
+ var windowsTried = [];
631
+ var logPhrasePath = function logPhrasePath(skipReason) {
632
+ if (!(0, _debugMode.isAutocompleteDebugVerbose)()) {
633
+ return;
634
+ }
635
+ var detail = skipReason ? skipReason : "windows: ".concat(windowsTried.map(function (w) {
636
+ return "\"".concat(w.window, "\"\u2192").concat(w.matches);
637
+ }).join(', ') || '(none)');
638
+ (0, _debugMode.ctcTag)('phrase', "loaded=".concat(phraseTermCount, " (").concat(phrasesLoaded ? 'ready' : 'not loaded', ") \xB7 ").concat(detail), _debugMode.CTC_STYLES.section);
639
+ };
640
+ if (phraseTermCount === 0) {
641
+ logPhrasePath('phrase trie empty — bigrams.json/phrases.json not loaded (or 0 terms)');
642
+ return [];
643
+ }
644
+ var tokenMatches = Array.from(trimmed.matchAll(SURFACE_TOKEN_REGEX));
645
+ var lastMatch = tokenMatches[tokenMatches.length - 1];
646
+ var lastStart = lastMatch === null || lastMatch === void 0 ? void 0 : lastMatch.index;
647
+ if (!lastMatch || lastStart === undefined || lastStart + lastMatch[0].length !== trimmed.length) {
648
+ logPhrasePath('no tokens in current sentence');
649
+ return [];
650
+ }
651
+ var trailingTokens = [{
652
+ start: lastStart,
653
+ value: lastMatch[0]
654
+ }];
655
+ for (var i = tokenMatches.length - 2; i >= 0; i--) {
656
+ var current = trailingTokens[0];
657
+ var match = tokenMatches[i];
658
+ var start = match.index;
659
+ if (start === undefined) {
660
+ break;
661
+ }
662
+ var end = start + match[0].length;
663
+ var gap = trimmed.slice(end, current.start);
664
+ if (!ONLY_WHITESPACE_REGEX.test(gap)) {
665
+ break;
666
+ }
667
+ trailingTokens.unshift({
668
+ start: start,
669
+ value: match[0]
670
+ });
671
+ if (trailingTokens.length >= PHRASE_MAX_WORDS) {
672
+ break;
673
+ }
674
+ }
675
+ var partial = trailingTokens[trailingTokens.length - 1].value;
676
+ var results = [];
677
+ var seen = new Set();
678
+ var maxPrev = trailingTokens.length - 1;
679
+
680
+ // Widest window first so the most specific (longest-context) match wins the
681
+ // dedup for a given phrase term. `prev` is the number of *preceding* complete
682
+ // words included ahead of the current partial:
683
+ // prev ≥ 1 → "continue a phrase I've started" (e.g. "root c" → root cause).
684
+ // Preceding-word context is strong, so a short partial is fine.
685
+ // prev = 0 → "the partial is a phrase's FIRST word" (e.g. "root" → root
686
+ // cause). No preceding context, so require the partial to be a
687
+ // one character so grouped LM work can begin early. UI remains
688
+ // gated at DISPLAY_MIN_PREFIX_LENGTH.
689
+ for (var prev = maxPrev; prev >= 0; prev--) {
690
+ if (prev === 0 && partial.length < MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH) {
691
+ continue;
692
+ }
693
+ var windowTokens = trailingTokens.slice(-(prev + 1));
694
+ var windowPrefix = windowTokens.map(function (token) {
695
+ return token.value;
696
+ }).join(' ').toLowerCase();
697
+ var windowLen = windowPrefix.length;
698
+ var matches = phraseTrie.getCandidates(windowPrefix, MAX_PHRASE_CANDIDATES);
699
+ windowsTried.push({
700
+ window: windowPrefix,
701
+ matches: matches.length
702
+ });
703
+ var _iterator0 = _createForOfIteratorHelper(matches),
704
+ _step0;
705
+ try {
706
+ for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
707
+ var _match = _step0.value;
708
+ if (seen.has(_match.word)) {
709
+ continue;
710
+ }
711
+ seen.add(_match.word);
712
+ results.push({
713
+ node: _match.node,
714
+ word: _match.word,
715
+ matchedPrefixLen: windowLen,
716
+ surfaceStart: windowTokens[0].start
717
+ });
718
+ }
719
+ } catch (err) {
720
+ _iterator0.e(err);
721
+ } finally {
722
+ _iterator0.f();
723
+ }
724
+ }
725
+ logPhrasePath();
726
+ return results;
727
+ };
728
+
349
729
  // ─── Debug Helpers ───────────────────────────────────────────────────────────
350
730
 
351
731
  /**
@@ -354,11 +734,25 @@ var extractPreviousWord = function extractPreviousWord(text) {
354
734
  * wordCount: number of words in vector store (0 if not loaded)
355
735
  */
356
736
  var getPredictorStatus = exports.getPredictorStatus = function getPredictorStatus() {
737
+ var slowLaneStatus = (0, _slowLaneClient.getDefaultSlowLaneClientStatus)();
357
738
  return {
739
+ canonicalScoringSupported: slowLaneStatus.canonicalScoringSupported,
740
+ canonicalSurfaceCount: slowLaneStatus.canonicalSurfaceCount,
741
+ clientId: slowLaneStatus.clientId,
742
+ clientKind: slowLaneStatus.clientKind,
743
+ localModelReady: slowLaneStatus.localModelReady,
744
+ slowLaneRegistered: slowLaneStatus.registered,
358
745
  vectorsLoaded: vectorStore !== null,
359
746
  wordCount: vectorStore ? Object.keys(vectorStore.wordIndex).length : 0,
360
747
  vectorsLoadStarted: vectorsLoadStarted,
361
- isInitialized: isInitialized
748
+ isInitialized: isInitialized,
749
+ phrasesLoaded: phrasesLoaded,
750
+ phraseCount: phraseTermCount,
751
+ maxFreqByType: {
752
+ word: wordTrie.maxTenantFreq,
753
+ bigram: maxBigramFreq,
754
+ phrase: maxPhraseFreq
755
+ }
362
756
  };
363
757
  };
364
758
 
@@ -370,19 +764,57 @@ var getLastPredictionDebug = exports.getLastPredictionDebug = function getLastPr
370
764
  return lastPredictionDebug;
371
765
  };
372
766
  var initVocabulary = exports.initVocabulary = function initVocabulary(vocabulary) {
373
- var _iterator8 = _createForOfIteratorHelper(vocabulary.terms),
374
- _step8;
767
+ var _iterator1 = _createForOfIteratorHelper(vocabulary.terms),
768
+ _step1;
375
769
  try {
376
- for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
377
- var term = _step8.value;
770
+ for (_iterator1.s(); !(_step1 = _iterator1.n()).done;) {
771
+ var term = _step1.value;
378
772
  wordTrie.insert(term.word, term.freq, term.docFreq, term.authorFreq);
379
773
  }
380
774
  } catch (err) {
381
- _iterator8.e(err);
775
+ _iterator1.e(err);
382
776
  } finally {
383
- _iterator8.f();
777
+ _iterator1.f();
384
778
  }
385
779
  isInitialized = true;
780
+ recallGeneration++;
781
+ };
782
+
783
+ /**
784
+ * Insert a producer-shipped bigram/phrase artifact into the phrase trie.
785
+ *
786
+ * :params:
787
+ * artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
788
+ * termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
789
+ * :returns:
790
+ * The number of terms inserted
791
+ */
792
+ var initPhrases = exports.initPhrases = function initPhrases(artifact, termType) {
793
+ var count = 0;
794
+ for (var _i3 = 0, _Object$entries = Object.entries(artifact); _i3 < _Object$entries.length; _i3++) {
795
+ var _stats$pos_seq, _stats$head_pos, _stats$association;
796
+ var _Object$entries$_i = (0, _slicedToArray2.default)(_Object$entries[_i3], 2),
797
+ term = _Object$entries$_i[0],
798
+ stats = _Object$entries$_i[1];
799
+ phraseTrie.insert(term, stats.freq, stats.doc_freq, stats.author_freq, {
800
+ termType: termType,
801
+ posSeq: (_stats$pos_seq = stats.pos_seq) !== null && _stats$pos_seq !== void 0 ? _stats$pos_seq : null,
802
+ headPos: (_stats$head_pos = stats.head_pos) !== null && _stats$head_pos !== void 0 ? _stats$head_pos : null,
803
+ association: (_stats$association = stats.association) !== null && _stats$association !== void 0 ? _stats$association : null
804
+ });
805
+ // Track each type's own frequency ceiling for per-type normalization.
806
+ if (termType === 'bigram') {
807
+ if (stats.freq > maxBigramFreq) {
808
+ maxBigramFreq = stats.freq;
809
+ }
810
+ } else if (stats.freq > maxPhraseFreq) {
811
+ maxPhraseFreq = stats.freq;
812
+ }
813
+ count++;
814
+ }
815
+ phraseTermCount += count;
816
+ recallGeneration++;
817
+ return count;
386
818
  };
387
819
 
388
820
  /**
@@ -409,141 +841,258 @@ var ingestDocumentPage = exports.ingestDocumentPage = function ingestDocumentPag
409
841
  }
410
842
  var words = tokenize(pageContent);
411
843
  var validBoostedWords = new Set();
412
- var _iterator9 = _createForOfIteratorHelper(words),
413
- _step9;
844
+ var _iterator10 = _createForOfIteratorHelper(words),
845
+ _step10;
414
846
  try {
415
- for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
416
- var word = _step9.value;
847
+ for (_iterator10.s(); !(_step10 = _iterator10.n()).done;) {
848
+ var word = _step10.value;
417
849
  var didBoost = wordTrie.incrementSessionFreq(word);
418
850
  if (didBoost) {
419
851
  validBoostedWords.add(word);
420
852
  }
421
853
  }
422
854
  } catch (err) {
423
- _iterator9.e(err);
855
+ _iterator10.e(err);
424
856
  } finally {
425
- _iterator9.f();
857
+ _iterator10.f();
426
858
  }
427
859
  if ((0, _debugMode.isAutocompleteDebugEnabled)() && validBoostedWords.size > 0) {
428
- // eslint-disable-next-line no-console
429
- console.groupCollapsed("%c[L1 Session] %cPrimed ".concat(validBoostedWords.size, " valid dictionary words from page"), 'color: #00b8d9; font-weight: bold;', 'color: inherit; font-style: italic;');
430
- // eslint-disable-next-line no-console
431
- console.dir(Array.from(validBoostedWords).sort());
432
- // eslint-disable-next-line no-console
433
- console.groupEnd();
860
+ (0, _debugMode.ctcTag)('init', "L1 session primed ".concat(validBoostedWords.size, " words from page \xB7 __atlCtcDebug__.session() to inspect"), _debugMode.CTC_STYLES.brand);
861
+ if ((0, _debugMode.isAutocompleteDebugVerbose)()) {
862
+ // eslint-disable-next-line no-console
863
+ console.dir(Array.from(validBoostedWords).sort());
864
+ }
434
865
  }
435
866
  };
436
- var predict = exports.predict = function predict(textBefore) {
437
- if (!isInitialized) {
438
- // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
439
- // and skip this keystroke; the plugin also primes it on focus, so the tries
440
- // are usually ready before the user types.
441
- void loadDefaultVocabulary().catch(function () {});
442
- return null;
443
- }
444
- var t0 = performance.now();
445
867
 
446
- // ── Step 1: Bigram-based next-word suggestion at word boundary ───────────
447
- // if (textBefore.length > 0 && /\s$/u.test(textBefore)) {
448
- // const words = textBefore.toLowerCase().trimEnd().split(/\s+/u);
449
- // const prevWord = words[words.length - 1];
450
- // const nextWords = bigramMap.get(prevWord);
451
- // if (nextWords) {
452
- // const sorted = Object.entries(nextWords).sort((a, b) => b[1] - a[1]);
453
- // let bestWord = '';
454
- // for (const [word] of sorted) {
455
- // if (!isGrammarAllowed(prevWord, word)) {
456
- // continue;
457
- // }
458
- // bestWord = word;
459
- // break;
460
- // }
461
- // if (bestWord) {
462
- // if (debugMode) {
463
- // const latencyMs = performance.now() - t0;
464
- // console.log(
465
- // '%c[autocomplete] BIGRAM',
466
- // 'color:cyan',
467
- // '| "' + prevWord + '" -> "' + bestWord + '" | ' + latencyMs.toFixed(1) + 'ms',
468
- // );
469
- // }
470
- // return bestWord;
471
- // }
472
- // }
473
- // if (debugMode) {
474
- // console.log(
475
- // '%c[autocomplete] BIGRAM-MISS',
476
- // 'color:gray',
477
- // '| no bigram for "' + words[words.length - 1] + '", skipping prefix completion',
478
- // );
479
- // }
480
- // return null;
481
- // }
482
-
483
- // ── Step 2: Prefix completion (≥3 chars typed) ──────────────────────────
484
- if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
485
- return null;
486
- }
487
- var trimmed = textBefore.trimEnd();
488
- var lastSpaceIdx = trimmed.lastIndexOf(' ');
489
- var currentWord = lastSpaceIdx === -1 ? trimmed : trimmed.slice(lastSpaceIdx + 1);
490
- if (currentWord.length < MIN_PREFIX_LENGTH) {
491
- return null;
492
- }
493
- var candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
868
+ /**
869
+ * How many boosted words `inspectSessionBoosts` lists.
870
+ *
871
+ * A page ingest can boost thousands, and a list that long is not read. The
872
+ * strongest boosts are the ones that change an ordering, and `boosted` still
873
+ * reports the full size, so the cap loses nothing but volume.
874
+ */
875
+ var MAX_LISTED_SESSION_WORDS = 50;
876
+ /**
877
+ * Read the session's L1 boosts, optionally narrowed to a prefix.
878
+ *
879
+ * Installed as `__atlCtcDebug__.session()`, with `__atlCtcDebug__.session('poll')`
880
+ * to ask about one family. Returned rather than logged, so the console renders it
881
+ * as an inspectable object and a caller can assert on it.
882
+ *
883
+ * Only words the vocabulary already holds can carry a boost, because both writers
884
+ * go through `incrementSessionFreq` and it only finds existing nodes. An ingested
885
+ * word absent from the vocabulary is therefore missing from here and always will
886
+ * be.
887
+ */
888
+ var inspectSessionBoosts = exports.inspectSessionBoosts = function inspectSessionBoosts(prefix) {
889
+ var boosted = wordTrie.collectSessionBoosted(prefix !== null && prefix !== void 0 ? prefix : '');
890
+ return _objectSpread(_objectSpread({
891
+ boosted: boosted.length,
892
+ limit: MAX_LISTED_SESSION_WORDS
893
+ }, prefix === undefined ? {} : {
894
+ prefix: prefix
895
+ }), {}, {
896
+ words: boosted.sort(function (a, b) {
897
+ return b.node.sessionFreq - a.node.sessionFreq || a.word.localeCompare(b.word, 'en', {
898
+ numeric: true,
899
+ sensitivity: 'base'
900
+ });
901
+ }).slice(0, MAX_LISTED_SESSION_WORDS).map(function (_ref) {
902
+ var node = _ref.node,
903
+ word = _ref.word;
904
+ return {
905
+ sessionFreq: node.sessionFreq,
906
+ sessionOnly: node.tenantFreq === 0,
907
+ surface: word,
908
+ tenantFreq: node.tenantFreq
909
+ };
910
+ })
911
+ });
912
+ };
913
+
914
+ // At module scope so the console answers before the first keystroke, which is
915
+ // when someone reaching for it usually asks.
916
+ (0, _debugMode.registerCtcSessionInspector)(inspectSessionBoosts);
917
+
918
+ /**
919
+ * Result of a prediction: the ghost tail to insert plus an immutable record of
920
+ * the evidence that authorized the UI commitment.
921
+ */
922
+
923
+ // ─── Recall memo ─────────────────────────────────────────────────────────────
924
+
925
+ /** Trie recall plus the canonical context derived for each matched term. */
926
+
927
+ /**
928
+ * Bumped whenever an artifact load changes what the tries can return, which is
929
+ * the only way the candidate *set* for a given prefix can change.
930
+ *
931
+ * Session-frequency boosts are deliberately not counted: they mutate trie nodes
932
+ * in place, and the memo holds those nodes by reference, so a boost is already
933
+ * visible through a cached entry.
934
+ */
935
+ var recallGeneration = 0;
936
+ var recallMemo = null;
937
+ var computeCanonicalRecall = function computeCanonicalRecall(trimmed, currentWord, currentWordStart) {
938
+ // Start recall at one character so grouped causal work can overlap later
939
+ // keystrokes. Nothing is displayed until three characters.
940
+ var wordCandidates = currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH ? wordTrie.getCandidates(currentWord, MAX_CANDIDATES) : [];
494
941
 
495
942
  // Gap-fill from the L3 general-English trie, requesting a full buffer so
496
943
  // enough survive de-duplication against the L2 results.
497
- if (candidates.length < MAX_CANDIDATES) {
944
+ if (currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH && wordCandidates.length < MAX_CANDIDATES) {
498
945
  var l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
499
- var existingWords = new Set(candidates.map(function (c) {
946
+ var existingWords = new Set(wordCandidates.map(function (c) {
500
947
  return c.word;
501
948
  }));
502
- var _iterator0 = _createForOfIteratorHelper(l3Candidates),
503
- _step0;
949
+ var _iterator11 = _createForOfIteratorHelper(l3Candidates),
950
+ _step11;
504
951
  try {
505
- for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
506
- var l3c = _step0.value;
507
- if (candidates.length >= MAX_CANDIDATES) break;
952
+ for (_iterator11.s(); !(_step11 = _iterator11.n()).done;) {
953
+ var l3c = _step11.value;
954
+ if (wordCandidates.length >= MAX_CANDIDATES) break;
508
955
  if (!existingWords.has(l3c.word)) {
509
- candidates.push(l3c);
956
+ wordCandidates.push(l3c);
510
957
  }
511
958
  }
512
959
  } catch (err) {
513
- _iterator0.e(err);
960
+ _iterator11.e(err);
514
961
  } finally {
515
- _iterator0.f();
962
+ _iterator11.f();
963
+ }
964
+ }
965
+
966
+ // Multi-word completion terms (bigrams/phrases) matched on the trailing
967
+ // window ending at the current partial token. Runs even when the unigram path
968
+ // was skipped for a short partial, so first-word/early phrase matches surface.
969
+ var phraseCandidates = getPhraseCandidates(trimmed);
970
+
971
+ // Unify: a word completes the current partial token; a phrase completes its
972
+ // matched multi-word window. Track the prefix length per term so the ghost
973
+ // tail is sliced correctly regardless of term type.
974
+ var matched = [].concat((0, _toConsumableArray2.default)(wordCandidates.map(function (_ref2) {
975
+ var word = _ref2.word,
976
+ node = _ref2.node;
977
+ return {
978
+ word: word,
979
+ node: node,
980
+ matchedPrefixLen: currentWord.length,
981
+ surfaceStart: currentWordStart
982
+ };
983
+ })), (0, _toConsumableArray2.default)(phraseCandidates));
984
+ var positionCache = (0, _canonicalLmScoring.createCanonicalContextPositionCache)();
985
+ var canonicalMatched = matched.map(function (candidate) {
986
+ return _objectSpread(_objectSpread({}, candidate), (0, _canonicalLmScoring.deriveCanonicalCandidateContext)(trimmed, candidate.matchedPrefixLen, candidate.word, _slowLaneClient.getCanonicalSurfaceTokenIds, candidate.surfaceStart, positionCache));
987
+ });
988
+ var prefixLenByWord = new Map();
989
+ var _iterator12 = _createForOfIteratorHelper(canonicalMatched),
990
+ _step12;
991
+ try {
992
+ for (_iterator12.s(); !(_step12 = _iterator12.n()).done;) {
993
+ var m = _step12.value;
994
+ if (!prefixLenByWord.has(m.word)) {
995
+ prefixLenByWord.set(m.word, m.matchedPrefixLen);
996
+ }
997
+ }
998
+ } catch (err) {
999
+ _iterator12.e(err);
1000
+ } finally {
1001
+ _iterator12.f();
1002
+ }
1003
+ return {
1004
+ canonicalMatched: canonicalMatched,
1005
+ prefixLenByWord: prefixLenByWord
1006
+ };
1007
+ };
1008
+
1009
+ /**
1010
+ * Recall and canonical derivation for `trimmed`, reusing the previous result
1011
+ * when nothing they depend on has changed.
1012
+ *
1013
+ * `predict()` runs several times for one keystroke — once when the decision
1014
+ * opens and again for each async evidence signal that lands inside the budget —
1015
+ * and only the evidence differs between those runs. Without this, every run
1016
+ * re-walks the tries and re-derives a canonical context per candidate.
1017
+ */
1018
+ var getCanonicalRecall = function getCanonicalRecall(trimmed, currentWord, currentWordStart) {
1019
+ var surfaceCount = (0, _slowLaneClient.getCanonicalSurfaceCount)();
1020
+ if (recallMemo !== null && recallMemo.trimmed === trimmed && recallMemo.generation === recallGeneration && recallMemo.surfaceCount === surfaceCount) {
1021
+ return recallMemo;
1022
+ }
1023
+ var recall = computeCanonicalRecall(trimmed, currentWord, currentWordStart);
1024
+ recallMemo = _objectSpread(_objectSpread({}, recall), {}, {
1025
+ generation: recallGeneration,
1026
+ surfaceCount: surfaceCount,
1027
+ trimmed: trimmed
1028
+ });
1029
+ return recall;
1030
+ };
1031
+ var predict = exports.predict = function predict(textBefore) {
1032
+ var _trimmed$match$, _trimmed$match, _canonicalMatched$fin, _eligible$, _eligible$0$score, _eligible$2, _judgedEvidenceFor3, _prefixLenByWord$get2, _runtimeBySurface$get5, _judgedEvidenceFor$me3, _judgedEvidenceFor6, _contextTotals$get$le4, _contextTotals$get4, _selectedEvidence$ver, _selectedEvidence$tot, _selectedEvidence$ver2;
1033
+ if (!isInitialized) {
1034
+ // Vocabulary artifacts load asynchronously over the autocomplete gateway.
1035
+ // Kick off the load and skip this keystroke; the plugin also primes it on
1036
+ // focus, so the tries are usually ready before the user types.
1037
+ void loadDefaultVocabulary({
1038
+ source: 'predict'
1039
+ }).catch(function () {});
1040
+ return null;
1041
+ }
1042
+ var t0 = performance.now();
1043
+
1044
+ // Advance the post-accept cooldown once per prediction. It stays active until
1045
+ // BOTH the keystroke budget and the time window have elapsed; while active it
1046
+ // suppresses re-offering the just-accepted surface (see arbitration below).
1047
+ var cooledSurface = null;
1048
+ if (acceptCooldown) {
1049
+ acceptCooldown.predictionsSince += 1;
1050
+ var stillByKeys = acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES;
1051
+ var stillByTime = performance.now() - acceptCooldown.ts < COOLDOWN_MS;
1052
+ if (stillByKeys || stillByTime) {
1053
+ cooledSurface = acceptCooldown.surface;
1054
+ } else {
1055
+ acceptCooldown = null;
1056
+ }
1057
+ }
1058
+
1059
+ // ── Boundary/early-prefix prefetch; display remains gated at ≥3 ─────────
1060
+ if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
1061
+ var boundaryContext = (0, _canonicalLmScoring.deriveWhitespaceBoundaryContext)(textBefore);
1062
+ if (boundaryContext) {
1063
+ (0, _slowLaneClient.primeBoundaryLm)(_objectSpread(_objectSpread({}, boundaryContext), {}, {
1064
+ familyKey: boundaryContext.contextKey,
1065
+ priority: 0
1066
+ }));
516
1067
  }
1068
+ return null;
517
1069
  }
1070
+ var trimmed = textBefore.trimEnd();
1071
+ var trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
1072
+ if (trailingSurfaceToken.length === 0) {
1073
+ return null;
1074
+ }
1075
+ var currentWord = trailingSurfaceToken;
1076
+ var currentWordStart = trimmed.length - currentWord.length;
1077
+ var _getCanonicalRecall = getCanonicalRecall(trimmed, currentWord, currentWordStart),
1078
+ canonicalMatched = _getCanonicalRecall.canonicalMatched,
1079
+ prefixLenByWord = _getCanonicalRecall.prefixLenByWord;
518
1080
 
519
- // If both Tries are completely empty for this prefix
520
- if (candidates.length === 0) {
1081
+ // If every trie was empty for this prefix
1082
+ if (canonicalMatched.length === 0) {
1083
+ if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
1084
+ // eslint-disable-next-line no-console
1085
+ console.log("%c[CTC]%c \u2014 abstain: no matches for \"".concat(currentWord, "\""), _debugMode.CTC_STYLES.brand, _debugMode.CTC_STYLES.body);
1086
+ }
521
1087
  return null;
522
1088
  }
523
1089
  var previousWord = extractPreviousWord(trimmed);
524
1090
  var contextVector = getContextVectorForScoring(trimmed);
525
1091
  var lmLogits = (0, _slowLaneClient.getStoredLmLogits)();
526
-
527
- // Raw LM Output Logger
528
- if ((0, _debugMode.isAutocompleteDebugEnabled)() && lmLogits && Object.keys(lmLogits).length > 0) {
529
- var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
530
- return b[1] - a[1];
531
- }).slice(0, 5).map(function (_ref) {
532
- var _ref2 = (0, _slicedToArray2.default)(_ref, 2),
533
- word = _ref2[0],
534
- score = _ref2[1];
535
- return {
536
- Word: word,
537
- Prob: Number(score.toFixed(5))
538
- };
539
- });
540
- // eslint-disable-next-line no-console
541
- console.log('%c[Raw LM Prediction]🧠', 'color: #e83e8c; font-weight: bold;', rawLmTop);
542
- }
543
1092
  var mode = contextVector ? 'warm' : 'cold';
544
1093
 
545
- // Build ScoringCandidate array from TrieNodes
546
- var scoringCandidates = candidates.map(function (_ref3) {
1094
+ // Build ScoringCandidate array from matched terms (words + phrases)
1095
+ var scoringCandidates = canonicalMatched.map(function (_ref3) {
547
1096
  var word = _ref3.word,
548
1097
  node = _ref3.node;
549
1098
  return {
@@ -551,98 +1100,1111 @@ var predict = exports.predict = function predict(textBefore) {
551
1100
  tenantFreq: node.tenantFreq,
552
1101
  docFreq: node.docFreq,
553
1102
  authorFreq: node.authorFreq,
554
- sessionFreq: node.sessionFreq
1103
+ sessionFreq: node.sessionFreq,
1104
+ termType: node.termType,
1105
+ posSeq: node.posSeq,
1106
+ headPos: node.headPos,
1107
+ association: node.association
555
1108
  };
556
1109
  });
557
1110
 
558
1111
  // Filter the LM payload to only words matching the current prefix so that
559
1112
  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
560
1113
  var prefix = currentWord.toLowerCase();
561
- var prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
1114
+ var currentWordSeparator = (_canonicalMatched$fin = canonicalMatched.find(function (candidate) {
1115
+ return candidate.node.termType === 'word';
1116
+ })) === null || _canonicalMatched$fin === void 0 ? void 0 : _canonicalMatched$fin.separatorKind;
1117
+ var prefixLmLogits = lmLogits && currentWordSeparator === 'whitespace' ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
562
1118
  var _ref5 = (0, _slicedToArray2.default)(_ref4, 1),
563
1119
  word = _ref5[0];
564
1120
  return word.startsWith(prefix);
565
1121
  })) : null;
566
- var _rankCandidates = (0, _scoringPipeline.rankCandidates)(scoringCandidates, contextVector, function (w) {
567
- return getWordVector(w);
568
- }, prefixLmLogits, wordTrie.maxTenantFreq, previousWord),
569
- ranked = _rankCandidates.candidates,
570
- grammarMeta = _rankCandidates.grammarMeta,
571
- pipelineDebug = _rankCandidates.pipelineDebug;
572
- var best = ranked[0];
573
- var suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
574
- if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
575
- var latencyMs = performance.now() - t0;
576
- var tokens = tokenize(trimmed);
577
- var contextWords = tokens.slice(-CONTEXT_WORDS);
578
- var belowThreshold = best && best.finalScore < MIN_SCORE_THRESHOLD;
579
- var suggestionLabel = belowThreshold ? '🚫 (below threshold)' : suggestion && suggestion.length > 0 ? "\u2728 \"".concat(suggestion, "\"") : '🚫 (no match)';
580
- lastPredictionDebug = {
581
- textBefore: trimmed,
582
- currentWord: currentWord,
583
- mode: mode,
584
- contextWords: contextWords,
585
- topCandidates: ranked.slice(0, 5).map(function (r) {
586
- return {
587
- word: r.word,
588
- finalScore: r.finalScore,
589
- semanticScore: r.semanticScore,
590
- freqScore: r.freqScore,
591
- lmScore: r.lmScore
1122
+ var canonicalScoringSupported = (0, _slowLaneClient.isCanonicalSurfaceScoringSupported)();
1123
+ var eligibleContextKeys = Array.from(new Set(canonicalMatched.filter(function (candidate) {
1124
+ return candidate.canonicalTokenIds !== null;
1125
+ }).map(function (candidate) {
1126
+ return candidate.contextKey;
1127
+ }))).sort();
1128
+ var familyKey = eligibleContextKeys.join("\x01");
1129
+ var primeRequests = (0, _canonicalLmScoring.selectBoundaryPrimeRequests)(familyKey, canonicalMatched, PHRASE_MAX_WORDS);
1130
+ var _iterator13 = _createForOfIteratorHelper(primeRequests),
1131
+ _step13;
1132
+ try {
1133
+ for (_iterator13.s(); !(_step13 = _iterator13.n()).done;) {
1134
+ var request = _step13.value;
1135
+ (0, _slowLaneClient.primeBoundaryLm)(request);
1136
+ }
1137
+ } catch (err) {
1138
+ _iterator13.e(err);
1139
+ } finally {
1140
+ _iterator13.f();
1141
+ }
1142
+ var runtimeBySurface = new Map(canonicalMatched.map(function (candidate) {
1143
+ return [candidate.word, candidate];
1144
+ }));
1145
+ var canonicalEvidence = new Map();
1146
+ var firstTokenGroups = new Map();
1147
+ var _iterator14 = _createForOfIteratorHelper(canonicalMatched),
1148
+ _step14;
1149
+ try {
1150
+ for (_iterator14.s(); !(_step14 = _iterator14.n()).done;) {
1151
+ var _firstTokenGroups$get;
1152
+ var _candidate = _step14.value;
1153
+ if (_candidate.canonicalTokenIds === null) {
1154
+ continue;
1155
+ }
1156
+ // This accessor also materializes an exact cache entry when all tokens
1157
+ // have already been covered by shared-prefix work.
1158
+ (0, _slowLaneClient.getProgressiveSurfaceEvidence)(_candidate.contextKey, _candidate.word);
1159
+ var exact = (0, _slowLaneClient.getSurfaceScore)(_candidate.contextKey, _candidate.word);
1160
+ if (exact) {
1161
+ canonicalEvidence.set(_candidate.word, {
1162
+ contextKey: _candidate.contextKey,
1163
+ meanTokenLogProbability: exact.meanTokenLogProbability,
1164
+ score: 0,
1165
+ separatorKind: 'whitespace',
1166
+ source: 'canonical-full-surface',
1167
+ totalLogProbability: exact.totalLogProbability,
1168
+ totalSurfaceCharCount: _candidate.word.length,
1169
+ totalTokenCount: exact.tokenCount,
1170
+ verifiedCharCount: _candidate.word.length,
1171
+ verifiedTokenCount: exact.tokenCount
1172
+ });
1173
+ continue;
1174
+ }
1175
+ var _group2 = (_firstTokenGroups$get = firstTokenGroups.get(_candidate.contextKey)) !== null && _firstTokenGroups$get !== void 0 ? _firstTokenGroups$get : [];
1176
+ _group2.push(_candidate);
1177
+ firstTokenGroups.set(_candidate.contextKey, _group2);
1178
+ }
1179
+ } catch (err) {
1180
+ _iterator14.e(err);
1181
+ } finally {
1182
+ _iterator14.f();
1183
+ }
1184
+ var _iterator15 = _createForOfIteratorHelper(firstTokenGroups),
1185
+ _step15;
1186
+ try {
1187
+ for (_iterator15.s(); !(_step15 = _iterator15.n()).done;) {
1188
+ var _step15$value = (0, _slicedToArray2.default)(_step15.value, 2),
1189
+ _contextKey = _step15$value[0],
1190
+ _group3 = _step15$value[1];
1191
+ var boundary = (0, _slowLaneClient.getBoundaryLmState)(_contextKey);
1192
+ if (!boundary) {
1193
+ continue;
1194
+ }
1195
+ var maxLogit = -Infinity;
1196
+ var _iterator22 = _createForOfIteratorHelper(_group3),
1197
+ _step22;
1198
+ try {
1199
+ for (_iterator22.s(); !(_step22 = _iterator22.n()).done;) {
1200
+ var _candidate2$canonical;
1201
+ var _candidate2 = _step22.value;
1202
+ var tokenId = (_candidate2$canonical = _candidate2.canonicalTokenIds) === null || _candidate2$canonical === void 0 ? void 0 : _candidate2$canonical[0];
1203
+ var rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
1204
+ if (rawLogit !== undefined && Number.isFinite(rawLogit) && rawLogit > maxLogit) {
1205
+ maxLogit = rawLogit;
1206
+ }
1207
+ }
1208
+ } catch (err) {
1209
+ _iterator22.e(err);
1210
+ } finally {
1211
+ _iterator22.f();
1212
+ }
1213
+ if (!Number.isFinite(maxLogit)) {
1214
+ continue;
1215
+ }
1216
+ var _iterator23 = _createForOfIteratorHelper(_group3),
1217
+ _step23;
1218
+ try {
1219
+ for (_iterator23.s(); !(_step23 = _iterator23.n()).done;) {
1220
+ var _candidate3$canonical, _candidate3$canonical2, _candidate3$canonical3;
1221
+ var _candidate3 = _step23.value;
1222
+ var _tokenId = (_candidate3$canonical = _candidate3.canonicalTokenIds) === null || _candidate3$canonical === void 0 ? void 0 : _candidate3$canonical[0];
1223
+ var _rawLogit = _tokenId === undefined ? undefined : boundary.rawLogits[_tokenId];
1224
+ if (_rawLogit === undefined || !Number.isFinite(_rawLogit)) {
1225
+ continue;
1226
+ }
1227
+ var tokenCount = (_candidate3$canonical2 = (_candidate3$canonical3 = _candidate3.canonicalTokenIds) === null || _candidate3$canonical3 === void 0 ? void 0 : _candidate3$canonical3.length) !== null && _candidate3$canonical2 !== void 0 ? _candidate3$canonical2 : 0;
1228
+ if (_tokenId !== undefined && tokenCount === 1) {
1229
+ var totalLogProbability = (0, _canonicalLmScoring.logSoftmaxAt)(boundary.rawLogits, _tokenId);
1230
+ if (!Number.isFinite(totalLogProbability)) {
1231
+ continue;
1232
+ }
1233
+ canonicalEvidence.set(_candidate3.word, {
1234
+ contextKey: _contextKey,
1235
+ meanTokenLogProbability: totalLogProbability,
1236
+ score: 0,
1237
+ separatorKind: 'whitespace',
1238
+ source: 'canonical-full-surface',
1239
+ totalLogProbability: totalLogProbability,
1240
+ totalSurfaceCharCount: _candidate3.word.length,
1241
+ totalTokenCount: 1,
1242
+ verifiedCharCount: _candidate3.word.length,
1243
+ verifiedTokenCount: 1
1244
+ });
1245
+ continue;
1246
+ }
1247
+ canonicalEvidence.set(_candidate3.word, {
1248
+ contextKey: _contextKey,
1249
+ rawLogit: _rawLogit,
1250
+ score: 0,
1251
+ separatorKind: 'whitespace',
1252
+ source: 'canonical-first-token',
1253
+ totalSurfaceCharCount: _candidate3.word.length,
1254
+ totalTokenCount: tokenCount,
1255
+ verifiedCharCount: 0,
1256
+ verifiedTokenCount: 1
1257
+ });
1258
+ }
1259
+ } catch (err) {
1260
+ _iterator23.e(err);
1261
+ } finally {
1262
+ _iterator23.f();
1263
+ }
1264
+ }
1265
+ } catch (err) {
1266
+ _iterator15.e(err);
1267
+ } finally {
1268
+ _iterator15.f();
1269
+ }
1270
+ var _rankCandidates = (0, _scoringPipeline.rankCandidates)(scoringCandidates, contextVector, function (w) {
1271
+ return getWordVector(w);
1272
+ }, prefixLmLogits, {
1273
+ word: wordTrie.maxTenantFreq,
1274
+ bigram: maxBigramFreq,
1275
+ phrase: maxPhraseFreq
1276
+ }, previousWord, function (surface) {
1277
+ var _canonicalEvidence$ge;
1278
+ return (_canonicalEvidence$ge = canonicalEvidence.get(surface)) !== null && _canonicalEvidence$ge !== void 0 ? _canonicalEvidence$ge : null;
1279
+ }),
1280
+ ranked = _rankCandidates.candidates,
1281
+ grammarMeta = _rankCandidates.grammarMeta,
1282
+ pipelineDebug = _rankCandidates.pipelineDebug;
1283
+ var progressiveEligible = ranked.filter(function (candidate) {
1284
+ var runtime = runtimeBySurface.get(candidate.word);
1285
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined;
1286
+ });
1287
+ if (canonicalScoringSupported && progressiveEligible.length > 0) {
1288
+ var byContext = new Map();
1289
+ var _iterator16 = _createForOfIteratorHelper(progressiveEligible),
1290
+ _step16;
1291
+ try {
1292
+ for (_iterator16.s(); !(_step16 = _iterator16.n()).done;) {
1293
+ var _byContext$get;
1294
+ var candidate = _step16.value;
1295
+ var runtime = runtimeBySurface.get(candidate.word);
1296
+ if (!runtime || runtime.canonicalTokenIds === null) {
1297
+ continue;
1298
+ }
1299
+ var group = (_byContext$get = byContext.get(runtime.contextKey)) !== null && _byContext$get !== void 0 ? _byContext$get : [];
1300
+ group.push({
1301
+ runtime: runtime,
1302
+ rankHint: candidate.stage1Score
1303
+ });
1304
+ byContext.set(runtime.contextKey, group);
1305
+ }
1306
+ } catch (err) {
1307
+ _iterator16.e(err);
1308
+ } finally {
1309
+ _iterator16.f();
1310
+ }
1311
+ var _iterator17 = _createForOfIteratorHelper(byContext),
1312
+ _step17;
1313
+ try {
1314
+ for (_iterator17.s(); !(_step17 = _iterator17.n()).done;) {
1315
+ var _step17$value = (0, _slicedToArray2.default)(_step17.value, 2),
1316
+ contextKey = _step17$value[0],
1317
+ _group = _step17$value[1];
1318
+ (0, _slowLaneClient.requestProgressiveSurfaceScores)({
1319
+ familyKey: familyKey,
1320
+ contextKey: contextKey,
1321
+ prompt: _group[0].runtime.contextBeforeSurface,
1322
+ candidates: _group.map(function (_ref6) {
1323
+ var _runtime$canonicalTok;
1324
+ var runtime = _ref6.runtime,
1325
+ rankHint = _ref6.rankHint;
1326
+ return {
1327
+ surface: runtime.word,
1328
+ tokenIds: (_runtime$canonicalTok = runtime.canonicalTokenIds) !== null && _runtime$canonicalTok !== void 0 ? _runtime$canonicalTok : [],
1329
+ rankHint: rankHint
1330
+ };
1331
+ })
1332
+ });
1333
+ }
1334
+ } catch (err) {
1335
+ _iterator17.e(err);
1336
+ } finally {
1337
+ _iterator17.f();
1338
+ }
1339
+ }
1340
+
1341
+ // The one/two-character passes exist only to hide model latency. They may
1342
+ // prime and expand token groups above, but cannot commit UI.
1343
+ var displayPrefixReady = currentWord.length >= DISPLAY_MIN_PREFIX_LENGTH;
1344
+
1345
+ // ── Arbitration: full-surface evidence → plausibility → winner margin ──
1346
+ var canonicalLmSupported = canonicalScoringSupported;
1347
+ var hasExactEvidence = function hasExactEvidence(candidate) {
1348
+ var _candidate$lmEvidence;
1349
+ return ((_candidate$lmEvidence = candidate.lmEvidence) === null || _candidate$lmEvidence === void 0 ? void 0 : _candidate$lmEvidence.source) === 'canonical-full-surface' && candidate.lmEvidence.meanTokenLogProbability !== undefined && Number.isFinite(candidate.lmEvidence.meanTokenLogProbability);
1350
+ };
1351
+
1352
+ /**
1353
+ * The evidence this candidate is judged on: the summed log-probability of
1354
+ * the tokens that have been read, their per-token mean, and whether they
1355
+ * cover the whole surface.
1356
+ *
1357
+ * `total` is the quantity the posterior is built from — it is a sequence
1358
+ * log-likelihood, so it is comparable across surfaces of different lengths
1359
+ * in a way the mean is not. `mean` is retained for reporting and for the
1360
+ * oracle arms, which compare aggregation rules against each other.
1361
+ *
1362
+ * Every token read costs a round trip, so a surface longer than
1363
+ * `REQUIRED_VERIFIED_TOKENS` is judged on its leading tokens and its tail is
1364
+ * left unread. Callers compensate with a wider winner margin rather than
1365
+ * treating the two kinds of evidence as equivalent.
1366
+ *
1367
+ * This is the depth-unaware read, and it is what the pool below is built
1368
+ * from. Arbitration goes through `judgedEvidenceFor`, which adds the one
1369
+ * requirement that can only be stated once the pool is known: a surface
1370
+ * cannot be judged against members it has not yet been read level with.
1371
+ */
1372
+ var readJudgedEvidence = function readJudgedEvidence(candidate) {
1373
+ var evidence = candidate.lmEvidence;
1374
+ var exactMean = evidence === null || evidence === void 0 ? void 0 : evidence.meanTokenLogProbability;
1375
+ if (hasExactEvidence(candidate) && evidence && exactMean !== undefined) {
1376
+ var _evidence$totalLogPro;
1377
+ var _total = (_evidence$totalLogPro = evidence.totalLogProbability) !== null && _evidence$totalLogPro !== void 0 ? _evidence$totalLogPro : exactMean * evidence.totalTokenCount;
1378
+ return Number.isFinite(_total) ? {
1379
+ complete: true,
1380
+ mean: exactMean,
1381
+ total: _total
1382
+ } : null;
1383
+ }
1384
+ var runtime = runtimeBySurface.get(candidate.word);
1385
+ if (!runtime || runtime.canonicalTokenIds === null) {
1386
+ return null;
1387
+ }
1388
+ var progressive = (0, _slowLaneClient.getProgressiveSurfaceEvidence)(runtime.contextKey, candidate.word);
1389
+ if (!progressive || progressive.scoredTokenCount < REQUIRED_VERIFIED_TOKENS) {
1390
+ return null;
1391
+ }
1392
+ var total = progressive.totalLogProbability;
1393
+ var mean = total / progressive.scoredTokenCount;
1394
+ return Number.isFinite(mean) && Number.isFinite(total) ? {
1395
+ complete: false,
1396
+ mean: mean,
1397
+ total: total
1398
+ } : null;
1399
+ };
1400
+
1401
+ // Normalise each context's shortlist into a posterior. `total` is a sequence
1402
+ // log-likelihood, so `exp(total - logSumExp(totals))` is the share of the
1403
+ // shortlist's probability mass the model puts on that surface.
1404
+ //
1405
+ // Unknown future token log-probabilities cannot exceed zero, so a partially
1406
+ // read candidate's total is an upper bound on its eventual total, and its
1407
+ // posterior an upper bound on its eventual posterior. That is what lets the
1408
+ // plausibility floor and the display gate be the same threshold — one
1409
+ // applied to the bound, one to the verified value — instead of two numbers
1410
+ // that have to be kept consistent by hand.
1411
+ //
1412
+ // Candidates with no evidence at all are left out of the sum rather than
1413
+ // defaulted. Any default would have to stand in for a log-probability, and a
1414
+ // value low enough not to distort the normaliser is indistinguishable from
1415
+ // omitting the candidate.
1416
+ var optimisticTotalByWord = new Map();
1417
+ var contextTotals = new Map();
1418
+ var contextSurfaces = new Map();
1419
+ // Counted before the total is looked up, so this is every candidate the
1420
+ // context could have scored rather than every candidate it did. The gap
1421
+ // between the two is what separates a context holding one match from a
1422
+ // context whose rivals have not come back yet.
1423
+ var contextRequestedCount = new Map();
1424
+ var bestTotalByContext = new Map();
1425
+ var bestCandidateByContext = new Map();
1426
+ var _iterator18 = _createForOfIteratorHelper(ranked),
1427
+ _step18;
1428
+ try {
1429
+ for (_iterator18.s(); !(_step18 = _iterator18.n()).done;) {
1430
+ var _contextRequestedCoun2, _judged$total;
1431
+ var _candidate4 = _step18.value;
1432
+ var _runtime = runtimeBySurface.get(_candidate4.word);
1433
+ if (!_runtime || _runtime.canonicalTokenIds === null) {
1434
+ continue;
1435
+ }
1436
+ contextRequestedCount.set(_runtime.contextKey, ((_contextRequestedCoun2 = contextRequestedCount.get(_runtime.contextKey)) !== null && _contextRequestedCoun2 !== void 0 ? _contextRequestedCoun2 : 0) + 1);
1437
+ var judged = readJudgedEvidence(_candidate4);
1438
+ var progressive = (0, _slowLaneClient.getProgressiveSurfaceEvidence)(_runtime.contextKey, _candidate4.word);
1439
+ // Once a candidate has been read far enough to judge, its verified total
1440
+ // replaces the bound.
1441
+ var optimisticTotal = (_judged$total = judged === null || judged === void 0 ? void 0 : judged.total) !== null && _judged$total !== void 0 ? _judged$total : progressive === null || progressive === void 0 ? void 0 : progressive.totalLogProbability;
1442
+ if (optimisticTotal === undefined || !Number.isFinite(optimisticTotal)) {
1443
+ continue;
1444
+ }
1445
+ optimisticTotalByWord.set(_candidate4.word, optimisticTotal);
1446
+ var totals = contextTotals.get(_runtime.contextKey);
1447
+ var surfaces = contextSurfaces.get(_runtime.contextKey);
1448
+ if (totals && surfaces) {
1449
+ totals.push(optimisticTotal);
1450
+ surfaces.push(_candidate4.word);
1451
+ } else {
1452
+ contextTotals.set(_runtime.contextKey, [optimisticTotal]);
1453
+ contextSurfaces.set(_runtime.contextKey, [_candidate4.word]);
1454
+ }
1455
+ var best = bestTotalByContext.get(_runtime.contextKey);
1456
+ if (best === undefined || optimisticTotal > best) {
1457
+ bestTotalByContext.set(_runtime.contextKey, optimisticTotal);
1458
+ bestCandidateByContext.set(_runtime.contextKey, _candidate4);
1459
+ }
1460
+ }
1461
+
1462
+ /**
1463
+ * Whether a shorter member of the same pool is this surface's word-boundary
1464
+ * prefix — `happy path` where `happy` is also present.
1465
+ *
1466
+ * Only a multi-word surface can extend another, and a pool is a set, so this
1467
+ * costs one lookup per space rather than a comparison against every member.
1468
+ */
1469
+ } catch (err) {
1470
+ _iterator18.e(err);
1471
+ } finally {
1472
+ _iterator18.f();
1473
+ }
1474
+ var extendsAPoolMember = function extendsAPoolMember(surface, pool) {
1475
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1476
+ if (pool.has(surface.slice(0, space))) {
1477
+ return true;
1478
+ }
1479
+ }
1480
+ return false;
1481
+ };
1482
+
1483
+ // A sequence total only falls as more of the sequence is read, so a partial
1484
+ // sum over `k` tokens sits above the same surface's total over `m > k`. An
1485
+ // extension read less far than the member it continues therefore carries a
1486
+ // total above that member's — and because the chain rule narrows the
1487
+ // denominator to the pool's minimal members, `exp(total - normaliser)` comes
1488
+ // out above 1 for a surface whose own continuation has not been looked at.
1489
+ // It needs a three-token first word to happen, which 2.6% of served bigrams
1490
+ // have, and it inflates both the posterior the gate reads and the
1491
+ // `confidenceScore` every other candidate's margin is measured against.
1492
+ //
1493
+ // A surface's canonical tokenization begins with the tokenization of its own
1494
+ // leading words — the producer encodes `" " + surface` in one call and the
1495
+ // ByteLevel pre-tokenizer splits on word boundaries, so no merge crosses a
1496
+ // space — which is what makes both corrections below exact rather than
1497
+ // approximate.
1498
+ //
1499
+ // The total is capped at what it continues: `P(happy path)` cannot exceed
1500
+ // `P(happy)`. Shortest-first, so a three-word phrase is capped against a
1501
+ // two-word prefix that has already been capped itself.
1502
+ //
1503
+ // And the surface is held unjudgeable until it has been read level with what
1504
+ // it continues, because the cap alone would leave it looking exactly as
1505
+ // certain as its prefix while saying nothing about its own tail.
1506
+ var underReadExtensions = new Set();
1507
+ var _iterator19 = _createForOfIteratorHelper(contextSurfaces),
1508
+ _step19;
1509
+ try {
1510
+ for (_iterator19.s(); !(_step19 = _iterator19.n()).done;) {
1511
+ var _step19$value = (0, _slicedToArray2.default)(_step19.value, 2),
1512
+ _contextKey2 = _step19$value[0],
1513
+ _surfaces = _step19$value[1];
1514
+ var pool = new Set(_surfaces);
1515
+ var _iterator24 = _createForOfIteratorHelper((0, _toConsumableArray2.default)(_surfaces).sort(function (a, b) {
1516
+ return a.length - b.length;
1517
+ })),
1518
+ _step24;
1519
+ try {
1520
+ for (_iterator24.s(); !(_step24 = _iterator24.n()).done;) {
1521
+ var _getProgressiveSurfac, _getProgressiveSurfac2, _runtimeBySurface$get8, _runtimeBySurface$get9;
1522
+ var surface = _step24.value;
1523
+ var own = optimisticTotalByWord.get(surface);
1524
+ if (own === undefined) {
1525
+ continue;
1526
+ }
1527
+ var cap = Infinity;
1528
+ var continuedDepth = 0;
1529
+ for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
1530
+ var _runtimeBySurface$get6, _runtimeBySurface$get7;
1531
+ var _prefix = surface.slice(0, space);
1532
+ if (!pool.has(_prefix)) {
1533
+ continue;
1534
+ }
1535
+ var prefixTotal = optimisticTotalByWord.get(_prefix);
1536
+ if (prefixTotal !== undefined && prefixTotal < cap) {
1537
+ cap = prefixTotal;
1538
+ }
1539
+ var prefixDepth = (_runtimeBySurface$get6 = (_runtimeBySurface$get7 = runtimeBySurface.get(_prefix)) === null || _runtimeBySurface$get7 === void 0 || (_runtimeBySurface$get7 = _runtimeBySurface$get7.canonicalTokenIds) === null || _runtimeBySurface$get7 === void 0 ? void 0 : _runtimeBySurface$get7.length) !== null && _runtimeBySurface$get6 !== void 0 ? _runtimeBySurface$get6 : 0;
1540
+ if (prefixDepth > continuedDepth) {
1541
+ continuedDepth = prefixDepth;
1542
+ }
1543
+ }
1544
+ if (own > cap) {
1545
+ optimisticTotalByWord.set(surface, cap);
1546
+ }
1547
+ var scoredDepth = (_getProgressiveSurfac = (_getProgressiveSurfac2 = (0, _slowLaneClient.getProgressiveSurfaceEvidence)(_contextKey2, surface)) === null || _getProgressiveSurfac2 === void 0 ? void 0 : _getProgressiveSurfac2.scoredTokenCount) !== null && _getProgressiveSurfac !== void 0 ? _getProgressiveSurfac : 0;
1548
+ var surfaceDepth = (_runtimeBySurface$get8 = (_runtimeBySurface$get9 = runtimeBySurface.get(surface)) === null || _runtimeBySurface$get9 === void 0 || (_runtimeBySurface$get9 = _runtimeBySurface$get9.canonicalTokenIds) === null || _runtimeBySurface$get9 === void 0 ? void 0 : _runtimeBySurface$get9.length) !== null && _runtimeBySurface$get8 !== void 0 ? _runtimeBySurface$get8 : 0;
1549
+ if (scoredDepth < continuedDepth && scoredDepth < surfaceDepth) {
1550
+ underReadExtensions.add(surface);
1551
+ }
1552
+ }
1553
+ } catch (err) {
1554
+ _iterator24.e(err);
1555
+ } finally {
1556
+ _iterator24.f();
1557
+ }
1558
+ }
1559
+
1560
+ /**
1561
+ * The evidence arbitration judges a candidate on.
1562
+ *
1563
+ * Wraps the depth-unaware read with the pool-relative requirement: a surface
1564
+ * that continues another pool member is not judgeable until it has been read
1565
+ * level with it, so the two totals being compared are taken at comparable
1566
+ * depths.
1567
+ */
1568
+ } catch (err) {
1569
+ _iterator19.e(err);
1570
+ } finally {
1571
+ _iterator19.f();
1572
+ }
1573
+ var judgedEvidenceFor = function judgedEvidenceFor(candidate) {
1574
+ return underReadExtensions.has(candidate.word) ? null : readJudgedEvidence(candidate);
1575
+ };
1576
+ var hasJudgeableEvidence = function hasJudgeableEvidence(candidate) {
1577
+ return judgedEvidenceFor(candidate) !== null;
1578
+ };
1579
+
1580
+ // Normalise over the pool's minimal members only, and score everyone —
1581
+ // extensions included — against that.
1582
+ //
1583
+ // A sequence total is its prefix's total plus a log-probability that cannot
1584
+ // be positive, so summing a surface and its own extension double-counts:
1585
+ // `exp(total)` for `happy` already contains every continuation of `happy`,
1586
+ // `happy path` among them. Dividing one pool's mass between the two capped
1587
+ // the shorter form at half however certain it was, and the longer form below
1588
+ // that — both under gates neither could then clear, for a reason that was
1589
+ // arithmetic rather than evidence.
1590
+ //
1591
+ // Excluding extensions from the denominator is the chain rule: the minimal
1592
+ // members partition the mass, and an extension's share of that same
1593
+ // denominator is `P(prefix) * P(continuation | prefix)`, which is exactly
1594
+ // what a multi-word suggestion should be held to. Extensions stay in
1595
+ // `contextTotals`, so how contested a context is still counts every scored
1596
+ // candidate.
1597
+ var logSumExpByContext = new Map();
1598
+ var _iterator20 = _createForOfIteratorHelper(contextSurfaces),
1599
+ _step20;
1600
+ try {
1601
+ for (_iterator20.s(); !(_step20 = _iterator20.n()).done;) {
1602
+ var _step20$value = (0, _slicedToArray2.default)(_step20.value, 2),
1603
+ _contextKey3 = _step20$value[0],
1604
+ _surfaces2 = _step20$value[1];
1605
+ var _pool = new Set(_surfaces2);
1606
+ var minimalTotals = [];
1607
+ var _iterator25 = _createForOfIteratorHelper(_surfaces2),
1608
+ _step25;
1609
+ try {
1610
+ for (_iterator25.s(); !(_step25 = _iterator25.n()).done;) {
1611
+ var _surface = _step25.value;
1612
+ var total = optimisticTotalByWord.get(_surface);
1613
+ if (total !== undefined && !extendsAPoolMember(_surface, _pool)) {
1614
+ minimalTotals.push(total);
1615
+ }
1616
+ }
1617
+ // An extension is strictly longer than what it extends, so the shortest
1618
+ // member of any non-empty pool is always minimal and this is never empty.
1619
+ } catch (err) {
1620
+ _iterator25.e(err);
1621
+ } finally {
1622
+ _iterator25.f();
1623
+ }
1624
+ logSumExpByContext.set(_contextKey3, (0, _canonicalLmScoring.logSumExp)(minimalTotals));
1625
+ }
1626
+
1627
+ /**
1628
+ * A candidate's share of its context shortlist's probability mass.
1629
+ *
1630
+ * Pass an optimistic (partially read) total for an upper bound on the
1631
+ * eventual posterior, or a verified total for the posterior itself.
1632
+ */
1633
+ } catch (err) {
1634
+ _iterator20.e(err);
1635
+ } finally {
1636
+ _iterator20.f();
1637
+ }
1638
+ var posteriorFor = function posteriorFor(candidate, total) {
1639
+ var runtime = runtimeBySurface.get(candidate.word);
1640
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
1641
+ if (total === undefined || normalizer === undefined || !Number.isFinite(normalizer)) {
1642
+ return 0;
1643
+ }
1644
+ return Math.exp(total - normalizer);
1645
+ };
1646
+ var optimisticPosterior = function optimisticPosterior(candidate) {
1647
+ return posteriorFor(candidate, optimisticTotalByWord.get(candidate.word));
1648
+ };
1649
+ var judgedPosterior = function judgedPosterior(candidate) {
1650
+ var _judgedEvidenceFor;
1651
+ return posteriorFor(candidate, (_judgedEvidenceFor = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor === void 0 ? void 0 : _judgedEvidenceFor.total);
1652
+ };
1653
+
1654
+ // A candidate stays in the race for as long as its optimistic posterior
1655
+ // could still clear the gate it will be held to. The bound only falls as
1656
+ // more tokens are read, so nothing dropped here could have gone on to win.
1657
+ var couldClearGate = function couldClearGate(candidate) {
1658
+ return optimisticPosterior(candidate) >= MIN_LM_POSTERIOR[candidate.termType];
1659
+ };
1660
+ var passesLmFloor = function passesLmFloor(candidate) {
1661
+ return judgedEvidenceFor(candidate) !== null && couldClearGate(candidate);
1662
+ };
1663
+ var hasRequiredEvidence = function hasRequiredEvidence(candidate) {
1664
+ var _candidate$lmEvidence2;
1665
+ if (canonicalLmSupported) {
1666
+ return passesLmFloor(candidate);
1667
+ }
1668
+ return candidate.termType === 'word' && ((_candidate$lmEvidence2 = candidate.lmEvidence) === null || _candidate$lmEvidence2 === void 0 ? void 0 : _candidate$lmEvidence2.source) === 'network-logit';
1669
+ };
1670
+
1671
+ // ── Stabilization guards (QI-2) ──────────────────────────────────────────
1672
+ // Whole-surface repetition: drop a candidate whose full surface already equals
1673
+ // the run of tokens immediately before the cursor — accepting it would
1674
+ // duplicate what was just typed (`end to end` → `end to end to end`).
1675
+ var beforeTokens = tokenize(trimmed);
1676
+ var duplicatesPreceding = function duplicatesPreceding(c) {
1677
+ var surfaceTokens = tokenize(c.word);
1678
+ var n = surfaceTokens.length;
1679
+ if (n === 0 || beforeTokens.length < n) {
1680
+ return false;
1681
+ }
1682
+ for (var i = 0; i < n; i++) {
1683
+ if (beforeTokens[beforeTokens.length - n + i] !== surfaceTokens[i]) {
1684
+ return false;
1685
+ }
1686
+ }
1687
+ return true;
1688
+ };
1689
+ // Post-accept cooldown: don't re-offer the surface the user just accepted while
1690
+ // its cooldown is still active (see cooldown advance near t0).
1691
+ var isCooledDown = function isCooledDown(c) {
1692
+ return cooledSurface !== null && c.word.toLowerCase() === cooledSurface;
1693
+ };
1694
+ var isStabilized = function isStabilized(candidate) {
1695
+ return !duplicatesPreceding(candidate) && !isCooledDown(candidate);
1696
+ };
1697
+ // A candidate the user has all but finished typing is not worth a ghost, and
1698
+ // a candidate we would never show is not worth waiting for either.
1699
+ var suggestionLengthFor = function suggestionLengthFor(candidate) {
1700
+ var _prefixLenByWord$get;
1701
+ return candidate.word.length - ((_prefixLenByWord$get = prefixLenByWord.get(candidate.word)) !== null && _prefixLenByWord$get !== void 0 ? _prefixLenByWord$get : currentWord.length);
1702
+ };
1703
+ var isWorthShowing = function isWorthShowing(candidate) {
1704
+ return suggestionLengthFor(candidate) >= MIN_SUGGESTION_LENGTH;
1705
+ };
1706
+ var evidenceBacked = ranked.filter(function (candidate) {
1707
+ return hasRequiredEvidence(candidate) && isStabilized(candidate);
1708
+ });
1709
+ var confidenceScore = function confidenceScore(candidate) {
1710
+ if (!canonicalLmSupported) {
1711
+ return candidate.finalScore;
1712
+ }
1713
+ var runtime = runtimeBySurface.get(candidate.word);
1714
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) === null || !optimisticTotalByWord.has(candidate.word)) {
1715
+ return candidate.stage1Score;
1716
+ }
1717
+ return _scoringPipeline.STAGE1_WEIGHT * candidate.stage1Score + _scoringPipeline.STAGE2_WEIGHT * optimisticPosterior(candidate);
1718
+ };
1719
+ var exactConfidenceScore = function exactConfidenceScore(candidate) {
1720
+ if (!canonicalLmSupported || judgedEvidenceFor(candidate) === null) {
1721
+ return candidate.finalScore;
1722
+ }
1723
+ return _scoringPipeline.STAGE1_WEIGHT * candidate.stage1Score + _scoringPipeline.STAGE2_WEIGHT * judgedPosterior(candidate);
1724
+ };
1725
+
1726
+ /**
1727
+ * How many candidates in this one's context were scored at all.
1728
+ *
1729
+ * Read off pool membership, which counts every candidate that contributed a
1730
+ * total, rather than off the normaliser, which the chain rule narrows to the
1731
+ * pool's minimal members.
1732
+ */
1733
+ var scoredPoolSize = function scoredPoolSize(candidate) {
1734
+ var _runtimeBySurface$get, _contextTotals$get$le, _contextTotals$get;
1735
+ var contextKey = (_runtimeBySurface$get = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get === void 0 ? void 0 : _runtimeBySurface$get.contextKey;
1736
+ return contextKey === undefined ? 0 : (_contextTotals$get$le = (_contextTotals$get = contextTotals.get(contextKey)) === null || _contextTotals$get === void 0 ? void 0 : _contextTotals$get.length) !== null && _contextTotals$get$le !== void 0 ? _contextTotals$get$le : 0;
1737
+ };
1738
+
1739
+ /**
1740
+ * How many candidates the context held before any of them were scored.
1741
+ *
1742
+ * A pool of one means two opposite things. Either the vocabulary offered a
1743
+ * single continuation of what was typed, which is the least contested case
1744
+ * there is; or rivals were offered and have not been read yet, which is the
1745
+ * least informed. Only the second deserves refusing, and the two are
1746
+ * distinguishable exactly here.
1747
+ */
1748
+ var requestedPoolSize = function requestedPoolSize(candidate) {
1749
+ var _runtimeBySurface$get2, _contextRequestedCoun;
1750
+ var contextKey = (_runtimeBySurface$get2 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get2 === void 0 ? void 0 : _runtimeBySurface$get2.contextKey;
1751
+ return contextKey === undefined ? 0 : (_contextRequestedCoun = contextRequestedCount.get(contextKey)) !== null && _contextRequestedCoun !== void 0 ? _contextRequestedCoun : 0;
1752
+ };
1753
+
1754
+ // A suppressed short completion still counts as competition below, so losing
1755
+ // to one abstains rather than promoting the runner-up in its place.
1756
+ var gateCleared = evidenceBacked.filter(isWorthShowing).map(function (candidate) {
1757
+ return {
1758
+ candidate: candidate,
1759
+ posterior: judgedPosterior(candidate),
1760
+ score: exactConfidenceScore(candidate)
1761
+ };
1762
+ })
1763
+ // The gate is the model's own confidence in the surface; the blended score
1764
+ // only orders what has already cleared it, so a strong corpus prior can no
1765
+ // longer carry a surface the model is unsure of onto the screen.
1766
+ .filter(function (_ref7) {
1767
+ var candidate = _ref7.candidate,
1768
+ posterior = _ref7.posterior;
1769
+ return posterior >= MIN_LM_POSTERIOR[candidate.termType];
1770
+ });
1771
+ /**
1772
+ * Whether the model finds the surface plausible on its own terms.
1773
+ *
1774
+ * The gate above asks which candidate won its pool; this asks whether
1775
+ * winning it meant anything. The two come apart exactly where a pool is
1776
+ * thin — a shortlist of one hands its only member 1.0 by construction — and
1777
+ * a relative test can never see through that however it is tuned.
1778
+ *
1779
+ * Waived without canonical scoring, where there is no judged mean to test
1780
+ * and the network-logit path would otherwise refuse everything.
1781
+ */
1782
+ var isPlausibleSurface = function isPlausibleSurface(candidate) {
1783
+ var _judgedEvidenceFor2;
1784
+ if (!canonicalLmSupported) {
1785
+ return true;
1786
+ }
1787
+ var mean = (_judgedEvidenceFor2 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor2 === void 0 ? void 0 : _judgedEvidenceFor2.mean;
1788
+ return mean !== undefined && mean >= MIN_MEAN_TOKEN_LOG_PROBABILITY;
1789
+ };
1790
+ // Kept as its own stage rather than folded into the gate so the three
1791
+ // populations stay separable in the abstain cascade: cleared its pool and was
1792
+ // implausible, was plausible and had nothing to clear the pool against, or
1793
+ // cleared both. A candidate refused here stays in `ranked` and so still
1794
+ // counts as competition below — promoting the runner-up in place of an
1795
+ // implausible leader would show something worse, not something better.
1796
+ var plausible = gateCleared.filter(function (_ref8) {
1797
+ var candidate = _ref8.candidate;
1798
+ return isPlausibleSurface(candidate);
1799
+ });
1800
+ // Applied after the gate rather than folded into it, so the two populations
1801
+ // stay separable: a surface refused here cleared its threshold and was
1802
+ // refused for having had nothing to clear it against.
1803
+ var eligible = plausible.filter(function (_ref9) {
1804
+ var candidate = _ref9.candidate;
1805
+ return !canonicalLmSupported || scoredPoolSize(candidate) >= MIN_SCORED_POOL_SIZE || requestedPoolSize(candidate) <= 1;
1806
+ }).sort(function (a, b) {
1807
+ return b.score - a.score;
1808
+ });
1809
+ var selected = (_eligible$ = eligible[0]) === null || _eligible$ === void 0 ? void 0 : _eligible$.candidate;
1810
+ var selectedDisplayScore = (_eligible$0$score = (_eligible$2 = eligible[0]) === null || _eligible$2 === void 0 ? void 0 : _eligible$2.score) !== null && _eligible$0$score !== void 0 ? _eligible$0$score : 0;
1811
+ var selectedEvidence = selected && hasRequiredEvidence(selected) ? selected.lmEvidence : null;
1812
+ /**
1813
+ * Whether `longer` is `shorter` continued past a word boundary, in the same
1814
+ * normaliser.
1815
+ *
1816
+ * Shared by the two places that have to agree on what an extension is: the
1817
+ * chain rule, which stops dividing a pool's mass between a sequence and its
1818
+ * own prefix, and the winner margin below, which stops treating one as the
1819
+ * other's rival. They were the same category error and are now the same test.
1820
+ */
1821
+ var extendsInSamePool = function extendsInSamePool(shorter, longer) {
1822
+ var _runtimeBySurface$get3, _runtimeBySurface$get4;
1823
+ if (longer.word === shorter.word || !optimisticTotalByWord.has(longer.word)) {
1824
+ return false;
1825
+ }
1826
+ var pool = (_runtimeBySurface$get3 = runtimeBySurface.get(shorter.word)) === null || _runtimeBySurface$get3 === void 0 ? void 0 : _runtimeBySurface$get3.contextKey;
1827
+ if (pool === undefined || ((_runtimeBySurface$get4 = runtimeBySurface.get(longer.word)) === null || _runtimeBySurface$get4 === void 0 ? void 0 : _runtimeBySurface$get4.contextKey) !== pool) {
1828
+ return false;
1829
+ }
1830
+ return longer.word.length > shorter.word.length && longer.word.startsWith(shorter.word) && longer.word[shorter.word.length] === ' ';
1831
+ };
1832
+ // A candidate that extends the selection is not competing with it. Whichever
1833
+ // continuation the user meant, the shorter surface is a correct ghost — it is
1834
+ // the prefix of both — so the margin has nothing to arbitrate and charging the
1835
+ // selection for its own extension abstains on a case that could not be wrong.
1836
+ // The chain rule already removed this from the denominator; leaving it in the
1837
+ // margin meant a nested leader cleared its gate and then lost to the candidate
1838
+ // that had just been freed from dividing its mass. It penalised bigrams and
1839
+ // phrases specifically, because a nested pool is where they mostly appear.
1840
+ var runnerUpScore = selected ? Math.max.apply(Math, (0, _toConsumableArray2.default)(ranked.filter(function (candidate) {
1841
+ return candidate.word !== selected.word && isStabilized(candidate) && !extendsInSamePool(selected, candidate);
1842
+ }).map(confidenceScore)).concat([0])) : 0;
1843
+ var winnerMargin = selected ? selectedDisplayScore - runnerUpScore : 0;
1844
+ var hasMissingArtifactCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1845
+ var runtime = runtimeBySurface.get(candidate.word);
1846
+ return (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) === 'whitespace' && runtime.canonicalTokenIds === null;
1847
+ });
1848
+ // A candidate with canonical tokens but no evidence at all cannot join the
1849
+ // posterior: it has no likelihood to contribute, and any stand-in value would
1850
+ // distort the normaliser for every other candidate in its context. Absent
1851
+ // evidence is not evidence of absence though — the context's boundary may
1852
+ // simply not be primed yet — so it holds the ghost back instead of being
1853
+ // silently ignored. In a primed context every candidate has at least
1854
+ // first-token evidence, so this only fires while a context is still cold.
1855
+ var hasUnscoredCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
1856
+ var runtime = runtimeBySurface.get(candidate.word);
1857
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !optimisticTotalByWord.has(candidate.word) && isStabilized(candidate) && isWorthShowing(candidate);
1858
+ });
1859
+ // A rival only holds the ghost back while it is still too unread to judge.
1860
+ // Under full-surface evidence that meant any long surface blocked everything
1861
+ // until every one of its tokens had been paid for, which abstained far more
1862
+ // often than it ever changed the winner. A rival now becomes judgeable after
1863
+ // `REQUIRED_VERIFIED_TOKENS`, so this settles within the budget.
1864
+ var hasUnresolvedPotential = canonicalLmSupported && ranked.some(function (candidate) {
1865
+ var runtime = runtimeBySurface.get(candidate.word);
1866
+ return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !hasJudgeableEvidence(candidate) && couldClearGate(candidate) && isStabilized(candidate) && isWorthShowing(candidate);
1867
+ });
1868
+ var requiredWinnerMargin = _scoringPipeline.MIN_WINNER_MARGIN + (selected && ((_judgedEvidenceFor3 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor3 === void 0 ? void 0 : _judgedEvidenceFor3.complete) === false ? PARTIAL_EVIDENCE_MARGIN_PREMIUM : 0);
1869
+ // Everything the display decision needs except the veto itself. Splitting it
1870
+ // out is what lets the veto be priced: the value here at the moment the veto
1871
+ // fires is exactly the ghost the veto cost us.
1872
+ var clearsMarginBeforeColdVeto = selected !== undefined && !hasMissingArtifactCompetitor && winnerMargin >= requiredWinnerMargin;
1873
+ var clearsWinnerMargin = clearsMarginBeforeColdVeto && !hasUnscoredCompetitor;
1874
+ var suggestion = displayPrefixReady && selected && clearsWinnerMargin ? selected.word.slice((_prefixLenByWord$get2 = prefixLenByWord.get(selected.word)) !== null && _prefixLenByWord$get2 !== void 0 ? _prefixLenByWord$get2 : currentWord.length) : null;
1875
+
1876
+ // Which constraint bound. Computed on every evaluation rather than only under
1877
+ // debug, because it is counted for the session as well as printed, and
1878
+ // deriving the debug header from it keeps the two from disagreeing.
1879
+ var resolveAbstainReason = function resolveAbstainReason() {
1880
+ if (suggestion !== null && suggestion.length > 0) {
1881
+ return null;
1882
+ }
1883
+ if (!displayPrefixReady) {
1884
+ return 'prefetch';
1885
+ }
1886
+ if (ranked.length === 0) {
1887
+ return 'no-candidate';
1888
+ }
1889
+ if (hasMissingArtifactCompetitor) {
1890
+ return 'missing-artifact';
1891
+ }
1892
+ if (hasUnscoredCompetitor) {
1893
+ return 'cold-competitor';
1894
+ }
1895
+ if (hasUnresolvedPotential && !clearsWinnerMargin) {
1896
+ return 'unresolved-rival';
1897
+ }
1898
+ if (eligible.length > 0) {
1899
+ return selected && !clearsWinnerMargin ? 'winner-margin' : 'empty-completion';
1900
+ }
1901
+ if (plausible.length > 0) {
1902
+ // Cleared its gate, was plausible on its own terms, and was refused for
1903
+ // the shortlist it cleared that gate against holding nobody else.
1904
+ return 'lone-candidate';
1905
+ }
1906
+ if (gateCleared.length > 0) {
1907
+ // Held the largest share of its shortlist and still was not a surface
1908
+ // the model found plausible — which is only possible because a share is
1909
+ // relative and this is not.
1910
+ return 'implausible-surface';
1911
+ }
1912
+ if (evidenceBacked.length > 0 && evidenceBacked.every(function (candidate) {
1913
+ return !isWorthShowing(candidate);
1914
+ })) {
1915
+ return 'short-completion';
1916
+ }
1917
+ // The floor and the gate are one threshold, applied to the bound and to
1918
+ // the verified value, so a candidate short of it is dropped before
1919
+ // `evidenceBacked` is built. Asking whether anything was judgeable
1920
+ // separates a gate that rejected read candidates from a context where
1921
+ // nothing was read at all — the first is a threshold to calibrate, the
1922
+ // second is scheduling.
1923
+ return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
1924
+ };
1925
+ var abstainReason = resolveAbstainReason();
1926
+
1927
+ // The leader is the best-supported candidate, not the selected one: an
1928
+ // evaluation that showed nothing is exactly the one whose posterior needs
1929
+ // recording, and it has no selection to report.
1930
+ var posteriorLeader = ranked.reduce(function (best, candidate) {
1931
+ var posterior = optimisticPosterior(candidate);
1932
+ return posterior > 0 && (best === null || posterior > best.posterior) ? {
1933
+ candidate: candidate,
1934
+ posterior: posterior
1935
+ } : best;
1936
+ }, null);
1937
+ /**
1938
+ * The longer surface in the leader's normaliser that extends the given
1939
+ * candidate, if the pool holds one.
1940
+ *
1941
+ * `logSumExp` divides one pool's mass between a sequence and its own prefix,
1942
+ * and the longer total is the shorter one plus a log-probability that cannot
1943
+ * be positive — so the longer form holds at most half, or at most a third
1944
+ * where both shorter forms are present. Both sit under the gate it is held
1945
+ * to, so it cannot display however certain its continuation is.
1946
+ *
1947
+ * Asked from the shorter form because that is the one that leads: the
1948
+ * extending candidate is strictly lower in the same pool and so can never be
1949
+ * the leader.
1950
+ */
1951
+ var poolExtensionOf = function poolExtensionOf(shorter) {
1952
+ return ranked.find(function (candidate) {
1953
+ return extendsInSamePool(shorter, candidate);
1954
+ });
1955
+ };
1956
+ if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
1957
+ var _posteriorLeader$cand, _posteriorLeader$post, _selectedEvidence$mea, _selectedEvidence$mea2, _acceptCooldown$predi, _acceptCooldown;
1958
+ var verbose = (0, _debugMode.isAutocompleteDebugVerbose)();
1959
+ var latencyMs = performance.now() - t0;
1960
+ var tokens = tokenize(trimmed);
1961
+ var contextWords = tokens.slice(-CONTEXT_WORDS);
1962
+
1963
+ // Decision drives the (collapsed) group header. Abstains carry the reason
1964
+ // the session counters recorded, plus the numbers behind it, so it is clear
1965
+ // *why* nothing showed without expanding the group.
1966
+ var abstainDetail = {
1967
+ 'below-posterior-gate': "posterior below ".concat(MIN_LM_POSTERIOR[((_posteriorLeader$cand = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.candidate) !== null && _posteriorLeader$cand !== void 0 ? _posteriorLeader$cand : ranked[0]).termType], " (best ").concat(((_posteriorLeader$post = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.posterior) !== null && _posteriorLeader$post !== void 0 ? _posteriorLeader$post : 0).toFixed(2), ")"),
1968
+ 'cold-competitor': 'competitor has no LM evidence yet',
1969
+ 'empty-completion': 'empty completion',
1970
+ 'implausible-surface': "mean per-token log-probability below ".concat(MIN_MEAN_TOKEN_LOG_PROBABILITY, " (best ").concat(gateCleared.map(function (_ref0) {
1971
+ var _judgedEvidenceFor$me, _judgedEvidenceFor4;
1972
+ var candidate = _ref0.candidate;
1973
+ return (_judgedEvidenceFor$me = (_judgedEvidenceFor4 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor4 === void 0 ? void 0 : _judgedEvidenceFor4.mean) !== null && _judgedEvidenceFor$me !== void 0 ? _judgedEvidenceFor$me : -Infinity;
1974
+ }).reduce(function (best, mean) {
1975
+ return Math.max(best, mean);
1976
+ }, -Infinity).toFixed(2), ")"),
1977
+ 'lone-candidate': "cleared its gate against fewer than ".concat(MIN_SCORED_POOL_SIZE, " scored candidates, rivals unread"),
1978
+ 'missing-artifact': 'canonical artifact coverage missing',
1979
+ 'no-candidate': 'nothing cleared scoring',
1980
+ 'no-evidence': 'full-surface evidence absent',
1981
+ prefetch: "prefetch: ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " chars"),
1982
+ 'short-completion': "completion shorter than ".concat(MIN_SUGGESTION_LENGTH, " chars"),
1983
+ 'unresolved-rival': 'expanding plausible token-prefix groups',
1984
+ 'winner-margin': "winner margin ".concat(winnerMargin.toFixed(2), " < ").concat(requiredWinnerMargin.toFixed(2))
1985
+ };
1986
+ var decision = abstainReason === null ? "\u2728 \"".concat(suggestion !== null && suggestion !== void 0 ? suggestion : '', "\"") : abstainReason === 'prefetch' ? "\u23F3 ".concat(abstainDetail.prefetch) : "\u2014 abstain: ".concat(abstainDetail[abstainReason]);
1987
+ lastPredictionDebug = {
1988
+ awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
1989
+ textBefore: trimmed,
1990
+ currentWord: currentWord,
1991
+ decision: decision,
1992
+ mode: mode,
1993
+ contextWords: contextWords,
1994
+ topCandidates: ranked.slice(0, 5).map(function (r) {
1995
+ return {
1996
+ word: r.word,
1997
+ finalScore: r.finalScore,
1998
+ semanticScore: r.semanticScore,
1999
+ freqScore: r.freqScore,
2000
+ lmScore: r.lmScore
592
2001
  };
593
2002
  }),
594
2003
  suggestion: suggestion && suggestion.length > 0 ? suggestion : null
595
2004
  };
596
2005
 
597
- // ── Mode label: COLD / WARM(local) / WARM(BE) ───────────────────────
2006
+ // Mode describes the semantic-vector source without implying that a
2007
+ // network backend ran; the snapshot may come from the local MLC embedder.
598
2008
  var slowLaneVec = (0, _slowLaneClient.getStoredContextVector)();
599
2009
  var isUsingSlowLaneVector = slowLaneVec !== null && vectorStore !== null && slowLaneVec.length === vectorStore.dim;
600
- var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(BE)' : 'WARM(local)';
601
- var modeColor = modeLabel === 'WARM(BE)' ? 'color: #ff9800; font-weight: bold;' : modeLabel === 'WARM(local)' ? 'color: #4caf50; font-weight: bold;' : 'color: #9e9e9e; font-weight: bold;';
2010
+ var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(semantic)' : 'WARM(local-avg)';
2011
+ var modeColor = modeLabel === 'WARM(semantic)' ? _debugMode.CTC_STYLES.good : modeLabel === 'WARM(local-avg)' ? _debugMode.CTC_STYLES.warn : _debugMode.CTC_STYLES.cold;
602
2012
 
603
- // 1. Collapsible group header
2013
+ // Group header: [CTC:<mode>] <decision> · <mode> · <type> · <latency>
2014
+ // arbitrationMode ('v1') tags the line so this deploy's logs diff cleanly
2015
+ // against the baseline deploy's.
604
2016
  // eslint-disable-next-line no-console
605
- console.groupCollapsed("%c[Autocomplete] %c".concat(modeLabel, " %c| \"").concat(currentWord, "\" \u2794 ").concat(suggestionLabel, " | \u23F1 ").concat(latencyMs.toFixed(1), "ms"), 'color: #00b8d9; font-weight: bold;', modeColor, 'color: inherit; font-weight: normal;');
2017
+ console.groupCollapsed("%c[CTC:".concat(ARBITRATION_MODE, "]%c ").concat(decision, " %c\xB7 ").concat(modeLabel).concat(selected ? " \xB7 ".concat(selected.termType) : '', " \xB7 \u23F1 ").concat(latencyMs.toFixed(1), "ms"), _debugMode.CTC_STYLES.brand, _debugMode.CTC_STYLES.body, modeColor);
2018
+ var semanticInput = (0, _slowLaneClient.getStoredContextInput)();
2019
+ var causalPrompts = primeRequests.map(function (request, index) {
2020
+ var tail = request.prompt.slice(-DEBUG_TEXT_TAIL_CHARS);
2021
+ return "#".concat(index + 1, ":").concat(JSON.stringify(tail));
2022
+ });
606
2023
 
607
- // 2. Context window (what local vector averaging sees)
608
- // eslint-disable-next-line no-console
609
- console.log('%cContext Window:', 'color: #888; font-style: italic;', contextWords.length ? contextWords.join(' ') : '(none)');
2024
+ // Keep the exact model inputs separate from normalized lookup tokens.
2025
+ (0, _debugMode.ctcSection)('INPUT', "raw: ".concat(JSON.stringify(trimmed.slice(-DEBUG_TEXT_TAIL_CHARS)), " \xB7 prev: ").concat(previousWord ? JSON.stringify(previousWord) : '—'));
2026
+ (0, _debugMode.ctcSection)('LOOKUP', "normalized(last ".concat(CONTEXT_WORDS, "): ").concat(contextWords.length ? JSON.stringify(contextWords.join(' ')) : '(none)'));
2027
+ (0, _debugMode.ctcSection)('SEMANTIC', "snapshot: ".concat(semanticInput ? JSON.stringify(semanticInput.slice(-DEBUG_TEXT_TAIL_CHARS)) : '(unavailable)'));
2028
+ (0, _debugMode.ctcSection)('CAUSAL', "prompt".concat(causalPrompts.length === 1 ? '' : 's', ": ").concat(causalPrompts.join(' · ') || '(not ready)'));
610
2029
 
611
- // 3. Slow-lane status
612
- var vectorStatus = isUsingSlowLaneVector ? "\u2705 BE semantic vector (dim=".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, ")") : vectorStore ? '⚠️ Local vector average (slow-lane not yet returned)' : '❌ No vectors (cold)';
613
- var logitsStatus = lmLogits && Object.keys(lmLogits).length > 0 ? "\u2705 LM logits active (".concat(Object.keys(lmLogits).length, " tokens)") : '⏳ No LM logits (slow-lane pending or failed)';
614
- // eslint-disable-next-line no-console
615
- console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
2030
+ // SIGNALS — semantic state plus context-keyed canonical LM coverage.
2031
+ var logitCount = lmLogits ? Object.keys(lmLogits).length : 0;
2032
+ var semanticStatus = isUsingSlowLaneVector ? "semantic \u2705 snapshot(".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, "d)") : vectorStore ? 'semantic ⚠️ local-avg' : 'semantic ❌ cold';
2033
+ var tierACount = ranked.filter(function (candidate) {
2034
+ var _candidate$lmEvidence3;
2035
+ return ((_candidate$lmEvidence3 = candidate.lmEvidence) === null || _candidate$lmEvidence3 === void 0 ? void 0 : _candidate$lmEvidence3.source) === 'canonical-first-token';
2036
+ }).length;
2037
+ var exactCount = ranked.filter(function (candidate) {
2038
+ var _candidate$lmEvidence4;
2039
+ return ((_candidate$lmEvidence4 = candidate.lmEvidence) === null || _candidate$lmEvidence4 === void 0 ? void 0 : _candidate$lmEvidence4.source) === 'canonical-full-surface';
2040
+ }).length;
2041
+ var boundaryReady = primeRequests.filter(function (request) {
2042
+ return (0, _slowLaneClient.getBoundaryLmState)(request.contextKey) !== null;
2043
+ }).length;
2044
+ var artifactEligible = canonicalMatched.filter(function (candidate) {
2045
+ return candidate.canonicalTokenIds !== null;
2046
+ }).length;
2047
+ var documentStartFallbacks = canonicalMatched.filter(function (candidate) {
2048
+ return candidate.separatorKind === 'document-start';
2049
+ }).length;
2050
+ var punctuationFallbacks = canonicalMatched.filter(function (candidate) {
2051
+ return candidate.separatorKind === 'non-space';
2052
+ }).length;
2053
+ var logitsStatus = tierACount + exactCount > 0 ? "canonical \u2705 first-token:".concat(tierACount, " exact:").concat(exactCount) : logitCount > 0 ? "network logits \u2705 ".concat(logitCount) : 'canonical ⏳/absent';
2054
+ var formula = tierACount + exactCount + logitCount > 0 ? "Stage1\xD7".concat(_scoringPipeline.STAGE1_WEIGHT, "+LM\xD7").concat(_scoringPipeline.STAGE2_WEIGHT) : 'Stage1-only';
2055
+ (0, _debugMode.ctcSection)('SIGNALS', "".concat(semanticStatus, " \xB7 ").concat(logitsStatus, " \xB7 ").concat(formula));
2056
+ (0, _debugMode.ctcSection)('CANONICAL', "eligible ".concat(artifactEligible, "/").concat(canonicalMatched.length, " \xB7 separator fallback doc:").concat(documentStartFallbacks, " punct:").concat(punctuationFallbacks, " \xB7 contexts ").concat(boundaryReady, "/").concat(primeRequests.length, " ready (\u2264").concat(PHRASE_MAX_WORDS, ") \xB7 family=").concat(familyKey.slice(0, 72) || 'none'));
2057
+ if (verbose && logitCount > 0 && lmLogits) {
2058
+ var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
2059
+ return b[1] - a[1];
2060
+ }).slice(0, 5).map(function (_ref1) {
2061
+ var _ref10 = (0, _slicedToArray2.default)(_ref1, 2),
2062
+ word = _ref10[0],
2063
+ score = _ref10[1];
2064
+ return "".concat(word, ":").concat(score.toFixed(3));
2065
+ }).join(', ');
2066
+ (0, _debugMode.ctcSection)(' rawLM', "\uD83E\uDDE0 ".concat(rawLmTop));
2067
+ }
616
2068
 
617
- // 4. Scoring formula active this prediction
618
- var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? "Stage1(\xD7".concat(_scoringPipeline.STAGE1_WEIGHT, ") + LM(\xD7").concat(_scoringPipeline.STAGE2_WEIGHT, ")") : 'Stage1 only (no LM logits)';
619
- // eslint-disable-next-line no-console
620
- console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
2069
+ // GENERATE — how many candidates matched, split by term type
2070
+ var genByType = {
2071
+ word: 0,
2072
+ bigram: 0,
2073
+ phrase: 0
2074
+ };
2075
+ var _iterator21 = _createForOfIteratorHelper(canonicalMatched),
2076
+ _step21;
2077
+ try {
2078
+ for (_iterator21.s(); !(_step21 = _iterator21.n()).done;) {
2079
+ var m = _step21.value;
2080
+ genByType[m.node.termType] += 1;
2081
+ }
2082
+ } catch (err) {
2083
+ _iterator21.e(err);
2084
+ } finally {
2085
+ _iterator21.f();
2086
+ }
2087
+ (0, _debugMode.ctcSection)('GENERATE', "matched ".concat(canonicalMatched.length, " \u2192 word:").concat(genByType.word, " bigram:").concat(genByType.bigram, " phrase:").concat(genByType.phrase).concat(canonicalLmSupported ? ' · display needs exact surface evidence' : ''));
621
2088
 
622
- // 5. Grammar filter result (collected inside rankCandidates, logged here)
2089
+ // SCORE — Stage-1 + grammar funnel and canonical evidence depth.
2090
+ var lmCoverage = ranked.slice(0, 10).filter(function (r) {
2091
+ return r.lmScore > 0;
2092
+ }).length;
2093
+ (0, _debugMode.ctcSection)('SCORE', "in ".concat(pipelineDebug.initial, " \u2192 stage1(<").concat(_scoringPipeline.MIN_STAGE1_SCORE, ") \u2212").concat(pipelineDebug.stage1Rejected.length, " \u2192 grammar \u2212").concat(pipelineDebug.grammarRejected.length, " \u2192 final ").concat(pipelineDebug.final, " \xB7 LM cov ").concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " \xB7 first-token ").concat(tierACount, " \xB7 exact ").concat(exactCount));
623
2094
  if (grammarMeta) {
624
- // eslint-disable-next-line no-console
625
- console.log("%c[Grammar] \"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] \u2192 ").concat(grammarMeta.before, " candidates \u2192 ").concat(grammarMeta.after, " after filter"), 'color: #4caf50; font-weight: bold;');
626
- if (grammarMeta.dropped.length > 0) {
627
- // eslint-disable-next-line no-console
628
- console.log("%c\uD83D\uDEAB Dropped: ".concat(grammarMeta.dropped.join(', ')), 'color: #f44336; font-style: italic;');
629
- }
2095
+ (0, _debugMode.ctcSection)(' grammar', "\"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] ").concat(grammarMeta.before, "\u2192").concat(grammarMeta.after).concat(verbose && grammarMeta.dropped.length > 0 ? " \xB7 dropped: ".concat(grammarMeta.dropped.join(', ')) : ''));
630
2096
  }
2097
+ var gateReasonFor = function gateReasonFor(candidate) {
2098
+ var runtime = runtimeBySurface.get(candidate.word);
2099
+ if ((runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace') {
2100
+ var _runtime$separatorKin;
2101
+ return "separator:".concat((_runtime$separatorKin = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin !== void 0 ? _runtime$separatorKin : 'unknown');
2102
+ }
2103
+ if (runtime.canonicalTokenIds === null) {
2104
+ return 'missing-artifact';
2105
+ }
2106
+ if (!hasJudgeableEvidence(candidate)) {
2107
+ return "awaiting-tokens:".concat(REQUIRED_VERIFIED_TOKENS);
2108
+ }
2109
+ if (!passesLmFloor(candidate)) {
2110
+ return "bound:".concat(optimisticPosterior(candidate).toFixed(2), "<").concat(MIN_LM_POSTERIOR[candidate.termType]);
2111
+ }
2112
+ if (duplicatesPreceding(candidate)) {
2113
+ return 'repetition';
2114
+ }
2115
+ if (isCooledDown(candidate)) {
2116
+ return 'cooldown';
2117
+ }
2118
+ if (!isWorthShowing(candidate)) {
2119
+ return "suffix:".concat(suggestionLengthFor(candidate), "<").concat(MIN_SUGGESTION_LENGTH);
2120
+ }
2121
+ var floor = MIN_LM_POSTERIOR[candidate.termType];
2122
+ var posterior = judgedPosterior(candidate);
2123
+ if (posterior < floor) {
2124
+ return "posterior:".concat(posterior.toFixed(2), "<").concat(floor);
2125
+ }
2126
+ if (!isPlausibleSurface(candidate)) {
2127
+ var _judgedEvidenceFor$me2, _judgedEvidenceFor5;
2128
+ return "implausible:".concat(((_judgedEvidenceFor$me2 = (_judgedEvidenceFor5 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor5 === void 0 ? void 0 : _judgedEvidenceFor5.mean) !== null && _judgedEvidenceFor$me2 !== void 0 ? _judgedEvidenceFor$me2 : NaN).toFixed(2), "<").concat(MIN_MEAN_TOKEN_LOG_PROBABILITY);
2129
+ }
2130
+ return 'eligible';
2131
+ };
2132
+ var gateFunnel = function gateFunnel(termType) {
2133
+ var typeRanked = ranked.filter(function (candidate) {
2134
+ return candidate.termType === termType;
2135
+ });
2136
+ var exact = typeRanked.filter(hasJudgeableEvidence);
2137
+ var absolute = exact.filter(passesLmFloor);
2138
+ var stabilized = absolute.filter(isStabilized);
2139
+ var longEnough = stabilized.filter(isWorthShowing);
2140
+ var floorPassed = longEnough.filter(function (candidate) {
2141
+ return judgedPosterior(candidate) >= MIN_LM_POSTERIOR[termType];
2142
+ });
2143
+ var plausiblePassed = floorPassed.filter(isPlausibleSurface);
2144
+ return "".concat(termType, " m:").concat(genByType[termType], " r:").concat(typeRanked.length, " exact:").concat(exact.length, " abs:").concat(absolute.length, " stable:").concat(stabilized.length, " suffix:").concat(longEnough.length, " floor:").concat(floorPassed.length, " plausible:").concat(plausiblePassed.length, " eligible:").concat(eligible.filter(function (_ref11) {
2145
+ var candidate = _ref11.candidate;
2146
+ return candidate.termType === termType;
2147
+ }).length);
2148
+ };
2149
+ (0, _debugMode.ctcSection)('GATES', "".concat(gateFunnel('word'), " | ").concat(gateFunnel('bigram'), " | ").concat(gateFunnel('phrase')));
631
2150
 
632
- // 6. Pipeline funnel
633
- // eslint-disable-next-line no-console
634
- console.log("%c[Pipeline Funnel] %c\uD83D\uDCE5 In: ".concat(pipelineDebug.initial, " | \u274C Stage 1 (< ").concat(_scoringPipeline.MIN_STAGE1_SCORE, "): -").concat(pipelineDebug.stage1Rejected.length, " | \u274C Grammar: -").concat(pipelineDebug.grammarRejected.length, " | \u2705 Final: ").concat(pipelineDebug.final), 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
2151
+ // One line per context: how many candidates share the normaliser, and how
2152
+ // much of the mass the leader holds. A leader well under its threshold
2153
+ // means the context is contested, which is the abstention we want.
2154
+ var contextLeaders = Array.from(bestCandidateByContext.entries()).slice(0, PHRASE_MAX_WORDS).map(function (_ref12) {
2155
+ var _contextTotals$get$le2, _contextTotals$get2;
2156
+ var _ref13 = (0, _slicedToArray2.default)(_ref12, 2),
2157
+ contextKey = _ref13[0],
2158
+ leader = _ref13[1];
2159
+ var shortlistSize = (_contextTotals$get$le2 = (_contextTotals$get2 = contextTotals.get(contextKey)) === null || _contextTotals$get2 === void 0 ? void 0 : _contextTotals$get2.length) !== null && _contextTotals$get$le2 !== void 0 ? _contextTotals$get$le2 : 0;
2160
+ var evidenceKind = hasExactEvidence(leader) ? 'exact' : 'upper';
2161
+ return "".concat(contextKey.slice(0, 32), " \u2192 n=").concat(shortlistSize, " ").concat(leader.termType, ":\"").concat(leader.word, "\" ").concat(evidenceKind, " p=").concat(optimisticPosterior(leader).toFixed(3), "/").concat(MIN_LM_POSTERIOR[leader.termType]);
2162
+ }).join(' | ');
2163
+ (0, _debugMode.ctcSection)('NORMALIZE', contextLeaders.length > 0 ? contextLeaders : 'no scored candidate in any context');
2164
+ (0, _debugMode.ctcSection)('READINESS', "prefix ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " \xB7 boundary ").concat(boundaryReady, "/").concat(primeRequests.length, " \xB7 exact word:").concat(ranked.filter(function (candidate) {
2165
+ return candidate.termType === 'word' && hasExactEvidence(candidate);
2166
+ }).length, " bigram:").concat(ranked.filter(function (candidate) {
2167
+ return candidate.termType === 'bigram' && hasExactEvidence(candidate);
2168
+ }).length, " phrase:").concat(ranked.filter(function (candidate) {
2169
+ return candidate.termType === 'phrase' && hasExactEvidence(candidate);
2170
+ }).length, " \xB7 ").concat(suggestion ? 'display-ready' : decision));
635
2171
 
636
- // 7. Candidate table
637
- if (ranked.length > 0) {
638
- var lmCoverage = ranked.slice(0, 10).filter(function (r) {
639
- return r.lmScore > 0;
640
- }).length;
641
- // eslint-disable-next-line no-console
642
- console.log("%cLM coverage: ".concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " candidates had real logit scores"), 'color: #888; font-style: italic;');
2172
+ // ARBITRATE — expose every condition that can authorize or suppress ghost.
2173
+ var topOfType = function topOfType(t) {
2174
+ return ranked.find(function (c) {
2175
+ return c.termType === t;
2176
+ });
2177
+ };
2178
+ var fmtType = function fmtType(t) {
2179
+ var _top$lmEvidence$sourc, _top$lmEvidence;
2180
+ var top = topOfType(t);
2181
+ if (!top) {
2182
+ return "".concat(t, ":\u2014");
2183
+ }
2184
+ var floor = MIN_LM_POSTERIOR[t];
2185
+ var bound = optimisticPosterior(top);
2186
+ var evidence = hasRequiredEvidence(top) ? 'TierB✅' : "".concat((_top$lmEvidence$sourc = (_top$lmEvidence = top.lmEvidence) === null || _top$lmEvidence === void 0 ? void 0 : _top$lmEvidence.source) !== null && _top$lmEvidence$sourc !== void 0 ? _top$lmEvidence$sourc : 'absent', "\u274C");
2187
+ return "".concat(t, ":\"").concat(top.word, "\" p\u2264").concat(bound.toFixed(3)).concat(bound >= floor ? '≥' : '<').concat(floor, " fs\u2264").concat(confidenceScore(top).toFixed(2), " ").concat(evidence);
2188
+ };
2189
+ (0, _debugMode.ctcSection)('ARBITRATE', "".concat(selected ? "\u25B8 ".concat(selected.termType, " \"").concat(selected.word, "\" fs=").concat(selectedDisplayScore.toFixed(2), " mean=").concat((_selectedEvidence$mea = selectedEvidence === null || selectedEvidence === void 0 || (_selectedEvidence$mea2 = selectedEvidence.meanTokenLogProbability) === null || _selectedEvidence$mea2 === void 0 ? void 0 : _selectedEvidence$mea2.toFixed(2)) !== null && _selectedEvidence$mea !== void 0 ? _selectedEvidence$mea : 'network', " margin=").concat(winnerMargin.toFixed(2)).concat(clearsWinnerMargin ? '✅' : '❌') : '▸ (none)', " \xB7 prefix ").concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH).concat(displayPrefixReady ? '✅' : '⏳', " \xB7 grouped pending ").concat(hasUnresolvedPotential ? 'yes' : 'no', " | ").concat(fmtType('word'), " ").concat(fmtType('bigram'), " ").concat(fmtType('phrase')));
2190
+
2191
+ // STABILIZE — post-accept cooldown + whole-surface repetition (QI-2)
2192
+ var repetitionBlocked = ranked.filter(function (c) {
2193
+ return hasRequiredEvidence(c) && duplicatesPreceding(c);
2194
+ }).map(function (c) {
2195
+ return c.word;
2196
+ });
2197
+ var cooldownLabel = cooledSurface ? "cooldown \"".concat(cooledSurface, "\" (").concat((_acceptCooldown$predi = (_acceptCooldown = acceptCooldown) === null || _acceptCooldown === void 0 ? void 0 : _acceptCooldown.predictionsSince) !== null && _acceptCooldown$predi !== void 0 ? _acceptCooldown$predi : 0, "/").concat(COOLDOWN_KEYSTROKES, " keys \xB7 ").concat(COOLDOWN_MS, "ms)") : 'cooldown —';
2198
+ (0, _debugMode.ctcSection)('STABILIZE', "".concat(cooldownLabel, " \xB7 repetition-blocked: ").concat(repetitionBlocked.length > 0 ? repetitionBlocked.join(', ') : '—'));
2199
+
2200
+ // CANDIDATES — full scored table (verbose only)
2201
+ if (verbose && ranked.length > 0) {
643
2202
  var tableData = ranked.slice(0, 10).map(function (r) {
2203
+ var _r$lmEvidence, _contextTotals$get$le3, _contextTotals$get3, _r$lmEvidence$source, _r$lmEvidence2, _runtime$separatorKin2, _runtime$canonicalTok2, _runtime$canonicalTok3, _runtime$contextKey, _r$lmEvidence3, _r$lmEvidence4;
644
2204
  var rawLogit = 'Not in Payload';
645
- if (prefixLmLogits) {
2205
+ if (((_r$lmEvidence = r.lmEvidence) === null || _r$lmEvidence === void 0 ? void 0 : _r$lmEvidence.rawLogit) !== undefined) {
2206
+ rawLogit = Number(r.lmEvidence.rawLogit.toFixed(5));
2207
+ } else if (prefixLmLogits) {
646
2208
  var val = prefixLmLogits[r.word.toLowerCase()];
647
2209
  if (val !== undefined) {
648
2210
  rawLogit = Number(val.toFixed(5));
@@ -651,9 +2213,19 @@ var predict = exports.predict = function predict(textBefore) {
651
2213
  var original = scoringCandidates.find(function (sc) {
652
2214
  return sc.word === r.word;
653
2215
  });
2216
+ var runtime = runtimeBySurface.get(r.word);
2217
+ var progress = runtime ? (0, _slowLaneClient.getProgressiveSurfaceEvidence)(runtime.contextKey, r.word) : null;
2218
+ var optimisticTotal = optimisticTotalByWord.get(r.word);
2219
+ var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
2220
+ var shortlistSize = runtime ? (_contextTotals$get$le3 = (_contextTotals$get3 = contextTotals.get(runtime.contextKey)) === null || _contextTotals$get3 === void 0 ? void 0 : _contextTotals$get3.length) !== null && _contextTotals$get$le3 !== void 0 ? _contextTotals$get$le3 : 0 : 0;
2221
+ var contextLeader = runtime ? bestCandidateByContext.get(runtime.contextKey) : undefined;
654
2222
  var source = 'Unknown';
655
2223
  if (original) {
656
- if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
2224
+ if (original.termType === 'bigram') {
2225
+ source = '🔗 L2 (Bigram)';
2226
+ } else if (original.termType === 'phrase') {
2227
+ source = '🧩 L2 (Phrase)';
2228
+ } else if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
657
2229
  source = '🌍 L3 (Generic)';
658
2230
  } else if (original.sessionFreq > 0 && original.tenantFreq === 0) {
659
2231
  source = '👤 L1 (Session Only)';
@@ -661,13 +2233,40 @@ var predict = exports.predict = function predict(textBefore) {
661
2233
  source = '🏢 L2 (Domain)';
662
2234
  }
663
2235
  }
2236
+ var floor = MIN_LM_POSTERIOR[r.termType];
664
2237
  return {
665
2238
  Candidate: r.word,
666
2239
  Source: source,
2240
+ Type: r.termType,
667
2241
  'Final Score': Number(r.finalScore.toFixed(4)),
2242
+ 'Bounded Score': Number(confidenceScore(r).toFixed(4)),
2243
+ 'Exact Gate Score': hasExactEvidence(r) ? Number(exactConfidenceScore(r).toFixed(4)) : '—',
2244
+ Floor: floor,
2245
+ 'Posterior Floor': optimisticPosterior(r) >= floor ? '✅' : '❌',
2246
+ 'LM Floor': hasExactEvidence(r) ? passesLmFloor(r) ? '✅' : '❌' : '⏳',
2247
+ 'Gate Reason': gateReasonFor(r),
2248
+ 'Display Eligible': hasRequiredEvidence(r) && isStabilized(r) && judgedPosterior(r) >= floor ? '✅' : '❌',
668
2249
  Semantics: Number(r.semanticScore.toFixed(4)),
669
2250
  Freq: Number(r.freqScore.toFixed(4)),
670
2251
  'LM Score': Number(r.lmScore.toFixed(4)),
2252
+ 'LM Evidence': (_r$lmEvidence$source = (_r$lmEvidence2 = r.lmEvidence) === null || _r$lmEvidence2 === void 0 ? void 0 : _r$lmEvidence2.source) !== null && _r$lmEvidence$source !== void 0 ? _r$lmEvidence$source : 'absent',
2253
+ 'Verified Tokens': r.lmEvidence ? "".concat(r.lmEvidence.verifiedTokenCount, "/").concat(r.lmEvidence.totalTokenCount) : '0/0',
2254
+ 'Progressive Tokens': progress ? "".concat(progress.scoredTokenCount, "/").concat(progress.totalTokenCount) : '0/0',
2255
+ 'LM Mean Upper': progress !== null ? Number(progress.meanTokenLogProbabilityUpperBound.toFixed(5)) : '—',
2256
+ 'Optimistic Total': optimisticTotal !== undefined ? Number(optimisticTotal.toFixed(5)) : '—',
2257
+ 'Context logSumExp': normalizer !== undefined && Number.isFinite(normalizer) ? Number(normalizer.toFixed(5)) : '—',
2258
+ 'Shortlist Size': shortlistSize,
2259
+ 'Posterior (bound)': Number(optimisticPosterior(r).toFixed(5)),
2260
+ 'Posterior (judged)': hasJudgeableEvidence(r) ? Number(judgedPosterior(r).toFixed(5)) : '—',
2261
+ 'Context Leader': contextLeader ? "".concat(contextLeader.termType, ":").concat(contextLeader.word) : '—',
2262
+ 'Verified Chars': r.lmEvidence ? "".concat(r.lmEvidence.verifiedCharCount, "/").concat(r.lmEvidence.totalSurfaceCharCount) : "0/".concat(r.word.length),
2263
+ Separator: (_runtime$separatorKin2 = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin2 !== void 0 ? _runtime$separatorKin2 : 'unknown',
2264
+ 'Canonical Eligibility': (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace' ? 'incompatible-separator' : runtime.canonicalTokenIds === null ? 'missing-artifact' : 'eligible',
2265
+ 'Artifact Tokens': (_runtime$canonicalTok2 = runtime === null || runtime === void 0 || (_runtime$canonicalTok3 = runtime.canonicalTokenIds) === null || _runtime$canonicalTok3 === void 0 ? void 0 : _runtime$canonicalTok3.join(',')) !== null && _runtime$canonicalTok2 !== void 0 ? _runtime$canonicalTok2 : 'absent',
2266
+ 'Context Key': (_runtime$contextKey = runtime === null || runtime === void 0 ? void 0 : runtime.contextKey) !== null && _runtime$contextKey !== void 0 ? _runtime$contextKey : '—',
2267
+ 'Exact Total': ((_r$lmEvidence3 = r.lmEvidence) === null || _r$lmEvidence3 === void 0 ? void 0 : _r$lmEvidence3.totalLogProbability) !== undefined ? Number(r.lmEvidence.totalLogProbability.toFixed(5)) : '—',
2268
+ 'Exact Mean': ((_r$lmEvidence4 = r.lmEvidence) === null || _r$lmEvidence4 === void 0 ? void 0 : _r$lmEvidence4.meanTokenLogProbability) !== undefined ? Number(r.lmEvidence.meanTokenLogProbability.toFixed(5)) : '—',
2269
+ Assoc: typeof (original === null || original === void 0 ? void 0 : original.association) === 'number' ? Number(original.association.toFixed(4)) : '—',
671
2270
  'Raw Logit': rawLogit,
672
2271
  'Session Freq': (original === null || original === void 0 ? void 0 : original.sessionFreq) || 0
673
2272
  };
@@ -675,23 +2274,120 @@ var predict = exports.predict = function predict(textBefore) {
675
2274
 
676
2275
  // eslint-disable-next-line no-console
677
2276
  console.table(tableData);
678
- } else {
679
- // eslint-disable-next-line no-console
680
- console.log('No candidates found.');
681
2277
  }
682
2278
 
683
2279
  // eslint-disable-next-line no-console
684
2280
  console.groupEnd();
685
2281
  }
686
- return suggestion && suggestion.length > 0 ? suggestion : null;
2282
+ var selectedContextKey = selected ? (_runtimeBySurface$get5 = runtimeBySurface.get(selected.word)) === null || _runtimeBySurface$get5 === void 0 ? void 0 : _runtimeBySurface$get5.contextKey : undefined;
2283
+ return suggestion && suggestion.length > 0 && selected && selectedEvidence ? {
2284
+ text: suggestion,
2285
+ surface: selected.word,
2286
+ termType: selected.termType,
2287
+ posterior: judgedPosterior(selected),
2288
+ meanTokenLogProbability: (_judgedEvidenceFor$me3 = (_judgedEvidenceFor6 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor6 === void 0 ? void 0 : _judgedEvidenceFor6.mean) !== null && _judgedEvidenceFor$me3 !== void 0 ? _judgedEvidenceFor$me3 : NaN,
2289
+ poolHeldExtension: poolExtensionOf(selected) !== undefined,
2290
+ shortlistSize: selectedContextKey === undefined ? 1 : (_contextTotals$get$le4 = (_contextTotals$get4 = contextTotals.get(selectedContextKey)) === null || _contextTotals$get4 === void 0 ? void 0 : _contextTotals$get4.length) !== null && _contextTotals$get$le4 !== void 0 ? _contextTotals$get$le4 : 1,
2291
+ rankScore: selectedDisplayScore,
2292
+ evidenceTier: (selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.source) === 'network-logit' ? 'network-logit' : 'canonical-full-surface',
2293
+ evidenceDepth: {
2294
+ verifiedTokens: (_selectedEvidence$ver = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedTokenCount) !== null && _selectedEvidence$ver !== void 0 ? _selectedEvidence$ver : 0,
2295
+ totalTokens: (_selectedEvidence$tot = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.totalTokenCount) !== null && _selectedEvidence$tot !== void 0 ? _selectedEvidence$tot : 0,
2296
+ verifiedChars: (_selectedEvidence$ver2 = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedCharCount) !== null && _selectedEvidence$ver2 !== void 0 ? _selectedEvidence$ver2 : 0,
2297
+ totalChars: selected.word.length
2298
+ },
2299
+ winnerMargin: winnerMargin
2300
+ } : null;
687
2301
  };
688
2302
 
689
2303
  // ─── Data Loading ────────────────────────────────────────────────────────────
690
2304
 
2305
+ var isVocabularyJson = function isVocabularyJson(payload) {
2306
+ if (payload == null || (0, _typeof2.default)(payload) !== 'object') {
2307
+ return false;
2308
+ }
2309
+ var words = payload.words;
2310
+ return words != null && (0, _typeof2.default)(words) === 'object';
2311
+ };
2312
+ var isStringArray = function isStringArray(payload) {
2313
+ return Array.isArray(payload) && payload.every(function (entry) {
2314
+ return typeof entry === 'string';
2315
+ });
2316
+ };
2317
+ var isWordToOffsetMap = function isWordToOffsetMap(value) {
2318
+ return value != null && (0, _typeof2.default)(value) === 'object' && Object.values(value).every(function (entry) {
2319
+ return typeof entry === 'number';
2320
+ });
2321
+ };
2322
+ var isWordIndexPayload = function isWordIndexPayload(payload) {
2323
+ if (payload == null || (0, _typeof2.default)(payload) !== 'object') {
2324
+ return false;
2325
+ }
2326
+ var index = payload.index;
2327
+ return index === undefined ? isWordToOffsetMap(payload) : isWordToOffsetMap(index);
2328
+ };
2329
+
2330
+ /**
2331
+ * The word index has shipped both bare and wrapped in `{ index: ... }`, so
2332
+ * accept either rather than silently building an empty vector store.
2333
+ */
2334
+ var unwrapWordIndex = function unwrapWordIndex(payload) {
2335
+ var wrapped = payload.index;
2336
+ return wrapped !== null && wrapped !== void 0 ? wrapped : payload;
2337
+ };
2338
+
2339
+ /**
2340
+ * Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
2341
+ * confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
2342
+ * + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
2343
+ * are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
2344
+ * a number.
2345
+ */
2346
+
2347
+ /** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
2348
+
2349
+ var isPhraseStats = function isPhraseStats(value) {
2350
+ return value != null && (0, _typeof2.default)(value) === 'object' && typeof value.freq === 'number';
2351
+ };
2352
+ var isTermStatsMap = function isTermStatsMap(value) {
2353
+ if (value == null || (0, _typeof2.default)(value) !== 'object') {
2354
+ return false;
2355
+ }
2356
+ var values = Object.values(value);
2357
+ return values.length > 0 && values.every(isPhraseStats);
2358
+ };
2359
+
2360
+ /**
2361
+ * Normalize a `bigrams.json` / `phrases.json` payload into a flat
2362
+ * `{term: stats}` map. Tolerant of the exact wire shape (not yet finalized by
2363
+ * the producer): accepts either the flat map from spec §9 or a single wrapper
2364
+ * key (`terms`/`bigrams`/`phrases`/`words`). Returns `null` if unrecognized.
2365
+ *
2366
+ * :params:
2367
+ * payload: Raw JSON parsed from the artifact endpoint
2368
+ * :returns:
2369
+ * A `{term: stats}` map, or `null` when the shape is not a term-stats map
2370
+ */
2371
+ var normalizePhraseArtifact = function normalizePhraseArtifact(payload) {
2372
+ if (payload == null || (0, _typeof2.default)(payload) !== 'object') {
2373
+ return null;
2374
+ }
2375
+ if (isTermStatsMap(payload)) {
2376
+ return payload;
2377
+ }
2378
+ var obj = payload;
2379
+ for (var _i4 = 0, _arr = ['terms', 'bigrams', 'phrases', 'words']; _i4 < _arr.length; _i4++) {
2380
+ var key = _arr[_i4];
2381
+ if (isTermStatsMap(obj[key])) {
2382
+ return obj[key];
2383
+ }
2384
+ }
2385
+ return null;
2386
+ };
691
2387
  var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
692
- var _ref6 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(options) {
2388
+ var _ref14 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(options) {
693
2389
  var _options$isLocalLLM;
694
- var isLocalLLM, surface, url, res, buffer, float32, wordIndexPayload, wrappedIndex, wordIndex, nWords, dim, _t, _t2;
2390
+ var isLocalLLM, surface, buffer, float32, wordIndexPayload, wordIndex, nWords, dim, _t;
695
2391
  return _regenerator.default.wrap(function (_context) {
696
2392
  while (1) switch (_context.prev = _context.next) {
697
2393
  case 0:
@@ -711,65 +2407,23 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
711
2407
  } : {}));
712
2408
  _context.prev = 2;
713
2409
  _context.next = 3;
714
- return (0, _artifactsManifest.getArtifactUrl)(_artifactsManifest.ARTIFACT_NAME.WORD_VECTORS);
2410
+ return (0, _artifactLoader.fetchAutocompleteArtifactBinary)(_artifactsManifest.ARTIFACT_NAME.WORD_VECTORS, 'word_vectors_10k');
715
2411
  case 3:
716
- url = _context.sent;
717
- _context.next = 5;
718
- break;
719
- case 4:
720
- _context.prev = 4;
721
- _t = _context["catch"](2);
722
- vectorsLoadStarted = false;
723
- (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
724
- isLocalLLM: isLocalLLM,
725
- errorType: 'resolve_url'
726
- }, surface ? {
727
- surface: surface
728
- } : {}));
729
- // eslint-disable-next-line no-console
730
- console.warn('[text-predictor] Failed to resolve vectors URL:', _t);
731
- return _context.abrupt("return");
732
- case 5:
733
- _context.prev = 5;
734
- _context.next = 6;
735
- return fetch(url);
736
- case 6:
737
- res = _context.sent;
738
- if (res.ok) {
739
- _context.next = 7;
740
- break;
741
- }
742
- vectorsLoadStarted = false;
743
- (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
744
- isLocalLLM: isLocalLLM,
745
- status: res.status,
746
- errorType: 'http_error'
747
- }, surface ? {
748
- surface: surface
749
- } : {}));
750
- // eslint-disable-next-line no-console
751
- console.warn("[text-predictor] Failed to load vectors: ".concat(res.status));
752
- return _context.abrupt("return");
753
- case 7:
754
- _context.next = 8;
755
- return res.arrayBuffer();
756
- case 8:
757
2412
  buffer = _context.sent;
758
- float32 = new Float32Array(buffer); // The word index is published either bare or wrapped as `{ "index": {…} }`
759
- // (the wrapper exists so no real entry, e.g. the word "default", can shadow
760
- // a bundler-synthesised ESM export). Detect the wrapper by value type, not
761
- // by key presence — "index" is itself a plausible vocabulary word, and in a
762
- // bare index every value is a number.
763
- _context.next = 9;
764
- return (0, _artifactsManifest.fetchArtifactJson)(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX);
765
- case 9:
2413
+ float32 = new Float32Array(buffer);
2414
+ _context.next = 4;
2415
+ return (0, _artifactLoader.fetchAutocompleteArtifactJson)(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX, {
2416
+ summarize: function summarize(payload) {
2417
+ return "".concat(Object.keys(unwrapWordIndex(payload)).length, " entries");
2418
+ },
2419
+ validate: isWordIndexPayload
2420
+ });
2421
+ case 4:
766
2422
  wordIndexPayload = _context.sent;
767
- wrappedIndex = wordIndexPayload === null || wordIndexPayload === void 0 ? void 0 : wordIndexPayload.index;
768
- wordIndex = (0, _typeof2.default)(wrappedIndex) === 'object' && wrappedIndex !== null ? wrappedIndex : wordIndexPayload !== null && wordIndexPayload !== void 0 ? wordIndexPayload : {};
2423
+ wordIndex = unwrapWordIndex(wordIndexPayload);
769
2424
  nWords = Object.keys(wordIndex).length;
770
2425
  if (nWords === 0) {
771
- // eslint-disable-next-line no-console
772
- console.warn("[text-predictor] ".concat(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX, " was empty \u2014 semantic scoring will be a no-op."));
2426
+ (0, _debugMode.ctcTag)('init', '⚠️ word_index_10k.json was empty — semantic scoring will be a no-op', _debugMode.CTC_STYLES.warn);
773
2427
  }
774
2428
  dim = float32.length / nWords;
775
2429
  vectorStore = {
@@ -785,19 +2439,12 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
785
2439
  }, surface ? {
786
2440
  surface: surface
787
2441
  } : {}));
788
- if ((0, _debugMode.isAutocompleteDebugEnabled)()) {
789
- // eslint-disable-next-line no-console
790
- console.log('[text-predictor] Vectors loaded:', {
791
- wordCount: nWords,
792
- dim: dim,
793
- sizeBytes: float32.byteLength
794
- });
795
- }
796
- _context.next = 11;
2442
+ (0, _debugMode.ctcTag)('init', "vectors loaded: ".concat(nWords, " words \xB7 dim ").concat(dim, " \xB7 ").concat(float32.byteLength, "B"));
2443
+ _context.next = 6;
797
2444
  break;
798
- case 10:
799
- _context.prev = 10;
800
- _t2 = _context["catch"](5);
2445
+ case 5:
2446
+ _context.prev = 5;
2447
+ _t = _context["catch"](2);
801
2448
  vectorsLoadStarted = false;
802
2449
  (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
803
2450
  isLocalLLM: isLocalLLM,
@@ -805,60 +2452,197 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
805
2452
  }, surface ? {
806
2453
  surface: surface
807
2454
  } : {}));
808
- // eslint-disable-next-line no-console
809
- console.warn('[text-predictor] Failed to load vectors:', _t2);
810
- case 11:
2455
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F failed to load vectors: ".concat(String(_t)), _debugMode.CTC_STYLES.bad);
2456
+ case 6:
811
2457
  case "end":
812
2458
  return _context.stop();
813
2459
  }
814
- }, _callee, null, [[2, 4], [5, 10]]);
2460
+ }, _callee, null, [[2, 5]]);
815
2461
  }));
816
2462
  return function loadVectorsAsync(_x) {
817
- return _ref6.apply(this, arguments);
2463
+ return _ref14.apply(this, arguments);
818
2464
  };
819
2465
  }();
820
2466
  var initVectors = exports.initVectors = function initVectors(store) {
821
2467
  vectorStore = store;
822
2468
  };
2469
+
2470
+ /**
2471
+ * Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
2472
+ * over the TDP-OS gateway and insert them into the phrase trie.
2473
+ *
2474
+ * Each file is fetched independently (`Promise.allSettled`) so a missing or
2475
+ * late-published artifact only skips that term type — word completion and the
2476
+ * other term type are unaffected. Phrase VECTORS are not fetched here: per the
2477
+ * producer's `extend` decision they are appended into the existing
2478
+ * `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
2479
+ * covers them.
2480
+ *
2481
+ * :params:
2482
+ * options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
2483
+ * :returns:
2484
+ * A promise that resolves once both fetches have settled
2485
+ */
2486
+ var loadPhraseArtifacts = exports.loadPhraseArtifacts = /*#__PURE__*/function () {
2487
+ var _ref15 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee3(options) {
2488
+ var _options$isLocalLLM2;
2489
+ var isLocalLLM, loadOne, _yield$Promise$allSet, _yield$Promise$allSet2, bigramsResult, phrasesResult, bigramCount, phraseCount;
2490
+ return _regenerator.default.wrap(function (_context3) {
2491
+ while (1) switch (_context3.prev = _context3.next) {
2492
+ case 0:
2493
+ if (!phrasesLoadStarted) {
2494
+ _context3.next = 1;
2495
+ break;
2496
+ }
2497
+ return _context3.abrupt("return");
2498
+ case 1:
2499
+ phrasesLoadStarted = true;
2500
+ isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2501
+ (0, _ufo.startExp)(_ufo.EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2502
+ isLocalLLM: isLocalLLM
2503
+ });
2504
+ loadOne = /*#__PURE__*/function () {
2505
+ var _ref16 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee2(artifactName, termType, label) {
2506
+ var payload, normalized;
2507
+ return _regenerator.default.wrap(function (_context2) {
2508
+ while (1) switch (_context2.prev = _context2.next) {
2509
+ case 0:
2510
+ _context2.next = 1;
2511
+ return (0, _artifactLoader.fetchAutocompleteArtifactJson)(artifactName, {
2512
+ label: label,
2513
+ validate: function validate(p) {
2514
+ return p != null && (0, _typeof2.default)(p) === 'object';
2515
+ }
2516
+ });
2517
+ case 1:
2518
+ payload = _context2.sent;
2519
+ normalized = normalizePhraseArtifact(payload);
2520
+ if (normalized) {
2521
+ _context2.next = 2;
2522
+ break;
2523
+ }
2524
+ throw new Error("[text-predictor] ".concat(label, " payload was not a recognised term-stats map"));
2525
+ case 2:
2526
+ return _context2.abrupt("return", initPhrases(normalized, termType));
2527
+ case 3:
2528
+ case "end":
2529
+ return _context2.stop();
2530
+ }
2531
+ }, _callee2);
2532
+ }));
2533
+ return function loadOne(_x3, _x4, _x5) {
2534
+ return _ref16.apply(this, arguments);
2535
+ };
2536
+ }();
2537
+ _context3.next = 2;
2538
+ return Promise.allSettled([loadOne(_artifactsManifest.ARTIFACT_NAME.BIGRAMS, 'bigram', 'bigrams'), loadOne(_artifactsManifest.ARTIFACT_NAME.PHRASES, 'phrase', 'phrases')]);
2539
+ case 2:
2540
+ _yield$Promise$allSet = _context3.sent;
2541
+ _yield$Promise$allSet2 = (0, _slicedToArray2.default)(_yield$Promise$allSet, 2);
2542
+ bigramsResult = _yield$Promise$allSet2[0];
2543
+ phrasesResult = _yield$Promise$allSet2[1];
2544
+ bigramCount = bigramsResult.status === 'fulfilled' ? bigramsResult.value : 0;
2545
+ phraseCount = phrasesResult.status === 'fulfilled' ? phrasesResult.value : 0;
2546
+ if (bigramsResult.status === 'rejected') {
2547
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F failed to load bigrams.json: ".concat(String(bigramsResult.reason)), _debugMode.CTC_STYLES.warn);
2548
+ }
2549
+ if (phrasesResult.status === 'rejected') {
2550
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F failed to load phrases.json: ".concat(String(phrasesResult.reason)), _debugMode.CTC_STYLES.warn);
2551
+ }
2552
+ if (bigramsResult.status === 'fulfilled' || phrasesResult.status === 'fulfilled') {
2553
+ phrasesLoaded = true;
2554
+ (0, _ufo.succeedExp)(_ufo.EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2555
+ isLocalLLM: isLocalLLM,
2556
+ bigramCount: bigramCount,
2557
+ phraseCount: phraseCount
2558
+ });
2559
+ (0, _debugMode.ctcTag)('init', "phrase artifacts loaded: ".concat(bigramCount, " bigrams \xB7 ").concat(phraseCount, " phrases"));
2560
+ } else {
2561
+ // Both failed — e.g. artifacts not published to the tenant manifest yet.
2562
+ // Reset the guard so a later focus/predict can retry once they land.
2563
+ phrasesLoadStarted = false;
2564
+ (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
2565
+ isLocalLLM: isLocalLLM,
2566
+ errorType: 'network'
2567
+ });
2568
+ }
2569
+ case 3:
2570
+ case "end":
2571
+ return _context3.stop();
2572
+ }
2573
+ }, _callee3);
2574
+ }));
2575
+ return function loadPhraseArtifacts(_x2) {
2576
+ return _ref15.apply(this, arguments);
2577
+ };
2578
+ }();
823
2579
  var vocabularyLoadPromise;
824
2580
  var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefaultVocabulary(options) {
825
- var _options$isLocalLLM2;
2581
+ var _options$isLocalLLM3, _options$source;
826
2582
  if (isInitialized) {
827
2583
  return Promise.resolve();
828
2584
  }
829
2585
  if (vocabularyLoadPromise) {
830
2586
  return vocabularyLoadPromise;
831
2587
  }
832
- var isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
2588
+ var isLocalLLM = (_options$isLocalLLM3 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM3 !== void 0 ? _options$isLocalLLM3 : false;
2589
+ var source = (_options$source = options === null || options === void 0 ? void 0 : options.source) !== null && _options$source !== void 0 ? _options$source : 'predict';
833
2590
  var surface = options === null || options === void 0 ? void 0 : options.surface;
834
- vocabularyLoadPromise = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee2() {
835
- var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t3;
836
- return _regenerator.default.wrap(function (_context2) {
837
- while (1) switch (_context2.prev = _context2.next) {
2591
+ vocabularyLoadPromise = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee4() {
2592
+ var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t2;
2593
+ return _regenerator.default.wrap(function (_context4) {
2594
+ while (1) switch (_context4.prev = _context4.next) {
838
2595
  case 0:
839
2596
  (0, _ufo.startExp)(_ufo.EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
840
2597
  isLocalLLM: isLocalLLM
841
2598
  }, surface ? {
842
2599
  surface: surface
843
2600
  } : {}));
844
- _context2.prev = 1;
845
- _context2.next = 2;
846
- return Promise.all([(0, _artifactsManifest.fetchArtifactJson)(_artifactsManifest.ARTIFACT_NAME.VOCABULARY), (0, _artifactsManifest.fetchArtifactJson)(_artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY)]);
2601
+ _context4.prev = 1;
2602
+ (0, _debugMode.ctcTag)('init', "loading artifacts (trigger: ".concat(source, " \xB7 localLLM: ").concat(isLocalLLM, ")"));
2603
+ if ((0, _debugMode.isAutocompleteDebugVerbose)()) {
2604
+ // eslint-disable-next-line no-console
2605
+ console.groupCollapsed('%c[CTC:init]%c artifact manifest', _debugMode.CTC_STYLES.brand, _debugMode.CTC_STYLES.body);
2606
+ // eslint-disable-next-line no-console
2607
+ console.log('Eager:', [_artifactsManifest.ARTIFACT_NAME.VOCABULARY, _artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY, _artifactsManifest.ARTIFACT_NAME.POS_TAGS, _artifactsManifest.ARTIFACT_NAME.GHOST_POS_TAGS, _artifactsManifest.ARTIFACT_NAME.GRAMMAR_TRANSITIONS]);
2608
+ // eslint-disable-next-line no-console
2609
+ console.log('Conditional:', ["".concat(_artifactsManifest.ARTIFACT_NAME.WORD_INDEX, " (only when vectors load; phrase/bigram surfaces share this index)"), "".concat(_artifactsManifest.ARTIFACT_NAME.BIGRAMS, " + ").concat(_artifactsManifest.ARTIFACT_NAME.PHRASES, " (completion-term stats; fire-and-forget)"), "".concat(_artifactsManifest.ARTIFACT_NAME.PHRASE_CONTINUATION_TOKENS, " (local model only; canonical token IDs for the full served union)")]);
2610
+ // eslint-disable-next-line no-console
2611
+ console.groupEnd();
2612
+ }
2613
+ _context4.next = 2;
2614
+ return Promise.all([(0, _artifactLoader.fetchAutocompleteArtifactJson)(_artifactsManifest.ARTIFACT_NAME.VOCABULARY, {
2615
+ summarize: function summarize(payload) {
2616
+ return "".concat(Object.keys(payload.words).length, " words");
2617
+ },
2618
+ validate: isVocabularyJson
2619
+ }), (0, _artifactLoader.fetchAutocompleteArtifactJson)(_artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY, {
2620
+ summarize: function summarize(payload) {
2621
+ return "".concat(payload.length, " words");
2622
+ },
2623
+ validate: isStringArray
2624
+ }),
2625
+ // Awaited alongside the vocabulary so the grammar filter is settled
2626
+ // before the first suggestion can be produced. Without this the
2627
+ // predictor initialises on the smaller vocabulary payload and serves
2628
+ // candidates the filter would have dropped until the grammar lands.
2629
+ // Only settlement is required: a failed load leaves the filter as a
2630
+ // pass-through, which is the same degradation as an absent artifact.
2631
+ (0, _scoringPipeline.loadGrammarDataAsync)({
2632
+ isLocalLLM: isLocalLLM,
2633
+ surface: surface
2634
+ }).catch(function (error) {
2635
+ (0, _debugMode.ctcTag)('init', "\u26A0\uFE0F grammar data unavailable; filtering skipped: ".concat(String(error)), _debugMode.CTC_STYLES.warn);
2636
+ })]);
847
2637
  case 2:
848
- _yield$Promise$all = _context2.sent;
2638
+ _yield$Promise$all = _context4.sent;
849
2639
  _yield$Promise$all2 = (0, _slicedToArray2.default)(_yield$Promise$all, 2);
850
2640
  vocabularyData = _yield$Promise$all2[0];
851
2641
  l3VocabularyData = _yield$Promise$all2[1];
852
- if (!((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData))) {
853
- _context2.next = 3;
854
- break;
855
- }
856
- throw new Error("[text-predictor] unexpected vocabulary payload \u2014 " + "".concat(_artifactsManifest.ARTIFACT_NAME.VOCABULARY, " ").concat((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null ? 'is missing its `words` wrapper' : 'is valid', ", ") + "".concat(_artifactsManifest.ARTIFACT_NAME.L3_VOCABULARY, " ").concat(Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'));
857
- case 3:
858
- terms = Object.entries(vocabularyData.words).map(function (_ref8) {
859
- var _ref9 = (0, _slicedToArray2.default)(_ref8, 2),
860
- word = _ref9[0],
861
- stats = _ref9[1];
2642
+ terms = Object.entries(vocabularyData.words).map(function (_ref18) {
2643
+ var _ref19 = (0, _slicedToArray2.default)(_ref18, 2),
2644
+ word = _ref19[0],
2645
+ stats = _ref19[1];
862
2646
  return {
863
2647
  word: word,
864
2648
  freq: stats.freq,
@@ -872,6 +2656,12 @@ var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefault
872
2656
  initVocabulary({
873
2657
  terms: terms
874
2658
  });
2659
+
2660
+ // Phrase/bigram artifacts load independently and must never affect the
2661
+ // word path — fire-and-forget with its own error handling inside.
2662
+ void loadPhraseArtifacts({
2663
+ isLocalLLM: isLocalLLM
2664
+ }).catch(function () {});
875
2665
  (0, _ufo.succeedExp)(_ufo.EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
876
2666
  isLocalLLM: isLocalLLM,
877
2667
  l2WordCount: terms.length,
@@ -879,25 +2669,25 @@ var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefault
879
2669
  }, surface ? {
880
2670
  surface: surface
881
2671
  } : {}));
882
- _context2.next = 5;
2672
+ _context4.next = 4;
883
2673
  break;
884
- case 4:
885
- _context2.prev = 4;
886
- _t3 = _context2["catch"](1);
2674
+ case 3:
2675
+ _context4.prev = 3;
2676
+ _t2 = _context4["catch"](1);
887
2677
  (0, _ufo.failExp)(_ufo.EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
888
2678
  isLocalLLM: isLocalLLM,
889
- errorType: 'load_error'
2679
+ errorType: 'parse_error'
890
2680
  }, surface ? {
891
2681
  surface: surface
892
2682
  } : {}));
893
2683
  // Allow a later call to retry the load rather than caching the failure.
894
2684
  vocabularyLoadPromise = undefined;
895
- throw _t3;
896
- case 5:
2685
+ throw _t2;
2686
+ case 4:
897
2687
  case "end":
898
- return _context2.stop();
2688
+ return _context4.stop();
899
2689
  }
900
- }, _callee2, null, [[1, 4]]);
2690
+ }, _callee4, null, [[1, 3]]);
901
2691
  }))();
902
2692
  return vocabularyLoadPromise;
903
2693
  };