@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/dist/cjs/analytics/ufo.js +2 -1
- package/dist/cjs/pm-plugins/artifact-loader.js +135 -0
- package/dist/cjs/pm-plugins/artifacts-manifest.js +3 -0
- package/dist/cjs/pm-plugins/autocomplete-plugin.js +498 -160
- package/dist/cjs/pm-plugins/canonical-lm-scoring.js +371 -0
- package/dist/cjs/pm-plugins/debug-mode.js +105 -7
- package/dist/cjs/pm-plugins/ghost-text-decoration.js +12 -3
- package/dist/cjs/pm-plugins/local-slow-lane-client.js +1202 -453
- package/dist/cjs/pm-plugins/scoring-pipeline.js +164 -19
- package/dist/cjs/pm-plugins/slow-lane-client.js +100 -37
- package/dist/cjs/pm-plugins/text-predictor.js +2099 -309
- package/dist/es2019/analytics/ufo.js +2 -0
- package/dist/es2019/pm-plugins/artifact-loader.js +70 -0
- package/dist/es2019/pm-plugins/artifacts-manifest.js +3 -0
- package/dist/es2019/pm-plugins/autocomplete-plugin.js +489 -152
- package/dist/es2019/pm-plugins/canonical-lm-scoring.js +270 -0
- package/dist/es2019/pm-plugins/debug-mode.js +102 -7
- package/dist/es2019/pm-plugins/ghost-text-decoration.js +12 -3
- package/dist/es2019/pm-plugins/local-slow-lane-client.js +940 -326
- package/dist/es2019/pm-plugins/scoring-pipeline.js +137 -13
- package/dist/es2019/pm-plugins/slow-lane-client.js +94 -37
- package/dist/es2019/pm-plugins/text-predictor.js +1692 -200
- package/dist/esm/analytics/ufo.js +2 -1
- package/dist/esm/pm-plugins/artifact-loader.js +128 -0
- package/dist/esm/pm-plugins/artifacts-manifest.js +3 -0
- package/dist/esm/pm-plugins/autocomplete-plugin.js +501 -163
- package/dist/esm/pm-plugins/canonical-lm-scoring.js +364 -0
- package/dist/esm/pm-plugins/debug-mode.js +104 -6
- package/dist/esm/pm-plugins/ghost-text-decoration.js +12 -3
- package/dist/esm/pm-plugins/local-slow-lane-client.js +1200 -451
- package/dist/esm/pm-plugins/scoring-pipeline.js +163 -18
- package/dist/esm/pm-plugins/slow-lane-client.js +100 -37
- package/dist/esm/pm-plugins/text-predictor.js +2101 -313
- package/dist/types/analytics/ufo.d.ts +1 -0
- package/dist/types/pm-plugins/artifact-loader.d.ts +44 -0
- package/dist/types/pm-plugins/artifacts-manifest.d.ts +3 -0
- package/dist/types/pm-plugins/autocomplete-plugin.d.ts +39 -0
- package/dist/types/pm-plugins/canonical-lm-scoring.d.ts +158 -0
- package/dist/types/pm-plugins/debug-mode.d.ts +55 -1
- package/dist/types/pm-plugins/ghost-text-decoration.d.ts +5 -2
- package/dist/types/pm-plugins/local-slow-lane-client.d.ts +87 -54
- package/dist/types/pm-plugins/scoring-pipeline.d.ts +33 -1
- package/dist/types/pm-plugins/slow-lane-client.d.ts +51 -3
- package/dist/types/pm-plugins/text-predictor.d.ts +156 -3
- package/package.json +2 -2
- package/src/analytics/ufo.ts +5 -0
- package/src/pm-plugins/artifact-loader.ts +92 -0
- package/src/pm-plugins/artifacts-manifest.ts +3 -0
- package/src/pm-plugins/autocomplete-plugin.ts +584 -154
- package/src/pm-plugins/canonical-lm-scoring.ts +412 -0
- package/src/pm-plugins/debug-mode.ts +134 -7
- package/src/pm-plugins/ghost-text-decoration.ts +13 -4
- package/src/pm-plugins/local-slow-lane-client.ts +1302 -386
- package/src/pm-plugins/scoring-pipeline.ts +167 -17
- package/src/pm-plugins/slow-lane-client.ts +147 -62
- package/src/pm-plugins/text-predictor.ts +2119 -254
- package/CANONICAL_FIX__DO_NOT_USE_ME_A/package.json +0 -8
- package/CANONICAL_FIX__DO_NOT_USE_ME_B/package.json +0 -8
- package/CANONICAL_FIX__DO_NOT_USE_ME_C/package.json +0 -8
- package/dist/cjs/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -18
- package/dist/cjs/entry-points/src-pm-plugins-slow-lane-client.js +0 -36
- package/dist/cjs/entry-points/src-pm-plugins-text-predictor.js +0 -66
- package/dist/es2019/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
- package/dist/es2019/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
- package/dist/es2019/entry-points/src-pm-plugins-text-predictor.js +0 -2
- package/dist/esm/entry-points/src-pm-plugins-autocomplete-plugin.js +0 -2
- package/dist/esm/entry-points/src-pm-plugins-slow-lane-client.js +0 -2
- package/dist/esm/entry-points/src-pm-plugins-text-predictor.js +0 -2
- package/dist/types/entry-points/src-pm-plugins-autocomplete-plugin.d.ts +0 -2
- package/dist/types/entry-points/src-pm-plugins-slow-lane-client.d.ts +0 -2
- package/dist/types/entry-points/src-pm-plugins-text-predictor.d.ts +0 -2
- package/scripts/gen_first_token_to_words.py +0 -170
- package/src/entry-points/src-pm-plugins-autocomplete-plugin.ts +0 -7
- package/src/entry-points/src-pm-plugins-slow-lane-client.ts +0 -13
- package/src/entry-points/src-pm-plugins-text-predictor.ts +0 -14
- package/src/pm-plugins/autocomplete-plugin/package.json +0 -8
- package/src/pm-plugins/slow-lane-client/package.json +0 -8
- package/src/pm-plugins/text-predictor/package.json +0 -8
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import _typeof from "@babel/runtime/helpers/typeof";
|
|
2
1
|
import _asyncToGenerator from "@babel/runtime/helpers/asyncToGenerator";
|
|
2
|
+
import _typeof from "@babel/runtime/helpers/typeof";
|
|
3
|
+
import _toConsumableArray from "@babel/runtime/helpers/toConsumableArray";
|
|
3
4
|
import _slicedToArray from "@babel/runtime/helpers/slicedToArray";
|
|
4
5
|
import _createClass from "@babel/runtime/helpers/createClass";
|
|
5
6
|
import _classCallCheck from "@babel/runtime/helpers/classCallCheck";
|
|
@@ -15,7 +16,7 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
|
|
|
15
16
|
*
|
|
16
17
|
* Two prediction modes:
|
|
17
18
|
* 1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
|
|
18
|
-
* 2. Mid-word (≥
|
|
19
|
+
* 2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
|
|
19
20
|
*
|
|
20
21
|
* Scoring is delegated to scoring-pipeline.ts which handles:
|
|
21
22
|
* Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
|
|
@@ -24,18 +25,19 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
|
|
|
24
25
|
* Falls back to cold mode (freq-only) when vectors not yet loaded.
|
|
25
26
|
*
|
|
26
27
|
* Session personalization (L1): words the user types are incrementally boosted
|
|
27
|
-
* via incrementSessionFreq(), called on word boundaries from the plugin
|
|
28
|
+
* via incrementSessionFreq(), called on word boundaries from the plugin, and
|
|
29
|
+
* words in ingested context text via ingestDocumentPage(). What the session has
|
|
30
|
+
* boosted is visible at any time from the console: `__atlCtcDebug__.session()`,
|
|
31
|
+
* or `__atlCtcDebug__.session('poll')` for one family — see inspectSessionBoosts.
|
|
28
32
|
*/
|
|
29
33
|
|
|
30
34
|
import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
|
|
38
|
-
import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
|
|
35
|
+
import { fetchAutocompleteArtifactBinary, fetchAutocompleteArtifactJson } from './artifact-loader';
|
|
36
|
+
import { ARTIFACT_NAME } from './artifacts-manifest';
|
|
37
|
+
import { createCanonicalContextPositionCache, deriveCanonicalCandidateContext, deriveWhitespaceBoundaryContext, logSoftmaxAt, logSumExp, selectBoundaryPrimeRequests } from './canonical-lm-scoring';
|
|
38
|
+
import { CTC_STYLES, ctcSection, ctcTag, isAutocompleteDebugEnabled, isAutocompleteDebugVerbose, registerCtcSessionInspector } from './debug-mode';
|
|
39
|
+
import { loadGrammarDataAsync, rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE, MIN_WINNER_MARGIN } from './scoring-pipeline';
|
|
40
|
+
import { getBoundaryLmState, getCanonicalSurfaceCount, getCanonicalSurfaceTokenIds, getDefaultSlowLaneClientStatus, getProgressiveSurfaceEvidence, getStoredContextInput, getStoredContextVector, getStoredLmLogits, getSurfaceScore, isCanonicalSurfaceScoringSupported, primeBoundaryLm, requestProgressiveSurfaceScores } from './slow-lane-client';
|
|
39
41
|
|
|
40
42
|
// ─── Constants ───────────────────────────────────────────────────────────────
|
|
41
43
|
|
|
@@ -47,13 +49,152 @@ var WHITESPACE_SPLIT_REGEX = /\s+/;
|
|
|
47
49
|
var SENTENCE_BOUNDARY_REGEX = /[\n.?!]+/;
|
|
48
50
|
// eslint-disable-next-line require-unicode-regexp
|
|
49
51
|
var TRAILING_WHITESPACE_REGEX = /\s$/;
|
|
50
|
-
var
|
|
52
|
+
var TRAILING_SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+$/;
|
|
53
|
+
var SURFACE_TOKEN_REGEX = /(?:['\x2D0-9A-Z_a-z\xAA\xB2\xB3\xB5\xB9\xBA\xBC-\xBE\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u0660-\u0669\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07C0-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088F\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0966-\u096F\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09E6-\u09F1\u09F4-\u09F9\u09FC\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AE6-\u0AEF\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71-\u0B77\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0BE6-\u0BF2\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5C\u0C5D\u0C60\u0C61\u0C66-\u0C6F\u0C78-\u0C7E\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDC-\u0CDE\u0CE0\u0CE1\u0CE6-\u0CEF\u0CF1\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D58-\u0D61\u0D66-\u0D78\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DE6-\u0DEF\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F20-\u0F33\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F-\u1049\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u1090-\u1099\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A16\u1A20-\u1A54\u1A80-\u1A89\u1A90-\u1A99\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B50-\u1B59\u1B83-\u1BA0\u1BAE-\u1BE5\u1C00-\u1C23\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA7DC\uA7F1-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA830-\uA835\uA840-\uA873\uA882-\uA8B3\uA8D0-\uA8D9\uA8F2-\uA8F7\uA8FB\uA8FD\uA8FE\uA900-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF-\uA9D9\uA9E0-\uA9E4\uA9E6-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA50-\uAA59\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]|\uD800[\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDE80-\uDE9C\uDEA0-\uDED0\uDEE1-\uDEFB\uDF00-\uDF23\uDF2D-\uDF4A\uDF50-\uDF75\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5]|\uD801[\uDC00-\uDC9D\uDCA0-\uDCA9\uDCB0-\uDCD3\uDCD8-\uDCFB\uDD00-\uDD27\uDD30-\uDD63\uDD70-\uDD7A\uDD7C-\uDD8A\uDD8C-\uDD92\uDD94\uDD95\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC\uDDC0-\uDDF3\uDE00-\uDF36\uDF40-\uDF55\uDF60-\uDF67\uDF80-\uDF85\uDF87-\uDFB0\uDFB2-\uDFBA]|\uD802[\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDC58-\uDC76\uDC79-\uDC9E\uDCA7-\uDCAF\uDCE0-\uDCF2\uDCF4\uDCF5\uDCFB-\uDD1B\uDD20-\uDD39\uDD40-\uDD59\uDD80-\uDDB7\uDDBC-\uDDCF\uDDD2-\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE35\uDE40-\uDE48\uDE60-\uDE7E\uDE80-\uDE9F\uDEC0-\uDEC7\uDEC9-\uDEE4\uDEEB-\uDEEF\uDF00-\uDF35\uDF40-\uDF55\uDF58-\uDF72\uDF78-\uDF91\uDFA9-\uDFAF]|\uD803[\uDC00-\uDC48\uDC80-\uDCB2\uDCC0-\uDCF2\uDCFA-\uDD23\uDD30-\uDD39\uDD40-\uDD65\uDD6F-\uDD85\uDE60-\uDE7E\uDE80-\uDEA9\uDEB0\uDEB1\uDEC2-\uDEC7\uDF00-\uDF27\uDF30-\uDF45\uDF51-\uDF54\uDF70-\uDF81\uDFB0-\uDFCB\uDFE0-\uDFF6]|\uD804[\uDC03-\uDC37\uDC52-\uDC6F\uDC71\uDC72\uDC75\uDC83-\uDCAF\uDCD0-\uDCE8\uDCF0-\uDCF9\uDD03-\uDD26\uDD36-\uDD3F\uDD44\uDD47\uDD50-\uDD72\uDD76\uDD83-\uDDB2\uDDC1-\uDDC4\uDDD0-\uDDDA\uDDDC\uDDE1-\uDDF4\uDE00-\uDE11\uDE13-\uDE2B\uDE3F\uDE40\uDE80-\uDE86\uDE88\uDE8A-\uDE8D\uDE8F-\uDE9D\uDE9F-\uDEA8\uDEB0-\uDEDE\uDEF0-\uDEF9\uDF05-\uDF0C\uDF0F\uDF10\uDF13-\uDF28\uDF2A-\uDF30\uDF32\uDF33\uDF35-\uDF39\uDF3D\uDF50\uDF5D-\uDF61\uDF80-\uDF89\uDF8B\uDF8E\uDF90-\uDFB5\uDFB7\uDFD1\uDFD3]|\uD805[\uDC00-\uDC34\uDC47-\uDC4A\uDC50-\uDC59\uDC5F-\uDC61\uDC80-\uDCAF\uDCC4\uDCC5\uDCC7\uDCD0-\uDCD9\uDD80-\uDDAE\uDDD8-\uDDDB\uDE00-\uDE2F\uDE44\uDE50-\uDE59\uDE80-\uDEAA\uDEB8\uDEC0-\uDEC9\uDED0-\uDEE3\uDF00-\uDF1A\uDF30-\uDF3B\uDF40-\uDF46]|\uD806[\uDC00-\uDC2B\uDCA0-\uDCF2\uDCFF-\uDD06\uDD09\uDD0C-\uDD13\uDD15\uDD16\uDD18-\uDD2F\uDD3F\uDD41\uDD50-\uDD59\uDDA0-\uDDA7\uDDAA-\uDDD0\uDDE1\uDDE3\uDE00\uDE0B-\uDE32\uDE3A\uDE50\uDE5C-\uDE89\uDE9D\uDEB0-\uDEF8\uDFC0-\uDFE0\uDFF0-\uDFF9]|\uD807[\uDC00-\uDC08\uDC0A-\uDC2E\uDC40\uDC50-\uDC6C\uDC72-\uDC8F\uDD00-\uDD06\uDD08\uDD09\uDD0B-\uDD30\uDD46\uDD50-\uDD59\uDD60-\uDD65\uDD67\uDD68\uDD6A-\uDD89\uDD98\uDDA0-\uDDA9\uDDB0-\uDDDB\uDDE0-\uDDE9\uDEE0-\uDEF2\uDF02\uDF04-\uDF10\uDF12-\uDF33\uDF50-\uDF59\uDFB0\uDFC0-\uDFD4]|\uD808[\uDC00-\uDF99]|\uD809[\uDC00-\uDC6E\uDC80-\uDD43]|\uD80B[\uDF90-\uDFF0]|[\uD80C\uD80E\uD80F\uD81C-\uD822\uD840-\uD868\uD86A-\uD86D\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD88C][\uDC00-\uDFFF]|\uD80D[\uDC00-\uDC2F\uDC41-\uDC46\uDC60-\uDFFF]|\uD810[\uDC00-\uDFFA]|\uD811[\uDC00-\uDE46]|\uD818[\uDD00-\uDD1D\uDD30-\uDD39]|\uD81A[\uDC00-\uDE38\uDE40-\uDE5E\uDE60-\uDE69\uDE70-\uDEBE\uDEC0-\uDEC9\uDED0-\uDEED\uDF00-\uDF2F\uDF40-\uDF43\uDF50-\uDF59\uDF5B-\uDF61\uDF63-\uDF77\uDF7D-\uDF8F]|\uD81B[\uDD40-\uDD6C\uDD70-\uDD79\uDE40-\uDE96\uDEA0-\uDEB8\uDEBB-\uDED3\uDF00-\uDF4A\uDF50\uDF93-\uDF9F\uDFE0\uDFE1\uDFE3\uDFF2-\uDFF6]|\uD823[\uDC00-\uDCD5\uDCFF-\uDD1E\uDD80-\uDDF2]|\uD82B[\uDFF0-\uDFF3\uDFF5-\uDFFB\uDFFD\uDFFE]|\uD82C[\uDC00-\uDD22\uDD32\uDD50-\uDD52\uDD55\uDD64-\uDD67\uDD70-\uDEFB]|\uD82F[\uDC00-\uDC6A\uDC70-\uDC7C\uDC80-\uDC88\uDC90-\uDC99]|\uD833[\uDCF0-\uDCF9]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB\uDFCE-\uDFFF]|\uD837[\uDF00-\uDF1E\uDF25-\uDF2A]|\uD838[\uDC30-\uDC6D\uDD00-\uDD2C\uDD37-\uDD3D\uDD40-\uDD49\uDD4E\uDE90-\uDEAD\uDEC0-\uDEEB\uDEF0-\uDEF9]|\uD839[\uDCD0-\uDCEB\uDCF0-\uDCF9\uDDD0-\uDDED\uDDF0-\uDDFA\uDEC0-\uDEDE\uDEE0-\uDEE2\uDEE4\uDEE5\uDEE7-\uDEED\uDEF0-\uDEF4\uDEFE\uDEFF\uDFE0-\uDFE6\uDFE8-\uDFEB\uDFED\uDFEE\uDFF0-\uDFFE]|\uD83A[\uDC00-\uDCC4\uDCC7-\uDCCF\uDD00-\uDD43\uDD4B\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEAD\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0\uDFF0-\uDFFF]|\uD87B[\uDC00-\uDE5D]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD88D[\uDC00-\uDC79])+/g;
|
|
54
|
+
var ONLY_WHITESPACE_REGEX = /^[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+$/;
|
|
55
|
+
var UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH = 1;
|
|
56
|
+
var MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH = 1;
|
|
57
|
+
var DISPLAY_MIN_PREFIX_LENGTH = 3;
|
|
51
58
|
var MAX_CANDIDATES = 200;
|
|
52
59
|
var CONTEXT_WORDS = 10;
|
|
53
|
-
var MIN_SCORE_THRESHOLD = 0.35;
|
|
54
60
|
var L3_BASELINE_FREQ = 0.001;
|
|
61
|
+
// Max tokens in a completion term (bigram=2, phrase=3). Mirrors the producer's
|
|
62
|
+
// `phrase_max_words` default so the FE window never exceeds what was mined.
|
|
63
|
+
var PHRASE_MAX_WORDS = 3;
|
|
64
|
+
// Per-window cap on phrase-trie subtree collection (keeps the hot path bounded).
|
|
65
|
+
var MAX_PHRASE_CANDIDATES = 50;
|
|
66
|
+
/**
|
|
67
|
+
* Share of its context shortlist's probability mass the model must put on a
|
|
68
|
+
* surface before that surface may be shown.
|
|
69
|
+
*
|
|
70
|
+
* Candidates competing at the same boundary are normalised into a posterior
|
|
71
|
+
* over that shortlist, so this reads directly as "the model is at least this
|
|
72
|
+
* sure". Measuring distance from the best candidate instead cannot express
|
|
73
|
+
* being unsure: the leader scores full marks whether it won by a nose or a
|
|
74
|
+
* mile, which pushes the entire question of how contested a context is onto
|
|
75
|
+
* `MIN_WINNER_MARGIN` — and there the blended score's only remaining spread is
|
|
76
|
+
* the corpus prior, the weakest ranking signal we have.
|
|
77
|
+
*
|
|
78
|
+
* Each extra token raises the bar. A wrong multi-word ghost costs the reader
|
|
79
|
+
* more to notice and undo than a wrong single word, and it is offered from the
|
|
80
|
+
* same keystroke, so precision has to be bought per token of commitment.
|
|
81
|
+
*/
|
|
82
|
+
var MIN_LM_POSTERIOR = {
|
|
83
|
+
word: 0.5,
|
|
84
|
+
bigram: 0.6,
|
|
85
|
+
phrase: 0.7
|
|
86
|
+
};
|
|
87
|
+
/**
|
|
88
|
+
* How many scored candidates a context must hold before its leader may show.
|
|
89
|
+
*
|
|
90
|
+
* A posterior is a share of a shortlist, so a shortlist of one hands its only
|
|
91
|
+
* member 1.0 whatever the model thinks of it — `logSumExp` of a single value
|
|
92
|
+
* returns that value. Such a ghost sits in the top bucket without having beaten
|
|
93
|
+
* anything, and it is the gate's blind spot rather than a case the gate ruled
|
|
94
|
+
* on: 1 in 14 of them were accepted against 34% for word ghosts overall, and
|
|
95
|
+
* every phrase shown through this hole was rejected.
|
|
96
|
+
*
|
|
97
|
+
* Counted over pool membership rather than over the normaliser, because those
|
|
98
|
+
* two deliberately differ. Where a pool holds a surface and its own extension
|
|
99
|
+
* the chain rule leaves one term in the denominator, and the resulting 1.0 is
|
|
100
|
+
* earned — whichever the user meant, the shorter form is a correct ghost.
|
|
101
|
+
*
|
|
102
|
+
* Waived where the context never held a second candidate at all. The 1-in-14
|
|
103
|
+
* acceptance above was measured over a population dominated by pools that read
|
|
104
|
+
* as one because rivals had not come back, and it is that absence of evidence
|
|
105
|
+
* the floor is for. A vocabulary offering a single continuation of a long
|
|
106
|
+
* distinctive prefix is the opposite situation, and it is also where a ghost
|
|
107
|
+
* saves the most keystrokes, so refusing it spends the most to learn the least.
|
|
108
|
+
*/
|
|
109
|
+
var MIN_SCORED_POOL_SIZE = 2;
|
|
110
|
+
/**
|
|
111
|
+
* Mean per-token log-probability a surface must hold, in nats, independently of
|
|
112
|
+
* anything it was competing against.
|
|
113
|
+
*
|
|
114
|
+
* Every other display test is relative: a posterior is a share of a shortlist,
|
|
115
|
+
* and a winner margin is a distance from a rival. So a surface reaches the
|
|
116
|
+
* screen by being the best of what happened to be in its pool, and nothing ever
|
|
117
|
+
* asks whether the model finds it plausible at all. That gap was tolerable while
|
|
118
|
+
* little was being read, and stopped being tolerable once the scheduler started
|
|
119
|
+
* reading roughly twice as much per decision: shown-per-decision went from 3.2%
|
|
120
|
+
* to 7.4% with no threshold moved, and acceptance of what showed fell from 32%
|
|
121
|
+
* to 24.5%. More reading does not weaken a relative test — it hands it far more
|
|
122
|
+
* pools to be the winner of, and the marginal winner is worse than the average.
|
|
123
|
+
*
|
|
124
|
+
* Set at the point where a uniform distribution over the model's vocabulary
|
|
125
|
+
* sits: 49,152 tokens, so `ln(1/49152)` is about -10.8 nats. Below it the model
|
|
126
|
+
* assigns the surface less mass than a token drawn at random, which is the only
|
|
127
|
+
* statement about a surface that needs no reference to what it competed with.
|
|
128
|
+
*
|
|
129
|
+
* It was first tried at -8, chosen from the bucket where acceptance was measured
|
|
130
|
+
* to fall away, and that reasoning does not survive more data. Acceptance against
|
|
131
|
+
* this quantity turned out to be a smooth slope — about 19% below -8, 26% from -8
|
|
132
|
+
* to -6, then flat near 35% above - with no discontinuity to place a threshold
|
|
133
|
+
* at. A floor on a smooth slope is a coverage dial rather than a test: -8 removed
|
|
134
|
+
* a band converting at 26% and cost 3.7 points of show rate, for an acceptance
|
|
135
|
+
* movement no sample of this size can resolve. Anchoring to uniform instead makes
|
|
136
|
+
* the value an argument about what is indefensible rather than a point picked off
|
|
137
|
+
* a histogram, which is what stops it from being retuned every batch.
|
|
138
|
+
*
|
|
139
|
+
* Applied to the mean over the verified prefix, the same quantity
|
|
140
|
+
* `ghostsByMeanLogProb` buckets, and deliberately not to
|
|
141
|
+
* `meanTokenLogProbabilityUpperBound`. The bound is the right instrument for
|
|
142
|
+
* keeping a candidate in the race, where dropping something that could still
|
|
143
|
+
* clear the bar would be an error, but it is systematically higher than the
|
|
144
|
+
* quantity acceptance is measured against. Judging a surface on the prefix that
|
|
145
|
+
* was actually read is also what every other test here does.
|
|
146
|
+
*
|
|
147
|
+
* One value across all three shapes. Splitting it per shape needs an acceptance
|
|
148
|
+
* curve per shape, and 19 bigrams and 1 phrase across five sessions cannot
|
|
149
|
+
* support one.
|
|
150
|
+
*/
|
|
151
|
+
var MIN_MEAN_TOKEN_LOG_PROBABILITY = -10.8;
|
|
152
|
+
// Tab replaces the whole remaining suffix, so an N-character completion saves
|
|
153
|
+
// N-1 keystrokes: at one character it saves nothing at all and at two it saves
|
|
154
|
+
// one, which does not repay noticing the ghost and reaching for Tab. Offering
|
|
155
|
+
// them anyway spends the acceptance-rate denominator on completions nobody
|
|
156
|
+
// wants and teaches the user to stop reading ghosts. `DISPLAY_MIN_PREFIX_LENGTH`
|
|
157
|
+
// is the same guard on the side the user has already typed.
|
|
158
|
+
var MIN_SUGGESTION_LENGTH = 3;
|
|
159
|
+
/**
|
|
160
|
+
* How many leading tokens of a surface must be scored before it may be shown.
|
|
161
|
+
*
|
|
162
|
+
* Scoring a token costs one model round trip, and a round trip costs about the
|
|
163
|
+
* same no matter how much is in it, so the price of a candidate is its token
|
|
164
|
+
* count. Demanding every token therefore caps what can ever reach the screen at
|
|
165
|
+
* roughly two tokens inside the decision budget — which is why no phrase has
|
|
166
|
+
* ever been displayed. Verifying a fixed prefix makes the cost of a candidate
|
|
167
|
+
* independent of its length.
|
|
168
|
+
*/
|
|
169
|
+
var REQUIRED_VERIFIED_TOKENS = 2;
|
|
170
|
+
/**
|
|
171
|
+
* Extra winner margin demanded of a surface judged on a verified prefix rather
|
|
172
|
+
* than on every one of its tokens, to buy back the precision given up by
|
|
173
|
+
* leaving the tail unread.
|
|
174
|
+
*/
|
|
175
|
+
var PARTIAL_EVIDENCE_MARGIN_PREMIUM = 0.08;
|
|
176
|
+
var ARBITRATION_MODE = 'confidence-v2';
|
|
177
|
+
var DEBUG_TEXT_TAIL_CHARS = 120;
|
|
55
178
|
|
|
56
179
|
// ─── Types ───────────────────────────────────────────────────────────────────
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Which constraint stopped an evaluation from putting a ghost on screen.
|
|
183
|
+
*
|
|
184
|
+
* Each one implies different work: `below-posterior-gate` is a threshold to
|
|
185
|
+
* calibrate, `winner-margin` is two candidates the model cannot separate,
|
|
186
|
+
* `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
|
|
187
|
+
* is vocabulary coverage.
|
|
188
|
+
*/
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* A candidate paired with the length of the already-typed prefix it completes.
|
|
192
|
+
* For a single word this is the current partial token length; for a phrase it
|
|
193
|
+
* is the length of the matched multi-word window (previous words + partial),
|
|
194
|
+
* so the ghost tail is `word.slice(matchedPrefixLen)`.
|
|
195
|
+
*/
|
|
196
|
+
|
|
197
|
+
/** Optional per-term metadata carried by bigram/phrase completion terms. */
|
|
57
198
|
var TrieNode = /*#__PURE__*/_createClass(function TrieNode() {
|
|
58
199
|
_classCallCheck(this, TrieNode);
|
|
59
200
|
_defineProperty(this, "children", new Map());
|
|
@@ -62,6 +203,12 @@ var TrieNode = /*#__PURE__*/_createClass(function TrieNode() {
|
|
|
62
203
|
_defineProperty(this, "docFreq", 0);
|
|
63
204
|
_defineProperty(this, "authorFreq", 0);
|
|
64
205
|
_defineProperty(this, "sessionFreq", 0);
|
|
206
|
+
// Term-type + phrase metadata. Plain words keep `termType='word'` and null
|
|
207
|
+
// metadata; bigram/phrase nodes carry the producer-shipped POS + association.
|
|
208
|
+
_defineProperty(this, "termType", 'word');
|
|
209
|
+
_defineProperty(this, "posSeq", null);
|
|
210
|
+
_defineProperty(this, "headPos", null);
|
|
211
|
+
_defineProperty(this, "association", null);
|
|
65
212
|
});
|
|
66
213
|
var WeightedWordTrie = /*#__PURE__*/function () {
|
|
67
214
|
function WeightedWordTrie() {
|
|
@@ -72,7 +219,8 @@ var WeightedWordTrie = /*#__PURE__*/function () {
|
|
|
72
219
|
}
|
|
73
220
|
return _createClass(WeightedWordTrie, [{
|
|
74
221
|
key: "insert",
|
|
75
|
-
value: function insert(word, tenantFreq, docFreq, authorFreq) {
|
|
222
|
+
value: function insert(word, tenantFreq, docFreq, authorFreq, meta) {
|
|
223
|
+
var _meta$termType, _meta$posSeq, _meta$headPos, _meta$association;
|
|
76
224
|
var node = this.root;
|
|
77
225
|
var _iterator = _createForOfIteratorHelper(word.toLowerCase()),
|
|
78
226
|
_step;
|
|
@@ -95,6 +243,10 @@ var WeightedWordTrie = /*#__PURE__*/function () {
|
|
|
95
243
|
node.tenantFreq = tenantFreq;
|
|
96
244
|
node.docFreq = docFreq;
|
|
97
245
|
node.authorFreq = authorFreq;
|
|
246
|
+
node.termType = (_meta$termType = meta === null || meta === void 0 ? void 0 : meta.termType) !== null && _meta$termType !== void 0 ? _meta$termType : 'word';
|
|
247
|
+
node.posSeq = (_meta$posSeq = meta === null || meta === void 0 ? void 0 : meta.posSeq) !== null && _meta$posSeq !== void 0 ? _meta$posSeq : null;
|
|
248
|
+
node.headPos = (_meta$headPos = meta === null || meta === void 0 ? void 0 : meta.headPos) !== null && _meta$headPos !== void 0 ? _meta$headPos : null;
|
|
249
|
+
node.association = (_meta$association = meta === null || meta === void 0 ? void 0 : meta.association) !== null && _meta$association !== void 0 ? _meta$association : null;
|
|
98
250
|
if (tenantFreq > this.maxTenantFreq) {
|
|
99
251
|
this.maxTenantFreq = tenantFreq;
|
|
100
252
|
}
|
|
@@ -207,6 +359,65 @@ var WeightedWordTrie = /*#__PURE__*/function () {
|
|
|
207
359
|
node.sessionFreq += 1;
|
|
208
360
|
return true;
|
|
209
361
|
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Every word carrying a session boost, optionally limited to one prefix's
|
|
365
|
+
* subtree. Unlike `getCandidates` a word equal to the prefix is included,
|
|
366
|
+
* since the question here is what the session holds rather than what could
|
|
367
|
+
* still be typed.
|
|
368
|
+
*
|
|
369
|
+
* Walks the trie instead of reading an index, so nothing has to be kept in
|
|
370
|
+
* step on the write path for the sake of being able to ask.
|
|
371
|
+
*/
|
|
372
|
+
}, {
|
|
373
|
+
key: "collectSessionBoosted",
|
|
374
|
+
value: function collectSessionBoosted() {
|
|
375
|
+
var prefix = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : '';
|
|
376
|
+
var node = this.root;
|
|
377
|
+
var _iterator5 = _createForOfIteratorHelper(prefix.toLowerCase()),
|
|
378
|
+
_step5;
|
|
379
|
+
try {
|
|
380
|
+
for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
|
|
381
|
+
var char = _step5.value;
|
|
382
|
+
var next = node.children.get(char);
|
|
383
|
+
if (!next) {
|
|
384
|
+
return [];
|
|
385
|
+
}
|
|
386
|
+
node = next;
|
|
387
|
+
}
|
|
388
|
+
} catch (err) {
|
|
389
|
+
_iterator5.e(err);
|
|
390
|
+
} finally {
|
|
391
|
+
_iterator5.f();
|
|
392
|
+
}
|
|
393
|
+
var boosted = [];
|
|
394
|
+
var stack = [node];
|
|
395
|
+
while (stack.length > 0) {
|
|
396
|
+
var current = stack.pop();
|
|
397
|
+
if (!current) {
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
if (current.word !== null && current.sessionFreq > 0) {
|
|
401
|
+
boosted.push({
|
|
402
|
+
word: current.word,
|
|
403
|
+
node: current
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
var _iterator6 = _createForOfIteratorHelper(current.children.values()),
|
|
407
|
+
_step6;
|
|
408
|
+
try {
|
|
409
|
+
for (_iterator6.s(); !(_step6 = _iterator6.n()).done;) {
|
|
410
|
+
var child = _step6.value;
|
|
411
|
+
stack.push(child);
|
|
412
|
+
}
|
|
413
|
+
} catch (err) {
|
|
414
|
+
_iterator6.e(err);
|
|
415
|
+
} finally {
|
|
416
|
+
_iterator6.f();
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return boosted;
|
|
420
|
+
}
|
|
210
421
|
}]);
|
|
211
422
|
}(); // L1/L2 Trie (Session + Atlassian Domain)
|
|
212
423
|
var wordTrie = new WeightedWordTrie();
|
|
@@ -214,29 +425,36 @@ var wordTrie = new WeightedWordTrie();
|
|
|
214
425
|
// L3 Trie (General English Fallback)
|
|
215
426
|
var l3Trie = new WeightedWordTrie();
|
|
216
427
|
|
|
428
|
+
/**
|
|
429
|
+
* L2 phrase/bigram trie. Kept SEPARATE from `wordTrie` so single-word
|
|
430
|
+
* completion + L3 gap-fill stay untouched; multi-word units are only surfaced
|
|
431
|
+
* by the (12b) multiword prefix-window matcher. Keys are the full normalized
|
|
432
|
+
* surface string (spaces included), e.g. `"return on investment"`, so a lookup
|
|
433
|
+
* is a plain char-trie walk over the whole phrase.
|
|
434
|
+
*/
|
|
435
|
+
var phraseTrie = new WeightedWordTrie();
|
|
436
|
+
|
|
217
437
|
/**
|
|
218
438
|
* Loads the General English vocabulary.
|
|
219
439
|
* expects a simple array of strings: ["about", "above", "actually", ...]
|
|
220
440
|
*/
|
|
221
441
|
export var initL3Vocabulary = function initL3Vocabulary(l3Words) {
|
|
222
|
-
var
|
|
223
|
-
|
|
442
|
+
var _iterator7 = _createForOfIteratorHelper(l3Words),
|
|
443
|
+
_step7;
|
|
224
444
|
try {
|
|
225
|
-
for (
|
|
226
|
-
var word =
|
|
445
|
+
for (_iterator7.s(); !(_step7 = _iterator7.n()).done;) {
|
|
446
|
+
var word = _step7.value;
|
|
227
447
|
// Insert with a tiny baseline frequency so it mathematically
|
|
228
448
|
// loses to any domain word in Stage 1, but still scores above 0.
|
|
229
449
|
l3Trie.insert(word, L3_BASELINE_FREQ, 0, 0);
|
|
230
450
|
}
|
|
231
451
|
} catch (err) {
|
|
232
|
-
|
|
452
|
+
_iterator7.e(err);
|
|
233
453
|
} finally {
|
|
234
|
-
|
|
235
|
-
}
|
|
236
|
-
if (isAutocompleteDebugEnabled()) {
|
|
237
|
-
// eslint-disable-next-line no-console
|
|
238
|
-
console.log("[text-predictor] L3 General English loaded: ".concat(l3Words.length, " words"));
|
|
454
|
+
_iterator7.f();
|
|
239
455
|
}
|
|
456
|
+
recallGeneration++;
|
|
457
|
+
ctcTag('init', "L3 general English loaded: ".concat(l3Words.length, " words"));
|
|
240
458
|
};
|
|
241
459
|
|
|
242
460
|
// const bigramMap: Map<string, Record<string, number>> = new Map(
|
|
@@ -246,8 +464,50 @@ export var initL3Vocabulary = function initL3Vocabulary(l3Words) {
|
|
|
246
464
|
var isInitialized = false;
|
|
247
465
|
var vectorStore = null;
|
|
248
466
|
var vectorsLoadStarted = false;
|
|
467
|
+
|
|
468
|
+
// Phrase/bigram artifacts load independently of the word vocabulary so a
|
|
469
|
+
// missing/late `bigrams.json`/`phrases.json` never breaks word completion.
|
|
470
|
+
var phrasesLoadStarted = false;
|
|
471
|
+
var phrasesLoaded = false;
|
|
472
|
+
var phraseTermCount = 0;
|
|
473
|
+
// Per-type frequency maxima, tracked separately because bigrams and phrases
|
|
474
|
+
// share one trie but have very different absolute frequency ranges (bigrams are
|
|
475
|
+
// far more frequent than 3-word phrases). Normalizing each type by its OWN max
|
|
476
|
+
// stops phrase freq scores from being crushed against the top bigram's count.
|
|
477
|
+
var maxBigramFreq = 1;
|
|
478
|
+
var maxPhraseFreq = 1;
|
|
249
479
|
var lastPredictionDebug = null;
|
|
250
480
|
|
|
481
|
+
// ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
|
|
482
|
+
// Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
|
|
483
|
+
// generally keep a just-accepted unit from being re-offered:
|
|
484
|
+
// • repetition: never surface a candidate whose full surface already equals the
|
|
485
|
+
// run of tokens immediately before the cursor (handled inline in predict()).
|
|
486
|
+
// • cooldown: after an accept, briefly suppress re-offering that same unit. The
|
|
487
|
+
// cooldown lifts only once BOTH the keystroke budget and the time window have
|
|
488
|
+
// elapsed (either one still counting keeps it active).
|
|
489
|
+
var COOLDOWN_KEYSTROKES = 5;
|
|
490
|
+
var COOLDOWN_MS = 2000;
|
|
491
|
+
|
|
492
|
+
// Active post-accept cooldown, or null when none.
|
|
493
|
+
var acceptCooldown = null;
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* Start a short cooldown for the exact surface the editor inserted. The caller
|
|
497
|
+
* passes the committed snapshot's surface so background re-ranking can never
|
|
498
|
+
* move cooldown bookkeeping away from what the user actually accepted.
|
|
499
|
+
*/
|
|
500
|
+
export var noteSuggestionAccepted = function noteSuggestionAccepted(surface) {
|
|
501
|
+
var normalizedSurface = surface.trim().toLowerCase();
|
|
502
|
+
if (normalizedSurface) {
|
|
503
|
+
acceptCooldown = {
|
|
504
|
+
predictionsSince: 0,
|
|
505
|
+
surface: normalizedSurface,
|
|
506
|
+
ts: performance.now()
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
};
|
|
510
|
+
|
|
251
511
|
/** Get vector for a word from the store. */
|
|
252
512
|
var getWordVector = function getWordVector(word) {
|
|
253
513
|
if (!vectorStore) {
|
|
@@ -272,20 +532,20 @@ var computeContextVectorLocal = function computeContextVectorLocal(textBefore) {
|
|
|
272
532
|
var tokens = tokenize(textBefore);
|
|
273
533
|
var words = tokens.slice(-CONTEXT_WORDS);
|
|
274
534
|
var vectors = [];
|
|
275
|
-
var
|
|
276
|
-
|
|
535
|
+
var _iterator8 = _createForOfIteratorHelper(words),
|
|
536
|
+
_step8;
|
|
277
537
|
try {
|
|
278
|
-
for (
|
|
279
|
-
var word =
|
|
538
|
+
for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
|
|
539
|
+
var word = _step8.value;
|
|
280
540
|
var _v = getWordVector(word);
|
|
281
541
|
if (_v) {
|
|
282
542
|
vectors.push(_v);
|
|
283
543
|
}
|
|
284
544
|
}
|
|
285
545
|
} catch (err) {
|
|
286
|
-
|
|
546
|
+
_iterator8.e(err);
|
|
287
547
|
} finally {
|
|
288
|
-
|
|
548
|
+
_iterator8.f();
|
|
289
549
|
}
|
|
290
550
|
if (vectors.length === 0) {
|
|
291
551
|
return null;
|
|
@@ -318,11 +578,11 @@ var getContextVectorForScoring = function getContextVectorForScoring(textBefore)
|
|
|
318
578
|
var tokenize = function tokenize(text) {
|
|
319
579
|
var tokens = [];
|
|
320
580
|
// eslint-disable-next-line @atlassian/perf-linting/no-expensive-split-replace
|
|
321
|
-
var
|
|
322
|
-
|
|
581
|
+
var _iterator9 = _createForOfIteratorHelper(text.toLowerCase().split(WHITESPACE_SPLIT_REGEX)),
|
|
582
|
+
_step9;
|
|
323
583
|
try {
|
|
324
|
-
for (
|
|
325
|
-
var raw =
|
|
584
|
+
for (_iterator9.s(); !(_step9 = _iterator9.n()).done;) {
|
|
585
|
+
var raw = _step9.value;
|
|
326
586
|
// eslint-disable-next-line @atlassian/perf-linting/no-expensive-split-replace
|
|
327
587
|
var clean = raw.replace(PUNCTUATION_BOUNDARY_REGEX, '');
|
|
328
588
|
if (clean.length >= 2) {
|
|
@@ -330,9 +590,9 @@ var tokenize = function tokenize(text) {
|
|
|
330
590
|
}
|
|
331
591
|
}
|
|
332
592
|
} catch (err) {
|
|
333
|
-
|
|
593
|
+
_iterator9.e(err);
|
|
334
594
|
} finally {
|
|
335
|
-
|
|
595
|
+
_iterator9.f();
|
|
336
596
|
}
|
|
337
597
|
return tokens;
|
|
338
598
|
};
|
|
@@ -344,6 +604,124 @@ var extractPreviousWord = function extractPreviousWord(text) {
|
|
|
344
604
|
return words.length >= 2 ? words[words.length - 2] : '';
|
|
345
605
|
};
|
|
346
606
|
|
|
607
|
+
/**
|
|
608
|
+
* Collect phrase/bigram completion candidates for the multi-word window ending
|
|
609
|
+
* at the current partial token.
|
|
610
|
+
*
|
|
611
|
+
* Builds windows of the last `2..PHRASE_MAX_WORDS` tokens of the current
|
|
612
|
+
* sentence (widest first) and prefix-searches the phrase trie for each, so
|
|
613
|
+
* `quarterly pla` matches `quarterly planning meeting`. Windows never cross a
|
|
614
|
+
* sentence boundary (mirrors `extractPreviousWord`). Each result records the
|
|
615
|
+
* matched window length so the caller can compute the ghost tail.
|
|
616
|
+
*
|
|
617
|
+
* :params:
|
|
618
|
+
* trimmed: Text before the cursor, trailing whitespace already removed
|
|
619
|
+
* :returns:
|
|
620
|
+
* Matched phrase candidates (deduped, widest-window match wins per term)
|
|
621
|
+
*/
|
|
622
|
+
var getPhraseCandidates = function getPhraseCandidates(trimmed) {
|
|
623
|
+
// Records every prefix window we probe and its match count so the (verbose)
|
|
624
|
+
// debug line makes it obvious whether the phrase path found nothing because the
|
|
625
|
+
// trie is empty, or because no phrase begins with the typed window.
|
|
626
|
+
var windowsTried = [];
|
|
627
|
+
var logPhrasePath = function logPhrasePath(skipReason) {
|
|
628
|
+
if (!isAutocompleteDebugVerbose()) {
|
|
629
|
+
return;
|
|
630
|
+
}
|
|
631
|
+
var detail = skipReason ? skipReason : "windows: ".concat(windowsTried.map(function (w) {
|
|
632
|
+
return "\"".concat(w.window, "\"\u2192").concat(w.matches);
|
|
633
|
+
}).join(', ') || '(none)');
|
|
634
|
+
ctcTag('phrase', "loaded=".concat(phraseTermCount, " (").concat(phrasesLoaded ? 'ready' : 'not loaded', ") \xB7 ").concat(detail), CTC_STYLES.section);
|
|
635
|
+
};
|
|
636
|
+
if (phraseTermCount === 0) {
|
|
637
|
+
logPhrasePath('phrase trie empty — bigrams.json/phrases.json not loaded (or 0 terms)');
|
|
638
|
+
return [];
|
|
639
|
+
}
|
|
640
|
+
var tokenMatches = Array.from(trimmed.matchAll(SURFACE_TOKEN_REGEX));
|
|
641
|
+
var lastMatch = tokenMatches[tokenMatches.length - 1];
|
|
642
|
+
var lastStart = lastMatch === null || lastMatch === void 0 ? void 0 : lastMatch.index;
|
|
643
|
+
if (!lastMatch || lastStart === undefined || lastStart + lastMatch[0].length !== trimmed.length) {
|
|
644
|
+
logPhrasePath('no tokens in current sentence');
|
|
645
|
+
return [];
|
|
646
|
+
}
|
|
647
|
+
var trailingTokens = [{
|
|
648
|
+
start: lastStart,
|
|
649
|
+
value: lastMatch[0]
|
|
650
|
+
}];
|
|
651
|
+
for (var i = tokenMatches.length - 2; i >= 0; i--) {
|
|
652
|
+
var current = trailingTokens[0];
|
|
653
|
+
var match = tokenMatches[i];
|
|
654
|
+
var start = match.index;
|
|
655
|
+
if (start === undefined) {
|
|
656
|
+
break;
|
|
657
|
+
}
|
|
658
|
+
var end = start + match[0].length;
|
|
659
|
+
var gap = trimmed.slice(end, current.start);
|
|
660
|
+
if (!ONLY_WHITESPACE_REGEX.test(gap)) {
|
|
661
|
+
break;
|
|
662
|
+
}
|
|
663
|
+
trailingTokens.unshift({
|
|
664
|
+
start: start,
|
|
665
|
+
value: match[0]
|
|
666
|
+
});
|
|
667
|
+
if (trailingTokens.length >= PHRASE_MAX_WORDS) {
|
|
668
|
+
break;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
var partial = trailingTokens[trailingTokens.length - 1].value;
|
|
672
|
+
var results = [];
|
|
673
|
+
var seen = new Set();
|
|
674
|
+
var maxPrev = trailingTokens.length - 1;
|
|
675
|
+
|
|
676
|
+
// Widest window first so the most specific (longest-context) match wins the
|
|
677
|
+
// dedup for a given phrase term. `prev` is the number of *preceding* complete
|
|
678
|
+
// words included ahead of the current partial:
|
|
679
|
+
// prev ≥ 1 → "continue a phrase I've started" (e.g. "root c" → root cause).
|
|
680
|
+
// Preceding-word context is strong, so a short partial is fine.
|
|
681
|
+
// prev = 0 → "the partial is a phrase's FIRST word" (e.g. "root" → root
|
|
682
|
+
// cause). No preceding context, so require the partial to be a
|
|
683
|
+
// one character so grouped LM work can begin early. UI remains
|
|
684
|
+
// gated at DISPLAY_MIN_PREFIX_LENGTH.
|
|
685
|
+
for (var prev = maxPrev; prev >= 0; prev--) {
|
|
686
|
+
if (prev === 0 && partial.length < MULTIWORD_PREFETCH_MIN_PREFIX_LENGTH) {
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
var windowTokens = trailingTokens.slice(-(prev + 1));
|
|
690
|
+
var windowPrefix = windowTokens.map(function (token) {
|
|
691
|
+
return token.value;
|
|
692
|
+
}).join(' ').toLowerCase();
|
|
693
|
+
var windowLen = windowPrefix.length;
|
|
694
|
+
var matches = phraseTrie.getCandidates(windowPrefix, MAX_PHRASE_CANDIDATES);
|
|
695
|
+
windowsTried.push({
|
|
696
|
+
window: windowPrefix,
|
|
697
|
+
matches: matches.length
|
|
698
|
+
});
|
|
699
|
+
var _iterator0 = _createForOfIteratorHelper(matches),
|
|
700
|
+
_step0;
|
|
701
|
+
try {
|
|
702
|
+
for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
|
|
703
|
+
var _match = _step0.value;
|
|
704
|
+
if (seen.has(_match.word)) {
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
seen.add(_match.word);
|
|
708
|
+
results.push({
|
|
709
|
+
node: _match.node,
|
|
710
|
+
word: _match.word,
|
|
711
|
+
matchedPrefixLen: windowLen,
|
|
712
|
+
surfaceStart: windowTokens[0].start
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
} catch (err) {
|
|
716
|
+
_iterator0.e(err);
|
|
717
|
+
} finally {
|
|
718
|
+
_iterator0.f();
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
logPhrasePath();
|
|
722
|
+
return results;
|
|
723
|
+
};
|
|
724
|
+
|
|
347
725
|
// ─── Debug Helpers ───────────────────────────────────────────────────────────
|
|
348
726
|
|
|
349
727
|
/**
|
|
@@ -352,11 +730,25 @@ var extractPreviousWord = function extractPreviousWord(text) {
|
|
|
352
730
|
* wordCount: number of words in vector store (0 if not loaded)
|
|
353
731
|
*/
|
|
354
732
|
export var getPredictorStatus = function getPredictorStatus() {
|
|
733
|
+
var slowLaneStatus = getDefaultSlowLaneClientStatus();
|
|
355
734
|
return {
|
|
735
|
+
canonicalScoringSupported: slowLaneStatus.canonicalScoringSupported,
|
|
736
|
+
canonicalSurfaceCount: slowLaneStatus.canonicalSurfaceCount,
|
|
737
|
+
clientId: slowLaneStatus.clientId,
|
|
738
|
+
clientKind: slowLaneStatus.clientKind,
|
|
739
|
+
localModelReady: slowLaneStatus.localModelReady,
|
|
740
|
+
slowLaneRegistered: slowLaneStatus.registered,
|
|
356
741
|
vectorsLoaded: vectorStore !== null,
|
|
357
742
|
wordCount: vectorStore ? Object.keys(vectorStore.wordIndex).length : 0,
|
|
358
743
|
vectorsLoadStarted: vectorsLoadStarted,
|
|
359
|
-
isInitialized: isInitialized
|
|
744
|
+
isInitialized: isInitialized,
|
|
745
|
+
phrasesLoaded: phrasesLoaded,
|
|
746
|
+
phraseCount: phraseTermCount,
|
|
747
|
+
maxFreqByType: {
|
|
748
|
+
word: wordTrie.maxTenantFreq,
|
|
749
|
+
bigram: maxBigramFreq,
|
|
750
|
+
phrase: maxPhraseFreq
|
|
751
|
+
}
|
|
360
752
|
};
|
|
361
753
|
};
|
|
362
754
|
|
|
@@ -368,19 +760,57 @@ export var getLastPredictionDebug = function getLastPredictionDebug() {
|
|
|
368
760
|
return lastPredictionDebug;
|
|
369
761
|
};
|
|
370
762
|
export var initVocabulary = function initVocabulary(vocabulary) {
|
|
371
|
-
var
|
|
372
|
-
|
|
763
|
+
var _iterator1 = _createForOfIteratorHelper(vocabulary.terms),
|
|
764
|
+
_step1;
|
|
373
765
|
try {
|
|
374
|
-
for (
|
|
375
|
-
var term =
|
|
766
|
+
for (_iterator1.s(); !(_step1 = _iterator1.n()).done;) {
|
|
767
|
+
var term = _step1.value;
|
|
376
768
|
wordTrie.insert(term.word, term.freq, term.docFreq, term.authorFreq);
|
|
377
769
|
}
|
|
378
770
|
} catch (err) {
|
|
379
|
-
|
|
771
|
+
_iterator1.e(err);
|
|
380
772
|
} finally {
|
|
381
|
-
|
|
773
|
+
_iterator1.f();
|
|
382
774
|
}
|
|
383
775
|
isInitialized = true;
|
|
776
|
+
recallGeneration++;
|
|
777
|
+
};
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* Insert a producer-shipped bigram/phrase artifact into the phrase trie.
|
|
781
|
+
*
|
|
782
|
+
* :params:
|
|
783
|
+
* artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
|
|
784
|
+
* termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
|
|
785
|
+
* :returns:
|
|
786
|
+
* The number of terms inserted
|
|
787
|
+
*/
|
|
788
|
+
export var initPhrases = function initPhrases(artifact, termType) {
|
|
789
|
+
var count = 0;
|
|
790
|
+
for (var _i3 = 0, _Object$entries = Object.entries(artifact); _i3 < _Object$entries.length; _i3++) {
|
|
791
|
+
var _stats$pos_seq, _stats$head_pos, _stats$association;
|
|
792
|
+
var _Object$entries$_i = _slicedToArray(_Object$entries[_i3], 2),
|
|
793
|
+
term = _Object$entries$_i[0],
|
|
794
|
+
stats = _Object$entries$_i[1];
|
|
795
|
+
phraseTrie.insert(term, stats.freq, stats.doc_freq, stats.author_freq, {
|
|
796
|
+
termType: termType,
|
|
797
|
+
posSeq: (_stats$pos_seq = stats.pos_seq) !== null && _stats$pos_seq !== void 0 ? _stats$pos_seq : null,
|
|
798
|
+
headPos: (_stats$head_pos = stats.head_pos) !== null && _stats$head_pos !== void 0 ? _stats$head_pos : null,
|
|
799
|
+
association: (_stats$association = stats.association) !== null && _stats$association !== void 0 ? _stats$association : null
|
|
800
|
+
});
|
|
801
|
+
// Track each type's own frequency ceiling for per-type normalization.
|
|
802
|
+
if (termType === 'bigram') {
|
|
803
|
+
if (stats.freq > maxBigramFreq) {
|
|
804
|
+
maxBigramFreq = stats.freq;
|
|
805
|
+
}
|
|
806
|
+
} else if (stats.freq > maxPhraseFreq) {
|
|
807
|
+
maxPhraseFreq = stats.freq;
|
|
808
|
+
}
|
|
809
|
+
count++;
|
|
810
|
+
}
|
|
811
|
+
phraseTermCount += count;
|
|
812
|
+
recallGeneration++;
|
|
813
|
+
return count;
|
|
384
814
|
};
|
|
385
815
|
|
|
386
816
|
/**
|
|
@@ -407,141 +837,258 @@ export var ingestDocumentPage = function ingestDocumentPage(pageContent) {
|
|
|
407
837
|
}
|
|
408
838
|
var words = tokenize(pageContent);
|
|
409
839
|
var validBoostedWords = new Set();
|
|
410
|
-
var
|
|
411
|
-
|
|
840
|
+
var _iterator10 = _createForOfIteratorHelper(words),
|
|
841
|
+
_step10;
|
|
412
842
|
try {
|
|
413
|
-
for (
|
|
414
|
-
var word =
|
|
843
|
+
for (_iterator10.s(); !(_step10 = _iterator10.n()).done;) {
|
|
844
|
+
var word = _step10.value;
|
|
415
845
|
var didBoost = wordTrie.incrementSessionFreq(word);
|
|
416
846
|
if (didBoost) {
|
|
417
847
|
validBoostedWords.add(word);
|
|
418
848
|
}
|
|
419
849
|
}
|
|
420
850
|
} catch (err) {
|
|
421
|
-
|
|
851
|
+
_iterator10.e(err);
|
|
422
852
|
} finally {
|
|
423
|
-
|
|
853
|
+
_iterator10.f();
|
|
424
854
|
}
|
|
425
855
|
if (isAutocompleteDebugEnabled() && validBoostedWords.size > 0) {
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
console.groupEnd();
|
|
856
|
+
ctcTag('init', "L1 session primed ".concat(validBoostedWords.size, " words from page \xB7 __atlCtcDebug__.session() to inspect"), CTC_STYLES.brand);
|
|
857
|
+
if (isAutocompleteDebugVerbose()) {
|
|
858
|
+
// eslint-disable-next-line no-console
|
|
859
|
+
console.dir(Array.from(validBoostedWords).sort());
|
|
860
|
+
}
|
|
432
861
|
}
|
|
433
862
|
};
|
|
434
|
-
export var predict = function predict(textBefore) {
|
|
435
|
-
if (!isInitialized) {
|
|
436
|
-
// Vocabulary JSON is code-split and loads asynchronously. Kick off the load
|
|
437
|
-
// and skip this keystroke; the plugin also primes it on focus, so the tries
|
|
438
|
-
// are usually ready before the user types.
|
|
439
|
-
void loadDefaultVocabulary().catch(function () {});
|
|
440
|
-
return null;
|
|
441
|
-
}
|
|
442
|
-
var t0 = performance.now();
|
|
443
863
|
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
864
|
+
/**
|
|
865
|
+
* How many boosted words `inspectSessionBoosts` lists.
|
|
866
|
+
*
|
|
867
|
+
* A page ingest can boost thousands, and a list that long is not read. The
|
|
868
|
+
* strongest boosts are the ones that change an ordering, and `boosted` still
|
|
869
|
+
* reports the full size, so the cap loses nothing but volume.
|
|
870
|
+
*/
|
|
871
|
+
var MAX_LISTED_SESSION_WORDS = 50;
|
|
872
|
+
/**
|
|
873
|
+
* Read the session's L1 boosts, optionally narrowed to a prefix.
|
|
874
|
+
*
|
|
875
|
+
* Installed as `__atlCtcDebug__.session()`, with `__atlCtcDebug__.session('poll')`
|
|
876
|
+
* to ask about one family. Returned rather than logged, so the console renders it
|
|
877
|
+
* as an inspectable object and a caller can assert on it.
|
|
878
|
+
*
|
|
879
|
+
* Only words the vocabulary already holds can carry a boost, because both writers
|
|
880
|
+
* go through `incrementSessionFreq` and it only finds existing nodes. An ingested
|
|
881
|
+
* word absent from the vocabulary is therefore missing from here and always will
|
|
882
|
+
* be.
|
|
883
|
+
*/
|
|
884
|
+
export var inspectSessionBoosts = function inspectSessionBoosts(prefix) {
|
|
885
|
+
var boosted = wordTrie.collectSessionBoosted(prefix !== null && prefix !== void 0 ? prefix : '');
|
|
886
|
+
return _objectSpread(_objectSpread({
|
|
887
|
+
boosted: boosted.length,
|
|
888
|
+
limit: MAX_LISTED_SESSION_WORDS
|
|
889
|
+
}, prefix === undefined ? {} : {
|
|
890
|
+
prefix: prefix
|
|
891
|
+
}), {}, {
|
|
892
|
+
words: boosted.sort(function (a, b) {
|
|
893
|
+
return b.node.sessionFreq - a.node.sessionFreq || a.word.localeCompare(b.word, 'en', {
|
|
894
|
+
numeric: true,
|
|
895
|
+
sensitivity: 'base'
|
|
896
|
+
});
|
|
897
|
+
}).slice(0, MAX_LISTED_SESSION_WORDS).map(function (_ref) {
|
|
898
|
+
var node = _ref.node,
|
|
899
|
+
word = _ref.word;
|
|
900
|
+
return {
|
|
901
|
+
sessionFreq: node.sessionFreq,
|
|
902
|
+
sessionOnly: node.tenantFreq === 0,
|
|
903
|
+
surface: word,
|
|
904
|
+
tenantFreq: node.tenantFreq
|
|
905
|
+
};
|
|
906
|
+
})
|
|
907
|
+
});
|
|
908
|
+
};
|
|
909
|
+
|
|
910
|
+
// At module scope so the console answers before the first keystroke, which is
|
|
911
|
+
// when someone reaching for it usually asks.
|
|
912
|
+
registerCtcSessionInspector(inspectSessionBoosts);
|
|
913
|
+
|
|
914
|
+
/**
|
|
915
|
+
* Result of a prediction: the ghost tail to insert plus an immutable record of
|
|
916
|
+
* the evidence that authorized the UI commitment.
|
|
917
|
+
*/
|
|
918
|
+
|
|
919
|
+
// ─── Recall memo ─────────────────────────────────────────────────────────────
|
|
920
|
+
|
|
921
|
+
/** Trie recall plus the canonical context derived for each matched term. */
|
|
922
|
+
|
|
923
|
+
/**
|
|
924
|
+
* Bumped whenever an artifact load changes what the tries can return, which is
|
|
925
|
+
* the only way the candidate *set* for a given prefix can change.
|
|
926
|
+
*
|
|
927
|
+
* Session-frequency boosts are deliberately not counted: they mutate trie nodes
|
|
928
|
+
* in place, and the memo holds those nodes by reference, so a boost is already
|
|
929
|
+
* visible through a cached entry.
|
|
930
|
+
*/
|
|
931
|
+
var recallGeneration = 0;
|
|
932
|
+
var recallMemo = null;
|
|
933
|
+
var computeCanonicalRecall = function computeCanonicalRecall(trimmed, currentWord, currentWordStart) {
|
|
934
|
+
// Start recall at one character so grouped causal work can overlap later
|
|
935
|
+
// keystrokes. Nothing is displayed until three characters.
|
|
936
|
+
var wordCandidates = currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH ? wordTrie.getCandidates(currentWord, MAX_CANDIDATES) : [];
|
|
492
937
|
|
|
493
938
|
// Gap-fill from the L3 general-English trie, requesting a full buffer so
|
|
494
939
|
// enough survive de-duplication against the L2 results.
|
|
495
|
-
if (
|
|
940
|
+
if (currentWord.length >= UNIGRAM_PREFETCH_MIN_PREFIX_LENGTH && wordCandidates.length < MAX_CANDIDATES) {
|
|
496
941
|
var l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
|
|
497
|
-
var existingWords = new Set(
|
|
942
|
+
var existingWords = new Set(wordCandidates.map(function (c) {
|
|
498
943
|
return c.word;
|
|
499
944
|
}));
|
|
500
|
-
var
|
|
501
|
-
|
|
945
|
+
var _iterator11 = _createForOfIteratorHelper(l3Candidates),
|
|
946
|
+
_step11;
|
|
502
947
|
try {
|
|
503
|
-
for (
|
|
504
|
-
var l3c =
|
|
505
|
-
if (
|
|
948
|
+
for (_iterator11.s(); !(_step11 = _iterator11.n()).done;) {
|
|
949
|
+
var l3c = _step11.value;
|
|
950
|
+
if (wordCandidates.length >= MAX_CANDIDATES) break;
|
|
506
951
|
if (!existingWords.has(l3c.word)) {
|
|
507
|
-
|
|
952
|
+
wordCandidates.push(l3c);
|
|
508
953
|
}
|
|
509
954
|
}
|
|
510
955
|
} catch (err) {
|
|
511
|
-
|
|
956
|
+
_iterator11.e(err);
|
|
512
957
|
} finally {
|
|
513
|
-
|
|
958
|
+
_iterator11.f();
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
// Multi-word completion terms (bigrams/phrases) matched on the trailing
|
|
963
|
+
// window ending at the current partial token. Runs even when the unigram path
|
|
964
|
+
// was skipped for a short partial, so first-word/early phrase matches surface.
|
|
965
|
+
var phraseCandidates = getPhraseCandidates(trimmed);
|
|
966
|
+
|
|
967
|
+
// Unify: a word completes the current partial token; a phrase completes its
|
|
968
|
+
// matched multi-word window. Track the prefix length per term so the ghost
|
|
969
|
+
// tail is sliced correctly regardless of term type.
|
|
970
|
+
var matched = [].concat(_toConsumableArray(wordCandidates.map(function (_ref2) {
|
|
971
|
+
var word = _ref2.word,
|
|
972
|
+
node = _ref2.node;
|
|
973
|
+
return {
|
|
974
|
+
word: word,
|
|
975
|
+
node: node,
|
|
976
|
+
matchedPrefixLen: currentWord.length,
|
|
977
|
+
surfaceStart: currentWordStart
|
|
978
|
+
};
|
|
979
|
+
})), _toConsumableArray(phraseCandidates));
|
|
980
|
+
var positionCache = createCanonicalContextPositionCache();
|
|
981
|
+
var canonicalMatched = matched.map(function (candidate) {
|
|
982
|
+
return _objectSpread(_objectSpread({}, candidate), deriveCanonicalCandidateContext(trimmed, candidate.matchedPrefixLen, candidate.word, getCanonicalSurfaceTokenIds, candidate.surfaceStart, positionCache));
|
|
983
|
+
});
|
|
984
|
+
var prefixLenByWord = new Map();
|
|
985
|
+
var _iterator12 = _createForOfIteratorHelper(canonicalMatched),
|
|
986
|
+
_step12;
|
|
987
|
+
try {
|
|
988
|
+
for (_iterator12.s(); !(_step12 = _iterator12.n()).done;) {
|
|
989
|
+
var m = _step12.value;
|
|
990
|
+
if (!prefixLenByWord.has(m.word)) {
|
|
991
|
+
prefixLenByWord.set(m.word, m.matchedPrefixLen);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
} catch (err) {
|
|
995
|
+
_iterator12.e(err);
|
|
996
|
+
} finally {
|
|
997
|
+
_iterator12.f();
|
|
998
|
+
}
|
|
999
|
+
return {
|
|
1000
|
+
canonicalMatched: canonicalMatched,
|
|
1001
|
+
prefixLenByWord: prefixLenByWord
|
|
1002
|
+
};
|
|
1003
|
+
};
|
|
1004
|
+
|
|
1005
|
+
/**
|
|
1006
|
+
* Recall and canonical derivation for `trimmed`, reusing the previous result
|
|
1007
|
+
* when nothing they depend on has changed.
|
|
1008
|
+
*
|
|
1009
|
+
* `predict()` runs several times for one keystroke — once when the decision
|
|
1010
|
+
* opens and again for each async evidence signal that lands inside the budget —
|
|
1011
|
+
* and only the evidence differs between those runs. Without this, every run
|
|
1012
|
+
* re-walks the tries and re-derives a canonical context per candidate.
|
|
1013
|
+
*/
|
|
1014
|
+
var getCanonicalRecall = function getCanonicalRecall(trimmed, currentWord, currentWordStart) {
|
|
1015
|
+
var surfaceCount = getCanonicalSurfaceCount();
|
|
1016
|
+
if (recallMemo !== null && recallMemo.trimmed === trimmed && recallMemo.generation === recallGeneration && recallMemo.surfaceCount === surfaceCount) {
|
|
1017
|
+
return recallMemo;
|
|
1018
|
+
}
|
|
1019
|
+
var recall = computeCanonicalRecall(trimmed, currentWord, currentWordStart);
|
|
1020
|
+
recallMemo = _objectSpread(_objectSpread({}, recall), {}, {
|
|
1021
|
+
generation: recallGeneration,
|
|
1022
|
+
surfaceCount: surfaceCount,
|
|
1023
|
+
trimmed: trimmed
|
|
1024
|
+
});
|
|
1025
|
+
return recall;
|
|
1026
|
+
};
|
|
1027
|
+
export var predict = function predict(textBefore) {
|
|
1028
|
+
var _trimmed$match$, _trimmed$match, _canonicalMatched$fin, _eligible$, _eligible$0$score, _eligible$2, _judgedEvidenceFor3, _prefixLenByWord$get2, _runtimeBySurface$get5, _judgedEvidenceFor$me3, _judgedEvidenceFor6, _contextTotals$get$le4, _contextTotals$get4, _selectedEvidence$ver, _selectedEvidence$tot, _selectedEvidence$ver2;
|
|
1029
|
+
if (!isInitialized) {
|
|
1030
|
+
// Vocabulary artifacts load asynchronously over the autocomplete gateway.
|
|
1031
|
+
// Kick off the load and skip this keystroke; the plugin also primes it on
|
|
1032
|
+
// focus, so the tries are usually ready before the user types.
|
|
1033
|
+
void loadDefaultVocabulary({
|
|
1034
|
+
source: 'predict'
|
|
1035
|
+
}).catch(function () {});
|
|
1036
|
+
return null;
|
|
1037
|
+
}
|
|
1038
|
+
var t0 = performance.now();
|
|
1039
|
+
|
|
1040
|
+
// Advance the post-accept cooldown once per prediction. It stays active until
|
|
1041
|
+
// BOTH the keystroke budget and the time window have elapsed; while active it
|
|
1042
|
+
// suppresses re-offering the just-accepted surface (see arbitration below).
|
|
1043
|
+
var cooledSurface = null;
|
|
1044
|
+
if (acceptCooldown) {
|
|
1045
|
+
acceptCooldown.predictionsSince += 1;
|
|
1046
|
+
var stillByKeys = acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES;
|
|
1047
|
+
var stillByTime = performance.now() - acceptCooldown.ts < COOLDOWN_MS;
|
|
1048
|
+
if (stillByKeys || stillByTime) {
|
|
1049
|
+
cooledSurface = acceptCooldown.surface;
|
|
1050
|
+
} else {
|
|
1051
|
+
acceptCooldown = null;
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
// ── Boundary/early-prefix prefetch; display remains gated at ≥3 ─────────
|
|
1056
|
+
if (textBefore.length > 0 && TRAILING_WHITESPACE_REGEX.test(textBefore)) {
|
|
1057
|
+
var boundaryContext = deriveWhitespaceBoundaryContext(textBefore);
|
|
1058
|
+
if (boundaryContext) {
|
|
1059
|
+
primeBoundaryLm(_objectSpread(_objectSpread({}, boundaryContext), {}, {
|
|
1060
|
+
familyKey: boundaryContext.contextKey,
|
|
1061
|
+
priority: 0
|
|
1062
|
+
}));
|
|
514
1063
|
}
|
|
1064
|
+
return null;
|
|
515
1065
|
}
|
|
1066
|
+
var trimmed = textBefore.trimEnd();
|
|
1067
|
+
var trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
|
|
1068
|
+
if (trailingSurfaceToken.length === 0) {
|
|
1069
|
+
return null;
|
|
1070
|
+
}
|
|
1071
|
+
var currentWord = trailingSurfaceToken;
|
|
1072
|
+
var currentWordStart = trimmed.length - currentWord.length;
|
|
1073
|
+
var _getCanonicalRecall = getCanonicalRecall(trimmed, currentWord, currentWordStart),
|
|
1074
|
+
canonicalMatched = _getCanonicalRecall.canonicalMatched,
|
|
1075
|
+
prefixLenByWord = _getCanonicalRecall.prefixLenByWord;
|
|
516
1076
|
|
|
517
|
-
// If
|
|
518
|
-
if (
|
|
1077
|
+
// If every trie was empty for this prefix
|
|
1078
|
+
if (canonicalMatched.length === 0) {
|
|
1079
|
+
if (isAutocompleteDebugEnabled()) {
|
|
1080
|
+
// eslint-disable-next-line no-console
|
|
1081
|
+
console.log("%c[CTC]%c \u2014 abstain: no matches for \"".concat(currentWord, "\""), CTC_STYLES.brand, CTC_STYLES.body);
|
|
1082
|
+
}
|
|
519
1083
|
return null;
|
|
520
1084
|
}
|
|
521
1085
|
var previousWord = extractPreviousWord(trimmed);
|
|
522
1086
|
var contextVector = getContextVectorForScoring(trimmed);
|
|
523
1087
|
var lmLogits = getStoredLmLogits();
|
|
524
|
-
|
|
525
|
-
// Raw LM Output Logger
|
|
526
|
-
if (isAutocompleteDebugEnabled() && lmLogits && Object.keys(lmLogits).length > 0) {
|
|
527
|
-
var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
|
|
528
|
-
return b[1] - a[1];
|
|
529
|
-
}).slice(0, 5).map(function (_ref) {
|
|
530
|
-
var _ref2 = _slicedToArray(_ref, 2),
|
|
531
|
-
word = _ref2[0],
|
|
532
|
-
score = _ref2[1];
|
|
533
|
-
return {
|
|
534
|
-
Word: word,
|
|
535
|
-
Prob: Number(score.toFixed(5))
|
|
536
|
-
};
|
|
537
|
-
});
|
|
538
|
-
// eslint-disable-next-line no-console
|
|
539
|
-
console.log('%c[Raw LM Prediction]🧠', 'color: #e83e8c; font-weight: bold;', rawLmTop);
|
|
540
|
-
}
|
|
541
1088
|
var mode = contextVector ? 'warm' : 'cold';
|
|
542
1089
|
|
|
543
|
-
// Build ScoringCandidate array from
|
|
544
|
-
var scoringCandidates =
|
|
1090
|
+
// Build ScoringCandidate array from matched terms (words + phrases)
|
|
1091
|
+
var scoringCandidates = canonicalMatched.map(function (_ref3) {
|
|
545
1092
|
var word = _ref3.word,
|
|
546
1093
|
node = _ref3.node;
|
|
547
1094
|
return {
|
|
@@ -549,98 +1096,1111 @@ export var predict = function predict(textBefore) {
|
|
|
549
1096
|
tenantFreq: node.tenantFreq,
|
|
550
1097
|
docFreq: node.docFreq,
|
|
551
1098
|
authorFreq: node.authorFreq,
|
|
552
|
-
sessionFreq: node.sessionFreq
|
|
1099
|
+
sessionFreq: node.sessionFreq,
|
|
1100
|
+
termType: node.termType,
|
|
1101
|
+
posSeq: node.posSeq,
|
|
1102
|
+
headPos: node.headPos,
|
|
1103
|
+
association: node.association
|
|
553
1104
|
};
|
|
554
1105
|
});
|
|
555
1106
|
|
|
556
1107
|
// Filter the LM payload to only words matching the current prefix so that
|
|
557
1108
|
// lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
|
|
558
1109
|
var prefix = currentWord.toLowerCase();
|
|
559
|
-
var
|
|
1110
|
+
var currentWordSeparator = (_canonicalMatched$fin = canonicalMatched.find(function (candidate) {
|
|
1111
|
+
return candidate.node.termType === 'word';
|
|
1112
|
+
})) === null || _canonicalMatched$fin === void 0 ? void 0 : _canonicalMatched$fin.separatorKind;
|
|
1113
|
+
var prefixLmLogits = lmLogits && currentWordSeparator === 'whitespace' ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
|
|
560
1114
|
var _ref5 = _slicedToArray(_ref4, 1),
|
|
561
1115
|
word = _ref5[0];
|
|
562
1116
|
return word.startsWith(prefix);
|
|
563
1117
|
})) : null;
|
|
564
|
-
var
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
var
|
|
571
|
-
var
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
1118
|
+
var canonicalScoringSupported = isCanonicalSurfaceScoringSupported();
|
|
1119
|
+
var eligibleContextKeys = Array.from(new Set(canonicalMatched.filter(function (candidate) {
|
|
1120
|
+
return candidate.canonicalTokenIds !== null;
|
|
1121
|
+
}).map(function (candidate) {
|
|
1122
|
+
return candidate.contextKey;
|
|
1123
|
+
}))).sort();
|
|
1124
|
+
var familyKey = eligibleContextKeys.join("\x01");
|
|
1125
|
+
var primeRequests = selectBoundaryPrimeRequests(familyKey, canonicalMatched, PHRASE_MAX_WORDS);
|
|
1126
|
+
var _iterator13 = _createForOfIteratorHelper(primeRequests),
|
|
1127
|
+
_step13;
|
|
1128
|
+
try {
|
|
1129
|
+
for (_iterator13.s(); !(_step13 = _iterator13.n()).done;) {
|
|
1130
|
+
var request = _step13.value;
|
|
1131
|
+
primeBoundaryLm(request);
|
|
1132
|
+
}
|
|
1133
|
+
} catch (err) {
|
|
1134
|
+
_iterator13.e(err);
|
|
1135
|
+
} finally {
|
|
1136
|
+
_iterator13.f();
|
|
1137
|
+
}
|
|
1138
|
+
var runtimeBySurface = new Map(canonicalMatched.map(function (candidate) {
|
|
1139
|
+
return [candidate.word, candidate];
|
|
1140
|
+
}));
|
|
1141
|
+
var canonicalEvidence = new Map();
|
|
1142
|
+
var firstTokenGroups = new Map();
|
|
1143
|
+
var _iterator14 = _createForOfIteratorHelper(canonicalMatched),
|
|
1144
|
+
_step14;
|
|
1145
|
+
try {
|
|
1146
|
+
for (_iterator14.s(); !(_step14 = _iterator14.n()).done;) {
|
|
1147
|
+
var _firstTokenGroups$get;
|
|
1148
|
+
var _candidate = _step14.value;
|
|
1149
|
+
if (_candidate.canonicalTokenIds === null) {
|
|
1150
|
+
continue;
|
|
1151
|
+
}
|
|
1152
|
+
// This accessor also materializes an exact cache entry when all tokens
|
|
1153
|
+
// have already been covered by shared-prefix work.
|
|
1154
|
+
getProgressiveSurfaceEvidence(_candidate.contextKey, _candidate.word);
|
|
1155
|
+
var exact = getSurfaceScore(_candidate.contextKey, _candidate.word);
|
|
1156
|
+
if (exact) {
|
|
1157
|
+
canonicalEvidence.set(_candidate.word, {
|
|
1158
|
+
contextKey: _candidate.contextKey,
|
|
1159
|
+
meanTokenLogProbability: exact.meanTokenLogProbability,
|
|
1160
|
+
score: 0,
|
|
1161
|
+
separatorKind: 'whitespace',
|
|
1162
|
+
source: 'canonical-full-surface',
|
|
1163
|
+
totalLogProbability: exact.totalLogProbability,
|
|
1164
|
+
totalSurfaceCharCount: _candidate.word.length,
|
|
1165
|
+
totalTokenCount: exact.tokenCount,
|
|
1166
|
+
verifiedCharCount: _candidate.word.length,
|
|
1167
|
+
verifiedTokenCount: exact.tokenCount
|
|
1168
|
+
});
|
|
1169
|
+
continue;
|
|
1170
|
+
}
|
|
1171
|
+
var _group2 = (_firstTokenGroups$get = firstTokenGroups.get(_candidate.contextKey)) !== null && _firstTokenGroups$get !== void 0 ? _firstTokenGroups$get : [];
|
|
1172
|
+
_group2.push(_candidate);
|
|
1173
|
+
firstTokenGroups.set(_candidate.contextKey, _group2);
|
|
1174
|
+
}
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
_iterator14.e(err);
|
|
1177
|
+
} finally {
|
|
1178
|
+
_iterator14.f();
|
|
1179
|
+
}
|
|
1180
|
+
var _iterator15 = _createForOfIteratorHelper(firstTokenGroups),
|
|
1181
|
+
_step15;
|
|
1182
|
+
try {
|
|
1183
|
+
for (_iterator15.s(); !(_step15 = _iterator15.n()).done;) {
|
|
1184
|
+
var _step15$value = _slicedToArray(_step15.value, 2),
|
|
1185
|
+
_contextKey = _step15$value[0],
|
|
1186
|
+
_group3 = _step15$value[1];
|
|
1187
|
+
var boundary = getBoundaryLmState(_contextKey);
|
|
1188
|
+
if (!boundary) {
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
var maxLogit = -Infinity;
|
|
1192
|
+
var _iterator22 = _createForOfIteratorHelper(_group3),
|
|
1193
|
+
_step22;
|
|
1194
|
+
try {
|
|
1195
|
+
for (_iterator22.s(); !(_step22 = _iterator22.n()).done;) {
|
|
1196
|
+
var _candidate2$canonical;
|
|
1197
|
+
var _candidate2 = _step22.value;
|
|
1198
|
+
var tokenId = (_candidate2$canonical = _candidate2.canonicalTokenIds) === null || _candidate2$canonical === void 0 ? void 0 : _candidate2$canonical[0];
|
|
1199
|
+
var rawLogit = tokenId === undefined ? undefined : boundary.rawLogits[tokenId];
|
|
1200
|
+
if (rawLogit !== undefined && Number.isFinite(rawLogit) && rawLogit > maxLogit) {
|
|
1201
|
+
maxLogit = rawLogit;
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
} catch (err) {
|
|
1205
|
+
_iterator22.e(err);
|
|
1206
|
+
} finally {
|
|
1207
|
+
_iterator22.f();
|
|
1208
|
+
}
|
|
1209
|
+
if (!Number.isFinite(maxLogit)) {
|
|
1210
|
+
continue;
|
|
1211
|
+
}
|
|
1212
|
+
var _iterator23 = _createForOfIteratorHelper(_group3),
|
|
1213
|
+
_step23;
|
|
1214
|
+
try {
|
|
1215
|
+
for (_iterator23.s(); !(_step23 = _iterator23.n()).done;) {
|
|
1216
|
+
var _candidate3$canonical, _candidate3$canonical2, _candidate3$canonical3;
|
|
1217
|
+
var _candidate3 = _step23.value;
|
|
1218
|
+
var _tokenId = (_candidate3$canonical = _candidate3.canonicalTokenIds) === null || _candidate3$canonical === void 0 ? void 0 : _candidate3$canonical[0];
|
|
1219
|
+
var _rawLogit = _tokenId === undefined ? undefined : boundary.rawLogits[_tokenId];
|
|
1220
|
+
if (_rawLogit === undefined || !Number.isFinite(_rawLogit)) {
|
|
1221
|
+
continue;
|
|
1222
|
+
}
|
|
1223
|
+
var tokenCount = (_candidate3$canonical2 = (_candidate3$canonical3 = _candidate3.canonicalTokenIds) === null || _candidate3$canonical3 === void 0 ? void 0 : _candidate3$canonical3.length) !== null && _candidate3$canonical2 !== void 0 ? _candidate3$canonical2 : 0;
|
|
1224
|
+
if (_tokenId !== undefined && tokenCount === 1) {
|
|
1225
|
+
var totalLogProbability = logSoftmaxAt(boundary.rawLogits, _tokenId);
|
|
1226
|
+
if (!Number.isFinite(totalLogProbability)) {
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
1229
|
+
canonicalEvidence.set(_candidate3.word, {
|
|
1230
|
+
contextKey: _contextKey,
|
|
1231
|
+
meanTokenLogProbability: totalLogProbability,
|
|
1232
|
+
score: 0,
|
|
1233
|
+
separatorKind: 'whitespace',
|
|
1234
|
+
source: 'canonical-full-surface',
|
|
1235
|
+
totalLogProbability: totalLogProbability,
|
|
1236
|
+
totalSurfaceCharCount: _candidate3.word.length,
|
|
1237
|
+
totalTokenCount: 1,
|
|
1238
|
+
verifiedCharCount: _candidate3.word.length,
|
|
1239
|
+
verifiedTokenCount: 1
|
|
1240
|
+
});
|
|
1241
|
+
continue;
|
|
1242
|
+
}
|
|
1243
|
+
canonicalEvidence.set(_candidate3.word, {
|
|
1244
|
+
contextKey: _contextKey,
|
|
1245
|
+
rawLogit: _rawLogit,
|
|
1246
|
+
score: 0,
|
|
1247
|
+
separatorKind: 'whitespace',
|
|
1248
|
+
source: 'canonical-first-token',
|
|
1249
|
+
totalSurfaceCharCount: _candidate3.word.length,
|
|
1250
|
+
totalTokenCount: tokenCount,
|
|
1251
|
+
verifiedCharCount: 0,
|
|
1252
|
+
verifiedTokenCount: 1
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
} catch (err) {
|
|
1256
|
+
_iterator23.e(err);
|
|
1257
|
+
} finally {
|
|
1258
|
+
_iterator23.f();
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
} catch (err) {
|
|
1262
|
+
_iterator15.e(err);
|
|
1263
|
+
} finally {
|
|
1264
|
+
_iterator15.f();
|
|
1265
|
+
}
|
|
1266
|
+
var _rankCandidates = rankCandidates(scoringCandidates, contextVector, function (w) {
|
|
1267
|
+
return getWordVector(w);
|
|
1268
|
+
}, prefixLmLogits, {
|
|
1269
|
+
word: wordTrie.maxTenantFreq,
|
|
1270
|
+
bigram: maxBigramFreq,
|
|
1271
|
+
phrase: maxPhraseFreq
|
|
1272
|
+
}, previousWord, function (surface) {
|
|
1273
|
+
var _canonicalEvidence$ge;
|
|
1274
|
+
return (_canonicalEvidence$ge = canonicalEvidence.get(surface)) !== null && _canonicalEvidence$ge !== void 0 ? _canonicalEvidence$ge : null;
|
|
1275
|
+
}),
|
|
1276
|
+
ranked = _rankCandidates.candidates,
|
|
1277
|
+
grammarMeta = _rankCandidates.grammarMeta,
|
|
1278
|
+
pipelineDebug = _rankCandidates.pipelineDebug;
|
|
1279
|
+
var progressiveEligible = ranked.filter(function (candidate) {
|
|
1280
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1281
|
+
return (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined;
|
|
1282
|
+
});
|
|
1283
|
+
if (canonicalScoringSupported && progressiveEligible.length > 0) {
|
|
1284
|
+
var byContext = new Map();
|
|
1285
|
+
var _iterator16 = _createForOfIteratorHelper(progressiveEligible),
|
|
1286
|
+
_step16;
|
|
1287
|
+
try {
|
|
1288
|
+
for (_iterator16.s(); !(_step16 = _iterator16.n()).done;) {
|
|
1289
|
+
var _byContext$get;
|
|
1290
|
+
var candidate = _step16.value;
|
|
1291
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1292
|
+
if (!runtime || runtime.canonicalTokenIds === null) {
|
|
1293
|
+
continue;
|
|
1294
|
+
}
|
|
1295
|
+
var group = (_byContext$get = byContext.get(runtime.contextKey)) !== null && _byContext$get !== void 0 ? _byContext$get : [];
|
|
1296
|
+
group.push({
|
|
1297
|
+
runtime: runtime,
|
|
1298
|
+
rankHint: candidate.stage1Score
|
|
1299
|
+
});
|
|
1300
|
+
byContext.set(runtime.contextKey, group);
|
|
1301
|
+
}
|
|
1302
|
+
} catch (err) {
|
|
1303
|
+
_iterator16.e(err);
|
|
1304
|
+
} finally {
|
|
1305
|
+
_iterator16.f();
|
|
1306
|
+
}
|
|
1307
|
+
var _iterator17 = _createForOfIteratorHelper(byContext),
|
|
1308
|
+
_step17;
|
|
1309
|
+
try {
|
|
1310
|
+
for (_iterator17.s(); !(_step17 = _iterator17.n()).done;) {
|
|
1311
|
+
var _step17$value = _slicedToArray(_step17.value, 2),
|
|
1312
|
+
contextKey = _step17$value[0],
|
|
1313
|
+
_group = _step17$value[1];
|
|
1314
|
+
requestProgressiveSurfaceScores({
|
|
1315
|
+
familyKey: familyKey,
|
|
1316
|
+
contextKey: contextKey,
|
|
1317
|
+
prompt: _group[0].runtime.contextBeforeSurface,
|
|
1318
|
+
candidates: _group.map(function (_ref6) {
|
|
1319
|
+
var _runtime$canonicalTok;
|
|
1320
|
+
var runtime = _ref6.runtime,
|
|
1321
|
+
rankHint = _ref6.rankHint;
|
|
1322
|
+
return {
|
|
1323
|
+
surface: runtime.word,
|
|
1324
|
+
tokenIds: (_runtime$canonicalTok = runtime.canonicalTokenIds) !== null && _runtime$canonicalTok !== void 0 ? _runtime$canonicalTok : [],
|
|
1325
|
+
rankHint: rankHint
|
|
1326
|
+
};
|
|
1327
|
+
})
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
} catch (err) {
|
|
1331
|
+
_iterator17.e(err);
|
|
1332
|
+
} finally {
|
|
1333
|
+
_iterator17.f();
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
// The one/two-character passes exist only to hide model latency. They may
|
|
1338
|
+
// prime and expand token groups above, but cannot commit UI.
|
|
1339
|
+
var displayPrefixReady = currentWord.length >= DISPLAY_MIN_PREFIX_LENGTH;
|
|
1340
|
+
|
|
1341
|
+
// ── Arbitration: full-surface evidence → plausibility → winner margin ──
|
|
1342
|
+
var canonicalLmSupported = canonicalScoringSupported;
|
|
1343
|
+
var hasExactEvidence = function hasExactEvidence(candidate) {
|
|
1344
|
+
var _candidate$lmEvidence;
|
|
1345
|
+
return ((_candidate$lmEvidence = candidate.lmEvidence) === null || _candidate$lmEvidence === void 0 ? void 0 : _candidate$lmEvidence.source) === 'canonical-full-surface' && candidate.lmEvidence.meanTokenLogProbability !== undefined && Number.isFinite(candidate.lmEvidence.meanTokenLogProbability);
|
|
1346
|
+
};
|
|
1347
|
+
|
|
1348
|
+
/**
|
|
1349
|
+
* The evidence this candidate is judged on: the summed log-probability of
|
|
1350
|
+
* the tokens that have been read, their per-token mean, and whether they
|
|
1351
|
+
* cover the whole surface.
|
|
1352
|
+
*
|
|
1353
|
+
* `total` is the quantity the posterior is built from — it is a sequence
|
|
1354
|
+
* log-likelihood, so it is comparable across surfaces of different lengths
|
|
1355
|
+
* in a way the mean is not. `mean` is retained for reporting and for the
|
|
1356
|
+
* oracle arms, which compare aggregation rules against each other.
|
|
1357
|
+
*
|
|
1358
|
+
* Every token read costs a round trip, so a surface longer than
|
|
1359
|
+
* `REQUIRED_VERIFIED_TOKENS` is judged on its leading tokens and its tail is
|
|
1360
|
+
* left unread. Callers compensate with a wider winner margin rather than
|
|
1361
|
+
* treating the two kinds of evidence as equivalent.
|
|
1362
|
+
*
|
|
1363
|
+
* This is the depth-unaware read, and it is what the pool below is built
|
|
1364
|
+
* from. Arbitration goes through `judgedEvidenceFor`, which adds the one
|
|
1365
|
+
* requirement that can only be stated once the pool is known: a surface
|
|
1366
|
+
* cannot be judged against members it has not yet been read level with.
|
|
1367
|
+
*/
|
|
1368
|
+
var readJudgedEvidence = function readJudgedEvidence(candidate) {
|
|
1369
|
+
var evidence = candidate.lmEvidence;
|
|
1370
|
+
var exactMean = evidence === null || evidence === void 0 ? void 0 : evidence.meanTokenLogProbability;
|
|
1371
|
+
if (hasExactEvidence(candidate) && evidence && exactMean !== undefined) {
|
|
1372
|
+
var _evidence$totalLogPro;
|
|
1373
|
+
var _total = (_evidence$totalLogPro = evidence.totalLogProbability) !== null && _evidence$totalLogPro !== void 0 ? _evidence$totalLogPro : exactMean * evidence.totalTokenCount;
|
|
1374
|
+
return Number.isFinite(_total) ? {
|
|
1375
|
+
complete: true,
|
|
1376
|
+
mean: exactMean,
|
|
1377
|
+
total: _total
|
|
1378
|
+
} : null;
|
|
1379
|
+
}
|
|
1380
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1381
|
+
if (!runtime || runtime.canonicalTokenIds === null) {
|
|
1382
|
+
return null;
|
|
1383
|
+
}
|
|
1384
|
+
var progressive = getProgressiveSurfaceEvidence(runtime.contextKey, candidate.word);
|
|
1385
|
+
if (!progressive || progressive.scoredTokenCount < REQUIRED_VERIFIED_TOKENS) {
|
|
1386
|
+
return null;
|
|
1387
|
+
}
|
|
1388
|
+
var total = progressive.totalLogProbability;
|
|
1389
|
+
var mean = total / progressive.scoredTokenCount;
|
|
1390
|
+
return Number.isFinite(mean) && Number.isFinite(total) ? {
|
|
1391
|
+
complete: false,
|
|
1392
|
+
mean: mean,
|
|
1393
|
+
total: total
|
|
1394
|
+
} : null;
|
|
1395
|
+
};
|
|
1396
|
+
|
|
1397
|
+
// Normalise each context's shortlist into a posterior. `total` is a sequence
|
|
1398
|
+
// log-likelihood, so `exp(total - logSumExp(totals))` is the share of the
|
|
1399
|
+
// shortlist's probability mass the model puts on that surface.
|
|
1400
|
+
//
|
|
1401
|
+
// Unknown future token log-probabilities cannot exceed zero, so a partially
|
|
1402
|
+
// read candidate's total is an upper bound on its eventual total, and its
|
|
1403
|
+
// posterior an upper bound on its eventual posterior. That is what lets the
|
|
1404
|
+
// plausibility floor and the display gate be the same threshold — one
|
|
1405
|
+
// applied to the bound, one to the verified value — instead of two numbers
|
|
1406
|
+
// that have to be kept consistent by hand.
|
|
1407
|
+
//
|
|
1408
|
+
// Candidates with no evidence at all are left out of the sum rather than
|
|
1409
|
+
// defaulted. Any default would have to stand in for a log-probability, and a
|
|
1410
|
+
// value low enough not to distort the normaliser is indistinguishable from
|
|
1411
|
+
// omitting the candidate.
|
|
1412
|
+
var optimisticTotalByWord = new Map();
|
|
1413
|
+
var contextTotals = new Map();
|
|
1414
|
+
var contextSurfaces = new Map();
|
|
1415
|
+
// Counted before the total is looked up, so this is every candidate the
|
|
1416
|
+
// context could have scored rather than every candidate it did. The gap
|
|
1417
|
+
// between the two is what separates a context holding one match from a
|
|
1418
|
+
// context whose rivals have not come back yet.
|
|
1419
|
+
var contextRequestedCount = new Map();
|
|
1420
|
+
var bestTotalByContext = new Map();
|
|
1421
|
+
var bestCandidateByContext = new Map();
|
|
1422
|
+
var _iterator18 = _createForOfIteratorHelper(ranked),
|
|
1423
|
+
_step18;
|
|
1424
|
+
try {
|
|
1425
|
+
for (_iterator18.s(); !(_step18 = _iterator18.n()).done;) {
|
|
1426
|
+
var _contextRequestedCoun2, _judged$total;
|
|
1427
|
+
var _candidate4 = _step18.value;
|
|
1428
|
+
var _runtime = runtimeBySurface.get(_candidate4.word);
|
|
1429
|
+
if (!_runtime || _runtime.canonicalTokenIds === null) {
|
|
1430
|
+
continue;
|
|
1431
|
+
}
|
|
1432
|
+
contextRequestedCount.set(_runtime.contextKey, ((_contextRequestedCoun2 = contextRequestedCount.get(_runtime.contextKey)) !== null && _contextRequestedCoun2 !== void 0 ? _contextRequestedCoun2 : 0) + 1);
|
|
1433
|
+
var judged = readJudgedEvidence(_candidate4);
|
|
1434
|
+
var progressive = getProgressiveSurfaceEvidence(_runtime.contextKey, _candidate4.word);
|
|
1435
|
+
// Once a candidate has been read far enough to judge, its verified total
|
|
1436
|
+
// replaces the bound.
|
|
1437
|
+
var optimisticTotal = (_judged$total = judged === null || judged === void 0 ? void 0 : judged.total) !== null && _judged$total !== void 0 ? _judged$total : progressive === null || progressive === void 0 ? void 0 : progressive.totalLogProbability;
|
|
1438
|
+
if (optimisticTotal === undefined || !Number.isFinite(optimisticTotal)) {
|
|
1439
|
+
continue;
|
|
1440
|
+
}
|
|
1441
|
+
optimisticTotalByWord.set(_candidate4.word, optimisticTotal);
|
|
1442
|
+
var totals = contextTotals.get(_runtime.contextKey);
|
|
1443
|
+
var surfaces = contextSurfaces.get(_runtime.contextKey);
|
|
1444
|
+
if (totals && surfaces) {
|
|
1445
|
+
totals.push(optimisticTotal);
|
|
1446
|
+
surfaces.push(_candidate4.word);
|
|
1447
|
+
} else {
|
|
1448
|
+
contextTotals.set(_runtime.contextKey, [optimisticTotal]);
|
|
1449
|
+
contextSurfaces.set(_runtime.contextKey, [_candidate4.word]);
|
|
1450
|
+
}
|
|
1451
|
+
var best = bestTotalByContext.get(_runtime.contextKey);
|
|
1452
|
+
if (best === undefined || optimisticTotal > best) {
|
|
1453
|
+
bestTotalByContext.set(_runtime.contextKey, optimisticTotal);
|
|
1454
|
+
bestCandidateByContext.set(_runtime.contextKey, _candidate4);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
/**
|
|
1459
|
+
* Whether a shorter member of the same pool is this surface's word-boundary
|
|
1460
|
+
* prefix — `happy path` where `happy` is also present.
|
|
1461
|
+
*
|
|
1462
|
+
* Only a multi-word surface can extend another, and a pool is a set, so this
|
|
1463
|
+
* costs one lookup per space rather than a comparison against every member.
|
|
1464
|
+
*/
|
|
1465
|
+
} catch (err) {
|
|
1466
|
+
_iterator18.e(err);
|
|
1467
|
+
} finally {
|
|
1468
|
+
_iterator18.f();
|
|
1469
|
+
}
|
|
1470
|
+
var extendsAPoolMember = function extendsAPoolMember(surface, pool) {
|
|
1471
|
+
for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
|
|
1472
|
+
if (pool.has(surface.slice(0, space))) {
|
|
1473
|
+
return true;
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
return false;
|
|
1477
|
+
};
|
|
1478
|
+
|
|
1479
|
+
// A sequence total only falls as more of the sequence is read, so a partial
|
|
1480
|
+
// sum over `k` tokens sits above the same surface's total over `m > k`. An
|
|
1481
|
+
// extension read less far than the member it continues therefore carries a
|
|
1482
|
+
// total above that member's — and because the chain rule narrows the
|
|
1483
|
+
// denominator to the pool's minimal members, `exp(total - normaliser)` comes
|
|
1484
|
+
// out above 1 for a surface whose own continuation has not been looked at.
|
|
1485
|
+
// It needs a three-token first word to happen, which 2.6% of served bigrams
|
|
1486
|
+
// have, and it inflates both the posterior the gate reads and the
|
|
1487
|
+
// `confidenceScore` every other candidate's margin is measured against.
|
|
1488
|
+
//
|
|
1489
|
+
// A surface's canonical tokenization begins with the tokenization of its own
|
|
1490
|
+
// leading words — the producer encodes `" " + surface` in one call and the
|
|
1491
|
+
// ByteLevel pre-tokenizer splits on word boundaries, so no merge crosses a
|
|
1492
|
+
// space — which is what makes both corrections below exact rather than
|
|
1493
|
+
// approximate.
|
|
1494
|
+
//
|
|
1495
|
+
// The total is capped at what it continues: `P(happy path)` cannot exceed
|
|
1496
|
+
// `P(happy)`. Shortest-first, so a three-word phrase is capped against a
|
|
1497
|
+
// two-word prefix that has already been capped itself.
|
|
1498
|
+
//
|
|
1499
|
+
// And the surface is held unjudgeable until it has been read level with what
|
|
1500
|
+
// it continues, because the cap alone would leave it looking exactly as
|
|
1501
|
+
// certain as its prefix while saying nothing about its own tail.
|
|
1502
|
+
var underReadExtensions = new Set();
|
|
1503
|
+
var _iterator19 = _createForOfIteratorHelper(contextSurfaces),
|
|
1504
|
+
_step19;
|
|
1505
|
+
try {
|
|
1506
|
+
for (_iterator19.s(); !(_step19 = _iterator19.n()).done;) {
|
|
1507
|
+
var _step19$value = _slicedToArray(_step19.value, 2),
|
|
1508
|
+
_contextKey2 = _step19$value[0],
|
|
1509
|
+
_surfaces = _step19$value[1];
|
|
1510
|
+
var pool = new Set(_surfaces);
|
|
1511
|
+
var _iterator24 = _createForOfIteratorHelper(_toConsumableArray(_surfaces).sort(function (a, b) {
|
|
1512
|
+
return a.length - b.length;
|
|
1513
|
+
})),
|
|
1514
|
+
_step24;
|
|
1515
|
+
try {
|
|
1516
|
+
for (_iterator24.s(); !(_step24 = _iterator24.n()).done;) {
|
|
1517
|
+
var _getProgressiveSurfac, _getProgressiveSurfac2, _runtimeBySurface$get8, _runtimeBySurface$get9;
|
|
1518
|
+
var surface = _step24.value;
|
|
1519
|
+
var own = optimisticTotalByWord.get(surface);
|
|
1520
|
+
if (own === undefined) {
|
|
1521
|
+
continue;
|
|
1522
|
+
}
|
|
1523
|
+
var cap = Infinity;
|
|
1524
|
+
var continuedDepth = 0;
|
|
1525
|
+
for (var space = surface.indexOf(' '); space !== -1; space = surface.indexOf(' ', space + 1)) {
|
|
1526
|
+
var _runtimeBySurface$get6, _runtimeBySurface$get7;
|
|
1527
|
+
var _prefix = surface.slice(0, space);
|
|
1528
|
+
if (!pool.has(_prefix)) {
|
|
1529
|
+
continue;
|
|
1530
|
+
}
|
|
1531
|
+
var prefixTotal = optimisticTotalByWord.get(_prefix);
|
|
1532
|
+
if (prefixTotal !== undefined && prefixTotal < cap) {
|
|
1533
|
+
cap = prefixTotal;
|
|
1534
|
+
}
|
|
1535
|
+
var prefixDepth = (_runtimeBySurface$get6 = (_runtimeBySurface$get7 = runtimeBySurface.get(_prefix)) === null || _runtimeBySurface$get7 === void 0 || (_runtimeBySurface$get7 = _runtimeBySurface$get7.canonicalTokenIds) === null || _runtimeBySurface$get7 === void 0 ? void 0 : _runtimeBySurface$get7.length) !== null && _runtimeBySurface$get6 !== void 0 ? _runtimeBySurface$get6 : 0;
|
|
1536
|
+
if (prefixDepth > continuedDepth) {
|
|
1537
|
+
continuedDepth = prefixDepth;
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
if (own > cap) {
|
|
1541
|
+
optimisticTotalByWord.set(surface, cap);
|
|
1542
|
+
}
|
|
1543
|
+
var scoredDepth = (_getProgressiveSurfac = (_getProgressiveSurfac2 = getProgressiveSurfaceEvidence(_contextKey2, surface)) === null || _getProgressiveSurfac2 === void 0 ? void 0 : _getProgressiveSurfac2.scoredTokenCount) !== null && _getProgressiveSurfac !== void 0 ? _getProgressiveSurfac : 0;
|
|
1544
|
+
var surfaceDepth = (_runtimeBySurface$get8 = (_runtimeBySurface$get9 = runtimeBySurface.get(surface)) === null || _runtimeBySurface$get9 === void 0 || (_runtimeBySurface$get9 = _runtimeBySurface$get9.canonicalTokenIds) === null || _runtimeBySurface$get9 === void 0 ? void 0 : _runtimeBySurface$get9.length) !== null && _runtimeBySurface$get8 !== void 0 ? _runtimeBySurface$get8 : 0;
|
|
1545
|
+
if (scoredDepth < continuedDepth && scoredDepth < surfaceDepth) {
|
|
1546
|
+
underReadExtensions.add(surface);
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
} catch (err) {
|
|
1550
|
+
_iterator24.e(err);
|
|
1551
|
+
} finally {
|
|
1552
|
+
_iterator24.f();
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
/**
|
|
1557
|
+
* The evidence arbitration judges a candidate on.
|
|
1558
|
+
*
|
|
1559
|
+
* Wraps the depth-unaware read with the pool-relative requirement: a surface
|
|
1560
|
+
* that continues another pool member is not judgeable until it has been read
|
|
1561
|
+
* level with it, so the two totals being compared are taken at comparable
|
|
1562
|
+
* depths.
|
|
1563
|
+
*/
|
|
1564
|
+
} catch (err) {
|
|
1565
|
+
_iterator19.e(err);
|
|
1566
|
+
} finally {
|
|
1567
|
+
_iterator19.f();
|
|
1568
|
+
}
|
|
1569
|
+
var judgedEvidenceFor = function judgedEvidenceFor(candidate) {
|
|
1570
|
+
return underReadExtensions.has(candidate.word) ? null : readJudgedEvidence(candidate);
|
|
1571
|
+
};
|
|
1572
|
+
var hasJudgeableEvidence = function hasJudgeableEvidence(candidate) {
|
|
1573
|
+
return judgedEvidenceFor(candidate) !== null;
|
|
1574
|
+
};
|
|
1575
|
+
|
|
1576
|
+
// Normalise over the pool's minimal members only, and score everyone —
|
|
1577
|
+
// extensions included — against that.
|
|
1578
|
+
//
|
|
1579
|
+
// A sequence total is its prefix's total plus a log-probability that cannot
|
|
1580
|
+
// be positive, so summing a surface and its own extension double-counts:
|
|
1581
|
+
// `exp(total)` for `happy` already contains every continuation of `happy`,
|
|
1582
|
+
// `happy path` among them. Dividing one pool's mass between the two capped
|
|
1583
|
+
// the shorter form at half however certain it was, and the longer form below
|
|
1584
|
+
// that — both under gates neither could then clear, for a reason that was
|
|
1585
|
+
// arithmetic rather than evidence.
|
|
1586
|
+
//
|
|
1587
|
+
// Excluding extensions from the denominator is the chain rule: the minimal
|
|
1588
|
+
// members partition the mass, and an extension's share of that same
|
|
1589
|
+
// denominator is `P(prefix) * P(continuation | prefix)`, which is exactly
|
|
1590
|
+
// what a multi-word suggestion should be held to. Extensions stay in
|
|
1591
|
+
// `contextTotals`, so how contested a context is still counts every scored
|
|
1592
|
+
// candidate.
|
|
1593
|
+
var logSumExpByContext = new Map();
|
|
1594
|
+
var _iterator20 = _createForOfIteratorHelper(contextSurfaces),
|
|
1595
|
+
_step20;
|
|
1596
|
+
try {
|
|
1597
|
+
for (_iterator20.s(); !(_step20 = _iterator20.n()).done;) {
|
|
1598
|
+
var _step20$value = _slicedToArray(_step20.value, 2),
|
|
1599
|
+
_contextKey3 = _step20$value[0],
|
|
1600
|
+
_surfaces2 = _step20$value[1];
|
|
1601
|
+
var _pool = new Set(_surfaces2);
|
|
1602
|
+
var minimalTotals = [];
|
|
1603
|
+
var _iterator25 = _createForOfIteratorHelper(_surfaces2),
|
|
1604
|
+
_step25;
|
|
1605
|
+
try {
|
|
1606
|
+
for (_iterator25.s(); !(_step25 = _iterator25.n()).done;) {
|
|
1607
|
+
var _surface = _step25.value;
|
|
1608
|
+
var total = optimisticTotalByWord.get(_surface);
|
|
1609
|
+
if (total !== undefined && !extendsAPoolMember(_surface, _pool)) {
|
|
1610
|
+
minimalTotals.push(total);
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
// An extension is strictly longer than what it extends, so the shortest
|
|
1614
|
+
// member of any non-empty pool is always minimal and this is never empty.
|
|
1615
|
+
} catch (err) {
|
|
1616
|
+
_iterator25.e(err);
|
|
1617
|
+
} finally {
|
|
1618
|
+
_iterator25.f();
|
|
1619
|
+
}
|
|
1620
|
+
logSumExpByContext.set(_contextKey3, logSumExp(minimalTotals));
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
/**
|
|
1624
|
+
* A candidate's share of its context shortlist's probability mass.
|
|
1625
|
+
*
|
|
1626
|
+
* Pass an optimistic (partially read) total for an upper bound on the
|
|
1627
|
+
* eventual posterior, or a verified total for the posterior itself.
|
|
1628
|
+
*/
|
|
1629
|
+
} catch (err) {
|
|
1630
|
+
_iterator20.e(err);
|
|
1631
|
+
} finally {
|
|
1632
|
+
_iterator20.f();
|
|
1633
|
+
}
|
|
1634
|
+
var posteriorFor = function posteriorFor(candidate, total) {
|
|
1635
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1636
|
+
var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
|
|
1637
|
+
if (total === undefined || normalizer === undefined || !Number.isFinite(normalizer)) {
|
|
1638
|
+
return 0;
|
|
1639
|
+
}
|
|
1640
|
+
return Math.exp(total - normalizer);
|
|
1641
|
+
};
|
|
1642
|
+
var optimisticPosterior = function optimisticPosterior(candidate) {
|
|
1643
|
+
return posteriorFor(candidate, optimisticTotalByWord.get(candidate.word));
|
|
1644
|
+
};
|
|
1645
|
+
var judgedPosterior = function judgedPosterior(candidate) {
|
|
1646
|
+
var _judgedEvidenceFor;
|
|
1647
|
+
return posteriorFor(candidate, (_judgedEvidenceFor = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor === void 0 ? void 0 : _judgedEvidenceFor.total);
|
|
1648
|
+
};
|
|
1649
|
+
|
|
1650
|
+
// A candidate stays in the race for as long as its optimistic posterior
|
|
1651
|
+
// could still clear the gate it will be held to. The bound only falls as
|
|
1652
|
+
// more tokens are read, so nothing dropped here could have gone on to win.
|
|
1653
|
+
var couldClearGate = function couldClearGate(candidate) {
|
|
1654
|
+
return optimisticPosterior(candidate) >= MIN_LM_POSTERIOR[candidate.termType];
|
|
1655
|
+
};
|
|
1656
|
+
var passesLmFloor = function passesLmFloor(candidate) {
|
|
1657
|
+
return judgedEvidenceFor(candidate) !== null && couldClearGate(candidate);
|
|
1658
|
+
};
|
|
1659
|
+
var hasRequiredEvidence = function hasRequiredEvidence(candidate) {
|
|
1660
|
+
var _candidate$lmEvidence2;
|
|
1661
|
+
if (canonicalLmSupported) {
|
|
1662
|
+
return passesLmFloor(candidate);
|
|
1663
|
+
}
|
|
1664
|
+
return candidate.termType === 'word' && ((_candidate$lmEvidence2 = candidate.lmEvidence) === null || _candidate$lmEvidence2 === void 0 ? void 0 : _candidate$lmEvidence2.source) === 'network-logit';
|
|
1665
|
+
};
|
|
1666
|
+
|
|
1667
|
+
// ── Stabilization guards (QI-2) ──────────────────────────────────────────
|
|
1668
|
+
// Whole-surface repetition: drop a candidate whose full surface already equals
|
|
1669
|
+
// the run of tokens immediately before the cursor — accepting it would
|
|
1670
|
+
// duplicate what was just typed (`end to end` → `end to end to end`).
|
|
1671
|
+
var beforeTokens = tokenize(trimmed);
|
|
1672
|
+
var duplicatesPreceding = function duplicatesPreceding(c) {
|
|
1673
|
+
var surfaceTokens = tokenize(c.word);
|
|
1674
|
+
var n = surfaceTokens.length;
|
|
1675
|
+
if (n === 0 || beforeTokens.length < n) {
|
|
1676
|
+
return false;
|
|
1677
|
+
}
|
|
1678
|
+
for (var i = 0; i < n; i++) {
|
|
1679
|
+
if (beforeTokens[beforeTokens.length - n + i] !== surfaceTokens[i]) {
|
|
1680
|
+
return false;
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
return true;
|
|
1684
|
+
};
|
|
1685
|
+
// Post-accept cooldown: don't re-offer the surface the user just accepted while
|
|
1686
|
+
// its cooldown is still active (see cooldown advance near t0).
|
|
1687
|
+
var isCooledDown = function isCooledDown(c) {
|
|
1688
|
+
return cooledSurface !== null && c.word.toLowerCase() === cooledSurface;
|
|
1689
|
+
};
|
|
1690
|
+
var isStabilized = function isStabilized(candidate) {
|
|
1691
|
+
return !duplicatesPreceding(candidate) && !isCooledDown(candidate);
|
|
1692
|
+
};
|
|
1693
|
+
// A candidate the user has all but finished typing is not worth a ghost, and
|
|
1694
|
+
// a candidate we would never show is not worth waiting for either.
|
|
1695
|
+
var suggestionLengthFor = function suggestionLengthFor(candidate) {
|
|
1696
|
+
var _prefixLenByWord$get;
|
|
1697
|
+
return candidate.word.length - ((_prefixLenByWord$get = prefixLenByWord.get(candidate.word)) !== null && _prefixLenByWord$get !== void 0 ? _prefixLenByWord$get : currentWord.length);
|
|
1698
|
+
};
|
|
1699
|
+
var isWorthShowing = function isWorthShowing(candidate) {
|
|
1700
|
+
return suggestionLengthFor(candidate) >= MIN_SUGGESTION_LENGTH;
|
|
1701
|
+
};
|
|
1702
|
+
var evidenceBacked = ranked.filter(function (candidate) {
|
|
1703
|
+
return hasRequiredEvidence(candidate) && isStabilized(candidate);
|
|
1704
|
+
});
|
|
1705
|
+
var confidenceScore = function confidenceScore(candidate) {
|
|
1706
|
+
if (!canonicalLmSupported) {
|
|
1707
|
+
return candidate.finalScore;
|
|
1708
|
+
}
|
|
1709
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1710
|
+
if ((runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) === null || !optimisticTotalByWord.has(candidate.word)) {
|
|
1711
|
+
return candidate.stage1Score;
|
|
1712
|
+
}
|
|
1713
|
+
return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * optimisticPosterior(candidate);
|
|
1714
|
+
};
|
|
1715
|
+
var exactConfidenceScore = function exactConfidenceScore(candidate) {
|
|
1716
|
+
if (!canonicalLmSupported || judgedEvidenceFor(candidate) === null) {
|
|
1717
|
+
return candidate.finalScore;
|
|
1718
|
+
}
|
|
1719
|
+
return STAGE1_WEIGHT * candidate.stage1Score + STAGE2_WEIGHT * judgedPosterior(candidate);
|
|
1720
|
+
};
|
|
1721
|
+
|
|
1722
|
+
/**
|
|
1723
|
+
* How many candidates in this one's context were scored at all.
|
|
1724
|
+
*
|
|
1725
|
+
* Read off pool membership, which counts every candidate that contributed a
|
|
1726
|
+
* total, rather than off the normaliser, which the chain rule narrows to the
|
|
1727
|
+
* pool's minimal members.
|
|
1728
|
+
*/
|
|
1729
|
+
var scoredPoolSize = function scoredPoolSize(candidate) {
|
|
1730
|
+
var _runtimeBySurface$get, _contextTotals$get$le, _contextTotals$get;
|
|
1731
|
+
var contextKey = (_runtimeBySurface$get = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get === void 0 ? void 0 : _runtimeBySurface$get.contextKey;
|
|
1732
|
+
return contextKey === undefined ? 0 : (_contextTotals$get$le = (_contextTotals$get = contextTotals.get(contextKey)) === null || _contextTotals$get === void 0 ? void 0 : _contextTotals$get.length) !== null && _contextTotals$get$le !== void 0 ? _contextTotals$get$le : 0;
|
|
1733
|
+
};
|
|
1734
|
+
|
|
1735
|
+
/**
|
|
1736
|
+
* How many candidates the context held before any of them were scored.
|
|
1737
|
+
*
|
|
1738
|
+
* A pool of one means two opposite things. Either the vocabulary offered a
|
|
1739
|
+
* single continuation of what was typed, which is the least contested case
|
|
1740
|
+
* there is; or rivals were offered and have not been read yet, which is the
|
|
1741
|
+
* least informed. Only the second deserves refusing, and the two are
|
|
1742
|
+
* distinguishable exactly here.
|
|
1743
|
+
*/
|
|
1744
|
+
var requestedPoolSize = function requestedPoolSize(candidate) {
|
|
1745
|
+
var _runtimeBySurface$get2, _contextRequestedCoun;
|
|
1746
|
+
var contextKey = (_runtimeBySurface$get2 = runtimeBySurface.get(candidate.word)) === null || _runtimeBySurface$get2 === void 0 ? void 0 : _runtimeBySurface$get2.contextKey;
|
|
1747
|
+
return contextKey === undefined ? 0 : (_contextRequestedCoun = contextRequestedCount.get(contextKey)) !== null && _contextRequestedCoun !== void 0 ? _contextRequestedCoun : 0;
|
|
1748
|
+
};
|
|
1749
|
+
|
|
1750
|
+
// A suppressed short completion still counts as competition below, so losing
|
|
1751
|
+
// to one abstains rather than promoting the runner-up in its place.
|
|
1752
|
+
var gateCleared = evidenceBacked.filter(isWorthShowing).map(function (candidate) {
|
|
1753
|
+
return {
|
|
1754
|
+
candidate: candidate,
|
|
1755
|
+
posterior: judgedPosterior(candidate),
|
|
1756
|
+
score: exactConfidenceScore(candidate)
|
|
1757
|
+
};
|
|
1758
|
+
})
|
|
1759
|
+
// The gate is the model's own confidence in the surface; the blended score
|
|
1760
|
+
// only orders what has already cleared it, so a strong corpus prior can no
|
|
1761
|
+
// longer carry a surface the model is unsure of onto the screen.
|
|
1762
|
+
.filter(function (_ref7) {
|
|
1763
|
+
var candidate = _ref7.candidate,
|
|
1764
|
+
posterior = _ref7.posterior;
|
|
1765
|
+
return posterior >= MIN_LM_POSTERIOR[candidate.termType];
|
|
1766
|
+
});
|
|
1767
|
+
/**
|
|
1768
|
+
* Whether the model finds the surface plausible on its own terms.
|
|
1769
|
+
*
|
|
1770
|
+
* The gate above asks which candidate won its pool; this asks whether
|
|
1771
|
+
* winning it meant anything. The two come apart exactly where a pool is
|
|
1772
|
+
* thin — a shortlist of one hands its only member 1.0 by construction — and
|
|
1773
|
+
* a relative test can never see through that however it is tuned.
|
|
1774
|
+
*
|
|
1775
|
+
* Waived without canonical scoring, where there is no judged mean to test
|
|
1776
|
+
* and the network-logit path would otherwise refuse everything.
|
|
1777
|
+
*/
|
|
1778
|
+
var isPlausibleSurface = function isPlausibleSurface(candidate) {
|
|
1779
|
+
var _judgedEvidenceFor2;
|
|
1780
|
+
if (!canonicalLmSupported) {
|
|
1781
|
+
return true;
|
|
1782
|
+
}
|
|
1783
|
+
var mean = (_judgedEvidenceFor2 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor2 === void 0 ? void 0 : _judgedEvidenceFor2.mean;
|
|
1784
|
+
return mean !== undefined && mean >= MIN_MEAN_TOKEN_LOG_PROBABILITY;
|
|
1785
|
+
};
|
|
1786
|
+
// Kept as its own stage rather than folded into the gate so the three
|
|
1787
|
+
// populations stay separable in the abstain cascade: cleared its pool and was
|
|
1788
|
+
// implausible, was plausible and had nothing to clear the pool against, or
|
|
1789
|
+
// cleared both. A candidate refused here stays in `ranked` and so still
|
|
1790
|
+
// counts as competition below — promoting the runner-up in place of an
|
|
1791
|
+
// implausible leader would show something worse, not something better.
|
|
1792
|
+
var plausible = gateCleared.filter(function (_ref8) {
|
|
1793
|
+
var candidate = _ref8.candidate;
|
|
1794
|
+
return isPlausibleSurface(candidate);
|
|
1795
|
+
});
|
|
1796
|
+
// Applied after the gate rather than folded into it, so the two populations
|
|
1797
|
+
// stay separable: a surface refused here cleared its threshold and was
|
|
1798
|
+
// refused for having had nothing to clear it against.
|
|
1799
|
+
var eligible = plausible.filter(function (_ref9) {
|
|
1800
|
+
var candidate = _ref9.candidate;
|
|
1801
|
+
return !canonicalLmSupported || scoredPoolSize(candidate) >= MIN_SCORED_POOL_SIZE || requestedPoolSize(candidate) <= 1;
|
|
1802
|
+
}).sort(function (a, b) {
|
|
1803
|
+
return b.score - a.score;
|
|
1804
|
+
});
|
|
1805
|
+
var selected = (_eligible$ = eligible[0]) === null || _eligible$ === void 0 ? void 0 : _eligible$.candidate;
|
|
1806
|
+
var selectedDisplayScore = (_eligible$0$score = (_eligible$2 = eligible[0]) === null || _eligible$2 === void 0 ? void 0 : _eligible$2.score) !== null && _eligible$0$score !== void 0 ? _eligible$0$score : 0;
|
|
1807
|
+
var selectedEvidence = selected && hasRequiredEvidence(selected) ? selected.lmEvidence : null;
|
|
1808
|
+
/**
|
|
1809
|
+
* Whether `longer` is `shorter` continued past a word boundary, in the same
|
|
1810
|
+
* normaliser.
|
|
1811
|
+
*
|
|
1812
|
+
* Shared by the two places that have to agree on what an extension is: the
|
|
1813
|
+
* chain rule, which stops dividing a pool's mass between a sequence and its
|
|
1814
|
+
* own prefix, and the winner margin below, which stops treating one as the
|
|
1815
|
+
* other's rival. They were the same category error and are now the same test.
|
|
1816
|
+
*/
|
|
1817
|
+
var extendsInSamePool = function extendsInSamePool(shorter, longer) {
|
|
1818
|
+
var _runtimeBySurface$get3, _runtimeBySurface$get4;
|
|
1819
|
+
if (longer.word === shorter.word || !optimisticTotalByWord.has(longer.word)) {
|
|
1820
|
+
return false;
|
|
1821
|
+
}
|
|
1822
|
+
var pool = (_runtimeBySurface$get3 = runtimeBySurface.get(shorter.word)) === null || _runtimeBySurface$get3 === void 0 ? void 0 : _runtimeBySurface$get3.contextKey;
|
|
1823
|
+
if (pool === undefined || ((_runtimeBySurface$get4 = runtimeBySurface.get(longer.word)) === null || _runtimeBySurface$get4 === void 0 ? void 0 : _runtimeBySurface$get4.contextKey) !== pool) {
|
|
1824
|
+
return false;
|
|
1825
|
+
}
|
|
1826
|
+
return longer.word.length > shorter.word.length && longer.word.startsWith(shorter.word) && longer.word[shorter.word.length] === ' ';
|
|
1827
|
+
};
|
|
1828
|
+
// A candidate that extends the selection is not competing with it. Whichever
|
|
1829
|
+
// continuation the user meant, the shorter surface is a correct ghost — it is
|
|
1830
|
+
// the prefix of both — so the margin has nothing to arbitrate and charging the
|
|
1831
|
+
// selection for its own extension abstains on a case that could not be wrong.
|
|
1832
|
+
// The chain rule already removed this from the denominator; leaving it in the
|
|
1833
|
+
// margin meant a nested leader cleared its gate and then lost to the candidate
|
|
1834
|
+
// that had just been freed from dividing its mass. It penalised bigrams and
|
|
1835
|
+
// phrases specifically, because a nested pool is where they mostly appear.
|
|
1836
|
+
var runnerUpScore = selected ? Math.max.apply(Math, _toConsumableArray(ranked.filter(function (candidate) {
|
|
1837
|
+
return candidate.word !== selected.word && isStabilized(candidate) && !extendsInSamePool(selected, candidate);
|
|
1838
|
+
}).map(confidenceScore)).concat([0])) : 0;
|
|
1839
|
+
var winnerMargin = selected ? selectedDisplayScore - runnerUpScore : 0;
|
|
1840
|
+
var hasMissingArtifactCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
|
|
1841
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1842
|
+
return (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) === 'whitespace' && runtime.canonicalTokenIds === null;
|
|
1843
|
+
});
|
|
1844
|
+
// A candidate with canonical tokens but no evidence at all cannot join the
|
|
1845
|
+
// posterior: it has no likelihood to contribute, and any stand-in value would
|
|
1846
|
+
// distort the normaliser for every other candidate in its context. Absent
|
|
1847
|
+
// evidence is not evidence of absence though — the context's boundary may
|
|
1848
|
+
// simply not be primed yet — so it holds the ghost back instead of being
|
|
1849
|
+
// silently ignored. In a primed context every candidate has at least
|
|
1850
|
+
// first-token evidence, so this only fires while a context is still cold.
|
|
1851
|
+
var hasUnscoredCompetitor = canonicalLmSupported && ranked.some(function (candidate) {
|
|
1852
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1853
|
+
return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !optimisticTotalByWord.has(candidate.word) && isStabilized(candidate) && isWorthShowing(candidate);
|
|
1854
|
+
});
|
|
1855
|
+
// A rival only holds the ghost back while it is still too unread to judge.
|
|
1856
|
+
// Under full-surface evidence that meant any long surface blocked everything
|
|
1857
|
+
// until every one of its tokens had been paid for, which abstained far more
|
|
1858
|
+
// often than it ever changed the winner. A rival now becomes judgeable after
|
|
1859
|
+
// `REQUIRED_VERIFIED_TOKENS`, so this settles within the budget.
|
|
1860
|
+
var hasUnresolvedPotential = canonicalLmSupported && ranked.some(function (candidate) {
|
|
1861
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
1862
|
+
return candidate.word !== (selected === null || selected === void 0 ? void 0 : selected.word) && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== null && (runtime === null || runtime === void 0 ? void 0 : runtime.canonicalTokenIds) !== undefined && !hasJudgeableEvidence(candidate) && couldClearGate(candidate) && isStabilized(candidate) && isWorthShowing(candidate);
|
|
1863
|
+
});
|
|
1864
|
+
var requiredWinnerMargin = MIN_WINNER_MARGIN + (selected && ((_judgedEvidenceFor3 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor3 === void 0 ? void 0 : _judgedEvidenceFor3.complete) === false ? PARTIAL_EVIDENCE_MARGIN_PREMIUM : 0);
|
|
1865
|
+
// Everything the display decision needs except the veto itself. Splitting it
|
|
1866
|
+
// out is what lets the veto be priced: the value here at the moment the veto
|
|
1867
|
+
// fires is exactly the ghost the veto cost us.
|
|
1868
|
+
var clearsMarginBeforeColdVeto = selected !== undefined && !hasMissingArtifactCompetitor && winnerMargin >= requiredWinnerMargin;
|
|
1869
|
+
var clearsWinnerMargin = clearsMarginBeforeColdVeto && !hasUnscoredCompetitor;
|
|
1870
|
+
var suggestion = displayPrefixReady && selected && clearsWinnerMargin ? selected.word.slice((_prefixLenByWord$get2 = prefixLenByWord.get(selected.word)) !== null && _prefixLenByWord$get2 !== void 0 ? _prefixLenByWord$get2 : currentWord.length) : null;
|
|
1871
|
+
|
|
1872
|
+
// Which constraint bound. Computed on every evaluation rather than only under
|
|
1873
|
+
// debug, because it is counted for the session as well as printed, and
|
|
1874
|
+
// deriving the debug header from it keeps the two from disagreeing.
|
|
1875
|
+
var resolveAbstainReason = function resolveAbstainReason() {
|
|
1876
|
+
if (suggestion !== null && suggestion.length > 0) {
|
|
1877
|
+
return null;
|
|
1878
|
+
}
|
|
1879
|
+
if (!displayPrefixReady) {
|
|
1880
|
+
return 'prefetch';
|
|
1881
|
+
}
|
|
1882
|
+
if (ranked.length === 0) {
|
|
1883
|
+
return 'no-candidate';
|
|
1884
|
+
}
|
|
1885
|
+
if (hasMissingArtifactCompetitor) {
|
|
1886
|
+
return 'missing-artifact';
|
|
1887
|
+
}
|
|
1888
|
+
if (hasUnscoredCompetitor) {
|
|
1889
|
+
return 'cold-competitor';
|
|
1890
|
+
}
|
|
1891
|
+
if (hasUnresolvedPotential && !clearsWinnerMargin) {
|
|
1892
|
+
return 'unresolved-rival';
|
|
1893
|
+
}
|
|
1894
|
+
if (eligible.length > 0) {
|
|
1895
|
+
return selected && !clearsWinnerMargin ? 'winner-margin' : 'empty-completion';
|
|
1896
|
+
}
|
|
1897
|
+
if (plausible.length > 0) {
|
|
1898
|
+
// Cleared its gate, was plausible on its own terms, and was refused for
|
|
1899
|
+
// the shortlist it cleared that gate against holding nobody else.
|
|
1900
|
+
return 'lone-candidate';
|
|
1901
|
+
}
|
|
1902
|
+
if (gateCleared.length > 0) {
|
|
1903
|
+
// Held the largest share of its shortlist and still was not a surface
|
|
1904
|
+
// the model found plausible — which is only possible because a share is
|
|
1905
|
+
// relative and this is not.
|
|
1906
|
+
return 'implausible-surface';
|
|
1907
|
+
}
|
|
1908
|
+
if (evidenceBacked.length > 0 && evidenceBacked.every(function (candidate) {
|
|
1909
|
+
return !isWorthShowing(candidate);
|
|
1910
|
+
})) {
|
|
1911
|
+
return 'short-completion';
|
|
1912
|
+
}
|
|
1913
|
+
// The floor and the gate are one threshold, applied to the bound and to
|
|
1914
|
+
// the verified value, so a candidate short of it is dropped before
|
|
1915
|
+
// `evidenceBacked` is built. Asking whether anything was judgeable
|
|
1916
|
+
// separates a gate that rejected read candidates from a context where
|
|
1917
|
+
// nothing was read at all — the first is a threshold to calibrate, the
|
|
1918
|
+
// second is scheduling.
|
|
1919
|
+
return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
|
|
1920
|
+
};
|
|
1921
|
+
var abstainReason = resolveAbstainReason();
|
|
1922
|
+
|
|
1923
|
+
// The leader is the best-supported candidate, not the selected one: an
|
|
1924
|
+
// evaluation that showed nothing is exactly the one whose posterior needs
|
|
1925
|
+
// recording, and it has no selection to report.
|
|
1926
|
+
var posteriorLeader = ranked.reduce(function (best, candidate) {
|
|
1927
|
+
var posterior = optimisticPosterior(candidate);
|
|
1928
|
+
return posterior > 0 && (best === null || posterior > best.posterior) ? {
|
|
1929
|
+
candidate: candidate,
|
|
1930
|
+
posterior: posterior
|
|
1931
|
+
} : best;
|
|
1932
|
+
}, null);
|
|
1933
|
+
/**
|
|
1934
|
+
* The longer surface in the leader's normaliser that extends the given
|
|
1935
|
+
* candidate, if the pool holds one.
|
|
1936
|
+
*
|
|
1937
|
+
* `logSumExp` divides one pool's mass between a sequence and its own prefix,
|
|
1938
|
+
* and the longer total is the shorter one plus a log-probability that cannot
|
|
1939
|
+
* be positive — so the longer form holds at most half, or at most a third
|
|
1940
|
+
* where both shorter forms are present. Both sit under the gate it is held
|
|
1941
|
+
* to, so it cannot display however certain its continuation is.
|
|
1942
|
+
*
|
|
1943
|
+
* Asked from the shorter form because that is the one that leads: the
|
|
1944
|
+
* extending candidate is strictly lower in the same pool and so can never be
|
|
1945
|
+
* the leader.
|
|
1946
|
+
*/
|
|
1947
|
+
var poolExtensionOf = function poolExtensionOf(shorter) {
|
|
1948
|
+
return ranked.find(function (candidate) {
|
|
1949
|
+
return extendsInSamePool(shorter, candidate);
|
|
1950
|
+
});
|
|
1951
|
+
};
|
|
1952
|
+
if (isAutocompleteDebugEnabled()) {
|
|
1953
|
+
var _posteriorLeader$cand, _posteriorLeader$post, _selectedEvidence$mea, _selectedEvidence$mea2, _acceptCooldown$predi, _acceptCooldown;
|
|
1954
|
+
var verbose = isAutocompleteDebugVerbose();
|
|
1955
|
+
var latencyMs = performance.now() - t0;
|
|
1956
|
+
var tokens = tokenize(trimmed);
|
|
1957
|
+
var contextWords = tokens.slice(-CONTEXT_WORDS);
|
|
1958
|
+
|
|
1959
|
+
// Decision drives the (collapsed) group header. Abstains carry the reason
|
|
1960
|
+
// the session counters recorded, plus the numbers behind it, so it is clear
|
|
1961
|
+
// *why* nothing showed without expanding the group.
|
|
1962
|
+
var abstainDetail = {
|
|
1963
|
+
'below-posterior-gate': "posterior below ".concat(MIN_LM_POSTERIOR[((_posteriorLeader$cand = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.candidate) !== null && _posteriorLeader$cand !== void 0 ? _posteriorLeader$cand : ranked[0]).termType], " (best ").concat(((_posteriorLeader$post = posteriorLeader === null || posteriorLeader === void 0 ? void 0 : posteriorLeader.posterior) !== null && _posteriorLeader$post !== void 0 ? _posteriorLeader$post : 0).toFixed(2), ")"),
|
|
1964
|
+
'cold-competitor': 'competitor has no LM evidence yet',
|
|
1965
|
+
'empty-completion': 'empty completion',
|
|
1966
|
+
'implausible-surface': "mean per-token log-probability below ".concat(MIN_MEAN_TOKEN_LOG_PROBABILITY, " (best ").concat(gateCleared.map(function (_ref0) {
|
|
1967
|
+
var _judgedEvidenceFor$me, _judgedEvidenceFor4;
|
|
1968
|
+
var candidate = _ref0.candidate;
|
|
1969
|
+
return (_judgedEvidenceFor$me = (_judgedEvidenceFor4 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor4 === void 0 ? void 0 : _judgedEvidenceFor4.mean) !== null && _judgedEvidenceFor$me !== void 0 ? _judgedEvidenceFor$me : -Infinity;
|
|
1970
|
+
}).reduce(function (best, mean) {
|
|
1971
|
+
return Math.max(best, mean);
|
|
1972
|
+
}, -Infinity).toFixed(2), ")"),
|
|
1973
|
+
'lone-candidate': "cleared its gate against fewer than ".concat(MIN_SCORED_POOL_SIZE, " scored candidates, rivals unread"),
|
|
1974
|
+
'missing-artifact': 'canonical artifact coverage missing',
|
|
1975
|
+
'no-candidate': 'nothing cleared scoring',
|
|
1976
|
+
'no-evidence': 'full-surface evidence absent',
|
|
1977
|
+
prefetch: "prefetch: ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " chars"),
|
|
1978
|
+
'short-completion': "completion shorter than ".concat(MIN_SUGGESTION_LENGTH, " chars"),
|
|
1979
|
+
'unresolved-rival': 'expanding plausible token-prefix groups',
|
|
1980
|
+
'winner-margin': "winner margin ".concat(winnerMargin.toFixed(2), " < ").concat(requiredWinnerMargin.toFixed(2))
|
|
1981
|
+
};
|
|
1982
|
+
var decision = abstainReason === null ? "\u2728 \"".concat(suggestion !== null && suggestion !== void 0 ? suggestion : '', "\"") : abstainReason === 'prefetch' ? "\u23F3 ".concat(abstainDetail.prefetch) : "\u2014 abstain: ".concat(abstainDetail[abstainReason]);
|
|
1983
|
+
lastPredictionDebug = {
|
|
1984
|
+
awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
|
|
1985
|
+
textBefore: trimmed,
|
|
1986
|
+
currentWord: currentWord,
|
|
1987
|
+
decision: decision,
|
|
1988
|
+
mode: mode,
|
|
1989
|
+
contextWords: contextWords,
|
|
1990
|
+
topCandidates: ranked.slice(0, 5).map(function (r) {
|
|
1991
|
+
return {
|
|
1992
|
+
word: r.word,
|
|
1993
|
+
finalScore: r.finalScore,
|
|
1994
|
+
semanticScore: r.semanticScore,
|
|
1995
|
+
freqScore: r.freqScore,
|
|
1996
|
+
lmScore: r.lmScore
|
|
590
1997
|
};
|
|
591
1998
|
}),
|
|
592
1999
|
suggestion: suggestion && suggestion.length > 0 ? suggestion : null
|
|
593
2000
|
};
|
|
594
2001
|
|
|
595
|
-
//
|
|
2002
|
+
// Mode describes the semantic-vector source without implying that a
|
|
2003
|
+
// network backend ran; the snapshot may come from the local MLC embedder.
|
|
596
2004
|
var slowLaneVec = getStoredContextVector();
|
|
597
2005
|
var isUsingSlowLaneVector = slowLaneVec !== null && vectorStore !== null && slowLaneVec.length === vectorStore.dim;
|
|
598
|
-
var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(
|
|
599
|
-
var modeColor = modeLabel === 'WARM(
|
|
2006
|
+
var modeLabel = !contextVector ? 'COLD' : isUsingSlowLaneVector ? 'WARM(semantic)' : 'WARM(local-avg)';
|
|
2007
|
+
var modeColor = modeLabel === 'WARM(semantic)' ? CTC_STYLES.good : modeLabel === 'WARM(local-avg)' ? CTC_STYLES.warn : CTC_STYLES.cold;
|
|
600
2008
|
|
|
601
|
-
//
|
|
2009
|
+
// Group header: [CTC:<mode>] <decision> · <mode> · <type> · <latency>
|
|
2010
|
+
// arbitrationMode ('v1') tags the line so this deploy's logs diff cleanly
|
|
2011
|
+
// against the baseline deploy's.
|
|
602
2012
|
// eslint-disable-next-line no-console
|
|
603
|
-
console.groupCollapsed("%c[
|
|
2013
|
+
console.groupCollapsed("%c[CTC:".concat(ARBITRATION_MODE, "]%c ").concat(decision, " %c\xB7 ").concat(modeLabel).concat(selected ? " \xB7 ".concat(selected.termType) : '', " \xB7 \u23F1 ").concat(latencyMs.toFixed(1), "ms"), CTC_STYLES.brand, CTC_STYLES.body, modeColor);
|
|
2014
|
+
var semanticInput = getStoredContextInput();
|
|
2015
|
+
var causalPrompts = primeRequests.map(function (request, index) {
|
|
2016
|
+
var tail = request.prompt.slice(-DEBUG_TEXT_TAIL_CHARS);
|
|
2017
|
+
return "#".concat(index + 1, ":").concat(JSON.stringify(tail));
|
|
2018
|
+
});
|
|
604
2019
|
|
|
605
|
-
//
|
|
606
|
-
|
|
607
|
-
|
|
2020
|
+
// Keep the exact model inputs separate from normalized lookup tokens.
|
|
2021
|
+
ctcSection('INPUT', "raw: ".concat(JSON.stringify(trimmed.slice(-DEBUG_TEXT_TAIL_CHARS)), " \xB7 prev: ").concat(previousWord ? JSON.stringify(previousWord) : '—'));
|
|
2022
|
+
ctcSection('LOOKUP', "normalized(last ".concat(CONTEXT_WORDS, "): ").concat(contextWords.length ? JSON.stringify(contextWords.join(' ')) : '(none)'));
|
|
2023
|
+
ctcSection('SEMANTIC', "snapshot: ".concat(semanticInput ? JSON.stringify(semanticInput.slice(-DEBUG_TEXT_TAIL_CHARS)) : '(unavailable)'));
|
|
2024
|
+
ctcSection('CAUSAL', "prompt".concat(causalPrompts.length === 1 ? '' : 's', ": ").concat(causalPrompts.join(' · ') || '(not ready)'));
|
|
608
2025
|
|
|
609
|
-
//
|
|
610
|
-
var
|
|
611
|
-
var
|
|
612
|
-
|
|
613
|
-
|
|
2026
|
+
// SIGNALS — semantic state plus context-keyed canonical LM coverage.
|
|
2027
|
+
var logitCount = lmLogits ? Object.keys(lmLogits).length : 0;
|
|
2028
|
+
var semanticStatus = isUsingSlowLaneVector ? "semantic \u2705 snapshot(".concat(slowLaneVec === null || slowLaneVec === void 0 ? void 0 : slowLaneVec.length, "d)") : vectorStore ? 'semantic ⚠️ local-avg' : 'semantic ❌ cold';
|
|
2029
|
+
var tierACount = ranked.filter(function (candidate) {
|
|
2030
|
+
var _candidate$lmEvidence3;
|
|
2031
|
+
return ((_candidate$lmEvidence3 = candidate.lmEvidence) === null || _candidate$lmEvidence3 === void 0 ? void 0 : _candidate$lmEvidence3.source) === 'canonical-first-token';
|
|
2032
|
+
}).length;
|
|
2033
|
+
var exactCount = ranked.filter(function (candidate) {
|
|
2034
|
+
var _candidate$lmEvidence4;
|
|
2035
|
+
return ((_candidate$lmEvidence4 = candidate.lmEvidence) === null || _candidate$lmEvidence4 === void 0 ? void 0 : _candidate$lmEvidence4.source) === 'canonical-full-surface';
|
|
2036
|
+
}).length;
|
|
2037
|
+
var boundaryReady = primeRequests.filter(function (request) {
|
|
2038
|
+
return getBoundaryLmState(request.contextKey) !== null;
|
|
2039
|
+
}).length;
|
|
2040
|
+
var artifactEligible = canonicalMatched.filter(function (candidate) {
|
|
2041
|
+
return candidate.canonicalTokenIds !== null;
|
|
2042
|
+
}).length;
|
|
2043
|
+
var documentStartFallbacks = canonicalMatched.filter(function (candidate) {
|
|
2044
|
+
return candidate.separatorKind === 'document-start';
|
|
2045
|
+
}).length;
|
|
2046
|
+
var punctuationFallbacks = canonicalMatched.filter(function (candidate) {
|
|
2047
|
+
return candidate.separatorKind === 'non-space';
|
|
2048
|
+
}).length;
|
|
2049
|
+
var logitsStatus = tierACount + exactCount > 0 ? "canonical \u2705 first-token:".concat(tierACount, " exact:").concat(exactCount) : logitCount > 0 ? "network logits \u2705 ".concat(logitCount) : 'canonical ⏳/absent';
|
|
2050
|
+
var formula = tierACount + exactCount + logitCount > 0 ? "Stage1\xD7".concat(STAGE1_WEIGHT, "+LM\xD7").concat(STAGE2_WEIGHT) : 'Stage1-only';
|
|
2051
|
+
ctcSection('SIGNALS', "".concat(semanticStatus, " \xB7 ").concat(logitsStatus, " \xB7 ").concat(formula));
|
|
2052
|
+
ctcSection('CANONICAL', "eligible ".concat(artifactEligible, "/").concat(canonicalMatched.length, " \xB7 separator fallback doc:").concat(documentStartFallbacks, " punct:").concat(punctuationFallbacks, " \xB7 contexts ").concat(boundaryReady, "/").concat(primeRequests.length, " ready (\u2264").concat(PHRASE_MAX_WORDS, ") \xB7 family=").concat(familyKey.slice(0, 72) || 'none'));
|
|
2053
|
+
if (verbose && logitCount > 0 && lmLogits) {
|
|
2054
|
+
var rawLmTop = Object.entries(lmLogits).sort(function (a, b) {
|
|
2055
|
+
return b[1] - a[1];
|
|
2056
|
+
}).slice(0, 5).map(function (_ref1) {
|
|
2057
|
+
var _ref10 = _slicedToArray(_ref1, 2),
|
|
2058
|
+
word = _ref10[0],
|
|
2059
|
+
score = _ref10[1];
|
|
2060
|
+
return "".concat(word, ":").concat(score.toFixed(3));
|
|
2061
|
+
}).join(', ');
|
|
2062
|
+
ctcSection(' rawLM', "\uD83E\uDDE0 ".concat(rawLmTop));
|
|
2063
|
+
}
|
|
614
2064
|
|
|
615
|
-
//
|
|
616
|
-
var
|
|
617
|
-
|
|
618
|
-
|
|
2065
|
+
// GENERATE — how many candidates matched, split by term type
|
|
2066
|
+
var genByType = {
|
|
2067
|
+
word: 0,
|
|
2068
|
+
bigram: 0,
|
|
2069
|
+
phrase: 0
|
|
2070
|
+
};
|
|
2071
|
+
var _iterator21 = _createForOfIteratorHelper(canonicalMatched),
|
|
2072
|
+
_step21;
|
|
2073
|
+
try {
|
|
2074
|
+
for (_iterator21.s(); !(_step21 = _iterator21.n()).done;) {
|
|
2075
|
+
var m = _step21.value;
|
|
2076
|
+
genByType[m.node.termType] += 1;
|
|
2077
|
+
}
|
|
2078
|
+
} catch (err) {
|
|
2079
|
+
_iterator21.e(err);
|
|
2080
|
+
} finally {
|
|
2081
|
+
_iterator21.f();
|
|
2082
|
+
}
|
|
2083
|
+
ctcSection('GENERATE', "matched ".concat(canonicalMatched.length, " \u2192 word:").concat(genByType.word, " bigram:").concat(genByType.bigram, " phrase:").concat(genByType.phrase).concat(canonicalLmSupported ? ' · display needs exact surface evidence' : ''));
|
|
619
2084
|
|
|
620
|
-
//
|
|
2085
|
+
// SCORE — Stage-1 + grammar funnel and canonical evidence depth.
|
|
2086
|
+
var lmCoverage = ranked.slice(0, 10).filter(function (r) {
|
|
2087
|
+
return r.lmScore > 0;
|
|
2088
|
+
}).length;
|
|
2089
|
+
ctcSection('SCORE', "in ".concat(pipelineDebug.initial, " \u2192 stage1(<").concat(MIN_STAGE1_SCORE, ") \u2212").concat(pipelineDebug.stage1Rejected.length, " \u2192 grammar \u2212").concat(pipelineDebug.grammarRejected.length, " \u2192 final ").concat(pipelineDebug.final, " \xB7 LM cov ").concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " \xB7 first-token ").concat(tierACount, " \xB7 exact ").concat(exactCount));
|
|
621
2090
|
if (grammarMeta) {
|
|
622
|
-
|
|
623
|
-
console.log("%c[Grammar] \"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] \u2192 ").concat(grammarMeta.before, " candidates \u2192 ").concat(grammarMeta.after, " after filter"), 'color: #4caf50; font-weight: bold;');
|
|
624
|
-
if (grammarMeta.dropped.length > 0) {
|
|
625
|
-
// eslint-disable-next-line no-console
|
|
626
|
-
console.log("%c\uD83D\uDEAB Dropped: ".concat(grammarMeta.dropped.join(', ')), 'color: #f44336; font-style: italic;');
|
|
627
|
-
}
|
|
2091
|
+
ctcSection(' grammar', "\"".concat(grammarMeta.prevWord, "\" [").concat(grammarMeta.prevTags.join('|'), "] ").concat(grammarMeta.before, "\u2192").concat(grammarMeta.after).concat(verbose && grammarMeta.dropped.length > 0 ? " \xB7 dropped: ".concat(grammarMeta.dropped.join(', ')) : ''));
|
|
628
2092
|
}
|
|
2093
|
+
var gateReasonFor = function gateReasonFor(candidate) {
|
|
2094
|
+
var runtime = runtimeBySurface.get(candidate.word);
|
|
2095
|
+
if ((runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace') {
|
|
2096
|
+
var _runtime$separatorKin;
|
|
2097
|
+
return "separator:".concat((_runtime$separatorKin = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin !== void 0 ? _runtime$separatorKin : 'unknown');
|
|
2098
|
+
}
|
|
2099
|
+
if (runtime.canonicalTokenIds === null) {
|
|
2100
|
+
return 'missing-artifact';
|
|
2101
|
+
}
|
|
2102
|
+
if (!hasJudgeableEvidence(candidate)) {
|
|
2103
|
+
return "awaiting-tokens:".concat(REQUIRED_VERIFIED_TOKENS);
|
|
2104
|
+
}
|
|
2105
|
+
if (!passesLmFloor(candidate)) {
|
|
2106
|
+
return "bound:".concat(optimisticPosterior(candidate).toFixed(2), "<").concat(MIN_LM_POSTERIOR[candidate.termType]);
|
|
2107
|
+
}
|
|
2108
|
+
if (duplicatesPreceding(candidate)) {
|
|
2109
|
+
return 'repetition';
|
|
2110
|
+
}
|
|
2111
|
+
if (isCooledDown(candidate)) {
|
|
2112
|
+
return 'cooldown';
|
|
2113
|
+
}
|
|
2114
|
+
if (!isWorthShowing(candidate)) {
|
|
2115
|
+
return "suffix:".concat(suggestionLengthFor(candidate), "<").concat(MIN_SUGGESTION_LENGTH);
|
|
2116
|
+
}
|
|
2117
|
+
var floor = MIN_LM_POSTERIOR[candidate.termType];
|
|
2118
|
+
var posterior = judgedPosterior(candidate);
|
|
2119
|
+
if (posterior < floor) {
|
|
2120
|
+
return "posterior:".concat(posterior.toFixed(2), "<").concat(floor);
|
|
2121
|
+
}
|
|
2122
|
+
if (!isPlausibleSurface(candidate)) {
|
|
2123
|
+
var _judgedEvidenceFor$me2, _judgedEvidenceFor5;
|
|
2124
|
+
return "implausible:".concat(((_judgedEvidenceFor$me2 = (_judgedEvidenceFor5 = judgedEvidenceFor(candidate)) === null || _judgedEvidenceFor5 === void 0 ? void 0 : _judgedEvidenceFor5.mean) !== null && _judgedEvidenceFor$me2 !== void 0 ? _judgedEvidenceFor$me2 : NaN).toFixed(2), "<").concat(MIN_MEAN_TOKEN_LOG_PROBABILITY);
|
|
2125
|
+
}
|
|
2126
|
+
return 'eligible';
|
|
2127
|
+
};
|
|
2128
|
+
var gateFunnel = function gateFunnel(termType) {
|
|
2129
|
+
var typeRanked = ranked.filter(function (candidate) {
|
|
2130
|
+
return candidate.termType === termType;
|
|
2131
|
+
});
|
|
2132
|
+
var exact = typeRanked.filter(hasJudgeableEvidence);
|
|
2133
|
+
var absolute = exact.filter(passesLmFloor);
|
|
2134
|
+
var stabilized = absolute.filter(isStabilized);
|
|
2135
|
+
var longEnough = stabilized.filter(isWorthShowing);
|
|
2136
|
+
var floorPassed = longEnough.filter(function (candidate) {
|
|
2137
|
+
return judgedPosterior(candidate) >= MIN_LM_POSTERIOR[termType];
|
|
2138
|
+
});
|
|
2139
|
+
var plausiblePassed = floorPassed.filter(isPlausibleSurface);
|
|
2140
|
+
return "".concat(termType, " m:").concat(genByType[termType], " r:").concat(typeRanked.length, " exact:").concat(exact.length, " abs:").concat(absolute.length, " stable:").concat(stabilized.length, " suffix:").concat(longEnough.length, " floor:").concat(floorPassed.length, " plausible:").concat(plausiblePassed.length, " eligible:").concat(eligible.filter(function (_ref11) {
|
|
2141
|
+
var candidate = _ref11.candidate;
|
|
2142
|
+
return candidate.termType === termType;
|
|
2143
|
+
}).length);
|
|
2144
|
+
};
|
|
2145
|
+
ctcSection('GATES', "".concat(gateFunnel('word'), " | ").concat(gateFunnel('bigram'), " | ").concat(gateFunnel('phrase')));
|
|
629
2146
|
|
|
630
|
-
//
|
|
631
|
-
//
|
|
632
|
-
|
|
2147
|
+
// One line per context: how many candidates share the normaliser, and how
|
|
2148
|
+
// much of the mass the leader holds. A leader well under its threshold
|
|
2149
|
+
// means the context is contested, which is the abstention we want.
|
|
2150
|
+
var contextLeaders = Array.from(bestCandidateByContext.entries()).slice(0, PHRASE_MAX_WORDS).map(function (_ref12) {
|
|
2151
|
+
var _contextTotals$get$le2, _contextTotals$get2;
|
|
2152
|
+
var _ref13 = _slicedToArray(_ref12, 2),
|
|
2153
|
+
contextKey = _ref13[0],
|
|
2154
|
+
leader = _ref13[1];
|
|
2155
|
+
var shortlistSize = (_contextTotals$get$le2 = (_contextTotals$get2 = contextTotals.get(contextKey)) === null || _contextTotals$get2 === void 0 ? void 0 : _contextTotals$get2.length) !== null && _contextTotals$get$le2 !== void 0 ? _contextTotals$get$le2 : 0;
|
|
2156
|
+
var evidenceKind = hasExactEvidence(leader) ? 'exact' : 'upper';
|
|
2157
|
+
return "".concat(contextKey.slice(0, 32), " \u2192 n=").concat(shortlistSize, " ").concat(leader.termType, ":\"").concat(leader.word, "\" ").concat(evidenceKind, " p=").concat(optimisticPosterior(leader).toFixed(3), "/").concat(MIN_LM_POSTERIOR[leader.termType]);
|
|
2158
|
+
}).join(' | ');
|
|
2159
|
+
ctcSection('NORMALIZE', contextLeaders.length > 0 ? contextLeaders : 'no scored candidate in any context');
|
|
2160
|
+
ctcSection('READINESS', "prefix ".concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH, " \xB7 boundary ").concat(boundaryReady, "/").concat(primeRequests.length, " \xB7 exact word:").concat(ranked.filter(function (candidate) {
|
|
2161
|
+
return candidate.termType === 'word' && hasExactEvidence(candidate);
|
|
2162
|
+
}).length, " bigram:").concat(ranked.filter(function (candidate) {
|
|
2163
|
+
return candidate.termType === 'bigram' && hasExactEvidence(candidate);
|
|
2164
|
+
}).length, " phrase:").concat(ranked.filter(function (candidate) {
|
|
2165
|
+
return candidate.termType === 'phrase' && hasExactEvidence(candidate);
|
|
2166
|
+
}).length, " \xB7 ").concat(suggestion ? 'display-ready' : decision));
|
|
633
2167
|
|
|
634
|
-
//
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
return
|
|
638
|
-
})
|
|
639
|
-
|
|
640
|
-
|
|
2168
|
+
// ARBITRATE — expose every condition that can authorize or suppress ghost.
|
|
2169
|
+
var topOfType = function topOfType(t) {
|
|
2170
|
+
return ranked.find(function (c) {
|
|
2171
|
+
return c.termType === t;
|
|
2172
|
+
});
|
|
2173
|
+
};
|
|
2174
|
+
var fmtType = function fmtType(t) {
|
|
2175
|
+
var _top$lmEvidence$sourc, _top$lmEvidence;
|
|
2176
|
+
var top = topOfType(t);
|
|
2177
|
+
if (!top) {
|
|
2178
|
+
return "".concat(t, ":\u2014");
|
|
2179
|
+
}
|
|
2180
|
+
var floor = MIN_LM_POSTERIOR[t];
|
|
2181
|
+
var bound = optimisticPosterior(top);
|
|
2182
|
+
var evidence = hasRequiredEvidence(top) ? 'TierB✅' : "".concat((_top$lmEvidence$sourc = (_top$lmEvidence = top.lmEvidence) === null || _top$lmEvidence === void 0 ? void 0 : _top$lmEvidence.source) !== null && _top$lmEvidence$sourc !== void 0 ? _top$lmEvidence$sourc : 'absent', "\u274C");
|
|
2183
|
+
return "".concat(t, ":\"").concat(top.word, "\" p\u2264").concat(bound.toFixed(3)).concat(bound >= floor ? '≥' : '<').concat(floor, " fs\u2264").concat(confidenceScore(top).toFixed(2), " ").concat(evidence);
|
|
2184
|
+
};
|
|
2185
|
+
ctcSection('ARBITRATE', "".concat(selected ? "\u25B8 ".concat(selected.termType, " \"").concat(selected.word, "\" fs=").concat(selectedDisplayScore.toFixed(2), " mean=").concat((_selectedEvidence$mea = selectedEvidence === null || selectedEvidence === void 0 || (_selectedEvidence$mea2 = selectedEvidence.meanTokenLogProbability) === null || _selectedEvidence$mea2 === void 0 ? void 0 : _selectedEvidence$mea2.toFixed(2)) !== null && _selectedEvidence$mea !== void 0 ? _selectedEvidence$mea : 'network', " margin=").concat(winnerMargin.toFixed(2)).concat(clearsWinnerMargin ? '✅' : '❌') : '▸ (none)', " \xB7 prefix ").concat(currentWord.length, "/").concat(DISPLAY_MIN_PREFIX_LENGTH).concat(displayPrefixReady ? '✅' : '⏳', " \xB7 grouped pending ").concat(hasUnresolvedPotential ? 'yes' : 'no', " | ").concat(fmtType('word'), " ").concat(fmtType('bigram'), " ").concat(fmtType('phrase')));
|
|
2186
|
+
|
|
2187
|
+
// STABILIZE — post-accept cooldown + whole-surface repetition (QI-2)
|
|
2188
|
+
var repetitionBlocked = ranked.filter(function (c) {
|
|
2189
|
+
return hasRequiredEvidence(c) && duplicatesPreceding(c);
|
|
2190
|
+
}).map(function (c) {
|
|
2191
|
+
return c.word;
|
|
2192
|
+
});
|
|
2193
|
+
var cooldownLabel = cooledSurface ? "cooldown \"".concat(cooledSurface, "\" (").concat((_acceptCooldown$predi = (_acceptCooldown = acceptCooldown) === null || _acceptCooldown === void 0 ? void 0 : _acceptCooldown.predictionsSince) !== null && _acceptCooldown$predi !== void 0 ? _acceptCooldown$predi : 0, "/").concat(COOLDOWN_KEYSTROKES, " keys \xB7 ").concat(COOLDOWN_MS, "ms)") : 'cooldown —';
|
|
2194
|
+
ctcSection('STABILIZE', "".concat(cooldownLabel, " \xB7 repetition-blocked: ").concat(repetitionBlocked.length > 0 ? repetitionBlocked.join(', ') : '—'));
|
|
2195
|
+
|
|
2196
|
+
// CANDIDATES — full scored table (verbose only)
|
|
2197
|
+
if (verbose && ranked.length > 0) {
|
|
641
2198
|
var tableData = ranked.slice(0, 10).map(function (r) {
|
|
2199
|
+
var _r$lmEvidence, _contextTotals$get$le3, _contextTotals$get3, _r$lmEvidence$source, _r$lmEvidence2, _runtime$separatorKin2, _runtime$canonicalTok2, _runtime$canonicalTok3, _runtime$contextKey, _r$lmEvidence3, _r$lmEvidence4;
|
|
642
2200
|
var rawLogit = 'Not in Payload';
|
|
643
|
-
if (
|
|
2201
|
+
if (((_r$lmEvidence = r.lmEvidence) === null || _r$lmEvidence === void 0 ? void 0 : _r$lmEvidence.rawLogit) !== undefined) {
|
|
2202
|
+
rawLogit = Number(r.lmEvidence.rawLogit.toFixed(5));
|
|
2203
|
+
} else if (prefixLmLogits) {
|
|
644
2204
|
var val = prefixLmLogits[r.word.toLowerCase()];
|
|
645
2205
|
if (val !== undefined) {
|
|
646
2206
|
rawLogit = Number(val.toFixed(5));
|
|
@@ -649,9 +2209,19 @@ export var predict = function predict(textBefore) {
|
|
|
649
2209
|
var original = scoringCandidates.find(function (sc) {
|
|
650
2210
|
return sc.word === r.word;
|
|
651
2211
|
});
|
|
2212
|
+
var runtime = runtimeBySurface.get(r.word);
|
|
2213
|
+
var progress = runtime ? getProgressiveSurfaceEvidence(runtime.contextKey, r.word) : null;
|
|
2214
|
+
var optimisticTotal = optimisticTotalByWord.get(r.word);
|
|
2215
|
+
var normalizer = runtime ? logSumExpByContext.get(runtime.contextKey) : undefined;
|
|
2216
|
+
var shortlistSize = runtime ? (_contextTotals$get$le3 = (_contextTotals$get3 = contextTotals.get(runtime.contextKey)) === null || _contextTotals$get3 === void 0 ? void 0 : _contextTotals$get3.length) !== null && _contextTotals$get$le3 !== void 0 ? _contextTotals$get$le3 : 0 : 0;
|
|
2217
|
+
var contextLeader = runtime ? bestCandidateByContext.get(runtime.contextKey) : undefined;
|
|
652
2218
|
var source = 'Unknown';
|
|
653
2219
|
if (original) {
|
|
654
|
-
if (original.
|
|
2220
|
+
if (original.termType === 'bigram') {
|
|
2221
|
+
source = '🔗 L2 (Bigram)';
|
|
2222
|
+
} else if (original.termType === 'phrase') {
|
|
2223
|
+
source = '🧩 L2 (Phrase)';
|
|
2224
|
+
} else if (original.docFreq === 0 && original.tenantFreq === L3_BASELINE_FREQ) {
|
|
655
2225
|
source = '🌍 L3 (Generic)';
|
|
656
2226
|
} else if (original.sessionFreq > 0 && original.tenantFreq === 0) {
|
|
657
2227
|
source = '👤 L1 (Session Only)';
|
|
@@ -659,13 +2229,40 @@ export var predict = function predict(textBefore) {
|
|
|
659
2229
|
source = '🏢 L2 (Domain)';
|
|
660
2230
|
}
|
|
661
2231
|
}
|
|
2232
|
+
var floor = MIN_LM_POSTERIOR[r.termType];
|
|
662
2233
|
return {
|
|
663
2234
|
Candidate: r.word,
|
|
664
2235
|
Source: source,
|
|
2236
|
+
Type: r.termType,
|
|
665
2237
|
'Final Score': Number(r.finalScore.toFixed(4)),
|
|
2238
|
+
'Bounded Score': Number(confidenceScore(r).toFixed(4)),
|
|
2239
|
+
'Exact Gate Score': hasExactEvidence(r) ? Number(exactConfidenceScore(r).toFixed(4)) : '—',
|
|
2240
|
+
Floor: floor,
|
|
2241
|
+
'Posterior Floor': optimisticPosterior(r) >= floor ? '✅' : '❌',
|
|
2242
|
+
'LM Floor': hasExactEvidence(r) ? passesLmFloor(r) ? '✅' : '❌' : '⏳',
|
|
2243
|
+
'Gate Reason': gateReasonFor(r),
|
|
2244
|
+
'Display Eligible': hasRequiredEvidence(r) && isStabilized(r) && judgedPosterior(r) >= floor ? '✅' : '❌',
|
|
666
2245
|
Semantics: Number(r.semanticScore.toFixed(4)),
|
|
667
2246
|
Freq: Number(r.freqScore.toFixed(4)),
|
|
668
2247
|
'LM Score': Number(r.lmScore.toFixed(4)),
|
|
2248
|
+
'LM Evidence': (_r$lmEvidence$source = (_r$lmEvidence2 = r.lmEvidence) === null || _r$lmEvidence2 === void 0 ? void 0 : _r$lmEvidence2.source) !== null && _r$lmEvidence$source !== void 0 ? _r$lmEvidence$source : 'absent',
|
|
2249
|
+
'Verified Tokens': r.lmEvidence ? "".concat(r.lmEvidence.verifiedTokenCount, "/").concat(r.lmEvidence.totalTokenCount) : '0/0',
|
|
2250
|
+
'Progressive Tokens': progress ? "".concat(progress.scoredTokenCount, "/").concat(progress.totalTokenCount) : '0/0',
|
|
2251
|
+
'LM Mean Upper': progress !== null ? Number(progress.meanTokenLogProbabilityUpperBound.toFixed(5)) : '—',
|
|
2252
|
+
'Optimistic Total': optimisticTotal !== undefined ? Number(optimisticTotal.toFixed(5)) : '—',
|
|
2253
|
+
'Context logSumExp': normalizer !== undefined && Number.isFinite(normalizer) ? Number(normalizer.toFixed(5)) : '—',
|
|
2254
|
+
'Shortlist Size': shortlistSize,
|
|
2255
|
+
'Posterior (bound)': Number(optimisticPosterior(r).toFixed(5)),
|
|
2256
|
+
'Posterior (judged)': hasJudgeableEvidence(r) ? Number(judgedPosterior(r).toFixed(5)) : '—',
|
|
2257
|
+
'Context Leader': contextLeader ? "".concat(contextLeader.termType, ":").concat(contextLeader.word) : '—',
|
|
2258
|
+
'Verified Chars': r.lmEvidence ? "".concat(r.lmEvidence.verifiedCharCount, "/").concat(r.lmEvidence.totalSurfaceCharCount) : "0/".concat(r.word.length),
|
|
2259
|
+
Separator: (_runtime$separatorKin2 = runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== null && _runtime$separatorKin2 !== void 0 ? _runtime$separatorKin2 : 'unknown',
|
|
2260
|
+
'Canonical Eligibility': (runtime === null || runtime === void 0 ? void 0 : runtime.separatorKind) !== 'whitespace' ? 'incompatible-separator' : runtime.canonicalTokenIds === null ? 'missing-artifact' : 'eligible',
|
|
2261
|
+
'Artifact Tokens': (_runtime$canonicalTok2 = runtime === null || runtime === void 0 || (_runtime$canonicalTok3 = runtime.canonicalTokenIds) === null || _runtime$canonicalTok3 === void 0 ? void 0 : _runtime$canonicalTok3.join(',')) !== null && _runtime$canonicalTok2 !== void 0 ? _runtime$canonicalTok2 : 'absent',
|
|
2262
|
+
'Context Key': (_runtime$contextKey = runtime === null || runtime === void 0 ? void 0 : runtime.contextKey) !== null && _runtime$contextKey !== void 0 ? _runtime$contextKey : '—',
|
|
2263
|
+
'Exact Total': ((_r$lmEvidence3 = r.lmEvidence) === null || _r$lmEvidence3 === void 0 ? void 0 : _r$lmEvidence3.totalLogProbability) !== undefined ? Number(r.lmEvidence.totalLogProbability.toFixed(5)) : '—',
|
|
2264
|
+
'Exact Mean': ((_r$lmEvidence4 = r.lmEvidence) === null || _r$lmEvidence4 === void 0 ? void 0 : _r$lmEvidence4.meanTokenLogProbability) !== undefined ? Number(r.lmEvidence.meanTokenLogProbability.toFixed(5)) : '—',
|
|
2265
|
+
Assoc: typeof (original === null || original === void 0 ? void 0 : original.association) === 'number' ? Number(original.association.toFixed(4)) : '—',
|
|
669
2266
|
'Raw Logit': rawLogit,
|
|
670
2267
|
'Session Freq': (original === null || original === void 0 ? void 0 : original.sessionFreq) || 0
|
|
671
2268
|
};
|
|
@@ -673,23 +2270,120 @@ export var predict = function predict(textBefore) {
|
|
|
673
2270
|
|
|
674
2271
|
// eslint-disable-next-line no-console
|
|
675
2272
|
console.table(tableData);
|
|
676
|
-
} else {
|
|
677
|
-
// eslint-disable-next-line no-console
|
|
678
|
-
console.log('No candidates found.');
|
|
679
2273
|
}
|
|
680
2274
|
|
|
681
2275
|
// eslint-disable-next-line no-console
|
|
682
2276
|
console.groupEnd();
|
|
683
2277
|
}
|
|
684
|
-
|
|
2278
|
+
var selectedContextKey = selected ? (_runtimeBySurface$get5 = runtimeBySurface.get(selected.word)) === null || _runtimeBySurface$get5 === void 0 ? void 0 : _runtimeBySurface$get5.contextKey : undefined;
|
|
2279
|
+
return suggestion && suggestion.length > 0 && selected && selectedEvidence ? {
|
|
2280
|
+
text: suggestion,
|
|
2281
|
+
surface: selected.word,
|
|
2282
|
+
termType: selected.termType,
|
|
2283
|
+
posterior: judgedPosterior(selected),
|
|
2284
|
+
meanTokenLogProbability: (_judgedEvidenceFor$me3 = (_judgedEvidenceFor6 = judgedEvidenceFor(selected)) === null || _judgedEvidenceFor6 === void 0 ? void 0 : _judgedEvidenceFor6.mean) !== null && _judgedEvidenceFor$me3 !== void 0 ? _judgedEvidenceFor$me3 : NaN,
|
|
2285
|
+
poolHeldExtension: poolExtensionOf(selected) !== undefined,
|
|
2286
|
+
shortlistSize: selectedContextKey === undefined ? 1 : (_contextTotals$get$le4 = (_contextTotals$get4 = contextTotals.get(selectedContextKey)) === null || _contextTotals$get4 === void 0 ? void 0 : _contextTotals$get4.length) !== null && _contextTotals$get$le4 !== void 0 ? _contextTotals$get$le4 : 1,
|
|
2287
|
+
rankScore: selectedDisplayScore,
|
|
2288
|
+
evidenceTier: (selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.source) === 'network-logit' ? 'network-logit' : 'canonical-full-surface',
|
|
2289
|
+
evidenceDepth: {
|
|
2290
|
+
verifiedTokens: (_selectedEvidence$ver = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedTokenCount) !== null && _selectedEvidence$ver !== void 0 ? _selectedEvidence$ver : 0,
|
|
2291
|
+
totalTokens: (_selectedEvidence$tot = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.totalTokenCount) !== null && _selectedEvidence$tot !== void 0 ? _selectedEvidence$tot : 0,
|
|
2292
|
+
verifiedChars: (_selectedEvidence$ver2 = selectedEvidence === null || selectedEvidence === void 0 ? void 0 : selectedEvidence.verifiedCharCount) !== null && _selectedEvidence$ver2 !== void 0 ? _selectedEvidence$ver2 : 0,
|
|
2293
|
+
totalChars: selected.word.length
|
|
2294
|
+
},
|
|
2295
|
+
winnerMargin: winnerMargin
|
|
2296
|
+
} : null;
|
|
685
2297
|
};
|
|
686
2298
|
|
|
687
2299
|
// ─── Data Loading ────────────────────────────────────────────────────────────
|
|
688
2300
|
|
|
2301
|
+
var isVocabularyJson = function isVocabularyJson(payload) {
|
|
2302
|
+
if (payload == null || _typeof(payload) !== 'object') {
|
|
2303
|
+
return false;
|
|
2304
|
+
}
|
|
2305
|
+
var words = payload.words;
|
|
2306
|
+
return words != null && _typeof(words) === 'object';
|
|
2307
|
+
};
|
|
2308
|
+
var isStringArray = function isStringArray(payload) {
|
|
2309
|
+
return Array.isArray(payload) && payload.every(function (entry) {
|
|
2310
|
+
return typeof entry === 'string';
|
|
2311
|
+
});
|
|
2312
|
+
};
|
|
2313
|
+
var isWordToOffsetMap = function isWordToOffsetMap(value) {
|
|
2314
|
+
return value != null && _typeof(value) === 'object' && Object.values(value).every(function (entry) {
|
|
2315
|
+
return typeof entry === 'number';
|
|
2316
|
+
});
|
|
2317
|
+
};
|
|
2318
|
+
var isWordIndexPayload = function isWordIndexPayload(payload) {
|
|
2319
|
+
if (payload == null || _typeof(payload) !== 'object') {
|
|
2320
|
+
return false;
|
|
2321
|
+
}
|
|
2322
|
+
var index = payload.index;
|
|
2323
|
+
return index === undefined ? isWordToOffsetMap(payload) : isWordToOffsetMap(index);
|
|
2324
|
+
};
|
|
2325
|
+
|
|
2326
|
+
/**
|
|
2327
|
+
* The word index has shipped both bare and wrapped in `{ index: ... }`, so
|
|
2328
|
+
* accept either rather than silently building an empty vector store.
|
|
2329
|
+
*/
|
|
2330
|
+
var unwrapWordIndex = function unwrapWordIndex(payload) {
|
|
2331
|
+
var wrapped = payload.index;
|
|
2332
|
+
return wrapped !== null && wrapped !== void 0 ? wrapped : payload;
|
|
2333
|
+
};
|
|
2334
|
+
|
|
2335
|
+
/**
|
|
2336
|
+
* Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
|
|
2337
|
+
* confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
|
|
2338
|
+
* + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
|
|
2339
|
+
* are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
|
|
2340
|
+
* a number.
|
|
2341
|
+
*/
|
|
2342
|
+
|
|
2343
|
+
/** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
|
|
2344
|
+
|
|
2345
|
+
var isPhraseStats = function isPhraseStats(value) {
|
|
2346
|
+
return value != null && _typeof(value) === 'object' && typeof value.freq === 'number';
|
|
2347
|
+
};
|
|
2348
|
+
var isTermStatsMap = function isTermStatsMap(value) {
|
|
2349
|
+
if (value == null || _typeof(value) !== 'object') {
|
|
2350
|
+
return false;
|
|
2351
|
+
}
|
|
2352
|
+
var values = Object.values(value);
|
|
2353
|
+
return values.length > 0 && values.every(isPhraseStats);
|
|
2354
|
+
};
|
|
2355
|
+
|
|
2356
|
+
/**
|
|
2357
|
+
* Normalize a `bigrams.json` / `phrases.json` payload into a flat
|
|
2358
|
+
* `{term: stats}` map. Tolerant of the exact wire shape (not yet finalized by
|
|
2359
|
+
* the producer): accepts either the flat map from spec §9 or a single wrapper
|
|
2360
|
+
* key (`terms`/`bigrams`/`phrases`/`words`). Returns `null` if unrecognized.
|
|
2361
|
+
*
|
|
2362
|
+
* :params:
|
|
2363
|
+
* payload: Raw JSON parsed from the artifact endpoint
|
|
2364
|
+
* :returns:
|
|
2365
|
+
* A `{term: stats}` map, or `null` when the shape is not a term-stats map
|
|
2366
|
+
*/
|
|
2367
|
+
var normalizePhraseArtifact = function normalizePhraseArtifact(payload) {
|
|
2368
|
+
if (payload == null || _typeof(payload) !== 'object') {
|
|
2369
|
+
return null;
|
|
2370
|
+
}
|
|
2371
|
+
if (isTermStatsMap(payload)) {
|
|
2372
|
+
return payload;
|
|
2373
|
+
}
|
|
2374
|
+
var obj = payload;
|
|
2375
|
+
for (var _i4 = 0, _arr = ['terms', 'bigrams', 'phrases', 'words']; _i4 < _arr.length; _i4++) {
|
|
2376
|
+
var key = _arr[_i4];
|
|
2377
|
+
if (isTermStatsMap(obj[key])) {
|
|
2378
|
+
return obj[key];
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
return null;
|
|
2382
|
+
};
|
|
689
2383
|
export var loadVectorsAsync = /*#__PURE__*/function () {
|
|
690
|
-
var
|
|
2384
|
+
var _ref14 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
|
|
691
2385
|
var _options$isLocalLLM;
|
|
692
|
-
var isLocalLLM, surface,
|
|
2386
|
+
var isLocalLLM, surface, buffer, float32, wordIndexPayload, wordIndex, nWords, dim, _t;
|
|
693
2387
|
return _regeneratorRuntime.wrap(function (_context) {
|
|
694
2388
|
while (1) switch (_context.prev = _context.next) {
|
|
695
2389
|
case 0:
|
|
@@ -709,65 +2403,23 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
|
|
|
709
2403
|
} : {}));
|
|
710
2404
|
_context.prev = 2;
|
|
711
2405
|
_context.next = 3;
|
|
712
|
-
return
|
|
2406
|
+
return fetchAutocompleteArtifactBinary(ARTIFACT_NAME.WORD_VECTORS, 'word_vectors_10k');
|
|
713
2407
|
case 3:
|
|
714
|
-
url = _context.sent;
|
|
715
|
-
_context.next = 5;
|
|
716
|
-
break;
|
|
717
|
-
case 4:
|
|
718
|
-
_context.prev = 4;
|
|
719
|
-
_t = _context["catch"](2);
|
|
720
|
-
vectorsLoadStarted = false;
|
|
721
|
-
failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
|
|
722
|
-
isLocalLLM: isLocalLLM,
|
|
723
|
-
errorType: 'resolve_url'
|
|
724
|
-
}, surface ? {
|
|
725
|
-
surface: surface
|
|
726
|
-
} : {}));
|
|
727
|
-
// eslint-disable-next-line no-console
|
|
728
|
-
console.warn('[text-predictor] Failed to resolve vectors URL:', _t);
|
|
729
|
-
return _context.abrupt("return");
|
|
730
|
-
case 5:
|
|
731
|
-
_context.prev = 5;
|
|
732
|
-
_context.next = 6;
|
|
733
|
-
return fetch(url);
|
|
734
|
-
case 6:
|
|
735
|
-
res = _context.sent;
|
|
736
|
-
if (res.ok) {
|
|
737
|
-
_context.next = 7;
|
|
738
|
-
break;
|
|
739
|
-
}
|
|
740
|
-
vectorsLoadStarted = false;
|
|
741
|
-
failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
|
|
742
|
-
isLocalLLM: isLocalLLM,
|
|
743
|
-
status: res.status,
|
|
744
|
-
errorType: 'http_error'
|
|
745
|
-
}, surface ? {
|
|
746
|
-
surface: surface
|
|
747
|
-
} : {}));
|
|
748
|
-
// eslint-disable-next-line no-console
|
|
749
|
-
console.warn("[text-predictor] Failed to load vectors: ".concat(res.status));
|
|
750
|
-
return _context.abrupt("return");
|
|
751
|
-
case 7:
|
|
752
|
-
_context.next = 8;
|
|
753
|
-
return res.arrayBuffer();
|
|
754
|
-
case 8:
|
|
755
2408
|
buffer = _context.sent;
|
|
756
|
-
float32 = new Float32Array(buffer);
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
2409
|
+
float32 = new Float32Array(buffer);
|
|
2410
|
+
_context.next = 4;
|
|
2411
|
+
return fetchAutocompleteArtifactJson(ARTIFACT_NAME.WORD_INDEX, {
|
|
2412
|
+
summarize: function summarize(payload) {
|
|
2413
|
+
return "".concat(Object.keys(unwrapWordIndex(payload)).length, " entries");
|
|
2414
|
+
},
|
|
2415
|
+
validate: isWordIndexPayload
|
|
2416
|
+
});
|
|
2417
|
+
case 4:
|
|
764
2418
|
wordIndexPayload = _context.sent;
|
|
765
|
-
|
|
766
|
-
wordIndex = _typeof(wrappedIndex) === 'object' && wrappedIndex !== null ? wrappedIndex : wordIndexPayload !== null && wordIndexPayload !== void 0 ? wordIndexPayload : {};
|
|
2419
|
+
wordIndex = unwrapWordIndex(wordIndexPayload);
|
|
767
2420
|
nWords = Object.keys(wordIndex).length;
|
|
768
2421
|
if (nWords === 0) {
|
|
769
|
-
|
|
770
|
-
console.warn("[text-predictor] ".concat(ARTIFACT_NAME.WORD_INDEX, " was empty \u2014 semantic scoring will be a no-op."));
|
|
2422
|
+
ctcTag('init', '⚠️ word_index_10k.json was empty — semantic scoring will be a no-op', CTC_STYLES.warn);
|
|
771
2423
|
}
|
|
772
2424
|
dim = float32.length / nWords;
|
|
773
2425
|
vectorStore = {
|
|
@@ -783,19 +2435,12 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
|
|
|
783
2435
|
}, surface ? {
|
|
784
2436
|
surface: surface
|
|
785
2437
|
} : {}));
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
console.log('[text-predictor] Vectors loaded:', {
|
|
789
|
-
wordCount: nWords,
|
|
790
|
-
dim: dim,
|
|
791
|
-
sizeBytes: float32.byteLength
|
|
792
|
-
});
|
|
793
|
-
}
|
|
794
|
-
_context.next = 11;
|
|
2438
|
+
ctcTag('init', "vectors loaded: ".concat(nWords, " words \xB7 dim ").concat(dim, " \xB7 ").concat(float32.byteLength, "B"));
|
|
2439
|
+
_context.next = 6;
|
|
795
2440
|
break;
|
|
796
|
-
case
|
|
797
|
-
_context.prev =
|
|
798
|
-
|
|
2441
|
+
case 5:
|
|
2442
|
+
_context.prev = 5;
|
|
2443
|
+
_t = _context["catch"](2);
|
|
799
2444
|
vectorsLoadStarted = false;
|
|
800
2445
|
failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', _objectSpread({
|
|
801
2446
|
isLocalLLM: isLocalLLM,
|
|
@@ -803,60 +2448,197 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
|
|
|
803
2448
|
}, surface ? {
|
|
804
2449
|
surface: surface
|
|
805
2450
|
} : {}));
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
case 11:
|
|
2451
|
+
ctcTag('init', "\u26A0\uFE0F failed to load vectors: ".concat(String(_t)), CTC_STYLES.bad);
|
|
2452
|
+
case 6:
|
|
809
2453
|
case "end":
|
|
810
2454
|
return _context.stop();
|
|
811
2455
|
}
|
|
812
|
-
}, _callee, null, [[2,
|
|
2456
|
+
}, _callee, null, [[2, 5]]);
|
|
813
2457
|
}));
|
|
814
2458
|
return function loadVectorsAsync(_x) {
|
|
815
|
-
return
|
|
2459
|
+
return _ref14.apply(this, arguments);
|
|
816
2460
|
};
|
|
817
2461
|
}();
|
|
818
2462
|
export var initVectors = function initVectors(store) {
|
|
819
2463
|
vectorStore = store;
|
|
820
2464
|
};
|
|
2465
|
+
|
|
2466
|
+
/**
|
|
2467
|
+
* Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
|
|
2468
|
+
* over the TDP-OS gateway and insert them into the phrase trie.
|
|
2469
|
+
*
|
|
2470
|
+
* Each file is fetched independently (`Promise.allSettled`) so a missing or
|
|
2471
|
+
* late-published artifact only skips that term type — word completion and the
|
|
2472
|
+
* other term type are unaffected. Phrase VECTORS are not fetched here: per the
|
|
2473
|
+
* producer's `extend` decision they are appended into the existing
|
|
2474
|
+
* `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
|
|
2475
|
+
* covers them.
|
|
2476
|
+
*
|
|
2477
|
+
* :params:
|
|
2478
|
+
* options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
|
|
2479
|
+
* :returns:
|
|
2480
|
+
* A promise that resolves once both fetches have settled
|
|
2481
|
+
*/
|
|
2482
|
+
export var loadPhraseArtifacts = /*#__PURE__*/function () {
|
|
2483
|
+
var _ref15 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee3(options) {
|
|
2484
|
+
var _options$isLocalLLM2;
|
|
2485
|
+
var isLocalLLM, loadOne, _yield$Promise$allSet, _yield$Promise$allSet2, bigramsResult, phrasesResult, bigramCount, phraseCount;
|
|
2486
|
+
return _regeneratorRuntime.wrap(function (_context3) {
|
|
2487
|
+
while (1) switch (_context3.prev = _context3.next) {
|
|
2488
|
+
case 0:
|
|
2489
|
+
if (!phrasesLoadStarted) {
|
|
2490
|
+
_context3.next = 1;
|
|
2491
|
+
break;
|
|
2492
|
+
}
|
|
2493
|
+
return _context3.abrupt("return");
|
|
2494
|
+
case 1:
|
|
2495
|
+
phrasesLoadStarted = true;
|
|
2496
|
+
isLocalLLM = (_options$isLocalLLM2 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM2 !== void 0 ? _options$isLocalLLM2 : false;
|
|
2497
|
+
startExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
|
|
2498
|
+
isLocalLLM: isLocalLLM
|
|
2499
|
+
});
|
|
2500
|
+
loadOne = /*#__PURE__*/function () {
|
|
2501
|
+
var _ref16 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee2(artifactName, termType, label) {
|
|
2502
|
+
var payload, normalized;
|
|
2503
|
+
return _regeneratorRuntime.wrap(function (_context2) {
|
|
2504
|
+
while (1) switch (_context2.prev = _context2.next) {
|
|
2505
|
+
case 0:
|
|
2506
|
+
_context2.next = 1;
|
|
2507
|
+
return fetchAutocompleteArtifactJson(artifactName, {
|
|
2508
|
+
label: label,
|
|
2509
|
+
validate: function validate(p) {
|
|
2510
|
+
return p != null && _typeof(p) === 'object';
|
|
2511
|
+
}
|
|
2512
|
+
});
|
|
2513
|
+
case 1:
|
|
2514
|
+
payload = _context2.sent;
|
|
2515
|
+
normalized = normalizePhraseArtifact(payload);
|
|
2516
|
+
if (normalized) {
|
|
2517
|
+
_context2.next = 2;
|
|
2518
|
+
break;
|
|
2519
|
+
}
|
|
2520
|
+
throw new Error("[text-predictor] ".concat(label, " payload was not a recognised term-stats map"));
|
|
2521
|
+
case 2:
|
|
2522
|
+
return _context2.abrupt("return", initPhrases(normalized, termType));
|
|
2523
|
+
case 3:
|
|
2524
|
+
case "end":
|
|
2525
|
+
return _context2.stop();
|
|
2526
|
+
}
|
|
2527
|
+
}, _callee2);
|
|
2528
|
+
}));
|
|
2529
|
+
return function loadOne(_x3, _x4, _x5) {
|
|
2530
|
+
return _ref16.apply(this, arguments);
|
|
2531
|
+
};
|
|
2532
|
+
}();
|
|
2533
|
+
_context3.next = 2;
|
|
2534
|
+
return Promise.allSettled([loadOne(ARTIFACT_NAME.BIGRAMS, 'bigram', 'bigrams'), loadOne(ARTIFACT_NAME.PHRASES, 'phrase', 'phrases')]);
|
|
2535
|
+
case 2:
|
|
2536
|
+
_yield$Promise$allSet = _context3.sent;
|
|
2537
|
+
_yield$Promise$allSet2 = _slicedToArray(_yield$Promise$allSet, 2);
|
|
2538
|
+
bigramsResult = _yield$Promise$allSet2[0];
|
|
2539
|
+
phrasesResult = _yield$Promise$allSet2[1];
|
|
2540
|
+
bigramCount = bigramsResult.status === 'fulfilled' ? bigramsResult.value : 0;
|
|
2541
|
+
phraseCount = phrasesResult.status === 'fulfilled' ? phrasesResult.value : 0;
|
|
2542
|
+
if (bigramsResult.status === 'rejected') {
|
|
2543
|
+
ctcTag('init', "\u26A0\uFE0F failed to load bigrams.json: ".concat(String(bigramsResult.reason)), CTC_STYLES.warn);
|
|
2544
|
+
}
|
|
2545
|
+
if (phrasesResult.status === 'rejected') {
|
|
2546
|
+
ctcTag('init', "\u26A0\uFE0F failed to load phrases.json: ".concat(String(phrasesResult.reason)), CTC_STYLES.warn);
|
|
2547
|
+
}
|
|
2548
|
+
if (bigramsResult.status === 'fulfilled' || phrasesResult.status === 'fulfilled') {
|
|
2549
|
+
phrasesLoaded = true;
|
|
2550
|
+
succeedExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
|
|
2551
|
+
isLocalLLM: isLocalLLM,
|
|
2552
|
+
bigramCount: bigramCount,
|
|
2553
|
+
phraseCount: phraseCount
|
|
2554
|
+
});
|
|
2555
|
+
ctcTag('init', "phrase artifacts loaded: ".concat(bigramCount, " bigrams \xB7 ").concat(phraseCount, " phrases"));
|
|
2556
|
+
} else {
|
|
2557
|
+
// Both failed — e.g. artifacts not published to the tenant manifest yet.
|
|
2558
|
+
// Reset the guard so a later focus/predict can retry once they land.
|
|
2559
|
+
phrasesLoadStarted = false;
|
|
2560
|
+
failExp(EXPERIENCE_NAME.LOAD_PHRASES, 'singleton', {
|
|
2561
|
+
isLocalLLM: isLocalLLM,
|
|
2562
|
+
errorType: 'network'
|
|
2563
|
+
});
|
|
2564
|
+
}
|
|
2565
|
+
case 3:
|
|
2566
|
+
case "end":
|
|
2567
|
+
return _context3.stop();
|
|
2568
|
+
}
|
|
2569
|
+
}, _callee3);
|
|
2570
|
+
}));
|
|
2571
|
+
return function loadPhraseArtifacts(_x2) {
|
|
2572
|
+
return _ref15.apply(this, arguments);
|
|
2573
|
+
};
|
|
2574
|
+
}();
|
|
821
2575
|
var vocabularyLoadPromise;
|
|
822
2576
|
export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
|
|
823
|
-
var _options$
|
|
2577
|
+
var _options$isLocalLLM3, _options$source;
|
|
824
2578
|
if (isInitialized) {
|
|
825
2579
|
return Promise.resolve();
|
|
826
2580
|
}
|
|
827
2581
|
if (vocabularyLoadPromise) {
|
|
828
2582
|
return vocabularyLoadPromise;
|
|
829
2583
|
}
|
|
830
|
-
var isLocalLLM = (_options$
|
|
2584
|
+
var isLocalLLM = (_options$isLocalLLM3 = options === null || options === void 0 ? void 0 : options.isLocalLLM) !== null && _options$isLocalLLM3 !== void 0 ? _options$isLocalLLM3 : false;
|
|
2585
|
+
var source = (_options$source = options === null || options === void 0 ? void 0 : options.source) !== null && _options$source !== void 0 ? _options$source : 'predict';
|
|
831
2586
|
var surface = options === null || options === void 0 ? void 0 : options.surface;
|
|
832
|
-
vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function
|
|
833
|
-
var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms,
|
|
834
|
-
return _regeneratorRuntime.wrap(function (
|
|
835
|
-
while (1) switch (
|
|
2587
|
+
vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee4() {
|
|
2588
|
+
var _yield$Promise$all, _yield$Promise$all2, vocabularyData, l3VocabularyData, terms, _t2;
|
|
2589
|
+
return _regeneratorRuntime.wrap(function (_context4) {
|
|
2590
|
+
while (1) switch (_context4.prev = _context4.next) {
|
|
836
2591
|
case 0:
|
|
837
2592
|
startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
|
|
838
2593
|
isLocalLLM: isLocalLLM
|
|
839
2594
|
}, surface ? {
|
|
840
2595
|
surface: surface
|
|
841
2596
|
} : {}));
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
2597
|
+
_context4.prev = 1;
|
|
2598
|
+
ctcTag('init', "loading artifacts (trigger: ".concat(source, " \xB7 localLLM: ").concat(isLocalLLM, ")"));
|
|
2599
|
+
if (isAutocompleteDebugVerbose()) {
|
|
2600
|
+
// eslint-disable-next-line no-console
|
|
2601
|
+
console.groupCollapsed('%c[CTC:init]%c artifact manifest', CTC_STYLES.brand, CTC_STYLES.body);
|
|
2602
|
+
// eslint-disable-next-line no-console
|
|
2603
|
+
console.log('Eager:', [ARTIFACT_NAME.VOCABULARY, ARTIFACT_NAME.L3_VOCABULARY, ARTIFACT_NAME.POS_TAGS, ARTIFACT_NAME.GHOST_POS_TAGS, ARTIFACT_NAME.GRAMMAR_TRANSITIONS]);
|
|
2604
|
+
// eslint-disable-next-line no-console
|
|
2605
|
+
console.log('Conditional:', ["".concat(ARTIFACT_NAME.WORD_INDEX, " (only when vectors load; phrase/bigram surfaces share this index)"), "".concat(ARTIFACT_NAME.BIGRAMS, " + ").concat(ARTIFACT_NAME.PHRASES, " (completion-term stats; fire-and-forget)"), "".concat(ARTIFACT_NAME.PHRASE_CONTINUATION_TOKENS, " (local model only; canonical token IDs for the full served union)")]);
|
|
2606
|
+
// eslint-disable-next-line no-console
|
|
2607
|
+
console.groupEnd();
|
|
2608
|
+
}
|
|
2609
|
+
_context4.next = 2;
|
|
2610
|
+
return Promise.all([fetchAutocompleteArtifactJson(ARTIFACT_NAME.VOCABULARY, {
|
|
2611
|
+
summarize: function summarize(payload) {
|
|
2612
|
+
return "".concat(Object.keys(payload.words).length, " words");
|
|
2613
|
+
},
|
|
2614
|
+
validate: isVocabularyJson
|
|
2615
|
+
}), fetchAutocompleteArtifactJson(ARTIFACT_NAME.L3_VOCABULARY, {
|
|
2616
|
+
summarize: function summarize(payload) {
|
|
2617
|
+
return "".concat(payload.length, " words");
|
|
2618
|
+
},
|
|
2619
|
+
validate: isStringArray
|
|
2620
|
+
}),
|
|
2621
|
+
// Awaited alongside the vocabulary so the grammar filter is settled
|
|
2622
|
+
// before the first suggestion can be produced. Without this the
|
|
2623
|
+
// predictor initialises on the smaller vocabulary payload and serves
|
|
2624
|
+
// candidates the filter would have dropped until the grammar lands.
|
|
2625
|
+
// Only settlement is required: a failed load leaves the filter as a
|
|
2626
|
+
// pass-through, which is the same degradation as an absent artifact.
|
|
2627
|
+
loadGrammarDataAsync({
|
|
2628
|
+
isLocalLLM: isLocalLLM,
|
|
2629
|
+
surface: surface
|
|
2630
|
+
}).catch(function (error) {
|
|
2631
|
+
ctcTag('init', "\u26A0\uFE0F grammar data unavailable; filtering skipped: ".concat(String(error)), CTC_STYLES.warn);
|
|
2632
|
+
})]);
|
|
845
2633
|
case 2:
|
|
846
|
-
_yield$Promise$all =
|
|
2634
|
+
_yield$Promise$all = _context4.sent;
|
|
847
2635
|
_yield$Promise$all2 = _slicedToArray(_yield$Promise$all, 2);
|
|
848
2636
|
vocabularyData = _yield$Promise$all2[0];
|
|
849
2637
|
l3VocabularyData = _yield$Promise$all2[1];
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
throw new Error("[text-predictor] unexpected vocabulary payload \u2014 " + "".concat(ARTIFACT_NAME.VOCABULARY, " ").concat((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null ? 'is missing its `words` wrapper' : 'is valid', ", ") + "".concat(ARTIFACT_NAME.L3_VOCABULARY, " ").concat(Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'));
|
|
855
|
-
case 3:
|
|
856
|
-
terms = Object.entries(vocabularyData.words).map(function (_ref8) {
|
|
857
|
-
var _ref9 = _slicedToArray(_ref8, 2),
|
|
858
|
-
word = _ref9[0],
|
|
859
|
-
stats = _ref9[1];
|
|
2638
|
+
terms = Object.entries(vocabularyData.words).map(function (_ref18) {
|
|
2639
|
+
var _ref19 = _slicedToArray(_ref18, 2),
|
|
2640
|
+
word = _ref19[0],
|
|
2641
|
+
stats = _ref19[1];
|
|
860
2642
|
return {
|
|
861
2643
|
word: word,
|
|
862
2644
|
freq: stats.freq,
|
|
@@ -870,6 +2652,12 @@ export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
|
|
|
870
2652
|
initVocabulary({
|
|
871
2653
|
terms: terms
|
|
872
2654
|
});
|
|
2655
|
+
|
|
2656
|
+
// Phrase/bigram artifacts load independently and must never affect the
|
|
2657
|
+
// word path — fire-and-forget with its own error handling inside.
|
|
2658
|
+
void loadPhraseArtifacts({
|
|
2659
|
+
isLocalLLM: isLocalLLM
|
|
2660
|
+
}).catch(function () {});
|
|
873
2661
|
succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
|
|
874
2662
|
isLocalLLM: isLocalLLM,
|
|
875
2663
|
l2WordCount: terms.length,
|
|
@@ -877,25 +2665,25 @@ export var loadDefaultVocabulary = function loadDefaultVocabulary(options) {
|
|
|
877
2665
|
}, surface ? {
|
|
878
2666
|
surface: surface
|
|
879
2667
|
} : {}));
|
|
880
|
-
|
|
2668
|
+
_context4.next = 4;
|
|
881
2669
|
break;
|
|
882
|
-
case
|
|
883
|
-
|
|
884
|
-
|
|
2670
|
+
case 3:
|
|
2671
|
+
_context4.prev = 3;
|
|
2672
|
+
_t2 = _context4["catch"](1);
|
|
885
2673
|
failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', _objectSpread({
|
|
886
2674
|
isLocalLLM: isLocalLLM,
|
|
887
|
-
errorType: '
|
|
2675
|
+
errorType: 'parse_error'
|
|
888
2676
|
}, surface ? {
|
|
889
2677
|
surface: surface
|
|
890
2678
|
} : {}));
|
|
891
2679
|
// Allow a later call to retry the load rather than caching the failure.
|
|
892
2680
|
vocabularyLoadPromise = undefined;
|
|
893
|
-
throw
|
|
894
|
-
case
|
|
2681
|
+
throw _t2;
|
|
2682
|
+
case 4:
|
|
895
2683
|
case "end":
|
|
896
|
-
return
|
|
2684
|
+
return _context4.stop();
|
|
897
2685
|
}
|
|
898
|
-
},
|
|
2686
|
+
}, _callee4, null, [[1, 3]]);
|
|
899
2687
|
}))();
|
|
900
2688
|
return vocabularyLoadPromise;
|
|
901
2689
|
};
|