@atlaskit/editor-plugin-autocomplete 2.5.2 → 3.1.0

package/dist/esm/pm-plugins/text-predictor.js

@@ -1,4 +1,5 @@
 import _asyncToGenerator from "@babel/runtime/helpers/asyncToGenerator";
+import _typeof from "@babel/runtime/helpers/typeof";
 import _slicedToArray from "@babel/runtime/helpers/slicedToArray";
 import _createClass from "@babel/runtime/helpers/createClass";
 import _classCallCheck from "@babel/runtime/helpers/classCallCheck";
@@ -26,11 +27,9 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
 
 import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
 
-// import bigramsData from './data/bigrams.json';
-import l3VocabularyData from './data/l3_vocabulary.json';
-import vocabularyData from './data/vocabulary_10k.json';
-import wordIndexData from './data/word_index_10k.json';
-// import { rankCandidates, isGrammarAllowed } from './scoring-pipeline';
+// The vocabulary, L3 and word-index JSON payloads are dynamically imported in
+// loadDefaultVocabulary / loadVectorsAsync so their (large) contents stay out of
+// the editor's main chunk and only load when autocomplete is initialised.
 import { isAutocompleteDebugEnabled } from './debug-mode';
 import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
 import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
@@ -42,7 +41,7 @@ var PUNCTUATION_BOUNDARY_REGEX = /^[.,;:!?()\[\]{}"'`]+|[.,;:!?()\[\]{}"'`]+$/g;
 var MIN_PREFIX_LENGTH = 3;
 var MAX_CANDIDATES = 200;
 var CONTEXT_WORDS = 10;
-var MIN_SCORE_THRESHOLD = 0.2;
+var MIN_SCORE_THRESHOLD = 0.35;
 var L3_BASELINE_FREQ = 0.001;
 
 // ─── Types ───────────────────────────────────────────────────────────────────
@@ -206,7 +205,6 @@ var wordTrie = new WeightedWordTrie();
 // L3 Trie (General English Fallback)
 var l3Trie = new WeightedWordTrie();
 
-// --- Initialization Function ---
 /**
  * Loads the General English vocabulary.
  * expects a simple array of strings: ["about", "above", "actually", ...]
@@ -330,14 +328,11 @@ var tokenize = function tokenize(text) {
   return tokens;
 };
 var extractPreviousWord = function extractPreviousWord(text) {
-  // 1. Split the text by newlines or punctuation (. ? !)
+  // Only consider the current sentence/line the user is typing in.
   // eslint-disable-next-line require-unicode-regexp
   var sentences = text.split(/[\n.?!]+/);
-
-  // 2. Only look at the current sentence/line the user is typing in
   var currentSentence = sentences[sentences.length - 1];
 
-  // 3. Extract the previous word as normal
   // eslint-disable-next-line require-unicode-regexp
   var words = currentSentence.trimEnd().split(/\s+/);
   return words.length >= 2 ? words[words.length - 2] : '';
@@ -400,7 +395,6 @@ export var incrementSessionFreq = function incrementSessionFreq(word) {
  * Pass `undefined` (or omit the argument) to skip priming — useful when the
  * calling context does not yet have a page value available.
  */
-// NOTE: We ingest full page context here
 export var ingestDocumentPage = function ingestDocumentPage(pageContent) {
   if (!pageContent) {
     return;
@@ -433,7 +427,11 @@ export var ingestDocumentPage = function ingestDocumentPage(pageContent) {
 };
 export var predict = function predict(textBefore) {
   if (!isInitialized) {
-    loadDefaultVocabulary();
+    // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
+    // and skip this keystroke; the plugin also primes it on focus, so the tries
+    // are usually ready before the user types.
+    void loadDefaultVocabulary().catch(function () {});
+    return null;
   }
   var t0 = performance.now();
 
@@ -485,14 +483,11 @@ export var predict = function predict(textBefore) {
   if (currentWord.length < MIN_PREFIX_LENGTH) {
     return null;
   }
-
-  // 1. Primary Query: Ask the L2 Domain Trie
   var candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
 
-  // 2. Fallback Query: Gap-fill with the L3 General English Trie
+  // Gap-fill from the L3 general-English trie, requesting a full buffer so
+  // enough survive de-duplication against the L2 results.
   if (candidates.length < MAX_CANDIDATES) {
-    // Ask L3 for MAX_CANDIDATES to guarantee we have enough buffer
-    // to survive the deduplication process.
     var l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
     var existingWords = new Set(candidates.map(function (c) {
       return c.word;
@@ -502,8 +497,7 @@ export var predict = function predict(textBefore) {
     try {
       for (_iterator0.s(); !(_step0 = _iterator0.n()).done;) {
         var l3c = _step0.value;
-        if (candidates.length >= MAX_CANDIDATES) break; // Stop exactly at the limit
-
+        if (candidates.length >= MAX_CANDIDATES) break;
         if (!existingWords.has(l3c.word)) {
           candidates.push(l3c);
         }
@@ -687,53 +681,99 @@ export var predict = function predict(textBefore) {
 
 // ─── Data Loading ────────────────────────────────────────────────────────────
 
+/**
+ * Unwrap a dynamically imported JSON module to its parsed value, handling both
+ * interop modes AFM's bundler chain emits: a `.default`-wrapped namespace
+ * (classic webpack) and a named-exports namespace (webpack 5 / atlaspack JSON
+ * modules, where `default` can be a misleading scalar). Named exports are
+ * preferred when present. The caller declares the JSON `shape` because a dense
+ * array and a sparse numeric-keyed object are emitted identically as named
+ * exports. Kept in lock-step with the matching helper in local-slow-lane-client.ts.
+ */
+var unwrapJsonModule = function unwrapJsonModule(mod, shape) {
+  if (mod == null || _typeof(mod) !== 'object') {
+    return null;
+  }
+  var namespace = mod;
+  var ownKeys = Object.keys(namespace).filter(function (k) {
+    return k !== 'default' && k !== '__esModule';
+  });
+  if (ownKeys.length > 0) {
+    if (shape === 'array') {
+      var len = ownKeys.length;
+      var arr = new Array(len);
+      for (var i = 0; i < len; i++) {
+        arr[i] = namespace[String(i)];
+      }
+      return arr;
+    }
+    var obj = {};
+    var _iterator1 = _createForOfIteratorHelper(ownKeys),
+      _step1;
+    try {
+      for (_iterator1.s(); !(_step1 = _iterator1.n()).done;) {
+        var k = _step1.value;
+        obj[k] = namespace[k];
+      }
+    } catch (err) {
+      _iterator1.e(err);
+    } finally {
+      _iterator1.f();
+    }
+    return obj;
+  }
+  if ('default' in namespace && namespace.default != null) {
+    return namespace.default;
+  }
+  return null;
+};
 export var loadVectorsAsync = /*#__PURE__*/function () {
   var _ref6 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
-    var url, res, buffer, float32, wordIndex, nWords, dim;
-    return _regeneratorRuntime.wrap(function _callee$(_context) {
+    var url, _wordIndexOuter$index, res, buffer, float32, wordIndexModule, wordIndexOuter, wordIndex, nWords, dim, _t, _t2;
+    return _regeneratorRuntime.wrap(function (_context) {
       while (1) switch (_context.prev = _context.next) {
         case 0:
           if (!(vectorStore || vectorsLoadStarted)) {
-            _context.next = 2;
+            _context.next = 1;
             break;
           }
           return _context.abrupt("return");
-        case 2:
+        case 1:
           if (options !== null && options !== void 0 && options.getBinaryUrl) {
-            _context.next = 5;
+            _context.next = 2;
             break;
           }
           // eslint-disable-next-line no-console
           console.warn('[text-predictor] loadVectorsAsync called without a getBinaryUrl — vectors will not load. Pass getVectorsBinaryUrl via plugin options.');
           return _context.abrupt("return");
-        case 5:
+        case 2:
           vectorsLoadStarted = true;
           startExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton');
-          _context.prev = 7;
-          _context.next = 10;
+          _context.prev = 3;
+          _context.next = 4;
           return options.getBinaryUrl();
-        case 10:
+        case 4:
           url = _context.sent;
-          _context.next = 19;
+          _context.next = 6;
           break;
-        case 13:
-          _context.prev = 13;
-          _context.t0 = _context["catch"](7);
+        case 5:
+          _context.prev = 5;
+          _t = _context["catch"](3);
           vectorsLoadStarted = false;
           failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', {
             errorType: 'resolve_url'
           });
           // eslint-disable-next-line no-console
-          console.warn('[text-predictor] Failed to resolve vectors URL:', _context.t0);
+          console.warn('[text-predictor] Failed to resolve vectors URL:', _t);
           return _context.abrupt("return");
-        case 19:
-          _context.prev = 19;
-          _context.next = 22;
+        case 6:
+          _context.prev = 6;
+          _context.next = 7;
           return fetch(url);
-        case 22:
+        case 7:
           res = _context.sent;
           if (res.ok) {
-            _context.next = 28;
+            _context.next = 8;
             break;
           }
           vectorsLoadStarted = false;
@@ -744,14 +784,25 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
           // eslint-disable-next-line no-console
           console.warn("[text-predictor] Failed to load vectors: ".concat(res.status));
           return _context.abrupt("return");
-        case 28:
-          _context.next = 30;
+        case 8:
+          _context.next = 9;
           return res.arrayBuffer();
-        case 30:
+        case 9:
           buffer = _context.sent;
-          float32 = new Float32Array(buffer);
-          wordIndex = wordIndexData;
+          float32 = new Float32Array(buffer); // word_index_10k.json is wrapped as `{ "index": {…} }` so no real entry
+          // (e.g. the word "default") can shadow the synthetic ESM `default` export
+          // the bundler creates for dynamically-imported JSON.
+          _context.next = 10;
+          return import( /* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-word-index-10k" */'./data/word_index_10k.json');
+        case 10:
+          wordIndexModule = _context.sent;
+          wordIndexOuter = unwrapJsonModule(wordIndexModule, 'object');
+          wordIndex = (_wordIndexOuter$index = wordIndexOuter === null || wordIndexOuter === void 0 ? void 0 : wordIndexOuter.index) !== null && _wordIndexOuter$index !== void 0 ? _wordIndexOuter$index : {};
           nWords = Object.keys(wordIndex).length;
+          if (nWords === 0) {
+            // eslint-disable-next-line no-console
+            console.warn('[text-predictor] word_index_10k.json missing its `index` wrapper — wordIndex is empty, semantic scoring will be a no-op.');
+          }
           dim = float32.length / nWords;
           vectorStore = {
             float32: float32,
@@ -771,22 +822,22 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
               sizeBytes: float32.byteLength
             });
           }
-          _context.next = 45;
+          _context.next = 12;
           break;
-        case 40:
-          _context.prev = 40;
-          _context.t1 = _context["catch"](19);
+        case 11:
+          _context.prev = 11;
+          _t2 = _context["catch"](6);
           vectorsLoadStarted = false;
           failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', {
             errorType: 'network'
           });
           // eslint-disable-next-line no-console
-          console.warn('[text-predictor] Failed to load vectors:', _context.t1);
-        case 45:
+          console.warn('[text-predictor] Failed to load vectors:', _t2);
+        case 12:
         case "end":
           return _context.stop();
       }
-    }, _callee, null, [[7, 13], [19, 40]]);
+    }, _callee, null, [[3, 5], [6, 11]]);
   }));
   return function loadVectorsAsync(_x) {
     return _ref6.apply(this, arguments);
@@ -795,40 +846,73 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
 export var initVectors = function initVectors(store) {
   vectorStore = store;
 };
+var vocabularyLoadPromise;
 export var loadDefaultVocabulary = function loadDefaultVocabulary() {
   if (isInitialized) {
-    return;
+    return Promise.resolve();
   }
-  startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton');
-  try {
-    // 1. Load the Atlassian Domain (L2)
-    var data = vocabularyData;
-    var terms = Object.entries(data.words).map(function (_ref7) {
-      var _ref8 = _slicedToArray(_ref7, 2),
-        word = _ref8[0],
-        stats = _ref8[1];
-      return {
-        word: word,
-        freq: stats.freq,
-        docFreq: stats.doc_freq,
-        authorFreq: stats.author_freq
-      };
-    });
-    initVocabulary({
-      terms: terms
-    });
-
-    // 2. Load General English (L3)
-    var l3Words = l3VocabularyData;
-    initL3Vocabulary(l3Words);
-    succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
-      l2WordCount: terms.length,
-      l3WordCount: l3Words.length
-    });
-  } catch (e) {
-    failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
-      errorType: 'parse_error'
-    });
-    throw e;
+  if (vocabularyLoadPromise) {
+    return vocabularyLoadPromise;
   }
+  vocabularyLoadPromise = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee2() {
+    var _yield$Promise$all, _yield$Promise$all2, vocabularyModule, l3VocabularyModule, vocabularyData, l3VocabularyData, terms, _t3;
+    return _regeneratorRuntime.wrap(function (_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
+        case 0:
+          startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton');
+          _context2.prev = 1;
+          _context2.next = 2;
+          return Promise.all([import( /* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-vocabulary-10k" */'./data/vocabulary_10k.json'), import( /* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-l3-vocabulary" */'./data/l3_vocabulary.json')]);
+        case 2:
+          _yield$Promise$all = _context2.sent;
+          _yield$Promise$all2 = _slicedToArray(_yield$Promise$all, 2);
+          vocabularyModule = _yield$Promise$all2[0];
+          l3VocabularyModule = _yield$Promise$all2[1];
+          vocabularyData = unwrapJsonModule(vocabularyModule, 'object');
+          l3VocabularyData = unwrapJsonModule(l3VocabularyModule, 'array');
+          if (!((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData))) {
+            _context2.next = 3;
+            break;
+          }
+          throw new Error('[text-predictor] vocabulary JSON modules could not be unwrapped');
+        case 3:
+          terms = Object.entries(vocabularyData.words).map(function (_ref8) {
+            var _ref9 = _slicedToArray(_ref8, 2),
+              word = _ref9[0],
+              stats = _ref9[1];
+            return {
+              word: word,
+              freq: stats.freq,
+              docFreq: stats.doc_freq,
+              authorFreq: stats.author_freq
+            };
+          }); // Load L3 before L2: initVocabulary flips `isInitialized = true`, so it
+          // must run last — otherwise a throw in initL3Vocabulary would strand
+          // `isInitialized` true and the retry path could never reload L3.
+          initL3Vocabulary(l3VocabularyData);
+          initVocabulary({
+            terms: terms
+          });
+          succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
+            l2WordCount: terms.length,
+            l3WordCount: l3VocabularyData.length
+          });
+          _context2.next = 5;
+          break;
+        case 4:
+          _context2.prev = 4;
+          _t3 = _context2["catch"](1);
+          failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
+            errorType: 'parse_error'
+          });
+          // Allow a later call to retry the load rather than caching the failure.
+          vocabularyLoadPromise = undefined;
+          throw _t3;
+        case 5:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2, null, [[1, 4]]);
+  }))();
+  return vocabularyLoadPromise;
 };

package/dist/types/pm-plugins/text-predictor.d.ts

@@ -84,5 +84,5 @@ export declare const loadVectorsAsync: (options?: {
     getBinaryUrl?: () => Promise<string>;
 }) => Promise<void>;
 export declare const initVectors: (store: VectorStore) => void;
-export declare const loadDefaultVocabulary: () => void;
+export declare const loadDefaultVocabulary: () => Promise<void>;
 export {};

package/dist/types-ts4.5/pm-plugins/text-predictor.d.ts

@@ -84,5 +84,5 @@ export declare const loadVectorsAsync: (options?: {
     getBinaryUrl?: () => Promise<string>;
 }) => Promise<void>;
 export declare const initVectors: (store: VectorStore) => void;
-export declare const loadDefaultVocabulary: () => void;
+export declare const loadDefaultVocabulary: () => Promise<void>;
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@atlaskit/editor-plugin-autocomplete",
-	"version": "2.5.2",
+	"version": "3.1.0",
 	"description": "Client-side text autocomplete plugin for @atlaskit/editor-core",
 	"author": "Atlassian Pty Ltd",
 	"license": "Apache-2.0",
@@ -35,8 +35,8 @@
 		"wink-nlp": "^2.4.0"
 	},
 	"peerDependencies": {
-		"@atlaskit/editor-common": "^114.47.0",
-		"@atlaskit/editor-plugin-analytics": "^10.1.0",
+		"@atlaskit/editor-common": "^115.6.0",
+		"@atlaskit/editor-plugin-analytics": "^11.0.0",
 		"react": "^18.2.0"
 	},
 	"techstack": {

package/src/pm-plugins/autocomplete-plugin.ts

@@ -56,7 +56,6 @@ const getTextBeforeCursor = (state: EditorState): string => {
 	const { $from } = state.selection;
 	const maxChars = 200;
 
-	// 1. Get the perfectly flattened text of the current block up to the cursor
 	const blockNode = $from.parent;
 	const offsetInBlock = $from.parentOffset;
 	const blockText = blockNode.textContent.slice(0, offsetInBlock);
@@ -67,7 +66,7 @@ const getTextBeforeCursor = (state: EditorState): string => {
 
 	let fullText = blockText;
 
-	// 2. Walk backwards through previous blocks
+	// Walk backwards through previous blocks until we have enough context.
 	let depth = $from.depth - 1;
 
 	while (fullText.length < maxChars && depth >= 0) {
@@ -309,7 +308,6 @@ export const createAutocompletePlugin = (
 				const { state } = view;
 				const { selection } = state;
 
-				// Only predict for cursor selections (not range selections)
 				if (!selection.empty) {
 					return;
 				}
@@ -323,12 +321,10 @@ export const createAutocompletePlugin = (
 				}
 				dismissedContext = null;
 
-				// Don't predict if there's not enough context
 				if (textBefore.trim().length < 3) {
 					return;
 				}
 
-				// Tier 1 prediction is synchronous -- no async needed
 				const prediction = predict(textBefore);
 
 				if (prediction && prediction.length > 0) {
@@ -402,8 +398,7 @@ export const createAutocompletePlugin = (
 					return { ...pluginState, ...meta };
 				}
 
-				// If the document changed, clear the ghost text
-				// (new prediction will be scheduled from view.update)
+				// A new prediction is scheduled from view.update.
 				if (tr.docChanged) {
 					return {
 						...pluginState,
@@ -413,7 +408,6 @@ export const createAutocompletePlugin = (
 					};
 				}
 
-				// If selection changed without doc change, clear ghost text
 				if (tr.selectionSet && pluginState.ghostText) {
 					return {
 						...pluginState,
@@ -484,13 +478,11 @@ export const createAutocompletePlugin = (
 					return false;
 				},
 				focus: () => {
-					try {
-						loadDefaultVocabulary();
-					} catch (error) {
+					loadDefaultVocabulary().catch((error) => {
 						logException(error as Error, {
 							location: 'editor-plugin-autocomplete/loadDefaultVocabulary',
 						});
-					}
+					});
 					loadVectorsAsync({ getBinaryUrl: options?.getVectorsBinaryUrl }).catch((error) => {
 						logException(error as Error, {
 							location: 'editor-plugin-autocomplete/loadVectorsAsync',
@@ -533,12 +525,10 @@ export const createAutocompletePlugin = (
 					if (justAccepted) {
 						justAccepted = false;
 
-						// ✨ THE FIX: Memorize the text state right after acceptance.
-						// Any follow-up transactions will hit the 'dismissedContext'
-						// block and abort until the user actually types a new character!
+						// Snapshot the post-acceptance text so follow-up transactions hit
+						// the dismissedContext guard and abort until the user types again.
 						dismissedContext = getTextBeforeCursor(view.state);
 
-						// Also clear any pending debounce timers from before the acceptance
 						if (debounceTimer) {
 							clearTimeout(debounceTimer);
 						}