@atlaskit/editor-plugin-autocomplete 3.0.0 → 3.2.0

package/dist/es2019/pm-plugins/scoring-pipeline.js

@@ -135,15 +135,16 @@ function applyGrammarFilter(candidates, previousWord) {
       dropped.push(entry.candidate.word);
     }
   }
-  const finalFiltered = filtered.length > 0 ? filtered : candidates;
+
+  // Grammar is authoritative.
   return {
-    filtered: finalFiltered,
+    filtered: filtered,
     grammarMeta: {
       prevWord: lowerPrev,
       prevTags,
       before: candidates.length,
-      after: finalFiltered.length,
-      dropped: filtered.length > 0 ? dropped : []
+      after: filtered.length,
+      dropped: dropped
     }
   };
 }

package/dist/es2019/pm-plugins/text-predictor.js

@@ -18,11 +18,9 @@ import _defineProperty from "@babel/runtime/helpers/defineProperty";
 
 import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
 
-// import bigramsData from './data/bigrams.json';
-import l3VocabularyData from './data/l3_vocabulary.json';
-import vocabularyData from './data/vocabulary_10k.json';
-import wordIndexData from './data/word_index_10k.json';
-// import { rankCandidates, isGrammarAllowed } from './scoring-pipeline';
+// The vocabulary, L3 and word-index JSON payloads are dynamically imported in
+// loadDefaultVocabulary / loadVectorsAsync so their (large) contents stay out of
+// the editor's main chunk and only load when autocomplete is initialised.
 import { isAutocompleteDebugEnabled } from './debug-mode';
 import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
 import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
@@ -34,7 +32,7 @@ const PUNCTUATION_BOUNDARY_REGEX = /^[.,;:!?()\[\]{}"'`]+|[.,;:!?()\[\]{}"'`]+$/
 const MIN_PREFIX_LENGTH = 3;
 const MAX_CANDIDATES = 200;
 const CONTEXT_WORDS = 10;
-const MIN_SCORE_THRESHOLD = 0.2;
+const MIN_SCORE_THRESHOLD = 0.35;
 const L3_BASELINE_FREQ = 0.001;
 
 // ─── Types ───────────────────────────────────────────────────────────────────
@@ -153,7 +151,6 @@ const wordTrie = new WeightedWordTrie();
 // L3 Trie (General English Fallback)
 const l3Trie = new WeightedWordTrie();
 
-// --- Initialization Function ---
 /**
  * Loads the General English vocabulary.
  * expects a simple array of strings: ["about", "above", "actually", ...]
@@ -249,14 +246,11 @@ const tokenize = text => {
   return tokens;
 };
 const extractPreviousWord = text => {
-  // 1. Split the text by newlines or punctuation (. ? !)
+  // Only consider the current sentence/line the user is typing in.
   // eslint-disable-next-line require-unicode-regexp
   const sentences = text.split(/[\n.?!]+/);
-
-  // 2. Only look at the current sentence/line the user is typing in
   const currentSentence = sentences[sentences.length - 1];
 
-  // 3. Extract the previous word as normal
   // eslint-disable-next-line require-unicode-regexp
   const words = currentSentence.trimEnd().split(/\s+/);
   return words.length >= 2 ? words[words.length - 2] : '';
@@ -310,7 +304,6 @@ export const incrementSessionFreq = word => {
  * Pass `undefined` (or omit the argument) to skip priming — useful when the
  * calling context does not yet have a page value available.
  */
-// NOTE: We ingest full page context here
 export const ingestDocumentPage = pageContent => {
   if (!pageContent) {
     return;
@@ -334,7 +327,11 @@ export const ingestDocumentPage = pageContent => {
 };
 export const predict = textBefore => {
   if (!isInitialized) {
-    loadDefaultVocabulary();
+    // Vocabulary JSON is code-split and loads asynchronously. Kick off the load
+    // and skip this keystroke; the plugin also primes it on focus, so the tries
+    // are usually ready before the user types.
+    void loadDefaultVocabulary().catch(() => {});
+    return null;
   }
   const t0 = performance.now();
 
@@ -386,19 +383,15 @@ export const predict = textBefore => {
   if (currentWord.length < MIN_PREFIX_LENGTH) {
     return null;
   }
-
-  // 1. Primary Query: Ask the L2 Domain Trie
   const candidates = wordTrie.getCandidates(currentWord, MAX_CANDIDATES);
 
-  // 2. Fallback Query: Gap-fill with the L3 General English Trie
+  // Gap-fill from the L3 general-English trie, requesting a full buffer so
+  // enough survive de-duplication against the L2 results.
   if (candidates.length < MAX_CANDIDATES) {
-    // Ask L3 for MAX_CANDIDATES to guarantee we have enough buffer
-    // to survive the deduplication process.
     const l3Candidates = l3Trie.getCandidates(currentWord, MAX_CANDIDATES);
     const existingWords = new Set(candidates.map(c => c.word));
     for (const l3c of l3Candidates) {
-      if (candidates.length >= MAX_CANDIDATES) break; // Stop exactly at the limit
-
+      if (candidates.length >= MAX_CANDIDATES) break;
       if (!existingWords.has(l3c.word)) {
         candidates.push(l3c);
       }
@@ -558,6 +551,41 @@ export const predict = textBefore => {
 
 // ─── Data Loading ────────────────────────────────────────────────────────────
 
+/**
+ * Unwrap a dynamically imported JSON module to its parsed value, handling both
+ * interop modes AFM's bundler chain emits: a `.default`-wrapped namespace
+ * (classic webpack) and a named-exports namespace (webpack 5 / atlaspack JSON
+ * modules, where `default` can be a misleading scalar). Named exports are
+ * preferred when present. The caller declares the JSON `shape` because a dense
+ * array and a sparse numeric-keyed object are emitted identically as named
+ * exports. Kept in lock-step with the matching helper in local-slow-lane-client.ts.
+ */
+const unwrapJsonModule = (mod, shape) => {
+  if (mod == null || typeof mod !== 'object') {
+    return null;
+  }
+  const namespace = mod;
+  const ownKeys = Object.keys(namespace).filter(k => k !== 'default' && k !== '__esModule');
+  if (ownKeys.length > 0) {
+    if (shape === 'array') {
+      const len = ownKeys.length;
+      const arr = new Array(len);
+      for (let i = 0; i < len; i++) {
+        arr[i] = namespace[String(i)];
+      }
+      return arr;
+    }
+    const obj = {};
+    for (const k of ownKeys) {
+      obj[k] = namespace[k];
+    }
+    return obj;
+  }
+  if ('default' in namespace && namespace.default != null) {
+    return namespace.default;
+  }
+  return null;
+};
 export const loadVectorsAsync = async options => {
   if (vectorStore || vectorsLoadStarted) {
     return;
@@ -582,6 +610,7 @@ export const loadVectorsAsync = async options => {
     return;
   }
   try {
+    var _wordIndexOuter$index;
     const res = await fetch(url);
     if (!res.ok) {
       vectorsLoadStarted = false;
@@ -595,8 +624,18 @@ export const loadVectorsAsync = async options => {
     }
     const buffer = await res.arrayBuffer();
     const float32 = new Float32Array(buffer);
-    const wordIndex = wordIndexData;
+
+    // word_index_10k.json is wrapped as `{ "index": {…} }` so no real entry
+    // (e.g. the word "default") can shadow the synthetic ESM `default` export
+    // the bundler creates for dynamically-imported JSON.
+    const wordIndexModule = await import( /* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-word-index-10k" */'./data/word_index_10k.json');
+    const wordIndexOuter = unwrapJsonModule(wordIndexModule, 'object');
+    const wordIndex = (_wordIndexOuter$index = wordIndexOuter === null || wordIndexOuter === void 0 ? void 0 : wordIndexOuter.index) !== null && _wordIndexOuter$index !== void 0 ? _wordIndexOuter$index : {};
     const nWords = Object.keys(wordIndex).length;
+    if (nWords === 0) {
+      // eslint-disable-next-line no-console
+      console.warn('[text-predictor] word_index_10k.json missing its `index` wrapper — wordIndex is empty, semantic scoring will be a no-op.');
+    }
     const dim = float32.length / nWords;
     vectorStore = {
       float32,
@@ -628,35 +667,51 @@ export const loadVectorsAsync = async options => {
 export const initVectors = store => {
   vectorStore = store;
 };
+let vocabularyLoadPromise;
 export const loadDefaultVocabulary = () => {
   if (isInitialized) {
-    return;
-  }
-  startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton');
-  try {
-    // 1. Load the Atlassian Domain (L2)
-    const data = vocabularyData;
-    const terms = Object.entries(data.words).map(([word, stats]) => ({
-      word,
-      freq: stats.freq,
-      docFreq: stats.doc_freq,
-      authorFreq: stats.author_freq
-    }));
-    initVocabulary({
-      terms
-    });
-
-    // 2. Load General English (L3)
-    const l3Words = l3VocabularyData;
-    initL3Vocabulary(l3Words);
-    succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
-      l2WordCount: terms.length,
-      l3WordCount: l3Words.length
-    });
-  } catch (e) {
-    failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
-      errorType: 'parse_error'
-    });
-    throw e;
-  }
+    return Promise.resolve();
+  }
+  if (vocabularyLoadPromise) {
+    return vocabularyLoadPromise;
+  }
+  vocabularyLoadPromise = (async () => {
+    startExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton');
+    try {
+      // The L2 vocabulary and L3 word list are code-split into their own async
+      // chunks so they stay out of the editor's main bundle.
+      const [vocabularyModule, l3VocabularyModule] = await Promise.all([import( /* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-vocabulary-10k" */'./data/vocabulary_10k.json'), import( /* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-l3-vocabulary" */'./data/l3_vocabulary.json')]);
+      const vocabularyData = unwrapJsonModule(vocabularyModule, 'object');
+      const l3VocabularyData = unwrapJsonModule(l3VocabularyModule, 'array');
+      if ((vocabularyData === null || vocabularyData === void 0 ? void 0 : vocabularyData.words) == null || !Array.isArray(l3VocabularyData)) {
+        throw new Error('[text-predictor] vocabulary JSON modules could not be unwrapped');
+      }
+      const terms = Object.entries(vocabularyData.words).map(([word, stats]) => ({
+        word,
+        freq: stats.freq,
+        docFreq: stats.doc_freq,
+        authorFreq: stats.author_freq
+      }));
+
+      // Load L3 before L2: initVocabulary flips `isInitialized = true`, so it
+      // must run last — otherwise a throw in initL3Vocabulary would strand
+      // `isInitialized` true and the retry path could never reload L3.
+      initL3Vocabulary(l3VocabularyData);
+      initVocabulary({
+        terms
+      });
+      succeedExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
+        l2WordCount: terms.length,
+        l3WordCount: l3VocabularyData.length
+      });
+    } catch (e) {
+      failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
+        errorType: 'parse_error'
+      });
+      // Allow a later call to retry the load rather than caching the failure.
+      vocabularyLoadPromise = undefined;
+      throw e;
+    }
+  })();
+  return vocabularyLoadPromise;
 };

package/dist/esm/pm-plugins/autocomplete-plugin.js

@@ -31,8 +31,6 @@ var createInitialState = function createInitialState() {
 var getTextBeforeCursor = function getTextBeforeCursor(state) {
   var $from = state.selection.$from;
   var maxChars = 200;
-
-  // 1. Get the perfectly flattened text of the current block up to the cursor
   var blockNode = $from.parent;
   var offsetInBlock = $from.parentOffset;
   var blockText = blockNode.textContent.slice(0, offsetInBlock);
@@ -41,7 +39,7 @@ var getTextBeforeCursor = function getTextBeforeCursor(state) {
   }
   var fullText = blockText;
 
-  // 2. Walk backwards through previous blocks
+  // Walk backwards through previous blocks until we have enough context.
   var depth = $from.depth - 1;
   while (fullText.length < maxChars && depth >= 0) {
     var parentNode = $from.node(depth);
@@ -242,8 +240,6 @@ export var createAutocompletePlugin = function createAutocompletePlugin(options,
       try {
         var state = view.state;
         var selection = state.selection;
-
-        // Only predict for cursor selections (not range selections)
         if (!selection.empty) {
           return;
         }
@@ -255,13 +251,9 @@ export var createAutocompletePlugin = function createAutocompletePlugin(options,
           return;
         }
         dismissedContext = null;
-
-        // Don't predict if there's not enough context
         if (textBefore.trim().length < 3) {
           return;
         }
-
-        // Tier 1 prediction is synchronous -- no async needed
         var prediction = predict(textBefore);
         if (prediction && prediction.length > 0) {
           var typedLength = getTypedLengthForPrediction(textBefore);
@@ -323,8 +315,7 @@ export var createAutocompletePlugin = function createAutocompletePlugin(options,
           return _objectSpread(_objectSpread({}, pluginState), meta);
         }
 
-        // If the document changed, clear the ghost text
-        // (new prediction will be scheduled from view.update)
+        // A new prediction is scheduled from view.update.
         if (tr.docChanged) {
           return _objectSpread(_objectSpread({}, pluginState), {}, {
             ghostText: '',
@@ -332,8 +323,6 @@ export var createAutocompletePlugin = function createAutocompletePlugin(options,
             decorationSet: DecorationSet.empty
           });
         }
-
-        // If selection changed without doc change, clear ghost text
         if (tr.selectionSet && pluginState.ghostText) {
           return _objectSpread(_objectSpread({}, pluginState), {}, {
             ghostText: '',
@@ -394,13 +383,11 @@ export var createAutocompletePlugin = function createAutocompletePlugin(options,
           return false;
         },
         focus: function focus() {
-          try {
-            loadDefaultVocabulary();
-          } catch (error) {
+          loadDefaultVocabulary().catch(function (error) {
             logException(error, {
               location: 'editor-plugin-autocomplete/loadDefaultVocabulary'
             });
-          }
+          });
           loadVectorsAsync({
             getBinaryUrl: options === null || options === void 0 ? void 0 : options.getVectorsBinaryUrl
           }).catch(function (error) {
@@ -442,12 +429,9 @@ export var createAutocompletePlugin = function createAutocompletePlugin(options,
             if (justAccepted) {
               justAccepted = false;
 
-              // ✨ THE FIX: Memorize the text state right after acceptance.
-              // Any follow-up transactions will hit the 'dismissedContext'
-              // block and abort until the user actually types a new character!
+              // Snapshot the post-acceptance text so follow-up transactions hit
+              // the dismissedContext guard and abort until the user types again.
               dismissedContext = getTextBeforeCursor(view.state);
-
-              // Also clear any pending debounce timers from before the acceptance
               if (debounceTimer) {
                 clearTimeout(debounceTimer);
               }