@atlaskit/editor-plugin-autocomplete 1.0.0 → 2.1.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @atlaskit/editor-plugin-autocomplete
 
+## 2.1.0
+
+### Minor Changes
+
+- [`6b36a63af0057`](https://bitbucket.org/atlassian/atlassian-frontend-monorepo/commits/6b36a63af0057) -
+  Updated scoring math for contextual typeahead autocomplete
+
+### Patch Changes
+
+- Updated dependencies
+
+## 2.0.0
+
+### Patch Changes
+
+- Updated dependencies
+
 ## 1.0.0
 
 ### Patch Changes

package/dist/cjs/pm-plugins/scoring-pipeline.js

@@ -4,6 +4,7 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
+exports.STAGE2_WEIGHT = exports.STAGE1_WEIGHT = exports.MIN_STAGE1_SCORE = void 0;
 exports.rankCandidates = rankCandidates;
 var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
 var _combined_l2_l3_pos_tags = _interopRequireDefault(require("./data/combined_l2_l3_pos_tags.json"));
@@ -26,10 +27,15 @@ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length)
 var ALPHA = 0.5;
 var BETA = 0.5;
 var NEUTRAL_SCORE = 0.5;
-var STAGE1_WEIGHT = 0.6;
-var STAGE2_WEIGHT = 0.4;
-var MIN_STAGE1_SCORE = 0.35;
+var STAGE1_WEIGHT = exports.STAGE1_WEIGHT = 0.35;
+var STAGE2_WEIGHT = exports.STAGE2_WEIGHT = 0.65;
+var MIN_STAGE1_SCORE = exports.MIN_STAGE1_SCORE = 0.35;
 var L1_SESSION_CAP = 1.2;
+// Minimum prefix-payload max LM probability before Stage 2 activates.
+// Below this threshold the LM signal is too weak to suppress Stage 1 — finalScore
+// falls back to stage1Score directly. Prevents weak prefixes (e.g. "ins" → "instances"
+// at 0.00024) from triggering re-ranking.
+var LM_GATE_THRESHOLD = 0.0005;
 
 // ─── Grammar Data (loaded once on import) ───────────────────
 
@@ -193,6 +199,7 @@ function getLmScore(word, lmLogits) {
 // ─── Public API ─────────────────────────────────────────────
 
 function rankCandidates(candidates, contextVector, getWordVector, lmLogits, maxTenantFreq, previousWord) {
+  var _grammarMeta$dropped;
   // Stage 1
   var stage1Results = candidates.map(function (candidate) {
     var _scoreStage = scoreStage1(candidate, contextVector, getWordVector, maxTenantFreq),
@@ -206,33 +213,48 @@ function rankCandidates(candidates, contextVector, getWordVector, lmLogits, maxT
       stage1Score: stage1Score
     };
   });
-  var stage1Survivors = stage1Results.filter(function (entry) {
-    return entry.stage1Score >= MIN_STAGE1_SCORE;
-  });
+  var stage1Survivors = [];
+  var stage1Rejected = [];
+  var _iterator3 = _createForOfIteratorHelper(stage1Results),
+    _step3;
+  try {
+    for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
+      var entry = _step3.value;
+      if (entry.stage1Score >= MIN_STAGE1_SCORE) {
+        stage1Survivors.push(entry);
+      } else {
+        stage1Rejected.push(entry.candidate.word);
+      }
+    }
 
-  // Grammar Filter
+    // Grammar Filter
+  } catch (err) {
+    _iterator3.e(err);
+  } finally {
+    _iterator3.f();
+  }
   var _applyGrammarFilter = applyGrammarFilter(stage1Survivors, previousWord),
     filtered = _applyGrammarFilter.filtered,
     grammarMeta = _applyGrammarFilter.grammarMeta;
 
   // Stage 2 + final assembly
   var lmMax = 0;
-  if (lmLogits && Object.keys(lmLogits).length > 0) {
+  if (lmLogits) {
     var values = Object.values(lmLogits);
-    lmMax = Math.max.apply(Math, values);
+    if (values.length > 0) lmMax = Math.max.apply(Math, values);
   }
   var scored = filtered.map(function (entry) {
     var lmScore = 0;
     var finalScore = entry.stage1Score;
-    if (lmLogits && Object.keys(lmLogits).length > 0) {
+    if (lmLogits && lmMax >= LM_GATE_THRESHOLD) {
       var rawLm = getLmScore(entry.candidate.word, lmLogits);
       if (rawLm !== 0) {
-        // The word was in the top_k! Score it normally.
         var logitDiff = Math.log(rawLm) - Math.log(lmMax);
         lmScore = Math.exp(logitDiff);
-      } else {
-        lmScore = 0.05;
       }
+      // Words absent from the prefix-filtered payload get lmScore = 0,
+      // not 0.05, so they don't outrank genuine LM predictions.
+
       finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
     }
     return {
@@ -244,13 +266,17 @@ function rankCandidates(candidates, contextVector, getWordVector, lmLogits, maxT
     };
   });
   scored.sort(function (a, b) {
-    if (b.finalScore !== a.finalScore) {
-      return b.finalScore - a.finalScore;
-    }
+    if (b.finalScore !== a.finalScore) return b.finalScore - a.finalScore;
     return a.word.length - b.word.length;
   });
   return {
     candidates: scored,
-    grammarMeta: grammarMeta
+    grammarMeta: grammarMeta,
+    pipelineDebug: {
+      initial: candidates.length,
+      stage1Rejected: stage1Rejected,
+      grammarRejected: (_grammarMeta$dropped = grammarMeta === null || grammarMeta === void 0 ? void 0 : grammarMeta.dropped) !== null && _grammarMeta$dropped !== void 0 ? _grammarMeta$dropped : [],
+      final: scored.length
+    }
   };
 }

package/dist/cjs/pm-plugins/text-predictor.js

@@ -573,11 +573,21 @@ var predict = exports.predict = function predict(textBefore) {
       sessionFreq: node.sessionFreq
     };
   });
+
+  // Filter the LM payload to only words matching the current prefix so that
+  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
+  var prefix = currentWord.toLowerCase();
+  var prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
+    var _ref5 = (0, _slicedToArray2.default)(_ref4, 1),
+      word = _ref5[0];
+    return word.startsWith(prefix);
+  })) : null;
   var _rankCandidates = (0, _scoringPipeline.rankCandidates)(scoringCandidates, contextVector, function (w) {
       return getWordVector(w);
-    }, lmLogits, wordTrie.maxTenantFreq, previousWord),
+    }, prefixLmLogits, wordTrie.maxTenantFreq, previousWord),
     ranked = _rankCandidates.candidates,
-    grammarMeta = _rankCandidates.grammarMeta;
+    grammarMeta = _rankCandidates.grammarMeta,
+    pipelineDebug = _rankCandidates.pipelineDebug;
   var best = ranked[0];
   var suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
   if (debugMode) {
@@ -624,7 +634,7 @@ var predict = exports.predict = function predict(textBefore) {
     console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
 
     // 4. Scoring formula active this prediction
-    var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? 'Stage1(×0.6) + LM(×0.4)' : 'Stage1 only (no LM logits)';
+    var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? "Stage1(\xD7".concat(_scoringPipeline.STAGE1_WEIGHT, ") + LM(\xD7").concat(_scoringPipeline.STAGE2_WEIGHT, ")") : 'Stage1 only (no LM logits)';
     // eslint-disable-next-line no-console
     console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
 
@@ -638,17 +648,21 @@ var predict = exports.predict = function predict(textBefore) {
       }
     }
 
-    // 6. Candidate table
+    // 6. Pipeline funnel
+    // eslint-disable-next-line no-console
+    console.log("%c[Pipeline Funnel] %c\uD83D\uDCE5 In: ".concat(pipelineDebug.initial, " | \u274C Stage 1 (< ").concat(_scoringPipeline.MIN_STAGE1_SCORE, "): -").concat(pipelineDebug.stage1Rejected.length, " | \u274C Grammar: -").concat(pipelineDebug.grammarRejected.length, " | \u2705 Final: ").concat(pipelineDebug.final), 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
+
+    // 7. Candidate table
     if (ranked.length > 0) {
       var lmCoverage = ranked.slice(0, 10).filter(function (r) {
-        return r.lmScore > 0.05;
+        return r.lmScore > 0;
       }).length;
       // eslint-disable-next-line no-console
       console.log("%cLM coverage: ".concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " candidates had real logit scores"), 'color: #888; font-style: italic;');
       var tableData = ranked.slice(0, 10).map(function (r) {
         var rawLogit = 'Not in Payload';
-        if (lmLogits) {
-          var val = lmLogits[r.word.toLowerCase()];
+        if (prefixLmLogits) {
+          var val = prefixLmLogits[r.word.toLowerCase()];
           if (val !== undefined) {
             rawLogit = Number(val.toFixed(5));
           }
@@ -694,7 +708,7 @@ var predict = exports.predict = function predict(textBefore) {
 // ─── Data Loading ────────────────────────────────────────────────────────────
 
 var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
-  var _ref4 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(options) {
+  var _ref6 = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee(options) {
     var url, res, buffer, float32, wordIndex, nWords, dim;
     return _regenerator.default.wrap(function _callee$(_context) {
       while (1) switch (_context.prev = _context.next) {
@@ -780,7 +794,7 @@ var loadVectorsAsync = exports.loadVectorsAsync = /*#__PURE__*/function () {
     }, _callee, null, [[6, 12], [17, 37]]);
   }));
   return function loadVectorsAsync(_x) {
-    return _ref4.apply(this, arguments);
+    return _ref6.apply(this, arguments);
   };
 }();
 var initVectors = exports.initVectors = function initVectors(store) {
@@ -789,10 +803,10 @@ var initVectors = exports.initVectors = function initVectors(store) {
 var loadDefaultVocabulary = exports.loadDefaultVocabulary = function loadDefaultVocabulary() {
   // 1. Load the Atlassian Domain (L2)
   var data = _vocabulary_10k.default;
-  var terms = Object.entries(data.words).map(function (_ref5) {
-    var _ref6 = (0, _slicedToArray2.default)(_ref5, 2),
-      word = _ref6[0],
-      stats = _ref6[1];
+  var terms = Object.entries(data.words).map(function (_ref7) {
+    var _ref8 = (0, _slicedToArray2.default)(_ref7, 2),
+      word = _ref8[0],
+      stats = _ref8[1];
     return {
       word: word,
       freq: stats.freq,

package/dist/es2019/pm-plugins/scoring-pipeline.js

@@ -18,10 +18,15 @@ import grammarTransitionsData from './data/grammar_transitions_10k.json';
 const ALPHA = 0.5;
 const BETA = 0.5;
 const NEUTRAL_SCORE = 0.5;
-const STAGE1_WEIGHT = 0.6;
-const STAGE2_WEIGHT = 0.4;
-const MIN_STAGE1_SCORE = 0.35;
+export const STAGE1_WEIGHT = 0.35;
+export const STAGE2_WEIGHT = 0.65;
+export const MIN_STAGE1_SCORE = 0.35;
 const L1_SESSION_CAP = 1.2;
+// Minimum prefix-payload max LM probability before Stage 2 activates.
+// Below this threshold the LM signal is too weak to suppress Stage 1 — finalScore
+// falls back to stage1Score directly. Prevents weak prefixes (e.g. "ins" → "instances"
+// at 0.00024) from triggering re-ranking.
+const LM_GATE_THRESHOLD = 0.0005;
 
 // ─── Grammar Data (loaded once on import) ───────────────────
 
@@ -158,6 +163,7 @@ function getLmScore(word, lmLogits) {
 // ─── Public API ─────────────────────────────────────────────
 
 export function rankCandidates(candidates, contextVector, getWordVector, lmLogits, maxTenantFreq, previousWord) {
+  var _grammarMeta$dropped;
   // Stage 1
   const stage1Results = candidates.map(candidate => {
     const {
@@ -172,7 +178,15 @@ export function rankCandidates(candidates, contextVector, getWordVector, lmLogit
       stage1Score
     };
   });
-  const stage1Survivors = stage1Results.filter(entry => entry.stage1Score >= MIN_STAGE1_SCORE);
+  const stage1Survivors = [];
+  const stage1Rejected = [];
+  for (const entry of stage1Results) {
+    if (entry.stage1Score >= MIN_STAGE1_SCORE) {
+      stage1Survivors.push(entry);
+    } else {
+      stage1Rejected.push(entry.candidate.word);
+    }
+  }
 
   // Grammar Filter
   const {
@@ -182,22 +196,22 @@ export function rankCandidates(candidates, contextVector, getWordVector, lmLogit
 
   // Stage 2 + final assembly
   let lmMax = 0;
-  if (lmLogits && Object.keys(lmLogits).length > 0) {
+  if (lmLogits) {
     const values = Object.values(lmLogits);
-    lmMax = Math.max(...values);
+    if (values.length > 0) lmMax = Math.max(...values);
   }
   const scored = filtered.map(entry => {
     let lmScore = 0;
     let finalScore = entry.stage1Score;
-    if (lmLogits && Object.keys(lmLogits).length > 0) {
+    if (lmLogits && lmMax >= LM_GATE_THRESHOLD) {
       const rawLm = getLmScore(entry.candidate.word, lmLogits);
       if (rawLm !== 0) {
-        // The word was in the top_k! Score it normally.
         const logitDiff = Math.log(rawLm) - Math.log(lmMax);
         lmScore = Math.exp(logitDiff);
-      } else {
-        lmScore = 0.05;
       }
+      // Words absent from the prefix-filtered payload get lmScore = 0,
+      // not 0.05, so they don't outrank genuine LM predictions.
+
       finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
     }
     return {
@@ -209,13 +223,17 @@ export function rankCandidates(candidates, contextVector, getWordVector, lmLogit
     };
   });
   scored.sort((a, b) => {
-    if (b.finalScore !== a.finalScore) {
-      return b.finalScore - a.finalScore;
-    }
+    if (b.finalScore !== a.finalScore) return b.finalScore - a.finalScore;
     return a.word.length - b.word.length;
   });
   return {
     candidates: scored,
-    grammarMeta
+    grammarMeta,
+    pipelineDebug: {
+      initial: candidates.length,
+      stage1Rejected,
+      grammarRejected: (_grammarMeta$dropped = grammarMeta === null || grammarMeta === void 0 ? void 0 : grammarMeta.dropped) !== null && _grammarMeta$dropped !== void 0 ? _grammarMeta$dropped : [],
+      final: scored.length
+    }
   };
 }

package/dist/es2019/pm-plugins/text-predictor.js

@@ -21,7 +21,7 @@ import l3VocabularyData from './data/l3_vocabulary.json';
 import vocabularyData from './data/vocabulary_10k.json';
 import wordIndexData from './data/word_index_10k.json';
 // import { rankCandidates, isGrammarAllowed } from './scoring-pipeline';
-import { rankCandidates } from './scoring-pipeline';
+import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
 import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
 
 // ─── Constants ───────────────────────────────────────────────────────────────
@@ -452,10 +452,16 @@ export const predict = textBefore => {
     authorFreq: node.authorFreq,
     sessionFreq: node.sessionFreq
   }));
+
+  // Filter the LM payload to only words matching the current prefix so that
+  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
+  const prefix = currentWord.toLowerCase();
+  const prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(([word]) => word.startsWith(prefix))) : null;
   const {
     candidates: ranked,
-    grammarMeta
-  } = rankCandidates(scoringCandidates, contextVector, w => getWordVector(w), lmLogits, wordTrie.maxTenantFreq, previousWord);
+    grammarMeta,
+    pipelineDebug
+  } = rankCandidates(scoringCandidates, contextVector, w => getWordVector(w), prefixLmLogits, wordTrie.maxTenantFreq, previousWord);
   const best = ranked[0];
   const suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
   if (debugMode) {
@@ -500,7 +506,7 @@ export const predict = textBefore => {
     console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
 
     // 4. Scoring formula active this prediction
-    const formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? 'Stage1(×0.6) + LM(×0.4)' : 'Stage1 only (no LM logits)';
+    const formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? `Stage1(×${STAGE1_WEIGHT}) + LM(×${STAGE2_WEIGHT})` : 'Stage1 only (no LM logits)';
     // eslint-disable-next-line no-console
     console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
 
@@ -514,15 +520,19 @@ export const predict = textBefore => {
       }
     }
 
-    // 6. Candidate table
+    // 6. Pipeline funnel
+    // eslint-disable-next-line no-console
+    console.log(`%c[Pipeline Funnel] %c📥 In: ${pipelineDebug.initial} | ❌ Stage 1 (< ${MIN_STAGE1_SCORE}): -${pipelineDebug.stage1Rejected.length} | ❌ Grammar: -${pipelineDebug.grammarRejected.length} | ✅ Final: ${pipelineDebug.final}`, 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
+
+    // 7. Candidate table
     if (ranked.length > 0) {
-      const lmCoverage = ranked.slice(0, 10).filter(r => r.lmScore > 0.05).length;
+      const lmCoverage = ranked.slice(0, 10).filter(r => r.lmScore > 0).length;
       // eslint-disable-next-line no-console
       console.log(`%cLM coverage: ${lmCoverage}/${Math.min(ranked.length, 10)} candidates had real logit scores`, 'color: #888; font-style: italic;');
       const tableData = ranked.slice(0, 10).map(r => {
         let rawLogit = 'Not in Payload';
-        if (lmLogits) {
-          const val = lmLogits[r.word.toLowerCase()];
+        if (prefixLmLogits) {
+          const val = prefixLmLogits[r.word.toLowerCase()];
           if (val !== undefined) {
             rawLogit = Number(val.toFixed(5));
           }

package/dist/esm/pm-plugins/scoring-pipeline.js

@@ -22,10 +22,15 @@ import grammarTransitionsData from './data/grammar_transitions_10k.json';
 var ALPHA = 0.5;
 var BETA = 0.5;
 var NEUTRAL_SCORE = 0.5;
-var STAGE1_WEIGHT = 0.6;
-var STAGE2_WEIGHT = 0.4;
-var MIN_STAGE1_SCORE = 0.35;
+export var STAGE1_WEIGHT = 0.35;
+export var STAGE2_WEIGHT = 0.65;
+export var MIN_STAGE1_SCORE = 0.35;
 var L1_SESSION_CAP = 1.2;
+// Minimum prefix-payload max LM probability before Stage 2 activates.
+// Below this threshold the LM signal is too weak to suppress Stage 1 — finalScore
+// falls back to stage1Score directly. Prevents weak prefixes (e.g. "ins" → "instances"
+// at 0.00024) from triggering re-ranking.
+var LM_GATE_THRESHOLD = 0.0005;
 
 // ─── Grammar Data (loaded once on import) ───────────────────
 
@@ -189,6 +194,7 @@ function getLmScore(word, lmLogits) {
 // ─── Public API ─────────────────────────────────────────────
 
 export function rankCandidates(candidates, contextVector, getWordVector, lmLogits, maxTenantFreq, previousWord) {
+  var _grammarMeta$dropped;
   // Stage 1
   var stage1Results = candidates.map(function (candidate) {
     var _scoreStage = scoreStage1(candidate, contextVector, getWordVector, maxTenantFreq),
@@ -202,33 +208,48 @@ export function rankCandidates(candidates, contextVector, getWordVector, lmLogit
       stage1Score: stage1Score
     };
   });
-  var stage1Survivors = stage1Results.filter(function (entry) {
-    return entry.stage1Score >= MIN_STAGE1_SCORE;
-  });
+  var stage1Survivors = [];
+  var stage1Rejected = [];
+  var _iterator3 = _createForOfIteratorHelper(stage1Results),
+    _step3;
+  try {
+    for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
+      var entry = _step3.value;
+      if (entry.stage1Score >= MIN_STAGE1_SCORE) {
+        stage1Survivors.push(entry);
+      } else {
+        stage1Rejected.push(entry.candidate.word);
+      }
+    }
 
-  // Grammar Filter
+    // Grammar Filter
+  } catch (err) {
+    _iterator3.e(err);
+  } finally {
+    _iterator3.f();
+  }
   var _applyGrammarFilter = applyGrammarFilter(stage1Survivors, previousWord),
     filtered = _applyGrammarFilter.filtered,
     grammarMeta = _applyGrammarFilter.grammarMeta;
 
   // Stage 2 + final assembly
   var lmMax = 0;
-  if (lmLogits && Object.keys(lmLogits).length > 0) {
+  if (lmLogits) {
     var values = Object.values(lmLogits);
-    lmMax = Math.max.apply(Math, values);
+    if (values.length > 0) lmMax = Math.max.apply(Math, values);
   }
   var scored = filtered.map(function (entry) {
     var lmScore = 0;
     var finalScore = entry.stage1Score;
-    if (lmLogits && Object.keys(lmLogits).length > 0) {
+    if (lmLogits && lmMax >= LM_GATE_THRESHOLD) {
       var rawLm = getLmScore(entry.candidate.word, lmLogits);
       if (rawLm !== 0) {
-        // The word was in the top_k! Score it normally.
         var logitDiff = Math.log(rawLm) - Math.log(lmMax);
         lmScore = Math.exp(logitDiff);
-      } else {
-        lmScore = 0.05;
       }
+      // Words absent from the prefix-filtered payload get lmScore = 0,
+      // not 0.05, so they don't outrank genuine LM predictions.
+
       finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
     }
     return {
@@ -240,13 +261,17 @@ export function rankCandidates(candidates, contextVector, getWordVector, lmLogit
     };
   });
   scored.sort(function (a, b) {
-    if (b.finalScore !== a.finalScore) {
-      return b.finalScore - a.finalScore;
-    }
+    if (b.finalScore !== a.finalScore) return b.finalScore - a.finalScore;
     return a.word.length - b.word.length;
   });
   return {
     candidates: scored,
-    grammarMeta: grammarMeta
+    grammarMeta: grammarMeta,
+    pipelineDebug: {
+      initial: candidates.length,
+      stage1Rejected: stage1Rejected,
+      grammarRejected: (_grammarMeta$dropped = grammarMeta === null || grammarMeta === void 0 ? void 0 : grammarMeta.dropped) !== null && _grammarMeta$dropped !== void 0 ? _grammarMeta$dropped : [],
+      final: scored.length
+    }
   };
 }

package/dist/esm/pm-plugins/text-predictor.js

@@ -29,7 +29,7 @@ import l3VocabularyData from './data/l3_vocabulary.json';
 import vocabularyData from './data/vocabulary_10k.json';
 import wordIndexData from './data/word_index_10k.json';
 // import { rankCandidates, isGrammarAllowed } from './scoring-pipeline';
-import { rankCandidates } from './scoring-pipeline';
+import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
 import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
 
 // ─── Constants ───────────────────────────────────────────────────────────────
@@ -570,11 +570,21 @@ export var predict = function predict(textBefore) {
       sessionFreq: node.sessionFreq
     };
   });
+
+  // Filter the LM payload to only words matching the current prefix so that
+  // lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
+  var prefix = currentWord.toLowerCase();
+  var prefixLmLogits = lmLogits ? Object.fromEntries(Object.entries(lmLogits).filter(function (_ref4) {
+    var _ref5 = _slicedToArray(_ref4, 1),
+      word = _ref5[0];
+    return word.startsWith(prefix);
+  })) : null;
   var _rankCandidates = rankCandidates(scoringCandidates, contextVector, function (w) {
       return getWordVector(w);
-    }, lmLogits, wordTrie.maxTenantFreq, previousWord),
+    }, prefixLmLogits, wordTrie.maxTenantFreq, previousWord),
     ranked = _rankCandidates.candidates,
-    grammarMeta = _rankCandidates.grammarMeta;
+    grammarMeta = _rankCandidates.grammarMeta,
+    pipelineDebug = _rankCandidates.pipelineDebug;
   var best = ranked[0];
   var suggestion = best && best.finalScore >= MIN_SCORE_THRESHOLD ? best.word.slice(currentWord.length) : null;
   if (debugMode) {
@@ -621,7 +631,7 @@ export var predict = function predict(textBefore) {
     console.log('%cSlow Lane:', 'color: #888; font-style: italic;', vectorStatus, '|', logitsStatus);
 
     // 4. Scoring formula active this prediction
-    var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? 'Stage1(×0.6) + LM(×0.4)' : 'Stage1 only (no LM logits)';
+    var formulaLabel = lmLogits && Object.keys(lmLogits).length > 0 ? "Stage1(\xD7".concat(STAGE1_WEIGHT, ") + LM(\xD7").concat(STAGE2_WEIGHT, ")") : 'Stage1 only (no LM logits)';
     // eslint-disable-next-line no-console
     console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
 
@@ -635,17 +645,21 @@ export var predict = function predict(textBefore) {
       }
     }
 
-    // 6. Candidate table
+    // 6. Pipeline funnel
+    // eslint-disable-next-line no-console
+    console.log("%c[Pipeline Funnel] %c\uD83D\uDCE5 In: ".concat(pipelineDebug.initial, " | \u274C Stage 1 (< ").concat(MIN_STAGE1_SCORE, "): -").concat(pipelineDebug.stage1Rejected.length, " | \u274C Grammar: -").concat(pipelineDebug.grammarRejected.length, " | \u2705 Final: ").concat(pipelineDebug.final), 'color: #9c27b0; font-weight: bold;', 'color: inherit;');
+
+    // 7. Candidate table
     if (ranked.length > 0) {
       var lmCoverage = ranked.slice(0, 10).filter(function (r) {
-        return r.lmScore > 0.05;
+        return r.lmScore > 0;
       }).length;
       // eslint-disable-next-line no-console
       console.log("%cLM coverage: ".concat(lmCoverage, "/").concat(Math.min(ranked.length, 10), " candidates had real logit scores"), 'color: #888; font-style: italic;');
       var tableData = ranked.slice(0, 10).map(function (r) {
         var rawLogit = 'Not in Payload';
-        if (lmLogits) {
-          var val = lmLogits[r.word.toLowerCase()];
+        if (prefixLmLogits) {
+          var val = prefixLmLogits[r.word.toLowerCase()];
           if (val !== undefined) {
             rawLogit = Number(val.toFixed(5));
           }
@@ -691,7 +705,7 @@ export var predict = function predict(textBefore) {
 // ─── Data Loading ────────────────────────────────────────────────────────────
 
 export var loadVectorsAsync = /*#__PURE__*/function () {
-  var _ref4 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
+  var _ref6 = _asyncToGenerator( /*#__PURE__*/_regeneratorRuntime.mark(function _callee(options) {
     var url, res, buffer, float32, wordIndex, nWords, dim;
     return _regeneratorRuntime.wrap(function _callee$(_context) {
       while (1) switch (_context.prev = _context.next) {
@@ -777,7 +791,7 @@ export var loadVectorsAsync = /*#__PURE__*/function () {
     }, _callee, null, [[6, 12], [17, 37]]);
   }));
   return function loadVectorsAsync(_x) {
-    return _ref4.apply(this, arguments);
+    return _ref6.apply(this, arguments);
   };
 }();
 export var initVectors = function initVectors(store) {
@@ -786,10 +800,10 @@ export var initVectors = function initVectors(store) {
 export var loadDefaultVocabulary = function loadDefaultVocabulary() {
   // 1. Load the Atlassian Domain (L2)
   var data = vocabularyData;
-  var terms = Object.entries(data.words).map(function (_ref5) {
-    var _ref6 = _slicedToArray(_ref5, 2),
-      word = _ref6[0],
-      stats = _ref6[1];
+  var terms = Object.entries(data.words).map(function (_ref7) {
+    var _ref8 = _slicedToArray(_ref7, 2),
+      word = _ref8[0],
+      stats = _ref8[1];
     return {
       word: word,
       freq: stats.freq,

package/dist/types/pm-plugins/scoring-pipeline.d.ts

@@ -26,8 +26,18 @@ export interface GrammarFilterMeta {
     prevTags: string[];
     prevWord: string;
 }
+export declare const STAGE1_WEIGHT = 0.35;
+export declare const STAGE2_WEIGHT = 0.65;
+export declare const MIN_STAGE1_SCORE = 0.35;
+export interface PipelineDebug {
+    final: number;
+    grammarRejected: string[];
+    initial: number;
+    stage1Rejected: string[];
+}
 export interface RankCandidatesResult {
     candidates: ScoredCandidate[];
     grammarMeta: GrammarFilterMeta | null;
+    pipelineDebug: PipelineDebug;
 }
 export declare function rankCandidates(candidates: ScoringCandidate[], contextVector: Float32Array | null, getWordVector: (word: string) => Float32Array | null, lmLogits: Record<string, number> | null, maxTenantFreq: number, previousWord: string): RankCandidatesResult;

package/dist/types-ts4.5/pm-plugins/scoring-pipeline.d.ts

@@ -26,8 +26,18 @@ export interface GrammarFilterMeta {
     prevTags: string[];
     prevWord: string;
 }
+export declare const STAGE1_WEIGHT = 0.35;
+export declare const STAGE2_WEIGHT = 0.65;
+export declare const MIN_STAGE1_SCORE = 0.35;
+export interface PipelineDebug {
+    final: number;
+    grammarRejected: string[];
+    initial: number;
+    stage1Rejected: string[];
+}
 export interface RankCandidatesResult {
     candidates: ScoredCandidate[];
     grammarMeta: GrammarFilterMeta | null;
+    pipelineDebug: PipelineDebug;
 }
 export declare function rankCandidates(candidates: ScoringCandidate[], contextVector: Float32Array | null, getWordVector: (word: string) => Float32Array | null, lmLogits: Record<string, number> | null, maxTenantFreq: number, previousWord: string): RankCandidatesResult;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@atlaskit/editor-plugin-autocomplete",
-	"version": "1.0.0",
+	"version": "2.1.0",
 	"description": "Client-side text autocomplete plugin for @atlaskit/editor-core",
 	"author": "Atlassian Pty Ltd",
 	"license": "Apache-2.0",
@@ -33,7 +33,7 @@
 		"wink-nlp": "^2.4.0"
 	},
 	"peerDependencies": {
-		"@atlaskit/editor-common": "^113.0.0",
+		"@atlaskit/editor-common": "^114.1.0",
 		"react": "^18.2.0"
 	},
 	"techstack": {

package/src/pm-plugins/scoring-pipeline.ts

@@ -49,10 +49,15 @@ interface GrammarTransitions {
 const ALPHA = 0.5;
 const BETA = 0.5;
 const NEUTRAL_SCORE = 0.5;
-const STAGE1_WEIGHT = 0.6;
-const STAGE2_WEIGHT = 0.4;
-const MIN_STAGE1_SCORE = 0.35;
+export const STAGE1_WEIGHT = 0.35;
+export const STAGE2_WEIGHT = 0.65;
+export const MIN_STAGE1_SCORE = 0.35;
 const L1_SESSION_CAP = 1.2;
+// Minimum prefix-payload max LM probability before Stage 2 activates.
+// Below this threshold the LM signal is too weak to suppress Stage 1 — finalScore
+// falls back to stage1Score directly. Prevents weak prefixes (e.g. "ins" → "instances"
+// at 0.00024) from triggering re-ranking.
+const LM_GATE_THRESHOLD = 0.0005;
 
 // ─── Grammar Data (loaded once on import) ───────────────────
 
@@ -217,9 +222,17 @@ function getLmScore(word: string, lmLogits: Record<string, number> | null): numb
 
 // ─── Public API ─────────────────────────────────────────────
 
+export interface PipelineDebug {
+	final: number;
+	grammarRejected: string[];
+	initial: number;
+	stage1Rejected: string[];
+}
+
 export interface RankCandidatesResult {
 	candidates: ScoredCandidate[];
 	grammarMeta: GrammarFilterMeta | null;
+	pipelineDebug: PipelineDebug;
 }
 
 export function rankCandidates(
@@ -240,33 +253,39 @@ export function rankCandidates(
 		);
 		return { candidate, semanticScore, freqScore, stage1Score };
 	});
-
-	const stage1Survivors = stage1Results.filter((entry) => entry.stage1Score >= MIN_STAGE1_SCORE);
+	const stage1Survivors: typeof stage1Results = [];
+	const stage1Rejected: string[] = [];
+	for (const entry of stage1Results) {
+		if (entry.stage1Score >= MIN_STAGE1_SCORE) {
+			stage1Survivors.push(entry);
+		} else {
+			stage1Rejected.push(entry.candidate.word);
+		}
+	}
 
 	// Grammar Filter
 	const { filtered, grammarMeta } = applyGrammarFilter(stage1Survivors, previousWord);
 
 	// Stage 2 + final assembly
 	let lmMax = 0;
-	if (lmLogits && Object.keys(lmLogits).length > 0) {
+	if (lmLogits) {
 		const values = Object.values(lmLogits);
-		lmMax = Math.max(...values);
+		if (values.length > 0) lmMax = Math.max(...values);
 	}
 
 	const scored: ScoredCandidate[] = filtered.map((entry) => {
 		let lmScore = 0;
 		let finalScore = entry.stage1Score;
 
-		if (lmLogits && Object.keys(lmLogits).length > 0) {
+		if (lmLogits && lmMax >= LM_GATE_THRESHOLD) {
 			const rawLm = getLmScore(entry.candidate.word, lmLogits);
 
 			if (rawLm !== 0) {
-				// The word was in the top_k! Score it normally.
 				const logitDiff = Math.log(rawLm) - Math.log(lmMax);
 				lmScore = Math.exp(logitDiff);
-			} else {
-				lmScore = 0.05;
 			}
+			// Words absent from the prefix-filtered payload get lmScore = 0,
+			// not 0.05, so they don't outrank genuine LM predictions.
 
 			finalScore = STAGE1_WEIGHT * entry.stage1Score + STAGE2_WEIGHT * lmScore;
 		}
@@ -281,11 +300,18 @@ export function rankCandidates(
 	});
 
 	scored.sort((a, b) => {
-		if (b.finalScore !== a.finalScore) {
-			return b.finalScore - a.finalScore;
-		}
+		if (b.finalScore !== a.finalScore) return b.finalScore - a.finalScore;
 		return a.word.length - b.word.length;
 	});
 
-	return { candidates: scored, grammarMeta };
+	return {
+		candidates: scored,
+		grammarMeta,
+		pipelineDebug: {
+			initial: candidates.length,
+			stage1Rejected,
+			grammarRejected: grammarMeta?.dropped ?? [],
+			final: scored.length,
+		},
+	};
 }

package/src/pm-plugins/text-predictor.ts

@@ -20,7 +20,7 @@ import l3VocabularyData from './data/l3_vocabulary.json';
 import vocabularyData from './data/vocabulary_10k.json';
 import wordIndexData from './data/word_index_10k.json';
 // import { rankCandidates, isGrammarAllowed } from './scoring-pipeline';
-import { rankCandidates } from './scoring-pipeline';
+import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
 import type { ScoringCandidate } from './scoring-pipeline';
 import { getStoredContextVector, getStoredLmLogits } from './slow-lane-client';
 
@@ -533,11 +533,19 @@ export const predict = (textBefore: string): string | null => {
 		sessionFreq: node.sessionFreq,
 	}));
 
-	const { candidates: ranked, grammarMeta } = rankCandidates(
+	// Filter the LM payload to only words matching the current prefix so that
+	// lmMax in rankCandidates reflects prefix-relevant signal, not the global distribution.
+	const prefix = currentWord.toLowerCase();
+	const prefixLmLogits = lmLogits
+		? Object.fromEntries(
+			Object.entries(lmLogits).filter(([word]) => word.startsWith(prefix))
+		)
+		: null;
+	const { candidates: ranked, grammarMeta, pipelineDebug } = rankCandidates(
 		scoringCandidates,
 		contextVector,
 		(w: string) => getWordVector(w),
-		lmLogits,
+		prefixLmLogits,
 		wordTrie.maxTenantFreq,
 		previousWord,
 	);
@@ -624,7 +632,7 @@ export const predict = (textBefore: string): string | null => {
 		// 4. Scoring formula active this prediction
 		const formulaLabel =
 			lmLogits && Object.keys(lmLogits).length > 0
-				? 'Stage1(×0.6) + LM(×0.4)'
+				? `Stage1(×${STAGE1_WEIGHT}) + LM(×${STAGE2_WEIGHT})`
 				: 'Stage1 only (no LM logits)';
 		// eslint-disable-next-line no-console
 		console.log('%cFormula:', 'color: #888; font-style: italic;', formulaLabel);
@@ -645,9 +653,17 @@ export const predict = (textBefore: string): string | null => {
 			}
 		}
 
-		// 6. Candidate table
+		// 6. Pipeline funnel
+		// eslint-disable-next-line no-console
+		console.log(
+			`%c[Pipeline Funnel] %c📥 In: ${pipelineDebug.initial} | ❌ Stage 1 (< ${MIN_STAGE1_SCORE}): -${pipelineDebug.stage1Rejected.length} | ❌ Grammar: -${pipelineDebug.grammarRejected.length} | ✅ Final: ${pipelineDebug.final}`,
+			'color: #9c27b0; font-weight: bold;',
+			'color: inherit;',
+		);
+
+		// 7. Candidate table
 		if (ranked.length > 0) {
-			const lmCoverage = ranked.slice(0, 10).filter((r) => r.lmScore > 0.05).length;
+			const lmCoverage = ranked.slice(0, 10).filter((r) => r.lmScore > 0).length;
 			// eslint-disable-next-line no-console
 			console.log(
 				`%cLM coverage: ${lmCoverage}/${Math.min(ranked.length, 10)} candidates had real logit scores`,
@@ -656,8 +672,8 @@ export const predict = (textBefore: string): string | null => {
 
 			const tableData = ranked.slice(0, 10).map((r) => {
 				let rawLogit: string | number = 'Not in Payload';
-				if (lmLogits) {
-					const val = lmLogits[r.word.toLowerCase()];
+				if (prefixLmLogits) {
+					const val = prefixLmLogits[r.word.toLowerCase()];
 					if (val !== undefined) {
 						rawLogit = Number(val.toFixed(5));
 					}