@atlaskit/editor-plugin-autocomplete 9.0.0 → 9.2.0

package/dist/es2019/pm-plugins/text-predictor.js

@@ -13,14 +13,17 @@ import _defineProperty from "@babel/runtime/helpers/defineProperty";
  * Falls back to cold mode (freq-only) when vectors not yet loaded.
  *
  * Session personalization (L1): words the user types are incrementally boosted
- * via incrementSessionFreq(), called on word boundaries from the plugin.
+ * via incrementSessionFreq(), called on word boundaries from the plugin, and
+ * words in ingested context text via ingestDocumentPage(). What the session has
+ * boosted is visible at any time from the console: `__atlCtcDebug__.session()`,
+ * or `__atlCtcDebug__.session('poll')` for one family — see inspectSessionBoosts.
  */
 
 import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
 import { fetchAutocompleteArtifactBinary, fetchAutocompleteArtifactJson } from './artifact-loader';
 import { ARTIFACT_NAME } from './artifacts-manifest';
 import { createCanonicalContextPositionCache, deriveCanonicalCandidateContext, deriveWhitespaceBoundaryContext, logSoftmaxAt, logSumExp, selectBoundaryPrimeRequests } from './canonical-lm-scoring';
-import { CTC_STYLES, ctcSection, ctcTag, isAutocompleteDebugEnabled, isAutocompleteDebugVerbose } from './debug-mode';
+import { CTC_STYLES, ctcSection, ctcTag, isAutocompleteDebugEnabled, isAutocompleteDebugVerbose, registerCtcSessionInspector } from './debug-mode';
 import { loadGrammarDataAsync, rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE, MIN_WINNER_MARGIN } from './scoring-pipeline';
 import { getBoundaryLmState, getCanonicalSurfaceCount, getCanonicalSurfaceTokenIds, getDefaultSlowLaneClientStatus, getProgressiveSurfaceEvidence, getStoredContextInput, getStoredContextVector, getStoredLmLogits, getSurfaceScore, isCanonicalSurfaceScoringSupported, primeBoundaryLm, requestProgressiveSurfaceScores } from './slow-lane-client';
 
@@ -172,6 +175,18 @@ const DEBUG_TEXT_TAIL_CHARS = 120;
  * is vocabulary coverage.
  */
 
+/**
+ * What the scored path concluded, recorded on every evaluation whether or not
+ * debug is on.
+ *
+ * This exists for the inline-code harvester, which may only offer a harvested
+ * surface once the scored path has finished and come away empty. The reason is
+ * the load-bearing part: `no-candidate` means no vocabulary reaches this prefix
+ * at all, while `winner-margin` means two known words the model cannot yet
+ * separate — the first is a gap worth filling and the second is a prefix
+ * ambiguous enough that filling it would be a guess.
+ */
+
 /**
  * A candidate paired with the length of the already-typed prefix it completes.
  * For a single word this is the current partial token length; for a phrase it
@@ -200,6 +215,17 @@ class TrieNode {
 class WeightedWordTrie {
   constructor() {
     _defineProperty(this, "root", new TrieNode());
+    /**
+     * Every node a session boost has been written to.
+     *
+     * Kept because dropping the boosts is no longer a rare event — it happens
+     * each time the reader changes page or conversation — and walking a vocabulary
+     * of tens of thousands of words to find the few hundred that were touched
+     * costs about 10ms of main thread every time. The two writers below are the
+     * only way a `sessionFreq` moves, so keeping this in step costs one Set
+     * insertion on a path that is already descending the trie.
+     */
+    _defineProperty(this, "boostedNodes", new Set());
     /** Highest tenantFreq seen — used to normalize freq scores at query time */
     _defineProperty(this, "maxTenantFreq", 1);
   }
@@ -273,6 +299,11 @@ class WeightedWordTrie {
     return node.word !== null ? node : null;
   }
 
+  /** Whether this exact surface is stored as a terminal word. */
+  hasWord(word) {
+    return this.findNode(word) !== null;
+  }
+
   /**
    * Set the session frequency for a word.
    * Returns true if the word exists in the trie.
@@ -283,6 +314,7 @@ class WeightedWordTrie {
       return false;
     }
     node.sessionFreq = count;
+    this.boostedNodes.add(node);
     return true;
   }
 
@@ -296,8 +328,58 @@ class WeightedWordTrie {
       return false;
     }
     node.sessionFreq += 1;
+    this.boostedNodes.add(node);
     return true;
   }
+
+  /**
+   * Zero every session boost, in the number of words boosted rather than the
+   * number of words known.
+   */
+  clearSessionBoosts() {
+    for (const node of this.boostedNodes) {
+      node.sessionFreq = 0;
+    }
+    this.boostedNodes.clear();
+  }
+
+  /**
+   * Every word carrying a session boost, optionally limited to one prefix's
+   * subtree. Unlike `getCandidates` a word equal to the prefix is included,
+   * since the question here is what the session holds rather than what could
+   * still be typed.
+   *
+   * Walks the trie rather than reading `boostedNodes`, since a prefix answer is
+   * a subtree question and this only runs when a human asks it.
+   */
+  collectSessionBoosted(prefix = '') {
+    let node = this.root;
+    for (const char of prefix.toLowerCase()) {
+      const next = node.children.get(char);
+      if (!next) {
+        return [];
+      }
+      node = next;
+    }
+    const boosted = [];
+    const stack = [node];
+    while (stack.length > 0) {
+      const current = stack.pop();
+      if (!current) {
+        continue;
+      }
+      if (current.word !== null && current.sessionFreq > 0) {
+        boosted.push({
+          word: current.word,
+          node: current
+        });
+      }
+      for (const child of current.children.values()) {
+        stack.push(child);
+      }
+    }
+    return boosted;
+  }
 }
 
 // L1/L2 Trie (Session + Atlassian Domain)
@@ -349,6 +431,13 @@ let phraseTermCount = 0;
 let maxBigramFreq = 1;
 let maxPhraseFreq = 1;
 let lastPredictionDebug = null;
+let lastPredictionOutcome = null;
+const recordPredictionOutcome = outcome => {
+  lastPredictionOutcome = outcome;
+};
+
+/** The verdict from the most recent `predict` call. Always populated. */
+export const getLastPredictionOutcome = () => lastPredictionOutcome;
 
 // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
 // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
@@ -380,6 +469,22 @@ export const noteSuggestionAccepted = surface => {
   }
 };
 
+/**
+ * Whether `surface` is the one the user just accepted and is still inside its
+ * cooldown window.
+ *
+ * Read-only, unlike the advance inside `predict`: the cooldown is measured in
+ * predictions, and a caller asking whether it is active must not consume one of
+ * them. Exported for the harvest path, which displays without going through
+ * arbitration and so would otherwise re-offer what was just accepted.
+ */
+export const isSurfaceInAcceptCooldown = surface => {
+  if (!acceptCooldown || acceptCooldown.surface !== surface.trim().toLowerCase()) {
+    return false;
+  }
+  return acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES || performance.now() - acceptCooldown.ts < COOLDOWN_MS;
+};
+
 /** Get vector for a word from the store. */
 const getWordVector = word => {
   if (!vectorStore) {
@@ -649,6 +754,36 @@ export const incrementSessionFreq = word => {
   wordTrie.incrementSessionFreq(word);
 };
 
+/**
+ * Drop every L1 boost this session has accumulated.
+ *
+ * The vocabulary itself is left alone: only `sessionFreq` is cleared, so the
+ * tenant and generic frequencies a boost was sitting on top of survive. Called
+ * when the plugin decides the session it was learning for has ended — a new
+ * conversation, or a different page — since a boost is a claim about what is
+ * being discussed and that claim does not carry over.
+ */
+export const resetSessionBoosts = () => {
+  wordTrie.clearSessionBoosts();
+  // Anything memoized against the old boosts is now describing a session that
+  // no longer exists.
+  recallGeneration++;
+};
+
+/**
+ * Which vocabulary already holds this surface, if any.
+ *
+ * Used by the inline-code harvester to drop terms the scored path can already
+ * serve, so that harvesting stays limited to words with no route to a
+ * suggestion today.
+ */
+export const lookupVocabularySource = word => {
+  if (wordTrie.hasWord(word)) {
+    return 'l2';
+  }
+  return l3Trie.hasWord(word) ? 'l3' : null;
+};
+
 /**
  * Prime session frequencies from a document page string.
  *
@@ -672,7 +807,7 @@ export const ingestDocumentPage = pageContent => {
     }
   }
   if (isAutocompleteDebugEnabled() && validBoostedWords.size > 0) {
-    ctcTag('init', `L1 session primed ${validBoostedWords.size} words from page`, CTC_STYLES.brand);
+    ctcTag('init', `L1 session primed ${validBoostedWords.size} words from page · __atlCtcDebug__.session() to inspect`, CTC_STYLES.brand);
     if (isAutocompleteDebugVerbose()) {
       // eslint-disable-next-line no-console
       console.dir(Array.from(validBoostedWords).sort());
@@ -680,6 +815,54 @@ export const ingestDocumentPage = pageContent => {
   }
 };
 
+/**
+ * How many boosted words `inspectSessionBoosts` lists.
+ *
+ * A page ingest can boost thousands, and a list that long is not read. The
+ * strongest boosts are the ones that change an ordering, and `boosted` still
+ * reports the full size, so the cap loses nothing but volume.
+ */
+const MAX_LISTED_SESSION_WORDS = 100;
+/**
+ * Read the session's L1 boosts, optionally narrowed to a prefix.
+ *
+ * Installed as `__atlCtcDebug__.session()`, with `__atlCtcDebug__.session('poll')`
+ * to ask about one family. Returned rather than logged, so the console renders it
+ * as an inspectable object and a caller can assert on it.
+ *
+ * Only words the vocabulary already holds can carry a boost, because both writers
+ * go through `incrementSessionFreq` and it only finds existing nodes. An ingested
+ * word absent from the vocabulary is therefore missing from here and always will
+ * be — that gap is what the inline-code harvester covers, and those surfaces show
+ * up under `__atlCtcDebug__.harvest()` instead.
+ */
+export const inspectSessionBoosts = prefix => {
+  const boosted = wordTrie.collectSessionBoosted(prefix !== null && prefix !== void 0 ? prefix : '');
+  return {
+    boosted: boosted.length,
+    limit: MAX_LISTED_SESSION_WORDS,
+    ...(prefix === undefined ? {} : {
+      prefix
+    }),
+    words: boosted.sort((a, b) => b.node.sessionFreq - a.node.sessionFreq || a.word.localeCompare(b.word, 'en', {
+      numeric: true,
+      sensitivity: 'base'
+    })).slice(0, MAX_LISTED_SESSION_WORDS).map(({
+      node,
+      word
+    }) => ({
+      sessionFreq: node.sessionFreq,
+      sessionOnly: node.tenantFreq === 0,
+      surface: word,
+      tenantFreq: node.tenantFreq
+    }))
+  };
+};
+
+// At module scope so the console answers before the first keystroke, which is
+// when someone reaching for it usually asks.
+registerCtcSessionInspector(inspectSessionBoosts);
+
 /**
  * Result of a prediction: the ghost tail to insert plus an immutable record of
  * the evidence that authorized the UI commitment.
@@ -783,6 +966,14 @@ export const predict = textBefore => {
     void loadDefaultVocabulary({
       source: 'predict'
     }).catch(() => {});
+    // Awaiting, not empty-handed: with no vocabulary loaded the harvester's own
+    // intake filter has not been applied to anything either.
+    recordPredictionOutcome({
+      abstainReason: 'not-initialized',
+      awaitingAsyncEvidence: true,
+      scoredCandidateCount: 0,
+      textBefore
+    });
     return null;
   }
   const t0 = performance.now();
@@ -812,11 +1003,23 @@ export const predict = textBefore => {
         priority: 0
       });
     }
+    recordPredictionOutcome({
+      abstainReason: 'prefetch',
+      awaitingAsyncEvidence: true,
+      scoredCandidateCount: 0,
+      textBefore
+    });
     return null;
   }
   const trimmed = textBefore.trimEnd();
   const trailingSurfaceToken = (_trimmed$match$ = (_trimmed$match = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)) === null || _trimmed$match === void 0 ? void 0 : _trimmed$match[0]) !== null && _trimmed$match$ !== void 0 ? _trimmed$match$ : '';
   if (trailingSurfaceToken.length === 0) {
+    recordPredictionOutcome({
+      abstainReason: 'no-surface-token',
+      awaitingAsyncEvidence: false,
+      scoredCandidateCount: 0,
+      textBefore
+    });
     return null;
   }
   const currentWord = trailingSurfaceToken;
@@ -828,6 +1031,14 @@ export const predict = textBefore => {
 
   // If every trie was empty for this prefix
   if (canonicalMatched.length === 0) {
+    // Terminal, and the only verdict that says the vocabulary has no claim on
+    // this prefix at all — which is what makes it the harvester's cue.
+    recordPredictionOutcome({
+      abstainReason: 'no-candidate',
+      awaitingAsyncEvidence: false,
+      scoredCandidateCount: 0,
+      textBefore
+    });
     if (isAutocompleteDebugEnabled()) {
       // eslint-disable-next-line no-console
       console.log(`%c[CTC]%c — abstain: no matches for "${currentWord}"`, CTC_STYLES.brand, CTC_STYLES.body);
@@ -1511,6 +1722,12 @@ export const predict = textBefore => {
     return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
   };
   const abstainReason = resolveAbstainReason();
+  recordPredictionOutcome({
+    abstainReason,
+    awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
+    scoredCandidateCount: ranked.length,
+    textBefore
+  });
 
   // The leader is the best-supported candidate, not the selected one: an
   // evaluation that showed nothing is exactly the one whose posterior needs
@@ -1561,6 +1778,9 @@ export const predict = textBefore => {
       'missing-artifact': 'canonical artifact coverage missing',
       'no-candidate': 'nothing cleared scoring',
       'no-evidence': 'full-surface evidence absent',
+      // Recorded at the two exits above this block, so they never print here.
+      'no-surface-token': 'no trailing surface token',
+      'not-initialized': 'vocabulary not loaded',
       prefetch: `prefetch: ${currentWord.length}/${DISPLAY_MIN_PREFIX_LENGTH} chars`,
       'short-completion': `completion shorter than ${MIN_SUGGESTION_LENGTH} chars`,
       'unresolved-rival': 'expanding plausible token-prefix groups',