@atlaskit/editor-plugin-autocomplete 9.1.0 → 9.2.1

package/src/pm-plugins/text-predictor.ts

@@ -212,7 +212,7 @@ const DEBUG_TEXT_TAIL_CHARS = 120;
  * `cold-competitor` and `unresolved-rival` are scheduling, and `no-candidate`
  * is vocabulary coverage.
  */
-type CtcAbstainReason =
+export type CtcAbstainReason =
 	| 'below-posterior-gate'
 	| 'cold-competitor'
 	| 'empty-completion'
@@ -221,11 +221,34 @@ type CtcAbstainReason =
 	| 'missing-artifact'
 	| 'no-candidate'
 	| 'no-evidence'
+	| 'no-surface-token'
+	| 'not-initialized'
 	| 'prefetch'
 	| 'short-completion'
 	| 'unresolved-rival'
 	| 'winner-margin';
 
+/**
+ * What the scored path concluded, recorded on every evaluation whether or not
+ * debug is on.
+ *
+ * This exists for the inline-code harvester, which may only offer a harvested
+ * surface once the scored path has finished and come away empty. The reason is
+ * the load-bearing part: `no-candidate` means no vocabulary reaches this prefix
+ * at all, while `winner-margin` means two known words the model cannot yet
+ * separate — the first is a gap worth filling and the second is a prefix
+ * ambiguous enough that filling it would be a guess.
+ */
+export interface PredictionOutcome {
+	abstainReason: CtcAbstainReason | null;
+	/** True while a pending async signal could still change the verdict. */
+	awaitingAsyncEvidence: boolean;
+	/** How many vocabulary candidates were scored for this prefix. */
+	scoredCandidateCount: number;
+	/** The exact string passed to `predict`, so a caller can confirm identity. */
+	textBefore: string;
+}
+
 interface Candidate {
 	node: TrieNode;
 	word: string;
@@ -288,6 +311,17 @@ class TrieNode {
 
 class WeightedWordTrie {
 	private root = new TrieNode();
+	/**
+	 * Every node a session boost has been written to.
+	 *
+	 * Kept because dropping the boosts is no longer a rare event — it happens
+	 * each time the reader changes page or conversation — and walking a vocabulary
+	 * of tens of thousands of words to find the few hundred that were touched
+	 * costs about 10ms of main thread every time. The two writers below are the
+	 * only way a `sessionFreq` moves, so keeping this in step costs one Set
+	 * insertion on a path that is already descending the trie.
+	 */
+	private boostedNodes = new Set<TrieNode>();
 	/** Highest tenantFreq seen — used to normalize freq scores at query time */
 	maxTenantFreq: number = 1;
 
@@ -367,6 +401,11 @@ class WeightedWordTrie {
 		return node.word !== null ? node : null;
 	}
 
+	/** Whether this exact surface is stored as a terminal word. */
+	hasWord(word: string): boolean {
+		return this.findNode(word) !== null;
+	}
+
 	/**
 	 * Set the session frequency for a word.
 	 * Returns true if the word exists in the trie.
@@ -377,6 +416,7 @@ class WeightedWordTrie {
 			return false;
 		}
 		node.sessionFreq = count;
+		this.boostedNodes.add(node);
 		return true;
 	}
 
@@ -390,17 +430,29 @@ class WeightedWordTrie {
 			return false;
 		}
 		node.sessionFreq += 1;
+		this.boostedNodes.add(node);
 		return true;
 	}
 
+	/**
+	 * Zero every session boost, in the number of words boosted rather than the
+	 * number of words known.
+	 */
+	clearSessionBoosts(): void {
+		for (const node of this.boostedNodes) {
+			node.sessionFreq = 0;
+		}
+		this.boostedNodes.clear();
+	}
+
 	/**
 	 * Every word carrying a session boost, optionally limited to one prefix's
 	 * subtree. Unlike `getCandidates` a word equal to the prefix is included,
 	 * since the question here is what the session holds rather than what could
 	 * still be typed.
 	 *
-	 * Walks the trie instead of reading an index, so nothing has to be kept in
-	 * step on the write path for the sake of being able to ask.
+	 * Walks the trie rather than reading `boostedNodes`, since a prefix answer is
+	 * a subtree question and this only runs when a human asks it.
 	 */
 	collectSessionBoosted(prefix: string = ''): Candidate[] {
 		let node = this.root;
@@ -500,6 +552,15 @@ let lastPredictionDebug: {
 	}>;
 } | null = null;
 
+let lastPredictionOutcome: PredictionOutcome | null = null;
+
+const recordPredictionOutcome = (outcome: PredictionOutcome): void => {
+	lastPredictionOutcome = outcome;
+};
+
+/** The verdict from the most recent `predict` call. Always populated. */
+export const getLastPredictionOutcome = (): PredictionOutcome | null => lastPredictionOutcome;
+
 // ── Stabilization (QI-2): post-accept cooldown + whole-surface repetition ────
 // Two guards that stop the accept→echo (`end to end` → `end to end to end`) and
 // generally keep a just-accepted unit from being re-offered:
@@ -530,6 +591,25 @@ export const noteSuggestionAccepted = (surface: string): void => {
 	}
 };
 
+/**
+ * Whether `surface` is the one the user just accepted and is still inside its
+ * cooldown window.
+ *
+ * Read-only, unlike the advance inside `predict`: the cooldown is measured in
+ * predictions, and a caller asking whether it is active must not consume one of
+ * them. Exported for the harvest path, which displays without going through
+ * arbitration and so would otherwise re-offer what was just accepted.
+ */
+export const isSurfaceInAcceptCooldown = (surface: string): boolean => {
+	if (!acceptCooldown || acceptCooldown.surface !== surface.trim().toLowerCase()) {
+		return false;
+	}
+	return (
+		acceptCooldown.predictionsSince <= COOLDOWN_KEYSTROKES ||
+		performance.now() - acceptCooldown.ts < COOLDOWN_MS
+	);
+};
+
 /** Get vector for a word from the store. */
 const getWordVector = (word: string): Float32Array | null => {
 	if (!vectorStore) {
@@ -839,6 +919,36 @@ export const incrementSessionFreq = (word: string): void => {
 	wordTrie.incrementSessionFreq(word);
 };
 
+/**
+ * Drop every L1 boost this session has accumulated.
+ *
+ * The vocabulary itself is left alone: only `sessionFreq` is cleared, so the
+ * tenant and generic frequencies a boost was sitting on top of survive. Called
+ * when the plugin decides the session it was learning for has ended — a new
+ * conversation, or a different page — since a boost is a claim about what is
+ * being discussed and that claim does not carry over.
+ */
+export const resetSessionBoosts = (): void => {
+	wordTrie.clearSessionBoosts();
+	// Anything memoized against the old boosts is now describing a session that
+	// no longer exists.
+	recallGeneration++;
+};
+
+/**
+ * Which vocabulary already holds this surface, if any.
+ *
+ * Used by the inline-code harvester to drop terms the scored path can already
+ * serve, so that harvesting stays limited to words with no route to a
+ * suggestion today.
+ */
+export const lookupVocabularySource = (word: string): 'l2' | 'l3' | null => {
+	if (wordTrie.hasWord(word)) {
+		return 'l2';
+	}
+	return l3Trie.hasWord(word) ? 'l3' : null;
+};
+
 /**
  * Prime session frequencies from a document page string.
  *
@@ -884,7 +994,7 @@ export const ingestDocumentPage = (pageContent: string | undefined): void => {
  * strongest boosts are the ones that change an ordering, and `boosted` still
  * reports the full size, so the cap loses nothing but volume.
  */
-const MAX_LISTED_SESSION_WORDS = 50;
+const MAX_LISTED_SESSION_WORDS = 100;
 
 interface SessionWordSnapshot {
 	/** Times this session has seen it: words typed plus words in ingested text. */
@@ -917,7 +1027,8 @@ export interface SessionSnapshot {
  * Only words the vocabulary already holds can carry a boost, because both writers
  * go through `incrementSessionFreq` and it only finds existing nodes. An ingested
  * word absent from the vocabulary is therefore missing from here and always will
- * be.
+ * be — that gap is what the inline-code harvester covers, and those surfaces show
+ * up under `__atlCtcDebug__.harvest()` instead.
  */
 export const inspectSessionBoosts = (prefix?: string): SessionSnapshot => {
 	const boosted = wordTrie.collectSessionBoosted(prefix ?? '');
@@ -957,8 +1068,15 @@ export interface PredictionResult {
 		verifiedChars: number;
 		verifiedTokens: number;
 	};
-	/** Evidence tier that authorized display. Tier A is never display-eligible. */
-	evidenceTier: 'canonical-full-surface' | 'network-logit';
+	/**
+	 * Evidence tier that authorized display. Tier A is never display-eligible.
+	 *
+	 * `session-harvest` never passes through this module: an inline-code surface
+	 * harvested from the session has no frequencies, vector or canonical token
+	 * ids, so it is authorized by being marked as code on a prefix the scored
+	 * path left unclaimed rather than by model evidence.
+	 */
+	evidenceTier: 'canonical-full-surface' | 'network-logit' | 'session-harvest';
 	/**
 	 * Mean per-token log-probability of the verified prefix.
 	 *
@@ -1127,6 +1245,14 @@ export const predict = (textBefore: string): PredictionResult | null => {
 		// Kick off the load and skip this keystroke; the plugin also primes it on
 		// focus, so the tries are usually ready before the user types.
 		void loadDefaultVocabulary({ source: 'predict' }).catch(() => {});
+		// Awaiting, not empty-handed: with no vocabulary loaded the harvester's own
+		// intake filter has not been applied to anything either.
+		recordPredictionOutcome({
+			abstainReason: 'not-initialized',
+			awaitingAsyncEvidence: true,
+			scoredCandidateCount: 0,
+			textBefore,
+		});
 		return null;
 	}
 
@@ -1157,12 +1283,24 @@ export const predict = (textBefore: string): PredictionResult | null => {
 				priority: 0,
 			});
 		}
+		recordPredictionOutcome({
+			abstainReason: 'prefetch',
+			awaitingAsyncEvidence: true,
+			scoredCandidateCount: 0,
+			textBefore,
+		});
 		return null;
 	}
 
 	const trimmed = textBefore.trimEnd();
 	const trailingSurfaceToken = trimmed.match(TRAILING_SURFACE_TOKEN_REGEX)?.[0] ?? '';
 	if (trailingSurfaceToken.length === 0) {
+		recordPredictionOutcome({
+			abstainReason: 'no-surface-token',
+			awaitingAsyncEvidence: false,
+			scoredCandidateCount: 0,
+			textBefore,
+		});
 		return null;
 	}
 	const currentWord = trailingSurfaceToken;
@@ -1176,6 +1314,14 @@ export const predict = (textBefore: string): PredictionResult | null => {
 
 	// If every trie was empty for this prefix
 	if (canonicalMatched.length === 0) {
+		// Terminal, and the only verdict that says the vocabulary has no claim on
+		// this prefix at all — which is what makes it the harvester's cue.
+		recordPredictionOutcome({
+			abstainReason: 'no-candidate',
+			awaitingAsyncEvidence: false,
+			scoredCandidateCount: 0,
+			textBefore,
+		});
 		if (isAutocompleteDebugEnabled()) {
 			// eslint-disable-next-line no-console
 			console.log(
@@ -1935,6 +2081,12 @@ export const predict = (textBefore: string): PredictionResult | null => {
 		return ranked.some(hasJudgeableEvidence) ? 'below-posterior-gate' : 'no-evidence';
 	};
 	const abstainReason: CtcAbstainReason | null = resolveAbstainReason();
+	recordPredictionOutcome({
+		abstainReason,
+		awaitingAsyncEvidence: hasUnresolvedPotential && !clearsWinnerMargin,
+		scoredCandidateCount: ranked.length,
+		textBefore,
+	});
 
 	// The leader is the best-supported candidate, not the selected one: an
 	// evaluation that showed nothing is exactly the one whose posterior needs
@@ -1988,6 +2140,9 @@ export const predict = (textBefore: string): PredictionResult | null => {
 			'missing-artifact': 'canonical artifact coverage missing',
 			'no-candidate': 'nothing cleared scoring',
 			'no-evidence': 'full-surface evidence absent',
+			// Recorded at the two exits above this block, so they never print here.
+			'no-surface-token': 'no trailing surface token',
+			'not-initialized': 'vocabulary not loaded',
 			prefetch: `prefetch: ${currentWord.length}/${DISPLAY_MIN_PREFIX_LENGTH} chars`,
 			'short-completion': `completion shorter than ${MIN_SUGGESTION_LENGTH} chars`,
 			'unresolved-rival': 'expanding plausible token-prefix groups',