@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0

package/dist/types/pm-plugins/slow-lane-client.d.ts

@@ -6,6 +6,7 @@
  *   Request:  { text, session_id }
  *   Response: { semantic_vector: number[], lm_logits: Record<string, number> }
  */
+import type { BoundaryLmState, BoundaryPrimeRequest, ProgressiveSurfaceEvidence, SurfaceScore, SurfaceScoreRequest } from './canonical-lm-scoring';
 /** Request payload for typeahead-encodings endpoint. */
 export interface TypeaheadEncodingsRequest {
     session_id: string;
@@ -34,14 +35,61 @@ export interface SlowLaneClientConfig {
     sessionId?: string;
     surface?: string;
 }
-export declare const createSlowLaneClient: (config: SlowLaneClientConfig) => {
+export interface SlowLaneClient {
+    getBoundaryLmState?: (contextKey: string) => BoundaryLmState | null;
+    getCanonicalSurfaceCount?: () => number;
+    getCanonicalSurfaceTokenIds?: (surface: string) => number[] | null;
+    getContextInput?: () => string | null;
     getContextVector: () => Float32Array | null;
     getLmLogits: () => Record<string, number> | null;
+    getProgressiveSurfaceEvidence?: (contextKey: string, surface: string) => ProgressiveSurfaceEvidence | null;
+    getSurfaceScore?: (contextKey: string, surface: string) => SurfaceScore | null;
+    isReady?: () => boolean;
     isWordBoundary: (text: string) => boolean;
+    primeBoundaryLm?: (input: BoundaryPrimeRequest) => void;
+    requestProgressiveSurfaceScores?: (input: SurfaceScoreRequest) => void;
     setContextVector: (vector: Float32Array | null) => void;
     setLmLogits: (logits: Record<string, number> | null) => void;
     updateContext: (text: string) => void;
-};
-export declare const setDefaultSlowLaneClient: (client: ReturnType<typeof createSlowLaneClient> | null) => void;
+}
+export declare const createSlowLaneClient: (config: SlowLaneClientConfig) => SlowLaneClient;
+export type SlowLaneClientKind = 'localLlm' | 'server';
+export interface SlowLaneClientRegistration {
+    clientId: string;
+    kind: SlowLaneClientKind;
+}
+export interface SlowLaneClientStatus {
+    canonicalScoringSupported: boolean;
+    canonicalSurfaceCount: number | null;
+    clientId: string | null;
+    clientKind: SlowLaneClientKind | null;
+    localModelReady: boolean | null;
+    registered: boolean;
+}
+export declare const setDefaultSlowLaneClient: (client: SlowLaneClient | null, registration?: SlowLaneClientRegistration) => void;
+/**
+ * Clear the shared client only when the caller still owns the registration.
+ *
+ * Multiple editor instances can briefly overlap during a React/editor remount.
+ * An older instance must not disconnect the newer instance when its delayed
+ * teardown runs after the newer client has registered.
+ */
+export declare const clearDefaultSlowLaneClient: (client: SlowLaneClient) => boolean;
+export declare const getDefaultSlowLaneClientStatus: () => SlowLaneClientStatus;
 export declare const getStoredContextVector: () => Float32Array | null;
+export declare const getStoredContextInput: () => string | null;
 export declare const getStoredLmLogits: () => Record<string, number> | null;
+export declare const getBoundaryLmState: (contextKey: string) => BoundaryLmState | null;
+export declare const getCanonicalSurfaceTokenIds: (surface: string) => number[] | null;
+export declare const getSurfaceScore: (contextKey: string, surface: string) => SurfaceScore | null;
+export declare const getProgressiveSurfaceEvidence: (contextKey: string, surface: string) => ProgressiveSurfaceEvidence | null;
+export declare const primeBoundaryLm: (input: BoundaryPrimeRequest) => void;
+export declare const requestProgressiveSurfaceScores: (input: SurfaceScoreRequest) => void;
+export declare const isCanonicalSurfaceScoringSupported: () => boolean;
+/**
+ * How many surfaces the canonical token map currently holds. Goes from zero to
+ * its final size once the artifact lands, which is the only point at which
+ * `getCanonicalSurfaceTokenIds` starts answering — so callers memoising over
+ * that lookup can use this to detect the transition.
+ */
+export declare const getCanonicalSurfaceCount: () => number;

package/dist/types/pm-plugins/text-predictor.d.ts

@@ -3,7 +3,7 @@
  *
  * Two prediction modes:
  *   1. Word boundary → bigram-based next-word suggestion (grammar-filtered)
- *   2. Mid-word (≥3 chars) → trie prefix search → scoring pipeline → top result
+ *   2. Mid-word (≥1 char) → grouped prefetch; ≥3 chars + exact evidence → ghost result
  *
  * Scoring is delegated to scoring-pipeline.ts which handles:
  *   Stage 1 (semantic + frequency), grammar filter, Stage 2 (optional LM re-ranking).
@@ -12,8 +12,12 @@
  * Falls back to cold mode (freq-only) when vectors not yet loaded.
  *
  * Session personalization (L1): words the user types are incrementally boosted
- * via incrementSessionFreq(), called on word boundaries from the plugin.
+ * via incrementSessionFreq(), called on word boundaries from the plugin, and
+ * words in ingested context text via ingestDocumentPage(). What the session has
+ * boosted is visible at any time from the console: `__atlCtcDebug__.session()`,
+ * or `__atlCtcDebug__.session('poll')` for one family — see inspectSessionBoosts.
  */
+import type { TermType } from './scoring-pipeline';
 export interface WeightedTerm {
     authorFreq: number;
     docFreq: number;
@@ -33,13 +37,32 @@ interface VectorStore {
  * expects a simple array of strings: ["about", "above", "actually", ...]
  */
 export declare const initL3Vocabulary: (l3Words: string[]) => void;
+/**
+ * Start a short cooldown for the exact surface the editor inserted. The caller
+ * passes the committed snapshot's surface so background re-ranking can never
+ * move cooldown bookkeeping away from what the user actually accepted.
+ */
+export declare const noteSuggestionAccepted: (surface: string) => void;
 /**
  * Get predictor status for debugging.
  * vectorsLoaded: true when semantic scoring is active
  * wordCount: number of words in vector store (0 if not loaded)
  */
 export declare const getPredictorStatus: () => {
+    canonicalScoringSupported: boolean;
+    canonicalSurfaceCount: number | null;
+    clientId: string | null;
+    clientKind: 'localLlm' | 'server' | null;
     isInitialized: boolean;
+    localModelReady: boolean | null;
+    maxFreqByType: {
+        bigram: number;
+        phrase: number;
+        word: number;
+    };
+    phraseCount: number;
+    phrasesLoaded: boolean;
+    slowLaneRegistered: boolean;
     vectorsLoaded: boolean;
     vectorsLoadStarted: boolean;
     wordCount: number;
@@ -49,8 +72,10 @@ export declare const getPredictorStatus: () => {
  * Returns null if no prediction has run yet or debug was off.
  */
 export declare const getLastPredictionDebug: () => {
+    awaitingAsyncEvidence: boolean;
     contextWords: string[];
     currentWord: string;
+    decision: string;
     mode: 'cold' | 'warm';
     suggestion: string | null;
     textBefore: string;
@@ -63,6 +88,16 @@ export declare const getLastPredictionDebug: () => {
     }>;
 } | null;
 export declare const initVocabulary: (vocabulary: TenantVocabulary) => void;
+/**
+ * Insert a producer-shipped bigram/phrase artifact into the phrase trie.
+ *
+ * :params:
+ *   artifact: Normalized `{term: {freq, doc_freq, author_freq, pos_seq, head_pos, association}}` map
+ *   termType: Whether these terms are `bigram` (2 tokens) or `phrase` (3 tokens)
+ * :returns:
+ *   The number of terms inserted
+ */
+export declare const initPhrases: (artifact: PhraseArtifactJson, termType: TermType) => number;
 /**
  * Increment L1 session frequency for a single word.
  * Called from the plugin on word boundaries for efficient incremental boosting.
@@ -79,14 +114,132 @@ export declare const incrementSessionFreq: (word: string) => void;
  * calling context does not yet have a page value available.
  */
 export declare const ingestDocumentPage: (pageContent: string | undefined) => void;
-export declare const predict: (textBefore: string) => string | null;
+interface SessionWordSnapshot {
+    /** Times this session has seen it: words typed plus words in ingested text. */
+    sessionFreq: number;
+    /** No corpus frequency behind it, so L1 is the whole of its standing. */
+    sessionOnly: boolean;
+    surface: string;
+    /** Corpus frequency shipped with the vocabulary, for scale against the boost. */
+    tenantFreq: number;
+}
+export interface SessionSnapshot {
+    /** How many words hold a boost, whether or not they are listed below. */
+    boosted: number;
+    /** Ceiling on `words`; beyond it the weakest boosts are left out of the listing. */
+    limit: number;
+    /** The prefix asked about, when one was passed. */
+    prefix?: string;
+    /** Strongest boost first, then alphabetically. */
+    words: SessionWordSnapshot[];
+}
+/**
+ * Read the session's L1 boosts, optionally narrowed to a prefix.
+ *
+ * Installed as `__atlCtcDebug__.session()`, with `__atlCtcDebug__.session('poll')`
+ * to ask about one family. Returned rather than logged, so the console renders it
+ * as an inspectable object and a caller can assert on it.
+ *
+ * Only words the vocabulary already holds can carry a boost, because both writers
+ * go through `incrementSessionFreq` and it only finds existing nodes. An ingested
+ * word absent from the vocabulary is therefore missing from here and always will
+ * be.
+ */
+export declare const inspectSessionBoosts: (prefix?: string) => SessionSnapshot;
+/**
+ * Result of a prediction: the ghost tail to insert plus an immutable record of
+ * the evidence that authorized the UI commitment.
+ */
+export interface PredictionResult {
+    /** Canonical evidence depth for the selected full surface. */
+    evidenceDepth: {
+        totalChars: number;
+        totalTokens: number;
+        verifiedChars: number;
+        verifiedTokens: number;
+    };
+    /** Evidence tier that authorized display. Tier A is never display-eligible. */
+    evidenceTier: 'canonical-full-surface' | 'network-logit';
+    /**
+     * Mean per-token log-probability of the verified prefix.
+     *
+     * Carried alongside the posterior because the two answer different
+     * questions and a plausibility floor is placed on this one. A surface alone
+     * in its normaliser holds the whole pool whatever the model thinks of it,
+     * so its posterior cannot say whether it was worth showing and this can.
+     */
+    meanTokenLogProbability: number;
+    /**
+     * Whether a longer candidate in the same pool extends the selected surface.
+     *
+     * The chain rule leaves such a surface holding its pool's mass undivided, so
+     * its posterior is high for a structural reason rather than a modelled one —
+     * whichever continuation was meant, the prefix was right. Carried so the top
+     * posterior bucket can be split on it, since a lifted nested leader and a
+     * genuinely preferred surface are otherwise the same number.
+     */
+    poolHeldExtension: boolean;
+    /** Share of its shortlist's mass the model put on the selected surface. */
+    posterior: number;
+    /** Final Stage-1 + LM ranking score. */
+    rankScore: number;
+    /** How many scored candidates the selected surface's normaliser divided between. */
+    shortlistSize: number;
+    /** Full surface of the selected candidate (identity for keep-vs-swap checks). */
+    surface: string;
+    /** Whether the selected surface is a single word, a bigram, or a phrase. */
+    termType: TermType;
+    /** Ghost tail actually shown/inserted (surface minus the already-typed prefix). */
+    text: string;
+    /** Final-score lead over the strongest evidence-backed runner-up. */
+    winnerMargin: number;
+}
+export declare const predict: (textBefore: string) => PredictionResult | null;
+/**
+ * Per-term stats shape for `bigrams.json` / `phrases.json`. Field names/types
+ * confirmed against the producer (`l2_vocabulary_creation._build_ngram_stat_payload`
+ * + `artifact_release._validate_ngram_payload`): `freq`/`doc_freq`/`author_freq`
+ * are ints, `pos_seq` a string array, `head_pos` a string or null, `association`
+ * a number.
+ */
+interface PhraseStatsJson {
+    association?: number;
+    author_freq: number;
+    doc_freq: number;
+    freq: number;
+    head_pos?: string | null;
+    pos_seq?: string[];
+}
+/** Normalized `{term: stats}` map after unwrapping any producer wrapper key. */
+type PhraseArtifactJson = Record<string, PhraseStatsJson>;
 export declare const loadVectorsAsync: (options?: {
     isLocalLLM?: boolean;
     surface?: string;
 }) => Promise<void>;
 export declare const initVectors: (store: VectorStore) => void;
+/**
+ * Load the producer's `bigrams.json` + `phrases.json` completion-term artifacts
+ * over the TDP-OS gateway and insert them into the phrase trie.
+ *
+ * Each file is fetched independently (`Promise.allSettled`) so a missing or
+ * late-published artifact only skips that term type — word completion and the
+ * other term type are unaffected. Phrase VECTORS are not fetched here: per the
+ * producer's `extend` decision they are appended into the existing
+ * `word-index-10k.json` / `word-vectors-10k.bin`, so `loadVectorsAsync` already
+ * covers them.
+ *
+ * :params:
+ *   options.isLocalLLM: Tags the UFO experience so latency/success feeds the same SLO
+ * :returns:
+ *   A promise that resolves once both fetches have settled
+ */
+export declare const loadPhraseArtifacts: (options?: {
+    isLocalLLM?: boolean;
+}) => Promise<void>;
+type VocabularyLoadSource = 'focus' | 'predict';
 export declare const loadDefaultVocabulary: (options?: {
     isLocalLLM?: boolean;
+    source?: VocabularyLoadSource;
     surface?: string;
 }) => Promise<void>;
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@atlaskit/editor-plugin-autocomplete",
-	"version": "8.0.0",
+	"version": "9.1.0",
 	"description": "Client-side text autocomplete plugin for @atlaskit/editor-core",
 	"author": "Atlassian Pty Ltd",
 	"license": "Apache-2.0",
@@ -27,7 +27,7 @@
 		"wink-nlp": "^2.4.0"
 	},
 	"peerDependencies": {
-		"@atlaskit/editor-common": "^119.0.0",
+		"@atlaskit/editor-common": "^119.9.0",
 		"@atlaskit/editor-plugin-analytics": "^15.0.0",
 		"react": "^18.2.0 || ^19.2.0"
 	},

package/src/analytics/ufo.ts

@@ -37,6 +37,7 @@ export const EXPERIENCE_NAME = {
 	SLOW_LANE_FETCH: 'slow-lane-fetch',
 	LOAD_VOCABULARY: 'load-vocabulary',
 	LOAD_VECTORS: 'load-vectors',
+	LOAD_PHRASES: 'load-phrases',
 	LOAD_GRAMMAR: 'load-grammar',
 } as const;
 
@@ -70,6 +71,10 @@ const experiences: Record<AutocompleteExperienceName, ConcurrentExperience> = {
 		EXPERIENCE_NAME.LOAD_VECTORS,
 		operationConfig,
 	),
+	[EXPERIENCE_NAME.LOAD_PHRASES]: new ConcurrentExperience(
+		EXPERIENCE_NAME.LOAD_PHRASES,
+		operationConfig,
+	),
 	[EXPERIENCE_NAME.LOAD_GRAMMAR]: new ConcurrentExperience(
 		EXPERIENCE_NAME.LOAD_GRAMMAR,
 		operationConfig,

package/src/pm-plugins/artifact-loader.ts

@@ -0,0 +1,92 @@
+/**
+ * Validation and diagnostics layer over the artifacts manifest.
+ *
+ * `artifacts-manifest` owns *where* a payload comes from (one manifest request,
+ * then a presigned CDN download). This module owns *whether the payload is
+ * usable*: every loader declares the shape it expects, so a mis-published
+ * artifact fails at the boundary with the artifact's name rather than surfacing
+ * later as an empty trie or a silent no-op in scoring.
+ */
+
+import { type ArtifactName, fetchArtifactJson, getArtifactUrl } from './artifacts-manifest';
+import { ctcTag } from './debug-mode';
+
+const TRAILING_SLASH_REGEX = /\/$/u;
+const DEFAULT_BASE_URL = '';
+const AUTOCOMPLETE_GATEWAY_BASE = '/gateway/api/v1/autocomplete';
+
+type ArtifactLoaderOptions<T> = {
+	label?: string;
+	summarize?: (payload: T) => string;
+	validate?: (payload: unknown) => payload is T;
+};
+
+/**
+ * Build a URL for one of the autocomplete gateway's own endpoints (e.g. the
+ * network slow lane). Model artifacts do NOT go through here — they are served
+ * from the CDN via the manifest.
+ */
+export const buildAutocompleteGatewayUrl = (
+	path: string,
+	baseUrl: string = DEFAULT_BASE_URL,
+): string => {
+	const normalizedBase = baseUrl.replace(TRAILING_SLASH_REGEX, '');
+	return `${normalizedBase}${AUTOCOMPLETE_GATEWAY_BASE}${path}`;
+};
+
+/**
+ * Download a JSON artifact and assert its shape before handing it to a loader.
+ *
+ * :params:
+ *   artifactName: Manifest key for the artifact
+ *   options.label: Name used in diagnostics; defaults to the manifest key
+ *   options.summarize: Renders a short size/count summary for the debug log
+ *   options.validate: Type guard the payload must satisfy
+ * :returns:
+ *   The parsed, validated payload
+ */
+export const fetchAutocompleteArtifactJson = async <T>(
+	artifactName: ArtifactName,
+	options: ArtifactLoaderOptions<T> = {},
+): Promise<T> => {
+	const label = options.label ?? artifactName;
+	// Presigned CDN URLs carry an auth token, so only the artifact name is logged.
+	ctcTag('init', `↓ loading ${label}`);
+
+	const payload = await fetchArtifactJson<unknown>(artifactName);
+	if (options.validate && !options.validate(payload)) {
+		throw new Error(`[autocomplete-artifacts] ${label} payload shape was invalid`);
+	}
+
+	const summary = options.summarize?.(payload as T);
+	ctcTag('init', `✓ loaded ${label}${summary ? ` (${summary})` : ''}`);
+
+	return payload as T;
+};
+
+/**
+ * Download a binary artifact (the word-vectors blob) as an ArrayBuffer.
+ *
+ * :params:
+ *   artifactName: Manifest key for the artifact
+ *   label: Name used in diagnostics; defaults to the manifest key
+ * :returns:
+ *   The raw bytes of the artifact
+ */
+export const fetchAutocompleteArtifactBinary = async (
+	artifactName: ArtifactName,
+	label: string = artifactName,
+): Promise<ArrayBuffer> => {
+	ctcTag('init', `↓ loading ${label}`);
+
+	const url = await getArtifactUrl(artifactName);
+	const response = await fetch(url);
+	if (!response.ok) {
+		throw new Error(`[autocomplete-artifacts] Failed to fetch ${label}: ${response.status}`);
+	}
+
+	const payload = await response.arrayBuffer();
+	ctcTag('init', `✓ loaded ${label} (${payload.byteLength} bytes)`);
+
+	return payload;
+};

package/src/pm-plugins/artifacts-manifest.ts

@@ -20,10 +20,13 @@ import { isAutocompleteDebugEnabled } from './debug-mode';
  * hyphenated, unlike the underscored filenames the artifacts are generated from.
  */
 export const ARTIFACT_NAME = {
+	BIGRAMS: 'bigrams.json',
 	FIRST_TOKEN_TO_WORDS: 'first-token-to-words.json',
 	GHOST_POS_TAGS: 'ghost-pos-tags.json',
 	GRAMMAR_TRANSITIONS: 'grammar-transitions-10k.json',
 	L3_VOCABULARY: 'l3-vocabulary.json',
+	PHRASE_CONTINUATION_TOKENS: 'phrase-continuation-tokens.json',
+	PHRASES: 'phrases.json',
 	POS_TAGS: 'combined-l2-l3-pos-tags.json',
 	VOCABULARY: 'vocabulary-10k.json',
 	WORD_INDEX: 'word-index-10k.json',