@atlaskit/editor-plugin-autocomplete 6.0.0 → 8.0.0

package/src/pm-plugins/local-slow-lane-client.ts

@@ -39,6 +39,7 @@ import type { MLCEngine, InitProgressReport, AppConfig, LogitProcessor } from '@
 
 import { abortExp, EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
 
+import { ARTIFACT_NAME, fetchArtifactJson } from './artifacts-manifest';
 import { isAutocompleteDebugEnabled } from './debug-mode';
 import { isWordBoundary } from './slow-lane-client';
 
@@ -349,78 +350,9 @@ let prefixMapTokenIds: number[] = [];
 let bePayloadDataPromise: Promise<void> | undefined;
 
 /**
- * Unwrap a dynamically imported JSON module to the parsed JSON value, working
- * across the two interop modes AFM's bundler chain emits:
- *
- *   1. **`.default`-wrapped namespace** — classic webpack (and Jest) hang the
- *      JSON value under the `default` export.
- *   2. **Named-exports namespace** — webpack 5 / atlaspack with JSON
- *      named-exports (or native ESM JSON modules) expose each top-level key as
- *      a named export and shadow `default`, so `mod.default` can be `undefined`
- *      (or some unrelated value) even though `mod` itself holds the data.
- *
- * The caller MUST declare the underlying JSON shape via `shape` because, in
- * named-exports mode, a dense array `["a","b"]` and a sparse numeric-keyed
- * object `{"5":"a","12":"b"}` are emitted identically (`{"0":..}` / `{"5":..}`);
- * no runtime heuristic can tell them apart, so only the caller knows which:
- *
- *   - `'object'` — the JSON is a `{...}` (including sparse maps keyed by integer
- *     IDs). The named exports are rebuilt into a plain object so `Object.entries`
- *     yields the real keys, not synthetic array indices.
- *   - `'array'` — the JSON is a `[...]`, reconstructed from the `0..n-1` indices.
- *
- * :param mod: The raw module object returned by `await import('./*.json')`.
- * :param shape: `'object'` if the source JSON is `{...}`, `'array'` if `[...]`.
- * :returns: The parsed JSON value, or `null` if neither interop mode applies.
- */
-function unwrapJsonModule<T>(mod: unknown, shape: 'object' | 'array'): T | null {
-	if (mod == null || typeof mod !== 'object') {
-		return null;
-	}
-	const namespace = mod as Record<string, unknown> & { default?: unknown };
-
-	// Compute the named-export own-keys (strip synthetic markers).
-	const ownKeys = Object.keys(namespace).filter((k) => k !== 'default' && k !== '__esModule');
-
-	// PREFER named exports when present — they always reflect the JSON's real
-	// top-level keys / indices, regardless of what `default` happens to be.
-	// Under JSON named-exports mode `default` is not necessarily the parsed
-	// value (e.g. for `{"service": 0, ...}` it can be the number `0`, with the
-	// real data in the named exports), so taking `default` first would corrupt it.
-	if (ownKeys.length > 0) {
-		if (shape === 'array') {
-			// JSON arrays are dense; reconstruct from `0..length-1` indices.
-			const len = ownKeys.length;
-			const arr = new Array(len);
-			for (let i = 0; i < len; i++) {
-				arr[i] = namespace[String(i)];
-			}
-			return arr as T;
-		}
-		// shape === 'object'. Rebuild a plain object from the (stripped) own
-		// keys so callers can `Object.entries()` it without iterating over
-		// `default` / `__esModule`, and to detach from the module-namespace
-		// object (which is sealed/non-extensible on some bundler outputs).
-		const obj: Record<string, unknown> = {};
-		for (const k of ownKeys) {
-			obj[k] = namespace[k];
-		}
-		return obj as T;
-	}
-
-	// Fallback: no named exports — classic webpack JSON-module interop where
-	// the whole parsed JSON value is hung under `default`. Trust it.
-	if ('default' in namespace && namespace.default != null) {
-		return namespace.default as T;
-	}
-
-	return null;
-}
-
-/**
- * Lazily load and build the BE-parity lookup tables from their JSON payloads.
- * The dynamic imports are split into their own async chunks so neither file is
- * bundled into the editor's main chunk unless local inference is initialised.
+ * Lazily load and build the BE-parity lookup tables from their CDN payloads.
+ * Both artifacts are fetched on demand so neither is downloaded unless local
+ * inference is initialised.
  *
  * :returns:
  *   A promise that resolves once `firstTokenToWords`, `l2Words` and
@@ -429,31 +361,17 @@ function unwrapJsonModule<T>(mod: unknown, shape: 'object' | 'array'): T | null
 const loadBePayloadData = (): Promise<void> => {
 	if (!bePayloadDataPromise) {
 		bePayloadDataPromise = (async () => {
-			const [firstTokenToWordsModule, vocabularyModule] = await Promise.all([
-				import(
-					/* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-first-token-to-words" */ './data/first_token_to_words.json'
-				),
-				import(
-					/* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-vocabulary-10k" */ './data/vocabulary_10k.json'
-				),
+			const [firstTokenToWordsData, vocabularyData] = await Promise.all([
+				fetchArtifactJson<Record<string, string[]>>(ARTIFACT_NAME.FIRST_TOKEN_TO_WORDS),
+				fetchArtifactJson<{ words: Record<string, unknown> }>(ARTIFACT_NAME.VOCABULARY),
 			]);
 
-			const firstTokenToWordsData = unwrapJsonModule<Record<string, string[]>>(
-				firstTokenToWordsModule,
-				'object',
-			);
-			const vocabularyData = unwrapJsonModule<{ words: Record<string, unknown> }>(
-				vocabularyModule,
-				'object',
-			);
-
 			if (firstTokenToWordsData == null || vocabularyData?.words == null) {
 				// Hard-fail with a precise message so the catch() in initEngine logs
-				// exactly which import couldn't be unwrapped, rather than the generic
-				// V8 "Cannot convert undefined or null to object" we hit before the
-				// helper was added.
+				// exactly which payload was malformed, rather than the generic V8
+				// "Cannot convert undefined or null to object".
 				throw new Error(
-					`[LocalSlowLane] JSON module could not be unwrapped — ` +
+					`[LocalSlowLane] unexpected artifact payload — ` +
 						`firstTokenToWordsData=${firstTokenToWordsData == null ? 'null/undefined' : 'defined'}, ` +
 						`vocabularyData=${vocabularyData == null ? 'null/undefined' : vocabularyData.words == null ? 'defined but missing .words' : 'defined'}`,
 				);

package/src/pm-plugins/scoring-pipeline.ts

@@ -3,11 +3,15 @@
  *
  * Operates synchronously on pre-loaded data. Each stage gracefully degrades
  * when its required data isn't available (cold → warm → full warm).
+ *
+ * The grammar/POS payloads are fetched from the CDN by loadGrammarDataAsync, so
+ * the grammar filter is a pass-through until that resolves.
  */
 
-import posTagsData from './data/combined_l2_l3_pos_tags.json';
-import ghostPosTagsData from './data/ghost_pos_tags.json';
-import grammarTransitionsData from './data/grammar_transitions_10k.json';
+import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
+
+import { ARTIFACT_NAME, fetchArtifactJson } from './artifacts-manifest';
+import { isAutocompleteDebugEnabled } from './debug-mode';
 
 // ─── Types ──────────────────────────────────────────────────
 
@@ -59,22 +63,114 @@ const L1_SESSION_CAP = 1.2;
 // at 0.00024) from triggering re-ranking.
 const LM_GATE_THRESHOLD = 0.0005;
 
-// ─── Grammar Data (loaded once on import) ───────────────────
+// ─── Grammar Data (fetched once from the CDN) ────────────────
 
-const posTags: Map<string, string[]> = new Map(
-	Object.entries(posTagsData as Record<string, string[]>),
-);
+let posTags: Map<string, string[]> = new Map();
 
-const grammarTransitions = grammarTransitionsData as GrammarTransitions;
+/**
+ * GHOST POS DICTIONARY
+ * A mapping of common structural English words that were stripped from the main
+ * domain vocabulary. This allows the grammar filter to understand context
+ * without suggesting these words to the user.
+ */
+let ghostPosTags: Record<string, string[]> = {};
 
 /**
- * Precomputed map from each POS tag to the set of allowed next POS tags.
- * Built once at module load from grammarTransitions so applyGrammarFilter
- * never re-iterates the transition rules per call.
+ * Precomputed map from each POS tag to the set of allowed next POS tags, so
+ * applyGrammarFilter never re-iterates the transition rules per call.
  */
-const precomputedAllowedByPos: Map<string, Set<string>> = new Map(
-	Object.entries(grammarTransitions.transitions).map(([pos, rule]) => [pos, new Set(rule.allowed)]),
-);
+let precomputedAllowedByPos: Map<string, Set<string>> = new Map();
+
+let grammarDataPromise: Promise<void> | undefined;
+
+const setGrammarData = (data: {
+	ghostPosTags: Record<string, string[]>;
+	grammarTransitions: GrammarTransitions;
+	posTags: Record<string, string[]>;
+}): void => {
+	posTags = new Map(Object.entries(data.posTags));
+	ghostPosTags = data.ghostPosTags;
+	precomputedAllowedByPos = new Map(
+		Object.entries(data.grammarTransitions.transitions ?? {}).map(([pos, rule]) => [
+			pos,
+			new Set(rule.allowed),
+		]),
+	);
+};
+
+export const isGrammarDataLoaded = (): boolean => precomputedAllowedByPos.size > 0;
+
+/**
+ * Fetch the POS tag and grammar transition payloads from the CDN.
+ *
+ * Safe to call repeatedly: the in-flight promise is shared, and a failure is not
+ * cached so a later call can retry.
+ */
+export const loadGrammarDataAsync = (options?: {
+	isLocalLLM?: boolean;
+	surface?: string;
+}): Promise<void> => {
+	if (grammarDataPromise) {
+		return grammarDataPromise;
+	}
+
+	const isLocalLLM = options?.isLocalLLM ?? false;
+	const surface = options?.surface;
+
+	grammarDataPromise = (async () => {
+		startExp(EXPERIENCE_NAME.LOAD_GRAMMAR, 'singleton', {
+			isLocalLLM,
+			...(surface ? { surface } : {}),
+		});
+
+		try {
+			const [posTagsData, ghostPosTagsData, grammarTransitionsData] = await Promise.all([
+				fetchArtifactJson<Record<string, string[]>>(ARTIFACT_NAME.POS_TAGS),
+				fetchArtifactJson<Record<string, string[]>>(ARTIFACT_NAME.GHOST_POS_TAGS),
+				fetchArtifactJson<GrammarTransitions>(ARTIFACT_NAME.GRAMMAR_TRANSITIONS),
+			]);
+
+			if (grammarTransitionsData?.transitions == null) {
+				throw new Error(
+					`[scoring-pipeline] ${ARTIFACT_NAME.GRAMMAR_TRANSITIONS} is missing its \`transitions\` wrapper`,
+				);
+			}
+
+			setGrammarData({
+				posTags: posTagsData ?? {},
+				ghostPosTags: ghostPosTagsData ?? {},
+				grammarTransitions: grammarTransitionsData,
+			});
+
+			succeedExp(EXPERIENCE_NAME.LOAD_GRAMMAR, 'singleton', {
+				isLocalLLM,
+				posTagCount: posTags.size,
+				transitionCount: precomputedAllowedByPos.size,
+				...(surface ? { surface } : {}),
+			});
+
+			if (isAutocompleteDebugEnabled()) {
+				// eslint-disable-next-line no-console
+				console.log('[scoring-pipeline] Grammar data loaded:', {
+					posTagCount: posTags.size,
+					ghostPosTagCount: Object.keys(ghostPosTags).length,
+					transitionCount: precomputedAllowedByPos.size,
+				});
+			}
+		} catch (e) {
+			failExp(EXPERIENCE_NAME.LOAD_GRAMMAR, 'singleton', {
+				isLocalLLM,
+				errorType: 'network',
+				...(surface ? { surface } : {}),
+			});
+			// Allow a later call to retry the load rather than caching the failure.
+			grammarDataPromise = undefined;
+			throw e;
+		}
+	})();
+
+	return grammarDataPromise;
+};
 
 // ─── Math ───────────────────────────────────────────────────
 
@@ -135,14 +231,6 @@ function scoreStage1(
 
 // ─── Grammar Filter ─────────────────────────────────────────
 
-/**
- * GHOST POS DICTIONARY
- * A hardcoded mapping of common structural English words that were stripped
- * from the main domain vocabulary. This allows the grammar filter to understand
- * context without suggesting these words to the user.
- */
-const ghostPosTags: Record<string, string[]> = ghostPosTagsData as Record<string, string[]>;
-
 type FilterEntry = {
 	candidate: ScoringCandidate;
 	freqScore: number;
@@ -154,7 +242,11 @@ function applyGrammarFilter(
 	candidates: FilterEntry[],
 	previousWord: string,
 ): { filtered: FilterEntry[]; grammarMeta: GrammarFilterMeta | null } {
-	if (!previousWord) return { filtered: candidates, grammarMeta: null };
+	// Pass everything through until loadGrammarDataAsync resolves — filtering on a
+	// partially loaded ruleset would drop valid candidates.
+	if (!previousWord || !isGrammarDataLoaded()) {
+		return { filtered: candidates, grammarMeta: null };
+	}
 
 	const lowerPrev = previousWord.toLowerCase();
 	const prevTags = ghostPosTags[lowerPrev] || posTags.get(lowerPrev);

package/src/pm-plugins/text-predictor.ts

@@ -17,9 +17,10 @@
 
 import { EXPERIENCE_NAME, failExp, startExp, succeedExp } from '../analytics/ufo';
 
-// The vocabulary, L3 and word-index JSON payloads are dynamically imported in
-// loadDefaultVocabulary / loadVectorsAsync so their (large) contents stay out of
-// the editor's main chunk and only load when autocomplete is initialised.
+// The vocabulary, L3 and word-index payloads are fetched from the CDN in
+// loadDefaultVocabulary / loadVectorsAsync so their (large) contents never enter
+// the editor's bundle and only load when autocomplete is initialised.
+import { ARTIFACT_NAME, fetchArtifactJson, getArtifactUrl } from './artifacts-manifest';
 import { isAutocompleteDebugEnabled } from './debug-mode';
 import { rankCandidates, STAGE1_WEIGHT, STAGE2_WEIGHT, MIN_STAGE1_SCORE } from './scoring-pipeline';
 import type { ScoringCandidate } from './scoring-pipeline';
@@ -709,60 +710,13 @@ interface VocabularyJson {
 	>;
 }
 
-/**
- * Unwrap a dynamically imported JSON module to its parsed value, handling both
- * interop modes AFM's bundler chain emits: a `.default`-wrapped namespace
- * (classic webpack) and a named-exports namespace (webpack 5 / atlaspack JSON
- * modules, where `default` can be a misleading scalar). Named exports are
- * preferred when present. The caller declares the JSON `shape` because a dense
- * array and a sparse numeric-keyed object are emitted identically as named
- * exports. Kept in lock-step with the matching helper in local-slow-lane-client.ts.
- */
-function unwrapJsonModule<T>(mod: unknown, shape: 'object' | 'array'): T | null {
-	if (mod == null || typeof mod !== 'object') {
-		return null;
-	}
-	const namespace = mod as Record<string, unknown> & { default?: unknown };
-	const ownKeys = Object.keys(namespace).filter((k) => k !== 'default' && k !== '__esModule');
-
-	if (ownKeys.length > 0) {
-		if (shape === 'array') {
-			const len = ownKeys.length;
-			const arr = new Array(len);
-			for (let i = 0; i < len; i++) {
-				arr[i] = namespace[String(i)];
-			}
-			return arr as T;
-		}
-		const obj: Record<string, unknown> = {};
-		for (const k of ownKeys) {
-			obj[k] = namespace[k];
-		}
-		return obj as T;
-	}
-
-	if ('default' in namespace && namespace.default != null) {
-		return namespace.default as T;
-	}
-
-	return null;
-}
-
 export const loadVectorsAsync = async (options?: {
-	getBinaryUrl?: () => Promise<string>;
 	isLocalLLM?: boolean;
 	surface?: string;
 }): Promise<void> => {
 	if (vectorStore || vectorsLoadStarted) {
 		return;
 	}
-	if (!options?.getBinaryUrl) {
-		// eslint-disable-next-line no-console
-		console.warn(
-			'[text-predictor] loadVectorsAsync called without a getBinaryUrl — vectors will not load. Pass getVectorsBinaryUrl via plugin options.',
-		);
-		return;
-	}
 	const isLocalLLM = options?.isLocalLLM ?? false;
 	const surface = options?.surface;
 	vectorsLoadStarted = true;
@@ -773,7 +727,7 @@ export const loadVectorsAsync = async (options?: {
 
 	let url: string;
 	try {
-		url = await options.getBinaryUrl();
+		url = await getArtifactUrl(ARTIFACT_NAME.WORD_VECTORS);
 	} catch (e) {
 		vectorsLoadStarted = false;
 		failExp(EXPERIENCE_NAME.LOAD_VECTORS, 'singleton', {
@@ -803,22 +757,25 @@ export const loadVectorsAsync = async (options?: {
 		const buffer = await res.arrayBuffer();
 		const float32 = new Float32Array(buffer);
 
-		// word_index_10k.json is wrapped as `{ "index": {…} }` so no real entry
-		// (e.g. the word "default") can shadow the synthetic ESM `default` export
-		// the bundler creates for dynamically-imported JSON.
-		const wordIndexModule = await import(
-			/* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-word-index-10k" */ './data/word_index_10k.json'
-		);
-		const wordIndexOuter = unwrapJsonModule<{ index?: Record<string, number> }>(
-			wordIndexModule,
-			'object',
+		// The word index is published either bare or wrapped as `{ "index": {…} }`
+		// (the wrapper exists so no real entry, e.g. the word "default", can shadow
+		// a bundler-synthesised ESM export). Detect the wrapper by value type, not
+		// by key presence — "index" is itself a plausible vocabulary word, and in a
+		// bare index every value is a number.
+		const wordIndexPayload = await fetchArtifactJson<Record<string, unknown>>(
+			ARTIFACT_NAME.WORD_INDEX,
 		);
-		const wordIndex = wordIndexOuter?.index ?? {};
+		const wrappedIndex = wordIndexPayload?.index;
+		const wordIndex = (
+			typeof wrappedIndex === 'object' && wrappedIndex !== null
+				? wrappedIndex
+				: (wordIndexPayload ?? {})
+		) as Record<string, number>;
 		const nWords = Object.keys(wordIndex).length;
 		if (nWords === 0) {
 			// eslint-disable-next-line no-console
 			console.warn(
-				'[text-predictor] word_index_10k.json missing its `index` wrapper — wordIndex is empty, semantic scoring will be a no-op.',
+				`[text-predictor] ${ARTIFACT_NAME.WORD_INDEX} was empty — semantic scoring will be a no-op.`,
 			);
 		}
 		const dim = float32.length / nWords;
@@ -877,22 +834,17 @@ export const loadDefaultVocabulary = (options?: {
 		});
 
 		try {
-			// The L2 vocabulary and L3 word list are code-split into their own async
-			// chunks so they stay out of the editor's main bundle.
-			const [vocabularyModule, l3VocabularyModule] = await Promise.all([
-				import(
-					/* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-vocabulary-10k" */ './data/vocabulary_10k.json'
-				),
-				import(
-					/* webpackChunkName: "@atlaskit-internal_editor-plugin-autocomplete-l3-vocabulary" */ './data/l3_vocabulary.json'
-				),
+			const [vocabularyData, l3VocabularyData] = await Promise.all([
+				fetchArtifactJson<VocabularyJson>(ARTIFACT_NAME.VOCABULARY),
+				fetchArtifactJson<string[]>(ARTIFACT_NAME.L3_VOCABULARY),
 			]);
 
-			const vocabularyData = unwrapJsonModule<VocabularyJson>(vocabularyModule, 'object');
-			const l3VocabularyData = unwrapJsonModule<string[]>(l3VocabularyModule, 'array');
-
 			if (vocabularyData?.words == null || !Array.isArray(l3VocabularyData)) {
-				throw new Error('[text-predictor] vocabulary JSON modules could not be unwrapped');
+				throw new Error(
+					`[text-predictor] unexpected vocabulary payload — ` +
+						`${ARTIFACT_NAME.VOCABULARY} ${vocabularyData?.words == null ? 'is missing its `words` wrapper' : 'is valid'}, ` +
+						`${ARTIFACT_NAME.L3_VOCABULARY} ${Array.isArray(l3VocabularyData) ? 'is valid' : 'is not an array'}`,
+				);
 			}
 
 			const terms = Object.entries(vocabularyData.words).map(([word, stats]) => ({
@@ -917,7 +869,7 @@ export const loadDefaultVocabulary = (options?: {
 		} catch (e) {
 			failExp(EXPERIENCE_NAME.LOAD_VOCABULARY, 'singleton', {
 				isLocalLLM,
-				errorType: 'parse_error',
+				errorType: 'load_error',
 				...(surface ? { surface } : {}),
 			});
 			// Allow a later call to retry the load rather than caching the failure.