@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0

package/dist/types/analytics/ufo.d.ts

@@ -30,6 +30,7 @@ export declare const EXPERIENCE_NAME: {
     readonly SLOW_LANE_FETCH: 'slow-lane-fetch';
     readonly LOAD_VOCABULARY: 'load-vocabulary';
     readonly LOAD_VECTORS: 'load-vectors';
+    readonly LOAD_PHRASES: 'load-phrases';
     readonly LOAD_GRAMMAR: 'load-grammar';
 };
 export type AutocompleteExperienceName = (typeof EXPERIENCE_NAME)[keyof typeof EXPERIENCE_NAME];

package/dist/types/pm-plugins/artifact-loader.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Validation and diagnostics layer over the artifacts manifest.
+ *
+ * `artifacts-manifest` owns *where* a payload comes from (one manifest request,
+ * then a presigned CDN download). This module owns *whether the payload is
+ * usable*: every loader declares the shape it expects, so a mis-published
+ * artifact fails at the boundary with the artifact's name rather than surfacing
+ * later as an empty trie or a silent no-op in scoring.
+ */
+import { type ArtifactName } from './artifacts-manifest';
+type ArtifactLoaderOptions<T> = {
+    label?: string;
+    summarize?: (payload: T) => string;
+    validate?: (payload: unknown) => payload is T;
+};
+/**
+ * Build a URL for one of the autocomplete gateway's own endpoints (e.g. the
+ * network slow lane). Model artifacts do NOT go through here — they are served
+ * from the CDN via the manifest.
+ */
+export declare const buildAutocompleteGatewayUrl: (path: string, baseUrl?: string) => string;
+/**
+ * Download a JSON artifact and assert its shape before handing it to a loader.
+ *
+ * :params:
+ *   artifactName: Manifest key for the artifact
+ *   options.label: Name used in diagnostics; defaults to the manifest key
+ *   options.summarize: Renders a short size/count summary for the debug log
+ *   options.validate: Type guard the payload must satisfy
+ * :returns:
+ *   The parsed, validated payload
+ */
+export declare const fetchAutocompleteArtifactJson: <T>(artifactName: ArtifactName, options?: ArtifactLoaderOptions<T>) => Promise<T>;
+/**
+ * Download a binary artifact (the word-vectors blob) as an ArrayBuffer.
+ *
+ * :params:
+ *   artifactName: Manifest key for the artifact
+ *   label: Name used in diagnostics; defaults to the manifest key
+ * :returns:
+ *   The raw bytes of the artifact
+ */
+export declare const fetchAutocompleteArtifactBinary: (artifactName: ArtifactName, label?: string) => Promise<ArrayBuffer>;
+export {};

package/dist/types/pm-plugins/artifacts-manifest.d.ts

@@ -15,10 +15,13 @@
  * hyphenated, unlike the underscored filenames the artifacts are generated from.
  */
 export declare const ARTIFACT_NAME: {
+    readonly BIGRAMS: 'bigrams.json';
     readonly FIRST_TOKEN_TO_WORDS: 'first-token-to-words.json';
     readonly GHOST_POS_TAGS: 'ghost-pos-tags.json';
     readonly GRAMMAR_TRANSITIONS: 'grammar-transitions-10k.json';
     readonly L3_VOCABULARY: 'l3-vocabulary.json';
+    readonly PHRASE_CONTINUATION_TOKENS: 'phrase-continuation-tokens.json';
+    readonly PHRASES: 'phrases.json';
     readonly POS_TAGS: 'combined-l2-l3-pos-tags.json';
     readonly VOCABULARY: 'vocabulary-10k.json';
     readonly WORD_INDEX: 'word-index-10k.json';

package/dist/types/pm-plugins/autocomplete-plugin.d.ts

@@ -3,7 +3,44 @@ import type { ExtractInjectionAPI } from '@atlaskit/editor-common/types';
 import { PluginKey } from '@atlaskit/editor-prosemirror/state';
 import { DecorationSet } from '@atlaskit/editor-prosemirror/view';
 import type { AutocompletePlugin } from '../autocompletePluginType';
+import { type PredictionResult } from './text-predictor';
 export declare const autocompletePluginKey: PluginKey;
+export interface CommittedSuggestion {
+    /** Wall-clock latency from the triggering editor update to the committed ghost. */
+    decisionLatencyMs: number;
+    /** Evidence tier that authorized this suggestion for display. */
+    evidenceTier: PredictionResult['evidenceTier'];
+    /** Ghost tail actually rendered and inserted. */
+    ghostText: string;
+    /**
+     * Mean per-token log-probability of the verified prefix, carried so an
+     * acceptance can be attributed to the evidence that authorized the display
+     * rather than to whatever the scorer holds by the time Tab arrives.
+     */
+    meanTokenLogProbability: number;
+    /**
+     * Whether a longer candidate in the same pool extended this surface, which
+     * hands it that pool's mass undivided under the chain rule and puts it near
+     * the top of the posterior range for a structural reason.
+     */
+    poolHeldExtension: boolean;
+    /** ProseMirror position at which the ghost is anchored. */
+    position: number;
+    /** Share of its shortlist's mass the model put on this surface. */
+    posterior: number;
+    /** Final confidence-v2 ranking score. */
+    rankScore: number;
+    /** Monotonic editor decision revision that owns this suggestion. */
+    revision: number;
+    /** How many scored candidates this surface's normaliser divided between. */
+    shortlistSize: number;
+    /** Full candidate surface, used by analytics and cooldown. */
+    surface: string;
+    /** Whether the surface is a single word, a bigram, or a phrase. */
+    termType: PredictionResult['termType'];
+    /** Difference between the selected rank score and its closest eligible rival. */
+    winnerMargin: number;
+}
 export interface AutocompletePluginState {
     /** The decoration set containing the ghost text widget */
     decorationSet: DecorationSet;
@@ -11,6 +48,8 @@ export interface AutocompletePluginState {
     ghostPosition: number;
     /** The predicted ghost text to display */
     ghostText: string;
+    /** Single authoritative snapshot for rendering, insertion, analytics, and cooldown. */
+    suggestion: CommittedSuggestion | null;
 }
 /**
  * Context provided to the autocomplete plugin on first editor focus.

package/dist/types/pm-plugins/canonical-lm-scoring.d.ts

@@ -0,0 +1,158 @@
+import type { LogitProcessor } from '@mlc-ai/web-llm';
+export type SeparatorKind = 'whitespace' | 'document-start' | 'non-space';
+export type CanonicalLmEvidenceSource = 'canonical-first-token' | 'canonical-full-surface' | 'network-logit';
+export interface BoundaryLmState {
+    contextKey: string;
+    prompt: string;
+    rawLogits: Float32Array;
+}
+export interface SurfaceScore {
+    contextKey: string;
+    meanTokenLogProbability: number;
+    surface: string;
+    tokenCount: number;
+    totalLogProbability: number;
+}
+export interface CanonicalCandidateContext {
+    canonicalTokenIds: number[] | null;
+    contextBeforeSurface: string;
+    contextKey: string;
+    separatorKind: SeparatorKind;
+    surfaceStart: number;
+}
+export interface CanonicalBoundaryContext {
+    contextKey: string;
+    prompt: string;
+}
+export interface CanonicalLmEvidence {
+    contextKey: string;
+    logProbability?: number;
+    meanTokenLogProbability?: number;
+    rawLogit?: number;
+    score: number;
+    separatorKind: 'whitespace';
+    source: CanonicalLmEvidenceSource;
+    totalLogProbability?: number;
+    totalSurfaceCharCount: number;
+    totalTokenCount: number;
+    verifiedCharCount: number;
+    verifiedTokenCount: number;
+}
+export interface BoundaryPrimeRequest {
+    contextKey: string;
+    familyKey: string;
+    priority: number;
+    prompt: string;
+}
+export interface SurfaceScoreRequest {
+    candidates: Array<{
+        rankHint?: number;
+        surface: string;
+        tokenIds: number[];
+    }>;
+    contextKey: string;
+    familyKey: string;
+    prompt: string;
+}
+export interface ProgressiveSurfaceEvidence {
+    meanTokenLogProbabilityUpperBound: number;
+    scoredTokenCount: number;
+    totalLogProbability: number;
+    totalTokenCount: number;
+}
+export interface TokenPrefixExpansion {
+    contextKey: string;
+    rawNextTokenLogits: Float32Array;
+    tokenPrefix: number[];
+    totalLogProbability: number;
+}
+export interface TokenPrefixGroup {
+    surfaces: string[];
+    tokenPrefix: number[];
+}
+/**
+ * Token trie for the producer's canonical leading-space token sequences.
+ * It lets the scorer expand a shared token prefix once for every surface below
+ * that node (`root cause`, `root directory`, ...), rather than guessing two
+ * arbitrary full surfaces from a broad one-character string prefix.
+ */
+export declare class CanonicalSurfaceTokenTrie {
+    private readonly root;
+    private readonly tokenIdsBySurface;
+    constructor(entries?: Iterable<readonly [string, number[]]>);
+    insert(surface: string, tokenIds: number[]): void;
+    getTokenIds(surface: string): number[] | null;
+    groupByScoredPrefix(candidates: Array<{
+        scoredTokenCount: number;
+        surface: string;
+        tokenIds: number[];
+    }>): TokenPrefixGroup[];
+}
+/** The part of a candidate's context that depends only on where its surface starts. */
+interface SurfacePositionContext {
+    contextBeforeSurface: string;
+    contextKey: string;
+    separatorKind: SeparatorKind;
+    surfaceStart: number;
+}
+/**
+ * Per-pass store for the position-derived half of a candidate's context.
+ *
+ * Every unigram candidate for a keystroke shares one surface start, and the
+ * phrase candidates share a handful more, so without this the same slice,
+ * `trimEnd` and context key are rebuilt for each of the couple of hundred
+ * candidates. Valid only for a single `textBeforeCursor`.
+ */
+export type CanonicalContextPositionCache = Map<number, SurfacePositionContext>;
+export declare const createCanonicalContextPositionCache: () => CanonicalContextPositionCache;
+/**
+ * Derive the causal prompt and artifact eligibility from the exact, untrimmed
+ * pre-cursor text. The shipped artifact contains only tokenizations of
+ * `" " + surface`, so it is valid only when the surface is actually preceded
+ * by whitespace.
+ */
+export declare const deriveCanonicalCandidateContext: (textBeforeCursor: string, matchedPrefixLen: number, surface: string, getTokenIds: (surface: string) => number[] | null, surfaceStartOverride?: number, positionCache?: CanonicalContextPositionCache) => CanonicalCandidateContext;
+/**
+ * Derive the next surface's leading-space context at an actual word boundary.
+ * This lets the local LM start its one-token prime on the space keystroke,
+ * before the user has typed the first character of the next word.
+ */
+export declare const deriveWhitespaceBoundaryContext: (textBeforeCursor: string) => CanonicalBoundaryContext | null;
+/**
+ * Return one prime per exact context, ordered from the latest surface start
+ * (normally the unigram window) to wider phrase windows.
+ */
+export declare const selectBoundaryPrimeRequests: (familyKey: string, candidates: Array<CanonicalCandidateContext>, maxPrimes: number) => BoundaryPrimeRequest[];
+/**
+ * `log(sum(exp(values)))`, shifted by the maximum so the sum cannot overflow or
+ * underflow to zero for the sequence log-likelihoods this is called with.
+ *
+ * Returns `-Infinity` for an empty or wholly non-finite input, which makes
+ * `exp(value - logSumExp(values))` evaluate to zero rather than `NaN`.
+ */
+export declare const logSumExp: (values: readonly number[]) => number;
+export declare const logSoftmaxAt: (logits: Float32Array, target: number) => number;
+/**
+ * One processor is registered for the causal model so the scheduler can read
+ * the next-token distribution off any forward pass.
+ *
+ * The scheduler drives decoding token by token — it prefills a prompt, then
+ * feeds each continuation token itself — so the processor never has to force
+ * the model onto a target path. It only has to hand back the distribution the
+ * forward pass produced, which it does without altering the logits.
+ */
+export declare class CanonicalLogitProcessor implements LogitProcessor {
+    private captured;
+    processLogits: (logits: Float32Array) => Float32Array;
+    processSampledToken: () => void;
+    resetState: () => void;
+    startCapture: () => void;
+    /**
+     * The captured distribution, or null when the forward pass never ran.
+     *
+     * `processLogits` already copied it, and every capture allocates afresh, so
+     * the buffer handed out here is never written to again.
+     */
+    getCapturedLogits: () => Float32Array | null;
+}
+export {};

package/dist/types/pm-plugins/debug-mode.d.ts

@@ -18,10 +18,64 @@
 declare global {
     interface Window {
         __atlCtcDebug__?: {
-            enable: () => void;
             disable: () => void;
+            /**
+             * Enable CTC debug logging for this session. Pass `'verbose'` to also
+             * emit the per-prediction candidate table, canonical LM, and grammar detail:
+             * `__atlCtcDebug__.enable('verbose')`.
+             */
+            enable: (level?: 'verbose') => void;
             isEnabled: () => boolean;
+            /** Whether verbose logging (candidate tables + extra detail) is on. */
+            isVerbose: () => boolean;
+            /**
+             * Snapshot of the L1 session boosts — the vocabulary words this session
+             * has seen and how often — or one family of them:
+             * `__atlCtcDebug__.session('poll')`.
+             *
+             * Installed by the predictor rather than declared with the rest of the
+             * API, and typed as `unknown` so the return shape can live with the module
+             * that owns it instead of creating a cycle back to this one.
+             */
+            session?: (prefix?: string) => unknown;
         };
     }
 }
+/**
+ * Shared `%c` styles for all CTC console output, so every log — lifecycle,
+ * async signals, and per-prediction groups — reads consistently.
+ */
+export declare const CTC_STYLES: {
+    brand: string;
+    section: string;
+    dim: string;
+    good: string;
+    warn: string;
+    bad: string;
+    cold: string;
+    lm: string;
+    body: string;
+};
+/**
+ * Log one aligned section line inside a per-prediction group, e.g.
+ * `INPUT      raw: "…"`. Label is padded so the bodies line up. No-ops unless
+ * debug is enabled, so callers don't need to guard.
+ */
+export declare const ctcSection: (label: string, body: string) => void;
+/**
+ * Log a lifecycle / async-signal line (outside the per-prediction groups),
+ * tagged `[CTC:<tag>]`. No-ops unless debug is enabled, so callers don't need to
+ * guard. Use tags like `init` (loads) and `signal` (slow-lane arrivals).
+ */
+export declare const ctcTag: (tag: string, body: string, tagStyle?: string) => void;
 export declare const isAutocompleteDebugEnabled: () => boolean;
+export declare const isAutocompleteDebugVerbose: () => boolean;
+/**
+ * Hang the L1 session-boost snapshot off the console API.
+ *
+ * Unlike the log helpers this is available whether or not debug is enabled:
+ * inspecting state on demand is not logging, and asking someone to turn on
+ * logging and retype to find out what the session already holds defeats the
+ * point of being able to ask.
+ */
+export declare const registerCtcSessionInspector: (inspect: (prefix?: string) => unknown) => void;

package/dist/types/pm-plugins/ghost-text-decoration.d.ts

@@ -1,7 +1,10 @@
-import type { EditorState } from '@atlaskit/editor-prosemirror/state';
+import type { Node as PMNode } from '@atlaskit/editor-prosemirror/model';
 import { DecorationSet } from '@atlaskit/editor-prosemirror/view';
 /**
  * Creates a DecorationSet containing a ghost text widget at the given position.
  * The ghost text is rendered as a styled <span> that appears after the cursor.
+ *
+ * Takes the document rather than the whole state so it can also be called from
+ * `apply`, where only the post-transaction doc exists.
  */
-export declare const createGhostTextDecorationSet: (state: EditorState, position: number, text: string) => DecorationSet;
+export declare const createGhostTextDecorationSet: (doc: PMNode, position: number, text: string) => DecorationSet;

package/dist/types/pm-plugins/local-slow-lane-client.d.ts

@@ -5,11 +5,9 @@
  * a backend API, this client runs two models in the browser via WebGPU, in a
  * single MLCEngine, to reproduce the BE encoder's outputs on-device:
  *
- *   - Causal LM (SmolLM2-135M-Instruct): one decode step per word boundary. A
- *     registered LogitProcessor captures the raw next-token logits, which
- *     `computeBePayload` turns into a whole-word `lm_logits` payload — a faithful
- *     port of the BE `CausalLMEncoder._get_top_k_probs` (masked softmax over the
- *     vocab's first-tokens, prefix expansion, L2 reservation, log-space pooling).
+ *   - Causal LM (SmolLM2-135M-Instruct): context-keyed one-step boundary primes
+ *     provide canonical first-token logits. A persistent token-prefix scheduler
+ *     expands shared paths and exact-scores only plausible finalists.
  *   - Semantic embedder (Snowflake Arctic Embed S): produces the real 384-d
  *     `semantic_vector`. Inputs are wrapped as passages (see `wrapForArctic`) so
  *     the runtime vector lands in the same space as the precomputed word bin.
@@ -19,7 +17,7 @@
  * the main thread is viable:
  *
  *   - WebGPU GPU compute is inherently async (doesn't block the main thread)
- *   - CPU overhead (BE-parity post-processing) is a few ms
+ *   - CPU overhead for named-token reads and cache bookkeeping is small
  *   - Per-inference latency is well within autocomplete expectations
  *     (~250 ms between word boundaries)
  *
@@ -30,10 +28,11 @@
  *   - Standard npm import — just works
  *
  * ── Interface ────────────────────────────────────────────────────────────
- * Same shape as createSlowLaneClient so text-predictor.ts needs zero changes.
- * The client exposes getContextVector() and getLmLogits() which are populated
- * asynchronously after each updateContext() call.
+ * Same base shape as createSlowLaneClient, plus on-device canonical-surface
+ * scoring. Semantic updates remain word-boundary timed; causal work is requested
+ * independently for exact pre-surface contexts.
  */
+import { type BoundaryLmState, type BoundaryPrimeRequest, type ProgressiveSurfaceEvidence, type SurfaceScore, type SurfaceScoreRequest } from './canonical-lm-scoring';
 export interface LocalSlowLaneClientConfig {
     /**
      * Optional custom model registration for models not in web-llm's
@@ -60,12 +59,25 @@ export interface LocalSlowLaneClientConfig {
      * `customModelConfig` with the model URL and WASM library URL.
      */
     modelId?: string;
+    /** Callback fired when a context-keyed Tier-A raw-logit vector is available. */
+    onBoundaryLmUpdate?: (opts: {
+        contextKey: string;
+        familyKey: string;
+        latencyMs: number;
+    }) => void;
     /** Callback fired when the engine fails to load/start. */
     onLoadError?: (error: LocalSlowLaneLoadError) => void;
     /** Callback fired when the engine successfully loads and is ready. */
     onLoadSuccess?: (info: LocalSlowLaneLoadSuccess) => void;
     /** Callback fired with status messages (model loading progress, etc.). */
     onStatus?: (message: string) => void;
+    /** Callback fired when grouped progress or exact surface scores become available. */
+    onSurfaceScoreUpdate?: (opts: {
+        contextKey: string;
+        count: number;
+        familyKey: string;
+        latencyMs: number;
+    }) => void;
     /** Callback fired when inference returns new results. */
     onUpdate?: (opts: {
         hasLmLogits: boolean;
@@ -78,11 +90,19 @@ export interface LocalSlowLaneClientConfig {
 export interface LocalSlowLaneClient {
     /** Clean up resources. */
     destroy: () => void;
+    getBoundaryLmState: (contextKey: string) => BoundaryLmState | null;
+    getCanonicalSurfaceCount: () => number;
+    getCanonicalSurfaceTokenIds: (surface: string) => number[] | null;
+    getContextInput: () => string | null;
     getContextVector: () => Float32Array | null;
     getLmLogits: () => Record<string, number> | null;
+    getProgressiveSurfaceEvidence: (contextKey: string, surface: string) => ProgressiveSurfaceEvidence | null;
+    getSurfaceScore: (contextKey: string, surface: string) => SurfaceScore | null;
     /** Whether the model is loaded and ready for inference. */
     isReady: () => boolean;
     isWordBoundary: (text: string) => boolean;
+    primeBoundaryLm: (input: BoundaryPrimeRequest) => void;
+    requestProgressiveSurfaceScores: (input: SurfaceScoreRequest) => void;
     setContextVector: (vector: Float32Array | null) => void;
     setLmLogits: (logits: Record<string, number> | null) => void;
     updateContext: (text: string) => void;
@@ -160,21 +180,7 @@ export declare const LOCAL_MLC_EMBEDDING_MODEL_ID = "snowflake-arctic-embed-s-q0
  * the relationship. Encode both sides as passages. See implementation.md §4.3.
  */
 export declare const wrapForArctic: (text: string) => string;
-/**
- * BE-parity constants — must match `CausalLMEncoder` defaults in the Python
- * sidecar (`cc-smarts/python-sidecar/src/causal_lm_encoder.py`) and
- * `SlowLaneEngine` (`typeahead_context_encoding.py`) so local payloads behave
- * identically to the server-client setup.
- */
-export declare const BE_PARITY: {
-    /** Final payload size cap (BE: `top_k_words`). */
-    readonly TOP_K_WORDS: 2000;
-    /** L2 (domain) words admitted unconditionally before pooling (BE: `reserved_l2_slots`). */
-    readonly RESERVED_L2_SLOTS: 500;
-    /** Log-space additive bias favouring L2 over L3 in the pool (BE: `l2_bias`). */
-    readonly L2_BIAS: 1;
-    /** Drop words below this probability from the final payload (BE: `> 0.00001`). */
-    readonly MIN_PROB: 0.00001;
+export declare const LOCAL_INFERENCE: {
     /**
      * Word-level approximation of the BE causal LM token limit.
      *
@@ -195,33 +201,61 @@ export declare const BE_PARITY: {
      */
     readonly MAX_CONTEXT_WORDS: 100;
 };
-/**
- * Convert a raw next-token logit vector into a whole-word probability payload,
- * faithfully porting the BE `CausalLMEncoder._get_top_k_probs`
- * (`cc-smarts/python-sidecar/src/causal_lm_encoder.py`).
- *
- * Steps: (1) numerically-stable masked softmax over only the token ids present
- * in the prefix-expansion map; (2) spread each token's probability to every
- * whole word sharing that first token, taking the max; (3) reserve the top L2
- * words unconditionally; (4) rank the remainder in a log-space pool with an
- * additive L2 bias; (5) emit raw probabilities for the survivors, lowercased
- * and trimmed at `MIN_PROB`.
- *
- * :params:
- *   rawLogits: Full-vocabulary logits from the LM's single decode step
- *   prefixMap: Map of first-token id to the words starting with that token
- *   domainWords: Set of L2 (domain) words, for tier-aware ranking
- * :returns:
- *   A record of lowercase word to probability — the BE `lm_logits` payload
- */
-export declare const computeBePayload: (rawLogits: Float32Array, prefixMap: Map<number, string[]>, domainWords: Set<string>, 
-/**
- * Pre-derived token-ID array for the softmax mask. Defaults to the
- * module-level `prefixMapTokenIds` (zero allocation in production). Pass
- * `Array.from(prefixMap.keys())` in tests that supply a custom prefixMap so
- * the softmax mask stays consistent with the iteration in Step 2.
- */
-validTokenIds?: number[]) => Record<string, number>;
+export declare const CANONICAL_SCORING: {
+    /**
+     * How many contexts keep their prefilled state before the oldest is dropped.
+     *
+     * Sizes every context-keyed cache together on purpose: an expansion needs
+     * both the boundary logits and the candidate list to still be resident, so
+     * bounding them separately would evict half of a context and strand the
+     * other half.
+     *
+     * The context of a keystroke is the text *before* the word being typed,
+     * which does not change while that word is typed, so one word should cost
+     * one prefill and then hit. Measured hit rate was 45% over the first 84s of
+     * a session and 24% over the following four minutes, well short of that, and
+     * prompt prefills per decision rose 0.80 → 1.25 across the same split while
+     * cost per prefill stayed flat. Contexts were being dropped while still live.
+     *
+     * A boundary entry holds a `Float32Array` over the 49,152-token vocabulary,
+     * so each one is ~192KB and this bound is the dominant term in the scorer's
+     * footprint: ~24MB resident here, against ~6MB at the 32 this replaced. That
+     * cost is what kept the bound low, not a hit rate anyone had measured.
+     */
+    readonly BOUNDARY_CACHE_MAX: 128;
+    readonly EXACT_MAX_TARGET_TOKENS: 8;
+    readonly PREFIX_CACHE_MAX: 256;
+    /**
+     * How many candidates compete for expansion in one context.
+     *
+     * Each distinct token prefix among them is a separate branch, and a branch
+     * the engine is not already standing on costs a prompt prefill before its
+     * first decode. A wide field therefore spreads a decision's round trips
+     * across candidates and finishes none of them inside the budget.
+     */
+    readonly PROGRESSIVE_CANDIDATES_MAX: 8;
+    /**
+     * Ceiling on distinct token prefixes explored per context.
+     *
+     * Raising this to 8 to give long surfaces more room did the opposite: model
+     * calls per decision went 2.35 → 4.32, the warm-KV extend share fell 40% →
+     * 31% as the extra branches displaced the live path, the prefix queue backed
+     * up to 29 deep, and acceptances per thousand decisions fell 9.3 → 6.7. No
+     * phrase was shown either way, so breadth was never the binding constraint.
+     *
+     * What blocked them then was normalisation against a short word's per-token
+     * mean, which has since been replaced by a posterior over sequence
+     * log-likelihoods. That removes the bias towards short surfaces but does not
+     * by itself make phrases reachable: a longer surface is strictly less likely
+     * than a shorter one, so a phrase sharing a shortlist with a unigram still
+     * holds little of its mass. Whether the remaining gap is the threshold or the
+     * comparison is still open.
+     */
+    readonly PROGRESSIVE_EXPANSIONS_PER_CONTEXT_MAX: 4;
+    readonly PROGRESSIVE_INPUT_MAX: 400;
+    readonly SURFACE_CACHE_MAX: 128;
+    readonly TIER_A_PRIMES_MAX: 3;
+};
 /**
  * Create a local slow-lane client powered by MLC WebLLM.
  *
@@ -233,9 +267,8 @@ validTokenIds?: number[]) => Record<string, number>;
  * const client = createLocalSlowLaneClient({ debounceMs: 300 });
  * // On word boundaries:
  * client.updateContext(docText);
- * // In scoring pipeline:
- * const vec = client.getContextVector();
- * const logits = client.getLmLogits();
+ * // Candidate scoring independently calls
+ * // primeBoundaryLm/requestProgressiveSurfaceScores.
  * // On plugin teardown:
  * client.destroy();
  * ```

package/dist/types/pm-plugins/scoring-pipeline.d.ts

@@ -7,18 +7,35 @@
  * The grammar/POS payloads are fetched from the CDN by loadGrammarDataAsync, so
  * the grammar filter is a pass-through until that resolves.
  */
+import type { CanonicalLmEvidence } from './canonical-lm-scoring';
+/**
+ * A completion term is `(term, termType)`. `word` is the v1 default; `bigram`
+ * (2 tokens) and `phrase` (3 tokens) are multi-word units mined by the producer.
+ */
+export type TermType = 'word' | 'bigram' | 'phrase';
 export interface ScoringCandidate {
+    /** NPMI-style association strength (bigram/phrase only). */
+    association?: number | null;
     authorFreq: number;
     docFreq: number;
+    /** Last-token POS of a multi-word term (bigram/phrase only). */
+    headPos?: string | null;
+    /** Per-token POS sequence of a multi-word term (bigram/phrase only). */
+    posSeq?: string[] | null;
     sessionFreq: number;
     tenantFreq: number;
+    /** Defaults to `'word'` when omitted (back-compat with word-only callers). */
+    termType?: TermType;
     word: string;
 }
 export interface ScoredCandidate {
     finalScore: number;
     freqScore: number;
+    lmEvidence: CanonicalLmEvidence | null;
     lmScore: number;
     semanticScore: number;
+    stage1Score: number;
+    termType: TermType;
     word: string;
 }
 /** Metadata returned by the grammar filter for debug logging in the caller. */
@@ -32,6 +49,18 @@ export interface GrammarFilterMeta {
 export declare const STAGE1_WEIGHT = 0.35;
 export declare const STAGE2_WEIGHT = 0.65;
 export declare const MIN_STAGE1_SCORE = 0.35;
+/**
+ * How far the leading candidate must sit above the runner-up on the blended
+ * score before a ghost is committed.
+ *
+ * Lives beside the weights it is measured in, because it is only meaningful on
+ * that scale — and because both the arbitration that applies it and the
+ * scheduler that decides when reading more tokens can no longer change the
+ * outcome have to use the same number. When those two drifted apart, the
+ * scheduler stopped early on a margin the arbitration did not recognise and the
+ * abstention that followed was invisible from either side.
+ */
+export declare const MIN_WINNER_MARGIN = 0.08;
 export declare const isGrammarDataLoaded: () => boolean;
 /**
  * Fetch the POS tag and grammar transition payloads from the CDN.
@@ -54,4 +83,7 @@ export interface RankCandidatesResult {
     grammarMeta: GrammarFilterMeta | null;
     pipelineDebug: PipelineDebug;
 }
-export declare function rankCandidates(candidates: ScoringCandidate[], contextVector: Float32Array | null, getWordVector: (word: string) => Float32Array | null, lmLogits: Record<string, number> | null, maxTenantFreq: number, previousWord: string): RankCandidatesResult;
+/**
+ * Apply Stage 1, grammar, and nullable context-local LM evidence to candidates.
+ */
+export declare function rankCandidates(candidates: ScoringCandidate[], contextVector: Float32Array | null, getWordVector: (word: string) => Float32Array | null, lmLogits: Record<string, number> | null, maxFreqByType: Record<TermType, number>, previousWord: string, getCanonicalLmEvidence?: (surface: string) => CanonicalLmEvidence | null): RankCandidatesResult;