@atlaskit/editor-plugin-autocomplete 8.0.0 → 9.1.0

package/dist/es2019/pm-plugins/canonical-lm-scoring.js

@@ -0,0 +1,270 @@
+import _defineProperty from "@babel/runtime/helpers/defineProperty";
+const createTokenTrieNode = () => ({
+  children: new Map()
+});
+
+/**
+ * Token trie for the producer's canonical leading-space token sequences.
+ * It lets the scorer expand a shared token prefix once for every surface below
+ * that node (`root cause`, `root directory`, ...), rather than guessing two
+ * arbitrary full surfaces from a broad one-character string prefix.
+ */
+export class CanonicalSurfaceTokenTrie {
+  constructor(entries = []) {
+    _defineProperty(this, "root", createTokenTrieNode());
+    _defineProperty(this, "tokenIdsBySurface", new Map());
+    for (const [surface, tokenIds] of entries) {
+      this.insert(surface, tokenIds);
+    }
+  }
+  insert(surface, tokenIds) {
+    if (tokenIds.length === 0) {
+      return;
+    }
+    const normalizedSurface = surface.toLowerCase();
+    this.tokenIdsBySurface.set(normalizedSurface, [...tokenIds]);
+    let node = this.root;
+    for (const tokenId of tokenIds) {
+      let child = node.children.get(tokenId);
+      if (!child) {
+        child = createTokenTrieNode();
+        node.children.set(tokenId, child);
+      }
+      node = child;
+    }
+  }
+  getTokenIds(surface) {
+    const tokenIds = this.tokenIdsBySurface.get(surface.toLowerCase());
+    return tokenIds ? [...tokenIds] : null;
+  }
+  groupByScoredPrefix(candidates) {
+    const groups = new Map();
+    for (const candidate of candidates) {
+      var _groups$get;
+      if (candidate.scoredTokenCount <= 0 || candidate.scoredTokenCount >= candidate.tokenIds.length) {
+        continue;
+      }
+      const tokenPrefix = candidate.tokenIds.slice(0, candidate.scoredTokenCount);
+      let node = this.root;
+      for (const tokenId of tokenPrefix) {
+        node = node.children.get(tokenId);
+        if (!node) {
+          break;
+        }
+      }
+      if (!node) {
+        continue;
+      }
+      const key = tokenPrefix.join(',');
+      const group = (_groups$get = groups.get(key)) !== null && _groups$get !== void 0 ? _groups$get : {
+        surfaces: [],
+        tokenPrefix
+      };
+      group.surfaces.push(candidate.surface);
+      groups.set(key, group);
+    }
+    return Array.from(groups.values());
+  }
+}
+const WHITESPACE_REGEX = /\s/u;
+const makeContextKey = (prompt, separatorKind) => `${separatorKind}\u0000${prompt}`;
+
+/** The part of a candidate's context that depends only on where its surface starts. */
+
+/**
+ * Per-pass store for the position-derived half of a candidate's context.
+ *
+ * Every unigram candidate for a keystroke shares one surface start, and the
+ * phrase candidates share a handful more, so without this the same slice,
+ * `trimEnd` and context key are rebuilt for each of the couple of hundred
+ * candidates. Valid only for a single `textBeforeCursor`.
+ */
+
+export const createCanonicalContextPositionCache = () => new Map();
+
+/**
+ * Derive the causal prompt and artifact eligibility from the exact, untrimmed
+ * pre-cursor text. The shipped artifact contains only tokenizations of
+ * `" " + surface`, so it is valid only when the surface is actually preceded
+ * by whitespace.
+ */
+export const deriveCanonicalCandidateContext = (textBeforeCursor, matchedPrefixLen, surface, getTokenIds, surfaceStartOverride, positionCache) => {
+  const derivedSurfaceStart = textBeforeCursor.length - matchedPrefixLen;
+  const surfaceStart = Math.max(0, Math.min(textBeforeCursor.length, surfaceStartOverride !== null && surfaceStartOverride !== void 0 ? surfaceStartOverride : derivedSurfaceStart));
+  let position = positionCache === null || positionCache === void 0 ? void 0 : positionCache.get(surfaceStart);
+  if (position === undefined) {
+    const precedingChar = surfaceStart > 0 ? textBeforeCursor[surfaceStart - 1] : null;
+    const separatorKind = precedingChar === null ? 'document-start' : WHITESPACE_REGEX.test(precedingChar) ? 'whitespace' : 'non-space';
+    const contextBeforeSurface = textBeforeCursor.slice(0, surfaceStart).trimEnd();
+    position = {
+      contextBeforeSurface,
+      contextKey: makeContextKey(contextBeforeSurface, separatorKind),
+      separatorKind,
+      surfaceStart
+    };
+    positionCache === null || positionCache === void 0 ? void 0 : positionCache.set(surfaceStart, position);
+  }
+  return {
+    ...position,
+    canonicalTokenIds: position.separatorKind === 'whitespace' ? getTokenIds(surface) : null
+  };
+};
+
+/**
+ * Derive the next surface's leading-space context at an actual word boundary.
+ * This lets the local LM start its one-token prime on the space keystroke,
+ * before the user has typed the first character of the next word.
+ */
+export const deriveWhitespaceBoundaryContext = textBeforeCursor => {
+  var _textBeforeCursor$at;
+  if (textBeforeCursor.length === 0 || !WHITESPACE_REGEX.test((_textBeforeCursor$at = textBeforeCursor.at(-1)) !== null && _textBeforeCursor$at !== void 0 ? _textBeforeCursor$at : '')) {
+    return null;
+  }
+  const prompt = textBeforeCursor.trimEnd();
+  if (prompt.length === 0) {
+    return null;
+  }
+  return {
+    contextKey: makeContextKey(prompt, 'whitespace'),
+    prompt
+  };
+};
+
+/**
+ * Return one prime per exact context, ordered from the latest surface start
+ * (normally the unigram window) to wider phrase windows.
+ */
+export const selectBoundaryPrimeRequests = (familyKey, candidates, maxPrimes) => {
+  const byContext = new Map();
+  for (const candidate of candidates) {
+    if (candidate.canonicalTokenIds === null) {
+      continue;
+    }
+    const current = byContext.get(candidate.contextKey);
+    if (!current || candidate.surfaceStart > current.surfaceStart) {
+      byContext.set(candidate.contextKey, candidate);
+    }
+  }
+  return Array.from(byContext.values()).sort((a, b) => b.surfaceStart - a.surfaceStart).slice(0, Math.max(0, maxPrimes)).map((candidate, priority) => ({
+    familyKey,
+    contextKey: candidate.contextKey,
+    prompt: candidate.contextBeforeSurface,
+    priority
+  }));
+};
+
+/**
+ * `log(sum(exp(values)))`, shifted by the maximum so the sum cannot overflow or
+ * underflow to zero for the sequence log-likelihoods this is called with.
+ *
+ * Returns `-Infinity` for an empty or wholly non-finite input, which makes
+ * `exp(value - logSumExp(values))` evaluate to zero rather than `NaN`.
+ */
+export const logSumExp = values => {
+  let max = -Infinity;
+  for (const value of values) {
+    if (Number.isFinite(value) && value > max) {
+      max = value;
+    }
+  }
+  if (!Number.isFinite(max)) {
+    return -Infinity;
+  }
+  let sumExp = 0;
+  for (const value of values) {
+    if (Number.isFinite(value)) {
+      sumExp += Math.exp(value - max);
+    }
+  }
+  return sumExp > 0 ? max + Math.log(sumExp) : -Infinity;
+};
+
+/** The two shift-and-sum terms of a distribution's log-partition. */
+
+/**
+ * The log-partition depends only on the buffer, but scoring reads the same
+ * buffer once per candidate — so without this the two full-vocabulary passes
+ * below run N times per boundary for an answer that cannot change.
+ *
+ * Keying on the buffer itself is safe because captured logits are never
+ * written to again (see `CanonicalLogitProcessor.processLogits`, which copies
+ * out of WebLLM's reused buffer), and the weak reference lets a retired
+ * boundary's entry go with it.
+ */
+const logPartitionTermsCache = new WeakMap();
+const getLogPartitionTerms = logits => {
+  const cached = logPartitionTermsCache.get(logits);
+  if (cached !== undefined) {
+    return cached;
+  }
+  let maxLogit = -Infinity;
+  for (let i = 0; i < logits.length; i++) {
+    if (logits[i] > maxLogit) {
+      maxLogit = logits[i];
+    }
+  }
+  let terms = null;
+  if (Number.isFinite(maxLogit)) {
+    let sumExp = 0;
+    for (let i = 0; i < logits.length; i++) {
+      sumExp += Math.exp(logits[i] - maxLogit);
+    }
+    if (sumExp > 0) {
+      terms = {
+        maxLogit,
+        logSumTerm: Math.log(sumExp)
+      };
+    }
+  }
+  logPartitionTermsCache.set(logits, terms);
+  return terms;
+};
+export const logSoftmaxAt = (logits, target) => {
+  if (target < 0 || target >= logits.length) {
+    return -Infinity;
+  }
+  const terms = getLogPartitionTerms(logits);
+  if (terms === null) {
+    return -Infinity;
+  }
+  return logits[target] - terms.maxLogit - terms.logSumTerm;
+};
+
+/**
+ * One processor is registered for the causal model so the scheduler can read
+ * the next-token distribution off any forward pass.
+ *
+ * The scheduler drives decoding token by token — it prefills a prompt, then
+ * feeds each continuation token itself — so the processor never has to force
+ * the model onto a target path. It only has to hand back the distribution the
+ * forward pass produced, which it does without altering the logits.
+ */
+export class CanonicalLogitProcessor {
+  constructor() {
+    _defineProperty(this, "captured", null);
+    _defineProperty(this, "processLogits", logits => {
+      // WebLLM reuses its logits buffer between forwards, so keep a private copy.
+      this.captured = new Float32Array(logits);
+      return logits;
+    });
+    _defineProperty(this, "processSampledToken", () => {
+      // The scheduler chooses the next token; WebLLM's sample is discarded.
+    });
+    _defineProperty(this, "resetState", () => {
+      // WebLLM resets the processor after the caller has armed a capture and
+      // immediately before it prefills a completion prompt, so this only ever
+      // clears a stale reading from the previous run.
+      this.captured = null;
+    });
+    _defineProperty(this, "startCapture", () => {
+      this.captured = null;
+    });
+    /**
+     * The captured distribution, or null when the forward pass never ran.
+     *
+     * `processLogits` already copied it, and every capture allocates afresh, so
+     * the buffer handed out here is never written to again.
+     */
+    _defineProperty(this, "getCapturedLogits", () => this.captured);
+  }
+}

package/dist/es2019/pm-plugins/debug-mode.js

@@ -16,16 +16,83 @@
  * block uncategorized localStorage/sessionStorage/cookie writes in some products.
  */
 
-const hasUrlDebugFlag = () => {
+/**
+ * Shared `%c` styles for all CTC console output, so every log — lifecycle,
+ * async signals, and per-prediction groups — reads consistently.
+ */
+export const CTC_STYLES = {
+  brand: 'color: #00b8d9; font-weight: bold;',
+  section: 'color: #9c27b0; font-weight: bold;',
+  dim: 'color: #888; font-style: italic;',
+  good: 'color: #4caf50; font-weight: bold;',
+  warn: 'color: #ff9800; font-weight: bold;',
+  bad: 'color: #f44336; font-weight: bold;',
+  cold: 'color: #9e9e9e; font-weight: bold;',
+  lm: 'color: #e83e8c; font-weight: bold;',
+  body: 'color: inherit; font-weight: normal;'
+};
+
+/**
+ * Log one aligned section line inside a per-prediction group, e.g.
+ * `INPUT      raw: "…"`. Label is padded so the bodies line up. No-ops unless
+ * debug is enabled, so callers don't need to guard.
+ */
+export const ctcSection = (label, body) => {
+  if (!isAutocompleteDebugEnabled()) {
+    return;
+  }
+  // eslint-disable-next-line no-console
+  console.log(`%c${label.padEnd(10)}%c${body}`, CTC_STYLES.section, CTC_STYLES.body);
+};
+
+/**
+ * Log a lifecycle / async-signal line (outside the per-prediction groups),
+ * tagged `[CTC:<tag>]`. No-ops unless debug is enabled, so callers don't need to
+ * guard. Use tags like `init` (loads) and `signal` (slow-lane arrivals).
+ */
+export const ctcTag = (tag, body, tagStyle = CTC_STYLES.brand) => {
+  if (!isAutocompleteDebugEnabled()) {
+    return;
+  }
+  // eslint-disable-next-line no-console
+  console.log(`%c[CTC:${tag}]%c ${body}`, tagStyle, CTC_STYLES.body);
+};
+const readUrlDebugFlag = () => {
   if (typeof window === 'undefined') {
-    return false;
+    return {
+      enabled: false,
+      verbose: false
+    };
   }
   try {
-    return new URLSearchParams(window.location.search).get('atlCtcDebug') === '1';
+    const value = new URLSearchParams(window.location.search).get('atlCtcDebug');
+    return {
+      enabled: value === '1' || value === 'verbose',
+      verbose: value === 'verbose'
+    };
   } catch {
-    return false;
+    return {
+      enabled: false,
+      verbose: false
+    };
   }
 };
+const printLegend = verbose => {
+  // eslint-disable-next-line no-console
+  console.groupCollapsed(`%c[CTC]%c debug enabled${verbose ? ' (verbose)' : ''} — Contextual Typeahead Completion`, CTC_STYLES.brand, CTC_STYLES.body);
+  // eslint-disable-next-line no-console
+  console.log('%cPipeline%c  INPUT → SIGNALS → CANONICAL → GENERATE → SCORE → ARBITRATE → STABILIZE → CANDIDATES', CTC_STYLES.section, CTC_STYLES.body);
+  // eslint-disable-next-line no-console
+  console.log('%cEvidence%c  Tier A = exact-context first token · prefix = one more decoded token · exact = every token of the surface scored · absent = Stage 1', CTC_STYLES.section, CTC_STYLES.body);
+  // eslint-disable-next-line no-console
+  console.log('%cLM queue%c  one serialised GPU queue drains both: prompt prefills and single-token decode steps', CTC_STYLES.section, CTC_STYLES.body);
+  // eslint-disable-next-line no-console
+  console.log('%cPlanes%c    [CTC:init] loads · [CTC:signal] semantic/network · [CTC:model] primes/exact · [CTC:model-cost] prefill/decode · [CTC:readiness] deadline progress', CTC_STYLES.section, CTC_STYLES.body);
+  // eslint-disable-next-line no-console
+  console.log('%cInspect%c   __atlCtcDebug__.enable("verbose") · .disable() · .session() (L1 boosts, narrow to a family with .session("poll"))', CTC_STYLES.section, CTC_STYLES.body);
+  // eslint-disable-next-line no-console
+  console.groupEnd();
+};
 
 // State lives on the window object (not a module closure) so duplicate copies of this
 // module across separate bundles/realms share one source of truth and the console API
@@ -35,15 +102,24 @@ const getDebugApi = () => {
     return undefined;
   }
   if (!window.__atlCtcDebug__) {
-    let debugEnabled = hasUrlDebugFlag();
+    const initial = readUrlDebugFlag();
+    let debugEnabled = initial.enabled;
+    let debugVerbose = initial.verbose;
+    if (debugEnabled) {
+      printLegend(debugVerbose);
+    }
     window.__atlCtcDebug__ = {
-      enable: () => {
+      enable: level => {
         debugEnabled = true;
+        debugVerbose = level === 'verbose';
+        printLegend(debugVerbose);
       },
       disable: () => {
         debugEnabled = false;
+        debugVerbose = false;
       },
-      isEnabled: () => debugEnabled
+      isEnabled: () => debugEnabled,
+      isVerbose: () => debugEnabled && debugVerbose
     };
   }
   return window.__atlCtcDebug__;
@@ -52,6 +128,25 @@ export const isAutocompleteDebugEnabled = () => {
   var _getDebugApi$isEnable, _getDebugApi;
   return (_getDebugApi$isEnable = (_getDebugApi = getDebugApi()) === null || _getDebugApi === void 0 ? void 0 : _getDebugApi.isEnabled()) !== null && _getDebugApi$isEnable !== void 0 ? _getDebugApi$isEnable : false;
 };
+export const isAutocompleteDebugVerbose = () => {
+  var _getDebugApi$isVerbos, _getDebugApi2;
+  return (_getDebugApi$isVerbos = (_getDebugApi2 = getDebugApi()) === null || _getDebugApi2 === void 0 ? void 0 : _getDebugApi2.isVerbose()) !== null && _getDebugApi$isVerbos !== void 0 ? _getDebugApi$isVerbos : false;
+};
+
+/**
+ * Hang the L1 session-boost snapshot off the console API.
+ *
+ * Unlike the log helpers this is available whether or not debug is enabled:
+ * inspecting state on demand is not logging, and asking someone to turn on
+ * logging and retype to find out what the session already holds defeats the
+ * point of being able to ask.
+ */
+export const registerCtcSessionInspector = inspect => {
+  const api = getDebugApi();
+  if (api) {
+    api.session = inspect;
+  }
+};
 
 // Eagerly install so the console API is available on load, regardless of call order.
 getDebugApi();

package/dist/es2019/pm-plugins/ghost-text-decoration.js

@@ -4,8 +4,11 @@ const GHOST_TEXT_CLASS = 'autocomplete-ghost-text';
 /**
  * Creates a DecorationSet containing a ghost text widget at the given position.
  * The ghost text is rendered as a styled <span> that appears after the cursor.
+ *
+ * Takes the document rather than the whole state so it can also be called from
+ * `apply`, where only the post-transaction doc exists.
  */
-export const createGhostTextDecorationSet = (state, position, text) => {
+export const createGhostTextDecorationSet = (doc, position, text) => {
   if (!text) {
     return DecorationSet.empty;
   }
@@ -30,7 +33,13 @@ export const createGhostTextDecorationSet = (state, position, text) => {
   }, {
     side: 1,
     // Render after content at this position
-    key: 'autocomplete-ghost-text'
+    // A matching key short-circuits `WidgetType.eq`, so ProseMirror reuses
+    // the rendered node and never calls `toDOM` again. Keying on the text
+    // keeps that reuse when nothing changed while still forcing a redraw
+    // when the ghost advances through a keystroke that confirmed it —
+    // otherwise the stale tail stays on screen and Tab inserts a
+    // character the user already typed.
+    key: `autocomplete-ghost-text:${text}`
   });
-  return DecorationSet.create(state.doc, [decoration]);
+  return DecorationSet.create(doc, [decoration]);
 };