@xelauvas/xela-cli 0.3.0 → 0.4.0

package/src/services/contextCollapse/index.ts

@@ -0,0 +1,322 @@
+import { randomUUID } from 'crypto'
+import { feature } from 'bun:bundle'
+import { queryHaiku } from '../../services/api/claude.js'
+import type { ToolUseContext } from '../../Tool.js'
+import type { Message } from '../../types/message.js'
+import { isEnvTruthy } from '../../utils/envUtils.js'
+import { logForDebugging } from '../../utils/debug.js'
+import {
+  recordContextCollapseCommit,
+  recordContextCollapseSnapshot,
+} from '../../utils/sessionStorage.js'
+import { asSystemPrompt } from '../../utils/systemPromptType.js'
+import { getEffectiveContextWindowSize } from '../compact/autoCompact.js'
+import {
+  type CommittedCollapse,
+  estimateTokens,
+  freshStore,
+  getStats as getStatsImpl,
+  getStore,
+  makeSummaryContent,
+  nextCollapseId,
+  notify,
+  projectView,
+  renderMessageForSummary,
+  type StagedSpan,
+} from './operations.js'
+
+export { projectView } from './operations.js'
+export { restoreFromEntries } from './persist.js'
+
+// Fractions of the effective context window. Each has an env override
+// (CLAUDE_CONTEXT_COLLAPSE_<NAME>) for tuning and tests.
+function fraction(name: string, fallback: number): number {
+  const v = Number(process.env[`CLAUDE_CONTEXT_COLLAPSE_${name}`])
+  return Number.isFinite(v) && v > 0 && v <= 1 ? v : fallback
+}
+function integer(name: string, fallback: number): number {
+  const v = Number(process.env[`CLAUDE_CONTEXT_COLLAPSE_${name}`])
+  return Number.isInteger(v) && v >= 0 ? v : fallback
+}
+const SPAWN_FRACTION = fraction('SPAWN_FRACTION', 0.5) // start summarizing the oldest span above this
+const COMMIT_FRACTION = fraction('COMMIT_FRACTION', 0.75) // swap staged spans for placeholders above this
+const PROTECTED_TAIL_FRACTION = fraction('PROTECTED_TAIL_FRACTION', 0.25) // never touch the most recent share of the context
+const SPAN_TARGET_FRACTION = fraction('SPAN_TARGET_FRACTION', 0.3) // a span summarizes about this much of the view
+const MIN_SPAWN_INTERVAL_TOKENS = integer('MIN_SPAWN_INTERVAL_TOKENS', 6_000) // growth needed between two spawns
+const MIN_SPAN_MESSAGES = integer('MIN_SPAN_MESSAGES', 4)
+const MAX_SUMMARY_INPUT_CHARS = 60_000
+const EMPTY_SPAWN_WARNING_AFTER = 3
+
+export function initContextCollapse(): void {
+  const store = getStore()
+  store.enabled =
+    feature('CONTEXT_COLLAPSE') &&
+    !isEnvTruthy(process.env.CLAUDE_CODE_DISABLE_CONTEXT_COLLAPSE) &&
+    process.env.CLAUDE_CONTEXT_COLLAPSE !== '0'
+  notify()
+}
+
+export function isContextCollapseEnabled(): boolean {
+  return feature('CONTEXT_COLLAPSE') && getStore().enabled
+}
+
+export function resetContextCollapse(): void {
+  const store = getStore()
+  const { enabled, listeners } = store
+  Object.assign(store, freshStore(), { enabled, listeners })
+  notify()
+}
+
+export function getStats(): ReturnType<typeof getStatsImpl> {
+  return getStatsImpl()
+}
+
+export function subscribe(listener: () => void): () => void {
+  const store = getStore()
+  store.listeners.add(listener)
+  return () => {
+    store.listeners.delete(listener)
+  }
+}
+
+function isMainThread(querySource: string, toolUseContext: ToolUseContext): boolean {
+  if ((toolUseContext as { agentId?: unknown }).agentId) return false
+  return querySource === 'repl_main_thread' || querySource === 'sdk'
+}
+
+function isRealUserTurn(message: Message): boolean {
+  if (message.type !== 'user') return false
+  if ((message as { isMeta?: boolean }).isMeta) return false
+  const content = (message as { message: { content: unknown } }).message.content
+  if (!Array.isArray(content)) return true
+  return !(content as Array<{ type: string }>).some(b => b.type === 'tool_result')
+}
+
+function isPlaceholder(message: Message): boolean {
+  const content = (message as { message?: { content?: unknown } }).message?.content
+  return typeof content === 'string' && content.startsWith('<collapsed id="')
+}
+
+/**
+ * Picks the oldest span worth summarizing: starts at the first real user
+ * turn after the last placeholder, ends right before a later real user turn,
+ * covers about SPAN_TARGET_FRACTION of the view and stays out of the
+ * protected tail.
+ */
+function chooseSpan(view: Message[], totalTokens: number): { start: number; end: number } | null {
+  let tailTokens = 0
+  let protectedFrom = view.length
+  for (let i = view.length - 1; i >= 0; i--) {
+    tailTokens += estimateTokens([view[i]!])
+    if (tailTokens >= totalTokens * PROTECTED_TAIL_FRACTION) {
+      protectedFrom = i
+      break
+    }
+  }
+  // Back protectedFrom up to a real user turn so spans end on turn boundaries
+  while (protectedFrom < view.length && protectedFrom > 0 && !isRealUserTurn(view[protectedFrom]!)) protectedFrom--
+
+  let start = 0
+  for (let i = 0; i < protectedFrom; i++) {
+    if (isPlaceholder(view[i]!)) start = i + 1
+  }
+  while (start < protectedFrom && !isRealUserTurn(view[start]!)) start++
+  if (start >= protectedFrom) return null
+
+  const target = totalTokens * SPAN_TARGET_FRACTION
+  let acc = 0
+  let end = -1
+  for (let i = start; i < protectedFrom; i++) {
+    acc += estimateTokens([view[i]!])
+    if (i + 1 < protectedFrom && isRealUserTurn(view[i + 1]!)) {
+      end = i
+      if (acc >= target) break
+    }
+  }
+  if (end < 0) end = protectedFrom - 1
+  if (end - start + 1 < MIN_SPAN_MESSAGES) return null
+  return { start, end }
+}
+
+function spanAlreadyStaged(store: ReturnType<typeof getStore>, startUuid: string): boolean {
+  return store.staged.some(s => s.startUuid === startUuid)
+}
+
+async function spawnSummarizer(
+  view: Message[],
+  span: { start: number; end: number },
+  toolUseContext: ToolUseContext,
+): Promise<void> {
+  const store = getStore()
+  store.spawning = true
+  store.health.totalSpawns++
+  notify()
+  const messages = view.slice(span.start, span.end + 1)
+  let transcript = messages.map(renderMessageForSummary).join('\n\n')
+  if (transcript.length > MAX_SUMMARY_INPUT_CHARS) {
+    transcript = transcript.slice(0, MAX_SUMMARY_INPUT_CHARS) + '\n…(truncated)'
+  }
+  const controller = new AbortController()
+  try {
+    const result = await queryHaiku({
+      systemPrompt: asSystemPrompt([
+        'You condense a slice of a coding-assistant conversation so it can be replaced in the context window. Keep every fact that later work could depend on: file paths, function and variable names, commands run and their outcomes, decisions and their reasons, errors seen, open questions, user preferences. Drop chit-chat, repeated tool output and dead ends (one line each). Write in compact prose or bullets, third person, no preamble.',
+      ]),
+      userPrompt: `Summarize this conversation slice:\n\n${transcript}`,
+      signal: controller.signal,
+      options: {
+        querySource: 'compact',
+        agents: [],
+        isNonInteractiveSession: toolUseContext.options.isNonInteractiveSession,
+        hasAppendSystemPrompt: false,
+        mcpTools: [],
+      },
+    })
+    const summary = result.message.content
+      .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
+      .map(b => b.text)
+      .join('\n')
+      .trim()
+    if (!summary) {
+      store.health.totalEmptySpawns++
+      if (store.health.totalEmptySpawns >= EMPTY_SPAWN_WARNING_AFTER) {
+        store.health.emptySpawnWarningEmitted = true
+      }
+      return
+    }
+    store.health.totalEmptySpawns = 0
+    store.health.emptySpawnWarningEmitted = false
+    const staged: StagedSpan = {
+      startUuid: messages[0]!.uuid,
+      endUuid: messages[messages.length - 1]!.uuid,
+      summary,
+      risk: 0,
+      stagedAt: Date.now(),
+    }
+    store.staged.push(staged)
+    void recordContextCollapseSnapshot({
+      staged: store.staged,
+      armed: store.armed,
+      lastSpawnTokens: store.lastSpawnTokens,
+    }).catch(() => {})
+    logForDebugging(`[context-collapse] staged span of ${messages.length} messages (${summary.length} chars)`)
+  } catch (err) {
+    store.health.totalErrors++
+    store.health.lastError = err instanceof Error ? err.message : String(err)
+    logForDebugging(`[context-collapse] summarizer failed: ${store.health.lastError}`)
+  } finally {
+    store.spawning = false
+    notify()
+  }
+}
+
+function commitStaged(view: Message[]): { view: Message[]; committed: number } {
+  const store = getStore()
+  let committed = 0
+  const remaining: StagedSpan[] = []
+  for (const s of store.staged) {
+    const first = view.findIndex(m => m.uuid === s.startUuid)
+    const last = view.findIndex(m => m.uuid === s.endUuid)
+    if (first < 0 || last < first) {
+      // span no longer exists in the view (compacted or snipped meanwhile)
+      continue
+    }
+    const collapseId = nextCollapseId()
+    const commit: CommittedCollapse = {
+      collapseId,
+      summaryUuid: randomUUID(),
+      summaryContent: makeSummaryContent(collapseId, s.summary),
+      summary: s.summary,
+      firstArchivedUuid: s.startUuid,
+      lastArchivedUuid: s.endUuid,
+      archived: view.slice(first, last + 1),
+    }
+    store.commits.push(commit)
+    committed++
+    void recordContextCollapseCommit({
+      collapseId: commit.collapseId,
+      summaryUuid: commit.summaryUuid,
+      summaryContent: commit.summaryContent,
+      summary: commit.summary,
+      firstArchivedUuid: commit.firstArchivedUuid,
+      lastArchivedUuid: commit.lastArchivedUuid,
+    }).catch(() => {})
+  }
+  store.staged = remaining
+  if (committed > 0) {
+    void recordContextCollapseSnapshot({
+      staged: store.staged,
+      armed: store.armed,
+      lastSpawnTokens: store.lastSpawnTokens,
+    }).catch(() => {})
+    notify()
+  }
+  return { view: projectView(view), committed }
+}
+
+/**
+ * Called on every main-thread query. Projects committed collapses, commits
+ * staged ones when the context is high, and kicks off a background
+ * summarizer when it is time to prepare the next span.
+ */
+export async function applyCollapsesIfNeeded(
+  messages: Message[],
+  toolUseContext: ToolUseContext,
+  querySource: string,
+): Promise<{ messages: Message[] }> {
+  if (!isContextCollapseEnabled() || !isMainThread(querySource, toolUseContext)) {
+    return { messages: projectView(messages) }
+  }
+  const store = getStore()
+  let view = projectView(messages)
+  const window = getEffectiveContextWindowSize(toolUseContext.options.mainLoopModel)
+  let tokens = estimateTokens(view)
+
+  if (store.staged.length > 0 && tokens >= window * COMMIT_FRACTION) {
+    const result = commitStaged(view)
+    view = result.view
+    tokens = estimateTokens(view)
+  }
+
+  if (
+    store.armed &&
+    !store.spawning &&
+    tokens >= window * SPAWN_FRACTION &&
+    tokens - store.lastSpawnTokens >= MIN_SPAWN_INTERVAL_TOKENS
+  ) {
+    const span = chooseSpan(view, tokens)
+    if (span && !spanAlreadyStaged(store, view[span.start]!.uuid)) {
+      store.lastSpawnTokens = tokens
+      void spawnSummarizer(view, span, toolUseContext)
+    }
+  }
+  return { messages: view }
+}
+
+/**
+ * Overflow recovery: the request was too long, so swap every staged span for
+ * its placeholder now and let the caller retry.
+ */
+export function recoverFromOverflow(
+  messages: Message[],
+  _querySource: string,
+): { messages: Message[]; committed: number } {
+  if (!isContextCollapseEnabled()) return { messages, committed: 0 }
+  const result = commitStaged(projectView(messages))
+  return { messages: result.view, committed: result.committed }
+}
+
+/**
+ * A prompt-too-long error is withheld from the conversation (and handled by
+ * recoverFromOverflow) only when there is staged material to commit.
+ */
+export function isWithheldPromptTooLong(
+  message: Message,
+  isPromptTooLongMessage: (m: Message) => boolean,
+  querySource: string,
+): boolean {
+  if (!isContextCollapseEnabled()) return false
+  if (querySource !== 'repl_main_thread' && querySource !== 'sdk') return false
+  if (getStore().staged.length === 0) return false
+  return isPromptTooLongMessage(message)
+}

package/src/services/contextCollapse/operations.ts

@@ -0,0 +1,187 @@
+import type { Message } from '../../types/message.js'
+import { createUserMessage } from '../../utils/messages.js'
+
+/**
+ * Context collapse: spans of older messages are summarized in the
+ * background ("staged") and later swapped for a placeholder ("committed").
+ * The store is process-wide on globalThis because several entry points load
+ * this module through the CJS require shim and would otherwise get copies.
+ */
+export type CommittedCollapse = {
+  collapseId: string
+  summaryUuid: string
+  summaryContent: string
+  summary: string
+  firstArchivedUuid: string
+  lastArchivedUuid: string
+  /** Original messages, filled lazily by projectView on first match. */
+  archived: Message[]
+}
+
+export type StagedSpan = {
+  startUuid: string
+  endUuid: string
+  summary: string
+  risk: number
+  stagedAt: number
+}
+
+export type CollapseHealth = {
+  totalSpawns: number
+  totalErrors: number
+  totalEmptySpawns: number
+  lastError?: string
+  emptySpawnWarningEmitted: boolean
+}
+
+export type CollapseStore = {
+  enabled: boolean
+  commits: CommittedCollapse[]
+  staged: StagedSpan[]
+  armed: boolean
+  lastSpawnTokens: number
+  spawning: boolean
+  idCounter: number
+  health: CollapseHealth
+  listeners: Set<() => void>
+}
+
+export function getStore(): CollapseStore {
+  const g = globalThis as unknown as { __xelaContextCollapse?: CollapseStore }
+  if (!g.__xelaContextCollapse) {
+    g.__xelaContextCollapse = freshStore()
+  }
+  return g.__xelaContextCollapse
+}
+
+export function freshStore(): CollapseStore {
+  return {
+    enabled: false,
+    commits: [],
+    staged: [],
+    armed: true,
+    lastSpawnTokens: 0,
+    spawning: false,
+    idCounter: 0,
+    health: {
+      totalSpawns: 0,
+      totalErrors: 0,
+      totalEmptySpawns: 0,
+      emptySpawnWarningEmitted: false,
+    },
+    listeners: new Set(),
+  }
+}
+
+export function notify(): void {
+  for (const l of getStore().listeners) {
+    try {
+      l()
+    } catch {
+      // listener errors must not break the query loop
+    }
+  }
+}
+
+export function nextCollapseId(): string {
+  const store = getStore()
+  store.idCounter += 1
+  return String(store.idCounter).padStart(16, '0')
+}
+
+export function makeSummaryContent(collapseId: string, summary: string): string {
+  return `<collapsed id="${collapseId}">\n${summary}\n</collapsed>`
+}
+
+export function makePlaceholder(commit: CommittedCollapse): Message {
+  return createUserMessage({
+    content: commit.summaryContent,
+    isMeta: true,
+    uuid: commit.summaryUuid as never,
+  })
+}
+
+/**
+ * Replays the commit log on a message list: each committed span, when both
+ * boundaries are present, is replaced by its placeholder. Pure; safe to run
+ * on every query.
+ */
+export function projectView<T extends Message>(messages: T[]): T[] {
+  const store = getStore()
+  if (store.commits.length === 0) return messages
+  let view: Message[] = messages
+  for (const commit of store.commits) {
+    const first = view.findIndex(m => m.uuid === commit.firstArchivedUuid)
+    if (first < 0) continue
+    const last = view.findIndex(m => m.uuid === commit.lastArchivedUuid)
+    if (last < first) continue
+    if (commit.archived.length === 0) {
+      commit.archived = view.slice(first, last + 1)
+    }
+    view = [...view.slice(0, first), makePlaceholder(commit), ...view.slice(last + 1)]
+  }
+  return view as T[]
+}
+
+export function estimateTokens(messages: readonly Message[]): number {
+  let total = 0
+  for (const m of messages) total += estimateMessageTokens(m)
+  return total
+}
+
+export function estimateMessageTokens(message: Message): number {
+  const m = message as { message?: { content?: unknown }; content?: unknown }
+  const content = m.message?.content ?? m.content ?? ''
+  const text = typeof content === 'string' ? content : JSON.stringify(content)
+  return Math.ceil(text.length / 4) + 8
+}
+
+export function getStats(): {
+  collapsedSpans: number
+  stagedSpans: number
+  collapsedMessages: number
+  health: CollapseHealth
+} {
+  const store = getStore()
+  return {
+    collapsedSpans: store.commits.length,
+    stagedSpans: store.staged.length,
+    collapsedMessages: store.commits.reduce((n, c) => n + c.archived.length, 0),
+    health: store.health,
+  }
+}
+
+/** Plain-text rendering of a message for the summarizer prompt. */
+export function renderMessageForSummary(message: Message): string {
+  const m = message as {
+    type: string
+    message?: { role?: string; content?: unknown }
+    content?: unknown
+  }
+  const role = m.type === 'assistant' ? 'Assistant' : m.type === 'user' ? 'User' : m.type
+  const content = m.message?.content ?? m.content ?? ''
+  if (typeof content === 'string') return `${role}: ${content}`
+  if (!Array.isArray(content)) return `${role}: ${JSON.stringify(content)}`
+  const parts: string[] = []
+  for (const block of content as Array<Record<string, unknown>>) {
+    switch (block.type) {
+      case 'text':
+        parts.push(String(block.text ?? ''))
+        break
+      case 'tool_use':
+        parts.push(`[tool ${String(block.name)} ${JSON.stringify(block.input).slice(0, 400)}]`)
+        break
+      case 'tool_result': {
+        const c = block.content
+        const text = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => (x as { text?: string }).text ?? '').join('\n') : ''
+        parts.push(`[result ${text.slice(0, 1500)}${text.length > 1500 ? '…' : ''}]`)
+        break
+      }
+      case 'thinking':
+        break
+      default:
+        parts.push(`[${String(block.type)}]`)
+    }
+  }
+  return `${role}: ${parts.join('\n')}`
+}

package/src/services/contextCollapse/persist.ts

@@ -0,0 +1,40 @@
+import type {
+  ContextCollapseCommitEntry,
+  ContextCollapseSnapshotEntry,
+} from '../../types/logs.js'
+import { freshStore, getStore, notify } from './operations.js'
+
+/**
+ * Rebuilds the store from transcript entries on resume. Commits replay in
+ * order with an empty archive (projectView fills it lazily); the snapshot
+ * restores the staged queue and the spawn clock. Always resets first so a
+ * resume into a session with no entries drops the previous session's state.
+ */
+export function restoreFromEntries(
+  commits: ContextCollapseCommitEntry[],
+  snapshot: ContextCollapseSnapshotEntry | undefined,
+): void {
+  const store = getStore()
+  const enabled = store.enabled
+  const listeners = store.listeners
+  Object.assign(store, freshStore(), { enabled, listeners })
+  for (const c of commits) {
+    store.commits.push({
+      collapseId: c.collapseId,
+      summaryUuid: c.summaryUuid,
+      summaryContent: c.summaryContent,
+      summary: c.summary,
+      firstArchivedUuid: c.firstArchivedUuid,
+      lastArchivedUuid: c.lastArchivedUuid,
+      archived: [],
+    })
+    const n = Number(c.collapseId)
+    if (Number.isFinite(n) && n > store.idCounter) store.idCounter = n
+  }
+  if (snapshot) {
+    store.staged = snapshot.staged.map(s => ({ ...s }))
+    store.armed = snapshot.armed
+    store.lastSpawnTokens = snapshot.lastSpawnTokens
+  }
+  notify()
+}

package/src/services/extractMemories/extractMemories.ts

@@ -26,6 +26,7 @@ import {
   getAutoMemPath,
   isAutoMemoryEnabled,
   isAutoMemPath,
+  isExtractGateOpen,
 } from '../../memdir/paths.js'
 import type { Tool } from '../../Tool.js'
 import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js'
@@ -376,9 +377,17 @@ export function initExtractMemories(): void {
     // process already-committed work that should not be throttled.
     if (!isTrailingRun) {
       turnsSinceLastExtraction++
+      // Local backend: the extraction is a full model call on the same GPU
+      // as the conversation, so default to every 3 turns
+      // (CLAUDE_CODE_MEMORY_EXTRACT_EVERY overrides).
+      const localEvery = process.env.OPENAI_API_KEY
+        ? Math.max(1, Number(process.env.CLAUDE_CODE_MEMORY_EXTRACT_EVERY) || 3)
+        : null
       if (
         turnsSinceLastExtraction <
-        (getFeatureValue_CACHED_MAY_BE_STALE('tengu_bramble_lintel', null) ?? 1)
+        (localEvery ??
+          getFeatureValue_CACHED_MAY_BE_STALE('tengu_bramble_lintel', null) ??
+          1)
       ) {
         return
       }
@@ -533,7 +542,7 @@ export function initExtractMemories(): void {
       return
     }
 
-    if (!getFeatureValue_CACHED_MAY_BE_STALE('tengu_passport_quail', false)) {
+    if (!isExtractGateOpen()) {
       if (process.env.USER_TYPE === 'ant' && !hasLoggedGateFailure) {
         hasLoggedGateFailure = true
         logEvent('tengu_extract_memories_gate_disabled', {})

package/src/services/skillSearch/featureCheck.ts

@@ -0,0 +1,12 @@
+import { feature } from 'bun:bundle'
+import { isEnvTruthy } from '../../utils/envUtils.js'
+
+/**
+ * Skill search replaces the full skill listing in the system prompt with
+ * per-turn "skills relevant to your task" suggestions. Local-only here: the
+ * index is lexical, over the skills installed on this machine.
+ */
+export function isSkillSearchEnabled(): boolean {
+  if (!feature('EXPERIMENTAL_SKILL_SEARCH')) return false
+  return !isEnvTruthy(process.env.CLAUDE_CODE_DISABLE_SKILL_SEARCH)
+}

package/src/services/skillSearch/localSearch.ts

@@ -0,0 +1,98 @@
+import type { Command } from '../../commands.js'
+
+export type SkillIndexEntry = {
+  name: string
+  description: string
+  /** lowercased tokens of name + description, with a name-weight prefix */
+  tokens: Map<string, number>
+  loadedFrom?: string
+}
+
+export type SkillHit = { name: string; description: string; score: number }
+
+const STOP = new Set(
+  'the a an and or of to in on for with is are be this that it as at by from into your you my our we i me do does how what which when where can should would could please make use using about need want like just also then than over under not no yes if else all any some more most very really'.split(
+    ' ',
+  ),
+)
+
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9àâîșşțţăéèêë\-_/\s]/g, ' ')
+    .split(/[\s/_\-]+/)
+    .map(t => t.trim())
+    .filter(t => t.length > 2 && !STOP.has(t))
+    .map(stem)
+}
+
+// Tiny stemmer: enough to match "testing" with "tests" and "deploying" with "deploy"
+function stem(word: string): string {
+  return word
+    .replace(/(ations?|ing|ers?|ed|es|s)$/u, (m, _g, offset) => (offset > 3 ? '' : m))
+}
+
+let cache: { key: string; index: SkillIndexEntry[] } | null = null
+
+function indexKey(commands: Command[]): string {
+  return commands.map(c => c.name).join('|')
+}
+
+/** Lexical index over prompt-type commands (skills). Memoized per command set. */
+export function getSkillIndex(commands: Command[]): SkillIndexEntry[] {
+  const key = indexKey(commands)
+  if (cache && cache.key === key) return cache.index
+  const index: SkillIndexEntry[] = []
+  for (const c of commands) {
+    if (c.type !== 'prompt') continue
+    if ((c as { isHidden?: boolean }).isHidden) continue
+    const description = c.description ?? ''
+    const tokens = new Map<string, number>()
+    for (const t of tokenize(c.name.replace(/[:]/g, ' '))) tokens.set(t, (tokens.get(t) ?? 0) + 2)
+    for (const t of tokenize(description)) tokens.set(t, (tokens.get(t) ?? 0) + 1)
+    index.push({
+      name: c.name,
+      description,
+      tokens,
+      loadedFrom: (c as { loadedFrom?: string }).loadedFrom ?? (c as { source?: string }).source,
+    })
+  }
+  cache = { key, index }
+  return index
+}
+
+export function clearSkillIndexCache(): void {
+  cache = null
+}
+
+/**
+ * Scores skills against a query: sum of matched token weights, scaled by
+ * inverse document frequency so generic words ("code", "file") count less.
+ */
+export function searchSkills(
+  query: string,
+  commands: Command[],
+  options: { limit?: number; minScore?: number; exclude?: Set<string> } = {},
+): SkillHit[] {
+  const index = getSkillIndex(commands)
+  const q = tokenize(query)
+  if (q.length === 0 || index.length === 0) return []
+  const df = new Map<string, number>()
+  for (const entry of index) for (const t of entry.tokens.keys()) df.set(t, (df.get(t) ?? 0) + 1)
+  const n = index.length
+  const hits: SkillHit[] = []
+  for (const entry of index) {
+    if (options.exclude?.has(entry.name)) continue
+    let score = 0
+    for (const t of new Set(q)) {
+      const w = entry.tokens.get(t)
+      if (!w) continue
+      const idf = Math.log(1 + n / (df.get(t) ?? 1))
+      score += w * idf
+    }
+    if (score > 0) hits.push({ name: entry.name, description: entry.description, score })
+  }
+  hits.sort((a, b) => b.score - a.score)
+  const minScore = options.minScore ?? 3
+  return hits.filter(h => h.score >= minScore).slice(0, options.limit ?? 5)
+}