@wei840222/qmd 2026.8.23

package/dist/llm.d.ts

@@ -0,0 +1,566 @@
+/**
+ * llm.ts - LLM abstraction layer for QMD using node-llama-cpp
+ *
+ * Provides embeddings, text generation, and reranking using local GGUF models.
+ */
+import type { Llama, Token as LlamaToken } from "node-llama-cpp";
+type NodeLlamaCppModule = {
+    getLlama: (options: Record<string, unknown>) => Promise<Llama>;
+    getLlamaGpuTypes?: (include?: "supported" | "allValid") => Promise<LlamaGpuMode[]>;
+    resolveModelFile: (model: string, optionsOrDirectory?: string | {
+        directory?: string;
+        cli?: boolean;
+    }) => Promise<string>;
+    LlamaChatSession: new (options: {
+        contextSequence: unknown;
+    }) => {
+        prompt: (prompt: string, options?: Record<string, unknown>) => Promise<string>;
+    };
+    LlamaLogLevel: {
+        error: unknown;
+    };
+};
+export declare function setNodeLlamaCppModuleForTest(module: NodeLlamaCppModule | null): void;
+/**
+ * Some node-llama-cpp native build/probe paths write library noise to stdout.
+ * JSON APIs must reserve stdout for machine-readable payloads, so route that
+ * noise to stderr while native llama initialization is in progress.
+ */
+export declare function withNativeStdoutRedirectedToStderr<T>(fn: () => Promise<T>): Promise<T>;
+/**
+ * Detect if a model URI uses the Qwen3-Embedding format.
+ * Qwen3-Embedding uses a different prompting style than nomic/embeddinggemma.
+ */
+export declare function isQwen3EmbeddingModel(modelUri: string): boolean;
+/**
+ * Format a query for embedding.
+ * Uses nomic-style task prefix format for embeddinggemma (default).
+ * Uses Qwen3-Embedding instruct format when a Qwen embedding model is active.
+ */
+export declare function formatQueryForEmbedding(query: string, modelUri?: string): string;
+/**
+ * Format a document for embedding.
+ * Uses nomic-style format with title and text fields (default).
+ * Qwen3-Embedding encodes documents as raw text without special prefixes.
+ */
+export declare function formatDocForEmbedding(text: string, title?: string, modelUri?: string): string;
+export declare function setLlamaDirWritableForTest(writable: boolean | undefined): void;
+/** Whether node-llama-cpp can write to its llama/ directory (false on NixOS). */
+export declare function canWriteLlamaDir(pkgDir?: string): boolean;
+/**
+ * Token with log probability
+ */
+export type TokenLogProb = {
+    token: string;
+    logprob: number;
+};
+/**
+ * Embedding result
+ */
+export type EmbeddingResult = {
+    embedding: number[];
+    model: string;
+};
+/**
+ * Generation result with optional logprobs
+ */
+export type GenerateResult = {
+    text: string;
+    model: string;
+    logprobs?: TokenLogProb[];
+    done: boolean;
+};
+/**
+ * Rerank result for a single document
+ */
+export type RerankDocumentResult = {
+    file: string;
+    score: number;
+    index: number;
+};
+/**
+ * Batch rerank result
+ */
+export type RerankResult = {
+    results: RerankDocumentResult[];
+    model: string;
+};
+/**
+ * Model info
+ */
+export type ModelInfo = {
+    name: string;
+    exists: boolean;
+    path?: string;
+};
+/**
+ * Options for embedding
+ */
+export type EmbedOptions = {
+    model?: string;
+    isQuery?: boolean;
+    title?: string;
+};
+/**
+ * Options for text generation
+ */
+export type GenerateOptions = {
+    model?: string;
+    maxTokens?: number;
+    temperature?: number;
+};
+/**
+ * Options for reranking
+ */
+export type RerankOptions = {
+    model?: string;
+};
+/**
+ * Options for LLM sessions
+ */
+export type LLMSessionOptions = {
+    /** Max session duration in ms (default: 10 minutes) */
+    maxDuration?: number;
+    /** External abort signal */
+    signal?: AbortSignal;
+    /** Debug name for logging */
+    name?: string;
+};
+/**
+ * Session interface for scoped LLM access with lifecycle guarantees
+ */
+export interface ILLMSession {
+    /** Embedding model actually loaded by the session owner. */
+    readonly embeddingModel: string;
+    embed(text: string, options?: EmbedOptions): Promise<EmbeddingResult | null>;
+    embedBatch(texts: string[], options?: EmbedOptions): Promise<(EmbeddingResult | null)[]>;
+    expandQuery(query: string, options?: {
+        context?: string;
+        includeLexical?: boolean;
+    }): Promise<Queryable[]>;
+    rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult>;
+    /** Whether this session is still valid (not released or aborted) */
+    readonly isValid: boolean;
+    /** Abort signal for this session (aborts on release or maxDuration) */
+    readonly signal: AbortSignal;
+}
+/**
+ * Supported query types for different search backends
+ */
+export type QueryType = 'lex' | 'vec' | 'hyde';
+/**
+ * A single query and its target backend type
+ */
+export type Queryable = {
+    type: QueryType;
+    text: string;
+};
+/**
+ * Document to rerank
+ */
+export type RerankDocument = {
+    file: string;
+    text: string;
+    title?: string;
+};
+export declare const LFM2_GENERATE_MODEL = "hf:LiquidAI/LFM2-1.2B-GGUF/LFM2-1.2B-Q4_K_M.gguf";
+export declare const LFM2_INSTRUCT_MODEL = "hf:LiquidAI/LFM2.5-1.2B-Instruct-GGUF/LFM2.5-1.2B-Instruct-Q4_K_M.gguf";
+export declare const DEFAULT_EMBED_MODEL_URI = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
+export declare const DEFAULT_RERANK_MODEL_URI = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf";
+export declare const DEFAULT_GENERATE_MODEL_URI = "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf";
+export type ModelResolutionConfig = {
+    embed?: string;
+    generate?: string;
+    rerank?: string;
+    embed_api_model?: string;
+    generate_api_model?: string;
+    rerank_api_model?: string;
+    embed_api_url?: string;
+    generate_api_url?: string;
+    generate_url?: string;
+    generate_base_url?: string;
+    rerank_api_url?: string;
+    rerank_url?: string;
+    rerank_base_url?: string;
+    [key: string]: any;
+};
+export declare function resolveEmbedModel(config?: ModelResolutionConfig): string;
+export declare function resolveGenerateModel(config?: ModelResolutionConfig): string;
+export declare function resolveRerankModel(config?: ModelResolutionConfig): string;
+export declare function resolveModels(config?: ModelResolutionConfig): Required<Pick<ModelResolutionConfig, "embed" | "generate" | "rerank">>;
+export declare const DEFAULT_MODEL_CACHE_DIR: string;
+export type PullResult = {
+    model: string;
+    path: string;
+    sizeBytes: number;
+    refreshed: boolean;
+};
+export type GgufFileInspection = {
+    exists: boolean;
+    valid: boolean;
+    kind: "missing" | "gguf" | "html" | "invalid";
+    sizeBytes?: number;
+    magic?: string;
+    details: string;
+};
+/**
+ * Inspect a potential GGUF model file without mutating it.
+ * Used by doctor for early diagnostics and by runtime validation before load.
+ */
+export declare function inspectGgufFile(filePath: string): GgufFileInspection;
+export declare function pullModels(models: string[], options?: {
+    refresh?: boolean;
+    cacheDir?: string;
+    cli?: boolean;
+}): Promise<PullResult[]>;
+/**
+ * Abstract LLM interface - implement this for different backends
+ */
+export interface LLM {
+    /**
+     * Get embeddings for text
+     */
+    embed(text: string, options?: EmbedOptions): Promise<EmbeddingResult | null>;
+    /**
+     * Generate text completion
+     */
+    generate(prompt: string, options?: GenerateOptions): Promise<GenerateResult | null>;
+    /**
+     * Check if a model exists/is available
+     */
+    modelExists(model: string): Promise<ModelInfo>;
+    /**
+     * Expand a search query into multiple variations for different backends.
+     * Returns a list of Queryable objects.
+     */
+    expandQuery(query: string, options?: {
+        context?: string;
+        includeLexical?: boolean;
+    }): Promise<Queryable[]>;
+    /**
+     * Rerank documents by relevance to a query
+     * Returns list of documents with relevance scores (higher = more relevant)
+     */
+    rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult>;
+    /**
+     * Dispose of resources
+     */
+    dispose(): Promise<void>;
+}
+export type LlamaCppConfig = {
+    embedModel?: string;
+    generateModel?: string;
+    rerankModel?: string;
+    modelCacheDir?: string;
+    /**
+     * Context size used for query expansion generation contexts.
+     * Default: 2048. Can also be set via QMD_EXPAND_CONTEXT_SIZE.
+     */
+    expandContextSize?: number;
+    /**
+     * Inactivity timeout in ms before unloading contexts (default: 2 minutes, 0 to disable).
+     *
+     * Per node-llama-cpp lifecycle guidance, we prefer keeping models loaded and only disposing
+     * contexts when idle, since contexts (and their sequences) are the heavy per-session objects.
+     * @see https://node-llama-cpp.withcat.ai/guide/objects-lifecycle
+     */
+    inactivityTimeoutMs?: number;
+    /**
+     * Whether to dispose models on inactivity (default: false).
+     *
+     * Keeping models loaded avoids repeated VRAM thrash; set to true only if you need aggressive
+     * memory reclaim.
+     */
+    disposeModelsOnInactivity?: boolean;
+};
+export type LlamaGpuMode = "auto" | "metal" | "vulkan" | "cuda" | false;
+type ParallelismOptions = {
+    gpu: string | false;
+    platform?: NodeJS.Platform;
+    computed: number;
+    envValue?: string;
+};
+export declare function resolveParallelismOverride(envValue?: string | undefined): number | undefined;
+export declare function resolveSafeParallelism(options: ParallelismOptions): number;
+/** Measured nomic-embed / embeddinggemma-300M embedding-context cost at 2048 tokens. */
+export declare const BASELINE_EMBED_CONTEXT_MB = 150;
+/**
+ * VRAM to leave free when sizing the embedding-context pool so `query` can
+ * still create a rerank context afterwards. Matches the 1000 MB figure
+ * `ensureRerankContexts` already uses as its per-context cost.
+ */
+export declare const EMBED_POOL_RERANK_RESERVE_MB = 1000;
+/**
+ * GPU embedding-context pool size from free VRAM.
+ *
+ * Uses 25% of (free − reserve), then clamps to `[1, cap]`. The reserve keeps
+ * the pool from consuming the memory `query` needs next for the reranker (#799).
+ */
+export declare function computeGpuContextPoolSize(options: {
+    freeMB: number;
+    perContextMB: number;
+    reserveMB?: number;
+    cap?: number;
+}): number;
+/**
+ * Estimate one embedding context's VRAM from the GGUF weight-file size.
+ *
+ * Small models (nomic / embeddinggemma-class, ≲350 MB) stay at the measured
+ * 150 MB baseline so default `qmd embed` throughput is unchanged. Larger
+ * files — Qwen3-Embedding-0.6B-Q8 is ~640 MB — were measured at ~1190 MB per
+ * 2048-token context, about 1.85× the weight file, because the KV cache
+ * dominates (#799).
+ */
+export declare function estimateEmbedContextMB(options: {
+    modelBytes: number;
+    contextSize?: number;
+}): number;
+export declare function resolveLlamaGpuMode(envValue?: string | undefined, forceCpuValue?: string | undefined): LlamaGpuMode;
+export declare class LlamaCpp implements LLM {
+    private readonly _ciMode;
+    private llama;
+    private embedModel;
+    private embedModelPath;
+    private embedContexts;
+    private generateModel;
+    private rerankModel;
+    private rerankContexts;
+    private embedModelUri;
+    private generateModelUri;
+    private rerankModelUri;
+    private modelCacheDir;
+    private expandContextSize;
+    private embedModelLoadPromise;
+    private generateModelLoadPromise;
+    private rerankModelLoadPromise;
+    private rerankContextsCreatePromise;
+    private llamaLoadPromise;
+    private inactivityTimer;
+    private idleUnloadPromise;
+    private inactivityTimeoutMs;
+    private disposeModelsOnInactivity;
+    private closing;
+    private disposed;
+    private disposePromise;
+    constructor(config?: LlamaCppConfig);
+    get embedModelName(): string;
+    get generateModelName(): string;
+    get rerankModelName(): string;
+    /**
+     * Reset the inactivity timer. Called after each model operation.
+     * When timer fires, models are unloaded to free memory (if no active sessions).
+     */
+    private touchActivity;
+    /**
+     * Check if any contexts are currently loaded (and therefore worth unloading on inactivity).
+     */
+    private hasLoadedContexts;
+    /**
+     * Unload idle resources but keep the instance alive for future use.
+     *
+     * By default, this disposes contexts (and their dependent sequences), while keeping models loaded.
+     * This matches the intended lifecycle: model → context → sequence, where contexts are per-session.
+     */
+    unloadIdleResources(): Promise<void>;
+    /** Wait for an inactivity unload that already owns this instance's resources. */
+    waitForIdleUnload(): Promise<void>;
+    /**
+     * Acquire a lease without yielding between the idle-unload check and the
+     * caller's synchronous lease registration.
+     */
+    acquireAfterIdleUnload<T>(acquire: () => T): Promise<T>;
+    private disposeIdleResources;
+    /**
+     * Ensure model cache directory exists
+     */
+    private ensureModelCacheDir;
+    /**
+     * Initialize the llama instance (lazy)
+     */
+    private ensureLlama;
+    private loadLlamaRuntime;
+    private isCpuOffloadForced;
+    private modelLoadOptions;
+    /**
+     * Resolve a model URI to a local path, downloading if needed.
+     * Validates the downloaded file is actually a GGUF model (not an HTML error page
+     * from a proxy or firewall).
+     */
+    private resolveModel;
+    /**
+     * Load embedding model (lazy)
+     */
+    private ensureEmbedModel;
+    /**
+     * Compute how many parallel contexts to create.
+     *
+     * GPU: constrained by VRAM (25% of free, capped at 8).
+     * CPU: constrained by cores. Splitting threads across contexts enables
+     *      true parallelism (each context runs on its own cores). Use at most
+     *      half the math cores, with at least 4 threads per context.
+     */
+    private computeParallelism;
+    /**
+     * Get the number of threads each context should use, given N parallel contexts.
+     * Splits available math cores evenly across contexts.
+     */
+    private threadsPerContext;
+    /**
+     * Load embedding contexts (lazy). Creates multiple for parallel embedding.
+     * Uses promise guard to prevent concurrent context creation race condition.
+     */
+    private embedContextsCreatePromise;
+    private ensureEmbedContexts;
+    /**
+     * Get a single embed context (for single-embed calls). Uses first from pool.
+     */
+    private ensureEmbedContext;
+    /**
+     * Load generation model (lazy) - context is created fresh per call
+     */
+    private ensureGenerateModel;
+    /**
+     * Load rerank model (lazy)
+     */
+    private ensureRerankModel;
+    /**
+     * Load rerank contexts (lazy). Creates multiple contexts for parallel ranking.
+     * Each context has its own sequence, so they can evaluate independently.
+     *
+     * VRAM per context is governed by contextSize alone —
+     * LlamaRankingContextOptions has no flashAttention option.
+     */
+    private static readonly RERANK_CONTEXT_SIZE;
+    private static readonly EMBED_CONTEXT_SIZE;
+    private ensureRerankContexts;
+    /**
+     * Tokenize text using the embedding model's tokenizer
+     * Returns tokenizer tokens (opaque type from node-llama-cpp)
+     */
+    tokenize(text: string): Promise<readonly LlamaToken[]>;
+    /**
+     * Count tokens in text using the embedding model's tokenizer
+     */
+    countTokens(text: string): Promise<number>;
+    /**
+     * Detokenize token IDs back to text
+     */
+    detokenize(tokens: readonly LlamaToken[]): Promise<string>;
+    /**
+     * Truncate text to fit within the embedding model's context window.
+     * Uses the model's own tokenizer for accurate token counting, then
+     * detokenizes back to text if truncation is needed.
+     * Returns the (possibly truncated) text and whether truncation occurred.
+     */
+    private resolveEmbedTokenLimit;
+    private truncateToContextSize;
+    embed(text: string, options?: EmbedOptions): Promise<EmbeddingResult | null>;
+    /**
+     * Batch embed multiple texts efficiently
+     * Uses Promise.all for parallel embedding - node-llama-cpp handles batching internally
+     */
+    embedBatch(texts: string[], options?: EmbedOptions): Promise<(EmbeddingResult | null)[]>;
+    generate(prompt: string, options?: GenerateOptions): Promise<GenerateResult | null>;
+    modelExists(modelUri: string): Promise<ModelInfo>;
+    expandQuery(query: string, options?: {
+        context?: string;
+        includeLexical?: boolean;
+    }): Promise<Queryable[]>;
+    private static readonly RERANK_TEMPLATE_OVERHEAD;
+    private static readonly RERANK_TARGET_DOCS_PER_CONTEXT;
+    rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult>;
+    /**
+     * Get device/GPU info for status display.
+     * Initializes llama if not already done.
+     */
+    getDeviceInfo(options?: {
+        allowBuild?: boolean;
+    }): Promise<{
+        gpu: string | false;
+        gpuOffloading: boolean;
+        gpuDevices: string[];
+        vram?: {
+            total: number;
+            used: number;
+            free: number;
+        };
+        cpuCores: number;
+    }>;
+    dispose(): Promise<void>;
+    private disposeAfterDrain;
+}
+/**
+ * Error thrown when an operation is attempted on a released or aborted session.
+ */
+export declare class SessionReleasedError extends Error {
+    constructor(message?: string);
+}
+/**
+ * Execute a function with a scoped LLM session.
+ * The session provides lifecycle guarantees - resources won't be disposed mid-operation.
+ *
+ * @example
+ * ```typescript
+ * await withLLMSession(async (session) => {
+ *   const expanded = await session.expandQuery(query);
+ *   const embeddings = await session.embedBatch(texts);
+ *   const reranked = await session.rerank(query, docs);
+ *   return reranked;
+ * }, { maxDuration: 10 * 60 * 1000, name: 'querySearch' });
+ * ```
+ */
+export declare function withLLMSession<T>(fn: (session: ILLMSession) => Promise<T>, options?: LLMSessionOptions): Promise<T>;
+/**
+ * Execute a function with a scoped LLM session using a specific LlamaCpp instance.
+ * Unlike withLLMSession, this does not use the global singleton.
+ */
+export declare function withLLMSessionForLlm<T>(llm: LlamaCpp, fn: (session: ILLMSession) => Promise<T>, options?: LLMSessionOptions): Promise<T>;
+/** Wait until all scoped sessions using a specific LlamaCpp instance have settled. */
+export declare function waitForLLMSessionsToDrain(llm: LlamaCpp): Promise<void>;
+/**
+ * Check if idle unload is safe (no active sessions or operations).
+ * Used internally by LlamaCpp idle timer.
+ */
+export declare function canUnloadLLM(llm?: LlamaCpp): boolean;
+/**
+ * Whether QMD's darwin Metal exit-crash mitigation is active in this process:
+ *   true  → residency sets disabled, process exit completes silently
+ *   false → either non-darwin, or `QMD_METAL_KEEP_RESIDENCY=1` overrode it,
+ *           in which case the libggml-metal teardown assertion may fire
+ */
+export declare function isDarwinMetalMitigationActive(): boolean;
+/**
+ * Compatibility shim: previous releases installed a `process.on('exit')` hook
+ * that tried to skip the C++ static destructor by calling `process.reallyExit`.
+ * That mechanism didn't work on Node (Environment::Exit still calls libc
+ * `exit()`), so it was replaced by `GGML_METAL_NO_RESIDENCY=1` from bin/qmd.
+ * Kept as a no-op for code paths that still call it; safe to remove once no
+ * production launcher predates the residency-set fix.
+ */
+export declare function installDarwinExitGuard(): void;
+/** @deprecated Replaced by isDarwinMetalMitigationActive. */
+export declare function isDarwinExitGuardInstalled(): boolean;
+/**
+ * Get the default LlamaCpp instance (creates one if needed). The LlamaCpp
+ * constructor installs the darwin exit guard, so any code path that obtains
+ * the singleton is protected.
+ */
+export declare function getDefaultLlamaCpp(): LlamaCpp;
+/**
+ * Set a custom default LlamaCpp instance (useful for testing). Setting a
+ * non-null instance also ensures the darwin exit guard is installed — keeps
+ * the invariant intact for test doubles that didn't go through the real
+ * constructor.
+ */
+export declare function setDefaultLlamaCpp(llm: LlamaCpp | null): void;
+/**
+ * Peek at the default LlamaCpp instance without instantiating one. Used by
+ * doctor and lifecycle diagnostics.
+ */
+export declare function hasDefaultLlamaCpp(): boolean;
+/**
+ * Dispose the default LlamaCpp instance if it exists.
+ * Call this before process exit to prevent NAPI crashes.
+ */
+export declare function disposeDefaultLlamaCpp(): Promise<void>;
+export {};