@wei840222/qmd 2026.8.23

package/dist/index.js

@@ -0,0 +1,418 @@
+/**
+ * QMD SDK - Library mode for programmatic access to QMD search and indexing.
+ *
+ * Usage:
+ *   import { createStore } from '@wei840222/qmd'
+ *
+ *   const store = await createStore({
+ *     dbPath: './my-index.sqlite',
+ *     config: {
+ *       collections: {
+ *         docs: { path: '/path/to/docs', pattern: '**\/*.md' }
+ *       }
+ *     }
+ *   })
+ *
+ *   const results = await store.search({ query: "how does auth work?" })
+ *   await store.close()
+ */
+import { existsSync } from "node:fs";
+import { createStore as createStoreInternal, hybridQuery, structuredSearch, extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES, reindexCollection, generateEmbeddings, listCollections as storeListCollections, syncConfigToDb, getStoreCollections, getStoreCollection, getStoreGlobalContext, getStoreContexts, upsertStoreCollection, removeCollection as removeCollectionWithDocuments, renameCollection as renameCollectionWithDocuments, updateStoreContext, removeStoreContext, setStoreGlobalContext, vacuumDatabase, cleanupOrphanedContent, cleanupOrphanedVectors, deleteLLMCache, deleteInactiveDocuments, clearAllEmbeddings, getPendingEmbeddingDocsReadOnly, getIndexHealthReadOnly, getStatusReadOnly, } from "./store.js";
+import { DEFAULT_EMBED_MODEL_URI, LlamaCpp, waitForLLMSessionsToDrain, } from "./llm.js";
+import { LocalEmbeddingProviderOwner } from "./embedding/local.js";
+import { OpenAIEmbeddingProvider, UnavailableOpenAIEmbeddingProvider, } from "./embedding/openai.js";
+import { authorizeRemoteEmbeddingRequest, remoteEmbeddingIdentity, } from "./embedding/remote-embedding.js";
+import { readStoredEmbeddingIdentity } from "./embedding/identity.js";
+import { inspectIndexDiagnostics } from "./diagnostics.js";
+import { EmbeddingConfigError, readCanonicalEmbeddingConfig, resolveEmbeddingConfig, writeCanonicalEmbeddingConfig, } from "./embedding/config.js";
+import { rebuildCjkLexicalIndex } from "./search/cjk-index.js";
+import { RemoteLLM } from "./remote-llm.js";
+import { HybridLLM } from "./hybrid-llm.js";
+import { createCollectionConfigSource, loadConfig, addCollection as collectionsAddCollection, removeCollection as collectionsRemoveCollection, renameCollection as collectionsRenameCollection, addContext as collectionsAddContext, removeContext as collectionsRemoveContext, setGlobalContext as collectionsSetGlobalContext, } from "./collections.js";
+// Re-export utility functions and types used by frontends
+export { extractSnippet, addLineNumbers, DEFAULT_MULTI_GET_MAX_BYTES };
+// Re-export getDefaultDbPath for CLI/MCP that need the default database location
+export { getDefaultDbPath } from "./store.js";
+// Re-export Maintenance class for CLI housekeeping operations
+export { Maintenance } from "./maintenance.js";
+/**
+ * Create a QMD store for programmatic access to search and indexing.
+ *
+ * @example
+ * ```typescript
+ * // With a YAML config file
+ * const store = await createStore({
+ *   dbPath: './index.sqlite',
+ *   configPath: './qmd.yml',
+ * })
+ *
+ * // With inline config (no files needed besides the DB)
+ * const store = await createStore({
+ *   dbPath: './index.sqlite',
+ *   config: {
+ *     collections: {
+ *       docs: { path: '/path/to/docs', pattern: '**\/*.md' }
+ *     }
+ *   }
+ * })
+ *
+ * const results = await store.search({ query: "authentication flow" })
+ * await store.close()
+ * ```
+ */
+export async function createStore(options) {
+    if (!options.dbPath) {
+        throw new Error("dbPath is required");
+    }
+    if (options.configPath && options.config) {
+        throw new Error("Provide either configPath or config, not both");
+    }
+    // A missing read-only index uses an initialized in-memory store without
+    // creating the requested on-disk file.
+    const useTemporaryStore = options.readOnly === true && !existsSync(options.dbPath);
+    const internal = createStoreInternal(useTemporaryStore ? ":memory:" : options.dbPath, { readOnly: options.readOnly === true && !useTemporaryStore });
+    const db = internal.db;
+    const externalConfigSource = options.configPath
+        ? createCollectionConfigSource({ configPath: options.configPath })
+        : options.config
+            ? createCollectionConfigSource({ config: options.config })
+            : undefined;
+    // Sync config into SQLite store_collections
+    let config;
+    if (externalConfigSource) {
+        config = loadConfig(externalConfigSource);
+        if (!options.readOnly || useTemporaryStore)
+            syncConfigToDb(db, config);
+    }
+    // else: DB-only mode — no external config, use existing store_collections
+    const embedding = resolveEmbeddingConfig({
+        config,
+        dbConfig: readCanonicalEmbeddingConfig(db),
+        defaultLocalModel: DEFAULT_EMBED_MODEL_URI,
+    });
+    if (!options.readOnly || useTemporaryStore) {
+        writeCanonicalEmbeddingConfig(db, embedding.canonical);
+    }
+    // Create a per-store LlamaCpp instance — lazy-loads models on first use,
+    // auto-unloads after 5 min inactivity to free VRAM.
+    const localLlm = new LlamaCpp({
+        embedModel: embedding.canonical.provider === "local"
+            ? embedding.canonical.model
+            : DEFAULT_EMBED_MODEL_URI,
+        generateModel: config?.models?.generate,
+        rerankModel: config?.models?.rerank,
+        inactivityTimeoutMs: 5 * 60 * 1000,
+        disposeModelsOnInactivity: true,
+    });
+    const rawGenerateUrl = config?.models?.generate_url ?? config?.models?.generate_base_url ?? config?.models?.generate_api_url;
+    const rawRerankUrl = config?.models?.rerank_url ?? config?.models?.rerank_base_url ?? config?.models?.rerank_api_url;
+    const hasRemoteLLM = Boolean(rawGenerateUrl || rawRerankUrl);
+    const remoteLlm = hasRemoteLLM
+        ? new RemoteLLM({
+            generateUrl: config?.models?.generate_url,
+            generateBaseUrl: config?.models?.generate_base_url,
+            generateApiUrl: config?.models?.generate_api_url,
+            generateApiModel: config?.models?.generate_api_model,
+            generateApiKey: config?.models?.generate_api_key,
+            rerankUrl: config?.models?.rerank_url,
+            rerankBaseUrl: config?.models?.rerank_base_url,
+            rerankApiUrl: config?.models?.rerank_api_url,
+            rerankApiModel: config?.models?.rerank_api_model,
+            rerankApiKey: config?.models?.rerank_api_key,
+            timeoutMs: options.remoteRequestTimeoutMs,
+        })
+        : undefined;
+    const llm = remoteLlm ? new HybridLLM(localLlm, remoteLlm) : localLlm;
+    internal.llm = llm;
+    let closeEmbeddingResources;
+    let remoteKeyConfigured = false;
+    if (embedding.canonical.provider === "local") {
+        const embeddingOwner = new LocalEmbeddingProviderOwner(localLlm, {
+            model: embedding.canonical.model,
+            dimension: embedding.canonical.dimension ?? undefined,
+        });
+        internal.embeddingProvider = embeddingOwner.provider;
+        closeEmbeddingResources = () => embeddingOwner.close();
+    }
+    else {
+        const apiKey = config?.models?.embed_api_key?.trim() ||
+            process.env.OPENAI_API_KEY?.trim();
+        remoteKeyConfigured = apiKey != null && apiKey !== "";
+        const configuredModel = embedding.canonical.model;
+        const configuredDimension = embedding.canonical.dimension;
+        const configuredBaseUrl = embedding.canonical.baseUrl;
+        // For non-default endpoints (self-hosted / proxy), API key is optional
+        const canConstruct = remoteKeyConfigured || embedding.credentialAvailable;
+        const provider = canConstruct
+            ? new OpenAIEmbeddingProvider({
+                apiKey: apiKey || undefined,
+                model: configuredModel,
+                dimension: configuredDimension,
+                baseUrl: configuredBaseUrl,
+                requestTimeoutMs: options.remoteRequestTimeoutMs,
+                authorizeRequest: request => {
+                    const activeProvider = internal.embeddingProvider;
+                    if (!activeProvider?.remote) {
+                        throw new EmbeddingConfigError("Remote embedding provider is not active.");
+                    }
+                    const storedIdentity = readStoredEmbeddingIdentity(db);
+                    const requestIdentity = [
+                        storedIdentity,
+                        remoteEmbeddingIdentity(activeProvider, "regex"),
+                        remoteEmbeddingIdentity(activeProvider, "auto"),
+                    ].find(identity => identity?.fingerprint === request.fingerprint);
+                    if (!requestIdentity) {
+                        throw new EmbeddingConfigError("Remote request fingerprint does not match an active embedding identity.");
+                    }
+                    authorizeRemoteEmbeddingRequest(db, requestIdentity, request.purpose, {
+                        lease: request.buildLease,
+                        requestFingerprint: request.fingerprint,
+                    });
+                },
+            })
+            : new UnavailableOpenAIEmbeddingProvider({ model: configuredModel, dimension: configuredDimension, baseUrl: configuredBaseUrl });
+        internal.embeddingProvider = provider;
+        closeEmbeddingResources = async () => {
+            try {
+                await provider.close();
+            }
+            finally {
+                try {
+                    await waitForLLMSessionsToDrain(localLlm);
+                }
+                finally {
+                    await llm.dispose();
+                }
+            }
+        };
+    }
+    internal.authorizeRemoteRequest = (purpose, context) => {
+        const activeProvider = internal.embeddingProvider;
+        if (!activeProvider?.remote) {
+            throw new EmbeddingConfigError("Remote embedding provider is not active.");
+        }
+        authorizeRemoteEmbeddingRequest(db, context.identity ?? remoteEmbeddingIdentity(activeProvider), purpose, { lease: context.lease });
+    };
+    let closePromise;
+    const store = {
+        internal,
+        dbPath: internal.dbPath,
+        // Search
+        search: async (opts) => {
+            if (!opts.query && !opts.queries) {
+                throw new Error("search() requires either 'query' or 'queries'");
+            }
+            // Normalize collection/collections
+            const collections = [
+                ...(typeof opts.collection === "string" ? [opts.collection] : (opts.collection ?? [])),
+                ...(opts.collections ?? []),
+            ];
+            const skipRerank = opts.rerank === false;
+            if (opts.queries) {
+                // Pre-expanded queries — use structuredSearch
+                return structuredSearch(internal, opts.queries, {
+                    collections: collections.length > 0 ? collections : undefined,
+                    limit: opts.limit,
+                    minScore: opts.minScore,
+                    explain: opts.explain,
+                    rerankContext: opts.rerankContext,
+                    candidateLimit: opts.candidateLimit,
+                    skipRerank,
+                    chunkStrategy: opts.chunkStrategy,
+                });
+            }
+            // Simple query string — use hybridQuery (expand + search + rerank)
+            return hybridQuery(internal, opts.query, {
+                collections: collections.length > 0 ? collections : undefined,
+                collection: collections.length === 1 ? collections[0] : (collections.length > 0 ? collections : undefined),
+                limit: opts.limit,
+                minScore: opts.minScore,
+                explain: opts.explain,
+                expansionContext: opts.expansionContext,
+                rerankContext: opts.rerankContext,
+                expansion: opts.expansion,
+                hooks: opts.hooks,
+                candidateLimit: opts.candidateLimit,
+                skipRerank,
+                chunkStrategy: opts.chunkStrategy,
+            });
+        },
+        searchLex: async (q, opts) => internal.searchFTS(q, opts?.limit, opts?.collection),
+        searchVector: async (q, opts) => {
+            const provider = internal.embeddingProvider;
+            return internal.searchVec(q, provider?.model ?? internal.llm?.embedModelName ?? DEFAULT_EMBED_MODEL_URI, opts?.limit, opts?.collection);
+        },
+        expandQuery: async (q, opts) => internal.expandQuery(q, undefined, opts?.expansionContext),
+        get: async (pathOrDocid, opts) => internal.findDocument(pathOrDocid, opts),
+        getDocumentBody: async (pathOrDocid, opts) => {
+            const result = internal.findDocument(pathOrDocid, { includeBody: false });
+            if ("error" in result)
+                return null;
+            return internal.getDocumentBody(result, opts?.fromLine, opts?.maxLines);
+        },
+        multiGet: async (pattern, opts) => internal.findDocuments(pattern, opts),
+        // Collection Management — external config wins when configured; reconciliation mutates SQLite.
+        addCollection: async (name, opts) => {
+            if (externalConfigSource) {
+                collectionsAddCollection(name, opts.path, opts.pattern, opts.ignore, externalConfigSource);
+                syncConfigToDb(db, loadConfig(externalConfigSource));
+            }
+            else {
+                upsertStoreCollection(db, name, { path: opts.path, pattern: opts.pattern, ignore: opts.ignore });
+            }
+        },
+        removeCollection: async (name) => {
+            if (externalConfigSource) {
+                if (!collectionsRemoveCollection(name, externalConfigSource))
+                    return false;
+                syncConfigToDb(db, loadConfig(externalConfigSource));
+            }
+            else {
+                if (!getStoreCollection(db, name))
+                    return false;
+                removeCollectionWithDocuments(db, name);
+            }
+            return true;
+        },
+        renameCollection: async (oldName, newName) => {
+            if (externalConfigSource) {
+                if (!collectionsRenameCollection(oldName, newName, externalConfigSource))
+                    return false;
+                syncConfigToDb(db, loadConfig(externalConfigSource));
+            }
+            else {
+                if (!getStoreCollection(db, oldName))
+                    return false;
+                renameCollectionWithDocuments(db, oldName, newName);
+            }
+            return true;
+        },
+        listCollections: async () => storeListCollections(db),
+        getDefaultCollectionNames: async () => {
+            const collections = storeListCollections(db);
+            return collections.filter(c => c.includeByDefault).map(c => c.name);
+        },
+        // Context Management follows the same external-config-first reconciliation contract.
+        addContext: async (collectionName, pathPrefix, contextText) => {
+            if (externalConfigSource) {
+                if (!collectionsAddContext(collectionName, pathPrefix, contextText, externalConfigSource))
+                    return false;
+                syncConfigToDb(db, loadConfig(externalConfigSource));
+                return true;
+            }
+            return updateStoreContext(db, collectionName, pathPrefix, contextText);
+        },
+        removeContext: async (collectionName, pathPrefix) => {
+            if (externalConfigSource) {
+                if (!collectionsRemoveContext(collectionName, pathPrefix, externalConfigSource))
+                    return false;
+                syncConfigToDb(db, loadConfig(externalConfigSource));
+                return true;
+            }
+            return removeStoreContext(db, collectionName, pathPrefix);
+        },
+        setGlobalContext: async (context) => {
+            if (externalConfigSource) {
+                collectionsSetGlobalContext(context, externalConfigSource);
+                syncConfigToDb(db, loadConfig(externalConfigSource));
+            }
+            else {
+                setStoreGlobalContext(db, context);
+            }
+        },
+        getGlobalContext: async () => getStoreGlobalContext(db),
+        listContexts: async () => getStoreContexts(db),
+        // Indexing — reads collections from SQLite
+        update: async (updateOpts) => {
+            const collections = getStoreCollections(db);
+            const filtered = updateOpts?.collections
+                ? collections.filter(c => updateOpts.collections.includes(c.name))
+                : collections;
+            internal.clearCache();
+            let totalIndexed = 0, totalUpdated = 0, totalUnchanged = 0, totalRemoved = 0, totalSkipped = 0;
+            for (const col of filtered) {
+                const result = await reindexCollection(internal, col.path, col.pattern || "**/*.md", col.name, {
+                    ignorePatterns: col.ignore,
+                    onProgress: updateOpts?.onProgress
+                        ? (info) => updateOpts.onProgress({ collection: col.name, ...info })
+                        : undefined,
+                });
+                totalIndexed += result.indexed;
+                totalUpdated += result.updated;
+                totalUnchanged += result.unchanged;
+                totalRemoved += result.removed;
+                totalSkipped += result.skipped;
+            }
+            await rebuildCjkLexicalIndex(options.dbPath);
+            return {
+                collections: filtered.length,
+                indexed: totalIndexed,
+                updated: totalUpdated,
+                unchanged: totalUnchanged,
+                removed: totalRemoved,
+                skipped: totalSkipped,
+                needsEmbedding: internal.getHashesNeedingEmbedding(),
+            };
+        },
+        embed: async (embedOpts) => {
+            const provider = internal.embeddingProvider;
+            if (provider?.remote) {
+                if (embedOpts?.force && embedOpts.collection) {
+                    throw new EmbeddingConfigError("Remote destructive embedding rebuilds cannot be collection-scoped.");
+                }
+                const identity = remoteEmbeddingIdentity(provider, embedOpts?.chunkStrategy);
+                const pending = getPendingEmbeddingDocsReadOnly(db, embedOpts?.collection, identity.model, identity.fingerprint);
+                if (pending.length === 0 && !embedOpts?.force) {
+                    return { docsProcessed: 0, chunksEmbedded: 0, errors: 0, failures: [], durationMs: 0 };
+                }
+                if (!remoteKeyConfigured && !embedding.credentialAvailable) {
+                    throw new EmbeddingConfigError("OpenAI document embedding is authorized, but OPENAI_API_KEY is not configured.");
+                }
+            }
+            return generateEmbeddings(internal, {
+                force: embedOpts?.force,
+                model: embedOpts?.model,
+                collection: embedOpts?.collection,
+                maxDocsPerBatch: embedOpts?.maxDocsPerBatch,
+                maxBatchBytes: embedOpts?.maxBatchBytes,
+                chunkStrategy: embedOpts?.chunkStrategy,
+                onProgress: embedOpts?.onProgress,
+            });
+        },
+        // Index Health
+        getStatus: async () => {
+            const diagnostics = inspectIndexDiagnostics(db, {
+                fallbackModel: embedding.canonical.model,
+                provider: internal.embeddingProvider,
+                keyConfigured: remoteKeyConfigured,
+            });
+            return {
+                ...getStatusReadOnly(db, diagnostics.embedding.chunks.pendingDocuments),
+                diagnostics,
+            };
+        },
+        getIndexHealth: async () => {
+            const diagnostics = inspectIndexDiagnostics(db, {
+                fallbackModel: embedding.canonical.model,
+                provider: internal.embeddingProvider,
+                keyConfigured: remoteKeyConfigured,
+            });
+            return getIndexHealthReadOnly(db, diagnostics.embedding.chunks.pendingDocuments);
+        },
+        // Lifecycle
+        close: () => {
+            if (!closePromise) {
+                closePromise = (async () => {
+                    try {
+                        await closeEmbeddingResources();
+                    }
+                    finally {
+                        internal.close();
+                    }
+                })();
+            }
+            return closePromise;
+        },
+    };
+    return store;
+}