contextqmd 0.1.0

package/README.md

@@ -0,0 +1,87 @@
+# contextqmd-cli
+
+`contextqmd` is a standalone CLI for ContextQMD. It talks to the registry API directly and manages a local docs cache and search index powered by QMD (SQLite + BM25 + vector search).
+
+Requires Node.js >= 22.0.0.
+
+## Commands
+
+### Library Management
+
+```bash
+contextqmd libraries search "inertia rails"
+contextqmd libraries install "inertia rails" --version 3.17.0
+contextqmd libraries list
+contextqmd libraries update inertiajs/inertia-rails
+contextqmd libraries remove inertiajs/inertia-rails --version 3.17.0
+```
+
+### Documentation Search & Retrieval
+
+```bash
+contextqmd docs search "file uploads" --library inertiajs/inertia-rails --version 3.17.0 --mode hybrid
+contextqmd docs get --library inertiajs/inertia-rails --version 3.17.0 --doc-path guides/forms.md --from-line 120 --max-lines 80
+```
+
+Search modes: `fts` (full-text), `vector`, `hybrid`, `auto` (default — heuristic picks based on query shape).
+
+## Global Flags
+
+| Flag | Description |
+|------|-------------|
+| `--json` | Structured JSON output instead of human-readable text |
+| `--registry <url>` | Registry URL (default: `https://contextqmd.com`) |
+| `--token <token>` | Authentication token |
+| `--cache-dir <path>` | Local cache directory (default: `~/.cache/contextqmd`) |
+
+## Architecture
+
+```
+src/
+  index.ts              CLI entrypoint (Commander.js argument parsing, IO abstraction)
+  lib/
+    service.ts          Business logic — install, search, update, remove orchestration
+    registry-client.ts  HTTP client for the ContextQMD registry API (uses built-in fetch)
+    local-cache.ts      Filesystem cache with atomic installs and backup/restore
+    doc-indexer.ts      Search index wrapping @tobilu/qmd (SQLite + BM25 + vector)
+    config.ts           Config loader (~/.config/contextqmd/config.json with defaults)
+    types.ts            Shared API contract types (Library, Version, Manifest, etc.)
+```
+
+**Install pipeline:** Resolves library via registry → fetches manifest → tries bundle download (tar.gz with SHA-256 verification) → falls back to page-by-page API download → indexes pages for local search.
+
+**Search pipeline:** Classifies query (code patterns → FTS, conceptual questions → vector/hybrid) → searches local QMD index → falls back to FTS if vector/hybrid returns empty.
+
+## Configuration
+
+Optional config file at `~/.config/contextqmd/config.json`:
+
+```json
+{
+  "registry_url": "https://contextqmd.com",
+  "default_install_mode": "slim",
+  "preferred_search_mode": "auto",
+  "local_cache_dir": "~/.cache/contextqmd",
+  "allow_origin_fetch": true,
+  "allow_remote_bundles": true,
+  "verify_registry_signatures": true
+}
+```
+
+All fields are optional — defaults are used for anything not specified.
+
+## Development
+
+```bash
+npm install
+npm run build       # compile TypeScript to dist/
+npm run check       # type-check without emitting
+npm test            # run tests (vitest)
+npm link            # symlink the contextqmd binary globally
+```
+
+## Dependencies
+
+- **[commander](https://github.com/tj/commander.js)** — CLI argument parsing
+- **[@tobilu/qmd](https://github.com/tobilu/qmd)** — local search index (SQLite + BM25 + vector + LLM reranking)
+- **[zod](https://github.com/colinhacks/zod)** — schema validation

package/dist/index.js

@@ -0,0 +1,199 @@
+#!/usr/bin/env node
+import { realpathSync } from "node:fs";
+import { join, resolve as resolvePath } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Command, CommanderError } from "commander";
+import { loadConfig } from "./lib/config.js";
+import { DocIndexer } from "./lib/doc-indexer.js";
+import { LocalCache } from "./lib/local-cache.js";
+import { RegistryClient } from "./lib/registry-client.js";
+import { getDoc, installDocs, listInstalledDocs, removeDocs, searchDocs, searchLibraries, updateDocs, } from "./lib/service.js";
+function defaultIo() {
+    return {
+        env: process.env,
+        writeStdout: chunk => process.stdout.write(chunk),
+        writeStderr: chunk => process.stderr.write(chunk),
+    };
+}
+function resolveEntrypointPath(path) {
+    try {
+        return realpathSync(path);
+    }
+    catch {
+        return resolvePath(path);
+    }
+}
+export function isCliEntrypoint(argvPath = process.argv[1], moduleUrl = import.meta.url) {
+    if (!argvPath)
+        return false;
+    return resolveEntrypointPath(argvPath) === resolveEntrypointPath(fileURLToPath(moduleUrl));
+}
+async function withDeps(env, options, io, run) {
+    const config = loadConfig();
+    const registryUrl = options.registry ?? config.registry_url;
+    const token = options.token ?? env.CONTEXTQMD_API_TOKEN;
+    const cacheDir = options.cacheDir ?? config.local_cache_dir;
+    const registryClient = new RegistryClient(registryUrl, token);
+    const cache = new LocalCache(cacheDir);
+    const indexer = new DocIndexer(join(cacheDir, "index.sqlite"), cache);
+    try {
+        return await run({
+            registryClient,
+            cache,
+            indexer,
+            reportProgress: message => io.writeStderr(`${message}\n`),
+        });
+    }
+    finally {
+        await indexer.close();
+    }
+}
+function renderResult(result, options, io) {
+    const output = options.json
+        ? JSON.stringify(result.structuredContent ?? {}, null, 2)
+        : result.text;
+    const normalized = output.endsWith("\n") ? output : `${output}\n`;
+    if (result.isError) {
+        io.writeStderr(normalized);
+        return 1;
+    }
+    io.writeStdout(normalized);
+    return 0;
+}
+function addGlobalOptions(command) {
+    return command
+        .option("--json", "Print structured output as JSON")
+        .option("--registry <url>", "Registry URL override")
+        .option("--token <token>", "Registry token override")
+        .option("--cache-dir <path>", "Local cache directory override");
+}
+function globalOptionsFor(command) {
+    return command.optsWithGlobals();
+}
+function createProgram(io, onExitCode) {
+    const program = addGlobalOptions(new Command());
+    program
+        .name("contextqmd")
+        .description("CLI for ContextQMD")
+        .exitOverride()
+        .configureOutput({
+        writeOut: chunk => io.writeStdout(chunk),
+        writeErr: chunk => io.writeStderr(chunk),
+    });
+    const libraries = program.command("libraries").description("Library package workflows");
+    libraries
+        .command("search")
+        .argument("<query>", "Search query")
+        .option("--limit <count>", "Maximum libraries to return")
+        .action(async function (query, options) {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => searchLibraries(deps, { query, ...(options.limit ? { limit: Number(options.limit) } : {}) }));
+        onExitCode(renderResult(result, global, io));
+    });
+    libraries
+        .command("install")
+        .argument("<library>", "Library query or slug")
+        .option("--version <version>", "Version to install")
+        .action(async function (library, options) {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => installDocs(deps, { library, version: options.version }));
+        onExitCode(renderResult(result, global, io));
+    });
+    libraries
+        .command("list")
+        .action(async function () {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => listInstalledDocs(deps));
+        onExitCode(renderResult(result, global, io));
+    });
+    libraries
+        .command("update")
+        .argument("[library]", "Optional library slug (namespace/name)")
+        .action(async function (library) {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => updateDocs(deps, library ? { library } : {}));
+        onExitCode(renderResult(result, global, io));
+    });
+    libraries
+        .command("remove")
+        .argument("<library>", "Library slug (namespace/name)")
+        .option("--version <version>", "Installed version to remove")
+        .action(async function (library, options) {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => removeDocs(deps, { library, version: options.version }));
+        onExitCode(renderResult(result, global, io));
+    });
+    const docs = program.command("docs").description("Installed documentation operations");
+    docs
+        .command("search")
+        .argument("<query>", "Search query")
+        .option("--library <library>", "Filter to a specific library")
+        .option("--version <version>", "Filter to a specific version")
+        .option("--mode <mode>", "Search mode")
+        .option("--max-results <count>", "Maximum results to return")
+        .action(async function (query, options) {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => searchDocs(deps, {
+            query,
+            ...(options.library ? { library: options.library } : {}),
+            ...(options.version ? { version: options.version } : {}),
+            ...(options.mode ? { mode: options.mode } : {}),
+            ...(options.maxResults ? { max_results: Number(options.maxResults) } : {}),
+        }));
+        onExitCode(renderResult(result, global, io));
+    });
+    docs
+        .command("get")
+        .requiredOption("--library <library>", "Library slug (namespace/name)")
+        .requiredOption("--version <version>", "Installed version")
+        .option("--doc-path <docPath>", "Canonical document path")
+        .option("--page-uid <pageUid>", "Page UID fallback")
+        .option("--from-line <line>", "Start line")
+        .option("--max-lines <count>", "Maximum lines")
+        .option("--around-line <line>", "Anchor line")
+        .option("--before <count>", "Lines before around-line")
+        .option("--after <count>", "Lines after around-line")
+        .option("--line-numbers", "Include line numbers")
+        .action(async function (options) {
+        const global = globalOptionsFor(this);
+        const result = await withDeps(io.env, global, io, deps => getDoc(deps, {
+            library: options.library,
+            version: options.version,
+            ...(options.docPath ? { doc_path: options.docPath } : {}),
+            ...(options.pageUid ? { page_uid: options.pageUid } : {}),
+            ...(options.fromLine ? { from_line: Number(options.fromLine) } : {}),
+            ...(options.maxLines ? { max_lines: Number(options.maxLines) } : {}),
+            ...(options.aroundLine ? { around_line: Number(options.aroundLine) } : {}),
+            ...(options.before ? { before: Number(options.before) } : {}),
+            ...(options.after ? { after: Number(options.after) } : {}),
+            ...(options.lineNumbers ? { line_numbers: true } : {}),
+        }));
+        onExitCode(renderResult(result, global, io));
+    });
+    return program;
+}
+export async function runCli(argv, inputIo = {}) {
+    const io = { ...defaultIo(), ...inputIo };
+    let exitCode = 0;
+    const program = createProgram(io, code => {
+        exitCode = code;
+    });
+    try {
+        await program.parseAsync(argv, { from: "user" });
+        return exitCode;
+    }
+    catch (error) {
+        if (error instanceof CommanderError) {
+            if (error.code !== "commander.helpDisplayed") {
+                io.writeStderr(`${error.message}\n`);
+            }
+            return error.exitCode;
+        }
+        io.writeStderr(`${error.message}\n`);
+        return 1;
+    }
+}
+if (isCliEntrypoint()) {
+    const exitCode = await runCli(process.argv.slice(2));
+    process.exit(exitCode);
+}

package/dist/lib/config.js

@@ -0,0 +1,22 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+const DEFAULT_CONFIG = {
+    registry_url: "https://contextqmd.com",
+    fallback_registries: [],
+    allow_origin_fetch: true,
+    allow_remote_bundles: true,
+    allow_public_fallback: false,
+    verify_registry_signatures: true,
+    default_install_mode: "slim",
+    preferred_search_mode: "auto",
+    local_cache_dir: join(homedir(), ".cache", "contextqmd"),
+};
+export function loadConfig(configPath) {
+    const path = configPath ?? join(homedir(), ".config", "contextqmd", "config.json");
+    if (!existsSync(path)) {
+        return { ...DEFAULT_CONFIG };
+    }
+    const raw = JSON.parse(readFileSync(path, "utf-8"));
+    return { ...DEFAULT_CONFIG, ...raw };
+}

package/dist/lib/doc-indexer.js

@@ -0,0 +1,270 @@
+import { createHash } from "node:crypto";
+import { createStore, extractSnippet, } from "@tobilu/qmd";
+import { normalizeDocPath } from "./local-cache.js";
+export function classifyQuery(query) {
+    const q = query.trim();
+    if (q.split(/\s+/).length <= 2 && !q.includes("?")) {
+        return "fts";
+    }
+    const codePatterns = [
+        /[a-z][A-Z]/,
+        /[a-z]_[a-z]/,
+        /\w+\.\w+\.\w+/,
+        /`[^`]+`/,
+        /^[A-Z_]{3,}$/,
+        /\berr(or)?[:\s]+/i,
+        /\d+\.\d+\.\d+/,
+        /[{}()\[\]<>]/,
+        /^(get|set|use|create|delete|update)\w+/i,
+        /\w+::\w+/,
+        /\w+\/\w+/,
+    ];
+    if (codePatterns.some(pattern => pattern.test(q))) {
+        return "fts";
+    }
+    const conceptualPatterns = [
+        /^(how|what|why|when|where|can|should|is it|does)\b/i,
+        /\b(best practice|pattern|approach|strategy|concept|overview|guide)\b/i,
+        /\b(difference between|compare|vs\.?|versus)\b/i,
+        /\b(explain|understand|learn|tutorial)\b/i,
+    ];
+    if (conceptualPatterns.some(pattern => pattern.test(q))) {
+        if (q.split(/\s+/).length >= 6) {
+            return "hybrid";
+        }
+        return "vector";
+    }
+    if (q.split(/\s+/).length >= 8) {
+        return "hybrid";
+    }
+    return "fts";
+}
+export class DocIndexer {
+    storePromise;
+    cache;
+    constructor(dbPath, cache) {
+        this.storePromise = createStore({
+            dbPath,
+            config: { collections: {} },
+        });
+        this.cache = cache;
+    }
+    async close() {
+        const store = await this.storePromise;
+        await store.close();
+    }
+    async getStore() {
+        return (await this.storePromise).internal;
+    }
+    static collectionName(namespace, name, version) {
+        return `${namespace}__${name}__${version}`;
+    }
+    static parseCollectionName(collectionName) {
+        const parts = collectionName.split("__");
+        if (parts.length !== 3)
+            return null;
+        return { namespace: parts[0], name: parts[1], version: parts[2] };
+    }
+    async indexLibraryVersion(namespace, name, version) {
+        const store = await this.getStore();
+        const collectionName = DocIndexer.collectionName(namespace, name, version);
+        const pageUids = this.cache.listPageUids(namespace, name, version);
+        const desiredPaths = new Set();
+        let indexed = 0;
+        for (const pageUid of pageUids) {
+            const content = this.cache.readPage(namespace, name, version, pageUid);
+            if (!content)
+                continue;
+            const page = this.cache.findPageByUid(namespace, name, version, pageUid);
+            const docPath = page ? normalizeDocPath(page.path) : `${pageUid}.md`;
+            desiredPaths.add(docPath);
+            const title = page?.title ?? extractTitle(content, pageUid);
+            const hash = await hashContent(content);
+            const now = new Date().toISOString();
+            const legacyPath = `${pageUid}.md`;
+            const existing = store.findActiveDocument(collectionName, docPath);
+            const legacyExisting = docPath === legacyPath
+                ? existing
+                : store.findActiveDocument(collectionName, legacyPath);
+            if (existing && existing.hash === hash)
+                continue;
+            if (!existing && legacyExisting && legacyExisting.hash === hash) {
+                store.insertDocument(collectionName, docPath, title, legacyExisting.hash, now, now);
+                if (legacyPath !== docPath) {
+                    store.deactivateDocument(collectionName, legacyPath);
+                }
+                indexed++;
+                continue;
+            }
+            store.insertContent(hash, content, now);
+            if (existing) {
+                store.updateDocument(existing.id, title, hash, now);
+            }
+            else {
+                store.insertDocument(collectionName, docPath, title, hash, now, now);
+                if (legacyExisting && legacyPath !== docPath) {
+                    store.deactivateDocument(collectionName, legacyPath);
+                }
+            }
+            indexed++;
+        }
+        for (const activePath of store.getActiveDocumentPaths(collectionName)) {
+            if (!desiredPaths.has(activePath)) {
+                store.deactivateDocument(collectionName, activePath);
+            }
+        }
+        return indexed;
+    }
+    async removeLibraryVersion(namespace, name, version) {
+        const store = await this.getStore();
+        const collectionName = DocIndexer.collectionName(namespace, name, version);
+        const paths = store.getActiveDocumentPaths(collectionName);
+        for (const path of paths) {
+            store.deactivateDocument(collectionName, path);
+        }
+    }
+    async search(query, options = {}) {
+        if (options.version && !options.library) {
+            console.warn(`[contextqmd] version filter "${options.version}" specified without library — version will be applied as a post-query filter`);
+        }
+        const requestedMode = options.mode ?? "auto";
+        const effectiveMode = requestedMode === "auto" ? classifyQuery(query) : requestedMode;
+        const searchMode = !options.library && effectiveMode !== "fts" ? "fts" : effectiveMode;
+        if (searchMode === "vector") {
+            const results = await this.searchVector(query, options);
+            if (results.length > 0)
+                return results;
+            return (await this.searchFTS(query, options)).map(result => ({ ...result, searchMode: "fts" }));
+        }
+        if (searchMode === "hybrid") {
+            const results = await this.searchHybrid(query, options);
+            if (results.length > 0)
+                return results;
+            return (await this.searchFTS(query, options)).map(result => ({ ...result, searchMode: "fts" }));
+        }
+        return this.searchFTS(query, options);
+    }
+    async searchFTS(query, options = {}) {
+        const store = await this.getStore();
+        const limit = options.maxResults ?? 10;
+        let collectionFilter;
+        if (options.library && options.version) {
+            const [namespace, name] = options.library.split("/");
+            collectionFilter = DocIndexer.collectionName(namespace, name, options.version);
+        }
+        const results = store.searchFTS(query, limit * 2, collectionFilter);
+        return this.mapAnyResults(results, query, options, "fts").slice(0, limit);
+    }
+    async searchVector(query, options = {}) {
+        const store = await this.storePromise;
+        const limit = options.maxResults ?? 10;
+        let collectionFilter;
+        if (options.library && options.version) {
+            const [namespace, name] = options.library.split("/");
+            collectionFilter = DocIndexer.collectionName(namespace, name, options.version);
+        }
+        try {
+            const results = await withTimeout(store.searchVector(query, {
+                collection: collectionFilter,
+                limit: limit * 2,
+            }), 10_000);
+            return this.mapAnyResults(results, query, options, "vector").slice(0, limit);
+        }
+        catch {
+            return [];
+        }
+    }
+    async searchHybrid(query, options = {}) {
+        const store = await this.storePromise;
+        const limit = options.maxResults ?? 10;
+        let collectionFilter;
+        if (options.library && options.version) {
+            const [namespace, name] = options.library.split("/");
+            collectionFilter = DocIndexer.collectionName(namespace, name, options.version);
+        }
+        try {
+            const results = await withTimeout(store.search({ query, collection: collectionFilter, limit }), 10_000);
+            return this.mapAnyResults(results, query, options, "hybrid", result => this.extractSnippetInfo(result.body ?? "", query, result.bestChunkPos)).slice(0, limit);
+        }
+        catch {
+            return [];
+        }
+    }
+    mapAnyResults(results, query, options, mode, snippetFn = result => this.extractSnippetInfo(result.body ?? "", query, result.chunkPos)) {
+        return results
+            .map(result => {
+            const collectionName = result.collectionName ?? result.displayPath.split("/")[0];
+            const parsed = DocIndexer.parseCollectionName(collectionName);
+            const library = parsed ? `${parsed.namespace}/${parsed.name}` : collectionName;
+            let docPath = result.displayPath;
+            if (docPath.startsWith(`${collectionName}/`)) {
+                docPath = docPath.slice(collectionName.length + 1);
+            }
+            docPath = normalizeDocPath(docPath);
+            const page = parsed
+                ? this.cache.findPageByPath(parsed.namespace, parsed.name, parsed.version, docPath)
+                : null;
+            const pageUid = page?.page_uid ?? docPath.replace(/\.md$/, "");
+            const contentMd = parsed
+                ? (this.cache.readPage(parsed.namespace, parsed.name, parsed.version, pageUid) ?? result.body ?? "")
+                : (result.body ?? "");
+            const snippet = snippetFn(result);
+            return {
+                pageUid,
+                title: page?.title ?? result.title,
+                path: result.displayPath,
+                docPath,
+                contentMd,
+                score: result.score,
+                snippet: snippet.snippet,
+                library,
+                searchMode: mode,
+                _version: parsed?.version,
+                lineStart: snippet.lineStart,
+                lineEnd: snippet.lineEnd,
+                url: page?.url,
+            };
+        })
+            .filter(result => {
+            if (options.library && result.library !== options.library)
+                return false;
+            if (options.version && result._version !== options.version)
+                return false;
+            return true;
+        })
+            .map(({ _version, ...result }) => ({ ...result, version: _version ?? "unknown" }));
+    }
+    extractSnippetInfo(body, query, chunkPos) {
+        if (!body.trim()) {
+            return { snippet: "", lineStart: null, lineEnd: null };
+        }
+        const { snippet, line, snippetLines } = extractSnippet(body, query, 500, chunkPos);
+        if (!line || line < 1) {
+            return { snippet, lineStart: null, lineEnd: null };
+        }
+        return {
+            snippet,
+            lineStart: line,
+            lineEnd: line + Math.max(snippetLines - 1, 0),
+        };
+    }
+}
+async function hashContent(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+function withTimeout(promise, ms) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`Timed out after ${ms}ms`)), ms);
+        promise.then(value => {
+            clearTimeout(timer);
+            resolve(value);
+        }, error => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+function extractTitle(content, fallback) {
+    const match = content.match(/^#\s+(.+)$/m);
+    return match ? match[1].trim() : fallback;
+}