@clarify-labs/cli 0.10.3 → 0.11.1

package/output/cli/commands/mcp.d.ts

@@ -0,0 +1,27 @@
+import { McpSiteConfig } from '../../plugins/mcp-search/mcp-config.js';
+export type McpCliOptions = {
+    /** Skip the on-disk index cache; always fetch fresh. */
+    noCache?: boolean;
+    /** Optional logger for diagnostics (defaults to stderr). */
+    log?: (message: string) => void;
+};
+/**
+ * Start a local MCP server (stdio transport) that exposes full-text search
+ * over one or more deployed Clarify sites' Orama indices.
+ *
+ * For each site:
+ *   1. Fetch `<site>/mcp.json` to learn the index path, default locale, and
+ *      document count. This is a tiny file and avoids hardcoding any
+ *      assumptions about where the index lives on the site.
+ *   2. Use the descriptor to fetch the binary `.msp` index (cached locally
+ *      keyed by site origin + document count + locale fingerprint) and
+ *      deserialize it into an in-memory Orama db with the correct tokenizer.
+ *
+ * Then register the `search_docs` tool, which searches across all loaded
+ * sites and merges results (each hit annotated with its source site), and
+ * serve over stdio.
+ *
+ * All logs go to stderr - stdout is reserved for the JSON-RPC protocol.
+ */
+export declare function runMcp(siteUrls: string[], options?: McpCliOptions): Promise<void>;
+export declare function fetchMcpConfig(siteUrl: string, fetchImpl: typeof fetch, log: (m: string) => void): Promise<McpSiteConfig>;

package/output/cli/mcp/orama-loader.d.ts

@@ -0,0 +1,56 @@
+import { McpSiteConfig } from '../../plugins/mcp-search/mcp-config.js';
+import { McpSearchResult } from '../../plugins/mcp-search/orama-index.js';
+/**
+ * A ready-to-search Orama index loaded for the MCP server.
+ *
+ * The underlying db is fully in-memory after load, so `search` is synchronous
+ * to the index. `dispose` drops the reference so the buffer can be GC'd; it's
+ * safe to call multiple times.
+ */
+export type LoadedSearchIndex = {
+    /** Search the index. Locale filter is optional. */
+    search: (params: {
+        query: string;
+        locale?: string;
+        limit?: number;
+    }) => McpSearchResult;
+    /** Release the in-memory index. Safe to call multiple times. */
+    dispose: () => void;
+};
+/** Re-export so callers can name the result type without reaching into orama-index. */
+export type { McpSearchResult as SearchResult } from '../../plugins/mcp-search/orama-index.js';
+export type OramaLoaderOptions = {
+    /**
+     * Root directory for the on-disk cache. Defaults to `~/.clarify/mcp-cache`.
+     * Each site gets a subdirectory keyed by a stable hash of its origin + path.
+     */
+    cacheDir?: string;
+    /**
+     * Bypass the cache and always fetch fresh. Implies purging the cached copy
+     * for this site on a successful refresh.
+     */
+    noCache?: boolean;
+    /** Optional fetch override (testing). */
+    fetchImpl?: typeof fetch;
+    /** Optional logger (testing). Defaults to no-op. */
+    log?: (message: string) => void;
+};
+/**
+ * Load the Orama binary index for a remote site and return a ready-to-search
+ * handle.
+ *
+ * The `.msp` index is a single binary file (msgpack-serialized Orama `save()`
+ * output). It is cached on disk so repeated MCP starts don't re-download.
+ * Cache validity is keyed by the document count and locale list recorded in
+ * `mcp.json`; if either changes, the cached index is stale and refetched.
+ *
+ * The caller must supply the same `defaultLocale` recorded in `mcp.json` so
+ * the tokenizer matches the one used at build time. A mismatch would not
+ * corrupt data but would cause CJK query terms to be tokenized incorrectly.
+ */
+export declare function loadSearchIndex(siteUrl: string, config: McpSiteConfig, options?: OramaLoaderOptions): Promise<LoadedSearchIndex>;
+/**
+ * Remove the on-disk cache for a site. Useful for `--no-cache` cleanup or for
+ * tests. No-op if the site has no cache directory.
+ */
+export declare function purgeSiteCache(siteUrl: string, cacheDir?: string): Promise<void>;

package/output/cli/mcp/search-server.d.ts

@@ -0,0 +1,34 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { McpSiteConfig } from '../../plugins/mcp-search/mcp-config.js';
+import { LoadedSearchIndex } from './orama-loader.js';
+export type SearchServerOptions = {
+    /** Optional logger routed to stderr (stdout is the JSON-RPC channel). */
+    log?: (message: string) => void;
+};
+/**
+ * A site loaded for the MCP server: its descriptor, ready-to-search index,
+ * and the origin URL the index was fetched from (used to resolve relative
+ * result paths and to label results by source site).
+ */
+export type LoadedMcpSite = {
+    siteUrl: string;
+    config: McpSiteConfig;
+    index: LoadedSearchIndex;
+};
+/**
+ * Build the MCP server exposing site search over one or more Clarify sites.
+ *
+ * Tools registered:
+ *   - `search_docs`: full-text search across all loaded sites, with each
+ *     result annotated by its source site.
+ *
+ * Results are serialized to a compact, LLM-friendly shape: title, URL, site,
+ * excerpt, score, and locale. Full page content is intentionally omitted to
+ * keep responses within token budgets; clients can fetch the URL themselves.
+ */
+export declare function createMcpSearchServer(sites: LoadedMcpSite[], options?: SearchServerOptions): McpServer;
+/**
+ * Run the search server over stdio until the client disconnects or the process
+ * is signaled. Resolves on clean shutdown.
+ */
+export declare function runMcpSearchServer(sites: LoadedMcpSite[], options?: SearchServerOptions): Promise<void>;

package/output/cli/program.d.ts

@@ -1,2 +1 @@
-export declare function createCli(): import('cac').CAC;
 export declare function main(argv?: string[]): Promise<void>;

package/output/cli.cjs

@@ -1,14 +1,19 @@
 #!/usr/bin/env node
-const require_engine = require("./engine-BDdPszs2.cjs");
+const require_engine = require("./engine-Cbn_coqg.cjs");
 let node_path = require("node:path");
 let node_fs_promises = require("node:fs/promises");
 let node_fs = require("node:fs");
 let vite = require("vite");
+let zod = require("zod");
 let node_url = require("node:url");
+let node_os = require("node:os");
 let node_module = require("node:module");
 let cac = require("cac");
 let _clarify_labs_templates = require("@clarify-labs/templates");
 let node_child_process = require("node:child_process");
+let node_crypto = require("node:crypto");
+let _modelcontextprotocol_sdk_server_mcp_js = require("@modelcontextprotocol/sdk/server/mcp.js");
+let _modelcontextprotocol_sdk_server_stdio_js = require("@modelcontextprotocol/sdk/server/stdio.js");
 //#region source/core/runtime/log.ts
 var LOG_FILE_NAME = ".clarify.log";
 var DEFAULT_LOG_ENTRIES = 50;
@@ -462,6 +467,285 @@ function runInit(options, force, template, install = false) {
 	console.log(`       ${suggestedRun}`);
 }
 //#endregion
+//#region source/cli/mcp/orama-loader.ts
+var DEFAULT_CACHE_DIR = () => (0, node_path.join)((0, node_os.homedir)(), ".clarify", "mcp-cache");
+var FETCH_TIMEOUT_MS = 6e4;
+/** Stable per-site cache key. Two URLs with the same origin+path share a cache. */
+function siteCacheKey(siteUrl) {
+	let url;
+	try {
+		url = new URL(siteUrl);
+	} catch {
+		throw new Error(`[clarify] Invalid site URL: ${siteUrl}`);
+	}
+	const normalized = `${url.origin}${url.pathname.replace(/\/+$/, "")}`;
+	return (0, node_crypto.createHash)("sha256").update(normalized).digest("hex").slice(0, 16);
+}
+async function fetchBufferWithTimeout(url, fetchImpl, log) {
+	const controller = new AbortController();
+	const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+	try {
+		log(`Fetching ${url}`);
+		const res = await fetchImpl(url, {
+			signal: controller.signal,
+			redirect: "follow"
+		});
+		if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText} fetching ${url}`);
+		const buf = await res.arrayBuffer();
+		return new Uint8Array(buf);
+	} finally {
+		clearTimeout(timer);
+	}
+}
+/**
+* Load the Orama binary index for a remote site and return a ready-to-search
+* handle.
+*
+* The `.msp` index is a single binary file (msgpack-serialized Orama `save()`
+* output). It is cached on disk so repeated MCP starts don't re-download.
+* Cache validity is keyed by the document count and locale list recorded in
+* `mcp.json`; if either changes, the cached index is stale and refetched.
+*
+* The caller must supply the same `defaultLocale` recorded in `mcp.json` so
+* the tokenizer matches the one used at build time. A mismatch would not
+* corrupt data but would cause CJK query terms to be tokenized incorrectly.
+*/
+async function loadSearchIndex(siteUrl, config, options = {}) {
+	const log = options.log ?? (() => {});
+	const fetchImpl = options.fetchImpl ?? fetch;
+	const siteCacheDir = (0, node_path.join)(options.cacheDir ?? DEFAULT_CACHE_DIR(), siteCacheKey(siteUrl));
+	const indexCachePath = (0, node_path.join)(siteCacheDir, "mcp-search.msp");
+	const manifestPath = (0, node_path.join)(siteCacheDir, "manifest.json");
+	const base = new URL(siteUrl).href;
+	log(`Site base resolved to ${base}`);
+	const search = config.capabilities.search;
+	if (!search) throw new Error(`[clarify] mcp.json for ${siteUrl} has no search capability.`);
+	const indexUrl = new URL(search.indexPath, base).href;
+	const fingerprint = localeFingerprint(config);
+	let useCache = !options.noCache;
+	if (useCache) try {
+		const manifest = JSON.parse(await (0, node_fs_promises.readFile)(manifestPath, "utf8"));
+		if (manifest.documentCount !== search.documentCount || manifest.fingerprint !== fingerprint) {
+			log("Cache stale (document count or locales changed); refetching");
+			useCache = false;
+		}
+	} catch {
+		useCache = false;
+	}
+	let buffer;
+	if (useCache) try {
+		buffer = await (0, node_fs_promises.readFile)(indexCachePath);
+		log("Using cached mcp-search.msp");
+	} catch {}
+	if (buffer === void 0) {
+		const fetched = await fetchBufferWithTimeout(indexUrl, fetchImpl, log);
+		buffer = fetched;
+		try {
+			await (0, node_fs_promises.mkdir)(siteCacheDir, { recursive: true });
+			await (0, node_fs_promises.writeFile)(indexCachePath, fetched);
+			await (0, node_fs_promises.writeFile)(manifestPath, JSON.stringify({
+				siteUrl: base,
+				fetchedAt: (/* @__PURE__ */ new Date()).toISOString(),
+				documentCount: search.documentCount,
+				fingerprint
+			}, null, 2), "utf8");
+			log(`Cached mcp-search.msp to ${indexCachePath}`);
+		} catch (err) {
+			log(`Warning: failed to cache index: ${err instanceof Error ? err.message : String(err)}`);
+		}
+	}
+	const defaultLocale = search.defaultLocale;
+	const db = require_engine.deserializeSearchIndex(buffer, defaultLocale);
+	let disposed = false;
+	return {
+		search: ({ query, locale, limit }) => {
+			if (disposed) throw new Error("[clarify] Search index already disposed.");
+			return require_engine.searchMcpIndex(db, {
+				query,
+				locale,
+				limit
+			});
+		},
+		dispose: () => {
+			disposed = true;
+		}
+	};
+}
+/**
+* Build a stable fingerprint of the indexed locales so the cache can detect
+* locale-set changes even when the document count happens to stay the same.
+*/
+function localeFingerprint(config) {
+	return (0, node_crypto.createHash)("sha256").update((config.capabilities.search?.locales ?? []).slice().sort().join("|")).digest("hex").slice(0, 16);
+}
+//#endregion
+//#region source/cli/mcp/search-server.ts
+/**
+* Build the MCP server exposing site search over one or more Clarify sites.
+*
+* Tools registered:
+*   - `search_docs`: full-text search across all loaded sites, with each
+*     result annotated by its source site.
+*
+* Results are serialized to a compact, LLM-friendly shape: title, URL, site,
+* excerpt, score, and locale. Full page content is intentionally omitted to
+* keep responses within token budgets; clients can fetch the URL themselves.
+*/
+function createMcpSearchServer(sites, options = {}) {
+	const log = options.log ?? (() => {});
+	const server = new _modelcontextprotocol_sdk_server_mcp_js.McpServer({
+		name: "clarify-search",
+		version: "1.0.0"
+	});
+	if (sites.length === 0) throw new Error("[clarify] createMcpSearchServer requires at least one loaded site.");
+	const totalDocs = sites.reduce((sum, site) => sum + (site.config.capabilities.search?.documentCount ?? 0), 0);
+	const localeSet = /* @__PURE__ */ new Set();
+	for (const site of sites) for (const locale of site.config.capabilities.search?.locales ?? []) localeSet.add(locale);
+	const localeList = [...localeSet];
+	const siteTitles = sites.map((site) => site.config.site.title ?? "site").filter((title, i, arr) => arr.indexOf(title) === i);
+	const siteHint = siteTitles.length === 1 ? `"${siteTitles[0]}"` : `${sites.length} sites (${siteTitles.join(", ")})`;
+	const localeHint = localeList.length > 0 ? ` Available locales: ${localeList.join(", ")}.` : "";
+	const defaultLocale = sites[0].config.capabilities.search?.defaultLocale;
+	const defaultLocaleHint = defaultLocale ? ` Defaults to "${defaultLocale}".` : "";
+	server.registerTool("search_docs", {
+		title: "Search documentation",
+		description: [
+			`Full-text search over ${siteHint} documentation index.`,
+			`Returns ranked results with excerpts, URLs, and source site. The index contains ${totalDocs} pages across ${localeList.length} locale(s).`,
+			localeHint,
+			defaultLocaleHint,
+			"Pass a `locale` filter to restrict results to a specific language."
+		].filter(Boolean).join(" "),
+		inputSchema: {
+			query: zod.z.string().min(1).describe("Search query. CJK text is word-segmented automatically; Latin text is split on whitespace/punctuation."),
+			locale: zod.z.string().optional().describe(`Filter results to a single locale. One of: ${localeList.join(", ")}.`),
+			limit: zod.z.number().int().min(1).max(50).optional().describe("Maximum results to return. Default 10, max 50.")
+		}
+	}, async ({ query, locale, limit }) => {
+		log(`search_docs query=${JSON.stringify(query)} locale=${JSON.stringify(locale)} limit=${limit ?? 10}`);
+		const allHits = [];
+		let errorResult;
+		for (const site of sites) try {
+			const result = site.index.search({
+				query,
+				locale,
+				limit
+			});
+			for (const hit of result.hits) allHits.push({
+				...hit,
+				site: site.siteUrl
+			});
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			log(`search failed for ${site.siteUrl}: ${message}`);
+			errorResult = { text: `Search failed for ${site.siteUrl}: ${message}` };
+		}
+		if (allHits.length === 0 && errorResult) return {
+			isError: true,
+			content: [{
+				type: "text",
+				text: errorResult.text
+			}]
+		};
+		allHits.sort((a, b) => b.score - a.score);
+		const capped = limit ? allHits.slice(0, limit) : allHits.slice(0, 10);
+		return { content: [{
+			type: "text",
+			text: [[
+				`Found ${allHits.length} result(s) across ${totalDocs} indexed pages from ${sites.length} site(s).`,
+				`Showing top ${capped.length}.`,
+				""
+			].join("\n"), ...capped.map((hit, i) => formatResultText(hit, i))].join("\n---\n")
+		}] };
+	});
+	return server;
+}
+/**
+* Run the search server over stdio until the client disconnects or the process
+* is signaled. Resolves on clean shutdown.
+*/
+async function runMcpSearchServer(sites, options = {}) {
+	const log = options.log ?? (() => {});
+	const server = createMcpSearchServer(sites, options);
+	const transport = new _modelcontextprotocol_sdk_server_stdio_js.StdioServerTransport();
+	transport.onclose = () => {
+		log("MCP transport closed; disposing search indices");
+		for (const site of sites) site.index.dispose();
+	};
+	await server.connect(transport);
+	log(`MCP search server listening on stdio (${sites.length} site(s))`);
+}
+function formatResultText(hit, index) {
+	const lines = [
+		`## ${index + 1}. ${hit.title}`,
+		`URL: ${hit.path}`,
+		`Site: ${hit.site}`,
+		`Score: ${hit.score}`
+	];
+	if (hit.locale) lines.push(`Locale: ${hit.locale}`);
+	if (hit.excerpt) lines.push("", hit.excerpt);
+	if (hit.description) lines.push("", `Description: ${hit.description}`);
+	if (hit.keywords?.length > 0) lines.push("", `Keywords: ${hit.keywords.join(", ")}`);
+	return lines.join("\n");
+}
+//#endregion
+//#region source/cli/commands/mcp.ts
+/**
+* Start a local MCP server (stdio transport) that exposes full-text search
+* over one or more deployed Clarify sites' Orama indices.
+*
+* For each site:
+*   1. Fetch `<site>/mcp.json` to learn the index path, default locale, and
+*      document count. This is a tiny file and avoids hardcoding any
+*      assumptions about where the index lives on the site.
+*   2. Use the descriptor to fetch the binary `.msp` index (cached locally
+*      keyed by site origin + document count + locale fingerprint) and
+*      deserialize it into an in-memory Orama db with the correct tokenizer.
+*
+* Then register the `search_docs` tool, which searches across all loaded
+* sites and merges results (each hit annotated with its source site), and
+* serve over stdio.
+*
+* All logs go to stderr - stdout is reserved for the JSON-RPC protocol.
+*/
+async function runMcp(siteUrls, options = {}) {
+	if (siteUrls.length === 0) throw new Error("[clarify] At least one site URL is required. Usage: clarify mcp <site> [site...]");
+	const log = options.log ?? stderrLog;
+	const sites = [];
+	for (const siteUrl of siteUrls) {
+		const config = await fetchMcpConfig(siteUrl, fetch, log);
+		const search = config.capabilities.search;
+		if (!search) throw new Error(`[clarify] mcp.json for ${siteUrl} has no search capability. Run a build with search enabled.`);
+		log(`Loaded mcp.json for ${siteUrl}: ${search.documentCount} doc(s), ${search.locales.length} locale(s)`);
+		const index = await loadSearchIndex(siteUrl, config, {
+			noCache: options.noCache,
+			fetchImpl: fetch,
+			log
+		});
+		sites.push({
+			siteUrl,
+			config,
+			index
+		});
+	}
+	await runMcpSearchServer(sites, { log });
+}
+function stderrLog(message) {
+	process.stderr.write(`[clarify] ${message}\n`);
+}
+async function fetchMcpConfig(siteUrl, fetchImpl, log) {
+	const configUrl = new URL("mcp.json", new URL(siteUrl).href).href;
+	log(`Fetching ${configUrl}`);
+	const res = await fetchImpl(configUrl, { redirect: "follow" });
+	if (!res.ok) throw new Error(`[clarify] Failed to fetch mcp.json from ${configUrl} (HTTP ${res.status}). Ensure the site was built with a recent version of Clarify that emits mcp.json.`);
+	const config = await res.json();
+	if (config.version !== 3) throw new Error(`[clarify] Unsupported mcp.json schema version ${config.version}. Please update the clarify CLI.`);
+	const search = config.capabilities.search;
+	if (!search || search.documentCount === 0) throw new Error(`[clarify] mcp.json at ${configUrl} has no indexed documents. Run a build with search enabled.`);
+	if (!search.indexPath) throw new Error(`[clarify] mcp.json at ${configUrl} is missing capabilities.search.indexPath.`);
+	return config;
+}
+//#endregion
 //#region source/cli/options.ts
 function resolvePort(port) {
 	if (port === void 0) return void 0;
@@ -484,32 +768,29 @@ function resolveCliOptions(options) {
 function withBuildOptions(command) {
 	return command.option("--content <dir>", "Content directory relative to root");
 }
-function resolveInitOptions(directory, options) {
-	return resolveOptions({
-		...options,
-		root: directory
-	});
-}
 function withSharedOptions(command) {
 	return command.option("--root <dir>", "Project root directory").option("--content <dir>", "Content directory relative to root").option("--output <dir>", "Build output directory relative to root");
 }
-function resolveOptions(options) {
-	return resolveCliOptions(options);
-}
 function createCli() {
 	const cli = (0, cac.cac)("clarify");
 	cli.version(require_engine.cliPackageVersion).help();
 	withSharedOptions(cli.command("dev", "Start the local documentation server")).option("--host [host]", "Dev server host").option("--port <port>", "Dev server port").option("--open [path]", "Open the dev server in a browser").action(async (options) => {
-		await runDev(resolveOptions(options));
+		await runDev(resolveCliOptions(options));
 	});
 	withSharedOptions(cli.command("build", "Build the static documentation site")).action(async (options) => {
-		await runBuild(resolveOptions(options));
+		await runBuild(resolveCliOptions(options));
 	});
 	withSharedOptions(cli.command("check", "Validate the documentation project")).option("--strict", "Treat warnings as failures").option("--format <format>", "Output format: text or json").action(async (options) => {
 		await runCheck(options);
 	});
 	withBuildOptions(cli.command("init [directory]", "Create a Clarify project scaffold")).option("--force", "Overwrite files created by init").option("--template <name>", "Template to use: minimal, standard, or complete").option("--install", "Install dependencies after init").action((directory, options) => {
-		runInit(resolveInitOptions(directory, options), options.force === true, options.template, options.install === true);
+		runInit(resolveCliOptions({
+			...options,
+			root: directory
+		}), options.force === true, options.template, options.install === true);
+	});
+	cli.command("mcp <...site>", "Start a local MCP server exposing full-text search over one or more deployed sites").option("--no-cache", "Skip the local search index cache and always fetch fresh").action(async (sites, options) => {
+		await runMcp(sites, { noCache: options.noCache === true });
 	});
 	return cli;
 }