@clarify-labs/cli 0.11.0 → 0.11.2

package/output/{engine-DfF5RJjC.js → engine-DI8Nd3v0.js}

@@ -18,6 +18,9 @@ import { createReadStream, existsSync, mkdirSync, mkdtempSync, readFileSync, rea
 import { build, loadConfigFromFile } from "vite";
 import { z } from "zod";
 import { stringify } from "yaml";
+import { createHash } from "node:crypto";
+import { decode, encode } from "@msgpack/msgpack";
+import { create, insertMultiple, load, save, search } from "@orama/orama";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import SwaggerParser from "@apidevtools/swagger-parser";
 import { tmpdir } from "node:os";
@@ -968,7 +971,7 @@ function featureSchema(shape, defaults) {
 	} : value);
 }
 var clarifyFeaturesConfigSchema = z.object({
-	search: featureSchema({ provider: z.literal("pagefind").default("pagefind") }, { provider: "pagefind" }),
+	search: featureSchema({ mcp: z.boolean().default(true) }, { mcp: true }),
 	repository: featureSchema(clarifyRepositoryConfigSchema.shape, {}),
 	themeEditor: featureSchema({}, {}),
 	openapi: featureSchema({ playground: z.boolean().default(true) }, { playground: true })
@@ -1554,7 +1557,7 @@ function createViteAdapter(engine) {
 }
 var cliPackageVersion = {
 	name: "@clarify-labs/cli",
-	version: "0.11.0",
+	version: "0.11.2",
 	license: "AGPL-3.0-only",
 	type: "module",
 	repository: {
@@ -1595,6 +1598,9 @@ var cliPackageVersion = {
 		"@clarify-labs/templates": "workspace:*",
 		"@mdx-js/mdx": "^3.1.1",
 		"@mdx-js/rollup": "^3.1.1",
+		"@modelcontextprotocol/sdk": "^1.29.0",
+		"@msgpack/msgpack": "^3.1.3",
+		"@orama/orama": "^3.1.18",
 		"@shikijs/rehype": "^4.3.0",
 		"@tailwindcss/vite": "^4.3.2",
 		"@vitejs/plugin-react": "^6.0.3",
@@ -1613,7 +1619,6 @@ var cliPackageVersion = {
 		"rehype-raw": "^7.0.0",
 		"rehype-slug": "^6.0.0",
 		"remark": "^15.0.1",
-		"remark-gfm": "^4.0.1",
 		"shiki": "4.3.0",
 		"unist-util-visit": "^5.1.0",
 		"vite": "^8.1.4",
@@ -1894,6 +1899,351 @@ function createHtmlShellPlugin() {
 	};
 }
 //#endregion
+//#region source/plugins/mcp-search/mcp-config.ts
+/**
+* Normalize a route prefix into a root-relative path segment.
+*
+* `'/'` -> `''`, `'/docs'` -> `'/docs'`, `'docs/'` -> `'/docs'`.
+*/
+function normalizePrefix(routePrefix) {
+	if (!routePrefix || routePrefix === "/") return "";
+	return `/${routePrefix.replace(/^\/+|\/+$/g, "")}`;
+}
+/**
+* Assemble the MCP site config from build-time metadata.
+*
+* Unlike the previous pagefind-based descriptor, this no longer reads an
+* entry JSON file - the Orama index is a single binary blob, so the caller
+* supplies the document count and locale list directly from the build step
+* that produced the index.
+*/
+function createMcpSiteConfig(projectConfig, options) {
+	const prefix = normalizePrefix(projectConfig.routePrefix);
+	return {
+		version: 3,
+		site: {
+			title: projectConfig.title,
+			description: projectConfig.description,
+			url: projectConfig.siteUrl,
+			routePrefix: projectConfig.routePrefix ?? "/"
+		},
+		capabilities: { search: {
+			type: "search",
+			indexPath: `${prefix}/mcp-search.msp`,
+			indexHash: options.indexHash,
+			defaultLocale: projectConfig.locales?.default,
+			documentCount: options.documentCount,
+			locales: options.locales
+		} }
+	};
+}
+//#endregion
+//#region source/plugins/mcp-search/orama-index.ts
+/**
+* Orama schema for the MCP search index.
+*
+* - `path`, `title`, `description`, `content` are full-text searchable strings.
+* - `locale` is an `enum` so we can use `where: { locale: { eq } }` filters at
+*   query time. (`string` fields cannot be filtered in Orama.)
+* - `keywords` is a `string[]` also indexed for full-text matching.
+*/
+var mcpSearchSchema = {
+	path: "string",
+	title: "string",
+	description: "string",
+	locale: "enum",
+	content: "string",
+	keywords: "string[]"
+};
+/**
+* Intl.Segmenter cache keyed by BCP 47 tag. Segmenter construction is not
+* cheap, and the MCP server issues many queries against the same locale, so
+* we reuse instances. The cache is module-scoped: build and runtime share it
+* in the same process, and separate processes get their own - which is fine.
+*/
+var segmenterCache = /* @__PURE__ */ new Map();
+function getSegmenter(locale) {
+	let seg = segmenterCache.get(locale);
+	if (!seg) {
+		seg = new Intl.Segmenter(locale, { granularity: "word" });
+		segmenterCache.set(locale, seg);
+	}
+	return seg;
+}
+/**
+* Map a Clarify locale code (e.g. `zh-CN`, `en-US`, `ja-JP`) to a BCP 47 tag
+* suitable for `Intl.Segmenter`. Returns `null` for locales we intentionally
+* want to tokenize with the Latin/whitespace fallback (faster, and avoids
+* forcing CJK-style word segmentation on Latin scripts).
+*/
+function segmenterLocale(locale) {
+	if (!locale) return null;
+	const lower = locale.toLowerCase();
+	if (lower.startsWith("zh")) return "zh-CN";
+	if (lower.startsWith("ja")) return "ja-JP";
+	if (lower.startsWith("ko")) return "ko-KR";
+	return null;
+}
+/**
+* Tokenize text for Orama indexing and querying.
+*
+* CJK locales use `Intl.Segmenter` (word granularity), which splits
+* spaceless scripts into words and individual characters. Latin/whitespace
+* locales fall back to a Unicode-aware split on spaces and punctuation.
+*
+* All tokens are lowercased so queries match regardless of case.
+*/
+function tokenizeForSearch(text, locale) {
+	if (!text) return [];
+	const segLocale = segmenterLocale(locale);
+	if (segLocale) {
+		const seg = getSegmenter(segLocale);
+		return Array.from(seg.segment(text)).filter((s) => s.isWordLike).map((s) => s.segment.toLowerCase());
+	}
+	return text.toLowerCase().split(/[\s\p{P}]+/u).filter(Boolean);
+}
+/**
+* Build an Orama tokenizer component bound to a specific default locale.
+*
+* The tokenizer receives the raw field value and the per-field language hint
+* from Orama (which we don't set per-field, so it falls back to the default
+* language). We resolve the segmenter locale from the default locale at
+* construction time so the same tokenizer handles all documents in a
+* predominantly single-locale index.
+*
+* For multi-locale indexes, the tokenizer is also used at query time. Since
+* the query locale is not known to Orama's tokenizer callback, we use the
+* index's default locale - this is acceptable because:
+*   - CJK queries get CJK segmentation (correct)
+*   - Latin queries against a CJK-default index still work because Latin
+*     text has no word boundaries for CJK segmenters to mis-split; the
+*     segmenter passes through Latin words as-is.
+*/
+function createMcpTokenizer(defaultLocale) {
+	return {
+		language: defaultLocale ?? "en",
+		normalizationCache: /* @__PURE__ */ new Map(),
+		tokenize(raw, language) {
+			return tokenizeForSearch(typeof raw === "string" ? raw : String(raw ?? ""), language ?? defaultLocale);
+		}
+	};
+}
+/**
+* Compose the searchable text for a route. Shared between the Pagefind
+* (browser) and Orama (MCP) indexing paths so both backends see the same
+* field order and content.
+*/
+function routeSearchContent(route) {
+	return [
+		route.meta.title,
+		route.meta.description,
+		route.meta.keywords?.join(" "),
+		route.meta.sections?.map((section) => section.title).join(" "),
+		route.source.content
+	].filter(Boolean).join("\n\n");
+}
+/**
+* Convert a `ContentRoute` into the document shape stored in the Orama index.
+*/
+function routeToSearchDocument(route, defaultLocale) {
+	if (route.isBareAlias) return null;
+	const content = routeSearchContent(route);
+	if (!content.trim()) return null;
+	return {
+		path: route.path,
+		title: route.meta.title ?? "",
+		description: route.meta.description ?? "",
+		locale: route.locale ?? defaultLocale ?? "en",
+		content,
+		keywords: route.meta.keywords ?? []
+	};
+}
+/**
+* Create an Orama index pre-configured with the MCP schema and CJK-aware
+* tokenizer. Both the build step and the runtime loader use this so the
+* tokenizer is guaranteed to match between indexing and querying.
+*
+* Annotated as `AnyOrama` (Orama's exported base type) rather than inferred
+* via `ReturnType<typeof create>` because the inference would pull in
+* Orama's internal component types (`DocumentsStore`, `Index`, `Sorter`)
+* which are not exported and break `.d.ts` generation (TS2883).
+*/
+function createMcpSearchIndex(defaultLocale) {
+	return create({
+		schema: mcpSearchSchema,
+		components: { tokenizer: createMcpTokenizer(defaultLocale) }
+	});
+}
+/**
+* Insert search documents into an Orama index.
+*/
+function indexSearchDocuments(db, documents) {
+	if (documents.length === 0) return;
+	insertMultiple(db, documents);
+}
+/**
+* Build a ready-to-search Orama index from content routes.
+*
+* Filters out bare aliases and empty routes. Returns the populated db.
+*/
+function buildSearchIndex(routes, defaultLocale) {
+	const db = createMcpSearchIndex(defaultLocale);
+	const documents = [];
+	for (const route of routes) {
+		const doc = routeToSearchDocument(route, defaultLocale);
+		if (doc) documents.push(doc);
+	}
+	indexSearchDocuments(db, documents);
+	return {
+		db,
+		documentCount: documents.length
+	};
+}
+/**
+* Collect the distinct locale codes that will be present in the Orama index,
+* in stable insertion order. Bare aliases are skipped (they duplicate the
+* default-locale route). Locales with no code fall back to the project
+* default so single-locale sites still surface one locale.
+*/
+function collectIndexedLocales(routes, defaultLocale) {
+	const seen = /* @__PURE__ */ new Set();
+	const locales = [];
+	for (const route of routes) {
+		if (route.isBareAlias) continue;
+		const code = route.locale ?? defaultLocale ?? "en";
+		if (!seen.has(code)) {
+			seen.add(code);
+			locales.push(code);
+		}
+	}
+	return locales;
+}
+/**
+* Search an Orama MCP index.
+*
+* Applies an optional locale filter (via the `enum` field) and returns a
+* compact, LLM-friendly hit shape. Excerpts are derived from the matched
+* content: Orama does not return highlighted excerpts, so we synthesize a
+* short window around the first query term occurrence.
+*/
+function searchMcpIndex(db, params) {
+	const limit = Math.max(1, Math.min(50, params.limit ?? 10));
+	const result = search(db, {
+		term: params.query,
+		limit,
+		where: params.locale ? { locale: { eq: params.locale } } : void 0
+	});
+	return {
+		hits: result.hits.map((hit) => {
+			const doc = hit.document;
+			return {
+				path: doc.path,
+				title: doc.title || guessTitleFromPath(doc.path),
+				description: doc.description,
+				locale: doc.locale,
+				score: hit.score,
+				excerpt: makeExcerpt(doc.content, params.query),
+				keywords: doc.keywords
+			};
+		}),
+		count: result.count
+	};
+}
+/**
+* Build a short excerpt around the first occurrence of any query term in the
+* content. Falls back to the beginning of the content if no term is found.
+*/
+function makeExcerpt(content, query, maxLen = 240) {
+	if (!content) return "";
+	const terms = query.toLowerCase().split(/[\s\p{P}]+/u).filter(Boolean);
+	const lower = content.toLowerCase();
+	let pos = -1;
+	for (const term of terms) {
+		const i = lower.indexOf(term);
+		if (i >= 0 && (pos < 0 || i < pos)) pos = i;
+	}
+	if (pos < 0) pos = 0;
+	const half = Math.floor(maxLen / 2);
+	const start = Math.max(0, pos - half);
+	const end = Math.min(content.length, start + maxLen);
+	const prefix = start > 0 ? "…" : "";
+	const suffix = end < content.length ? "…" : "";
+	return `${prefix}${content.slice(start, end).trim()}${suffix}`;
+}
+function guessTitleFromPath(path) {
+	if (!path) return "(untitled)";
+	const segments = path.split("/").filter(Boolean);
+	if (segments.length === 0) return "Home";
+	return segments[segments.length - 1].replace(/\.(md|mdx|html)$/, "").replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+}
+/**
+* Serialize an Orama index to a msgpack binary buffer.
+*
+* Uses `save()` to extract the raw index data, then msgpack-encodes it for a
+* compact binary representation. The same schema + tokenizer must be supplied
+* when loading so query-time tokenization matches indexing-time tokenization.
+*/
+function serializeSearchIndex(db) {
+	return encode(save(db));
+}
+/**
+* Deserialize a msgpack binary buffer into a ready-to-search Orama index.
+*
+* The caller must supply the same `defaultLocale` used when the index was
+* built, so the tokenizer matches. A mismatch would not corrupt data but
+* would cause CJK query terms to be tokenized incorrectly (e.g. "文档" left
+* as a single token instead of split into "文" + "档").
+*/
+function deserializeSearchIndex(buffer, defaultLocale) {
+	const raw = decode(buffer);
+	const db = createMcpSearchIndex(defaultLocale);
+	load(db, raw);
+	return db;
+}
+//#endregion
+//#region source/plugins/mcp-search/index.ts
+/**
+* MCP search index plugin.
+*
+* Generates the Orama binary index (`mcp-search.msp`) and the MCP site
+* descriptor (`mcp.json`) as build assets consumed by the `clarify mcp`
+* command to serve documentation search to MCP-aware clients (e.g. AI coding
+* assistants).
+*
+* This plugin is independent of `clarify:page-search` (Pagefind), which
+* powers in-browser search. The MCP server runs in Node and needs a
+* self-contained, compact index with CJK-aware tokenization that doesn't
+* depend on the browser-only Pagefind runtime.
+*/
+function createMcpSearchPlugin() {
+	return {
+		name: "clarify:mcp-search",
+		hooks: { "build:assets"(ctx) {
+			const searchConfig = ctx.projectConfig.features.search;
+			if (!searchConfig.enabled || !searchConfig.mcp) return [];
+			const defaultLocale = ctx.projectConfig.locales?.default;
+			const { db, documentCount } = buildSearchIndex(ctx.routes, defaultLocale);
+			const locales = collectIndexedLocales(ctx.routes, defaultLocale);
+			const indexBytes = serializeSearchIndex(db);
+			const indexHash = createHash("sha256").update(indexBytes).digest("hex");
+			const mcpConfig = createMcpSiteConfig(ctx.projectConfig, {
+				documentCount,
+				locales,
+				indexHash
+			});
+			const assets = [{
+				fileName: `${mcpConfig.capabilities.search?.indexPath.replace(/\/mcp-search\.msp$/, "") ?? ""}/mcp-search.msp`.replace(/^\//, ""),
+				source: indexBytes
+			}, {
+				fileName: "mcp.json",
+				source: `${JSON.stringify(mcpConfig, null, 2)}\n`
+			}];
+			console.log(`[clarify] MCP search index (${documentCount} docs, ${locales.length} locales) written to mcp-search.msp.`);
+			return assets;
+		} }
+	};
+}
+//#endregion
 //#region source/plugins/openapi/parser.ts
 var OPENAPI_HTTP_METHODS = [
 	"get",
@@ -2316,12 +2666,12 @@ function removeClarifyTempDir(path) {
 	});
 }
 //#endregion
-//#region source/plugins/search-index/search-language.ts
+//#region source/plugins/page-search/search-language.ts
 function toPagefindLanguage(locale) {
 	return locale?.trim().replace(/_/g, "-").split("-").filter(Boolean).join("-").toLowerCase() || "en";
 }
 //#endregion
-//#region source/plugins/search-index/index.ts
+//#region source/plugins/page-search/index.ts
 var pagefindContentTypes = {
 	".css": "text/css; charset=utf-8",
 	".js": "text/javascript; charset=utf-8",
@@ -2361,15 +2711,6 @@ async function generatePagefindIndex(options, pagefind) {
 		if (!pagefind) await runtime.close();
 	}
 }
-function routeSearchContent(route) {
-	return [
-		route.meta.title,
-		route.meta.description,
-		route.meta.keywords?.join(" "),
-		route.meta.sections?.map((section) => section.title).join(" "),
-		route.source.content
-	].filter(Boolean).join("\n\n");
-}
 async function generateDevSearchIndex(ctx, root, pagefind) {
 	const { errors, index } = await pagefind.createIndex({
 		includeCharacters: "._-/#:@",
@@ -2426,9 +2767,9 @@ function serveDevPagefind(server, getRoot, ctx) {
 		createReadStream(filePath).pipe(res);
 	});
 }
-function createSearchIndexPlugin() {
+function createPageSearchPlugin() {
 	return {
-		name: "clarify:search-index",
+		name: "clarify:page-search",
 		hooks: {
 			async "dev:configureServer"(server, ctx) {
 				if (!ctx.projectConfig.features.search.enabled) return;
@@ -2475,7 +2816,7 @@ function createSearchIndexPlugin() {
 				const outputDirectory = ctx.generateOptions.outputDirectory;
 				if (!outputDirectory) return;
 				const pageCount = await generatePagefindIndex({ outputRoot: resolve(ctx.generateOptions.projectRoot, outputDirectory) });
-				console.log(`[clarify] Search index generated for ${pageCount} pages.`);
+				console.log(`[clarify] Page search index generated for ${pageCount} pages.`);
 			}
 		}
 	};
@@ -2624,7 +2965,8 @@ function createBuiltinPlugins(options = {}) {
 		createSourceLinksPlugin(),
 		createContentArtifactsPlugin(),
 		createSeoPlugin(),
-		createSearchIndexPlugin()
+		createPageSearchPlugin(),
+		createMcpSearchPlugin()
 	];
 	if (options.htmlShell ?? true) plugins.push(createHtmlShellPlugin());
 	return plugins;
@@ -2971,16 +3313,19 @@ async function renderSSGRoutes(routes, contextOrConfig, outputDirectory, ssrBund
 	const { render } = await import(pathToFileURL(ssrBundlePath).href);
 	const template = readIndexHtml(outputDirectory);
 	if (!template) throw new Error(`[clarify] index.html not found in outputDirectory "${outputDirectory}". Make sure Vite build produces it.`);
-	for (const route of routes) try {
-		const finalHtml = injectSSRIntoTemplate(template, await render(route.path), contextOrConfig, route);
-		for (const outFile of routeOutputFiles(outputDirectory, route)) {
-			mkdirSync(dirname(outFile), { recursive: true });
-			writeFileSync(outFile, finalHtml, "utf-8");
+	await Promise.all(routes.map(async (route) => {
+		try {
+			const appHtml = await render(route.path);
+			const finalHtml = injectSSRIntoTemplate(template, appHtml, contextOrConfig, route);
+			for (const outFile of routeOutputFiles(outputDirectory, route)) {
+				mkdirSync(dirname(outFile), { recursive: true });
+				writeFileSync(outFile, finalHtml, "utf-8");
+			}
+		} catch (err) {
+			console.error(`[clarify] Failed to render route "${route.path}":`, err);
+			throw err;
 		}
-	} catch (err) {
-		console.error(`[clarify] Failed to render route "${route.path}":`, err);
-		throw err;
-	}
+	}));
 }
 //#endregion
 //#region source/core/runtime/startup.ts
@@ -3419,6 +3764,6 @@ function createClarifyEngine(options = {}) {
 	return new ClarifyEngine(options);
 }
 //#endregion
-export { cliPackageVersion as a, defineConfig as c, createClarifyTempDir as i, createClarifyEngine as n, cliPackageVersionWithCaret as o, createClarifyRuntimeAliases as r, createViteAdapter as s, ClarifyEngine as t };
+export { deserializeSearchIndex as a, cliPackageVersionWithCaret as c, createClarifyTempDir as i, createViteAdapter as l, createClarifyEngine as n, searchMcpIndex as o, createClarifyRuntimeAliases as r, cliPackageVersion as s, ClarifyEngine as t, defineConfig as u };
 
-//# sourceMappingURL=engine-DfF5RJjC.js.map
+//# sourceMappingURL=engine-DI8Nd3v0.js.map