blume 1.2.0 → 1.2.1

package/docs/configuration/search.mdx

@@ -67,10 +67,25 @@ search: {
 }
 ```
 
+#### Languages written without spaces
+
+Orama's standard tokenizer splits on word boundaries that only exist in space-separated scripts, so Japanese, Chinese, Korean, and Thai text would otherwise produce no matches at all. Blume handles this for you: when [`i18n.defaultLocale`](/docs/content/i18n) is one of those languages, the index switches to a word-segmenting tokenizer (built on the browser- and Node-native `Intl.Segmenter`). Declaring your site's language is all it takes:
+
+```ts blume.config.ts lineNumbers
+i18n: {
+  defaultLocale: "ja",
+  locales: [{ code: "ja", label: "日本語" }],
+}
+```
+
+The same tokenizer serves the search dialog, the MCP server's `search_docs` tool, and Ask AI grounding. On a mixed-language site the whole index shares the default locale's tokenizer — that's safe, because Latin words survive segmentation intact, so pages in English (or any spaced language) stay searchable alongside the default language.
+
 ### FlexSearch
 
 A second keyless, client-side option. It reuses the same `/blume-search.json` index Orama ships and builds a [FlexSearch](https://github.com/nextapps-de/flexsearch) document index in the browser. Works in `blume dev` and `blume build`.
 
+FlexSearch has no equivalent segmentation hook, so for sites in Japanese, Chinese, Korean, or Thai prefer Orama (the default) or [Pagefind](#pagefind), whose `pagefind_extended` binary segments those languages natively.
+
 ```ts blume.config.ts lineNumbers
 search: {
   provider: "flexsearch",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "blume",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "Documentation that's fast, AI-ready, and zero-config.",
   "keywords": [
     "astro",
@@ -109,7 +109,7 @@
     "satteri": "^0.9.5",
     "shiki": "^4.2.0",
     "simple-icons": "^13.0.0",
-    "tailwindcss": "^4",
+    "tailwindcss": "^4.3.0",
     "takumi-js": "^2.2.1",
     "tinyglobby": "^0.2.10",
     "twoslash": "^0.3.9",

package/src/ai/ask-context.ts

@@ -19,6 +19,12 @@ export interface AskPage {
  * `generated/ask-data.json` and built by {@link buildAskData}.
  */
 export interface AskData {
+  /**
+   * The site's `i18n.defaultLocale`, when i18n is configured. Selects a
+   * word-segmenting Orama tokenizer for languages written without spaces, so
+   * retrieval can match CJK/Thai content.
+   */
+  defaultLocale?: string;
   documents: OramaDoc[];
   site: string | null;
 }
@@ -203,7 +209,7 @@ export const createAskContext = (
   let dbPromise: Promise<Awaited<ReturnType<typeof buildOramaIndex>>> | null =
     null;
   const index = () => {
-    dbPromise ??= buildOramaIndex(data.documents);
+    dbPromise ??= buildOramaIndex(data.documents, data.defaultLocale);
     return dbPromise;
   };
   const byRoute = new Map(data.documents.map((doc) => [doc.route, doc]));

package/src/ai/ask-data.ts

@@ -19,6 +19,7 @@ export const buildAskData = async (project: BlumeProject): Promise<AskData> => {
     includeWhenDisabled: true,
   });
   return {
+    defaultLocale: project.config.i18n?.defaultLocale,
     documents: documents.map((doc) => ({
       content: doc.content,
       description: doc.description,

package/src/ai/mcp/data.ts

@@ -28,6 +28,12 @@ export interface McpData {
    * paths, matching the sitemap/llms.txt convention.
    */
   base: string;
+  /**
+   * The site's `i18n.defaultLocale`, when i18n is configured. Selects a
+   * word-segmenting Orama tokenizer for languages written without spaces, so
+   * `search_docs` can match CJK/Thai content.
+   */
+  defaultLocale?: string;
   documents: OramaDoc[];
   instructions?: string;
   name: string;
@@ -82,6 +88,7 @@ export const buildMcpData = async (project: BlumeProject): Promise<McpData> => {
 
   return {
     base: normalizeBasePath(config.deployment.base),
+    defaultLocale: config.i18n?.defaultLocale,
     documents: documents.map((doc) => ({
       content: doc.content,
       description: doc.description,

package/src/ai/mcp/server.ts

@@ -139,13 +139,18 @@ export type OramaIndexProvider = () => Promise<
   Awaited<ReturnType<typeof buildOramaIndex>>
 >;
 
-/** Memoize the search index so every server built from a snapshot shares it. */
+/**
+ * Memoize the search index so every server built from a snapshot shares it.
+ * `locale` is the snapshot's `defaultLocale`, forwarded so unspaced scripts
+ * (Japanese, Chinese, Korean, Thai) get a word-segmenting tokenizer.
+ */
 export const createIndexProvider = (
-  documents: OramaDoc[]
+  documents: OramaDoc[],
+  locale?: string
 ): OramaIndexProvider => {
   let dbPromise: ReturnType<OramaIndexProvider> | null = null;
   return () => {
-    dbPromise ??= buildOramaIndex(documents);
+    dbPromise ??= buildOramaIndex(documents, locale);
     return dbPromise;
   };
 };
@@ -240,7 +245,7 @@ export const buildServer = (
 export const createMcpFetchHandler = (
   data: McpData
 ): ((request: Request) => Promise<Response>) => {
-  const index = createIndexProvider(data.documents);
+  const index = createIndexProvider(data.documents, data.defaultLocale);
 
   return async (request: Request): Promise<Response> => {
     if (request.method === "OPTIONS") {

package/src/ai/mcp/stdio.ts

@@ -28,7 +28,10 @@ export const serveMcpStdio = async (
   const stdin = streams.stdin ?? process.stdin;
   const stdout = streams.stdout ?? process.stdout;
   const transport = new StdioServerTransport(stdin, stdout);
-  const server = buildServer(data, createIndexProvider(data.documents));
+  const server = buildServer(
+    data,
+    createIndexProvider(data.documents, data.defaultLocale)
+  );
   await server.connect(transport);
   await once(stdin, "end");
   await transport.close();

package/src/astro/templates.ts

@@ -976,12 +976,18 @@ const searchClientImport = (module: string): string =>
 const SEARCH_BASE_IMPORT =
   'import { joinBase } from "blume/components/islands/base-path.ts";\n';
 
-/** A client that loads a static `blume-search.json` index (Orama, FlexSearch). */
-const staticSearchClient = (module: string): string =>
+/**
+ * A client that loads a static `blume-search.json` index (Orama, FlexSearch).
+ * `locale` (Orama only) is the site's `i18n.defaultLocale`, which selects a
+ * word-segmenting tokenizer for languages written without spaces.
+ */
+const staticSearchClient = (module: string, locale?: string): string =>
   `${SEARCH_CLIENT_HEADER}${searchClientImport(module)}${SEARCH_BASE_IMPORT}
 const indexUrl = joinBase(import.meta.env.BASE_URL, "blume-search.json");
 
-export const createSearch = () => create({ indexUrl });
+export const createSearch = () => create({ indexUrl${
+    locale ? `, locale: ${JSON.stringify(locale)}` : ""
+  } });
 `;
 
 /** A client that passes public credentials straight to the provider SDK. */
@@ -1030,7 +1036,12 @@ export const searchClientTemplate = (config: ResolvedConfig): string => {
   const { search } = config;
 
   if (search.provider === "orama" || search.provider === "flexsearch") {
-    return staticSearchClient(search.provider);
+    // Only Orama derives a tokenizer from the locale; FlexSearch has no
+    // equivalent hook, so its client keeps the bare index URL.
+    return staticSearchClient(
+      search.provider,
+      search.provider === "orama" ? config.i18n?.defaultLocale : undefined
+    );
   }
 
   const hosted = hostedSearchOptions(search);

package/src/components/layout/NavTree.astro

@@ -137,7 +137,7 @@ const initialId =
             <div class="mb-3 flex items-center gap-0.5">
               <button
                 aria-label={n.back}
-                class="-ml-1 flex shrink-0 items-center justify-center self-stretch rounded px-1 text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
+                class="-ml-1 flex shrink-0 items-center justify-center self-stretch rounded-[0.65rem] px-1 text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
                 data-nav-back={panel.parentId}
                 type="button"
               >
@@ -145,7 +145,7 @@ const initialId =
               </button>
               <a
                 aria-current={panel.route === currentRoute ? "page" : undefined}
-                class="flex-1 truncate rounded px-1 py-1 font-semibold text-foreground text-sm transition-colors hover:bg-muted"
+                class="flex-1 truncate rounded-[0.65rem] px-1 py-1 font-semibold text-foreground text-sm transition-colors hover:bg-muted"
                 href={withBase(panel.route)}
               >
                 {panel.label}
@@ -154,7 +154,7 @@ const initialId =
           ) : (
             <button
               aria-label={`${n.back}: ${panel.label}`}
-              class="-ml-1 mb-3 flex w-full items-center gap-1.5 rounded p-1 text-left font-semibold text-foreground text-sm transition-colors hover:bg-muted"
+              class="-ml-1 mb-3 flex w-full items-center gap-1.5 rounded-[0.65rem] p-1 text-left font-semibold text-foreground text-sm transition-colors hover:bg-muted"
               data-nav-back={panel.parentId}
               type="button"
             >
@@ -317,7 +317,7 @@ const initialId =
               {item.route ? (
                 <a
                   aria-current={item.route === currentRoute ? "page" : undefined}
-                  class="-ml-1 flex flex-1 items-center gap-1.5 rounded px-1 py-0.5 text-foreground transition-colors hover:bg-muted aria-[current=page]:bg-muted"
+                  class="-ml-1 flex flex-1 items-center gap-1.5 rounded-[0.65rem] px-1 py-0.5 text-foreground transition-colors hover:bg-muted aria-[current=page]:bg-muted"
                   href={withBase(item.route)}
                 >
                   {item.icon && (

package/src/components/layout/RootLayout.astro

@@ -479,7 +479,7 @@ const bannerKey = banner?.dismissible ? banner.key : null;
         aria-label={navStrings.primary}
         data-blume-nav-drawer
         class:list={[
-          "fixed top-[var(--blume-drawer-top,4rem)] start-0 z-[35] h-[calc(100dvh-var(--blume-drawer-top,4rem))] w-64 max-w-[80vw] -translate-x-[105%] overflow-y-auto border-border border-e bg-background px-5 pt-4 pb-6 transition-transform rtl:translate-x-[105%] [:where([data-blume-nav-open])_&]:translate-x-0! lg:sticky lg:top-16 lg:z-auto lg:h-[calc(100dvh-4rem)] lg:w-auto lg:max-w-none lg:translate-x-0! lg:border-e-0 lg:bg-transparent lg:px-4",
+          "fixed top-[var(--blume-drawer-top,4rem)] start-0 z-[35] h-[calc(100dvh-var(--blume-drawer-top,4rem))] w-64 max-w-[80vw] -translate-x-[105%] overflow-y-auto border-border border-e bg-background px-5 pt-4 pb-6 transition-transform rtl:translate-x-[105%] [:where([data-blume-nav-open])_&]:translate-x-0! lg:sticky lg:top-16 lg:z-auto lg:h-[calc(100dvh-4rem)] lg:w-auto lg:max-w-none lg:translate-x-0! lg:scrollbar-thin lg:scrollbar-thumb-border lg:scrollbar-track-transparent lg:border-e-0 lg:bg-transparent lg:px-4",
           // A "bare" landing (the changelog index) has no sidebar column on
           // desktop, but the header hamburger still needs a drawer to open on
           // mobile — without it the toggle only locked page scroll.

package/src/components/layout/search/orama.ts

@@ -10,14 +10,17 @@ import type { IndexedDocument, SearchFn } from "./types.ts";
  * in-memory full-text database in the browser, and query it. Keyless and
  * available in both dev and the production build. A generous match pool is
  * pulled so the section pills can count across the whole result set before the
- * active filter and display limit are applied.
+ * active filter and display limit are applied. `locale` is the site's
+ * `i18n.defaultLocale`, baked into the generated client so unspaced scripts
+ * (Japanese, Chinese, Korean, Thai) get a word-segmenting tokenizer.
  */
 export const createSearch = async (opts: {
   indexUrl: string;
+  locale?: string;
 }): Promise<SearchFn> => {
   const response = await fetch(opts.indexUrl);
   const documents = (await response.json()) as IndexedDocument[];
-  const db = await buildOramaIndex(documents);
+  const db = await buildOramaIndex(documents, opts.locale);
 
   return async (query, options) => {
     const docs = await queryOramaIndex(db, query, RESULT_POOL, options?.locale);

package/src/search/orama-index.ts

@@ -1,5 +1,5 @@
 import { create, insertMultiple, search } from "@orama/orama";
-import type { AnyOrama } from "@orama/orama";
+import type { AnyOrama, Tokenizer } from "@orama/orama";
 
 /**
  * The minimal document shape both the client-side search dialog and the
@@ -31,15 +31,65 @@ const SCHEMA = {
 /** Title and description outrank body text, matching the search dialog. */
 const BOOST = { description: 2, title: 3 };
 
+/**
+ * Scripts written without spaces between words. Orama's default tokenizer
+ * splits on a Latin-centric delimiter class, so text in these languages
+ * collapses to zero tokens and every query silently returns no hits. Keyed by
+ * the primary language subtag of `i18n.defaultLocale`.
+ */
+const SEGMENTED_LANGUAGES = new Set(["ja", "ko", "th", "zh"]);
+
+/**
+ * A word-segmenting tokenizer for languages the default splitter can't handle,
+ * built on `Intl.Segmenter` (the same engine `@orama/tokenizers` wraps).
+ * Input is lowercased before segmenting — unlike the upstream tokenizers —
+ * so Latin terms ("GDPR", English pages on a mixed-locale site) still match
+ * case-insensitively. Returns `undefined` for languages the default tokenizer
+ * already serves, and on runtimes without `Intl.Segmenter`, where the caller
+ * falls back to Orama's default.
+ */
+const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
+  const language = locale?.toLowerCase().split(/[-_]/u)[0] ?? "";
+  if (!SEGMENTED_LANGUAGES.has(language)) {
+    return;
+  }
+  if (typeof Intl.Segmenter !== "function") {
+    return;
+  }
+  const segmenter = new Intl.Segmenter(language, { granularity: "word" });
+  return {
+    language,
+    normalizationCache: new Map(),
+    tokenize: (raw: string): string[] => {
+      const tokens = new Set<string>();
+      for (const segment of segmenter.segment(raw.toLowerCase())) {
+        if (segment.isWordLike) {
+          tokens.add(segment.segment);
+        }
+      }
+      return [...tokens];
+    },
+  };
+};
+
 /**
  * Build an in-memory Orama full-text index from search documents. Shared by the
- * Orama client loader (browser) and the MCP server (Node), so ranking is
- * identical wherever docs are queried.
+ * Orama client loader (browser), the MCP server, and Ask AI grounding (Node),
+ * so ranking is identical wherever docs are queried. `locale` — the site's
+ * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for languages
+ * written without spaces (Japanese, Chinese, Korean, Thai); the tokenizer
+ * belongs to the database, so on a mixed-locale site it applies to every
+ * document, which is safe because Latin words survive segmentation intact.
  */
 export const buildOramaIndex = async (
-  documents: OramaDoc[]
+  documents: OramaDoc[],
+  locale?: string
 ): Promise<AnyOrama> => {
-  const db = create({ schema: SCHEMA });
+  const tokenizer = segmentingTokenizer(locale);
+  const db = create({
+    schema: SCHEMA,
+    ...(tokenizer ? { components: { tokenizer } } : {}),
+  });
   await insertMultiple(db, documents);
   return db;
 };