blume 1.1.4 → 1.2.1

package/src/openapi/render-mdx.ts

@@ -1,4 +1,5 @@
 import type { ApiOperationRef, ApiSpecData } from "./model.ts";
+import type { ReferenceSource } from "./references.ts";
 
 /**
  * Lower a parsed spec into MDX for the staged content source. Each operation and
@@ -126,7 +127,11 @@ const withDescription = (description: string, component: string): string =>
 
 export const operationMdx = (
   spec: ApiSpecData,
-  operation: ApiOperationRef
+  operation: ApiOperationRef,
+  reference?: Pick<
+    ReferenceSource,
+    "includeInLlms" | "includeInSearch" | "noindex"
+  >
 ): RenderedPage => {
   const method = operation.method.toUpperCase();
   const title = operation.summary || `${method} ${operation.path}`;
@@ -142,9 +147,16 @@ export const operationMdx = (
       `<Operation source="${spec.slug}" id="${operation.key}" />`
     ),
     data: {
+      ...(reference?.includeInLlms === false ? { ai: { exclude: true } } : {}),
       ...(operation.deprecated ? { deprecated: true } : {}),
-      search: { tags: [operation.tag, method] },
-      seo: { description: operationDescription(spec, operation) },
+      search: {
+        ...(reference?.includeInSearch === false ? { exclude: true } : {}),
+        tags: [operation.tag, method],
+      },
+      seo: {
+        description: operationDescription(spec, operation),
+        ...(reference?.noindex ? { noindex: true } : {}),
+      },
       sidebar: { badge: method, label: operation.summary || operation.path },
       title,
       // Signals the two-column API layout (request panel instead of the TOC).
@@ -153,7 +165,13 @@ export const operationMdx = (
   };
 };
 
-export const overviewMdx = (spec: ApiSpecData): RenderedPage => {
+export const overviewMdx = (
+  spec: ApiSpecData,
+  reference?: Pick<
+    ReferenceSource,
+    "includeInLlms" | "includeInSearch" | "noindex"
+  >
+): RenderedPage => {
   // Tag sections: declared tags in spec order, then any tag an operation
   // references that isn't declared under `tags`. The section headings are
   // emitted as real markdown `##` (not markup inside a component) so the
@@ -205,10 +223,15 @@ export const overviewMdx = (spec: ApiSpecData): RenderedPage => {
       ...tagSections,
     ].join("\n\n"),
     data: {
+      ...(reference?.includeInLlms === false ? { ai: { exclude: true } } : {}),
+      ...(reference?.includeInSearch === false
+        ? { search: { exclude: true } }
+        : {}),
       seo: {
         description:
           clip(plainProse(spec.description), META_DESCRIPTION_MAX) ||
           `${apiName(spec)} API reference.`,
+        ...(reference?.noindex ? { noindex: true } : {}),
       },
       sidebar: { label: "Overview" },
       title: apiName(spec),

package/src/openapi/scalar.ts

@@ -167,6 +167,7 @@ export const buildReferenceFiles = async (options: {
           ...ref.scalar,
         },
         dataImport: `${"../".repeat(depth + 1)}generated/data.json`,
+        noindex: ref.noindex,
         route: ref.route,
         title: ref.label,
       }),

package/src/openapi/source.ts

@@ -53,17 +53,24 @@ const toEntry = (rendered: RenderedPage, ref: string): SourceEntry => {
 /** All staged entries for one spec: operations first, overview last. */
 const specEntries = (
   spec: ApiSpecData,
-  operations: ApiOperationRef[]
+  operations: ApiOperationRef[],
+  reference: ReferenceSource
 ): SourceEntry[] => {
   const entries = operations.map((operation) =>
-    toEntry(operationMdx(spec, operation), `${routeToRef(operation.route)}.mdx`)
+    toEntry(
+      operationMdx(spec, operation, reference),
+      `${routeToRef(operation.route)}.mdx`
+    )
   );
   // Overview last so an operation sets the section's routePath before the index
   // page is inserted (the group's routePath is derived from its first child).
   // A root-mounted reference refs `index.mdx`, not `/index.mdx`.
   const base = routeToRef(spec.route);
   entries.push(
-    toEntry(overviewMdx(spec), base ? `${base}/index.mdx` : "index.mdx")
+    toEntry(
+      overviewMdx(spec, reference),
+      base ? `${base}/index.mdx` : "index.mdx"
+    )
   );
   return entries;
 };
@@ -149,7 +156,7 @@ export const openApiSource = (
               ]
             : []),
         ],
-        entries: specEntries(spec, operations),
+        entries: specEntries(spec, operations, reference),
         slug: reference.slug,
         spec,
       };

package/src/registry/eject.ts

@@ -112,7 +112,15 @@ const askFiles = async (
   genDir: string
 ): Promise<{ content: string; path: string }[]> => {
   const { ask } = project.config.ai;
-  if (!ask?.enabled) {
+  if (!(ask?.enabled && !ask.endpoint)) {
+    const endpointPath = join(srcDir, "pages", "api", "ask.ts");
+    if (existsSync(endpointPath)) {
+      const content = await readFile(endpointPath, "utf-8");
+      if (content.startsWith("// Generated by Blume. Do not edit.")) {
+        await rm(endpointPath, { force: true });
+      }
+    }
+    await rm(join(genDir, "ask-data.json"), { force: true });
     return [];
   }
   const grounded = ask.provider !== "inkeep";
@@ -271,6 +279,15 @@ const examplesPreviewFiles = (
       ]
     : [];
 
+const ejectIntegrationBridge = (
+  config: BlumeProject["config"],
+  root: string,
+  configFile: string | null
+): Parameters<typeof astroConfigTemplate>[0]["integrationBridge"] =>
+  config.integrations.length > 0 && configFile
+    ? { configFile: toPosix(relative(root, configFile)) }
+    : undefined;
+
 /**
  * Promote the generated runtime into the project as an owned Astro app. After
  * eject the project has a normal `astro.config.mjs` and `src/`, the `blume` CLI
@@ -367,6 +384,11 @@ export const eject = async (
         dataPath: "./src/generated/data.json",
         examplesPath: "./src/generated/examples.ts",
         examplesThemePath: "./src/generated/examples.css",
+        integrationBridge: ejectIntegrationBridge(
+          config,
+          root,
+          context.configFile
+        ),
         needsReact,
         needsSvelte,
         needsVue,

package/src/search/build.ts

@@ -1,9 +1,10 @@
 import { join } from "pathe";
 
 /**
- * Build a local Pagefind search index over the built site. Pagefind only
- * indexes elements marked with `data-pagefind-body`, which Blume adds to the
- * content of indexable pages, so nav chrome and excluded pages are skipped.
+ * Build a local Pagefind search index over the built site. Pagefind indexes
+ * every rendered page except those whose `<html>` carries
+ * `data-pagefind-ignore`, which Blume stamps on non-indexable pages
+ * (search-excluded, or hidden without the opt-in), so those stay out.
  *
  * Returns the number of pages indexed.
  */

package/src/search/orama-index.ts

@@ -1,5 +1,5 @@
 import { create, insertMultiple, search } from "@orama/orama";
-import type { AnyOrama } from "@orama/orama";
+import type { AnyOrama, Tokenizer } from "@orama/orama";
 
 /**
  * The minimal document shape both the client-side search dialog and the
@@ -31,15 +31,65 @@ const SCHEMA = {
 /** Title and description outrank body text, matching the search dialog. */
 const BOOST = { description: 2, title: 3 };
 
+/**
+ * Scripts written without spaces between words. Orama's default tokenizer
+ * splits on a Latin-centric delimiter class, so text in these languages
+ * collapses to zero tokens and every query silently returns no hits. Keyed by
+ * the primary language subtag of `i18n.defaultLocale`.
+ */
+const SEGMENTED_LANGUAGES = new Set(["ja", "ko", "th", "zh"]);
+
+/**
+ * A word-segmenting tokenizer for languages the default splitter can't handle,
+ * built on `Intl.Segmenter` (the same engine `@orama/tokenizers` wraps).
+ * Input is lowercased before segmenting — unlike the upstream tokenizers —
+ * so Latin terms ("GDPR", English pages on a mixed-locale site) still match
+ * case-insensitively. Returns `undefined` for languages the default tokenizer
+ * already serves, and on runtimes without `Intl.Segmenter`, where the caller
+ * falls back to Orama's default.
+ */
+const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
+  const language = locale?.toLowerCase().split(/[-_]/u)[0] ?? "";
+  if (!SEGMENTED_LANGUAGES.has(language)) {
+    return;
+  }
+  if (typeof Intl.Segmenter !== "function") {
+    return;
+  }
+  const segmenter = new Intl.Segmenter(language, { granularity: "word" });
+  return {
+    language,
+    normalizationCache: new Map(),
+    tokenize: (raw: string): string[] => {
+      const tokens = new Set<string>();
+      for (const segment of segmenter.segment(raw.toLowerCase())) {
+        if (segment.isWordLike) {
+          tokens.add(segment.segment);
+        }
+      }
+      return [...tokens];
+    },
+  };
+};
+
 /**
  * Build an in-memory Orama full-text index from search documents. Shared by the
- * Orama client loader (browser) and the MCP server (Node), so ranking is
- * identical wherever docs are queried.
+ * Orama client loader (browser), the MCP server, and Ask AI grounding (Node),
+ * so ranking is identical wherever docs are queried. `locale` — the site's
+ * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for languages
+ * written without spaces (Japanese, Chinese, Korean, Thai); the tokenizer
+ * belongs to the database, so on a mixed-locale site it applies to every
+ * document, which is safe because Latin words survive segmentation intact.
  */
 export const buildOramaIndex = async (
-  documents: OramaDoc[]
+  documents: OramaDoc[],
+  locale?: string
 ): Promise<AnyOrama> => {
-  const db = create({ schema: SCHEMA });
+  const tokenizer = segmentingTokenizer(locale);
+  const db = create({
+    schema: SCHEMA,
+    ...(tokenizer ? { components: { tokenizer } } : {}),
+  });
   await insertMultiple(db, documents);
   return db;
 };