blume 0.0.0 → 0.1.0

package/src/core/server-features.ts

@@ -0,0 +1,23 @@
+import { searchProviderMeta } from "../search/providers.ts";
+import type { ResolvedConfig } from "./schema.ts";
+
+/**
+ * List the enabled features that require Astro server output. Static builds
+ * fail clearly when any of these are enabled.
+ */
+export const serverFeatures = (config: ResolvedConfig): string[] => {
+  const features: string[] = [];
+  if (config.ai.ask?.enabled) {
+    features.push("Ask AI");
+  }
+  // The hosted MCP server is a live JSON-RPC endpoint, so it needs a runtime.
+  if (config.mcp.enabled) {
+    features.push("MCP server");
+  }
+  // Mixedbread (and any future provider) that proxies queries through a secret
+  // server endpoint can't run on a static build.
+  if (searchProviderMeta(config.search.provider).requiresServer) {
+    features.push(`Search (${config.search.provider})`);
+  }
+  return features;
+};

package/src/core/sources/assets.ts

@@ -0,0 +1,77 @@
+import { mkdir, writeFile } from "node:fs/promises";
+
+import { extname, join } from "pathe";
+
+import type { Diagnostic } from "../types.ts";
+import { hashText } from "./cache.ts";
+
+const MD_IMAGE = /!\[(?<alt>[^\]]*)\]\((?<url>[^)\s]+)\)/gu;
+const REMOTE = /^https?:\/\//u;
+const SAFE_EXT = /^\.[a-z0-9]+$/iu;
+
+/** Where to write downloaded assets and how to reference them publicly. */
+export interface AssetContext {
+  assetsDir: string;
+  assetsBaseUrl: string;
+  /** Injected for tests; defaults to the global `fetch`. */
+  fetchImpl?: typeof fetch;
+}
+
+/** Pick a file extension from a URL, defaulting to `.png`. */
+const extFor = (url: string): string => {
+  const clean = url.split("?")[0] ?? url;
+  const ext = extname(clean);
+  return SAFE_EXT.test(ext) ? ext.toLowerCase() : ".png";
+};
+
+/**
+ * Download remote images referenced in a Markdown body into the asset dir and
+ * rewrite their `src` to the local public path. Remote CMS URLs (notably
+ * Notion's signed, expiring links) would otherwise rot a static build. Assets
+ * are content-addressed by URL hash, so repeated builds are stable and deduped.
+ */
+export const materializeAssets = async (
+  markdown: string,
+  ctx: AssetContext
+): Promise<{ markdown: string; diagnostics: Diagnostic[] }> => {
+  const doFetch = ctx.fetchImpl ?? globalThis.fetch;
+  const diagnostics: Diagnostic[] = [];
+
+  const urls = new Set<string>();
+  for (const match of markdown.matchAll(MD_IMAGE)) {
+    const url = match.groups?.url;
+    if (url && REMOTE.test(url)) {
+      urls.add(url);
+    }
+  }
+
+  const rewrites = new Map<string, string>();
+  await Promise.all(
+    [...urls].map(async (url) => {
+      try {
+        const res = await doFetch(url);
+        if (!res.ok) {
+          throw new Error(`${res.status}`);
+        }
+        const bytes = new Uint8Array(await res.arrayBuffer());
+        const file = `${hashText(url)}${extFor(url)}`;
+        await mkdir(ctx.assetsDir, { recursive: true });
+        await writeFile(join(ctx.assetsDir, file), bytes);
+        rewrites.set(url, `${ctx.assetsBaseUrl}/${file}`);
+      } catch (error) {
+        diagnostics.push({
+          code: "BLUME_ASSET_FETCH_FAILED",
+          message: `Failed to download asset ${url}: ${(error as Error).message}`,
+          severity: "warning",
+        });
+      }
+    })
+  );
+
+  const rewritten = markdown.replaceAll(MD_IMAGE, (match, alt, url) => {
+    const local = rewrites.get(url);
+    return local ? `![${alt}](${local})` : match;
+  });
+
+  return { diagnostics, markdown: rewritten };
+};

package/src/core/sources/cache.ts

@@ -0,0 +1,122 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+
+import { join } from "pathe";
+
+import { BlumeError } from "../diagnostics.ts";
+import type { Diagnostic } from "../types.ts";
+import type { SourceEntry, SourceLoadResult } from "./types.ts";
+
+/** Small, stable content hash for cache/HMR bookkeeping (non-bitwise). */
+export const hashText = (text: string): string => {
+  let hash = 5381;
+  for (let i = 0; i < text.length; i += 1) {
+    hash = (hash * 33 + (text.codePointAt(i) ?? 0)) % 2_147_483_647;
+  }
+  return hash.toString(36);
+};
+
+/** A stable digest of a source's entries, for change detection while polling. */
+export const entriesDigest = (entries: SourceEntry[]): string =>
+  hashText(
+    entries
+      .map((entry) => `${entry.ref}:${entry.hash ?? hashText(entry.body.text)}`)
+      .join("|")
+  );
+
+/**
+ * Build an opt-in polling watcher for a remote source: re-`load()` on an
+ * interval and fire `onChange` only when the entry digest changes, so a remote
+ * source can hot-reload in dev without refetching the world on every keystroke.
+ */
+export const pollingWatch =
+  (
+    load: () => Promise<SourceLoadResult>,
+    intervalSeconds: number
+  ): ((onChange: () => void) => () => void) =>
+  (onChange) => {
+    let last = "";
+    const tick = async (): Promise<void> => {
+      try {
+        const { entries } = await load();
+        const next = entriesDigest(entries);
+        if (last && next !== last) {
+          onChange();
+        }
+        last = next;
+      } catch {
+        // Ignore transient poll failures; the cache keeps serving last-known-good.
+      }
+    };
+    const timer = setInterval(() => {
+      void tick();
+    }, intervalSeconds * 1000);
+    return () => clearInterval(timer);
+  };
+
+/** A per-source snapshot of the last successful fetch, for offline tolerance. */
+export interface SnapshotCache {
+  read: () => Promise<SourceEntry[]>;
+  write: (entries: SourceEntry[]) => Promise<void>;
+}
+
+/** Build a JSON snapshot cache under `<cacheDir>/entries.json`. */
+export const snapshotCache = (cacheDir: string): SnapshotCache => {
+  const file = join(cacheDir, "entries.json");
+  return {
+    read: async () => {
+      try {
+        return JSON.parse(await readFile(file, "utf-8")) as SourceEntry[];
+      } catch {
+        return [];
+      }
+    },
+    write: async (entries) => {
+      try {
+        await mkdir(cacheDir, { recursive: true });
+        await writeFile(file, `${JSON.stringify(entries)}\n`, "utf-8");
+      } catch {
+        // Cache is best-effort; a write failure must not fail the build.
+      }
+    },
+  };
+};
+
+/**
+ * Run a remote `fetchEntries`, caching the result. When `refresh` is false and a
+ * snapshot exists, serve it without fetching (cache-first dev). On fetch failure,
+ * serve the last-known-good snapshot with a warning so a CMS/network outage
+ * doesn't fail the build; if there is no snapshot either, surface a hard error.
+ */
+export const loadWithCache = async (
+  name: string,
+  cache: SnapshotCache,
+  fetchEntries: () => Promise<SourceEntry[]>,
+  refresh = true
+): Promise<SourceLoadResult> => {
+  if (!refresh) {
+    const cached = await cache.read();
+    if (cached.length > 0) {
+      return { diagnostics: [], entries: cached };
+    }
+  }
+  try {
+    const entries = await fetchEntries();
+    await cache.write(entries);
+    return { diagnostics: [], entries };
+  } catch (error) {
+    const fallback = await cache.read();
+    if (fallback.length > 0) {
+      const diagnostic: Diagnostic = {
+        code: "BLUME_SOURCE_OFFLINE",
+        message: `Source "${name}" could not be fetched (${(error as Error).message}); served ${fallback.length} cached entries.`,
+        severity: "warning",
+      };
+      return { diagnostics: [diagnostic], entries: fallback };
+    }
+    throw new BlumeError({
+      code: "BLUME_SOURCE_FETCH_FAILED",
+      message: `Source "${name}" failed to load and no cache is available: ${(error as Error).message}`,
+      severity: "error",
+    });
+  }
+};

package/src/core/sources/filesystem.ts

@@ -0,0 +1,99 @@
+import { existsSync, watch as fsWatch } from "node:fs";
+import { readFile } from "node:fs/promises";
+
+import matter from "gray-matter";
+import { extname, isAbsolute, join, relative, resolve } from "pathe";
+import { glob } from "tinyglobby";
+
+import { BlumeError } from "../diagnostics.ts";
+import type { ContentSource, SourceEntry, SourceLoadResult } from "./types.ts";
+
+/** Options for the built-in filesystem source. */
+export interface FilesystemSourceOptions {
+  /** Stable source name; namespaces ids and diagnostics. */
+  name: string;
+  /** Optional route prefix. */
+  prefix?: string;
+  /** Content root, absolute or relative to `projectRoot`. */
+  root: string;
+  include: string[];
+  exclude: string[];
+  /** Absolute project root, used to resolve a relative `root`. */
+  projectRoot: string;
+  /** When set, a missing root throws this code instead of the generic one. */
+  missingCode?: string;
+}
+
+/**
+ * The built-in content source: discover and read `.md`/`.mdx` files under a
+ * content root. This is the extraction of the original `discoverContent()` scan
+ * into a `ContentSource`; every other adapter follows the same contract.
+ */
+export const filesystemSource = (
+  options: FilesystemSourceOptions
+): ContentSource & { readonly contentRoot: string } => {
+  const contentRoot = isAbsolute(options.root)
+    ? options.root
+    : join(resolve(options.projectRoot), options.root);
+
+  const load = async (): Promise<SourceLoadResult> => {
+    const files = await glob(options.include, {
+      absolute: true,
+      cwd: contentRoot,
+      ignore: options.exclude,
+      onlyFiles: true,
+    });
+    files.sort();
+
+    const entries = await Promise.all(
+      files.map(async (file): Promise<SourceEntry> => {
+        const source = await readFile(file, "utf-8");
+        const ext = extname(file).toLowerCase();
+        const format = ext === ".mdx" ? "mdx" : "md";
+        const parsed = matter(source);
+        return {
+          body: { format, text: parsed.content },
+          data: parsed.data,
+          ref: relative(contentRoot, file),
+          sourcePath: file,
+        };
+      })
+    );
+
+    return { diagnostics: [], entries };
+  };
+
+  const validate = (): void => {
+    if (!existsSync(contentRoot)) {
+      throw new BlumeError({
+        code: options.missingCode ?? "BLUME_CONTENT_ROOT_MISSING",
+        file: contentRoot,
+        message: `Content root not found: ${options.root}`,
+        severity: "error",
+        suggestion: `Create a "${options.root}" folder with at least one .md or .mdx file, or set content.root in blume.config.ts.`,
+      });
+    }
+  };
+
+  const watch = (onChange: () => void): (() => void) => {
+    if (!existsSync(contentRoot)) {
+      return () => {
+        // Nothing to dispose when the root doesn't exist yet.
+      };
+    }
+    const watcher = fsWatch(contentRoot, { recursive: true }, onChange);
+    return () => watcher.close();
+  };
+
+  return {
+    contentRoot,
+    load,
+    name: options.name,
+    prefix: options.prefix,
+    read: (ref: string): Promise<string> =>
+      readFile(join(contentRoot, ref), "utf-8"),
+    staged: false,
+    validate,
+    watch,
+  };
+};

package/src/core/sources/mdx-remote.ts

@@ -0,0 +1,216 @@
+import matter from "gray-matter";
+
+import { BlumeError } from "../diagnostics.ts";
+import {
+  hashText,
+  loadWithCache,
+  pollingWatch,
+  snapshotCache,
+} from "./cache.ts";
+import type {
+  ContentSource,
+  SourceContext,
+  SourceEntry,
+  SourceLoadResult,
+} from "./types.ts";
+
+/** Options for the built-in remote Markdown/MDX source. */
+export interface MdxRemoteSourceOptions {
+  name: string;
+  prefix?: string;
+  /** Raw base URL for `files`, e.g. `https://raw.githubusercontent.com/o/r/main/docs`. */
+  url?: string;
+  /** Explicit source-relative file paths to fetch from `url`. */
+  files?: string[];
+  /** Enumerate a GitHub repo subtree via the git-trees API. */
+  github?: { owner: string; repo: string; ref: string; path: string };
+  include: string[];
+  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
+  pollInterval?: number;
+  /** Injected for tests; defaults to the global `fetch`. */
+  fetchImpl?: typeof fetch;
+}
+
+const REGEX_SPECIAL = /[.*+?^${}()|[\]\\]/u;
+
+/** Escape a literal character for embedding in a RegExp. */
+const escapeChar = (char: string): string =>
+  REGEX_SPECIAL.test(char) ? `\\${char}` : char;
+
+/** Compile a glob (`**`, `*`, `?`, `{a,b}`) into an anchored RegExp. */
+const globToRegExp = (pattern: string): RegExp => {
+  let source = "";
+  let i = 0;
+  while (i < pattern.length) {
+    const char = pattern[i] ?? "";
+    if (char === "*") {
+      if (pattern[i + 1] === "*") {
+        source += ".*";
+        i += 2;
+        if (pattern[i] === "/") {
+          i += 1;
+        }
+        continue;
+      }
+      source += "[^/]*";
+      i += 1;
+      continue;
+    }
+    if (char === "?") {
+      source += "[^/]";
+      i += 1;
+      continue;
+    }
+    if (char === "{") {
+      const end = pattern.indexOf("}", i);
+      if (end !== -1) {
+        const options = pattern
+          .slice(i + 1, end)
+          .split(",")
+          .map((part) => [...part].map(escapeChar).join(""))
+          .join("|");
+        source += `(?:${options})`;
+        i = end + 1;
+        continue;
+      }
+    }
+    source += escapeChar(char);
+    i += 1;
+  }
+  return new RegExp(`^${source}$`, "u");
+};
+
+/** Whether a ref matches any of the include globs. */
+const matchesInclude = (ref: string, patterns: string[]): boolean =>
+  patterns.some((pattern) => globToRegExp(pattern).test(ref));
+
+/** A file to fetch: its source-local ref plus where to read it from. */
+interface RemoteRef {
+  ref: string;
+  fetchUrl: string;
+  editUrl?: string;
+}
+
+const githubHeaders = (): Record<string, string> => {
+  const token = process.env.GITHUB_TOKEN;
+  return token ? { authorization: `Bearer ${token}` } : {};
+};
+
+interface GithubTreeEntry {
+  path: string;
+  type: string;
+}
+
+/** Enumerate a GitHub repo subtree, mapping blobs to remote refs. */
+const enumerateGithub = async (
+  github: { owner: string; repo: string; ref: string; path: string },
+  include: string[],
+  doFetch: typeof fetch
+): Promise<RemoteRef[]> => {
+  const { owner, repo, ref } = github;
+  const base = github.path.replaceAll(/^\/|\/$/gu, "");
+  const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`;
+  const res = await doFetch(treeUrl, { headers: githubHeaders() });
+  if (!res.ok) {
+    throw new Error(`${treeUrl} -> ${res.status}`);
+  }
+  const body = (await res.json()) as { tree?: GithubTreeEntry[] };
+  const prefix = base ? `${base}/` : "";
+  return (body.tree ?? [])
+    .filter((node) => node.type === "blob" && node.path.startsWith(prefix))
+    .map((node) => node.path.slice(prefix.length))
+    .filter((rel) => matchesInclude(rel, include))
+    .map((rel) => ({
+      editUrl: `https://github.com/${owner}/${repo}/edit/${ref}/${prefix}${rel}`,
+      fetchUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${prefix}${rel}`,
+      ref: rel,
+    }));
+};
+
+/**
+ * Remote Markdown/MDX content source. Fetches raw `.md`/`.mdx` over HTTP and
+ * passes the text straight through `normalizeEntry`. A snapshot under
+ * `.blume/cache/<source>/` makes rebuilds offline-tolerant.
+ */
+export const mdxRemoteSource = (
+  options: MdxRemoteSourceOptions,
+  ctx: SourceContext
+): ContentSource => {
+  const doFetch = options.fetchImpl ?? globalThis.fetch;
+  const cache = snapshotCache(ctx.cacheDir);
+  let snapshot = new Map<string, SourceEntry>();
+
+  const enumerate = async (): Promise<RemoteRef[]> => {
+    if (options.github) {
+      return await enumerateGithub(options.github, options.include, doFetch);
+    }
+    if (options.files && options.url) {
+      const base = options.url.replace(/\/$/u, "");
+      return options.files
+        .filter((ref) => matchesInclude(ref, options.include))
+        .map((ref) => ({
+          editUrl: `${base}/${ref}`,
+          fetchUrl: `${base}/${ref}`,
+          ref,
+        }));
+    }
+    throw new BlumeError({
+      code: "BLUME_SOURCE_MISCONFIGURED",
+      message: `Source "${options.name}" needs either { github } or { url, files }.`,
+      severity: "error",
+    });
+  };
+
+  const fetchEntry = async (item: RemoteRef): Promise<SourceEntry> => {
+    const res = await doFetch(item.fetchUrl, { headers: githubHeaders() });
+    if (!res.ok) {
+      throw new Error(`${item.fetchUrl} -> ${res.status}`);
+    }
+    const text = await res.text();
+    const parsed = matter(text);
+    const format = item.ref.toLowerCase().endsWith(".mdx") ? "mdx" : "md";
+    return {
+      body: { format, text: parsed.content },
+      data: parsed.data,
+      editUrl: item.editUrl,
+      hash: hashText(text),
+      raw: text,
+      ref: item.ref,
+    };
+  };
+
+  const load = async (): Promise<SourceLoadResult> => {
+    const result = await loadWithCache(
+      options.name,
+      cache,
+      async () => {
+        const refs = await enumerate();
+        return await Promise.all(refs.map(fetchEntry));
+      },
+      ctx.refresh ?? true
+    );
+    snapshot = new Map(result.entries.map((entry) => [entry.ref, entry]));
+    return result;
+  };
+
+  const read = async (ref: string): Promise<string> => {
+    const cached = snapshot.get(ref);
+    if (cached) {
+      return cached.raw ?? cached.body.text;
+    }
+    const all = await cache.read();
+    const entry = all.find((e) => e.ref === ref);
+    return entry?.raw ?? entry?.body.text ?? "";
+  };
+
+  return {
+    load,
+    name: options.name,
+    prefix: options.prefix,
+    read,
+    staged: true,
+    watch: options.pollInterval
+      ? pollingWatch(load, options.pollInterval)
+      : undefined,
+  };
+};

package/src/core/sources/mintlify.ts

@@ -0,0 +1,161 @@
+import { existsSync, watch as fsWatch } from "node:fs";
+import { readFile } from "node:fs/promises";
+
+import matter from "gray-matter";
+import { isAbsolute, join, relative, resolve } from "pathe";
+import { glob } from "tinyglobby";
+
+import { transformMintlifyContent } from "../../migrate/mintlify/transform.ts";
+import { BlumeError } from "../diagnostics.ts";
+import type { Diagnostic } from "../types.ts";
+import type { ContentSource, SourceEntry, SourceLoadResult } from "./types.ts";
+
+/** Options for the Mintlify bridge content source. */
+export interface MintlifySourceOptions {
+  /** Stable source name; namespaces ids and diagnostics. */
+  name: string;
+  /** Optional route prefix. */
+  prefix?: string;
+  /** Content root, absolute or relative to `projectRoot` (Mintlify: `.`). */
+  root: string;
+  include: string[];
+  exclude: string[];
+  /** `docs.json` variables, inlined into content (`{{name}}`) at scan time. */
+  variables: Record<string, string>;
+  /** Absolute path of the `docs.json`/`mint.json`, watched for changes in dev. */
+  configFile?: string;
+  /** Absolute project root, used to resolve a relative `root`. */
+  projectRoot: string;
+}
+
+/**
+ * Folders Mintlify projects keep alongside content that are never pages:
+ * snippets are inlined as includes, and build/tooling dirs are noise. Merged
+ * with the user's `exclude` so bridge mode behaves like the one-shot migrator.
+ */
+const MINTLIFY_SOURCE_IGNORES = [
+  "node_modules/**",
+  ".blume/**",
+  "dist/**",
+  "build/**",
+  "public/**",
+  "snippets/**",
+];
+
+/**
+ * The Mintlify bridge content source. Reads an unconverted Mintlify project in
+ * place and transforms each page to Blume MDX at scan time (callouts → `:::`
+ * directives, snippet/variable inlining, etc.) via `transformMintlifyContent`.
+ * Staged: the transformed bodies are materialized under `.blume/content` and
+ * rendered through Astro's `staged` collection, so the rewrites actually reach
+ * the output. Components Blume already ships (Card, Tabs, Steps, …) render as-is.
+ */
+export const mintlifySource = (
+  options: MintlifySourceOptions
+): ContentSource & { readonly contentRoot: string } => {
+  const contentRoot = isAbsolute(options.root)
+    ? options.root
+    : join(resolve(options.projectRoot), options.root);
+  const ignore = [...new Set([...options.exclude, ...MINTLIFY_SOURCE_IGNORES])];
+
+  const transform = (
+    raw: string,
+    file: string
+  ): ReturnType<typeof transformMintlifyContent> =>
+    transformMintlifyContent(raw, {
+      filePath: file,
+      root: resolve(options.projectRoot),
+      variables: options.variables,
+    });
+
+  const load = async (): Promise<SourceLoadResult> => {
+    const files = await glob(options.include, {
+      absolute: true,
+      cwd: contentRoot,
+      ignore,
+      onlyFiles: true,
+    });
+    files.sort();
+
+    const unsupported = new Set<string>();
+    const entries = await Promise.all(
+      files.map(async (file): Promise<SourceEntry> => {
+        const result = await transform(await readFile(file, "utf-8"), file);
+        for (const name of result.unsupported) {
+          unsupported.add(name);
+        }
+        const parsed = matter(result.content);
+        // Force MDX: Mintlify pages are MDX-authored and the rewrites emit `:::`
+        // directives + JSX, neither of which the plain `.md` processor expands.
+        return {
+          body: { format: "mdx", text: parsed.content },
+          data: parsed.data,
+          raw: result.content,
+          ref: relative(contentRoot, file),
+          sourcePath: file,
+        };
+      })
+    );
+
+    const diagnostics: Diagnostic[] =
+      unsupported.size > 0
+        ? [
+            {
+              code: "BLUME_MINTLIFY_UNSUPPORTED",
+              message: `Mintlify components without a Blume equivalent were left as-is: ${[...unsupported].toSorted().join(", ")}. Use the OpenAPI reference for API parameters.`,
+              severity: "warning",
+            },
+          ]
+        : [];
+
+    return { diagnostics, entries };
+  };
+
+  const validate = (): void => {
+    if (!existsSync(contentRoot)) {
+      throw new BlumeError({
+        code: "BLUME_CONTENT_ROOT_MISSING",
+        file: contentRoot,
+        message: `Content root not found: ${options.root}`,
+        severity: "error",
+        suggestion: `Run "blume dev" from the directory that contains docs.json.`,
+      });
+    }
+  };
+
+  const watch = (onChange: () => void): (() => void) => {
+    const disposers: (() => void)[] = [];
+    if (existsSync(contentRoot)) {
+      const watcher = fsWatch(contentRoot, { recursive: true }, onChange);
+      disposers.push(() => watcher.close());
+    }
+    // Watch docs.json directly: it lives at the content root but a non-recursive
+    // single-file watch fires reliably on edits that recursive dir-watch can miss.
+    if (options.configFile && existsSync(options.configFile)) {
+      const watcher = fsWatch(options.configFile, onChange);
+      disposers.push(() => watcher.close());
+    }
+    return () => {
+      for (const dispose of disposers) {
+        dispose();
+      }
+    };
+  };
+
+  const read = async (ref: string): Promise<string> => {
+    const file = join(contentRoot, ref);
+    const result = await transform(await readFile(file, "utf-8"), file);
+    return result.content;
+  };
+
+  return {
+    contentRoot,
+    load,
+    name: options.name,
+    prefix: options.prefix,
+    read,
+    staged: true,
+    validate,
+    watch,
+  };
+};