blume 0.0.0 → 0.1.0

package/src/core/sources/portable-text.ts

@@ -0,0 +1,143 @@
+/**
+ * A minimal Portable Text → Markdown serializer. Covers the standard block,
+ * list, and mark types Sanity emits; custom block/mark types fall through to a
+ * user-supplied serializer or are skipped with a noted comment. Output is
+ * Markdown text that flows through Blume's normal pipeline.
+ */
+
+/** A single Portable Text node (block, image, or a custom type). */
+export interface PortableTextBlock {
+  _type: string;
+  _key?: string;
+  style?: string;
+  listItem?: string;
+  level?: number;
+  children?: PortableTextSpan[];
+  markDefs?: PortableTextMarkDef[];
+  [key: string]: unknown;
+}
+
+interface PortableTextSpan {
+  _type: string;
+  text?: string;
+  marks?: string[];
+}
+
+interface PortableTextMarkDef {
+  _key: string;
+  _type: string;
+  href?: string;
+}
+
+export interface PortableTextOptions {
+  /** Resolve an image block to a URL (the adapter knows project/dataset). */
+  imageUrl?: (block: PortableTextBlock) => string | null;
+  /** Custom block-type serializers, keyed by `_type`; return Markdown/MDX. */
+  serializers?: Record<string, (block: PortableTextBlock) => string>;
+}
+
+const HEADING_STYLES: Record<string, string> = {
+  h1: "# ",
+  h2: "## ",
+  h3: "### ",
+  h4: "#### ",
+  h5: "##### ",
+  h6: "###### ",
+};
+
+/** Wrap a span's text in Markdown for its marks (decorators + link defs). */
+const renderSpan = (
+  span: PortableTextSpan,
+  defs: Map<string, PortableTextMarkDef>
+): string => {
+  let text = span.text ?? "";
+  if (!span.marks || span.marks.length === 0) {
+    return text;
+  }
+  // Decorators wrap inline; a link def wraps last so its label keeps emphasis.
+  let link: PortableTextMarkDef | undefined;
+  for (const mark of span.marks) {
+    switch (mark) {
+      case "strong": {
+        text = `**${text}**`;
+        break;
+      }
+      case "em": {
+        text = `*${text}*`;
+        break;
+      }
+      case "code": {
+        text = `\`${text}\``;
+        break;
+      }
+      case "strike-through": {
+        text = `~~${text}~~`;
+        break;
+      }
+      default: {
+        const def = defs.get(mark);
+        if (def?._type === "link") {
+          link = def;
+        }
+      }
+    }
+  }
+  return link?.href ? `[${text}](${link.href})` : text;
+};
+
+/** Render the inline children of a block to a single Markdown string. */
+const renderChildren = (block: PortableTextBlock): string => {
+  const defs = new Map(
+    (block.markDefs ?? []).map((def) => [def._key, def] as const)
+  );
+  return (block.children ?? []).map((span) => renderSpan(span, defs)).join("");
+};
+
+const renderBlock = (
+  block: PortableTextBlock,
+  options: PortableTextOptions
+): string => {
+  const custom = options.serializers?.[block._type];
+  if (custom) {
+    return custom(block);
+  }
+  if (block._type === "image") {
+    const url = options.imageUrl?.(block);
+    const alt = typeof block.alt === "string" ? block.alt : "";
+    return url ? `![${alt}](${url})` : "";
+  }
+  if (block._type !== "block") {
+    return `<!-- unsupported Portable Text block: ${block._type} -->`;
+  }
+
+  const inline = renderChildren(block);
+  if (block.listItem) {
+    const indent = "  ".repeat(Math.max(0, (block.level ?? 1) - 1));
+    const marker = block.listItem === "number" ? "1." : "-";
+    return `${indent}${marker} ${inline}`;
+  }
+  if (block.style === "blockquote") {
+    return `> ${inline}`;
+  }
+  return `${HEADING_STYLES[block.style ?? "normal"] ?? ""}${inline}`;
+};
+
+/** Serialize a Portable Text array into a Markdown string. */
+export const portableTextToMarkdown = (
+  blocks: PortableTextBlock[],
+  options: PortableTextOptions = {}
+): string => {
+  const lines = blocks.map((block) => renderBlock(block, options));
+  // List items are single-newline separated; everything else gets a blank line.
+  const out: string[] = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i] ?? "";
+    const prevList = Boolean(blocks[i - 1]?.listItem);
+    const thisList = Boolean(blocks[i]?.listItem);
+    if (i > 0) {
+      out.push(prevList && thisList ? "\n" : "\n\n");
+    }
+    out.push(line);
+  }
+  return `${out.join("")}\n`;
+};

package/src/core/sources/read.ts

@@ -0,0 +1,36 @@
+import { readFile } from "node:fs/promises";
+
+import type { PageRecord } from "../types.ts";
+import type { ContentSource } from "./types.ts";
+
+/** The subset of a scanned project the entry reader needs. */
+export interface EntryReadContext {
+  /** The instantiated sources, keyed for `read()` lookups. */
+  sources?: ContentSource[];
+}
+
+/**
+ * Read an entry's raw body text without assuming a filesystem path. Prefers the
+ * body captured at scan time (staged sources), then the owning source's lazy
+ * `read()`, then the back-compat `sourcePath` (filesystem). Returns `""` when an
+ * entry can't be resolved, so callers degrade gracefully.
+ */
+export const readEntryText = async (
+  ctx: EntryReadContext,
+  page: PageRecord
+): Promise<string> => {
+  if (page.body) {
+    return page.body.text;
+  }
+  const name = page.source?.name;
+  const source = name
+    ? ctx.sources?.find((candidate) => candidate.name === name)
+    : undefined;
+  if (source?.read) {
+    return await source.read(page.source.ref);
+  }
+  if (page.sourcePath) {
+    return await readFile(page.sourcePath, "utf-8");
+  }
+  return "";
+};

package/src/core/sources/resolve.ts

@@ -0,0 +1,158 @@
+import { join } from "pathe";
+
+import type { ContentSourceConfig, ResolvedConfig } from "../schema.ts";
+import type { ProjectContext } from "../types.ts";
+import { filesystemSource } from "./filesystem.ts";
+import { mdxRemoteSource } from "./mdx-remote.ts";
+import { mintlifySource } from "./mintlify.ts";
+import { notionSource } from "./notion.ts";
+import { sanitySource } from "./sanity.ts";
+import type { ContentSource, SourceContext } from "./types.ts";
+
+/** Allocate a unique, stable source name from a base (prefix or type). */
+const uniqueNamer = (): ((base: string) => string) => {
+  const used = new Set<string>();
+  return (base) => {
+    let name = base;
+    let n = 2;
+    while (used.has(name)) {
+      name = `${base}-${n}`;
+      n += 1;
+    }
+    used.add(name);
+    return name;
+  };
+};
+
+/** Runtime knobs for a scan: dev/build mode, preview, and cache refresh. */
+export interface SourceRuntime {
+  mode: "dev" | "build";
+  preview?: boolean;
+  refresh?: boolean;
+}
+
+const sourceContext = (
+  context: ProjectContext,
+  name: string,
+  runtime: SourceRuntime
+): SourceContext => ({
+  assetsBaseUrl: `/blume-assets/${name}`,
+  assetsDir: join(context.outDir, "public", "blume-assets", name),
+  cacheDir: join(context.outDir, "cache", name),
+  mode: runtime.mode,
+  preview: runtime.preview,
+  projectRoot: context.root,
+  refresh: runtime.refresh ?? runtime.mode === "build",
+});
+
+const buildSource = (
+  def: ContentSourceConfig,
+  name: string,
+  context: ProjectContext,
+  runtime: SourceRuntime
+): ContentSource => {
+  if (def.type === "filesystem") {
+    return filesystemSource({
+      exclude: def.exclude,
+      include: def.include,
+      name,
+      prefix: def.prefix,
+      projectRoot: context.root,
+      root: def.root,
+    });
+  }
+  if (def.type === "mintlify") {
+    return mintlifySource({
+      configFile: def.configFile,
+      exclude: def.exclude,
+      include: def.include,
+      name,
+      prefix: def.prefix,
+      projectRoot: context.root,
+      root: def.root,
+      variables: def.variables,
+    });
+  }
+  if (def.type === "custom") {
+    // A user-provided instance manages its own context/caching; we only ensure
+    // its name is unique across the project for id namespacing.
+    return def.source.name === name ? def.source : { ...def.source, name };
+  }
+  if (def.type === "sanity") {
+    return sanitySource(
+      {
+        apiVersion: def.apiVersion,
+        dataset: def.dataset,
+        fields: def.fields,
+        name,
+        pollInterval: def.pollInterval,
+        prefix: def.prefix,
+        projectId: def.projectId,
+        query: def.query,
+      },
+      sourceContext(context, name, runtime)
+    );
+  }
+  if (def.type === "notion") {
+    return notionSource(
+      {
+        database: def.database,
+        name,
+        pollInterval: def.pollInterval,
+        prefix: def.prefix,
+        properties: def.properties,
+        publishedValue: def.publishedValue,
+      },
+      sourceContext(context, name, runtime)
+    );
+  }
+  return mdxRemoteSource(
+    {
+      files: def.files,
+      github: def.github,
+      include: def.include,
+      name,
+      pollInterval: def.pollInterval,
+      prefix: def.prefix,
+      url: def.url,
+    },
+    sourceContext(context, name, runtime)
+  );
+};
+
+/** The base name to allocate for a source config (before deduplication). */
+const baseName = (def: ContentSourceConfig): string => {
+  if (def.type === "custom") {
+    return def.source.name;
+  }
+  return def.prefix ?? def.type;
+};
+
+/**
+ * Build the ordered list of content sources for a project. With no
+ * `content.sources` configured, the top-level `root`/`include`/`exclude` desugar
+ * to a single implicit filesystem source, so existing projects are untouched.
+ */
+export const resolveSources = (
+  config: ResolvedConfig,
+  context: ProjectContext,
+  runtime: SourceRuntime
+): ContentSource[] => {
+  const defs = config.content.sources;
+  if (!defs || defs.length === 0) {
+    return [
+      filesystemSource({
+        exclude: config.content.exclude,
+        include: config.content.include,
+        name: "filesystem",
+        projectRoot: context.root,
+        root: config.content.root,
+      }),
+    ];
+  }
+
+  const nameFor = uniqueNamer();
+  return defs.map((def) =>
+    buildSource(def, nameFor(baseName(def)), context, runtime)
+  );
+};

package/src/core/sources/sanity.ts

@@ -0,0 +1,218 @@
+import matter from "gray-matter";
+import { join } from "pathe";
+
+import { BlumeError } from "../diagnostics.ts";
+import {
+  hashText,
+  loadWithCache,
+  pollingWatch,
+  snapshotCache,
+} from "./cache.ts";
+import { slugify } from "./normalize.ts";
+import { portableTextToMarkdown } from "./portable-text.ts";
+import type { PortableTextBlock } from "./portable-text.ts";
+import type {
+  ContentSource,
+  SourceContext,
+  SourceEntry,
+  SourceLoadResult,
+} from "./types.ts";
+
+/** The slice of `@sanity/client` this adapter relies on (so it's mockable). */
+export interface SanityClientLike {
+  fetch: <T = unknown>(query: string) => Promise<T>;
+}
+
+/** Field paths mapping a Sanity document onto Blume meta + body. */
+export interface SanityFieldMap {
+  /** Frontmatter title; default `title`. */
+  title?: string;
+  /** Frontmatter description; default `description`. */
+  description?: string;
+  /** Route slug (dot path); default `slug.current`. */
+  slug?: string;
+  /** Portable Text body field; default `body`. */
+  body?: string;
+  /** Last-modified ISO date; default `_updatedAt`. */
+  lastModified?: string;
+}
+
+export interface SanitySourceOptions {
+  name: string;
+  prefix?: string;
+  projectId: string;
+  dataset: string;
+  /** Sanity API version (a date); default `2024-01-01`. */
+  apiVersion?: string;
+  /** GROQ query selecting the documents to import. */
+  query: string;
+  fields?: SanityFieldMap;
+  /** Custom Portable Text block serializers, keyed by `_type`. */
+  serializers?: Record<string, (block: PortableTextBlock) => string>;
+  /** Read token for private datasets; defaults to `SANITY_TOKEN`. */
+  token?: string;
+  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
+  pollInterval?: number;
+  /** Injected for tests; otherwise built from `@sanity/client`. */
+  client?: SanityClientLike;
+}
+
+const IMAGE_REF = /^image-(?<id>[a-f0-9]+)-(?<dims>\d+x\d+)-(?<ext>\w+)$/u;
+
+/** Resolve a dot path (`slug.current`) against a document. */
+const getPath = (doc: Record<string, unknown>, path: string): unknown => {
+  let current: unknown = doc;
+  for (const key of path.split(".")) {
+    if (current && typeof current === "object") {
+      current = (current as Record<string, unknown>)[key];
+    } else {
+      return;
+    }
+  }
+  return current;
+};
+
+const asString = (value: unknown): string | undefined =>
+  typeof value === "string" ? value : undefined;
+
+/** Build a Sanity CDN URL from an image asset `_ref`. */
+const imageUrlFromRef = (
+  ref: string,
+  projectId: string,
+  dataset: string
+): string | null => {
+  const match = ref.match(IMAGE_REF);
+  if (!match?.groups) {
+    return null;
+  }
+  const { id, dims, ext } = match.groups;
+  return `https://cdn.sanity.io/images/${projectId}/${dataset}/${id}-${dims}.${ext}`;
+};
+
+const resolveClient = async (
+  options: SanitySourceOptions,
+  preview: boolean
+): Promise<SanityClientLike> => {
+  if (options.client) {
+    return options.client;
+  }
+  let createClient: (config: Record<string, unknown>) => SanityClientLike;
+  try {
+    ({ createClient } = (await import("@sanity/client")) as {
+      createClient: (config: Record<string, unknown>) => SanityClientLike;
+    });
+  } catch {
+    throw new BlumeError({
+      code: "BLUME_SOURCE_SDK_MISSING",
+      message: `Source "${options.name}" needs "@sanity/client". Install it (e.g. \`npm install @sanity/client\`).`,
+      severity: "error",
+    });
+  }
+  return createClient({
+    apiVersion: options.apiVersion ?? "2024-01-01",
+    dataset: options.dataset,
+    // Preview reads draft documents through the API; published builds use the CDN.
+    perspective: preview ? "previewDrafts" : "published",
+    projectId: options.projectId,
+    token: options.token ?? process.env.SANITY_TOKEN,
+    useCdn: !preview,
+  });
+};
+
+/**
+ * Sanity content source. Runs a GROQ query, maps each document's fields to Blume
+ * frontmatter and its Portable Text body to Markdown, and stages the result.
+ */
+export const sanitySource = (
+  options: SanitySourceOptions,
+  ctx?: SourceContext
+): ContentSource => {
+  const fields = options.fields ?? {};
+  // When constructed directly (custom-source SPI) without a context, cache under
+  // a name-derived dir relative to the project; the built-in type passes a ctx.
+  const cache = snapshotCache(
+    ctx?.cacheDir ?? join(".blume", "cache", options.name)
+  );
+  let snapshot = new Map<string, SourceEntry>();
+
+  const toEntry = (doc: Record<string, unknown>): SourceEntry => {
+    const slugValue =
+      asString(getPath(doc, fields.slug ?? "slug.current")) ??
+      asString(doc._id) ??
+      "untitled";
+    const slug = slugify(slugValue) || "untitled";
+
+    const data: Record<string, unknown> = {};
+    const title = asString(getPath(doc, fields.title ?? "title"));
+    const description = asString(
+      getPath(doc, fields.description ?? "description")
+    );
+    if (title) {
+      data.title = title;
+    }
+    if (description) {
+      data.description = description;
+    }
+
+    const blocks = (getPath(doc, fields.body ?? "body") ??
+      []) as PortableTextBlock[];
+    const markdown = Array.isArray(blocks)
+      ? portableTextToMarkdown(blocks, {
+          imageUrl: (block) => {
+            const ref = (block.asset as { _ref?: string } | undefined)?._ref;
+            return ref
+              ? imageUrlFromRef(ref, options.projectId, options.dataset)
+              : null;
+          },
+          serializers: options.serializers,
+        })
+      : "";
+
+    const raw = matter.stringify(markdown, data);
+    return {
+      body: { format: "md", text: markdown },
+      data,
+      hash: hashText(raw),
+      lastModified: asString(getPath(doc, fields.lastModified ?? "_updatedAt")),
+      raw,
+      ref: `${slug}.md`,
+    };
+  };
+
+  const load = async (): Promise<SourceLoadResult> => {
+    const result = await loadWithCache(
+      options.name,
+      cache,
+      async () => {
+        const client = await resolveClient(options, ctx?.preview ?? false);
+        const docs = await client.fetch<Record<string, unknown>[]>(
+          options.query
+        );
+        return docs.map(toEntry);
+      },
+      ctx?.refresh ?? true
+    );
+    snapshot = new Map(result.entries.map((entry) => [entry.ref, entry]));
+    return result;
+  };
+
+  const read = async (ref: string): Promise<string> => {
+    const cached = snapshot.get(ref);
+    if (cached) {
+      return cached.raw ?? cached.body.text;
+    }
+    const all = await cache.read();
+    return all.find((e) => e.ref === ref)?.raw ?? "";
+  };
+
+  return {
+    load,
+    name: options.name,
+    prefix: options.prefix,
+    read,
+    staged: true,
+    watch: options.pollInterval
+      ? pollingWatch(load, options.pollInterval)
+      : undefined,
+  };
+};

package/src/core/sources/types.ts

@@ -0,0 +1,105 @@
+import type { ResolvedI18nConfig } from "../schema.ts";
+import type { Diagnostic } from "../types.ts";
+
+/**
+ * A single content item, normalized by a source adapter. Adapters lower their
+ * native shape (files, Portable Text, Notion blocks, remote HTML) to Markdown/MDX
+ * *text* so Blume's markdown processors and component set apply uniformly.
+ */
+export interface SourceEntry {
+  /** Source-local stable id, e.g. `api/auth.mdx` or a CMS document id. */
+  ref: string;
+  /** Logical route input; defaults to `ref` if omitted. May include slashes. */
+  slug?: string;
+  /** Frontmatter-equivalent metadata, validated against the Blume meta schema. */
+  data: Record<string, unknown>;
+  /** The renderable body as Markdown/MDX source text (frontmatter stripped). */
+  body: { format: "md" | "mdx"; text: string };
+  /**
+   * Full source text including frontmatter, written verbatim to the staging
+   * dir so Astro re-parses the same frontmatter. Staged (non-filesystem) sources
+   * set this; filesystem entries (read from disk) omit it.
+   */
+  raw?: string;
+  /**
+   * Absolute filesystem path when the entry originates from disk. Populated by
+   * the filesystem adapter only; powers git last-modified and edit URLs and the
+   * `sourcePath` back-compat window. Omitted by remote/CMS adapters.
+   */
+  sourcePath?: string;
+  /** Optional provenance for "edit this page". */
+  editUrl?: string;
+  /** Optional last-modified ISO date supplied by the adapter (non-filesystem). */
+  lastModified?: string;
+  /** Content hash for cache invalidation / HMR; adapter-computed when cheap. */
+  hash?: string;
+}
+
+/** The result of a single `ContentSource.load()` call. */
+export interface SourceLoadResult {
+  entries: SourceEntry[];
+  /** Source-level diagnostics (e.g. an offline cache fallback warning). */
+  diagnostics: Diagnostic[];
+}
+
+/**
+ * Per-source runtime context, handed to an adapter factory at construction so
+ * `load`/`read`/`watch` can close over it without re-threading on every call.
+ */
+export interface SourceContext {
+  projectRoot: string;
+  /** Per-source cache dir under `.blume/cache/<source>/`. */
+  cacheDir: string;
+  mode: "dev" | "build";
+  /** Dir for downloaded assets (served from the site's public dir). */
+  assetsDir?: string;
+  /** Public URL prefix the downloaded assets are served under. */
+  assetsBaseUrl?: string;
+  /**
+   * Re-fetch remote content instead of serving the cached snapshot. True for
+   * builds and `blume sync`; false in dev (cache-first for fast, offline-tolerant
+   * restarts — refresh with `blume sync` or an opt-in `pollInterval`).
+   */
+  refresh?: boolean;
+  /**
+   * Preview unpublished content: drafts are kept and CMS adapters fetch draft
+   * documents (Sanity's `previewDrafts` perspective). Off for production builds.
+   */
+  preview?: boolean;
+}
+
+/**
+ * A pluggable content source. Adapters enumerate normalized entries and
+ * (optionally) re-read a single entry, validate themselves, and watch for
+ * changes. Identity is the opaque `SourceEntry.ref`, never a filesystem path.
+ */
+export interface ContentSource {
+  /** Unique, stable name; used for id namespacing and diagnostics. */
+  readonly name: string;
+  /**
+   * Whether entries render through the staging collection. Filesystem sources
+   * render through Astro's existing `docs` glob collection (`false`); every
+   * other source materializes MDX into `.blume/content` (`true`).
+   */
+  readonly staged: boolean;
+  /** Optional route prefix; the source's routes namespace under `/<prefix>/`. */
+  readonly prefix?: string;
+  /** Pull every entry. Called once per scan. */
+  load: () => Promise<SourceLoadResult>;
+  /** Validate the source is usable; throws a BlumeError when not. */
+  validate?: () => void;
+  /** Read a single entry's body lazily (search / AI / raw export). */
+  read?: (ref: string) => Promise<string>;
+  /**
+   * Notify on change in dev. Returns a disposer. Filesystem uses `fs.watch`;
+   * remote/static sources omit it (content is frozen for the session).
+   */
+  watch?: (onChange: () => void) => () => void;
+}
+
+/** Context passed to `normalizeEntry`, describing the owning source. */
+export interface NormalizeContext {
+  source: { name: string; prefix?: string; staged: boolean };
+  defaultType: string;
+  i18n?: ResolvedI18nConfig;
+}