blume 1.7.3 → 2.0.0

package/src/core/sources/notion.ts

@@ -6,6 +6,7 @@ import { join } from "pathe";
 import { parseYouTubeId } from "../../components/content/youtube.ts";
 import { BlumeError } from "../diagnostics.ts";
 import matter from "../frontmatter.ts";
+import { nodeRequire } from "../node-require.ts";
 import type { Diagnostic } from "../types.ts";
 import { materializeAssets } from "./assets.ts";
 import {
@@ -14,6 +15,7 @@ import {
   pollingWatch,
   snapshotCache,
 } from "./cache.ts";
+import { codeFence, image, renderInline, renderLink } from "./lower.ts";
 import { slugifyPath } from "./normalize.ts";
 import type {
   ContentSource,
@@ -152,24 +154,25 @@ interface NotionFrontmatter {
   [key: string]: boolean | string | { order: number } | undefined;
 }
 
+/**
+ * Rich text as Markdown for an MDX body. Each run goes through the shared
+ * lowering, so a literal `{`, `<`, `*`, or `[` typed in Notion renders as
+ * itself instead of opening a JSX expression, a tag, or emphasis, and a code
+ * run keeps its text verbatim inside a long-enough code span.
+ */
 const richToMarkdown = (rich: NotionRichText[] = []): string =>
   rich
-    .map((node) => {
-      let text = node.plain_text;
-      if (node.annotations?.code) {
-        text = `\`${text}\``;
-      }
-      if (node.annotations?.bold) {
-        text = `**${text}**`;
-      }
-      if (node.annotations?.italic) {
-        text = `*${text}*`;
-      }
-      if (node.annotations?.strikethrough) {
-        text = `~~${text}~~`;
-      }
-      return node.href ? `[${text}](${node.href})` : text;
-    })
+    .map(({ annotations = {}, href, plain_text: text }) =>
+      renderLink(
+        renderInline(text, {
+          bold: annotations.bold,
+          code: annotations.code,
+          italic: annotations.italic,
+          strike: annotations.strikethrough,
+        }),
+        href ?? undefined
+      )
+    )
     .join("");
 
 const isBlockPayload = (
@@ -334,11 +337,12 @@ const renderLeaf = (block: NotionBlock): string | null => {
       return "---";
     }
     case "code": {
-      return `\`\`\`${data.language ?? ""}\n${text}\n\`\`\``;
+      // The code's own text, verbatim: escapes would show inside the fence.
+      return codeFence(richToPlain(blockField(block)), data.language ?? "");
     }
     case "image": {
       const url = data.external?.url ?? data.file?.url;
-      return url ? `![${richToMarkdown(data.caption)}](${url})` : "";
+      return url ? image(richToPlain(data.caption), url) : "";
     }
     case "video": {
       return renderVideo(data);
@@ -369,6 +373,8 @@ export const notionSource = (
   // limited API, and a slow video must not hold API slots. One gate per source
   // bounds the fan-out across every page, not just within one.
   const downloads = pLimit(ASSET_DOWNLOAD_CONCURRENCY);
+  // Shared with the gate so two pages naming one asset share one download.
+  const inFlight = new Map<string, Promise<string>>();
   // Every Notion API call goes through the limiter, inside the retry — so a
   // call sleeping through a backoff doesn't hold a slot while it waits.
   const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
@@ -381,16 +387,18 @@ export const notionSource = (
   const assetsBaseUrl = ctx?.assetsBaseUrl ?? `/blume-assets/${options.name}`;
   let snapshot = new Map<string, SourceEntry>();
 
-  const resolveClient = async (): Promise<NotionClientLike> => {
+  const resolveClient = (): NotionClientLike => {
     if (options.client) {
       return options.client;
     }
     let Client: new (config: { auth?: string }) => NotionClientLike;
     try {
-      // The module is widened first because the SDK's response unions are
-      // broader than the NotionClientLike slice, so a direct assertion is
-      // rejected as non-overlapping.
-      const sdk: unknown = await import("@notionhq/client");
+      // `require`, not `import()`: an ejected app scans in
+      // `astro:build:done` (see `core/node-require.ts`). The module is
+      // widened first because the SDK's response unions are broader than the
+      // NotionClientLike slice, so a direct assertion is rejected as
+      // non-overlapping.
+      const sdk: unknown = nodeRequire("@notionhq/client");
       // SAFETY: `@notionhq/client` exports a `Client` class constructable with
       // an `auth` token whose instances cover the NotionClientLike slice; the
       // local type keeps the SDK mockable without importing its types.
@@ -438,7 +446,8 @@ export const notionSource = (
       return `<Callout>\n${body}\n</Callout>`;
     }
     if (block.type === "toggle") {
-      const title = jsxString(richToMarkdown(blockField(block)));
+      // The item's title renders as text, not Markdown.
+      const title = jsxString(richToPlain(blockField(block)));
       return `<Accordion>\n<AccordionItem title=${title}>\n${await children(block)}\n</AccordionItem>\n</Accordion>`;
     }
     if (block.type === "column_list") {
@@ -532,11 +541,12 @@ export const notionSource = (
 
   const frontmatter = (page: NotionPage) => {
     const data: NotionFrontmatter = {};
-    const title = richToMarkdown(titleProperty(page)?.title);
+    // Frontmatter holds plain text: Markdown marks or escapes would show.
+    const title = richToPlain(titleProperty(page)?.title);
     if (title) {
       data.title = title;
     }
-    const description = richToMarkdown(
+    const description = richToPlain(
       page.properties[props.description ?? "Description"]?.rich_text
     );
     if (description) {
@@ -549,7 +559,7 @@ export const notionSource = (
     if (order !== undefined) {
       data.sidebar = { order };
     }
-    const slugProp = richToMarkdown(
+    const slugProp = richToPlain(
       page.properties[props.slug ?? "Slug"]?.rich_text
     );
     // Path-aware: a `guides/setup` slug keeps its `/` (per-segment slugging)
@@ -568,6 +578,7 @@ export const notionSource = (
       assetsBaseUrl,
       assetsDir,
       fetchImpl: options.fetchImpl,
+      inFlight,
       limit: downloads,
     });
     const raw = matter.stringify(assets.markdown, data);
@@ -625,7 +636,7 @@ export const notionSource = (
       options.name,
       cache,
       async () => {
-        const client = await resolveClient();
+        const client = resolveClient();
         const dataSourceId = await resolveDataSource(client);
         const pages = await collectAll((cursor) =>
           queryDataSource(client, dataSourceId, cursor)

package/src/core/sources/payload.ts

@@ -0,0 +1,135 @@
+import type { JsonObject } from "./json.ts";
+import { asObject, asString, isJsonObject, objectsIn } from "./json.ts";
+import { lexicalToMarkdown } from "./lexical.ts";
+import { writesMdx } from "./lower.ts";
+import type { RemoteFieldMap, RestClient } from "./remote.ts";
+import {
+  documentEntry,
+  fetchJson,
+  queryString,
+  remoteSource,
+} from "./remote.ts";
+import type { ContentSource, SourceContext, SourceEntry } from "./types.ts";
+
+export interface PayloadSourceOptions {
+  name: string;
+  prefix?: string;
+  /** The Payload app's origin (`https://cms.acme.dev`); its REST API is at `/api`. */
+  url: string;
+  /** The collection slug whose documents become pages. */
+  collection: string;
+  /**
+   * The auth-enabled collection the API key belongs to; default `users`. It
+   * names the scheme in the `Authorization` header (`users API-Key <key>`).
+   */
+  authCollection?: string;
+  /** Relationship depth to populate; default `1`, enough for upload URLs. */
+  depth?: number;
+  /**
+   * Field paths mapping a document onto Blume meta + body. Defaults:
+   * `title`, `description`, `slug`, `content`, `updatedAt`.
+   */
+  fields?: RemoteFieldMap;
+  /** Extra query parameters for the collection request (`where[...]`, `sort`). */
+  params?: Record<string, string>;
+  /** Serializers for `block`/`inlineBlock` nodes, keyed by `blockType`. */
+  serializers?: Record<string, (fields: JsonObject) => string>;
+  /** API key; defaults to `PAYLOAD_API_KEY`. */
+  token?: string;
+  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
+  pollInterval?: number;
+  /** Injected for tests; defaults to the global `fetch`. */
+  fetchImpl?: typeof fetch;
+}
+
+const PAGE_SIZE = 100;
+
+const DEFAULT_FIELDS: Required<RemoteFieldMap> = {
+  body: "content",
+  description: "description",
+  lastModified: "updatedAt",
+  slug: "slug",
+  title: "title",
+};
+
+/**
+ * Payload content source. Pages through a collection's REST endpoint, maps
+ * each document's fields to frontmatter, and lowers its Lexical body to
+ * Markdown. Published documents only, unless `--preview` asks for drafts.
+ */
+export const payloadSource = (
+  options: PayloadSourceOptions,
+  ctx?: SourceContext
+): ContentSource => {
+  const fields = { ...DEFAULT_FIELDS, ...options.fields };
+  const origin = options.url.replace(/\/+$/u, "");
+
+  const lower = (body: JsonObject): string =>
+    lexicalToMarkdown(body, {
+      baseUrl: origin,
+      serializers: options.serializers,
+    });
+
+  const fetchEntries = async (): Promise<SourceEntry[]> => {
+    const preview = ctx?.preview ?? false;
+    const token = options.token ?? process.env.PAYLOAD_API_KEY;
+    const client: RestClient = {
+      fetchImpl: options.fetchImpl,
+      headers: token
+        ? {
+            authorization: `${options.authCollection ?? "users"} API-Key ${token}`,
+          }
+        : {},
+    };
+    const base = `${origin}/api/${options.collection}`;
+    const entries: SourceEntry[] = [];
+    let page = 1;
+    let more = true;
+    while (more) {
+      // The user's params go first so the paging controls always win — a
+      // `params` key that shadowed them would refetch the same page forever.
+      const query = queryString({
+        ...options.params,
+        depth: String(options.depth ?? 1),
+        draft: preview ? "true" : undefined,
+        limit: String(PAGE_SIZE),
+        page: String(page),
+      });
+      // oxlint-disable-next-line no-await-in-loop -- pages are sequential: each response says whether another exists.
+      const result = asObject(await fetchJson(`${base}?${query}`, client));
+      if (!result) {
+        throw new Error("Payload returned a non-object response");
+      }
+      for (const doc of objectsIn(result.docs)) {
+        const draft = asString(doc._status) === "draft";
+        if (draft && !preview) {
+          continue;
+        }
+        const id = asString(doc.id) ?? String(doc.id ?? "");
+        entries.push(
+          documentEntry(
+            doc,
+            fields,
+            id,
+            (body) => (isJsonObject(body) ? lower(body) : ""),
+            draft,
+            writesMdx(options.serializers)
+          )
+        );
+      }
+      more = result.hasNextPage === true;
+      page += 1;
+    }
+    return entries;
+  };
+
+  return remoteSource(
+    {
+      fetchEntries,
+      name: options.name,
+      pollInterval: options.pollInterval,
+      prefix: options.prefix,
+    },
+    ctx
+  );
+};

package/src/core/sources/portable-text.ts

@@ -4,6 +4,7 @@
  * user-supplied serializer or are skipped with a noted comment. Output is
  * Markdown text that flows through Blume's normal pipeline.
  */
+import { escapeMarkdownText, image, unsupported, writesMdx } from "./lower.ts";
 
 /**
  * A field value on a Portable Text node: the arbitrary JSON the CMS query
@@ -49,7 +50,10 @@ interface PortableTextMarkDef {
 export interface PortableTextOptions {
   /** Resolve an image block to a URL (the adapter knows project/dataset). */
   imageUrl?: (block: PortableTextBlock) => string | null;
-  /** Custom block-type serializers, keyed by `_type`; return Markdown/MDX. */
+  /**
+   * Custom block-type serializers, keyed by `_type`; return Markdown or MDX.
+   * Setting any writes the source's entries as MDX.
+   */
   serializers?: Record<string, (block: PortableTextBlock) => string>;
 }
 
@@ -62,18 +66,6 @@ const HEADING_STYLES = new Map([
   ["h6", "###### "],
 ]);
 
-// Markdown/raw-HTML structure characters. Portable Text spans are *plain
-// text* — formatting arrives as marks, never as syntax in the text — so a
-// literal `*`, `_`, `[`, backtick, `~`, or `<` typed in the CMS must render
-// as itself. Unescaped, it opened emphasis or a code span mid-paragraph, and
-// `<` let CMS prose inject raw HTML into the rendered page. CommonMark
-// backslash-escapes every ASCII punctuation character, so `\*` is always the
-// literal asterisk.
-const MARKDOWN_SPECIALS = /[\\`*_[\]~<]/gu;
-
-const escapeText = (text: string): string =>
-  text.replaceAll(MARKDOWN_SPECIALS, String.raw`\$&`);
-
 /** Wrap a span's text in Markdown for its marks (decorators + link defs). */
 const renderSpan = (
   span: PortableTextSpan,
@@ -82,7 +74,7 @@ const renderSpan = (
   // Code spans stay verbatim: their text is literal inside the backticks,
   // and backslash escapes would render as backslashes.
   const isCode = span.marks?.includes("code") ?? false;
-  let text = isCode ? (span.text ?? "") : escapeText(span.text ?? "");
+  let text = isCode ? (span.text ?? "") : escapeMarkdownText(span.text ?? "");
   if (!span.marks || span.marks.length === 0) {
     return text;
   }
@@ -140,10 +132,13 @@ const renderBlock = (
   if (block._type === "image") {
     const url = options.imageUrl?.(block);
     const alt = isAltText(block.alt) ? block.alt : "";
-    return url ? `![${alt}](${url})` : "";
+    return url ? image(alt, url) : "";
   }
   if (block._type !== "block") {
-    return `<!-- unsupported Portable Text block: ${block._type} -->`;
+    return unsupported(
+      `Portable Text block: ${block._type}`,
+      writesMdx(options.serializers)
+    );
   }
 
   const inline = renderChildren(block);

package/src/core/sources/remote.ts

@@ -0,0 +1,226 @@
+import { join } from "pathe";
+
+import matter from "../frontmatter.ts";
+import {
+  hashText,
+  loadWithCache,
+  pollingWatch,
+  snapshotCache,
+} from "./cache.ts";
+import type { JsonObject, JsonValue } from "./json.ts";
+import { asString, getPath, isStringValue } from "./json.ts";
+import { slugify, slugifyPath } from "./normalize.ts";
+import type {
+  ContentSource,
+  SourceContext,
+  SourceEntry,
+  SourceLoadResult,
+} from "./types.ts";
+
+/** What every REST-backed CMS source needs to be a staged, cached source. */
+export interface RemoteSourceOptions {
+  /** Pull every entry from the API; called on refresh and on each poll. */
+  fetchEntries: () => Promise<SourceEntry[]>;
+  name: string;
+  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
+  pollInterval?: number;
+  prefix?: string;
+}
+
+/**
+ * The staged-source scaffolding a CMS adapter shares: a snapshot cache under
+ * `.blume/cache/<source>/` (or a name-derived dir when constructed directly
+ * as a custom source, without a context), cache-first dev loads, offline
+ * fallback, `read()` for raw export, and an opt-in polling `watch`.
+ */
+export const remoteSource = (
+  options: RemoteSourceOptions,
+  ctx?: SourceContext
+): ContentSource => {
+  const cache = snapshotCache(
+    ctx?.cacheDir ?? join(".blume", "cache", options.name)
+  );
+  let snapshot = new Map<string, SourceEntry>();
+
+  const load = async (
+    refresh = ctx?.refresh ?? true
+  ): Promise<SourceLoadResult> => {
+    const result = await loadWithCache(
+      options.name,
+      cache,
+      options.fetchEntries,
+      refresh
+    );
+    snapshot = new Map(result.entries.map((entry) => [entry.ref, entry]));
+    return result;
+  };
+
+  const read = async (ref: string): Promise<string> => {
+    const cached = snapshot.get(ref);
+    if (cached) {
+      return cached.raw ?? cached.body.text;
+    }
+    const all = await cache.read();
+    return all.find((entry) => entry.ref === ref)?.raw ?? "";
+  };
+
+  return {
+    load,
+    name: options.name,
+    prefix: options.prefix,
+    read,
+    staged: true,
+    watch: options.pollInterval
+      ? pollingWatch(
+          () => load(true),
+          options.pollInterval,
+          () => load()
+        )
+      : undefined,
+  };
+};
+
+/** The frontmatter a CMS document maps to. */
+export interface RemoteFrontmatter {
+  description?: string;
+  draft?: boolean;
+  title?: string;
+}
+
+/**
+ * A staged entry: Markdown body plus the frontmatter it was mapped from,
+ * written as `.mdx` when the body carries serializer output (see
+ * `writesMdx`).
+ */
+export const stagedEntry = (
+  slug: string,
+  data: RemoteFrontmatter,
+  markdown: string,
+  lastModified: string | undefined,
+  format: "md" | "mdx" = "md"
+): SourceEntry => {
+  const raw = matter.stringify(markdown, data);
+  return {
+    body: { format, text: markdown },
+    // Spread into a fresh literal: `SourceEntry.data` wants an
+    // index-signature type, which the named interface lacks.
+    data: { ...data },
+    hash: hashText(raw),
+    lastModified,
+    raw,
+    ref: `${slug}.${format}`,
+  };
+};
+
+/** How a source calls its CMS: the API's auth headers and an injectable fetch. */
+export interface RestClient {
+  /** Injected for tests; defaults to the global `fetch`. */
+  fetchImpl?: typeof fetch;
+  headers?: Record<string, string>;
+}
+
+/**
+ * How long one CMS request may take before the source gives up on it. A
+ * stalled CMS would otherwise hold `scanProject()` — and every `blume dev`
+ * rescan — indefinitely, and the cache fallback can only kick in once the
+ * request rejects.
+ */
+export const REMOTE_TIMEOUT_MS = 30_000;
+
+/**
+ * GET a JSON endpoint. A non-2xx response is an error naming the status, so
+ * the source's cache fallback reports why (`401 Unauthorized`) instead of a
+ * parse failure on an error body; a request past {@link REMOTE_TIMEOUT_MS}
+ * rejects with a `TimeoutError`.
+ */
+export const fetchJson = async (
+  url: string,
+  client: RestClient
+): Promise<JsonValue> => {
+  const doFetch = client.fetchImpl ?? globalThis.fetch;
+  const res = await doFetch(url, {
+    headers: { accept: "application/json", ...client.headers },
+    signal: AbortSignal.timeout(REMOTE_TIMEOUT_MS),
+  });
+  if (!res.ok) {
+    throw new Error(`${url} responded ${res.status} ${res.statusText}`.trim());
+  }
+  // SAFETY: the endpoint answered 2xx with a JSON body; whatever shape it
+  // holds is narrowed by the caller's predicates before use.
+  return (await res.json()) as JsonValue;
+};
+
+/** A query string from the fixed params plus a user's extra ones. */
+export const queryString = (
+  params: Record<string, string | undefined>
+): string => {
+  const search = new URLSearchParams();
+  for (const [key, value] of Object.entries(params)) {
+    if (value !== undefined) {
+      search.set(key, value);
+    }
+  }
+  return search.toString();
+};
+
+/** Field paths mapping a CMS document onto Blume meta + body. */
+export interface RemoteFieldMap {
+  /** Field holding the renderable body (rich text, or a Markdown string). */
+  body?: string;
+  /** Field holding the page description. */
+  description?: string;
+  /** Field holding the last-modified ISO date. */
+  lastModified?: string;
+  /** Field holding the route slug. */
+  slug?: string;
+  /** Field holding the page title. */
+  title?: string;
+}
+
+/**
+ * Map a document to a staged entry. The slug falls back to the document's id
+ * when the slug field is missing or slugifies to nothing (pure punctuation),
+ * so distinct documents never collapse onto one `untitled.md`; a slashed
+ * slug keeps its segments. A string body is Markdown and passes through as
+ * `.md`; any other shape goes to the CMS's lowerer, and is written as `.mdx`
+ * when `lowersToMdx` (the source has serializers, see `writesMdx`).
+ */
+export const documentEntry = (
+  doc: JsonObject,
+  fields: Required<RemoteFieldMap>,
+  id: string,
+  lower: (body: JsonValue) => string,
+  draft = false,
+  lowersToMdx = false
+): SourceEntry => {
+  const slugValue = asString(getPath(doc, fields.slug)) ?? id;
+  const slug = slugifyPath(slugValue) || slugify(id) || "untitled";
+  const data: RemoteFrontmatter = {};
+  const title = asString(getPath(doc, fields.title));
+  const description = asString(getPath(doc, fields.description));
+  if (title) {
+    data.title = title;
+  }
+  if (description) {
+    data.description = description;
+  }
+  if (draft) {
+    data.draft = true;
+  }
+  const body = getPath(doc, fields.body);
+  let markdown = "";
+  let format: "md" | "mdx" = "md";
+  if (isStringValue(body)) {
+    markdown = `${body.trimEnd()}\n`;
+  } else if (body !== undefined) {
+    markdown = lower(body);
+    format = lowersToMdx ? "mdx" : "md";
+  }
+  return stagedEntry(
+    slug,
+    data,
+    markdown,
+    asString(getPath(doc, fields.lastModified)),
+    format
+  );
+};