blume 1.7.3 → 2.0.1

package/src/core/server-features.ts

@@ -1,5 +1,4 @@
 import { needsPlaygroundProxy } from "../openapi/references.ts";
-import { searchProviderMeta } from "../search/providers.ts";
 import type { ResolvedConfig } from "./schema.ts";
 
 /**
@@ -8,23 +7,23 @@ import type { ResolvedConfig } from "./schema.ts";
  */
 export const serverFeatures = (config: ResolvedConfig): string[] => {
   const features: string[] = [];
-  if (config.ai.ask?.enabled && !config.ai.ask.endpoint) {
-    features.push("Ask AI");
+  if (config.ai.assistant?.enabled && !config.ai.assistant.endpoint) {
+    features.push("Assistant");
   }
   // The hosted MCP server is a live JSON-RPC endpoint, so it needs a runtime.
-  if (config.ai.mcp.enabled) {
+  if (config.agents.mcp.enabled) {
     features.push("MCP server");
   }
-  // The built-in playground proxy (`playground.proxy: true` on the OpenAPI or
-  // GraphQL block) is a live fetch endpoint at `/_api-proxy`; an external
-  // proxy URL (string) or a proxy-less playground stays fully static.
+  // The built-in playground proxy (`playground.proxy: true` on an `openapi()`
+  // or `graphql()` adapter) is a live fetch endpoint at `/_api-proxy`; an
+  // external proxy URL (string) or a proxy-less playground stays fully static.
   if (needsPlaygroundProxy(config)) {
     features.push("API playground proxy");
   }
-  // Mixedbread (and any future provider) that proxies queries through a secret
-  // server endpoint can't run on a static build.
-  if (searchProviderMeta(config.search.provider).requiresServer) {
-    features.push(`Search (${config.search.provider})`);
+  // A `server` adapter (Mixedbread) proxies queries through a secret server
+  // endpoint, so it can't run on a static build.
+  if (config.search.provider.mode === "server") {
+    features.push(`Search (${config.search.provider.kind})`);
   }
   return features;
 };

package/src/core/sources/assets.ts

@@ -67,6 +67,13 @@ export interface AssetContext {
    * download at once. Defaults to no gate.
    */
   limit?: <T>(task: () => Promise<T>) => Promise<T>;
+  /**
+   * Downloads in progress, keyed by asset stem. A source shares one table
+   * (alongside `limit`) across every page it materializes, so two pages that
+   * reference the same asset share one download instead of racing to publish
+   * the same file. Defaults to a table private to this call.
+   */
+  inFlight?: Map<string, Promise<string>>;
   /** Abort a download that hasn't completed within this many milliseconds. */
   timeoutMs?: number;
 }
@@ -128,6 +135,7 @@ export const materializeAssets = async (
 ): Promise<{ markdown: string; diagnostics: Diagnostic[] }> => {
   const doFetch = ctx.fetchImpl ?? globalThis.fetch;
   const limit = ctx.limit ?? ((task) => task());
+  const inFlight = ctx.inFlight ?? new Map<string, Promise<string>>();
   const timeoutMs = ctx.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const diagnostics: Diagnostic[] = [];
 
@@ -151,13 +159,7 @@ export const materializeAssets = async (
   }
 
   /** Fetch one asset into the asset dir and return its file name. */
-  const download = async (url: string): Promise<string> => {
-    // Hash the query-less URL: CMS asset URLs are pre-signed, so the query
-    // changes on every fetch of the same file — hashing it would mint a new
-    // file each refresh and re-dirty the content digest. Two real assets
-    // sharing scheme+host+path and differing only in query are rare enough to
-    // accept colliding.
-    const stem = hashText(url.split("?")[0] ?? url);
+  const download = async (url: string, stem: string): Promise<string> => {
     const urlExt = extFromUrl(url);
     // The name is known up front whenever the path has an extension (every
     // Notion upload does), so a file from an earlier run is reused as is;
@@ -189,9 +191,8 @@ export const materializeAssets = async (
     // megabytes where an image was a hundred kilobytes. Write to a temporary
     // name unique to this attempt and rename on completion, so a download that
     // dies midway never leaves a truncated file the next run would trust as
-    // complete, and two pages fetching the same asset at once (the gate bounds
-    // concurrency, it doesn't dedupe) never write into each other's file —
-    // both publish identical bytes and the last rename wins.
+    // complete. Two pages fetching the same asset at once share one download
+    // through `inFlight` (below), so no two attempts ever publish one file.
     const part = `${target}.${randomUUID()}${PART_SUFFIX}`;
     try {
       await writeFile(part, res.body);
@@ -203,11 +204,46 @@ export const materializeAssets = async (
     return file;
   };
 
+  /**
+   * Publish one asset, joining a download another page already has in
+   * flight. The gate sits inside the shared promise, so a page waiting on
+   * another page's download holds no slot of its own. A waiter whose
+   * publisher failed makes its own attempt, exactly as it would have without
+   * sharing — one transient error must not fail every page naming the asset.
+   */
+  const publish = async (url: string): Promise<string> => {
+    // Hash the query-less URL: CMS asset URLs are pre-signed, so the query
+    // changes on every fetch of the same file — hashing it would mint a new
+    // file each refresh and re-dirty the content digest. Two real assets
+    // sharing scheme+host+path and differing only in query are rare enough to
+    // accept colliding.
+    const stem = hashText(url.split("?")[0] ?? url);
+    const pending = inFlight.get(stem);
+    if (pending) {
+      try {
+        return await pending;
+      } catch {
+        // The failed download has left the table by now: join whichever
+        // waiter retried first, or go it alone.
+        return await publish(url);
+      }
+    }
+    const own = (async () => {
+      try {
+        return await limit(() => download(url, stem));
+      } finally {
+        inFlight.delete(stem);
+      }
+    })();
+    inFlight.set(stem, own);
+    return await own;
+  };
+
   const rewrites = new Map<string, string>();
   await Promise.all(
     [...urls].map(async (url) => {
       try {
-        const file = await limit(() => download(url));
+        const file = await publish(url);
         rewrites.set(url, `${ctx.assetsBaseUrl}/${file}`);
       } catch (error) {
         // SAFETY: everything thrown in this block is an Error — the manual

package/src/core/sources/collection.ts

@@ -0,0 +1,67 @@
+import { isAbsolute, join, resolve } from "pathe";
+
+import {
+  DEFAULT_CONTENT_EXCLUDE,
+  DEFAULT_CONTENT_INCLUDE,
+} from "../../sources/filesystem.ts";
+import type { ContentSourceAdapter } from "../../sources/registry.ts";
+import type { ResolvedConfig } from "../schema.ts";
+
+/** Resolve a source `root` against the project root (absolute passes through). */
+export const resolveSourceRoot = (projectRoot: string, root: string): string =>
+  isAbsolute(root) ? root : join(resolve(projectRoot), root);
+
+/** The configured sources of one adapter `kind`, in config order. */
+export const sourcesOfKind = <Kind extends ContentSourceAdapter["kind"]>(
+  config: ResolvedConfig,
+  kind: Kind
+): Extract<ContentSourceAdapter, { kind: Kind }>[] =>
+  config.content.sources.filter(
+    (source): source is Extract<ContentSourceAdapter, { kind: Kind }> =>
+      source.kind === kind
+  );
+
+/**
+ * The generated `docs` glob collection: its base directory and the include /
+ * exclude globs applied under it. Astro's glob loader ids each entry by its
+ * path relative to `base`, and a filesystem source ids each entry relative to
+ * its own root — so the two only agree when the collection is rooted at that
+ * source. The collection therefore roots at the first `filesystem()` source
+ * (the implicit one, for a zero-config project). Further filesystem sources
+ * must share that root and partition it with globs: every filesystem source's
+ * `include` feeds the collection, and only a pattern every one of them
+ * excludes is excluded, so no source's pages are globbed away. A source
+ * rooted elsewhere can't share the base and is caught by the entry-id guard in
+ * `scanProject`. With no filesystem source at all the collection globs nothing
+ * and its base is the default `docs` directory, which only anchors paths.
+ */
+export interface DocsCollection {
+  base: string;
+  include: string[];
+  exclude: string[];
+}
+
+export const resolveDocsCollection = (
+  config: ResolvedConfig,
+  projectRoot: string
+): DocsCollection => {
+  const [first, ...rest] = sourcesOfKind(config, "filesystem");
+  if (!first) {
+    return {
+      base: resolveSourceRoot(projectRoot, "docs"),
+      exclude: DEFAULT_CONTENT_EXCLUDE,
+      include: DEFAULT_CONTENT_INCLUDE,
+    };
+  }
+  const include = [
+    ...new Set([first, ...rest].flatMap((source) => source.options.include)),
+  ];
+  const exclude = first.options.exclude.filter((pattern) =>
+    rest.every((source) => source.options.exclude.includes(pattern))
+  );
+  return {
+    base: resolveSourceRoot(projectRoot, first.options.root),
+    exclude,
+    include,
+  };
+};

package/src/core/sources/contentful-rich-text.ts

@@ -0,0 +1,285 @@
+/**
+ * Contentful rich text → Markdown. Covers the block, inline, and mark types
+ * the Rich Text editor emits; embedded entries fall through to a serializer
+ * keyed by content type or are noted in a comment. Links to assets and
+ * entries arrive unresolved from the Delivery API (`data.target` is a
+ * `sys` link), so the source hands in resolvers over the response's
+ * `includes`; a document the SDK already resolved (with `fields` inline)
+ * lowers the same way.
+ */
+import type { JsonObject } from "./json.ts";
+import { asString, getPath, objectsIn } from "./json.ts";
+import type { InlineMarks } from "./lower.ts";
+import {
+  absoluteUrl,
+  blockquote,
+  headingPrefix,
+  image,
+  joinBlocks,
+  listItem,
+  markdownDocument,
+  renderInline,
+  renderLink,
+  unsupported,
+  writesMdx,
+} from "./lower.ts";
+
+/** An asset's file as a Markdown image needs it. */
+export interface ContentfulAsset {
+  description?: string;
+  title?: string;
+  url: string;
+}
+
+export interface ContentfulRichTextOptions {
+  /** Resolve an asset link (`data.target.sys.id`) to its file. */
+  resolveAsset?: (id: string) => ContentfulAsset | null;
+  /** Resolve an entry link to the entry (`sys` plus `fields`). */
+  resolveEntry?: (id: string) => JsonObject | null;
+  /**
+   * Serializers for embedded entries, keyed by content type id; return
+   * Markdown or MDX. Setting any writes the source's entries as MDX.
+   */
+  serializers?: Record<string, (entry: JsonObject) => string>;
+}
+
+const HEADING = /^heading-(?<level>[1-6])$/u;
+const CELL_PIPE = /\|/gu;
+const CELL_BREAK = /\n+/gu;
+
+/** The file behind an asset entry, or null when it carries none. */
+export const assetFromEntry = (asset: JsonObject): ContentfulAsset | null => {
+  const url = asString(getPath(asset, "fields.file.url"));
+  if (!url) {
+    return null;
+  }
+  return {
+    description: asString(getPath(asset, "fields.description")),
+    title: asString(getPath(asset, "fields.title")),
+    url: absoluteUrl(url),
+  };
+};
+
+const children = (node: JsonObject): JsonObject[] => objectsIn(node.content);
+
+const marksOf = (node: JsonObject): InlineMarks => {
+  const marks: InlineMarks = {};
+  for (const mark of objectsIn(node.marks)) {
+    switch (asString(mark.type)) {
+      case "bold": {
+        marks.bold = true;
+        break;
+      }
+      case "italic": {
+        marks.italic = true;
+        break;
+      }
+      case "code": {
+        marks.code = true;
+        break;
+      }
+      case "strikethrough": {
+        marks.strike = true;
+        break;
+      }
+      default:
+    }
+  }
+  return marks;
+};
+
+/**
+ * The linked object a node points at: inline when the document was resolved
+ * (`data.target.fields`), else through the resolver by `data.target.sys.id`.
+ */
+const linkedTarget = (
+  node: JsonObject,
+  resolve?: (id: string) => JsonObject | null
+): JsonObject | null => {
+  const target = getPath(node, "data.target");
+  if (getPath(node, "data.target.fields") !== undefined) {
+    // SAFETY: `data.target.fields` resolved through the object, so `target`
+    // is that object.
+    return target as JsonObject;
+  }
+  const id = asString(getPath(node, "data.target.sys.id"));
+  return id ? (resolve?.(id) ?? null) : null;
+};
+
+const linkedAsset = (
+  node: JsonObject,
+  options: ContentfulRichTextOptions
+): ContentfulAsset | null => {
+  if (getPath(node, "data.target.fields") !== undefined) {
+    const inline = linkedTarget(node);
+    return inline ? assetFromEntry(inline) : null;
+  }
+  const id = asString(getPath(node, "data.target.sys.id"));
+  return id ? (options.resolveAsset?.(id) ?? null) : null;
+};
+
+const embeddedEntry = (
+  node: JsonObject,
+  options: ContentfulRichTextOptions
+): string => {
+  const entry = linkedTarget(node, options.resolveEntry);
+  const contentType = entry
+    ? asString(getPath(entry, "sys.contentType.sys.id"))
+    : undefined;
+  const serializer = contentType
+    ? options.serializers?.[contentType]
+    : undefined;
+  if (entry && serializer) {
+    return serializer(entry);
+  }
+  return unsupported(
+    `Contentful embedded entry${contentType ? `: ${contentType}` : ""}`,
+    writesMdx(options.serializers)
+  );
+};
+
+const renderInlines = (
+  nodes: JsonObject[],
+  options: ContentfulRichTextOptions
+): string =>
+  nodes
+    // oxlint-disable-next-line no-use-before-define -- mutual recursion: a link's label is itself inline content
+    .map((node) => renderInlineNode(node, options))
+    .join("");
+
+const renderInlineNode = (
+  node: JsonObject,
+  options: ContentfulRichTextOptions
+): string => {
+  switch (asString(node.nodeType)) {
+    case "text": {
+      return renderInline(asString(node.value) ?? "", marksOf(node));
+    }
+    case "hyperlink": {
+      return renderLink(
+        renderInlines(children(node), options),
+        asString(getPath(node, "data.uri"))
+      );
+    }
+    case "asset-hyperlink": {
+      return renderLink(
+        renderInlines(children(node), options),
+        linkedAsset(node, options)?.url
+      );
+    }
+    case "embedded-entry-inline": {
+      return embeddedEntry(node, options);
+    }
+    default: {
+      // entry-hyperlink (no route to point at) and anything unknown: the label.
+      return renderInlines(children(node), options);
+    }
+  }
+};
+
+const renderList = (
+  items: JsonObject[],
+  marker: (index: number) => string,
+  options: ContentfulRichTextOptions
+): string =>
+  items
+    .map((item, index) =>
+      listItem(
+        marker(index),
+        // A single newline between an item's paragraph and its nested list
+        // keeps the list tight.
+        children(item)
+          // oxlint-disable-next-line no-use-before-define -- mutual recursion: an item holds blocks, including nested lists
+          .map((child) => renderBlock(child, options))
+          .filter((block) => block !== "")
+          .join("\n")
+      )
+    )
+    .join("\n");
+
+const cellText = (
+  cell: JsonObject,
+  options: ContentfulRichTextOptions
+): string =>
+  children(cell)
+    .map((paragraph) => renderInlines(children(paragraph), options))
+    .join(" ")
+    .replaceAll(CELL_BREAK, " ")
+    .replaceAll(CELL_PIPE, String.raw`\|`);
+
+const tableRow = (cells: string[]): string => `| ${cells.join(" | ")} |`;
+
+const renderTable = (
+  node: JsonObject,
+  options: ContentfulRichTextOptions
+): string => {
+  const rows = children(node).map((row) =>
+    children(row).map((cell) => cellText(cell, options))
+  );
+  const [head, ...body] = rows;
+  if (!head) {
+    return "";
+  }
+  return [
+    tableRow(head),
+    tableRow(head.map(() => "---")),
+    ...body.map((cells) => tableRow(cells)),
+  ].join("\n");
+};
+
+const renderBlock = (
+  node: JsonObject,
+  options: ContentfulRichTextOptions
+): string => {
+  const type = asString(node.nodeType) ?? "";
+  const heading = HEADING.exec(type)?.groups?.level;
+  if (heading) {
+    return `${headingPrefix(Number(heading))}${renderInlines(children(node), options)}`;
+  }
+  switch (type) {
+    case "paragraph": {
+      return renderInlines(children(node), options);
+    }
+    case "blockquote": {
+      return blockquote(
+        joinBlocks(children(node).map((child) => renderBlock(child, options)))
+      );
+    }
+    case "unordered-list": {
+      return renderList(children(node), () => "-", options);
+    }
+    case "ordered-list": {
+      return renderList(children(node), (index) => `${index + 1}.`, options);
+    }
+    case "hr": {
+      return "---";
+    }
+    case "embedded-asset-block": {
+      const asset = linkedAsset(node, options);
+      // Contentful's own rich-text renderer uses the description as alt text
+      // and the title only as a fallback.
+      return asset
+        ? image(asset.description ?? asset.title ?? "", asset.url)
+        : "";
+    }
+    case "embedded-entry-block": {
+      return embeddedEntry(node, options);
+    }
+    case "table": {
+      return renderTable(node, options);
+    }
+    default: {
+      return unsupported(
+        `Contentful node: ${type}`,
+        writesMdx(options.serializers)
+      );
+    }
+  }
+};
+
+/** Serialize a rich text document (`nodeType: "document"`) to Markdown. */
+export const contentfulRichTextToMarkdown = (
+  doc: JsonObject,
+  options: ContentfulRichTextOptions = {}
+): string =>
+  markdownDocument(children(doc).map((node) => renderBlock(node, options)));

package/src/core/sources/contentful.ts

@@ -0,0 +1,173 @@
+import {
+  assetFromEntry,
+  contentfulRichTextToMarkdown,
+} from "./contentful-rich-text.ts";
+import type { JsonObject } from "./json.ts";
+import {
+  asNumber,
+  asObject,
+  asString,
+  getPath,
+  isJsonObject,
+  objectsIn,
+} from "./json.ts";
+import { writesMdx } from "./lower.ts";
+import type { RemoteFieldMap, RestClient } from "./remote.ts";
+import {
+  documentEntry,
+  fetchJson,
+  queryString,
+  remoteSource,
+} from "./remote.ts";
+import type { ContentSource, SourceContext, SourceEntry } from "./types.ts";
+
+export interface ContentfulSourceOptions {
+  name: string;
+  prefix?: string;
+  /** Space id. */
+  space: string;
+  /** Environment id; default `master`. */
+  environment?: string;
+  /** The content type id whose entries become pages. */
+  contentType: string;
+  /** Locale code to fetch; omit for the space's default locale. */
+  locale?: string;
+  /**
+   * Field ids mapping an entry onto Blume meta + body. Paths resolve against
+   * the entry's `fields`, with `sys` reachable as `sys.<key>`. Defaults:
+   * `title`, `description`, `slug`, `body`, `sys.updatedAt`.
+   */
+  fields?: RemoteFieldMap;
+  /** Extra query parameters for the entries request (`fields.section: "sdk"`). */
+  params?: Record<string, string>;
+  /** Serializers for embedded entries, keyed by content type id. */
+  serializers?: Record<string, (entry: JsonObject) => string>;
+  /** Delivery API token; defaults to `CONTENTFUL_ACCESS_TOKEN`. */
+  token?: string;
+  /** Preview API token for `--preview`; defaults to `CONTENTFUL_PREVIEW_TOKEN`. */
+  previewToken?: string;
+  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
+  pollInterval?: number;
+  /** Injected for tests; defaults to the global `fetch`. */
+  fetchImpl?: typeof fetch;
+}
+
+const DELIVERY_HOST = "https://cdn.contentful.com";
+const PREVIEW_HOST = "https://preview.contentful.com";
+const PAGE_SIZE = 100;
+
+const DEFAULT_FIELDS: Required<RemoteFieldMap> = {
+  body: "body",
+  description: "description",
+  lastModified: "sys.updatedAt",
+  slug: "slug",
+  title: "title",
+};
+
+/** The included objects of a page, keyed by `sys.id`. */
+const byId = (objects: JsonObject[]): Map<string, JsonObject> =>
+  new Map(
+    objects.map((object) => [asString(getPath(object, "sys.id")) ?? "", object])
+  );
+
+/**
+ * Contentful content source. Pages through the Delivery API (the Preview API
+ * under `--preview`), maps each entry's fields to frontmatter, and lowers its
+ * rich text body to Markdown with the page's `includes` resolving asset and
+ * entry links.
+ */
+export const contentfulSource = (
+  options: ContentfulSourceOptions,
+  ctx?: SourceContext
+): ContentSource => {
+  const fields = { ...DEFAULT_FIELDS, ...options.fields };
+
+  const toEntry = (
+    item: JsonObject,
+    assets: Map<string, JsonObject>,
+    linked: Map<string, JsonObject>
+  ): SourceEntry => {
+    const sys = asObject(item.sys) ?? {};
+    const id = asString(sys.id) ?? "";
+    // Field paths resolve against the entry's fields, with `sys` beside them.
+    const view: JsonObject = { ...asObject(item.fields), sys };
+    return documentEntry(
+      view,
+      fields,
+      id,
+      (body) =>
+        isJsonObject(body)
+          ? contentfulRichTextToMarkdown(body, {
+              resolveAsset: (assetId) => {
+                const asset = assets.get(assetId);
+                return asset ? assetFromEntry(asset) : null;
+              },
+              resolveEntry: (entryId) => linked.get(entryId) ?? null,
+              serializers: options.serializers,
+            })
+          : "",
+      false,
+      writesMdx(options.serializers)
+    );
+  };
+
+  const fetchEntries = async (): Promise<SourceEntry[]> => {
+    const preview = ctx?.preview ?? false;
+    const deliveryToken = options.token ?? process.env.CONTENTFUL_ACCESS_TOKEN;
+    const previewToken =
+      options.previewToken ?? process.env.CONTENTFUL_PREVIEW_TOKEN;
+    // The Preview API rejects delivery tokens, so falling back to one would
+    // only trade a clear config error for a 401.
+    if (preview && !previewToken) {
+      throw new Error(
+        "Contentful --preview needs a Preview API token: set previewToken or CONTENTFUL_PREVIEW_TOKEN."
+      );
+    }
+    const token = preview ? previewToken : deliveryToken;
+    const client: RestClient = {
+      fetchImpl: options.fetchImpl,
+      headers: token ? { authorization: `Bearer ${token}` } : {},
+    };
+    const host = preview ? PREVIEW_HOST : DELIVERY_HOST;
+    const base = `${host}/spaces/${options.space}/environments/${options.environment ?? "master"}/entries`;
+    const entries: SourceEntry[] = [];
+    let skip = 0;
+    let more = true;
+    while (more) {
+      // The user's params go first so the paging controls always win — a
+      // `params` key that shadowed them would refetch the same page forever.
+      const query = queryString({
+        ...options.params,
+        content_type: options.contentType,
+        include: "2",
+        limit: String(PAGE_SIZE),
+        locale: options.locale,
+        skip: String(skip),
+      });
+      // oxlint-disable-next-line no-await-in-loop -- pages are sequential: each response says whether another exists.
+      const page = asObject(await fetchJson(`${base}?${query}`, client));
+      if (!page) {
+        throw new Error("Contentful returned a non-object response");
+      }
+      const assets = byId(objectsIn(getPath(page, "includes.Asset")));
+      const linked = byId(objectsIn(getPath(page, "includes.Entry")));
+      const items = objectsIn(page.items);
+      for (const item of items) {
+        entries.push(toEntry(item, assets, linked));
+      }
+      skip += items.length;
+      more = items.length > 0 && skip < (asNumber(page.total) ?? 0);
+    }
+    return entries;
+  };
+
+  return remoteSource(
+    {
+      fetchEntries,
+      name: options.name,
+      pollInterval: options.pollInterval,
+      prefix: options.prefix,
+    },
+    ctx
+  );
+};