blume 2.1.0 → 2.1.1

package/src/core/sources/contentful.ts

@@ -1,4 +1,6 @@
 import { BlumeError } from "../diagnostics.ts";
+import type { ResolvedI18nConfig, ResolvedVersionsConfig } from "../schema.ts";
+import type { Diagnostic } from "../types.ts";
 import {
   assetFromEntry,
   contentfulRichTextToMarkdown,
@@ -13,10 +15,13 @@ import {
   objectsIn,
 } from "./json.ts";
 import { writesMdx } from "./lower.ts";
-import type { RemoteFieldMap, RestClient } from "./remote.ts";
+import { localizedRoute, resolveEntryRoute } from "./normalize.ts";
+import type { RemoteFieldMap, RemoteFields, RestClient } from "./remote.ts";
 import {
   documentEntry,
+  documentSlug,
   fetchJson,
+  missingSecretError,
   queryString,
   remoteSource,
 } from "./remote.ts";
@@ -29,10 +34,24 @@ export interface ContentfulSourceOptions {
   space: string;
   /** Environment id; default `master`. */
   environment?: string;
+  /**
+   * Delivery API host name; default `cdn.contentful.com`. A space with EU
+   * data residency reads from `cdn.eu.contentful.com`.
+   */
+  host?: string;
+  /**
+   * Preview API host name for `--preview`; default `preview.contentful.com`
+   * (`preview.eu.contentful.com` for EU data residency).
+   */
+  previewHost?: string;
   /** The content type id whose entries become pages. */
   contentType: string;
   /** Locale code to fetch; omit for the space's default locale. */
   locale?: string;
+  /** The project's i18n config, which places an entry's route by its slug. */
+  i18n?: ResolvedI18nConfig;
+  /** The project's versions config, which places an entry's route by its slug. */
+  versions?: ResolvedVersionsConfig;
   /**
    * Field ids mapping an entry onto Blume meta + body. Paths resolve against
    * the entry's `fields`, with `sys` reachable as `sys.<key>`. Defaults:
@@ -53,11 +72,11 @@ export interface ContentfulSourceOptions {
   fetchImpl?: typeof fetch;
 }
 
-const DELIVERY_HOST = "https://cdn.contentful.com";
-const PREVIEW_HOST = "https://preview.contentful.com";
+const DELIVERY_HOST = "cdn.contentful.com";
+const PREVIEW_HOST = "preview.contentful.com";
 const PAGE_SIZE = 100;
 
-const DEFAULT_FIELDS: Required<RemoteFieldMap> = {
+const DEFAULT_FIELDS: RemoteFields = {
   body: "body",
   description: "description",
   lastModified: "sys.updatedAt",
@@ -71,11 +90,34 @@ const byId = (objects: JsonObject[]): Map<string, JsonObject> =>
     objects.map((object) => [asString(getPath(object, "sys.id")) ?? "", object])
   );
 
+/** One page of the entries response: its items and what their links reach. */
+interface EntriesPage {
+  assets: Map<string, JsonObject>;
+  items: JsonObject[];
+  linked: Map<string, JsonObject>;
+}
+
+/** An entry's id, and the entry as its field paths read it. */
+interface EntryView {
+  id: string;
+  /** The entry's fields, with `sys` beside them. */
+  view: JsonObject;
+}
+
+const entryView = (item: JsonObject): EntryView => {
+  const sys = asObject(item.sys) ?? {};
+  return {
+    id: asString(sys.id) ?? "",
+    view: { ...asObject(item.fields), sys },
+  };
+};
+
 /**
  * Contentful content source. Pages through the Delivery API (the Preview API
  * under `--preview`), maps each entry's fields to frontmatter, and lowers its
  * rich text body to Markdown with the page's `includes` resolving asset and
- * entry links.
+ * entry links. An entry hyperlink to another of the source's entries links
+ * to that entry's page.
  */
 export const contentfulSource = (
   options: ContentfulSourceOptions,
@@ -83,33 +125,78 @@ export const contentfulSource = (
 ): ContentSource => {
   const fields = { ...DEFAULT_FIELDS, ...options.fields };
 
+  // The route an entry's page publishes at: the ref `documentEntry` names it
+  // by, through the resolver `normalizeEntry` uses, so an entry hyperlink
+  // lands on the page that's built. A slug places a page in one locale, and
+  // `.md` stands in for either extension, which the route drops.
+  const routeOf = (view: JsonObject, id: string): string => {
+    const route = resolveEntryRoute(
+      { ref: `${documentSlug(view, fields.slug, id)}.md` },
+      ".md",
+      undefined,
+      { i18n: options.i18n, prefix: options.prefix, versions: options.versions }
+    );
+    return localizedRoute(
+      route.logicalRoute,
+      route.locales[0] ?? "",
+      options.i18n
+    );
+  };
+
   const toEntry = (
     item: JsonObject,
-    assets: Map<string, JsonObject>,
-    linked: Map<string, JsonObject>
+    { assets, linked }: EntriesPage,
+    routes: Map<string, string>,
+    warn: (diagnostic: Diagnostic) => void
   ): SourceEntry => {
-    const sys = asObject(item.sys) ?? {};
-    const id = asString(sys.id) ?? "";
-    // Field paths resolve against the entry's fields, with `sys` beside them.
-    const view: JsonObject = { ...asObject(item.fields), sys };
-    return documentEntry(
+    const { id, view } = entryView(item);
+    // Links the response left out of `includes`: the API delivers no
+    // unpublished, archived, or deleted target, and the page renders without
+    // it.
+    const missing = new Set<string>();
+    const entry = documentEntry(
       view,
       fields,
       id,
       (body) =>
         isJsonObject(body)
           ? contentfulRichTextToMarkdown(body, {
+              // Only an entry the source itself turns into a page has a
+              // route: one of another content type, or one `params` filters
+              // out, keeps its text.
+              entryHref: (target) =>
+                routes.get(asString(getPath(target, "sys.id")) ?? ""),
               resolveAsset: (assetId) => {
                 const asset = assets.get(assetId);
-                return asset ? assetFromEntry(asset) : null;
+                if (!asset) {
+                  missing.add(`asset "${assetId}"`);
+                  return null;
+                }
+                return assetFromEntry(asset);
+              },
+              resolveEntry: (entryId) => {
+                const target = linked.get(entryId);
+                if (!target) {
+                  missing.add(`entry "${entryId}"`);
+                }
+                return target ?? null;
               },
-              resolveEntry: (entryId) => linked.get(entryId) ?? null,
               serializers: options.serializers,
             })
           : "",
       false,
       writesMdx(options.serializers)
     );
+    for (const link of missing) {
+      warn({
+        code: "BLUME_SOURCE_UNRESOLVED_LINK",
+        message: `Source "${options.name}": "${entry.ref}" links to Contentful ${link}, which the response didn't include, so the page renders without it.`,
+        severity: "warning",
+        suggestion:
+          "Publish the linked asset or entry, or remove the link: the Delivery API returns only published content.",
+      });
+    }
+    return entry;
   };
 
   const preview = ctx?.preview ?? false;
@@ -131,17 +218,26 @@ export const contentfulSource = (
     }
   };
 
-  const fetchEntries = async (): Promise<SourceEntry[]> => {
+  const fetchEntries = async (
+    warn: (diagnostic: Diagnostic) => void
+  ): Promise<SourceEntry[]> => {
     const token = preview
       ? previewToken()
       : (options.token ?? process.env.CONTENTFUL_ACCESS_TOKEN);
+    // The Delivery API answers nothing without a token (`assertConfigured`
+    // already required the preview one).
+    if (!token) {
+      throw missingSecretError(options.name, "CONTENTFUL_ACCESS_TOKEN");
+    }
     const client: RestClient = {
       fetchImpl: options.fetchImpl,
-      headers: token ? { authorization: `Bearer ${token}` } : {},
+      headers: { authorization: `Bearer ${token}` },
     };
-    const host = preview ? PREVIEW_HOST : DELIVERY_HOST;
-    const base = `${host}/spaces/${options.space}/environments/${options.environment ?? "master"}/entries`;
-    const entries: SourceEntry[] = [];
+    const host = preview
+      ? (options.previewHost ?? PREVIEW_HOST)
+      : (options.host ?? DELIVERY_HOST);
+    const base = `https://${host}/spaces/${options.space}/environments/${options.environment ?? "master"}/entries`;
+    const pages: EntriesPage[] = [];
     let skip = 0;
     let more = true;
     while (more) {
@@ -160,16 +256,30 @@ export const contentfulSource = (
       if (!page) {
         throw new Error("Contentful returned a non-object response");
       }
-      const assets = byId(objectsIn(getPath(page, "includes.Asset")));
-      const linked = byId(objectsIn(getPath(page, "includes.Entry")));
       const items = objectsIn(page.items);
-      for (const item of items) {
-        entries.push(toEntry(item, assets, linked));
-      }
+      pages.push({
+        assets: byId(objectsIn(getPath(page, "includes.Asset"))),
+        items,
+        // `includes` leaves out an entry the page's `items` already hold, so
+        // a link to one of those resolves from `items`.
+        linked: byId([...objectsIn(getPath(page, "includes.Entry")), ...items]),
+      });
       skip += items.length;
       more = items.length > 0 && skip < (asNumber(page.total) ?? 0);
     }
-    return entries;
+    // A link can reach an entry a later page delivers, so every entry's route
+    // is known before any body lowers.
+    const routes = new Map(
+      pages.flatMap(({ items }) =>
+        items.map((item): [string, string] => {
+          const { id, view } = entryView(item);
+          return [id, routeOf(view, id)];
+        })
+      )
+    );
+    return pages.flatMap((page) =>
+      page.items.map((item) => toEntry(item, page, routes, warn))
+    );
   };
 
   return remoteSource(

package/src/core/sources/filesystem.ts

@@ -1,13 +1,17 @@
 import { existsSync, watch as fsWatch } from "node:fs";
 import { readFile } from "node:fs/promises";
 
+import { YAMLException } from "js-yaml";
 import { extname, isAbsolute, join, relative, resolve } from "pathe";
 import { glob } from "tinyglobby";
 
 import { BlumeError } from "../diagnostics.ts";
 import matter from "../frontmatter.ts";
 import type { Diagnostic } from "../types.ts";
-import { unloadablePathDiagnostic } from "./normalize.ts";
+import {
+  frontmatterYamlDiagnostic,
+  unloadablePathDiagnostic,
+} from "./normalize.ts";
 import type { ContentSource, SourceEntry, SourceLoadResult } from "./types.ts";
 import {
   baselineScanIgnore,
@@ -72,12 +76,22 @@ export const filesystemSource = (
       }
     }
 
-    const entries = await Promise.all(
-      loadable.map(async (file): Promise<SourceEntry> => {
+    const read = await Promise.all(
+      loadable.map(async (file): Promise<SourceEntry | Diagnostic> => {
         const source = await readFile(file, "utf-8");
         const ext = extname(file).toLowerCase();
         const format = ext === ".mdx" ? "mdx" : "md";
-        const parsed = matter(source);
+        // Invalid YAML leaves this one file out, reported at its line,
+        // instead of failing the whole scan.
+        let parsed: ReturnType<typeof matter>;
+        try {
+          parsed = matter(source);
+        } catch (error) {
+          if (error instanceof YAMLException) {
+            return frontmatterYamlDiagnostic(error, file);
+          }
+          throw error;
+        }
         return {
           body: { format, text: parsed.content },
           data: parsed.data,
@@ -90,6 +104,14 @@ export const filesystemSource = (
         };
       })
     );
+    const entries: SourceEntry[] = [];
+    for (const item of read) {
+      if ("body" in item) {
+        entries.push(item);
+      } else {
+        diagnostics.push(item);
+      }
+    }
 
     return { diagnostics, entries };
   };

package/src/core/sources/github-releases.ts

@@ -284,6 +284,17 @@ interface ChangelogFrontmatter {
   type?: string;
 }
 
+/**
+ * The tag a release shows beside its title. A draft (read only with
+ * `drafts`) says so, since nothing else marks it as unreleased.
+ */
+const releaseCategory = (release: GithubRelease): string => {
+  if (release.draft) {
+    return "Draft";
+  }
+  return release.prerelease ? "Prerelease" : "Release";
+};
+
 /**
  * Lower one release to a staged Markdown entry: the notes become the body,
  * `type: changelog` frontmatter (title/date/version/category) drives the
@@ -294,7 +305,7 @@ const releaseToEntry = (release: GithubRelease, site?: string): SourceEntry => {
   const version = release.tag_name.replace(LEADING_V, "");
   const title = release.name?.trim() || release.tag_name;
   const date = release.published_at ?? release.created_at;
-  const category = release.prerelease ? "Prerelease" : "Release";
+  const category = releaseCategory(release);
   // Release notes are the repository's content, not the site author's, so a
   // link whose destination isn't a web, mail, or relative address
   // (`javascript:`, `data:`) keeps only its label (see `safe-links.ts`), and

package/src/core/sources/lexical.ts

@@ -26,6 +26,11 @@ import {
 export interface LexicalOptions {
   /** The CMS origin a relative upload URL (`/media/x.png`) resolves against. */
   baseUrl?: string;
+  /**
+   * Called with the label of each node left out as a comment (a block with
+   * no serializer, a node type with no Markdown), so the source can warn.
+   */
+  onUnsupported?: (what: string) => void;
   /** Serializers for `block`/`inlineBlock` nodes, keyed by `fields.blockType`. */
   serializers?: Record<string, (fields: JsonObject) => string>;
 }
@@ -52,6 +57,12 @@ const marksOf = (node: JsonObject): InlineMarks => {
 
 const children = (node: JsonObject): JsonObject[] => objectsIn(node.children);
 
+/** The comment for a node with no Markdown, reported to `onUnsupported`. */
+const leftOut = (what: string, options: LexicalOptions): string => {
+  options.onUnsupported?.(what);
+  return unsupported(what, writesMdx(options.serializers));
+};
+
 const typeOf = (node: JsonObject): string => asString(node.type) ?? "";
 
 const customBlock = (node: JsonObject, options: LexicalOptions): string => {
@@ -61,10 +72,7 @@ const customBlock = (node: JsonObject, options: LexicalOptions): string => {
   if (fields && serializer) {
     return serializer(fields);
   }
-  return unsupported(
-    `Lexical block${blockType ? `: ${blockType}` : ""}`,
-    writesMdx(options.serializers)
-  );
+  return leftOut(`Lexical block${blockType ? `: ${blockType}` : ""}`, options);
 };
 
 const inlineParts = (
@@ -119,9 +127,9 @@ const renderUpload = (node: JsonObject, options: LexicalOptions): string => {
   const value = asObject(node.value);
   const url = value ? asString(value.url) : undefined;
   if (!(value && url)) {
-    return unsupported(
+    return leftOut(
       "Lexical upload (fetch with depth 1 to populate it)",
-      writesMdx(options.serializers)
+      options
     );
   }
   const alt = asString(value.alt) ?? asString(value.filename) ?? "";
@@ -182,10 +190,7 @@ const renderBlock = (node: JsonObject, options: LexicalOptions): string => {
       return customBlock(node, options);
     }
     default: {
-      return unsupported(
-        `Lexical node: ${typeOf(node)}`,
-        writesMdx(options.serializers)
-      );
+      return leftOut(`Lexical node: ${typeOf(node)}`, options);
     }
   }
 };

package/src/core/sources/mdx-remote.ts

@@ -39,6 +39,26 @@ export interface MdxRemoteSourceOptions {
 // thing on every source type: negation, character classes, nested braces, and
 // extglobs included. Compiled once per enumeration, not per ref.
 
+/**
+ * Whether a ref is included, read the way tinyglobby reads an `include`
+ * array: a `!` pattern excludes what it matches, and the rest include. One
+ * picomatch matcher over the whole array would OR a negation in, and
+ * `!drafts/**` would include every other path, `.txt` files and all.
+ */
+const includeMatcher = (include: string[]): ((ref: string) => boolean) => {
+  // An empty pattern list matches nothing, so an array of only negations
+  // includes nothing, as in tinyglobby.
+  const included = picomatch(
+    include.filter((pattern) => !pattern.startsWith("!"))
+  );
+  const excluded = picomatch(
+    include
+      .filter((pattern) => pattern.startsWith("!"))
+      .map((pattern) => pattern.slice(1))
+  );
+  return (ref) => included(ref) && !excluded(ref);
+};
+
 /** A file to fetch: its source-local ref plus where to read it from. */
 interface RemoteRef {
   ref: string;
@@ -102,12 +122,23 @@ interface GithubTreeEntry {
   type: string;
 }
 
+/** The files a source reads, and what the listing said about them. */
+interface Enumeration {
+  /**
+   * The configured `path`, described, when the repository at the ref has no
+   * folder there, so the source read nothing.
+   */
+  missing?: string;
+  refs: RemoteRef[];
+  truncated: boolean;
+}
+
 /** Enumerate a GitHub repo subtree, mapping blobs to remote refs. */
 const enumerateGithub = async (
   github: { owner: string; repo: string; ref: string; path: string },
   include: string[],
   doFetch: typeof fetch
-): Promise<{ refs: RemoteRef[]; truncated: boolean }> => {
+): Promise<Enumeration> => {
   const { owner, repo, ref } = github;
   const base = github.path.replaceAll(/^\/|\/$/gu, "");
   const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`;
@@ -119,7 +150,7 @@ const enumerateGithub = async (
     truncated?: boolean;
   };
   const prefix = base ? `${base}/` : "";
-  const included = picomatch(include);
+  const included = includeMatcher(include);
   const refs = (body.tree ?? []).flatMap((node) => {
     if (!(node.type === "blob" && node.path.startsWith(prefix))) {
       return [];
@@ -138,7 +169,18 @@ const enumerateGithub = async (
   });
   // GitHub caps the recursive tree response (~100k entries / 7MB) and flags it
   // with `truncated`; ignoring it would silently import only part of the repo.
-  return { refs, truncated: body.truncated === true };
+  const truncated = body.truncated === true;
+  // Git has no empty folders, so a complete listing with nothing under
+  // `path` means the path is wrong at this ref (a moved docs folder, a typo).
+  const found =
+    prefix === "" ||
+    truncated ||
+    (body.tree ?? []).some((node) => node.path.startsWith(prefix));
+  return {
+    missing: found ? undefined : `"${base}" in ${owner}/${repo} at ${ref}`,
+    refs,
+    truncated,
+  };
 };
 
 /**
@@ -169,15 +211,12 @@ export const mdxRemoteSource = (
     });
   };
 
-  const enumerate = async (): Promise<{
-    refs: RemoteRef[];
-    truncated: boolean;
-  }> => {
+  const enumerate = async (): Promise<Enumeration> => {
     if (options.github) {
       return await enumerateGithub(options.github, options.include, doFetch);
     }
     const base = (options.url ?? "").replace(/\/$/u, "");
-    const included = picomatch(options.include);
+    const included = includeMatcher(options.include);
     const refs = (options.files ?? []).flatMap((ref) =>
       included(ref)
         ? [{ editUrl: `${base}/${ref}`, fetchUrl: `${base}/${ref}`, ref }]
@@ -209,12 +248,25 @@ export const mdxRemoteSource = (
       options.name,
       cache,
       async () => {
-        const { refs, truncated } = await enumerate();
+        const { missing, refs, truncated } = await enumerate();
+        if (missing) {
+          skipped.push({
+            code: "BLUME_SOURCE_PATH_MISSING",
+            message: `Source "${options.name}" read no files: there is no folder ${missing}.`,
+            severity: "warning",
+            suggestion:
+              "Check github.path and github.ref against the repository.",
+          });
+        }
         if (truncated) {
+          // The listing covers the whole repository at the ref, whatever
+          // `path` is, so narrowing it doesn't help.
           skipped.push({
             code: "BLUME_SOURCE_TRUNCATED",
-            message: `Source "${options.name}" hit GitHub's tree listing limit; some files were not enumerated. Narrow the source path or split the repo.`,
+            message: `Source "${options.name}" hit GitHub's tree listing limit for the whole repository; some files were not enumerated.`,
             severity: "warning",
+            suggestion:
+              "List the files to read with { url, files } instead, or split the repository.",
           });
         }
         const reasons: string[] = [];