@kirchdev/duxt 0.1.0

package/modules/git-meta.ts

@@ -0,0 +1,349 @@
+import { execFileSync } from 'node:child_process';
+import { dirname, relative, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import type { Nuxt } from '@nuxt/schema';
+import { readDuxtBuildConfig } from '../duxt-app-config';
+import { duxtManifest, duxtSectionTypes } from '../sections-resolve';
+import { resolveLatestRefs } from '../sources-git';
+
+/**
+ * "Last updated" and the contributor list, from the git history the file
+ * already has.
+ *
+ * Neither needs config: the `sources` entry names the repository, the ref and
+ * the folder, and the page names the file.
+ *
+ * A REMOTE source needs one thing more. Content clones it, but with
+ * `--depth 1`: the checkout on disk holds a single commit, so every file in it
+ * looks as if it were written by whoever cut the tip. That is wrong data, not
+ * missing data, and worse than nothing — so a remote source is read only when
+ * it says `history: true`, and the clone is unshallowed once before it is.
+ * A source read off disk is a full checkout already and needs no flag.
+ *
+ * Hooked on the parse, not on the build, so the answer is cached exactly as
+ * long as it is true: Content re-parses a file when its content changes, which
+ * is the same moment its last-modified date does.
+ */
+interface AfterParseContext {
+  collection?: { name?: string };
+  content?: Record<string, unknown>;
+  file?: { path?: string; id?: string };
+}
+
+export interface DuxtContributor {
+  name: string;
+  commits: number;
+  username?: string;
+}
+
+export default function duxtGitMeta(_options: unknown, nuxt: Nuxt) {
+  const layerDir = fileURLToPath(new URL('..', import.meta.url));
+
+  const dirs = [
+    ...nuxt.options._layers.flatMap((entry) => [
+      entry.config.rootDir,
+      entry.config.srcDir
+    ]),
+    layerDir
+  ].filter(Boolean) as string[];
+
+  const config = readDuxtBuildConfig(dirs);
+  const sources = duxtManifest(
+    resolveLatestRefs(config?.sources ?? [{ path: 'docs' }]),
+    config?.sourceOptions ?? {},
+    duxtSectionTypes(config?.sectionTypes)
+  );
+
+  const wanted = new Map(
+    sources.map((source) => [source.collection, source.history])
+  );
+
+  nuxt.hook(
+    'content:file:afterParse' as never,
+    ((ctx: AfterParseContext) => {
+      const file = ctx.file?.path;
+      const content = ctx.content;
+      if (!file || !content) return;
+
+      // A source that has not asked for its history is left alone — for a
+      // downloaded one that would otherwise answer out of a single-commit
+      // clone, which is wrong data rather than missing data.
+      if (!wanted.get(ctx.collection?.name ?? '')) return;
+
+      if (file.includes('/.data/content/')) unshallow(dirname(file));
+
+      const log = gitLog(file);
+      if (!log.length) return;
+
+      content.lastUpdated ??= log[0]!.date;
+      content.contributors ??= contributorsOf(log);
+    }) as never
+  );
+}
+
+interface Commit {
+  date: string;
+  name: string;
+  email: string;
+}
+
+/**
+ * Turn a `--depth 1` clone into one with a history, once per repository.
+ *
+ * Cached by the repository's own root rather than by the directory the file
+ * sits in: a fetch per page would download the same history once per page.
+ * A failure is silent on purpose — the caller falls back to no history, which
+ * is what the page showed before.
+ */
+const unshallowed = new Set<string>();
+
+function unshallow(dir: string) {
+  let root: string;
+
+  try {
+    root = execFileSync('git', ['-C', dir, 'rev-parse', '--show-toplevel'], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore']
+    }).trim();
+  } catch {
+    return;
+  }
+
+  if (!root || unshallowed.has(root)) return;
+  unshallowed.add(root);
+
+  try {
+    const shallow = execFileSync(
+      'git',
+      ['-C', root, 'rev-parse', '--is-shallow-repository'],
+      { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }
+    ).trim();
+
+    if (shallow !== 'true') return;
+
+    execFileSync('git', ['-C', root, 'fetch', '--unshallow', '--quiet'], {
+      stdio: 'ignore'
+    });
+  } catch {
+    // No network, no remote, or a repository that cannot be deepened. The
+    // page keeps the fields it would have had without this module.
+  }
+}
+
+/**
+ * The unit separator, not a comma or a pipe.
+ *
+ * A commit message is free text and an author name may contain anything a
+ * person is called; the one byte that cannot appear in either is one git will
+ * emit for us with `%x1f`.
+ */
+const SEPARATOR = '\u001f';
+
+/**
+ * The record separator, marking where one commit's header begins.
+ *
+ * `--name-status -z` already ends every path with NUL, so the header needs a
+ * byte of its own to be told apart from one — and `%x01` is a byte no path and
+ * no author name can contain.
+ */
+const RECORD = '\u0001';
+
+/** What `-z` writes between the fields `--name-status` emits. */
+const NUL = '\0';
+
+/**
+ * ONE `git log` PER REPOSITORY, NOT ONE PER PAGE.
+ *
+ * This used to run `git log --follow` against each file as it was parsed. It
+ * was correct, and it cost 40 of the 55 seconds a cold content build took. The
+ * work is not the history — this repository's entire log reads in 70ms — it is
+ * spawning six hundred processes, which on a Linux that is really a virtual
+ * machine costs about 50ms each no matter how little the process then does.
+ *
+ * So the log is read once, whole, and indexed by path. Every page after the
+ * first is a map lookup.
+ *
+ * `--follow` is what that gives up, and it is bought back rather than dropped:
+ * `--name-status -M` reports a rename as `R100 <old> <new>`, so walking the log
+ * newest-first and remembering that `<old>` is now `<new>` lets a file keep the
+ * commits it earned under its former name. It is in fact the wider answer —
+ * git's own documentation calls `--follow` a single-file heuristic, and a file
+ * renamed twice defeats it.
+ */
+export function parseGitLog(output: string): Map<string, Commit[]> {
+  const history = new Map<string, Commit[]>();
+
+  /**
+   * Where a path that no longer exists lives today.
+   *
+   * Filled walking backwards through time, so by the moment an older commit
+   * names `docs/old.md` the rename that made it `docs/new.md` has already been
+   * read, and the commit lands on the file the site actually serves.
+   */
+  const renamedTo = new Map<string, string>();
+  const today = (path: string) => renamedTo.get(path) ?? path;
+
+  for (const record of output.split(RECORD)) {
+    if (!record) continue;
+
+    const header = record.indexOf(NUL);
+    if (header === -1) continue;
+
+    const [date, name, email] = record.slice(0, header).split(SEPARATOR);
+    const commit: Commit = {
+      date: date ?? '',
+      name: name ?? '',
+      email: email ?? ''
+    };
+
+    // The newline git writes after the header belongs to the first token, not
+    // to a path — so every token is trimmed rather than the string patched.
+    const tokens = record
+      .slice(header + 1)
+      .split(NUL)
+      .map((token) => token.trim())
+      .filter(Boolean);
+
+    for (let index = 0; index < tokens.length;) {
+      const status = tokens[index]!;
+
+      // A status is a letter and an optional similarity score. Anything else
+      // means the walk has lost its place, and the rest of this commit is not
+      // worth guessing at.
+      if (!/^[A-Z]\d*$/.test(status)) break;
+
+      // R and C carry two paths; every other status carries one.
+      const pair = status.startsWith('R') || status.startsWith('C');
+      const from = tokens[index + 1];
+      const to = pair ? tokens[index + 2] : from;
+      index += pair ? 3 : 2;
+
+      if (!to) break;
+
+      const path = today(to);
+      const commits = history.get(path);
+
+      if (commits) commits.push(commit);
+      else history.set(path, [commit]);
+
+      // Only a rename carries a name forward. A copy leaves the original where
+      // it was, and that file's history stays its own.
+      if (pair && status.startsWith('R') && from) renamedTo.set(from, path);
+    }
+  }
+
+  return history;
+}
+
+/** One index per repository, built the first time a page asks for it. */
+const histories = new Map<string, Map<string, Commit[]>>();
+
+function historyOf(root: string): Map<string, Commit[]> {
+  const cached = histories.get(root);
+  if (cached) return cached;
+
+  let history: Map<string, Commit[]>;
+
+  try {
+    history = parseGitLog(
+      execFileSync(
+        'git',
+        [
+          '-C',
+          root,
+          'log',
+          `--format=${RECORD}%aI%x1f%an%x1f%ae`,
+          '--name-status',
+          '-M',
+          '-z'
+        ],
+        {
+          encoding: 'utf8',
+          stdio: ['ignore', 'pipe', 'ignore'],
+          // A documentation repository's whole log against execFileSync's
+          // 1 MB default is a truncated history reported as a git failure.
+          maxBuffer: Number.POSITIVE_INFINITY
+        }
+      )
+    );
+  } catch {
+    // Not a checkout, no git, or a history that cannot be read. All three mean
+    // the same thing here: nothing to show, which is not an error.
+    history = new Map();
+  }
+
+  histories.set(root, history);
+  return history;
+}
+
+/** The repository a directory belongs to, asked once per directory. */
+const roots = new Map<string, string | undefined>();
+
+function rootOf(dir: string): string | undefined {
+  if (roots.has(dir)) return roots.get(dir);
+
+  let root: string | undefined;
+
+  try {
+    root =
+      execFileSync('git', ['-C', dir, 'rev-parse', '--show-toplevel'], {
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'ignore']
+      }).trim() || undefined;
+  } catch {
+    root = undefined;
+  }
+
+  roots.set(dir, root);
+  return root;
+}
+
+function gitLog(file: string): Commit[] {
+  const root = rootOf(dirname(file));
+  if (!root) return [];
+
+  // git speaks repository-relative paths with forward slashes, whatever the
+  // platform's own separator is.
+  const path = relative(root, file).split(sep).join('/');
+
+  return historyOf(root).get(path) ?? [];
+}
+
+/**
+ * One entry per person, most commits first.
+ *
+ * Identity is the email, not the name — the same person commits as "Titus
+ * Kirch" and as "titus" and would otherwise appear twice. The GitHub username
+ * is read out of a noreply address, which is the only place git actually
+ * carries one; without it there is a name and no avatar, which is the truth.
+ */
+export function contributorsOf(commits: Commit[]): DuxtContributor[] {
+  const people = new Map<string, DuxtContributor>();
+
+  for (const commit of commits) {
+    const key = commit.email.toLowerCase();
+    const existing = people.get(key);
+
+    if (existing) {
+      existing.commits += 1;
+      continue;
+    }
+
+    people.set(key, {
+      name: commit.name,
+      commits: 1,
+      username: githubUsername(commit.email)
+    });
+  }
+
+  return [...people.values()].sort((a, b) => b.commits - a.commits);
+}
+
+/** `1234567+octocat@users.noreply.github.com` becomes `octocat`. */
+export function githubUsername(email: string): string | undefined {
+  const match = /^(?:\d+\+)?([^@]+)@users\.noreply\.github\.com$/i.exec(
+    email.trim()
+  );
+
+  return match?.[1];
+}

package/modules/redirects.ts

@@ -0,0 +1,152 @@
+import { fileURLToPath } from 'node:url';
+import type { Nuxt } from '@nuxt/schema';
+import { readDuxtBuildConfig } from '../duxt-app-config';
+import { duxtManifest, duxtSectionTypes } from '../sections-resolve';
+import { resolveLatestRefs } from '../sources-git';
+import { readContentCache } from '../content-cache';
+
+/**
+ * `redirectFrom` in a page's frontmatter, turned into route rules.
+ *
+ * A moved page is the one kind of broken link a documentation site creates for
+ * itself, and the fix has always lived outside the site — an nginx rule, a
+ * Netlify `_redirects`, a Cloudflare list — written by whoever deploys it and
+ * never by whoever moved the page. Putting it in the frontmatter puts it in the
+ * same commit as the move.
+ *
+ * The layer is also the only thing that CAN solve it once. An old URL has to be
+ * redirected under every prefix the page is served at — the repository segment,
+ * the version segment and the locale segment — and only the resolved manifest
+ * knows which of those exist. A consumer writing the rule by hand would have to
+ * rediscover that scheme for every page.
+ */
+export default function duxtRedirects(_options: unknown, nuxt: Nuxt) {
+  const layerDir = fileURLToPath(new URL('..', import.meta.url));
+
+  const dirs = [
+    ...nuxt.options._layers.flatMap((entry) => [
+      entry.config.rootDir,
+      entry.config.srcDir
+    ]),
+    layerDir
+  ].filter(Boolean) as string[];
+
+  const config = readDuxtBuildConfig(dirs);
+  const sources = duxtManifest(
+    resolveLatestRefs(config?.sources ?? [{ path: 'docs' }]),
+    config?.sourceOptions ?? {},
+    duxtSectionTypes(config?.sectionTypes)
+  );
+
+  // `nitro:config` rather than `build:done`: route rules are read when Nitro is
+  // configured, and Content has filled the cache by then (it parses in its own
+  // `modules:done` listener, which runs earlier).
+  nuxt.hook('nitro:config', (nitro) => {
+    const pages = readContentCache(
+      nuxt,
+      sources.map((source) => source.collection)
+    );
+
+    if (!pages?.length) return;
+
+    const prefixOf = new Map(
+      sources.map((source) => [source.collection, source.prefix])
+    );
+
+    const rules = redirectRules(
+      pages.map((page) => ({
+        path: page.path,
+        prefix: prefixOf.get(page.collection) ?? '',
+        redirectFrom: Array.isArray(page.content.redirectFrom)
+          ? (page.content.redirectFrom as string[])
+          : []
+      })),
+      localeSegments(nuxt)
+    );
+
+    if (!Object.keys(rules).length) return;
+
+    // Which of the shipped rules are the layer's own, recorded because nothing
+    // in a route rule says who wrote it. The devtools panel reads the rules
+    // back out of the running server rather than recomputing them — the honest
+    // source, and the reason it also lists foreign ones: @nuxtjs/sitemap ships
+    // `/sitemap.xml` the moment a site has more than one sitemap, and a reader
+    // debugging a moved page should not have to wonder which rules are theirs.
+    //
+    // Taken BEFORE the merge below and filtered by what was already claimed:
+    // an existing rule wins, so a path someone else had is not ours to own.
+    const owned = Object.keys(rules).filter(
+      (path) => !nitro.routeRules?.[path]
+    );
+
+    nitro.routeRules = { ...rules, ...nitro.routeRules };
+    nitro.runtimeConfig = {
+      ...nitro.runtimeConfig,
+      duxt: { ...(nitro.runtimeConfig?.duxt as object), redirects: owned }
+    };
+  });
+}
+
+/** Every locale that appears as a URL segment; the default one does not. */
+function localeSegments(nuxt: Nuxt): string[] {
+  const i18n = (
+    nuxt.options as {
+      i18n?: {
+        locales?: (string | { code: string })[];
+        defaultLocale?: string;
+        strategy?: string;
+      };
+    }
+  ).i18n;
+
+  if (!i18n?.locales?.length) return [];
+
+  const codes = i18n.locales.map((locale) =>
+    typeof locale === 'string' ? locale : locale.code
+  );
+
+  return i18n.strategy === 'prefix'
+    ? codes
+    : codes.filter((code) => code !== i18n.defaultLocale);
+}
+
+/**
+ * The rules themselves, over data rather than over a build.
+ *
+ * A `redirectFrom` entry is written the way the page's own links are — relative
+ * to its source, without the prefix the site happens to serve it under. So each
+ * one becomes one rule per locale plus the unprefixed form, all pointing at the
+ * page's real path.
+ */
+export function redirectRules(
+  pages: { path: string; prefix: string; redirectFrom: string[] }[],
+  locales: string[]
+): Record<string, { redirect: { to: string; statusCode: 301 } }> {
+  const rules: Record<string, { redirect: { to: string; statusCode: 301 } }> =
+    {};
+
+  for (const page of pages) {
+    for (const from of page.redirectFrom) {
+      if (!from.startsWith('/')) continue;
+
+      const source = isInsidePrefix(from, page.prefix)
+        ? from
+        : `${page.prefix}${from}`;
+
+      // A page redirecting to itself is a loop, not a redirect.
+      if (source === page.path) continue;
+
+      for (const locale of ['', ...locales]) {
+        const at = locale ? `/${locale}${source}` : source;
+        const to = locale ? `/${locale}${page.path}` : page.path;
+
+        rules[at] ??= { redirect: { to, statusCode: 301 } };
+      }
+    }
+  }
+
+  return rules;
+}
+
+const isInsidePrefix = (path: string, prefix: string) =>
+  Boolean(prefix) && (path === prefix || path.startsWith(`${prefix}/`));

package/modules/validate.ts

@@ -0,0 +1,137 @@
+import { fileURLToPath } from 'node:url';
+import type { Nuxt } from '@nuxt/schema';
+import { readDuxtBuildConfig } from '../duxt-app-config';
+import { duxtManifest, duxtSectionTypes } from '../sections-resolve';
+import { resolveLatestRefs } from '../sources-git';
+import { readContentCache } from '../content-cache';
+import type { Collected, PageRecord } from '../validate-report';
+import { report, walk } from '../validate-report';
+
+/**
+ * Catch the silent failures.
+ *
+ * Every bug this layer actually had was quiet: a collection Content dropped
+ * because its name was not an identifier, a page that 404ed because a link had
+ * gone stale, a navigation that came back empty. None of the three said
+ * anything — the build was green and the site was wrong.
+ *
+ * All four checks read ONE thing: the parse cache, through
+ * `readContentCache`. See that file for why the parse hook is the wrong seam.
+ *
+ * Severity is not uniform, on purpose:
+ *
+ *  - a URL collision or an empty collection is an ERROR, because the page it
+ *    costs is unreachable and no amount of reading the site would say why;
+ *  - a broken link or a missing `title` is a WARNING naming the file, because
+ *    the page still renders and a remote source can go stale between releases
+ *    without that being this build's fault;
+ *  - what each language carries, and what has stood still behind its original,
+ *    is a NOTE — a state of the site rather than a defect in it. It is here
+ *    because nothing else in the layer ever says it out loud, and a translation
+ *    that quietly stopped moving is the failure this whole file is about.
+ */
+
+export default function duxtValidate(_options: unknown, nuxt: Nuxt) {
+  const layerDir = fileURLToPath(new URL('..', import.meta.url));
+
+  const dirs = [
+    ...nuxt.options._layers.flatMap((entry) => [
+      entry.config.rootDir,
+      entry.config.srcDir
+    ]),
+    layerDir
+  ].filter(Boolean) as string[];
+
+  const config = readDuxtBuildConfig(dirs);
+  const sources = duxtManifest(
+    resolveLatestRefs(config?.sources ?? [{ path: 'docs' }]),
+    config?.sourceOptions ?? {},
+    duxtSectionTypes(config?.sectionTypes)
+  );
+
+  // `build:done` rather than `modules:done`: Content fills the cache in a
+  // `modules:done` listener of its own, and this module is registered before
+  // Content so its listener would run first and read the previous build's
+  // answer.
+  nuxt.hook('build:done', () => {
+    const cached = readContentCache(
+      nuxt,
+      sources.map((source) => source.collection)
+    );
+
+    if (!cached) return;
+
+    // The reports come from `modules/config.ts`, which read the artefacts once
+    // and left them on the manifest it wrote into `appConfig`. Merged rather
+    // than read again: a second parse of the same OpenAPI document, to reach
+    // the same answer, is a cost this build does not need — and two parses are
+    // two chances to disagree.
+    const reported = new Map(
+      (
+        (
+          nuxt.options.appConfig.duxt as
+            | { resolvedSources?: typeof sources }
+            | undefined
+        )?.resolvedSources ?? []
+      )
+        .filter((source) => source.generated?.report)
+        .map((source) => [source.collection, source.generated!.report] as const)
+    );
+
+    for (const source of sources) {
+      if (source.generated && reported.has(source.collection)) {
+        source.generated.report = reported.get(source.collection);
+      }
+    }
+
+    const pages: PageRecord[] = cached.map((entry) => {
+      const collected: Collected = {
+        anchors: new Set<string>(),
+        links: [],
+        commands: []
+      };
+      walk(entry.content.body, collected);
+
+      return {
+        collection: entry.collection,
+        path: entry.path,
+        file: entry.file,
+        title:
+          typeof entry.content.title === 'string'
+            ? entry.content.title
+            : undefined,
+        description:
+          typeof entry.content.description === 'string'
+            ? entry.content.description
+            : undefined,
+        anchors: collected.anchors,
+        links: collected.links,
+        commands: collected.commands,
+        lastUpdated:
+          typeof entry.content.lastUpdated === 'string'
+            ? entry.content.lastUpdated
+            : undefined
+      };
+    });
+
+    const problems = report(sources, pages);
+
+    // The translation report is INFORMATION, not a finding: an untranslated
+    // page is a state of the site rather than a defect in it, and printing it
+    // as a warning would train everyone to ignore the warnings.
+    if (problems.notes.length) {
+      console.info('[duxt] translations');
+      for (const note of problems.notes) console.info(`[duxt]   ${note}`);
+    }
+
+    for (const warning of problems.warnings) {
+      console.warn(`[duxt] ${warning}`);
+    }
+
+    if (problems.errors.length) {
+      throw new Error(
+        `duxt: the documentation did not validate.\n  - ${problems.errors.join('\n  - ')}`
+      );
+    }
+  });
+}