@kirchdev/duxt 0.3.4-dev-main.41-23ded41 → 0.3.5-dev-main.43-3310f4b

package/sections.ts

@@ -17,7 +17,6 @@
  * artefact, a type that read nothing out of one — is the severity policy, and
  * it lives in the pure half where a test can reach it.
  */
-import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { defineCollection, defineCollectionSource } from '@nuxt/content';
 import type {
@@ -37,7 +36,13 @@ import {
   resolveGeneratedSections,
   sectionPages
 } from './sections-resolve';
+import {
+  diskSectionInput,
+  sectionArtefactExists,
+  sectionInputKind
+} from './section-input';
 import { resolveLatestRefs } from './sources-git';
+import { localHistoryFor } from './git-contributors';
 import { pageSchema, repositoryRoot } from './sources';
 
 /**
@@ -79,11 +84,17 @@ export function duxtGeneratedCollections(
  * neither.
  */
 function localCollection(entry: DuxtResolvedSource, type: DuxtSectionType) {
-  const file = join(repositoryRoot(), entry.path);
-
-  const pages = existsSync(file)
-    ? sectionPages(entry, type, readFileSync(file, 'utf8'))
-    : missingSectionArtefact(entry, file);
+  // The checkout goes with the handle, which is what lets the release history
+  // read the commits between two tags. `remoteCollection` below has a
+  // `--depth 1` clone and passes one only where that clone is of this very
+  // repository — see `localHistoryFor`.
+  const root = repositoryRoot();
+  const file = join(root, entry.path);
+  const kind = sectionInputKind(type);
+
+  const pages = sectionArtefactExists(file, kind)
+    ? sectionPages(entry, type, diskSectionInput(file, entry.path, kind, root))
+    : missingSectionArtefact(entry, file, type);
 
   const source = defineCollectionSource({
     getKeys: async () => pages.map((page) => page.file),
@@ -137,10 +148,22 @@ function remoteCollection(entry: DuxtResolvedSource, type: DuxtSectionType) {
     // `cwd` is where Content put the checkout, filled in by the `prepare` it
     // installed on this source and run before the first `getKeys`.
     const file = join(source.cwd, entry.path);
-
-    pages = existsSync(file)
-      ? sectionPages(entry, type, readFileSync(file, 'utf8'))
-      : missingSectionArtefact(entry, file);
+    const kind = sectionInputKind(type);
+
+    // The artefact is read from the download; the HISTORY, where the download
+    // is this repository, from the checkout that has all of it.
+    pages = sectionArtefactExists(file, kind)
+      ? sectionPages(
+          entry,
+          type,
+          diskSectionInput(
+            file,
+            entry.path,
+            kind,
+            localHistoryFor(url, repositoryRoot())
+          )
+        )
+      : missingSectionArtefact(entry, file, type);
 
     return pages;
   };

package/server/devtools/shell.ts

@@ -586,7 +586,7 @@ export function page(active: string, body: string, preview = false): string {
    * The tab row is how the devtools panel is navigated, and it is exactly wrong
    * inside a documentation page: the frame sits under prose describing ONE
    * panel, and a reader who switched tabs in it would be reading about Sources
-   * while looking at Paths. The row is also the panel's own chrome — the
+   * while looking at Paths. The row is also the panel's own header — the
    * documentation already carries a heading, a description and a sidebar entry
    * saying which panel this is.
    */

package/server/mcp/tools/list-pages.ts

@@ -1,4 +1,18 @@
-import { queryCollection } from '@nuxt/content/nitro';
+import { z } from 'zod';
+import {
+  DUXT_PAGE_MAX,
+  DUXT_PAGE_SIZE,
+  duxtBadCursor,
+  duxtCursorPage,
+  duxtLocaleSetup,
+  duxtMcpEvent,
+  duxtPageLine,
+  duxtScope,
+  duxtScopedPages,
+  duxtSources,
+  duxtUnknownScope,
+  duxtValidCursor
+} from '../../utils/mcp-docs';
 
 /**
  * The documentation's table of contents, for a model.
@@ -6,47 +20,74 @@ import { queryCollection } from '@nuxt/content/nitro';
  * Built on @nuxtjs/mcp-toolkit rather than a JSON endpoint of our own: it
  * speaks the actual protocol through the official SDK, so a client connects to
  * `/mcp` instead of someone writing a server around our shape.
+ *
+ * BOUNDED AND SCOPED, where it used to return every page of every version and
+ * every language in one unlabelled block. A prefix from `list_versions` picks
+ * the edition; `limit` and `cursor` keep one call's answer readable. Omitting
+ * the prefix still lists the whole site — in its default language, because the
+ * locale is part of the prefix rather than an axis of its own.
  */
-/** Every collection the manifest names — a versioned site has none called `docs`. */
-function duxtCollections(): 'docs'[] {
-  const { duxt } = useAppConfig() as { duxt?: Partial<DuxtConfig> };
-  const names = duxt?.resolvedSources?.length
-    ? duxt.resolvedSources.map((source) => source.collection)
-    : ['docs'];
-  return names as 'docs'[];
-}
-
 export default defineMcpTool({
   name: 'list_pages',
   title: 'List documentation pages',
   description:
-    'Every page of this documentation with its path, title and description. ' +
-    'Start here, then read a page with `read_page`.',
+    'The pages of this documentation with their path, title and description. ' +
+    'Scope with a prefix from `list_versions`, or omit it for every version. ' +
+    'Then read a page with `read_page`.',
   annotations: { readOnlyHint: true },
 
-  // A tool without an `inputSchema` is called with ONE argument: the SDK hands
-  // the handler `extra` directly rather than `(args, extra)`. Written the other
-  // way round, `extra` was undefined and every call failed on `extra.event`.
-  async handler(extra) {
-    const pages = (
-      await Promise.all(
-        duxtCollections().map((name) =>
-          queryCollection(extra.event, name)
-            .select('path', 'title', 'description')
-            .all()
-        )
-      )
-    ).flat();
-
-    const listing = pages
-      .filter((page) => page.path)
-      .sort((a, b) => a.path!.localeCompare(b.path!))
-      .map(
-        (page) =>
-          `- ${page.path} — ${page.title}${page.description ? `: ${page.description}` : ''}`
-      )
-      .join('\n');
-
-    return { content: [{ type: 'text', text: listing }] };
+  inputSchema: {
+    prefix: z
+      .string()
+      .optional()
+      .describe(
+        'URL prefix from `list_versions`, for example /app or /de/app. ' +
+          'Omit for every version in the default language.'
+      ),
+    limit: z
+      .number()
+      .int()
+      .min(1)
+      .max(DUXT_PAGE_MAX)
+      .default(DUXT_PAGE_SIZE)
+      .describe(`How many pages to return, at most ${DUXT_PAGE_MAX}`),
+    cursor: z
+      .string()
+      .optional()
+      .describe('`nextCursor` from a previous call, to continue the listing')
+  },
+
+  async handler({ prefix, limit, cursor }) {
+    if (cursor && !duxtValidCursor(cursor)) return duxtBadCursor(cursor);
+
+    const event = duxtMcpEvent();
+    const setup = duxtLocaleSetup(event);
+    const scope = duxtScope(prefix, duxtSources(), setup);
+    if (!scope) return duxtUnknownScope(prefix!);
+
+    const pages = await duxtScopedPages(event, scope, setup);
+    const { items, nextCursor } = duxtCursorPage(pages, limit, cursor);
+
+    if (!items.length)
+      return {
+        content: [
+          {
+            type: 'text',
+            text: cursor
+              ? `No further pages under ${scope.prefix}.`
+              : `No pages are published under ${scope.prefix}.`
+          }
+        ]
+      };
+
+    const listing = [
+      `${items.length} page(s) under ${scope.prefix}:`,
+      ...items.map(duxtPageLine)
+    ];
+    // The cursor is stated rather than implied: a listing that simply stops at
+    // the limit reads as complete, and an agent then documents half a site.
+    if (nextCursor) listing.push('', `nextCursor: ${nextCursor}`);
+
+    return { content: [{ type: 'text', text: listing.join('\n') }] };
   }
 });

package/server/mcp/tools/list-versions.ts

@@ -0,0 +1,84 @@
+import {
+  duxtDocVersions,
+  duxtLocaleSetup,
+  duxtMcpEvent,
+  duxtSources
+} from '../../utils/mcp-docs';
+
+/**
+ * Which editions of this documentation exist, and which one to read.
+ *
+ * The first call an agent should make, and the one that was missing: a site
+ * publishing `/app`, `/app/v1` and four languages answered `list_pages` with
+ * every page of every one of them, and nothing said which was current. A
+ * reader has a version switcher for this; a model had to infer it from paths.
+ *
+ * What it hands back is the PREFIX — the same string a browser takes — because
+ * that is also the whole of the scope `list_pages` and `search_docs` accept.
+ * No opaque identifier, no repository/version/locale tuple to reassemble.
+ */
+export default defineMcpTool({
+  name: 'list_versions',
+  title: 'List documentation versions',
+  description:
+    'Every version of this documentation: the URL prefix that scopes it, its ' +
+    'label, lifecycle status, the languages it is published in and whether it ' +
+    'is the default. Pass a prefix to `list_pages` or `search_docs`.',
+  annotations: { readOnlyHint: true },
+
+  // NO PARAMETERS, AND THAT IS DELIBERATE. A tool without an `inputSchema` is
+  // called with ONE argument — the SDK hands the handler `extra` rather than
+  // `(args, extra)` — so a handler that names its parameters has to know which
+  // arity it is on. Nothing here needs either of them: the request comes from
+  // `duxtMcpEvent()`, which is where it always had to come from.
+  async handler() {
+    const setup = duxtLocaleSetup(duxtMcpEvent());
+    const versions = duxtDocVersions(duxtSources(), setup);
+
+    // A site whose every source is a generated section publishes no version to
+    // choose between. Saying so beats an empty block an agent reads as a fault.
+    if (!versions.length)
+      return {
+        content: [
+          {
+            type: 'text',
+            text:
+              'This site publishes no versioned documentation. ' +
+              'Call `list_pages` without a prefix.'
+          }
+        ]
+      };
+
+    const listing = versions
+      .map((version) => {
+        // The lifecycle and the default role are different questions, and a
+        // list where three entries look alike says nothing about which of them
+        // is still safe to read.
+        const roles = [version.status, version.isDefault && 'default'].filter(
+          Boolean
+        );
+        const lines = [
+          `- ${version.prefix} — ${version.label} (${roles.join(', ')})`
+        ];
+
+        if (version.locales.length)
+          lines.push(
+            `  locales: ${version.locales
+              .map((locale) => `${locale.code} → ${locale.prefix}`)
+              .join(', ')}`
+          );
+
+        const origin = [
+          version.repository,
+          version.ref && `${version.refKind ?? 'ref'} ${version.ref}`,
+          version.path
+        ].filter(Boolean);
+        if (origin.length) lines.push(`  source: ${origin.join(', ')}`);
+
+        return lines.join('\n');
+      })
+      .join('\n');
+
+    return { content: [{ type: 'text', text: listing }] };
+  }
+});

package/server/mcp/tools/read-page.ts

@@ -1,47 +1,97 @@
 import { queryCollection } from '@nuxt/content/nitro';
 import { z } from 'zod';
+import { splitLocalePath } from '../../../app/utils/locale-path';
+import { sourcesForRoute } from '../../../sources-resolve';
+import {
+  duxtAuthoredMarkdown,
+  duxtFrontmatter,
+  duxtLocaleSetup,
+  duxtMcpEvent,
+  duxtPublicPath,
+  duxtSources
+} from '../../utils/mcp-docs';
 
 /**
- * One page's prose. Returns the stored content rather than rendered HTML — a
- * model reading documentation wants the text, not the theme around it.
+ * One page, as the author wrote it.
+ *
+ * ONE response shape, and it is Markdown: duxt's own YAML frontmatter over the
+ * authored body with the author's frontmatter removed. Before this the tool
+ * returned a title, a description and `JSON.stringify(page.body)` — Content's
+ * parsed AST, which is longer than the page, harder to read than the page, and
+ * not what any other machine-readable surface of this site hands over.
+ *
+ * The frontmatter is duxt's because the author's is gone: it says which
+ * version and which language this text is, which is precisely what a model
+ * holding one page out of four editions cannot otherwise know. `title` rides
+ * along for the same reason — the authored frontmatter carried it, the body
+ * usually has no heading of its own, and stripping one without the other hands
+ * over an untitled page.
+ *
+ * The path is the PUBLIC one, the same string `list_pages` printed, so a round
+ * trip through the two tools needs no translation. A locale segment on it
+ * selects the language, exactly as it does in a browser.
  */
-/** Every collection the manifest names — a versioned site has none called `docs`. */
-function duxtCollections(): 'docs'[] {
-  const { duxt } = useAppConfig() as { duxt?: Partial<DuxtConfig> };
-  const names = duxt?.resolvedSources?.length
-    ? duxt.resolvedSources.map((source) => source.collection)
-    : ['docs'];
-  return names as 'docs'[];
-}
-
 export default defineMcpTool({
   name: 'read_page',
   title: 'Read a documentation page',
-  description: 'The full content of one page. Paths come from `list_pages`.',
+  description:
+    'One page as Markdown, with duxt frontmatter naming its version, language ' +
+    'and lifecycle status. Paths come from `list_pages`.',
   annotations: { readOnlyHint: true },
 
   inputSchema: {
-    path: z.string().describe('Page path, for example /guide/deploying')
+    path: z
+      .string()
+      .describe('Page path from `list_pages`, for example /app/deploying')
   },
 
-  async handler({ path }, extra) {
-    // Try each collection: a path carries its own prefix, so at most one has it.
-    const found = await Promise.all(
-      duxtCollections().map((name) =>
-        queryCollection(extra.event, name).path(path).first()
-      )
+  async handler({ path }) {
+    const event = duxtMcpEvent();
+    const setup = duxtLocaleSetup(event);
+    const sources = duxtSources();
+    const { locale, path: wanted } = splitLocalePath(path, setup.codes);
+
+    // The same chain the rendered page takes: the requested language, its
+    // siblings, the configured fallback, then the untranslated original.
+    const chain = sourcesForRoute(
+      wanted,
+      locale ?? setup.defaultLocale,
+      sources,
+      setup.fallbackLocale
     );
-    const page = found.find(Boolean);
 
-    if (!page) {
+    for (const source of chain) {
+      const page = (await queryCollection(
+        event,
+        source.collection as Parameters<typeof queryCollection>[1]
+      )
+        .path(wanted)
+        .select('title', 'description', 'rawbody')
+        .first()) as unknown as
+        | { title?: string; description?: string; rawbody?: string }
+        | undefined;
+
+      if (!page) continue;
+
+      const frontmatter = duxtFrontmatter({
+        path: duxtPublicPath(wanted, locale, setup),
+        prefix: duxtPublicPath(source.prefix, locale, setup),
+        title: page.title,
+        description: page.description,
+        version: source.version,
+        ref: source.ref,
+        refKind: source.refKind,
+        locale: source.locale,
+        status: source.status
+      });
+
       return {
         content: [
           {
             type: 'text',
-            text: `No page at ${path}. Use \`list_pages\` for valid paths.`
+            text: `${frontmatter}\n\n${duxtAuthoredMarkdown(page.rawbody)}`
           }
-        ],
-        isError: true
+        ]
       };
     }
 
@@ -49,9 +99,13 @@ export default defineMcpTool({
       content: [
         {
           type: 'text',
-          text: `# ${page.title}\n\n${page.description ?? ''}\n\n${JSON.stringify(page.body)}`
+          text:
+            `No page at ${path}. ` +
+            'Use `list_pages` for valid paths, or `list_versions` to pick a ' +
+            'version first.'
         }
-      ]
+      ],
+      isError: true
     };
   }
 });

package/server/mcp/tools/search-docs.ts

@@ -1,74 +1,108 @@
-import { queryCollection } from '@nuxt/content/nitro';
 import { z } from 'zod';
+import {
+  DUXT_PAGE_MAX,
+  DUXT_PAGE_SIZE,
+  duxtBadCursor,
+  duxtCursorPage,
+  duxtExcerpt,
+  duxtLocaleSetup,
+  duxtMcpEvent,
+  duxtPageLine,
+  duxtScope,
+  duxtScopedPages,
+  duxtSources,
+  duxtUnknownScope,
+  duxtValidCursor
+} from '../../utils/mcp-docs';
+import { duxtPageSearchable } from '../../../app/utils/page-controls';
 
 /**
  * Search, so a model does not have to read every page to find one.
  *
- * A LIKE over title and description rather than the FTS index the browser
- * uses: that index is built client-side from the collection dump, and a server
- * tool has the database itself.
+ * THE BODY, not only the title and the description. A LIKE over those two
+ * found "Deploying" and missed every page that merely explains deployment,
+ * which is most of them — a search matching only what is already in the table
+ * of contents tells an agent nothing `list_pages` had not.
+ *
+ * Matched in JavaScript rather than in SQL, and that is the trade written
+ * down: SQLite's LIKE is case-insensitive for ASCII ONLY, so a German page
+ * searched for "übersicht" matched nothing at all, and the excerpt needs the
+ * body in hand regardless. The scope is what keeps the cost bounded — an agent
+ * that names a version pays for that version.
  */
-/** Every collection the manifest names — a versioned site has none called `docs`. */
-function duxtCollections(): 'docs'[] {
-  const { duxt } = useAppConfig() as { duxt?: Partial<DuxtConfig> };
-  const names = duxt?.resolvedSources?.length
-    ? duxt.resolvedSources.map((source) => source.collection)
-    : ['docs'];
-  return names as 'docs'[];
-}
-
 export default defineMcpTool({
   name: 'search_docs',
   title: 'Search the documentation',
-  description: 'Find pages whose title or description matches a term.',
+  description:
+    'Find pages whose title, description or Markdown contains a term, with an ' +
+    'excerpt around the first match. Scope with a prefix from `list_versions`.',
   annotations: { readOnlyHint: true },
 
   inputSchema: {
-    query: z.string().describe('What to look for')
+    query: z.string().min(1).describe('What to look for'),
+    prefix: z
+      .string()
+      .optional()
+      .describe(
+        'URL prefix from `list_versions`, for example /app or /de/app. ' +
+          'Omit to search every version in the default language.'
+      ),
+    limit: z
+      .number()
+      .int()
+      .min(1)
+      .max(DUXT_PAGE_MAX)
+      .default(DUXT_PAGE_SIZE)
+      .describe(`How many results to return, at most ${DUXT_PAGE_MAX}`),
+    cursor: z
+      .string()
+      .optional()
+      .describe('`nextCursor` from a previous call, to continue the results')
   },
 
-  async handler({ query }, extra) {
-    const term = `%${query}%`;
+  async handler({ query, prefix, limit, cursor }) {
+    if (cursor && !duxtValidCursor(cursor)) return duxtBadCursor(cursor);
+
+    const event = duxtMcpEvent();
+    const setup = duxtLocaleSetup(event);
+    const scope = duxtScope(prefix, duxtSources(), setup);
+    if (!scope) return duxtUnknownScope(prefix!);
+
+    const term = query.toLowerCase();
+    const holds = (value: string | undefined) =>
+      Boolean(value?.toLowerCase().includes(term));
 
-    // Both conditions in ONE group. Content joins the query's top-level
-    // conditions with AND regardless of how each was added, so a `where()`
-    // followed by an `orWhere()` asks for a term in the title AND in the
-    // description — which matched nothing at all. Inside a group the operator
-    // is the one the group was opened with, so `orWhere` gives title OR
-    // description.
-    const pages = (
-      await Promise.all(
-        duxtCollections().map((name) =>
-          queryCollection(extra.event, name)
-            .select('path', 'title', 'description')
-            .orWhere((group) =>
-              group
-                .where('title', 'LIKE', term)
-                .where('description', 'LIKE', term)
-            )
-            .all()
-        )
-      )
-    ).flat();
+    // `search: false` is applied HERE rather than in `duxtScopedPages`, which
+    // also feeds `list_pages` and `list_versions`: the opt-out makes a page
+    // un-findable, not unpublished, so the tree and the page itself are left
+    // alone. Same predicate the client search and the llms indexes use.
+    const matches = (await duxtScopedPages(event, scope, setup, true)).filter(
+      (page) =>
+        duxtPageSearchable(page) &&
+        (holds(page.title) || holds(page.description) || holds(page.rawbody))
+    );
 
-    if (!pages.length) {
+    if (!matches.length)
       return {
-        content: [{ type: 'text', text: `Nothing matches "${query}".` }]
+        content: [
+          {
+            type: 'text',
+            text: `Nothing matches "${query}" under ${scope.prefix}.`
+          }
+        ]
       };
-    }
 
-    return {
-      content: [
-        {
-          type: 'text',
-          text: pages
-            .map(
-              (page) =>
-                `- ${page.path} — ${page.title}${page.description ? `: ${page.description}` : ''}`
-            )
-            .join('\n')
-        }
-      ]
-    };
+    const { items, nextCursor } = duxtCursorPage(matches, limit, cursor);
+    const listing = items.flatMap((page) => {
+      const excerpt = duxtExcerpt(page.rawbody, query);
+
+      return excerpt
+        ? [duxtPageLine(page), `  ${excerpt}`]
+        : [duxtPageLine(page)];
+    });
+
+    if (nextCursor) listing.push('', `nextCursor: ${nextCursor}`);
+
+    return { content: [{ type: 'text', text: listing.join('\n') }] };
   }
 });

package/server/middleware/raw-markdown.ts

@@ -3,7 +3,7 @@ import { defineEventHandler, setHeader } from 'h3';
 import { splitLocalePath } from '../../app/utils/locale-path';
 import { sourcesForRoute } from '../../sources-resolve';
 import { duxtDefaults, mergeDuxtConfig } from '../../app/utils/duxt-config';
-import { stripFrontmatter } from '../utils/duxt-server-text';
+import { stripFrontmatter } from '../../frontmatter';
 
 /**
  * `…/guide/deploying.md` serves the page's Markdown source.

package/server/routes/llms-full.txt.get.ts

@@ -3,17 +3,15 @@ import { llmsPages } from '../utils/llms-pages';
 // does not see its utils. The defaults have to come from the same file the
 // pages use, or the two descriptions drift apart.
 import { duxtDefaults, mergeDuxtConfig } from '../../app/utils/duxt-config';
-import {
-  resolveServerTexts,
-  stripFrontmatter
-} from '../utils/duxt-server-text';
+import { resolveServerTexts } from '../utils/duxt-server-text';
+import { stripFrontmatter } from '../../frontmatter';
 
 /**
  * llms-full.txt — the whole documentation as one Markdown file.
  *
- * The companion to `llms.txt`, which is an index: a model that has read the
- * index still has to fetch sixty pages, and a model given this one has read
- * them. The convention (llmstxt.org) names both.
+ * The duxt extension beside `llms.txt`, which is the llms.txt convention's
+ * index: a model that has read the index still has to fetch sixty pages, and a
+ * model given this one has read them.
  *
  * The Markdown as written, not the rendered HTML — which is why the collection
  * schema asks Content for `rawbody`. An MDC block reaches the reader as the
@@ -27,7 +25,7 @@ export default defineEventHandler(async (event) => {
     mergeDuxtConfig(appConfig.duxt, duxtDefaults)
   );
 
-  const pages = await llmsPages(event, duxt.resolvedSources);
+  const pages = await llmsPages(event, duxt.resolvedSources, true);
 
   const origin = getRequestURL(event).origin;
 

package/server/routes/llms.txt.get.ts

@@ -38,7 +38,7 @@ export default defineEventHandler(async (event) => {
       .sort((a, b) => a.path!.localeCompare(b.path!))
       .map(
         (page) =>
-          `- [${page.title ?? page.path}](${origin}${page.path})` +
+          `- [${page.title ?? page.path}](${origin}${page.path}.md)` +
           (page.description ? `: ${page.description}` : '')
       ),
     ''

package/server/utils/duxt-server-text.ts

@@ -31,16 +31,3 @@ function lookup(key: string): string | undefined {
 export function resolveServerTexts<T>(config: T): T {
   return resolveDuxtTexts(config, 'en-GB', lookup);
 }
-
-/**
- * The Markdown body without its frontmatter block.
- *
- * Content's `rawbody` is the file as it sits on disk, frontmatter included. A
- * reader wants the prose; a model handed `---\ntitle: …\n---` reads a page
- * whose first heading is a YAML fence.
- */
-export function stripFrontmatter(body: string): string {
-  const match = /^---\r?\n[\s\S]*?\r?\n---\r?\n?/.exec(body);
-
-  return match ? body.slice(match[0].length) : body;
-}