dogsbay 0.2.0-beta.114 → 0.2.0-beta.116

package/dist/audit/deliverable-vars.d.ts

@@ -1,4 +1,4 @@
-import type { Issue } from "./types.js";
+import type { Issue } from "@dogsbay/site-check";
 /**
  * Root identifiers referenced by an expression: quoted strings and
  * dotted property tails stripped, reserved words and numbers excluded.

package/dist/audit/rules/structure/index.d.ts

@@ -1,11 +1,2 @@
-/**
- * Idempotent — calling `registerStructureRules` more than once
- * is a no-op.
- */
-export declare function registerStructureRules(): void;
-/**
- * Test-only: reset the "registered" flag so unit tests can
- * register fresh after `_clearRegistry()`. Production code never
- * calls this.
- */
-export declare function _resetStructureRegistration(): void;
+/** Idempotent, like the shared register* functions. */
+export declare function registerCliStructureRules(): void;

package/dist/audit/rules/structure/index.js

@@ -1,47 +1,17 @@
 /**
- * Structure rule category — register all `structure/*` rules
- * into the global registry. Imported by the audit runner /
- * site-check command at startup.
- *
- * Rules landed in this category:
- *   - structure/namespace-coherence  (corpus, multi-source axis)
- *
- * Future additions:
- *   - structure/orphan-pages         (corpus; pages not reachable from nav)
- *   - structure/duplicate-slugs      (corpus; two sources produced the same URL)
+ * The Astro CLI's own structure rules, registered on top of the shared
+ * ones from `@dogsbay/site-check`. Both read nav files and topic maps
+ * from disk, which dogsbay-ssg resolves differently.
  */
-import { registerRule } from "../../registry.js";
-import { assetRefs } from "./asset-refs.js";
-import { internalLinks } from "./internal-links.js";
-import { localeCoherence } from "./locale-coherence.js";
-import { namespaceCoherence } from "./namespace-coherence.js";
+import { registerRule } from "@dogsbay/site-check";
 import { navTargetExists } from "./nav-target-exists.js";
-import { unresolvedDirectives } from "./unresolved-directives.js";
 import { topicmapValid } from "./topicmap-valid.js";
-import { versionCoherence } from "./version-coherence.js";
 let registered = false;
-/**
- * Idempotent — calling `registerStructureRules` more than once
- * is a no-op.
- */
-export function registerStructureRules() {
+/** Idempotent, like the shared register* functions. */
+export function registerCliStructureRules() {
     if (registered)
         return;
     registered = true;
-    registerRule(namespaceCoherence);
-    registerRule(versionCoherence);
-    registerRule(localeCoherence);
     registerRule(navTargetExists);
-    registerRule(internalLinks);
-    registerRule(assetRefs);
-    registerRule(unresolvedDirectives);
     registerRule(topicmapValid);
 }
-/**
- * Test-only: reset the "registered" flag so unit tests can
- * register fresh after `_clearRegistry()`. Production code never
- * calls this.
- */
-export function _resetStructureRegistration() {
-    registered = false;
-}

package/dist/audit/rules/structure/nav-target-exists.d.ts

@@ -1,2 +1,2 @@
-import type { Rule } from "../../types.js";
+import type { Rule } from "@dogsbay/site-check";
 export declare const navTargetExists: Rule;

package/dist/audit/rules/structure/topicmap-valid.d.ts

@@ -8,5 +8,5 @@
  * the errors to `AuditSourceInfo.topicmapErrors`; this rule just reports
  * them with the standard exit-code/severity plumbing.
  */
-import type { Rule } from "../../types.js";
+import type { Rule } from "@dogsbay/site-check";
 export declare const topicmapValid: Rule;

package/dist/child-links.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Realize `::child-links` on every page against the site's nav
+ * (plans/child-links.md). Shared by `site build` (after the nav is final) and
+ * `convert --to <anything but dogsbay-md>`; Dogsbay MD keeps the directive.
+ */
+import type { ExportPage, NavItem } from "@dogsbay/types";
+export interface SiteChildLinksResult {
+    realized: number;
+    /** `page: folder` for every listing that found no child pages. */
+    empty: string[];
+    /** `page: folder` for listings answered from the page directory, not the nav. */
+    fromDirectory: string[];
+}
+/**
+ * @param prefix the href prefix the nav was built with (`""` for `convert`,
+ *   urlBase + basePath for `site build`): a page's nav href is
+ *   `prefix/slug`, and a `folder=` path is site-relative, so it is prefixed too.
+ */
+export declare function realizeSiteChildLinks(pages: ExportPage[], nav: NavItem[], prefix: string): SiteChildLinksResult;

package/dist/child-links.js

@@ -0,0 +1,48 @@
+import { directoryChildren, hasChildLinks, navKey, realizeChildLinks } from "@dogsbay/serialize-core";
+/**
+ * @param prefix the href prefix the nav was built with (`""` for `convert`,
+ *   urlBase + basePath for `site build`): a page's nav href is
+ *   `prefix/slug`, and a `folder=` path is site-relative, so it is prefixed too.
+ */
+export function realizeSiteChildLinks(pages, nav, prefix) {
+    const out = { realized: 0, empty: [], fromDirectory: [] };
+    if (!pages.some((p) => hasChildLinks(p.tree)))
+        return out;
+    const base = prefix.replace(/\/+$/, "");
+    const hrefOf = (slug) => `${base}/${slug.replace(/(^|\/)index$/, "")}`;
+    const info = new Map();
+    for (const page of pages) {
+        const { description, tags } = page.frontmatter ?? {};
+        info.set(navKey(hrefOf(page.slug)), {
+            title: page.title || undefined,
+            description: typeof description === "string" ? description : undefined,
+            tags: Array.isArray(tags) ? tags.map(String) : undefined,
+            // For `match=` (front-matter filters).
+            data: page.frontmatter,
+        });
+    }
+    // Trees carry SITE-relative links: Astro adds the base path itself, and an
+    // export target (a vault, MkDocs) has none. The nav is built with it on.
+    const outputHref = (href) => base && (href === base || href.startsWith(`${base}/`)) ? href.slice(base.length) || "/" : href;
+    let fallbackChildren;
+    for (const page of pages) {
+        if (!hasChildLinks(page.tree))
+            continue;
+        // Fragments, partials (`_`-prefixed segments) and redirect stubs are not pages to list.
+        fallbackChildren ??= directoryChildren(pages.filter((p) => !p.fragment && !p.redirect && !p.slug.split("/").some((seg) => seg.startsWith("_"))), hrefOf);
+        const r = realizeChildLinks(page.tree, {
+            nav,
+            fallbackChildren,
+            outputHref,
+            pageHref: hrefOf(page.slug),
+            folderHref: (folder) => `${base}/${folder.replace(/^\/+/, "")}`,
+            lookup: (href) => info.get(navKey(href)),
+        });
+        out.realized += r.realized;
+        for (const e of r.empty)
+            out.empty.push(`${page.slug}: ${e}`);
+        for (const e of r.fromFallback)
+            out.fromDirectory.push(`${page.slug}: ${e}`);
+    }
+    return out;
+}

package/dist/commands/convert.js

@@ -4,6 +4,7 @@ import YAML from "yaml";
 import pc from "picocolors";
 import { existsSync, realpathSync } from "node:fs";
 import { realizeIncludes, hasIncludes } from "@dogsbay/serialize-core";
+import { realizeSiteChildLinks } from "../child-links.js";
 import { dogsbayMdToTree } from "@dogsbay/format-dogsbay-md";
 import { FORMATS as formats, DOCUSAURUS_ADAPTERS as docusaurusAdapters, loadAdapter, MDX_ADAPTERS as mdxAdapters, GATSBY_ADAPTERS as gatsbyAdapters, STARLIGHT_ADAPTERS as starlightAdapters, detectFormat, importerNames, exporterNames, } from "../formats.js";
 // Re-exported so tests and programmatic callers keep a stable import site.
@@ -606,6 +607,7 @@ export async function runConvert(source, options, log = console.log) {
         ["--distro", options.distro],
         ["--topic-map", options.topicMap],
         ["--loader", options.loader],
+        ["--api-schema", options.apiSchema],
     ]
         .filter(([flag, value]) => typeof value === "string" && !declared.has(flag) && !CONSUMED_BY_CONVERT.has(flag))
         .map(([flag]) => flag);
@@ -632,6 +634,18 @@ export async function runConvert(source, options, log = console.log) {
     // where the pages actually land.
     if (options.hrefPrefix === undefined)
         options.hrefPrefix = "";
+    // Template partials travel as `{% with %}{% include %}` TEXT that only a
+    // Dogsbay-MD build renders (as template pages). Any other target would show
+    // the directives as literal text, so refuse before importing anything.
+    if (options.partialMode !== undefined && !["inline", "include", "template"].includes(String(options.partialMode))) {
+        throw new ConvertError(`--partial-mode must be inline, include or template (got ${String(options.partialMode)})`, "");
+    }
+    if (options.partialMode === "template" && importer.name !== "starlight") {
+        throw new ConvertError(`--partial-mode template is supported for --from starlight (got --from ${importer.name})`, "Other importers keep their partials inline, or use --partial-mode include (Docusaurus).");
+    }
+    if (options.partialMode === "template" && exporter.name !== "dogsbay-md") {
+        throw new ConvertError(`--partial-mode template needs --to dogsbay-md (got --to ${exporter.name})`, "Template partials are rendered by a Dogsbay-MD build. For other targets use --partial-mode inline.");
+    }
     const importResult = await importer.import(sourceDir, options);
     const { pages, nav, contentRoot } = importResult;
     // Mount the importer's asset directories so the `/_assets/…` refs it
@@ -660,6 +674,18 @@ export async function runConvert(source, options, log = console.log) {
         // legitimately keep shared fragments above the content dir.
         const roots = contentRoot ? [resolve(contentRoot), sourceDir] : [sourceDir];
         includeLoss = realizePageIncludes(pages, roots, log, options.partialMode === "include");
+        // `::child-links` → the listing, from the importer's nav. Dogsbay MD keeps
+        // the directive (the next build lists pages added since).
+        const links = realizeSiteChildLinks(pages, nav, String(options.hrefPrefix ?? ""));
+        if (links.realized) {
+            log(`  Child links: ${links.realized} listing(s) realized from the nav` +
+                (links.fromDirectory.length ? ` (${links.fromDirectory.length} from the page directory: not in the nav)` : ""));
+        }
+        if (links.empty.length) {
+            log(pc.yellow(`  Warning: ${links.empty.length} child-links listing(s) found no child pages:`));
+            for (const e of links.empty.slice(0, 10))
+                log(pc.yellow(`    ${e}`));
+        }
     }
     const output = resolve(options.output || `${source}-${exporter.name}`);
     await exporter.export(pages, nav, output, options);

package/dist/commands/migrate-mkdocs.js

@@ -95,7 +95,7 @@ export async function migrateMkdocs(source, options) {
     console.log(`Output:  ${outputDir}`);
     console.log();
     // 1. Output skeleton. The canonical Dogsbay layout (see
-    //    packages/cli/skills/platform/migration-shape/SKILL.md) puts
+    //    packages/cli/skills/platform/dogsbay-migration-shape/SKILL.md) puts
     //    `content/` and `dogsbay.config.yml` at the root; the
     //    generated Astro project lives under `./astro/` (created by
     //    emitSiteScaffold below).
@@ -122,7 +122,7 @@ export async function migrateMkdocs(source, options) {
     console.log(pc.green(`Wrote`) + ` ${pageCount} pages to ./content/ as Dogsbay-MD`);
     // 3. Convert MkDocs nav: → ./content/nav.yml in canonical
     //    single-key-map shape (- Label: file.md / nested children).
-    //    See packages/cli/skills/platform/nav-file/SKILL.md.
+    //    See packages/cli/skills/platform/dogsbay-nav-file/SKILL.md.
     //    Do NOT emit nav.json — runtime loader prefers it and would
     //    silently override the human-edited yml. The loader auto-
     //    detects nav.yml at the content root.
@@ -305,7 +305,7 @@ async function collectAndWriteContent(sourceDir, fullDocsDir, outputDir, mkdocsC
         // Convert `.md` links to root-relative slugs at parse time.
         // baseUrl: "" gives `/foo` / `/tutorial/quickstart` form (no
         // basePath prefix — site build adds /docs at render time).
-        // See packages/cli/skills/platform/migration-shape/SKILL.md
+        // See packages/cli/skills/platform/dogsbay-migration-shape/SKILL.md
         // → "Internal links — root-relative absolute slugs".
         linkRewrite: { baseUrl: "" },
         // fileInclude: `{* ../../docs_src/foo.py *}` in MkDocs sources
@@ -388,7 +388,7 @@ async function collectAndWriteContent(sourceDir, fullDocsDir, outputDir, mkdocsC
             // resolved relative paths to root-relative `/img/...` form;
             // this walker just adds the `_assets/` prefix so the refs line
             // up with where copyAssets deposits files. See
-            // packages/cli/skills/platform/migration-shape/SKILL.md
+            // packages/cli/skills/platform/dogsbay-migration-shape/SKILL.md
             // → "Asset folder".
             rewriteImageRefsToAssets(tree);
             // No frontmatter title lift in Phase 3: format-mkdocs's
@@ -889,7 +889,7 @@ function copyAssets(srcDocsDir, destPublicDir) {
  *
  * NOT the `{ label: "...", file: "..." }` shape — the runtime
  * validator throws on multi-key entries. See
- * packages/cli/skills/platform/nav-file/SKILL.md.
+ * packages/cli/skills/platform/dogsbay-nav-file/SKILL.md.
  */
 function convertNav(nav) {
     if (!Array.isArray(nav))

package/dist/commands/migrate-myst.d.ts

@@ -5,5 +5,24 @@ export interface MigrateMystOptions {
     siteUrl?: string;
     local?: boolean;
     quiet?: boolean;
+    /** "sphinx" | "translate"; auto when omitted (see chooseMystLoader). */
+    loader?: string;
 }
+/**
+ * Which way to import a MyST project. A Sphinx project (conf.py) built by
+ * Sphinx resolves what the translator cannot see from one page at a time:
+ * `{ref}` to a label on another page, `{term}` to the glossary, extlinks
+ * roles, toctrees across documents. Measured on Ubuntu Server (255 MyST
+ * pages): 4 link errors through Sphinx against 657 translated. The
+ * translator stays the path for mystmd / Jupyter Book (no conf.py), for a
+ * machine without Sphinx, and for a repository whose code should not run.
+ */
+export declare function chooseMystLoader(sourceDir: string, requested: string | undefined, has: {
+    sphinxSrc: boolean;
+    sphinx: boolean;
+}): {
+    loader: "sphinx" | "translate";
+    note?: string;
+    error?: string;
+};
 export declare function migrateMyst(source: string, options: MigrateMystOptions): Promise<void>;

package/dist/commands/migrate-myst.js

@@ -15,12 +15,48 @@
  * Pure Markdown — no external runtime needed. Mirrors migrate-rst /
  * migrate-gitbook in shape.
  */
-import { copyFileSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync, } from "node:fs";
+import { existsSync, mkdirSync, rmSync, statSync, writeFileSync, } from "node:fs";
 import { basename, dirname, join, relative, resolve } from "node:path";
 import pc from "picocolors";
 import { stringify as stringifyYaml } from "yaml";
 import { emitSiteScaffold } from "@dogsbay/format-astro";
 import { importMyst } from "@dogsbay/format-myst/cli";
+import { findSphinxSrc, pythonHasModule, sphinxAvailable } from "@dogsbay/format-rst";
+import { migrateRst, mirrorAssets } from "./migrate-rst.js";
+import { copyOpenApiSpecs } from "../resolve-openapi.js";
+/**
+ * Which way to import a MyST project. A Sphinx project (conf.py) built by
+ * Sphinx resolves what the translator cannot see from one page at a time:
+ * `{ref}` to a label on another page, `{term}` to the glossary, extlinks
+ * roles, toctrees across documents. Measured on Ubuntu Server (255 MyST
+ * pages): 4 link errors through Sphinx against 657 translated. The
+ * translator stays the path for mystmd / Jupyter Book (no conf.py), for a
+ * machine without Sphinx, and for a repository whose code should not run.
+ */
+export function chooseMystLoader(sourceDir, requested, has) {
+    if (requested && requested !== "sphinx" && requested !== "translate") {
+        return { loader: "translate", error: `--loader must be 'sphinx' or 'translate', not '${requested}'` };
+    }
+    if (requested === "translate")
+        return { loader: "translate" };
+    if (requested === "sphinx") {
+        if (!has.sphinxSrc)
+            return { loader: "translate", error: `--loader sphinx needs a conf.py under ${sourceDir}` };
+        if (!has.sphinx)
+            return { loader: "translate", error: "--loader sphinx needs Sphinx (and the project's extensions) for the Python Dogsbay uses (DOGSBAY_PYTHON)" };
+        return { loader: "sphinx" };
+    }
+    if (has.sphinxSrc && has.sphinx) {
+        return { loader: "sphinx", note: "a Sphinx project (conf.py): building it with Sphinx. Use --loader translate to translate the MyST without running project code." };
+    }
+    if (has.sphinxSrc) {
+        return {
+            loader: "translate",
+            note: "a Sphinx project (conf.py), but Sphinx with myst-parser is not installed for DOGSBAY_PYTHON: translating the MyST directly. Cross-page {ref} labels, {term} and extlinks roles will not resolve; install the project's docs requirements for a Sphinx build.",
+        };
+    }
+    return { loader: "translate" };
+}
 function yamlQuote(s) {
     return /[:#&*!|>%@`,'"[\]{}]|^\s|\s$/.test(s) ? JSON.stringify(s) : s;
 }
@@ -65,68 +101,43 @@ function buildConfig(siteName, siteUrl) {
 function frontmatterBlock(title) {
     return `---\n${stringifyYaml({ title }).replace(/\n$/, "")}\n---\n\n`;
 }
-/**
- * Real page/doc assets — an allowlist (not a blocklist) so a MyST project
- * tree's `.py` config helpers, `conf.py`, `requirements.txt`, etc. don't get
- * swept into the migrated content/.
- */
-const ASSET_EXTS = new Set([
-    ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico", ".bmp",
-    ".pdf", ".css", ".js", ".mjs", ".json", ".woff", ".woff2", ".ttf", ".otf",
-    ".mp4", ".webm", ".mov", ".txt", ".csv", ".zip", ".tar", ".gz",
-]);
-const SKIP_DIRS = new Set(["node_modules", ".git", "_build", "_templates", "__pycache__"]);
-/** Mirror asset files (by extension allowlist) from the docs root into content/. */
-function mirrorAssets(root, contentDir) {
-    let count = 0;
-    const walk = (dir) => {
-        let entries;
-        try {
-            entries = readdirSync(dir);
-        }
-        catch {
-            return;
-        }
-        for (const name of entries) {
-            if (SKIP_DIRS.has(name))
-                continue;
-            const full = join(dir, name);
-            let st;
-            try {
-                st = statSync(full);
-            }
-            catch {
-                continue;
-            }
-            if (st.isDirectory()) {
-                if (name.startsWith("."))
-                    continue;
-                walk(full);
-                continue;
-            }
-            const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
-            if (!ASSET_EXTS.has(ext))
-                continue; // skip .md, .py, conf.py, etc.
-            const dest = join(contentDir, relative(root, full));
-            try {
-                mkdirSync(dirname(dest), { recursive: true });
-                copyFileSync(full, dest);
-                count += 1;
-            }
-            catch {
-                /* ignore individual asset failures */
-            }
-        }
-    };
-    walk(root);
-    return count;
-}
 export async function migrateMyst(source, options) {
     const sourceDir = resolve(source);
     if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
         console.log(pc.red(`Error: source must be a directory: ${sourceDir}`));
         process.exit(1);
     }
+    const choice = chooseMystLoader(sourceDir, options.loader, {
+        sphinxSrc: findSphinxSrc(sourceDir) !== null,
+        // Sphinx alone cannot read MyST: myst-parser must be there too, or the
+        // build fails on a machine that has Sphinx for some other project.
+        sphinx: options.loader === "translate" ? false : sphinxAvailable() && pythonHasModule("myst_parser"),
+    });
+    if (choice.error) {
+        console.log(pc.red(`Error: ${choice.error}`));
+        process.exit(1);
+    }
+    if (choice.note)
+        console.log(pc.yellow(`myst: ${choice.note}`));
+    if (choice.loader === "sphinx") {
+        // The same pipeline as migrate-rst: Sphinx builds the MyST pages through
+        // myst-parser, and its XML is converted (toctree nav, labels, glossary,
+        // redirects). It prints the trust warning about running conf.py.
+        try {
+            return await migrateRst(source, { ...options, loader: "sphinx", corpusLabel: "MyST (Sphinx)", throwOnImportError: true });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            // Asked for explicitly: a failure is the answer.
+            if (options.loader === "sphinx") {
+                console.log(pc.red(`Error: ${msg}`));
+                process.exit(1);
+            }
+            // Chosen automatically: the project's extensions may simply not be
+            // installed. Translate instead, and say so.
+            console.log(pc.yellow(`myst: the Sphinx build failed; translating the MyST directly instead.\n  ${msg.split("\n")[1] ?? msg}`));
+        }
+    }
     const siteName = options.siteName?.trim() || prettify(basename(sourceDir));
     const outputDir = resolve(options.output || `${basename(sourceDir)}-dogsbay`);
     const existingConfig = ["yml", "yaml", "json"]
@@ -146,9 +157,9 @@ export async function migrateMyst(source, options) {
     console.log(`Source:  ${sourceDir}`);
     console.log(`Output:  ${outputDir}`);
     console.log();
-    let pages, nav, docsDir, navSource;
+    let pages, nav, docsDir, navSource, excludeAsset;
     try {
-        ({ pages, nav, docsDir, navSource } = await importMyst(sourceDir, {}));
+        ({ pages, nav, docsDir, navSource, excludeAsset } = await importMyst(sourceDir, {}));
     }
     catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
@@ -184,9 +195,15 @@ export async function migrateMyst(source, options) {
     console.log(pc.green(`Converted`) +
         ` ${pages.length} page(s)` +
         (collisions ? pc.yellow(` (${collisions} collision(s) skipped)`) : ""));
-    const assetCount = mirrorAssets(docsDir, contentDir);
+    // migrate-rst's mirror: links are not followed (one in someone else's
+    // repo can name any file on this machine), and what conf.py excludes stays
+    // out.
+    const assetCount = mirrorAssets(docsDir, contentDir, excludeAsset);
     if (assetCount > 0)
         console.log(pc.green(`Mirrored`) + ` ${assetCount} asset(s)`);
+    const specCount = copyOpenApiSpecs(pages, docsDir, contentDir);
+    if (specCount > 0)
+        console.log(pc.green(`Copied`) + ` ${specCount} OpenAPI spec(s) the API reference pages embed`);
     writeFileSync(join(contentDir, "nav.yml"), `# Migrated by dogsbay migrate-myst.\n# Edit freely.\n\n${navToYaml(nav)}\n`);
     console.log(pc.green(`Wrote`) + ` content/nav.yml`);
     writeFileSync(join(outputDir, "dogsbay.config.yml"), buildConfig(siteName, options.siteUrl));

package/dist/commands/migrate-rst.d.ts

@@ -7,5 +7,12 @@ export interface MigrateRstOptions {
     loader?: string;
     local?: boolean;
     quiet?: boolean;
+    /** What the source is called in output; migrate-myst passes "MyST (Sphinx)". */
+    corpusLabel?: string;
+    /** Throw an import failure instead of exiting, so the caller can fall back. */
+    throwOnImportError?: boolean;
 }
+/** Mirror asset files (by extension allowlist) from the docs root into
+ *  content/, skipping what `exclude` names (a root-relative path). */
+export declare function mirrorAssets(root: string, contentDir: string, exclude?: (rel: string) => boolean): number;
 export declare function migrateRst(source: string, options: MigrateRstOptions): Promise<void>;