dogsbay 0.2.0-beta.91 → 0.2.0-beta.93

package/dist/commands/add.js

@@ -22,7 +22,7 @@ function getSourceDir() {
         return monorepo;
     throw new Error("Could not find component source files.");
 }
-function collectDependencies(names) {
+export function collectDependencies(names) {
     const visited = new Set();
     const components = [];
     const npmDeps = [];

package/dist/commands/migrate-asciidoc.js

@@ -8,11 +8,13 @@
  * and runs `dogsbay site build`; the AsciiDoc source can be
  * deleted.
  *
- * Phase 8b of plans/format-asciidoc-import.md. Plain-corpus loader
- * only; topology-aware loaders (AsciiBinder `_topic_map.yml`,
- * Antora playbooks, AAP `master.adoc` walks) ship in Phase 9 per
- * plans/asciidoc-corpus-loaders.md and will hook in here without
- * changing the output shape.
+ * Corpus discovery is delegated to the `@dogsbay/format-asciidoc`
+ * CorpusLoader registry (see `asciidocPlugin.import` below), so the
+ * topology-aware loaders are live: detection auto-picks AsciiBinder
+ * (`_topic_maps/_topic_map.yml`), AAP (`master.adoc`), Antora component
+ * (`antora.yml`), or the plain `.adoc` walk, and topology files never
+ * accidentally become pages. Override with `--loader`.
+ * See plans/asciidoc-corpus-loaders.md.
  *
  * Output layout — same convention as `migrate-mkdocs`:
  *
@@ -29,10 +31,11 @@ import { copyFileSync, cpSync, existsSync, lstatSync, mkdirSync, readFileSync, r
 import { basename, dirname, join, relative, resolve } from "node:path";
 import pc from "picocolors";
 import { emitSiteScaffold } from "@dogsbay/format-astro";
-import { plugin as asciidocPlugin } from "@dogsbay/format-asciidoc/cli";
+import { plugin as asciidocPlugin, pickLoader } from "@dogsbay/format-asciidoc/cli";
 import { asciidocToMarkdown } from "@dogsbay/adoc2md-modular";
 import { treeToDogsbayMd } from "@dogsbay/format-dogsbay-md";
 import { ensureGitignore } from "../utils/gitignore.js";
+import { inlineRemoteIncludes, hasRemoteIncludes } from "../utils/remote-includes.js";
 function parseAttributePairs(pairs) {
     const out = {};
     for (const raw of pairs ?? []) {
@@ -211,6 +214,9 @@ function convertFragmentAdocs(sourceRoot, outputContentDir, scopedRootDirs, page
     let converted = 0;
     let failed = 0;
     let skipped = 0;
+    // Fragment .md files that carry a `{% include "https://…" %}` directive —
+    // resolved by an async fetch pass after the (sync) walk completes. (#007)
+    const remoteIncludePaths = [];
     function walk(srcDir, dstDir, depth) {
         let entries;
         try {
@@ -302,6 +308,8 @@ function convertFragmentAdocs(sourceRoot, outputContentDir, scopedRootDirs, page
                 // pursue option A through a non-frontmatter signal that
                 // can't collide with Jinja whitespace control.
                 writeFileSync(mdPath, md, "utf-8");
+                if (hasRemoteIncludes(md))
+                    remoteIncludePaths.push(mdPath);
                 converted += 1;
             }
             catch (err) {
@@ -312,7 +320,7 @@ function convertFragmentAdocs(sourceRoot, outputContentDir, scopedRootDirs, page
         }
     }
     walk(sourceRoot, outputContentDir, 0);
-    return { converted, failed, skipped };
+    return { converted, failed, skipped, remoteIncludePaths };
 }
 /**
  * Serialize a NavItem[] (from a CorpusLoader) to nav.yml syntax.
@@ -400,6 +408,21 @@ function buildDogsbayConfig(opts) {
     return lines.join("\n");
 }
 function buildMigrationNotes(opts) {
+    // Report the loader that actually ran rather than assuming a plain
+    // walk. The non-plain loaders read a topology source for the nav and
+    // route only topology-declared pages; `modules/` / `snippets/` /
+    // `_attributes/` are pulled in as include targets, not pages.
+    const loaderDesc = {
+        asciibinder: "AsciiBinder topology (`_topic_maps/_topic_map.yml`)",
+        "master-adoc": "AAP `master.adoc` topology",
+        antora: "Antora component (`antora.yml`)",
+        plain: "plain AsciiDoc — flat `.adoc` walk",
+    };
+    const corpusDesc = loaderDesc[opts.loader] ?? `\`${opts.loader}\` loader`;
+    const distroNote = opts.distro ? `, distro \`${opts.distro}\`` : "";
+    const topologyDidnt = opts.loader === "plain"
+        ? "- **Topology**: the plain loader walked every `.adoc` flat and derived the nav from the directory layout. If your source is actually AsciiBinder (`_topic_maps/_topic_map.yml`), Antora (`antora.yml`), or AAP (`master.adoc`), re-run with `--loader <name>` (or rely on auto-detection) to use the curated topology and per-page distro filtering instead.\n"
+        : "";
     const hasAttrs = Object.keys(opts.attributes).length > 0;
     const attrSummary = hasAttrs
         ? Object.entries(opts.attributes)
@@ -416,7 +439,7 @@ Generated by \`dogsbay migrate-asciidoc\`.
 ## Summary
 
 - **Site name:** ${opts.siteName}
-- **Source corpus:** \`${opts.sourceDir}\` (plain AsciiDoc — \`.adoc\` walk)
+- **Source corpus:** \`${opts.sourceDir}\` (${corpusDesc}${distroNote})
 - **Output:** \`${opts.outputDir}\`
 - **Pages migrated:** ${opts.pageCount}
 
@@ -455,8 +478,7 @@ See \`docs/asciidoc.md\` for the full preprocessor behaviour.
 
 ## What didn't (yet)
 
-- **Topology**: this migration walked every \`.adoc\` under \`${opts.sourceDir}\` as a flat plain-AsciiDoc corpus. If your source uses AsciiBinder \`_topic_map.yml\`, Antora \`antora-playbook.yml\`, or AAP \`master.adoc\` discovery, those topologies are not yet understood — the dedicated corpus loaders are planned for a future release (see \`plans/asciidoc-corpus-loaders.md\`).
-- **Line-form admonition layout**: \`NOTE: foo\` lines always emit the HTML-shaped \`**📌 NOTE**\\\` prefix regardless of options. Block-form \`[NOTE]\\n====\\n…\\n====\` does respect the configured \`admonitionFormat\`.
+${topologyDidnt}- **Line-form admonition layout**: \`NOTE: foo\` lines always emit the HTML-shaped \`**📌 NOTE**\\\` prefix regardless of options. Block-form \`[NOTE]\\n====\\n…\\n====\` does respect the configured \`admonitionFormat\`.
 
 ## Next steps
 
@@ -540,6 +562,7 @@ export async function migrateAsciidoc(source, options) {
         attributes: engineAttributes,
         loader: options.loader,
         distro: options.distro,
+        topicMap: options.topicMap,
     });
     if (importedPages.length === 0) {
         console.log(pc.red(`Error: no .adoc files found under ${sourceDir}`));
@@ -548,6 +571,13 @@ export async function migrateAsciidoc(source, options) {
     }
     console.log(`Pages:   ${importedPages.length}`);
     console.log();
+    // Which loader did the import auto-pick? Re-run detection (cheap —
+    // filesystem existence checks) so MIGRATION.md reports the topology
+    // that actually ran, not a hardcoded "plain walk" string. The import
+    // above already succeeded, so this won't throw on a no-match.
+    const loaderName = pickLoader(sourceDir, options.loader, {
+        topicMap: options.topicMap,
+    }).name;
     // 3. Write each page to disk with title frontmatter.
     //
     // When the importer populated `page.bodyMarkdown` (currently only
@@ -620,6 +650,26 @@ export async function migrateAsciidoc(source, options) {
             (frag.skipped > 0 ? ` (${frag.skipped} already-written page paths skipped)` : "") +
             (frag.failed > 0 ? ` (${frag.failed} failed)` : ""));
     }
+    // 3c-bis. Resolve remote `include::https://…[]` directives: fetch each URL
+    // once and bake it into the fragment as a fenced code block, so the build
+    // has no runtime network dependency and the content isn't dropped. (#007)
+    if (frag.remoteIncludePaths.length > 0) {
+        const cache = new Map();
+        let inlined = 0;
+        for (const mdPath of frag.remoteIncludePaths) {
+            const before = readFileSync(mdPath, "utf-8");
+            const after = await inlineRemoteIncludes(before, { cache });
+            if (after !== before) {
+                writeFileSync(mdPath, after, "utf-8");
+                inlined += 1;
+            }
+        }
+        const failures = [...cache.values()].filter((v) => v === null).length;
+        console.log(pc.green(`Inlined`) +
+            ` remote includes in ${inlined} fragment(s) (${cache.size} URL(s)` +
+            (failures > 0 ? `, ${failures} unreachable` : "") +
+            `)`);
+    }
     // 3d. Mirror non-`.adoc` filesystem entries within the scoped
     //     set: symlinks (critical for AsciiBinder include resolution
     //     where per-section dirs symlink back to root `_attributes/`,
@@ -695,6 +745,8 @@ export async function migrateAsciidoc(source, options) {
         pageCount: importedPages.length,
         attributes,
         rerunCmd,
+        loader: loaderName,
+        distro: options.distro,
     }));
     console.log(pc.green(`Wrote`) + ` MIGRATION.md`);
     if (!options.quiet) {

package/dist/commands/migrate-fluidtopics-rest.js

@@ -0,0 +1,149 @@
+/**
+ * `migrate-fluidtopics-rest` — migrate a *live* Fluid Topics portal (via the
+ * khub REST API) into a scaffolded Dogsbay-MD site. Reverse-engineers DITA-OT
+ * HTML5 → TreeNode → Dogsbay-MD. Works on any public FT portal.
+ *
+ * Same output shape as `migrate-fluidtopics` (Option A, Git Markdown export);
+ * this is Option B (live REST).
+ */
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import pc from "picocolors";
+import YAML from "yaml";
+import { emitSiteScaffold } from "@dogsbay/format-astro";
+import { serializeConfig } from "../config/serialize.js";
+export async function migrateFluidtopicsRest(source, options) {
+    if (!/^https?:\/\//i.test(source)) {
+        console.log(pc.red(`Error: source must be a portal URL (e.g. https://www.servicenow.com/docs)`));
+        process.exit(1);
+    }
+    const outputDir = resolve(options.output || "fluidtopics-rest-dogsbay");
+    const existingConfig = ["yml", "yaml", "json"]
+        .map((ext) => join(outputDir, `dogsbay.config.${ext}`))
+        .find(existsSync);
+    if (existingConfig && !options.force) {
+        console.log(pc.red(`Error: ${outputDir} already contains a Dogsbay site (${existingConfig}). Pass --force.`));
+        process.exit(1);
+    }
+    const siteName = options.siteName || "Documentation";
+    const basePath = options.basePath !== undefined ? options.basePath : "";
+    const hrefPrefix = basePath;
+    console.log();
+    console.log(pc.bold(`Migrating Fluid Topics portal (REST) to Dogsbay-MD: ${siteName}`));
+    console.log(`Portal:  ${source}`);
+    console.log(`Output:  ${outputDir}`);
+    console.log();
+    const contentDir = join(outputDir, "content");
+    const astroDir = join(outputDir, "astro");
+    mkdirSync(contentDir, { recursive: true });
+    mkdirSync(astroDir, { recursive: true });
+    const { importFluidTopicsRest } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-fluidtopics-rest"));
+    const { treeToDogsbayMd } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-dogsbay-md"));
+    const { pages, nav, unresolved, mapsProcessed, assets } = await importFluidTopicsRest({
+        portalBase: source,
+        maps: options.map,
+        titleFilter: options.titleFilter,
+        groupBy: options.groupBy,
+        singlePageBooks: options.singlePageBooks,
+        keepDuplicateMaps: options.keepDuplicateMaps,
+        metadataAllow: options.metadata?.split(",").map((s) => s.trim()).filter(Boolean),
+        fullTopicMetadata: options.fullTopicMetadata,
+        // Custom facets: editorialType (book/article), version, and subject (the
+        // DITA subjectScheme "Subject matter" / PBS facet, now lifted per-topic).
+        // category & audience are built-in. parseMeta lifts these into
+        // page.meta.taxonomies → DocsLayout emits data-pagefind-filter → Pagefind
+        // facets. (#search-facets)
+        taxonomyNames: ["editorialType", "version", "subject"],
+        lang: options.lang,
+        hrefPrefix,
+        throttleMs: options.throttle ? Number(options.throttle) : undefined,
+        singlePage: options.singlePage,
+        onProgress: (m) => console.log(pc.dim(`  ${m}`)),
+    });
+    const unit = options.singlePage ? "page(s) (one per book)" : "topics";
+    console.log(pc.green(`Imported`) + ` ${pages.length} ${unit} from ${mapsProcessed} map(s)`);
+    // Write extracted images as real content-rooted assets (per _assets convention).
+    for (const a of assets) {
+        const dest = join(contentDir, a.path);
+        mkdirSync(dirname(dest), { recursive: true });
+        writeFileSync(dest, a.data);
+    }
+    if (assets.length)
+        console.log(pc.green(`Wrote`) + ` ${assets.length} images to content/_assets/`);
+    for (const page of pages) {
+        const dest = join(contentDir, `${page.slug || "index"}.md`);
+        mkdirSync(dirname(dest), { recursive: true });
+        const body = treeToDogsbayMd(page.tree, { frontmatter: page.frontmatter });
+        writeFileSync(dest, body.endsWith("\n") ? body : `${body}\n`);
+    }
+    console.log(pc.green(`Wrote`) + ` ${pages.length} pages to ./content/`);
+    writeFileSync(join(contentDir, "nav.yml"), YAML.stringify(navItemsToSingleKey(nav, basePath)));
+    console.log(pc.green(`Wrote`) + ` content/nav.yml (${nav.length} top-level items)`);
+    const config = {
+        schemaVersion: 1,
+        site: { name: siteName, url: options.siteUrl, basePath },
+        content: { sources: [{ path: "./content", from: "dogsbay-md" }] },
+        // FT faceted metadata → search filters. `category` / `audience` are
+        // built-in facets (no declaration needed); `editorialType` (book/article),
+        // `version`, and `subject` (the DITA subjectScheme "Subject matter" / PBS
+        // facet) are declared here so the build extracts them into each page's
+        // taxonomies, DocsLayout emits the data-pagefind-filter divs, and Pagefind
+        // surfaces them as facet columns. Also gives browseable term index pages
+        // (/format, /version, /subject). See docs-dev/fluidtopics-rest-importer.md.
+        taxonomies: {
+            editorialType: { indexPath: "/format", order: 2 },
+            version: { indexPath: "/version", order: 3 },
+            subject: { indexPath: "/subject", order: 4 },
+        },
+        agent: { llmsTxt: true, mdMirror: true },
+    };
+    writeFileSync(join(outputDir, "dogsbay.config.yml"), serializeConfig(config, "yaml"));
+    emitSiteScaffold(astroDir, siteName, { siteName, siteUrl: options.siteUrl, basePath, llmsTxt: true, mdMirror: true, local: options.local }, true);
+    const unresolvedCount = Object.values(unresolved).reduce((n, a) => n + a.length, 0);
+    writeFileSync(join(outputDir, "MIGRATION.md"), `# Migration notes — ${siteName} (Fluid Topics REST)\n\nMigrated live from \`${source}\` via the khub API; DITA-OT HTML5 reverse-engineered to Dogsbay-MD.\n\n- Maps: **${mapsProcessed}**\n- Pages: **${pages.length}**\n- Unresolved in-content links: **${unresolvedCount}**\n\nMedia and any conref/keyref/profiling were already resolved/flattened upstream by FT's DITA-OT run.\n`);
+    console.log(pc.green(`Scaffolded`) + ` Astro project + config + MIGRATION.md`);
+    if (!options.quiet) {
+        console.log();
+        console.log(pc.bold("Done."));
+        console.log(`  cd ${outputDir} && npx dogsbay site dev`);
+    }
+}
+function navItemsToSingleKey(items, basePath) {
+    const out = [];
+    for (const item of items) {
+        // A node carrying a single-page annotation uses the EXPLICIT nav form
+        // (`{ title, single-page, items }`) so the annotation survives into nav.yml;
+        // unannotated nodes keep the compact single-key form.
+        if (item.singlePage) {
+            const entry = { title: item.label, "single-page": item.singlePage };
+            if (item.children && item.children.length > 0) {
+                entry.items = navItemsToSingleKey(item.children, basePath);
+            }
+            else if (item.href) {
+                entry.file = hrefToNavPath(item.href, basePath);
+            }
+            out.push(entry);
+        }
+        else if (item.children && item.children.length > 0) {
+            out.push({ [item.label]: navItemsToSingleKey(item.children, basePath) });
+        }
+        else if (item.href) {
+            out.push({ [item.label]: hrefToNavPath(item.href, basePath) });
+        }
+        else {
+            out.push({ [item.label]: [] });
+        }
+    }
+    return out;
+}
+function hrefToNavPath(href, basePath) {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href))
+        return href;
+    let s = href.split("#")[0];
+    if (basePath && s.startsWith(basePath))
+        s = s.slice(basePath.length);
+    s = s.replace(/^\/+/, "");
+    return s ? `${s}.md` : "index.md";
+}

package/dist/commands/migrate-fluidtopics.js

@@ -0,0 +1,158 @@
+/**
+ * `migrate-fluidtopics` — one-way migrate a Fluid Topics Markdown export
+ * (e.g. the ServiceNow/ServiceNowDocs repo) into a scaffolded Dogsbay-MD site.
+ *
+ * Canonical migration layout (see the migration-shape skill):
+ *   <output>/
+ *     content/            <- Dogsbay-MD, the new source of truth
+ *       <pub>/<topic>.md
+ *       nav.yml
+ *     astro/              <- generated Astro project (emitSiteScaffold)
+ *     dogsbay.config.yml
+ *     MIGRATION.md
+ *
+ * Mirrors `migrate-mkdocs`, reusing the shared emitters
+ * (importer -> treeToDogsbayMd -> emitSiteScaffold + serializeConfig).
+ */
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join, resolve, basename, dirname } from "node:path";
+import pc from "picocolors";
+import YAML from "yaml";
+import { emitSiteScaffold } from "@dogsbay/format-astro";
+import { serializeConfig } from "../config/serialize.js";
+export async function migrateFluidtopics(source, options) {
+    const sourceDir = resolve(source);
+    const markdownDir = join(sourceDir, "markdown");
+    if (!existsSync(markdownDir)) {
+        console.log(pc.red(`Error: no markdown/ directory found in ${sourceDir}`));
+        console.log("Point this at a Fluid Topics Markdown export (e.g. a ServiceNowDocs checkout).");
+        process.exit(1);
+    }
+    const outputDir = resolve(options.output || `${basename(sourceDir)}-dogsbay`);
+    const existingConfig = ["yml", "yaml", "json"]
+        .map((ext) => join(outputDir, `dogsbay.config.${ext}`))
+        .find(existsSync);
+    if (existingConfig && !options.force) {
+        console.log(pc.red(`Error: ${outputDir} already contains a Dogsbay site (${basename(existingConfig)}).`));
+        console.log("Pass --force to overwrite.");
+        process.exit(1);
+    }
+    const siteName = options.siteName || "Documentation";
+    // A dedicated docs repo serves at the host root by default (matches the
+    // openshift-docs-dogsbay / GitHub Pages project-site convention).
+    const basePath = options.basePath !== undefined ? options.basePath : "";
+    const hrefPrefix = basePath; // root-relative slugs when basePath is ""
+    console.log();
+    console.log(pc.bold(`Migrating Fluid Topics Markdown export to Dogsbay-MD: ${siteName}`));
+    console.log(`Source:  ${sourceDir}`);
+    console.log(`Output:  ${outputDir}`);
+    console.log();
+    const contentDir = join(outputDir, "content");
+    const astroDir = join(outputDir, "astro");
+    mkdirSync(contentDir, { recursive: true });
+    mkdirSync(astroDir, { recursive: true });
+    // 1. Import the export -> pages + nav.
+    const { importFluidTopicsProject } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-fluidtopics-markdown"));
+    const { treeToDogsbayMd } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-dogsbay-md"));
+    const { pages, nav, unresolved } = await importFluidTopicsProject(sourceDir, {
+        hrefPrefix,
+        publications: options.publication,
+    });
+    if (options.publication && options.publication.length > 0) {
+        console.log(pc.dim(`Scoped to publications: ${options.publication.join(", ")}`));
+    }
+    // 2. Write each page to ./content/<slug>.md as Dogsbay-MD.
+    for (const page of pages) {
+        const rel = `${page.slug || "index"}.md`;
+        const dest = join(contentDir, rel);
+        mkdirSync(dirname(dest), { recursive: true });
+        const body = treeToDogsbayMd(page.tree, { frontmatter: page.frontmatter });
+        writeFileSync(dest, body.endsWith("\n") ? body : `${body}\n`);
+    }
+    console.log(pc.green("Wrote") + ` ${pages.length} pages to ./content/ as Dogsbay-MD`);
+    // 3. nav.yml (single-key-map; file paths relative to content/).
+    const navYml = navItemsToSingleKey(nav, basePath);
+    writeFileSync(join(contentDir, "nav.yml"), YAML.stringify(navYml));
+    console.log(pc.green("Extracted") + ` navigation to content/nav.yml (${navYml.length} top-level items)`);
+    // 4. dogsbay.config.yml.
+    const dogsbayConfig = {
+        schemaVersion: 1,
+        site: { name: siteName, url: options.siteUrl, basePath },
+        content: { sources: [{ path: "./content", from: "dogsbay-md" }] },
+        agent: { llmsTxt: true, mdMirror: true },
+    };
+    writeFileSync(join(outputDir, "dogsbay.config.yml"), serializeConfig(dogsbayConfig, "yaml"));
+    console.log(pc.green("Wrote") + " dogsbay.config.yml");
+    // 5. Scaffold the Astro project under ./astro/.
+    emitSiteScaffold(astroDir, siteName, { siteName, siteUrl: options.siteUrl, basePath, llmsTxt: true, mdMirror: true, local: options.local }, true);
+    console.log(pc.green("Scaffolded") + " Astro project at ./astro/");
+    // 6. MIGRATION.md.
+    const unresolvedCount = Object.values(unresolved).reduce((n, a) => n + a.length, 0);
+    writeFileSync(join(outputDir, "MIGRATION.md"), buildMigrationNotes({ sourceDir, siteName, pageCount: pages.length, unresolvedCount, basePath }));
+    console.log(pc.green("Wrote") + " MIGRATION.md");
+    if (!options.quiet) {
+        console.log();
+        console.log(pc.bold("Done! Your Dogsbay-MD site is ready."));
+        console.log();
+        console.log("Next steps:");
+        console.log(`  cd ${outputDir}`);
+        console.log("  npx dogsbay site dev          # live preview (auto-installs)");
+        console.log();
+        console.log("Edit ./content/*.md (canonical Dogsbay-MD) from here on.");
+    }
+}
+function navItemsToSingleKey(items, basePath) {
+    const out = [];
+    for (const item of items) {
+        if (item.children && item.children.length > 0) {
+            out.push({ [item.label]: navItemsToSingleKey(item.children, basePath) });
+        }
+        else if (item.href) {
+            out.push({ [item.label]: hrefToNavPath(item.href, basePath) });
+        }
+        else {
+            out.push({ [item.label]: [] });
+        }
+    }
+    return out;
+}
+/** Convert a resolved route href back to a content-relative .md path. */
+function hrefToNavPath(href, basePath) {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href))
+        return href; // external
+    let s = href.split("#")[0];
+    if (basePath && s.startsWith(basePath))
+        s = s.slice(basePath.length);
+    s = s.replace(/^\/+/, "");
+    if (!s)
+        return "index.md";
+    return `${s}.md`;
+}
+function buildMigrationNotes(args) {
+    return [
+        `# Migration notes — ${args.siteName}`,
+        "",
+        `Migrated from a Fluid Topics Markdown export at \`${args.sourceDir}\`.`,
+        "",
+        `- Pages: **${args.pageCount}**`,
+        `- Base path: \`${args.basePath || "(root)"}\``,
+        `- Unresolved internal links: **${args.unresolvedCount}** (broken cross-references in the source export are left as-is).`,
+        "",
+        "## What this did",
+        "",
+        "- Converted each topic to Dogsbay-MD under `./content/` (DITA `topic_type` → frontmatter `type`).",
+        "- Embedded HTML tables → Markdown tables; DITA nav-card grids → `:::cards`.",
+        "- Resolved both link schemes (relative `.md` + `raw.githubusercontent` TOC URLs) to routes.",
+        "- Built `content/nav.yml` from each publication's `index.md` TOC + the README order.",
+        "",
+        "## Known gaps",
+        "",
+        "- **Media is omitted** in the source export — image references may be dangling. Pull assets from the live portal or replace as needed.",
+        "- Conref/keyref/profiling were already resolved/flattened by the export (not recoverable here).",
+        "",
+        "Edit `./content/*.md` from here on; re-run the migration to refresh.",
+        "",
+    ].join("\n");
+}

package/dist/config/load.js

@@ -104,6 +104,15 @@ function validate(parsed, sourcePath) {
     const propSchema = validatePropSchema(obj.propSchema, sourcePath);
     const plugins = validatePlugins(obj.plugins, sourcePath);
     const attributes = validateAttributes(obj.attributes, "attributes", sourcePath);
+    if (obj.granularity !== undefined) {
+        if (typeof obj.granularity !== "object" || obj.granularity === null) {
+            throw new Error(`granularity must be an object in ${sourcePath}`);
+        }
+        const g = obj.granularity;
+        if (g.book !== undefined && typeof g.book !== "boolean") {
+            throw new Error(`granularity.book must be a boolean in ${sourcePath}`);
+        }
+    }
     // Pass through validated structure — applyDefaults handles missing
     // optionals and contentSignal sub-fields.
     return {
@@ -121,6 +130,7 @@ function validate(parsed, sourcePath) {
         propSchema,
         plugins,
         attributes,
+        granularity: obj.granularity,
     };
 }
 /**

package/dist/config/to-astro-options.js

@@ -29,6 +29,7 @@ export function configToAstroOptions(config) {
         navMode: config.build?.navMode,
         llmsTxt: config.agent?.llmsTxt,
         mdMirror: config.agent?.mdMirror,
+        granularity: config.granularity,
         aiTrain: config.agent?.contentSignal?.aiTrain,
         aiInput: config.agent?.contentSignal?.aiInput,
         aiSearch: config.agent?.contentSignal?.search,

package/dist/index.js

@@ -8,6 +8,8 @@ import { add, list } from "./commands/add.js";
 import { importMkdocs } from "./commands/import-mkdocs.js";
 import { migrateMkdocs } from "./commands/migrate-mkdocs.js";
 import { migrateAsciidoc } from "./commands/migrate-asciidoc.js";
+import { migrateFluidtopics } from "./commands/migrate-fluidtopics.js";
+import { migrateFluidtopicsRest } from "./commands/migrate-fluidtopics-rest.js";
 import { preprocessVariants } from "./commands/preprocess-variants.js";
 import { lighthouse } from "./commands/lighthouse.js";
 import { convert } from "./commands/convert.js";
@@ -211,10 +213,11 @@ program
     .action((source, options) => migrateMkdocs(source, options));
 program
     .command("migrate-asciidoc")
-    .description("One-way migrate a plain AsciiDoc corpus to a scaffolded Dogsbay-MD " +
+    .description("One-way migrate an AsciiDoc corpus to a scaffolded Dogsbay-MD " +
     "project. Source-of-truth becomes ./content/*.md; the AsciiDoc source " +
-    "can be deleted. Plain .adoc directory walk only — AsciiBinder, " +
-    "Antora, and AAP master.adoc loaders are planned for a later release.")
+    "can be deleted. Topology is auto-detected — AsciiBinder " +
+    "(_topic_maps/_topic_map.yml), AAP (master.adoc), Antora (antora.yml), " +
+    "or a plain .adoc directory walk; override with --loader.")
     .argument("<source>", "Path to a directory containing .adoc files")
     .option("-o, --output <dir>", "Output directory (default: {source}-dogsbay)")
     .option("--force", "Overwrite an existing Dogsbay site. Clean re-import: clears the " +
@@ -226,13 +229,59 @@ program
     .option("-a, --attribute <name=value>", "AsciiDoc attribute (repeatable). Resolved refs/conditionals get " +
     "baked into the migrated Dogsbay-MD; unresolved ones survive as " +
     "{{ var }} / {% if %} for a downstream Minja pass.", (val, prev = []) => [...prev, val])
-    .option("--loader <name>", "Force a corpus loader (plain, master-adoc). Auto-detected when omitted.")
+    .option("--loader <name>", "Force a corpus loader (plain, master-adoc, asciibinder, antora). " +
+    "Auto-detected when omitted.")
     .option("--distro <name>", "AsciiBinder distro filter (e.g. 'openshift-enterprise'). Topic-map " +
     "entries whose Distros: list excludes this value are dropped from " +
     "the page set and nav. Entries without Distros: are kept as universal.")
+    .option("--topic-map <path>", "AsciiBinder topic-map path, relative to the source root " +
+    "(default: _topic_maps/_topic_map.yml). Drives loader detection and " +
+    "loading for corpora that name their topic map differently.")
     .option("--local", "Use file: references to local monorepo packages (for development)")
     .option("--quiet", "Suppress the 'Next steps' trailer")
     .action((source, options) => migrateAsciidoc(source, options));
+program
+    .command("migrate-fluidtopics")
+    .description("One-way migrate a Fluid Topics Markdown export (e.g. a ServiceNowDocs " +
+    "checkout: markdown/{publication}/**) to a scaffolded Dogsbay-MD " +
+    "project. Source-of-truth becomes ./content/*.md.")
+    .argument("<source>", "Path to the FT Markdown export (containing markdown/)")
+    .option("-o, --output <dir>", "Output directory (default: {source}-dogsbay)")
+    .option("--force", "Overwrite an existing Dogsbay site at the output dir")
+    .option("--site-name <name>", "Site name written into dogsbay.config.yml")
+    .option("--site-url <url>", "Canonical URL of the destination site. Drives robots.txt, sitemap, " +
+    "canonical tags, and llms.txt.")
+    .option("--base-path <path>", 'Path under the host where docs are served (default "" = host root, ' +
+    "the dedicated-repo convention).")
+    .option("--publication <name>", "Limit the migration to one or more publications (top-level markdown/{pub} " +
+    "folder). Repeatable. Useful for previewing a slice of a huge corpus.", (val, prev = []) => [...prev, val])
+    .option("--local", "Use file: references to local monorepo packages (for development)")
+    .option("--quiet", "Suppress the 'Next steps' trailer")
+    .action((source, options) => migrateFluidtopics(source, options));
+program
+    .command("migrate-fluidtopics-rest")
+    .description("Migrate a LIVE Fluid Topics portal via the khub REST API (Option B): " +
+    "reverse-engineers DITA-OT HTML5 → Dogsbay-MD. Works on any public FT " +
+    "portal (ServiceNow, Johnson Controls, FT docs).")
+    .argument("<portal-url>", "FT portal root (e.g. https://www.servicenow.com/docs or https://docs.johnsoncontrols.com/bas)")
+    .option("-o, --output <dir>", "Output directory (default: fluidtopics-rest-dogsbay)")
+    .option("--force", "Overwrite an existing Dogsbay site at the output dir")
+    .option("--site-name <name>", "Site name written into dogsbay.config.yml")
+    .option("--site-url <url>", "Canonical URL of the destination site")
+    .option("--base-path <path>", 'Path under the host where docs are served (default "" = host root)')
+    .option("--map <id>", "Limit to one or more map ids (repeatable)", (v, p = []) => [...p, v])
+    .option("--title-filter <substr>", "Limit to maps whose title contains this substring (case-insensitive)")
+    .option("--lang <locale>", "Locale to request from the portal (e.g. en-US)")
+    .option("--throttle <ms>", "Delay between khub requests (default 150)")
+    .option("--single-page", "Roll each map into ONE page (whole book) instead of one page per topic; xrefs become in-page anchors")
+    .option("--group-by <facets>", "Group the nav by map metadata facet(s), comma-separated (e.g. Category or Product,Category). Default: auto (Category). Use 'none' to keep it flat")
+    .option("--single-page-books", "Mark book-type maps (editorialType=book) with single-page: yes — an additive 'Read as single page' rollup per guide; articles stay per-topic (size cap applies)")
+    .option("--keep-duplicate-maps", "Keep stale duplicate publications (same title+version). Default: keep only the freshest ft:lastEdition per guide, removing duplicate nav entries")
+    .option("--metadata <keys>", "Extra FT topic-metadata keys to lift into page frontmatter, comma-separated (beyond the curated subject/product/author/category/audience/version set)")
+    .option("--full-topic-metadata", "Fetch full per-topic metadata (one request per topic) for accurate per-topic ft:wordCount/ft:outputSize + 'updated' date. Roughly doubles requests; off by default (technical numerics emit at book level only)")
+    .option("--local", "Use file: references to local monorepo packages (for development)")
+    .option("--quiet", "Suppress the trailer")
+    .action((source, options) => migrateFluidtopicsRest(source, options));
 program
     .command("preprocess-variants")
     .description("Add variant tabs to file includes (e.g. Python 3.10+ / 3.9+ tabs)")

package/dist/utils/remote-includes.js

@@ -0,0 +1,88 @@
+/**
+ * Remote AsciiDoc include resolution (migrate time).
+ *
+ * AsciiDoc `include::https://…[]` inlines remote content at build time
+ * (AsciiBinder runs with `--allow-uri-read`). The adoc2md engine — which is
+ * pure and does no network I/O — emits the URL form as a Minja directive
+ * `{% include "./https://…" %}`. The build-time Minja loader is filesystem-only,
+ * so the directive would leak verbatim into a `<pre>` block, dropping the
+ * content (e.g. 6 CloudFormation templates on the AWS UPI install pages).
+ *
+ * This pass fetches each remote include ONCE (cached by URL) and inlines it as
+ * a fenced code block, baking the content into the migrated markdown so the
+ * build has no runtime network dependency. On fetch failure it leaves a visible
+ * link instead of a broken directive. Issue #007.
+ */
+// Code-fence language inferred from the URL's file extension.
+const LANG_BY_EXT = {
+    yaml: "yaml",
+    yml: "yaml",
+    json: "json",
+    sh: "bash",
+    bash: "bash",
+    py: "python",
+    rb: "ruby",
+    go: "go",
+    toml: "toml",
+    ini: "ini",
+    xml: "xml",
+    tf: "hcl",
+};
+// `{% include "./https://…" %}` (the `./` is the engine's local-path prefix,
+// spurious for URLs) on its own line, capturing leading indent + the URL.
+const remoteIncludeRx = /^([ \t]*)\{%\s*include\s*"\.?\/?(https?:\/\/[^"]+)"\s*%\}[ \t]*$/gm;
+function langForUrl(url) {
+    const clean = url.split(/[?#]/)[0];
+    const ext = clean.slice(clean.lastIndexOf(".") + 1).toLowerCase();
+    return LANG_BY_EXT[ext] ?? "";
+}
+/** Choose a backtick fence longer than any run inside the content. */
+function pickFence(content) {
+    const longest = (content.match(/`+/g) ?? []).reduce((n, run) => Math.max(n, run.length), 0);
+    return "`".repeat(Math.max(3, longest + 1));
+}
+/** True if `md` contains at least one remote include directive. */
+export function hasRemoteIncludes(md) {
+    remoteIncludeRx.lastIndex = 0;
+    return remoteIncludeRx.test(md);
+}
+/**
+ * Replace every `{% include "https://…" %}` directive in `md` with the fetched
+ * content as a fenced code block. Returns `md` unchanged if there are none.
+ */
+export async function inlineRemoteIncludes(md, options = {}) {
+    const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+    const cache = options.cache ?? new Map();
+    const onWarn = options.onWarn ?? ((m) => console.warn(m));
+    // Collect the distinct URLs first so each is fetched at most once.
+    const urls = new Set();
+    for (const m of md.matchAll(remoteIncludeRx))
+        urls.add(m[2]);
+    if (urls.size === 0)
+        return md;
+    for (const url of urls) {
+        if (cache.has(url))
+            continue;
+        try {
+            const res = await fetchImpl(url);
+            if (!res.ok)
+                throw new Error(`HTTP ${res.status}`);
+            cache.set(url, await res.text());
+        }
+        catch (err) {
+            cache.set(url, null);
+            onWarn(`remote include: failed to fetch ${url} (${err instanceof Error ? err.message : String(err)}); leaving a link`);
+        }
+    }
+    return md.replace(remoteIncludeRx, (_whole, indent, url) => {
+        const content = cache.get(url);
+        if (content == null) {
+            // Fetch failed (or not attempted): a visible link beats a broken directive.
+            return `${indent}See [${url}](${url}).`;
+        }
+        const body = content.replace(/\n$/, "");
+        const fence = pickFence(body);
+        const lang = langForUrl(url);
+        return `${indent}${fence}${lang}\n${body}\n${indent}${fence}`;
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dogsbay",
-  "version": "0.2.0-beta.91",
+  "version": "0.2.0-beta.93",
   "description": "CLI for Dogsbay — scaffold, build, and serve documentation sites with markdown / MkDocs / Obsidian / OpenAPI sources",
   "type": "module",
   "bin": {
@@ -33,20 +33,22 @@
     "picocolors": "^1.1.0",
     "prompts": "^2.4.2",
     "yaml": "^2.8.3",
-    "@dogsbay/autodoc-python": "0.2.0-beta.91",
-    "@dogsbay/format-mkdocs": "0.2.0-beta.91",
-    "@dogsbay/format-astro": "0.2.0-beta.91",
-    "@dogsbay/format-obsidian": "0.2.0-beta.91",
-    "@dogsbay/format-mdx": "0.2.0-beta.91",
-    "@dogsbay/format-starlight": "0.2.0-beta.91",
-    "@dogsbay/format-docusaurus": "0.2.0-beta.91",
-    "@dogsbay/docusaurus-adapter-tigera": "0.2.0-beta.91",
-    "@dogsbay/format-dogsbay-md": "0.2.0-beta.91",
-    "@dogsbay/format-openapi": "0.2.0-beta.91",
-    "@dogsbay/adoc2md-modular": "0.2.0-beta.91",
-    "@dogsbay/format-asciidoc": "0.2.0-beta.91",
-    "@dogsbay/minja": "0.2.0-beta.91",
-    "@dogsbay/types": "0.2.0-beta.91"
+    "@dogsbay/format-mkdocs": "0.2.0-beta.93",
+    "@dogsbay/autodoc-python": "0.2.0-beta.93",
+    "@dogsbay/format-astro": "0.2.0-beta.93",
+    "@dogsbay/format-obsidian": "0.2.0-beta.93",
+    "@dogsbay/format-mdx": "0.2.0-beta.93",
+    "@dogsbay/format-starlight": "0.2.0-beta.93",
+    "@dogsbay/format-docusaurus": "0.2.0-beta.93",
+    "@dogsbay/docusaurus-adapter-tigera": "0.2.0-beta.93",
+    "@dogsbay/format-dogsbay-md": "0.2.0-beta.93",
+    "@dogsbay/format-fluidtopics-markdown": "0.2.0-beta.93",
+    "@dogsbay/format-openapi": "0.2.0-beta.93",
+    "@dogsbay/format-fluidtopics-rest": "0.2.0-beta.93",
+    "@dogsbay/adoc2md-modular": "0.2.0-beta.93",
+    "@dogsbay/format-asciidoc": "0.2.0-beta.93",
+    "@dogsbay/minja": "0.2.0-beta.93",
+    "@dogsbay/types": "0.2.0-beta.93"
   },
   "devDependencies": {
     "@types/markdown-it": "^14.1.0",