dogsbay 0.2.0-beta.92 → 0.2.0-beta.93

package/dist/commands/add.js

@@ -22,7 +22,7 @@ function getSourceDir() {
         return monorepo;
     throw new Error("Could not find component source files.");
 }
-function collectDependencies(names) {
+export function collectDependencies(names) {
     const visited = new Set();
     const components = [];
     const npmDeps = [];

package/dist/commands/migrate-asciidoc.js

@@ -8,11 +8,13 @@
  * and runs `dogsbay site build`; the AsciiDoc source can be
  * deleted.
  *
- * Phase 8b of plans/format-asciidoc-import.md. Plain-corpus loader
- * only; topology-aware loaders (AsciiBinder `_topic_map.yml`,
- * Antora playbooks, AAP `master.adoc` walks) ship in Phase 9 per
- * plans/asciidoc-corpus-loaders.md and will hook in here without
- * changing the output shape.
+ * Corpus discovery is delegated to the `@dogsbay/format-asciidoc`
+ * CorpusLoader registry (see `asciidocPlugin.import` below), so the
+ * topology-aware loaders are live: detection auto-picks AsciiBinder
+ * (`_topic_maps/_topic_map.yml`), AAP (`master.adoc`), Antora component
+ * (`antora.yml`), or the plain `.adoc` walk, and topology files never
+ * accidentally become pages. Override with `--loader`.
+ * See plans/asciidoc-corpus-loaders.md.
  *
  * Output layout — same convention as `migrate-mkdocs`:
  *
@@ -29,10 +31,11 @@ import { copyFileSync, cpSync, existsSync, lstatSync, mkdirSync, readFileSync, r
 import { basename, dirname, join, relative, resolve } from "node:path";
 import pc from "picocolors";
 import { emitSiteScaffold } from "@dogsbay/format-astro";
-import { plugin as asciidocPlugin } from "@dogsbay/format-asciidoc/cli";
+import { plugin as asciidocPlugin, pickLoader } from "@dogsbay/format-asciidoc/cli";
 import { asciidocToMarkdown } from "@dogsbay/adoc2md-modular";
 import { treeToDogsbayMd } from "@dogsbay/format-dogsbay-md";
 import { ensureGitignore } from "../utils/gitignore.js";
+import { inlineRemoteIncludes, hasRemoteIncludes } from "../utils/remote-includes.js";
 function parseAttributePairs(pairs) {
     const out = {};
     for (const raw of pairs ?? []) {
@@ -211,6 +214,9 @@ function convertFragmentAdocs(sourceRoot, outputContentDir, scopedRootDirs, page
     let converted = 0;
     let failed = 0;
     let skipped = 0;
+    // Fragment .md files that carry a `{% include "https://…" %}` directive —
+    // resolved by an async fetch pass after the (sync) walk completes. (#007)
+    const remoteIncludePaths = [];
     function walk(srcDir, dstDir, depth) {
         let entries;
         try {
@@ -302,6 +308,8 @@ function convertFragmentAdocs(sourceRoot, outputContentDir, scopedRootDirs, page
                 // pursue option A through a non-frontmatter signal that
                 // can't collide with Jinja whitespace control.
                 writeFileSync(mdPath, md, "utf-8");
+                if (hasRemoteIncludes(md))
+                    remoteIncludePaths.push(mdPath);
                 converted += 1;
             }
             catch (err) {
@@ -312,7 +320,7 @@ function convertFragmentAdocs(sourceRoot, outputContentDir, scopedRootDirs, page
         }
     }
     walk(sourceRoot, outputContentDir, 0);
-    return { converted, failed, skipped };
+    return { converted, failed, skipped, remoteIncludePaths };
 }
 /**
  * Serialize a NavItem[] (from a CorpusLoader) to nav.yml syntax.
@@ -400,6 +408,21 @@ function buildDogsbayConfig(opts) {
     return lines.join("\n");
 }
 function buildMigrationNotes(opts) {
+    // Report the loader that actually ran rather than assuming a plain
+    // walk. The non-plain loaders read a topology source for the nav and
+    // route only topology-declared pages; `modules/` / `snippets/` /
+    // `_attributes/` are pulled in as include targets, not pages.
+    const loaderDesc = {
+        asciibinder: "AsciiBinder topology (`_topic_maps/_topic_map.yml`)",
+        "master-adoc": "AAP `master.adoc` topology",
+        antora: "Antora component (`antora.yml`)",
+        plain: "plain AsciiDoc — flat `.adoc` walk",
+    };
+    const corpusDesc = loaderDesc[opts.loader] ?? `\`${opts.loader}\` loader`;
+    const distroNote = opts.distro ? `, distro \`${opts.distro}\`` : "";
+    const topologyDidnt = opts.loader === "plain"
+        ? "- **Topology**: the plain loader walked every `.adoc` flat and derived the nav from the directory layout. If your source is actually AsciiBinder (`_topic_maps/_topic_map.yml`), Antora (`antora.yml`), or AAP (`master.adoc`), re-run with `--loader <name>` (or rely on auto-detection) to use the curated topology and per-page distro filtering instead.\n"
+        : "";
     const hasAttrs = Object.keys(opts.attributes).length > 0;
     const attrSummary = hasAttrs
         ? Object.entries(opts.attributes)
@@ -416,7 +439,7 @@ Generated by \`dogsbay migrate-asciidoc\`.
 ## Summary
 
 - **Site name:** ${opts.siteName}
-- **Source corpus:** \`${opts.sourceDir}\` (plain AsciiDoc — \`.adoc\` walk)
+- **Source corpus:** \`${opts.sourceDir}\` (${corpusDesc}${distroNote})
 - **Output:** \`${opts.outputDir}\`
 - **Pages migrated:** ${opts.pageCount}
 
@@ -455,8 +478,7 @@ See \`docs/asciidoc.md\` for the full preprocessor behaviour.
 
 ## What didn't (yet)
 
-- **Topology**: this migration walked every \`.adoc\` under \`${opts.sourceDir}\` as a flat plain-AsciiDoc corpus. If your source uses AsciiBinder \`_topic_map.yml\`, Antora \`antora-playbook.yml\`, or AAP \`master.adoc\` discovery, those topologies are not yet understood — the dedicated corpus loaders are planned for a future release (see \`plans/asciidoc-corpus-loaders.md\`).
-- **Line-form admonition layout**: \`NOTE: foo\` lines always emit the HTML-shaped \`**📌 NOTE**\\\` prefix regardless of options. Block-form \`[NOTE]\\n====\\n…\\n====\` does respect the configured \`admonitionFormat\`.
+${topologyDidnt}- **Line-form admonition layout**: \`NOTE: foo\` lines always emit the HTML-shaped \`**📌 NOTE**\\\` prefix regardless of options. Block-form \`[NOTE]\\n====\\n…\\n====\` does respect the configured \`admonitionFormat\`.
 
 ## Next steps
 
@@ -540,6 +562,7 @@ export async function migrateAsciidoc(source, options) {
         attributes: engineAttributes,
         loader: options.loader,
         distro: options.distro,
+        topicMap: options.topicMap,
     });
     if (importedPages.length === 0) {
         console.log(pc.red(`Error: no .adoc files found under ${sourceDir}`));
@@ -548,6 +571,13 @@ export async function migrateAsciidoc(source, options) {
     }
     console.log(`Pages:   ${importedPages.length}`);
     console.log();
+    // Which loader did the import auto-pick? Re-run detection (cheap —
+    // filesystem existence checks) so MIGRATION.md reports the topology
+    // that actually ran, not a hardcoded "plain walk" string. The import
+    // above already succeeded, so this won't throw on a no-match.
+    const loaderName = pickLoader(sourceDir, options.loader, {
+        topicMap: options.topicMap,
+    }).name;
     // 3. Write each page to disk with title frontmatter.
     //
     // When the importer populated `page.bodyMarkdown` (currently only
@@ -620,6 +650,26 @@ export async function migrateAsciidoc(source, options) {
             (frag.skipped > 0 ? ` (${frag.skipped} already-written page paths skipped)` : "") +
             (frag.failed > 0 ? ` (${frag.failed} failed)` : ""));
     }
+    // 3c-bis. Resolve remote `include::https://…[]` directives: fetch each URL
+    // once and bake it into the fragment as a fenced code block, so the build
+    // has no runtime network dependency and the content isn't dropped. (#007)
+    if (frag.remoteIncludePaths.length > 0) {
+        const cache = new Map();
+        let inlined = 0;
+        for (const mdPath of frag.remoteIncludePaths) {
+            const before = readFileSync(mdPath, "utf-8");
+            const after = await inlineRemoteIncludes(before, { cache });
+            if (after !== before) {
+                writeFileSync(mdPath, after, "utf-8");
+                inlined += 1;
+            }
+        }
+        const failures = [...cache.values()].filter((v) => v === null).length;
+        console.log(pc.green(`Inlined`) +
+            ` remote includes in ${inlined} fragment(s) (${cache.size} URL(s)` +
+            (failures > 0 ? `, ${failures} unreachable` : "") +
+            `)`);
+    }
     // 3d. Mirror non-`.adoc` filesystem entries within the scoped
     //     set: symlinks (critical for AsciiBinder include resolution
     //     where per-section dirs symlink back to root `_attributes/`,
@@ -695,6 +745,8 @@ export async function migrateAsciidoc(source, options) {
         pageCount: importedPages.length,
         attributes,
         rerunCmd,
+        loader: loaderName,
+        distro: options.distro,
     }));
     console.log(pc.green(`Wrote`) + ` MIGRATION.md`);
     if (!options.quiet) {

package/dist/commands/migrate-fluidtopics-rest.js

@@ -45,6 +45,17 @@ export async function migrateFluidtopicsRest(source, options) {
         portalBase: source,
         maps: options.map,
         titleFilter: options.titleFilter,
+        groupBy: options.groupBy,
+        singlePageBooks: options.singlePageBooks,
+        keepDuplicateMaps: options.keepDuplicateMaps,
+        metadataAllow: options.metadata?.split(",").map((s) => s.trim()).filter(Boolean),
+        fullTopicMetadata: options.fullTopicMetadata,
+        // Custom facets: editorialType (book/article), version, and subject (the
+        // DITA subjectScheme "Subject matter" / PBS facet, now lifted per-topic).
+        // category & audience are built-in. parseMeta lifts these into
+        // page.meta.taxonomies → DocsLayout emits data-pagefind-filter → Pagefind
+        // facets. (#search-facets)
+        taxonomyNames: ["editorialType", "version", "subject"],
         lang: options.lang,
         hrefPrefix,
         throttleMs: options.throttle ? Number(options.throttle) : undefined,
@@ -74,6 +85,18 @@ export async function migrateFluidtopicsRest(source, options) {
         schemaVersion: 1,
         site: { name: siteName, url: options.siteUrl, basePath },
         content: { sources: [{ path: "./content", from: "dogsbay-md" }] },
+        // FT faceted metadata → search filters. `category` / `audience` are
+        // built-in facets (no declaration needed); `editorialType` (book/article),
+        // `version`, and `subject` (the DITA subjectScheme "Subject matter" / PBS
+        // facet) are declared here so the build extracts them into each page's
+        // taxonomies, DocsLayout emits the data-pagefind-filter divs, and Pagefind
+        // surfaces them as facet columns. Also gives browseable term index pages
+        // (/format, /version, /subject). See docs-dev/fluidtopics-rest-importer.md.
+        taxonomies: {
+            editorialType: { indexPath: "/format", order: 2 },
+            version: { indexPath: "/version", order: 3 },
+            subject: { indexPath: "/subject", order: 4 },
+        },
         agent: { llmsTxt: true, mdMirror: true },
     };
     writeFileSync(join(outputDir, "dogsbay.config.yml"), serializeConfig(config, "yaml"));
@@ -90,7 +113,20 @@ export async function migrateFluidtopicsRest(source, options) {
 function navItemsToSingleKey(items, basePath) {
     const out = [];
     for (const item of items) {
-        if (item.children && item.children.length > 0) {
+        // A node carrying a single-page annotation uses the EXPLICIT nav form
+        // (`{ title, single-page, items }`) so the annotation survives into nav.yml;
+        // unannotated nodes keep the compact single-key form.
+        if (item.singlePage) {
+            const entry = { title: item.label, "single-page": item.singlePage };
+            if (item.children && item.children.length > 0) {
+                entry.items = navItemsToSingleKey(item.children, basePath);
+            }
+            else if (item.href) {
+                entry.file = hrefToNavPath(item.href, basePath);
+            }
+            out.push(entry);
+        }
+        else if (item.children && item.children.length > 0) {
             out.push({ [item.label]: navItemsToSingleKey(item.children, basePath) });
         }
         else if (item.href) {

package/dist/config/load.js

@@ -104,6 +104,15 @@ function validate(parsed, sourcePath) {
     const propSchema = validatePropSchema(obj.propSchema, sourcePath);
     const plugins = validatePlugins(obj.plugins, sourcePath);
     const attributes = validateAttributes(obj.attributes, "attributes", sourcePath);
+    if (obj.granularity !== undefined) {
+        if (typeof obj.granularity !== "object" || obj.granularity === null) {
+            throw new Error(`granularity must be an object in ${sourcePath}`);
+        }
+        const g = obj.granularity;
+        if (g.book !== undefined && typeof g.book !== "boolean") {
+            throw new Error(`granularity.book must be a boolean in ${sourcePath}`);
+        }
+    }
     // Pass through validated structure — applyDefaults handles missing
     // optionals and contentSignal sub-fields.
     return {
@@ -121,6 +130,7 @@ function validate(parsed, sourcePath) {
         propSchema,
         plugins,
         attributes,
+        granularity: obj.granularity,
     };
 }
 /**

package/dist/config/to-astro-options.js

@@ -29,6 +29,7 @@ export function configToAstroOptions(config) {
         navMode: config.build?.navMode,
         llmsTxt: config.agent?.llmsTxt,
         mdMirror: config.agent?.mdMirror,
+        granularity: config.granularity,
         aiTrain: config.agent?.contentSignal?.aiTrain,
         aiInput: config.agent?.contentSignal?.aiInput,
         aiSearch: config.agent?.contentSignal?.search,

package/dist/index.js

@@ -213,10 +213,11 @@ program
     .action((source, options) => migrateMkdocs(source, options));
 program
     .command("migrate-asciidoc")
-    .description("One-way migrate a plain AsciiDoc corpus to a scaffolded Dogsbay-MD " +
+    .description("One-way migrate an AsciiDoc corpus to a scaffolded Dogsbay-MD " +
     "project. Source-of-truth becomes ./content/*.md; the AsciiDoc source " +
-    "can be deleted. Plain .adoc directory walk only — AsciiBinder, " +
-    "Antora, and AAP master.adoc loaders are planned for a later release.")
+    "can be deleted. Topology is auto-detected — AsciiBinder " +
+    "(_topic_maps/_topic_map.yml), AAP (master.adoc), Antora (antora.yml), " +
+    "or a plain .adoc directory walk; override with --loader.")
     .argument("<source>", "Path to a directory containing .adoc files")
     .option("-o, --output <dir>", "Output directory (default: {source}-dogsbay)")
     .option("--force", "Overwrite an existing Dogsbay site. Clean re-import: clears the " +
@@ -228,10 +229,14 @@ program
     .option("-a, --attribute <name=value>", "AsciiDoc attribute (repeatable). Resolved refs/conditionals get " +
     "baked into the migrated Dogsbay-MD; unresolved ones survive as " +
     "{{ var }} / {% if %} for a downstream Minja pass.", (val, prev = []) => [...prev, val])
-    .option("--loader <name>", "Force a corpus loader (plain, master-adoc). Auto-detected when omitted.")
+    .option("--loader <name>", "Force a corpus loader (plain, master-adoc, asciibinder, antora). " +
+    "Auto-detected when omitted.")
     .option("--distro <name>", "AsciiBinder distro filter (e.g. 'openshift-enterprise'). Topic-map " +
     "entries whose Distros: list excludes this value are dropped from " +
     "the page set and nav. Entries without Distros: are kept as universal.")
+    .option("--topic-map <path>", "AsciiBinder topic-map path, relative to the source root " +
+    "(default: _topic_maps/_topic_map.yml). Drives loader detection and " +
+    "loading for corpora that name their topic map differently.")
     .option("--local", "Use file: references to local monorepo packages (for development)")
     .option("--quiet", "Suppress the 'Next steps' trailer")
     .action((source, options) => migrateAsciidoc(source, options));
@@ -269,6 +274,11 @@ program
     .option("--lang <locale>", "Locale to request from the portal (e.g. en-US)")
     .option("--throttle <ms>", "Delay between khub requests (default 150)")
     .option("--single-page", "Roll each map into ONE page (whole book) instead of one page per topic; xrefs become in-page anchors")
+    .option("--group-by <facets>", "Group the nav by map metadata facet(s), comma-separated (e.g. Category or Product,Category). Default: auto (Category). Use 'none' to keep it flat")
+    .option("--single-page-books", "Mark book-type maps (editorialType=book) with single-page: yes — an additive 'Read as single page' rollup per guide; articles stay per-topic (size cap applies)")
+    .option("--keep-duplicate-maps", "Keep stale duplicate publications (same title+version). Default: keep only the freshest ft:lastEdition per guide, removing duplicate nav entries")
+    .option("--metadata <keys>", "Extra FT topic-metadata keys to lift into page frontmatter, comma-separated (beyond the curated subject/product/author/category/audience/version set)")
+    .option("--full-topic-metadata", "Fetch full per-topic metadata (one request per topic) for accurate per-topic ft:wordCount/ft:outputSize + 'updated' date. Roughly doubles requests; off by default (technical numerics emit at book level only)")
     .option("--local", "Use file: references to local monorepo packages (for development)")
     .option("--quiet", "Suppress the trailer")
     .action((source, options) => migrateFluidtopicsRest(source, options));

package/dist/utils/remote-includes.js

@@ -0,0 +1,88 @@
+/**
+ * Remote AsciiDoc include resolution (migrate time).
+ *
+ * AsciiDoc `include::https://…[]` inlines remote content at build time
+ * (AsciiBinder runs with `--allow-uri-read`). The adoc2md engine — which is
+ * pure and does no network I/O — emits the URL form as a Minja directive
+ * `{% include "./https://…" %}`. The build-time Minja loader is filesystem-only,
+ * so the directive would leak verbatim into a `<pre>` block, dropping the
+ * content (e.g. 6 CloudFormation templates on the AWS UPI install pages).
+ *
+ * This pass fetches each remote include ONCE (cached by URL) and inlines it as
+ * a fenced code block, baking the content into the migrated markdown so the
+ * build has no runtime network dependency. On fetch failure it leaves a visible
+ * link instead of a broken directive. Issue #007.
+ */
+// Code-fence language inferred from the URL's file extension.
+const LANG_BY_EXT = {
+    yaml: "yaml",
+    yml: "yaml",
+    json: "json",
+    sh: "bash",
+    bash: "bash",
+    py: "python",
+    rb: "ruby",
+    go: "go",
+    toml: "toml",
+    ini: "ini",
+    xml: "xml",
+    tf: "hcl",
+};
+// `{% include "./https://…" %}` (the `./` is the engine's local-path prefix,
+// spurious for URLs) on its own line, capturing leading indent + the URL.
+const remoteIncludeRx = /^([ \t]*)\{%\s*include\s*"\.?\/?(https?:\/\/[^"]+)"\s*%\}[ \t]*$/gm;
+function langForUrl(url) {
+    const clean = url.split(/[?#]/)[0];
+    const ext = clean.slice(clean.lastIndexOf(".") + 1).toLowerCase();
+    return LANG_BY_EXT[ext] ?? "";
+}
+/** Choose a backtick fence longer than any run inside the content. */
+function pickFence(content) {
+    const longest = (content.match(/`+/g) ?? []).reduce((n, run) => Math.max(n, run.length), 0);
+    return "`".repeat(Math.max(3, longest + 1));
+}
+/** True if `md` contains at least one remote include directive. */
+export function hasRemoteIncludes(md) {
+    remoteIncludeRx.lastIndex = 0;
+    return remoteIncludeRx.test(md);
+}
+/**
+ * Replace every `{% include "https://…" %}` directive in `md` with the fetched
+ * content as a fenced code block. Returns `md` unchanged if there are none.
+ */
+export async function inlineRemoteIncludes(md, options = {}) {
+    const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+    const cache = options.cache ?? new Map();
+    const onWarn = options.onWarn ?? ((m) => console.warn(m));
+    // Collect the distinct URLs first so each is fetched at most once.
+    const urls = new Set();
+    for (const m of md.matchAll(remoteIncludeRx))
+        urls.add(m[2]);
+    if (urls.size === 0)
+        return md;
+    for (const url of urls) {
+        if (cache.has(url))
+            continue;
+        try {
+            const res = await fetchImpl(url);
+            if (!res.ok)
+                throw new Error(`HTTP ${res.status}`);
+            cache.set(url, await res.text());
+        }
+        catch (err) {
+            cache.set(url, null);
+            onWarn(`remote include: failed to fetch ${url} (${err instanceof Error ? err.message : String(err)}); leaving a link`);
+        }
+    }
+    return md.replace(remoteIncludeRx, (_whole, indent, url) => {
+        const content = cache.get(url);
+        if (content == null) {
+            // Fetch failed (or not attempted): a visible link beats a broken directive.
+            return `${indent}See [${url}](${url}).`;
+        }
+        const body = content.replace(/\n$/, "");
+        const fence = pickFence(body);
+        const lang = langForUrl(url);
+        return `${indent}${fence}${lang}\n${body}\n${indent}${fence}`;
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dogsbay",
-  "version": "0.2.0-beta.92",
+  "version": "0.2.0-beta.93",
   "description": "CLI for Dogsbay — scaffold, build, and serve documentation sites with markdown / MkDocs / Obsidian / OpenAPI sources",
   "type": "module",
   "bin": {
@@ -33,22 +33,22 @@
     "picocolors": "^1.1.0",
     "prompts": "^2.4.2",
     "yaml": "^2.8.3",
-    "@dogsbay/autodoc-python": "0.2.0-beta.92",
-    "@dogsbay/format-astro": "0.2.0-beta.92",
-    "@dogsbay/format-mkdocs": "0.2.0-beta.92",
-    "@dogsbay/format-obsidian": "0.2.0-beta.92",
-    "@dogsbay/format-mdx": "0.2.0-beta.92",
-    "@dogsbay/format-starlight": "0.2.0-beta.92",
-    "@dogsbay/format-docusaurus": "0.2.0-beta.92",
-    "@dogsbay/docusaurus-adapter-tigera": "0.2.0-beta.92",
-    "@dogsbay/format-dogsbay-md": "0.2.0-beta.92",
-    "@dogsbay/format-fluidtopics-markdown": "0.2.0-beta.92",
-    "@dogsbay/format-fluidtopics-rest": "0.2.0-beta.92",
-    "@dogsbay/format-openapi": "0.2.0-beta.92",
-    "@dogsbay/adoc2md-modular": "0.2.0-beta.92",
-    "@dogsbay/format-asciidoc": "0.2.0-beta.92",
-    "@dogsbay/minja": "0.2.0-beta.92",
-    "@dogsbay/types": "0.2.0-beta.92"
+    "@dogsbay/format-mkdocs": "0.2.0-beta.93",
+    "@dogsbay/autodoc-python": "0.2.0-beta.93",
+    "@dogsbay/format-astro": "0.2.0-beta.93",
+    "@dogsbay/format-obsidian": "0.2.0-beta.93",
+    "@dogsbay/format-mdx": "0.2.0-beta.93",
+    "@dogsbay/format-starlight": "0.2.0-beta.93",
+    "@dogsbay/format-docusaurus": "0.2.0-beta.93",
+    "@dogsbay/docusaurus-adapter-tigera": "0.2.0-beta.93",
+    "@dogsbay/format-dogsbay-md": "0.2.0-beta.93",
+    "@dogsbay/format-fluidtopics-markdown": "0.2.0-beta.93",
+    "@dogsbay/format-openapi": "0.2.0-beta.93",
+    "@dogsbay/format-fluidtopics-rest": "0.2.0-beta.93",
+    "@dogsbay/adoc2md-modular": "0.2.0-beta.93",
+    "@dogsbay/format-asciidoc": "0.2.0-beta.93",
+    "@dogsbay/minja": "0.2.0-beta.93",
+    "@dogsbay/types": "0.2.0-beta.93"
   },
   "devDependencies": {
     "@types/markdown-it": "^14.1.0",