dogsbay 0.2.0-beta.91 → 0.2.0-beta.92

package/dist/commands/migrate-fluidtopics-rest.js

@@ -0,0 +1,113 @@
+/**
+ * `migrate-fluidtopics-rest` — migrate a *live* Fluid Topics portal (via the
+ * khub REST API) into a scaffolded Dogsbay-MD site. Reverse-engineers DITA-OT
+ * HTML5 → TreeNode → Dogsbay-MD. Works on any public FT portal.
+ *
+ * Same output shape as `migrate-fluidtopics` (Option A, Git Markdown export);
+ * this is Option B (live REST).
+ */
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import pc from "picocolors";
+import YAML from "yaml";
+import { emitSiteScaffold } from "@dogsbay/format-astro";
+import { serializeConfig } from "../config/serialize.js";
+export async function migrateFluidtopicsRest(source, options) {
+    if (!/^https?:\/\//i.test(source)) {
+        console.log(pc.red(`Error: source must be a portal URL (e.g. https://www.servicenow.com/docs)`));
+        process.exit(1);
+    }
+    const outputDir = resolve(options.output || "fluidtopics-rest-dogsbay");
+    const existingConfig = ["yml", "yaml", "json"]
+        .map((ext) => join(outputDir, `dogsbay.config.${ext}`))
+        .find(existsSync);
+    if (existingConfig && !options.force) {
+        console.log(pc.red(`Error: ${outputDir} already contains a Dogsbay site (${existingConfig}). Pass --force.`));
+        process.exit(1);
+    }
+    const siteName = options.siteName || "Documentation";
+    const basePath = options.basePath !== undefined ? options.basePath : "";
+    const hrefPrefix = basePath;
+    console.log();
+    console.log(pc.bold(`Migrating Fluid Topics portal (REST) to Dogsbay-MD: ${siteName}`));
+    console.log(`Portal:  ${source}`);
+    console.log(`Output:  ${outputDir}`);
+    console.log();
+    const contentDir = join(outputDir, "content");
+    const astroDir = join(outputDir, "astro");
+    mkdirSync(contentDir, { recursive: true });
+    mkdirSync(astroDir, { recursive: true });
+    const { importFluidTopicsRest } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-fluidtopics-rest"));
+    const { treeToDogsbayMd } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-dogsbay-md"));
+    const { pages, nav, unresolved, mapsProcessed, assets } = await importFluidTopicsRest({
+        portalBase: source,
+        maps: options.map,
+        titleFilter: options.titleFilter,
+        lang: options.lang,
+        hrefPrefix,
+        throttleMs: options.throttle ? Number(options.throttle) : undefined,
+        singlePage: options.singlePage,
+        onProgress: (m) => console.log(pc.dim(`  ${m}`)),
+    });
+    const unit = options.singlePage ? "page(s) (one per book)" : "topics";
+    console.log(pc.green(`Imported`) + ` ${pages.length} ${unit} from ${mapsProcessed} map(s)`);
+    // Write extracted images as real content-rooted assets (per _assets convention).
+    for (const a of assets) {
+        const dest = join(contentDir, a.path);
+        mkdirSync(dirname(dest), { recursive: true });
+        writeFileSync(dest, a.data);
+    }
+    if (assets.length)
+        console.log(pc.green(`Wrote`) + ` ${assets.length} images to content/_assets/`);
+    for (const page of pages) {
+        const dest = join(contentDir, `${page.slug || "index"}.md`);
+        mkdirSync(dirname(dest), { recursive: true });
+        const body = treeToDogsbayMd(page.tree, { frontmatter: page.frontmatter });
+        writeFileSync(dest, body.endsWith("\n") ? body : `${body}\n`);
+    }
+    console.log(pc.green(`Wrote`) + ` ${pages.length} pages to ./content/`);
+    writeFileSync(join(contentDir, "nav.yml"), YAML.stringify(navItemsToSingleKey(nav, basePath)));
+    console.log(pc.green(`Wrote`) + ` content/nav.yml (${nav.length} top-level items)`);
+    const config = {
+        schemaVersion: 1,
+        site: { name: siteName, url: options.siteUrl, basePath },
+        content: { sources: [{ path: "./content", from: "dogsbay-md" }] },
+        agent: { llmsTxt: true, mdMirror: true },
+    };
+    writeFileSync(join(outputDir, "dogsbay.config.yml"), serializeConfig(config, "yaml"));
+    emitSiteScaffold(astroDir, siteName, { siteName, siteUrl: options.siteUrl, basePath, llmsTxt: true, mdMirror: true, local: options.local }, true);
+    const unresolvedCount = Object.values(unresolved).reduce((n, a) => n + a.length, 0);
+    writeFileSync(join(outputDir, "MIGRATION.md"), `# Migration notes — ${siteName} (Fluid Topics REST)\n\nMigrated live from \`${source}\` via the khub API; DITA-OT HTML5 reverse-engineered to Dogsbay-MD.\n\n- Maps: **${mapsProcessed}**\n- Pages: **${pages.length}**\n- Unresolved in-content links: **${unresolvedCount}**\n\nMedia and any conref/keyref/profiling were already resolved/flattened upstream by FT's DITA-OT run.\n`);
+    console.log(pc.green(`Scaffolded`) + ` Astro project + config + MIGRATION.md`);
+    if (!options.quiet) {
+        console.log();
+        console.log(pc.bold("Done."));
+        console.log(`  cd ${outputDir} && npx dogsbay site dev`);
+    }
+}
+function navItemsToSingleKey(items, basePath) {
+    const out = [];
+    for (const item of items) {
+        if (item.children && item.children.length > 0) {
+            out.push({ [item.label]: navItemsToSingleKey(item.children, basePath) });
+        }
+        else if (item.href) {
+            out.push({ [item.label]: hrefToNavPath(item.href, basePath) });
+        }
+        else {
+            out.push({ [item.label]: [] });
+        }
+    }
+    return out;
+}
+function hrefToNavPath(href, basePath) {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href))
+        return href;
+    let s = href.split("#")[0];
+    if (basePath && s.startsWith(basePath))
+        s = s.slice(basePath.length);
+    s = s.replace(/^\/+/, "");
+    return s ? `${s}.md` : "index.md";
+}

package/dist/commands/migrate-fluidtopics.js

@@ -0,0 +1,158 @@
+/**
+ * `migrate-fluidtopics` — one-way migrate a Fluid Topics Markdown export
+ * (e.g. the ServiceNow/ServiceNowDocs repo) into a scaffolded Dogsbay-MD site.
+ *
+ * Canonical migration layout (see the migration-shape skill):
+ *   <output>/
+ *     content/            <- Dogsbay-MD, the new source of truth
+ *       <pub>/<topic>.md
+ *       nav.yml
+ *     astro/              <- generated Astro project (emitSiteScaffold)
+ *     dogsbay.config.yml
+ *     MIGRATION.md
+ *
+ * Mirrors `migrate-mkdocs`, reusing the shared emitters
+ * (importer -> treeToDogsbayMd -> emitSiteScaffold + serializeConfig).
+ */
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join, resolve, basename, dirname } from "node:path";
+import pc from "picocolors";
+import YAML from "yaml";
+import { emitSiteScaffold } from "@dogsbay/format-astro";
+import { serializeConfig } from "../config/serialize.js";
+export async function migrateFluidtopics(source, options) {
+    const sourceDir = resolve(source);
+    const markdownDir = join(sourceDir, "markdown");
+    if (!existsSync(markdownDir)) {
+        console.log(pc.red(`Error: no markdown/ directory found in ${sourceDir}`));
+        console.log("Point this at a Fluid Topics Markdown export (e.g. a ServiceNowDocs checkout).");
+        process.exit(1);
+    }
+    const outputDir = resolve(options.output || `${basename(sourceDir)}-dogsbay`);
+    const existingConfig = ["yml", "yaml", "json"]
+        .map((ext) => join(outputDir, `dogsbay.config.${ext}`))
+        .find(existsSync);
+    if (existingConfig && !options.force) {
+        console.log(pc.red(`Error: ${outputDir} already contains a Dogsbay site (${basename(existingConfig)}).`));
+        console.log("Pass --force to overwrite.");
+        process.exit(1);
+    }
+    const siteName = options.siteName || "Documentation";
+    // A dedicated docs repo serves at the host root by default (matches the
+    // openshift-docs-dogsbay / GitHub Pages project-site convention).
+    const basePath = options.basePath !== undefined ? options.basePath : "";
+    const hrefPrefix = basePath; // root-relative slugs when basePath is ""
+    console.log();
+    console.log(pc.bold(`Migrating Fluid Topics Markdown export to Dogsbay-MD: ${siteName}`));
+    console.log(`Source:  ${sourceDir}`);
+    console.log(`Output:  ${outputDir}`);
+    console.log();
+    const contentDir = join(outputDir, "content");
+    const astroDir = join(outputDir, "astro");
+    mkdirSync(contentDir, { recursive: true });
+    mkdirSync(astroDir, { recursive: true });
+    // 1. Import the export -> pages + nav.
+    const { importFluidTopicsProject } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-fluidtopics-markdown"));
+    const { treeToDogsbayMd } = (await import(
+    /* @vite-ignore */ "@dogsbay/format-dogsbay-md"));
+    const { pages, nav, unresolved } = await importFluidTopicsProject(sourceDir, {
+        hrefPrefix,
+        publications: options.publication,
+    });
+    if (options.publication && options.publication.length > 0) {
+        console.log(pc.dim(`Scoped to publications: ${options.publication.join(", ")}`));
+    }
+    // 2. Write each page to ./content/<slug>.md as Dogsbay-MD.
+    for (const page of pages) {
+        const rel = `${page.slug || "index"}.md`;
+        const dest = join(contentDir, rel);
+        mkdirSync(dirname(dest), { recursive: true });
+        const body = treeToDogsbayMd(page.tree, { frontmatter: page.frontmatter });
+        writeFileSync(dest, body.endsWith("\n") ? body : `${body}\n`);
+    }
+    console.log(pc.green("Wrote") + ` ${pages.length} pages to ./content/ as Dogsbay-MD`);
+    // 3. nav.yml (single-key-map; file paths relative to content/).
+    const navYml = navItemsToSingleKey(nav, basePath);
+    writeFileSync(join(contentDir, "nav.yml"), YAML.stringify(navYml));
+    console.log(pc.green("Extracted") + ` navigation to content/nav.yml (${navYml.length} top-level items)`);
+    // 4. dogsbay.config.yml.
+    const dogsbayConfig = {
+        schemaVersion: 1,
+        site: { name: siteName, url: options.siteUrl, basePath },
+        content: { sources: [{ path: "./content", from: "dogsbay-md" }] },
+        agent: { llmsTxt: true, mdMirror: true },
+    };
+    writeFileSync(join(outputDir, "dogsbay.config.yml"), serializeConfig(dogsbayConfig, "yaml"));
+    console.log(pc.green("Wrote") + " dogsbay.config.yml");
+    // 5. Scaffold the Astro project under ./astro/.
+    emitSiteScaffold(astroDir, siteName, { siteName, siteUrl: options.siteUrl, basePath, llmsTxt: true, mdMirror: true, local: options.local }, true);
+    console.log(pc.green("Scaffolded") + " Astro project at ./astro/");
+    // 6. MIGRATION.md.
+    const unresolvedCount = Object.values(unresolved).reduce((n, a) => n + a.length, 0);
+    writeFileSync(join(outputDir, "MIGRATION.md"), buildMigrationNotes({ sourceDir, siteName, pageCount: pages.length, unresolvedCount, basePath }));
+    console.log(pc.green("Wrote") + " MIGRATION.md");
+    if (!options.quiet) {
+        console.log();
+        console.log(pc.bold("Done! Your Dogsbay-MD site is ready."));
+        console.log();
+        console.log("Next steps:");
+        console.log(`  cd ${outputDir}`);
+        console.log("  npx dogsbay site dev          # live preview (auto-installs)");
+        console.log();
+        console.log("Edit ./content/*.md (canonical Dogsbay-MD) from here on.");
+    }
+}
+function navItemsToSingleKey(items, basePath) {
+    const out = [];
+    for (const item of items) {
+        if (item.children && item.children.length > 0) {
+            out.push({ [item.label]: navItemsToSingleKey(item.children, basePath) });
+        }
+        else if (item.href) {
+            out.push({ [item.label]: hrefToNavPath(item.href, basePath) });
+        }
+        else {
+            out.push({ [item.label]: [] });
+        }
+    }
+    return out;
+}
+/** Convert a resolved route href back to a content-relative .md path. */
+function hrefToNavPath(href, basePath) {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href))
+        return href; // external
+    let s = href.split("#")[0];
+    if (basePath && s.startsWith(basePath))
+        s = s.slice(basePath.length);
+    s = s.replace(/^\/+/, "");
+    if (!s)
+        return "index.md";
+    return `${s}.md`;
+}
+function buildMigrationNotes(args) {
+    return [
+        `# Migration notes — ${args.siteName}`,
+        "",
+        `Migrated from a Fluid Topics Markdown export at \`${args.sourceDir}\`.`,
+        "",
+        `- Pages: **${args.pageCount}**`,
+        `- Base path: \`${args.basePath || "(root)"}\``,
+        `- Unresolved internal links: **${args.unresolvedCount}** (broken cross-references in the source export are left as-is).`,
+        "",
+        "## What this did",
+        "",
+        "- Converted each topic to Dogsbay-MD under `./content/` (DITA `topic_type` → frontmatter `type`).",
+        "- Embedded HTML tables → Markdown tables; DITA nav-card grids → `:::cards`.",
+        "- Resolved both link schemes (relative `.md` + `raw.githubusercontent` TOC URLs) to routes.",
+        "- Built `content/nav.yml` from each publication's `index.md` TOC + the README order.",
+        "",
+        "## Known gaps",
+        "",
+        "- **Media is omitted** in the source export — image references may be dangling. Pull assets from the live portal or replace as needed.",
+        "- Conref/keyref/profiling were already resolved/flattened by the export (not recoverable here).",
+        "",
+        "Edit `./content/*.md` from here on; re-run the migration to refresh.",
+        "",
+    ].join("\n");
+}

package/dist/index.js

@@ -8,6 +8,8 @@ import { add, list } from "./commands/add.js";
 import { importMkdocs } from "./commands/import-mkdocs.js";
 import { migrateMkdocs } from "./commands/migrate-mkdocs.js";
 import { migrateAsciidoc } from "./commands/migrate-asciidoc.js";
+import { migrateFluidtopics } from "./commands/migrate-fluidtopics.js";
+import { migrateFluidtopicsRest } from "./commands/migrate-fluidtopics-rest.js";
 import { preprocessVariants } from "./commands/preprocess-variants.js";
 import { lighthouse } from "./commands/lighthouse.js";
 import { convert } from "./commands/convert.js";
@@ -233,6 +235,43 @@ program
     .option("--local", "Use file: references to local monorepo packages (for development)")
     .option("--quiet", "Suppress the 'Next steps' trailer")
     .action((source, options) => migrateAsciidoc(source, options));
+program
+    .command("migrate-fluidtopics")
+    .description("One-way migrate a Fluid Topics Markdown export (e.g. a ServiceNowDocs " +
+    "checkout: markdown/{publication}/**) to a scaffolded Dogsbay-MD " +
+    "project. Source-of-truth becomes ./content/*.md.")
+    .argument("<source>", "Path to the FT Markdown export (containing markdown/)")
+    .option("-o, --output <dir>", "Output directory (default: {source}-dogsbay)")
+    .option("--force", "Overwrite an existing Dogsbay site at the output dir")
+    .option("--site-name <name>", "Site name written into dogsbay.config.yml")
+    .option("--site-url <url>", "Canonical URL of the destination site. Drives robots.txt, sitemap, " +
+    "canonical tags, and llms.txt.")
+    .option("--base-path <path>", 'Path under the host where docs are served (default "" = host root, ' +
+    "the dedicated-repo convention).")
+    .option("--publication <name>", "Limit the migration to one or more publications (top-level markdown/{pub} " +
+    "folder). Repeatable. Useful for previewing a slice of a huge corpus.", (val, prev = []) => [...prev, val])
+    .option("--local", "Use file: references to local monorepo packages (for development)")
+    .option("--quiet", "Suppress the 'Next steps' trailer")
+    .action((source, options) => migrateFluidtopics(source, options));
+program
+    .command("migrate-fluidtopics-rest")
+    .description("Migrate a LIVE Fluid Topics portal via the khub REST API (Option B): " +
+    "reverse-engineers DITA-OT HTML5 → Dogsbay-MD. Works on any public FT " +
+    "portal (ServiceNow, Johnson Controls, FT docs).")
+    .argument("<portal-url>", "FT portal root (e.g. https://www.servicenow.com/docs or https://docs.johnsoncontrols.com/bas)")
+    .option("-o, --output <dir>", "Output directory (default: fluidtopics-rest-dogsbay)")
+    .option("--force", "Overwrite an existing Dogsbay site at the output dir")
+    .option("--site-name <name>", "Site name written into dogsbay.config.yml")
+    .option("--site-url <url>", "Canonical URL of the destination site")
+    .option("--base-path <path>", 'Path under the host where docs are served (default "" = host root)')
+    .option("--map <id>", "Limit to one or more map ids (repeatable)", (v, p = []) => [...p, v])
+    .option("--title-filter <substr>", "Limit to maps whose title contains this substring (case-insensitive)")
+    .option("--lang <locale>", "Locale to request from the portal (e.g. en-US)")
+    .option("--throttle <ms>", "Delay between khub requests (default 150)")
+    .option("--single-page", "Roll each map into ONE page (whole book) instead of one page per topic; xrefs become in-page anchors")
+    .option("--local", "Use file: references to local monorepo packages (for development)")
+    .option("--quiet", "Suppress the trailer")
+    .action((source, options) => migrateFluidtopicsRest(source, options));
 program
     .command("preprocess-variants")
     .description("Add variant tabs to file includes (e.g. Python 3.10+ / 3.9+ tabs)")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dogsbay",
-  "version": "0.2.0-beta.91",
+  "version": "0.2.0-beta.92",
   "description": "CLI for Dogsbay — scaffold, build, and serve documentation sites with markdown / MkDocs / Obsidian / OpenAPI sources",
   "type": "module",
   "bin": {
@@ -33,20 +33,22 @@
     "picocolors": "^1.1.0",
     "prompts": "^2.4.2",
     "yaml": "^2.8.3",
-    "@dogsbay/autodoc-python": "0.2.0-beta.91",
-    "@dogsbay/format-mkdocs": "0.2.0-beta.91",
-    "@dogsbay/format-astro": "0.2.0-beta.91",
-    "@dogsbay/format-obsidian": "0.2.0-beta.91",
-    "@dogsbay/format-mdx": "0.2.0-beta.91",
-    "@dogsbay/format-starlight": "0.2.0-beta.91",
-    "@dogsbay/format-docusaurus": "0.2.0-beta.91",
-    "@dogsbay/docusaurus-adapter-tigera": "0.2.0-beta.91",
-    "@dogsbay/format-dogsbay-md": "0.2.0-beta.91",
-    "@dogsbay/format-openapi": "0.2.0-beta.91",
-    "@dogsbay/adoc2md-modular": "0.2.0-beta.91",
-    "@dogsbay/format-asciidoc": "0.2.0-beta.91",
-    "@dogsbay/minja": "0.2.0-beta.91",
-    "@dogsbay/types": "0.2.0-beta.91"
+    "@dogsbay/autodoc-python": "0.2.0-beta.92",
+    "@dogsbay/format-astro": "0.2.0-beta.92",
+    "@dogsbay/format-mkdocs": "0.2.0-beta.92",
+    "@dogsbay/format-obsidian": "0.2.0-beta.92",
+    "@dogsbay/format-mdx": "0.2.0-beta.92",
+    "@dogsbay/format-starlight": "0.2.0-beta.92",
+    "@dogsbay/format-docusaurus": "0.2.0-beta.92",
+    "@dogsbay/docusaurus-adapter-tigera": "0.2.0-beta.92",
+    "@dogsbay/format-dogsbay-md": "0.2.0-beta.92",
+    "@dogsbay/format-fluidtopics-markdown": "0.2.0-beta.92",
+    "@dogsbay/format-fluidtopics-rest": "0.2.0-beta.92",
+    "@dogsbay/format-openapi": "0.2.0-beta.92",
+    "@dogsbay/adoc2md-modular": "0.2.0-beta.92",
+    "@dogsbay/format-asciidoc": "0.2.0-beta.92",
+    "@dogsbay/minja": "0.2.0-beta.92",
+    "@dogsbay/types": "0.2.0-beta.92"
   },
   "devDependencies": {
     "@types/markdown-it": "^14.1.0",