@dogsbay/format-fluidtopics-rest 0.2.0-beta.101

package/dist/html-to-tree.js

@@ -0,0 +1,309 @@
+/**
+ * Convert a DITA-OT **HTML5** topic (as served by FT's khub `…/content`
+ * endpoint) into Dogsbay TreeNode[].
+ *
+ * DITA-OT's HTML5 transtype preserves the DITA element vocabulary in `class`
+ * names, so the conversion is deterministic:
+ *   div.body.conbody|taskbody|refbody → topic type (concept/task/reference)
+ *   p.shortdesc / p.p / p             → paragraph
+ *   section.section + h2.sectiontitle → flattened to heading + content
+ *   table.table (CALS)                → table/thead/tbody/tr/th/td
+ *   ul/ol/li, dl/dt/dd, pre/codeblock, note → lists / deflists / code / callout
+ *   span.ph etc.                      → inline (see inline-html.ts)
+ * FT chrome (zDocs*, ai-summary, toggle/summary/tooltip, css-…) is stripped.
+ */
+import { load } from "cheerio";
+import { inlineFromChildren, textContent } from "./inline-html.js";
+const CHROME_CLASS = /\b(zDocs[\w-]*|ai-summary[\w-]*|summarize[\w-]*|summary-[\w-]*|toggle-[\w-]*|tooltip[\w-]*|disclaimer[\w-]*|sparkle[\w-]*|info-icon|info-tooltip[\w-]*|table--title-label|spacer)\b/;
+const SKIP_TAGS = new Set(["script", "style", "svg", "button", "nav", "head", "noscript"]);
+/** Block-level tags — a list item containing any of these is walked as blocks. */
+const BLOCK_TAGS = new Set([
+    "p", "ul", "ol", "dl", "table", "pre", "div", "section", "figure", "fig", "blockquote",
+    "h1", "h2", "h3", "h4", "h5", "h6", "note",
+]);
+export function htmlToTree(html, opts = {}) {
+    const $ = load(html);
+    const title = $("h1.topictitle1").first().text().trim() || $("h1").first().text().trim() || undefined;
+    const bodyEl = ($("div.body").get(0) ?? $("article").get(0) ?? $("main").get(0) ?? $("body").get(0));
+    const topicType = bodyTypeFrom(bodyEl);
+    const tree = bodyEl ? walkBlocks(bodyEl.children ?? [], opts) : [];
+    return { tree, title, topicType };
+}
+function bodyTypeFrom(el) {
+    const cls = el?.attribs?.class ?? "";
+    if (/\bconbody\b/.test(cls))
+        return "concept";
+    if (/\btaskbody\b/.test(cls))
+        return "task";
+    if (/\b(refbody|reference)\b/.test(cls))
+        return "reference";
+    if (/\bglossbody\b/.test(cls))
+        return "glossentry";
+    return "topic";
+}
+function isChrome(n) {
+    const name = (n.name ?? "").toLowerCase();
+    if (SKIP_TAGS.has(name))
+        return true;
+    const cls = n.attribs?.class ?? "";
+    if (name === "details" && /ai-summary/.test(cls))
+        return true;
+    if (CHROME_CLASS.test(cls))
+        return true;
+    return false;
+}
+/** An element is "inline" if it isn't a block tag or a heading. */
+function isInline(name) {
+    return !BLOCK_TAGS.has(name) && !/^h[1-6]$/.test(name);
+}
+/**
+ * Extract a YouTube video id from an embed/watch/short URL, else undefined.
+ * Handles youtube.com/embed/<id>, youtube-nocookie.com/embed/<id>,
+ * youtu.be/<id>, and watch?v=<id>.
+ */
+export function youtubeIdFromUrl(url) {
+    const m = url.match(/(?:youtube(?:-nocookie)?\.com\/embed\/|youtu\.be\/|youtube\.com\/watch\?(?:[^"]*&)?v=)([\w-]{6,})/);
+    return m?.[1];
+}
+function walkBlocks(nodes, opts) {
+    const out = [];
+    // Accumulate consecutive inline-level children (text, spans, xref spans,
+    // <a>, <img>, emphasis…) and flush them as ONE paragraph via the inline
+    // walker — so a mixed-content block (e.g. an <li> with "Updated the <xref>
+    // section:" followed by a nested <ul>) keeps its prose together and its
+    // xref spans resolve to links, instead of fragmenting into one-word paragraphs.
+    let buf = [];
+    const flush = () => {
+        if (buf.length === 0)
+            return;
+        const inline = inlineFromChildren(buf, opts);
+        buf = [];
+        if (inline.length)
+            out.push(paragraph(inline));
+    };
+    for (const n of nodes) {
+        if (n.type === "text") {
+            buf.push(n);
+            continue;
+        }
+        if (n.type !== "tag")
+            continue;
+        if (isChrome(n))
+            continue;
+        const name = (n.name ?? "").toLowerCase();
+        const cls = n.attribs?.class ?? "";
+        // Video embed → a `youtube` node. FT embeds training videos as
+        // `<iframe src="…youtube(-nocookie).com/embed/<id>…">`. The default walker
+        // classifies the iframe as inline and drops it (no inline rep), losing the
+        // video entirely (e.g. the FT101 lessons). Intercept before the inline gate.
+        if (name === "iframe") {
+            const id = youtubeIdFromUrl(n.attribs?.src ?? "");
+            if (id) {
+                flush();
+                const title = (n.attribs?.title ?? "").trim();
+                out.push({ type: "youtube", props: title ? { id, title } : { id } });
+            }
+            // Non-YouTube iframes are dropped (no safe generic embed); revisit if a
+            // portal uses Vimeo/Wistia etc.
+            continue;
+        }
+        // Inline element (incl. <img>, xref spans) → accumulate into the run.
+        if (isInline(name)) {
+            buf.push(n);
+            continue;
+        }
+        // Block element → flush the pending inline run, then handle the block.
+        flush();
+        const hm = /^h([1-6])$/.exec(name);
+        if (hm) {
+            out.push({
+                type: "heading",
+                props: { level: Number(hm[1]) },
+                children: [{ type: "prose", inline: inlineFromChildren(n.children, opts) }],
+            });
+            continue;
+        }
+        switch (name) {
+            case "p": {
+                const inline = inlineFromChildren(n.children, opts);
+                if (inline.length)
+                    out.push(paragraph(inline));
+                break;
+            }
+            case "section":
+            case "div": {
+                if (/\bnote\b/.test(cls)) {
+                    out.push({
+                        type: "callout",
+                        props: { variant: noteVariant(cls) },
+                        children: walkBlocks(n.children ?? [], opts),
+                    });
+                }
+                else {
+                    // section.section, div.p wrapper, fig wrapper, generic div → unwrap
+                    out.push(...walkBlocks(n.children ?? [], opts));
+                }
+                break;
+            }
+            case "ul":
+            case "ol": {
+                const items = (n.children ?? [])
+                    .filter((c) => c.type === "tag" && (c.name ?? "").toLowerCase() === "li")
+                    .map((li) => {
+                    // Inline-only <li> → one tight prose item; <li> with block children
+                    // (e.g. a nested list) → walk as blocks (inline runs coalesce there too).
+                    const hasBlock = (li.children ?? []).some((c) => c.type === "tag" && BLOCK_TAGS.has((c.name ?? "").toLowerCase()));
+                    const children = hasBlock
+                        ? walkBlocks(li.children ?? [], opts)
+                        : [{ type: "prose", inline: inlineFromChildren(li.children, opts) }];
+                    return { type: "list-item", children };
+                });
+                out.push({ type: name === "ol" ? "ordered-list" : "unordered-list", children: items });
+                break;
+            }
+            case "dl": {
+                const children = [];
+                for (const c of n.children ?? []) {
+                    const cn = (c.name ?? "").toLowerCase();
+                    // Canonical dt shape is inline-on-the-node (matches the parser +
+                    // renderDeflist, which reads dt.inline); dd carries block children.
+                    if (cn === "dt")
+                        children.push({ type: "dt", inline: inlineFromChildren(c.children, opts) });
+                    else if (cn === "dd")
+                        children.push({ type: "dd", children: walkBlocks(c.children ?? [], opts) });
+                }
+                out.push({ type: "deflist", children });
+                break;
+            }
+            case "pre": {
+                out.push({ type: "code", props: { code: rawText(n), lang: langFrom(cls) } });
+                break;
+            }
+            case "table": {
+                out.push(convertTable(n, opts));
+                break;
+            }
+            case "figure":
+            case "fig":
+            default:
+                out.push(...walkBlocks(n.children ?? [], opts));
+        }
+    }
+    flush();
+    return out;
+}
+function convertTable(t, opts) {
+    const caption = findFirst(t, "caption");
+    const children = [];
+    const section = (name) => {
+        const sec = findFirst(t, name);
+        if (!sec)
+            return null;
+        const rows = childrenNamed(sec, "tr").map((tr) => ({
+            type: "tr",
+            children: childrenNamed(tr, name === "thead" ? "th" : "td", true).map((cell) => {
+                const node = {
+                    type: name === "thead" ? "th" : "td",
+                    // Cells can hold block content (lists, multiple paragraphs) — walk as
+                    // blocks when present so structure survives; else a single prose run.
+                    children: cellChildren(cell, opts),
+                };
+                // Carry CALS spans (DITA-OT emits them as HTML colspan/rowspan) so the
+                // serializer can emit a spanning HTML table — otherwise every cell
+                // below a row-spanned cell shifts into the wrong column.
+                const spans = cellSpans(cell);
+                if (spans)
+                    node.props = spans;
+                return node;
+            }),
+        }));
+        return rows.length ? { type: name, children: rows } : null;
+    };
+    const thead = section("thead");
+    const tbody = section("tbody");
+    if (thead)
+        children.push(thead);
+    if (tbody)
+        children.push(tbody);
+    const node = { type: "table", children };
+    if (caption) {
+        const cap = textContent(caption);
+        if (cap)
+            node.props = { caption: cap };
+    }
+    return node;
+}
+// ── helpers ─────────────────────────────────────────────
+/** Read a cell's colspan/rowspan into TreeNode props (only when > 1). */
+function cellSpans(cell) {
+    const cs = parseInt(cell.attribs?.colspan ?? "", 10);
+    const rs = parseInt(cell.attribs?.rowspan ?? "", 10);
+    const props = {};
+    if (Number.isFinite(cs) && cs > 1)
+        props.colspan = cs;
+    if (Number.isFinite(rs) && rs > 1)
+        props.rowspan = rs;
+    return Object.keys(props).length ? props : undefined;
+}
+/** Cell content: block-walk when it has block children (lists, paragraphs), else inline. */
+function cellChildren(cell, opts) {
+    const hasBlock = (cell.children ?? []).some((c) => c.type === "tag" && BLOCK_TAGS.has((c.name ?? "").toLowerCase()));
+    return hasBlock
+        ? walkBlocks(cell.children ?? [], opts)
+        : [{ type: "prose", inline: inlineFromChildren(cell.children, opts) }];
+}
+function paragraph(inline) {
+    return { type: "paragraph", children: [{ type: "prose", inline }] };
+}
+function noteVariant(cls) {
+    const m = /\bnote_(\w+)\b/.exec(cls) || /\b(warning|caution|danger|tip|important|note)\b/.exec(cls);
+    return m ? (m[1] === "note" ? "note" : m[1]) : "note";
+}
+function langFrom(cls) {
+    const m = /\blanguage-([\w+-]+)\b/.exec(cls);
+    return m ? m[1] : undefined;
+}
+/** Raw text (no whitespace collapse) — for code blocks. */
+function rawText(n) {
+    let s = "";
+    const rec = (nodes) => {
+        for (const c of nodes) {
+            if (c.type === "text")
+                s += c.data ?? "";
+            else if (c.children)
+                rec(c.children);
+        }
+    };
+    rec(n.children ?? []);
+    return s.replace(/^\n+|\s+$/g, "");
+}
+/** First descendant element with the given tag name. */
+function findFirst(n, name) {
+    for (const c of n.children ?? []) {
+        if (c.type === "tag") {
+            if ((c.name ?? "").toLowerCase() === name)
+                return c;
+            const deep = findFirst(c, name);
+            if (deep)
+                return deep;
+        }
+    }
+    return undefined;
+}
+/** Direct (or near) child elements with the given tag name. `deep` searches descendants. */
+function childrenNamed(n, name, deep = false) {
+    const out = [];
+    const scan = (nodes) => {
+        for (const c of nodes) {
+            if (c.type !== "tag")
+                continue;
+            if ((c.name ?? "").toLowerCase() === name)
+                out.push(c);
+            else if (deep && c.children)
+                scan(c.children);
+        }
+    };
+    scan(n.children ?? []);
+    return out;
+}
+//# sourceMappingURL=html-to-tree.js.map

package/dist/html-to-tree.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-tree.js","sourceRoot":"","sources":["../src/html-to-tree.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAE/B,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAoC,MAAM,kBAAkB,CAAC;AAUrG,MAAM,YAAY,GAChB,qLAAqL,CAAC;AACxL,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;AAC3F,kFAAkF;AAClF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY;IACtF,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM;CAC3C,CAAC,CAAC;AAEH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAsB,EAAE;IAC/D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAErB,MAAM,KAAK,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;IAEtG,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAEtF,CAAC;IAEd,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACpC,CAAC;AAED,SAAS,YAAY,CAAC,EAAuB;IAC3C,MAAM,GAAG,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IACrC,IAAI,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5C,IAAI,yBAAyB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,WAAW,CAAC;IAC5D,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,YAAY,CAAC;IACnD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,CAAU;IAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1C,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IACnC,IAAI,IAAI,KAAK,SAAS,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9D,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mEAAmE;AACnE,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CACjB,mGAAmG,CACpG,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAgB,EAAE,IAAmB;IACvD,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,yEAAyE;IACzE,wEAAwE;IACxE,2EAA2E;IAC3E,wEAAwE;IACxE,gFAAgF;IAChF,IAAI,GAAG,GAAc,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC7C,GAAG,GAAG,EAAE,CAAC;QACT,IAAI,MAAM,CAAC,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACZ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK;YAAE,SAAS;QAC/B,IAAI,QAAQ,CAAC,CAAC,CAAC;YAAE,SAAS;QAE1B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QAEnC,+DAA+D;QAC/D,2EAA2E;QAC3E,2EAA2E;QAC3E,6EAA6E;QAC7E,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,EAAE,GAAG,gBAAgB,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;YAClD,IAAI,EAAE,EAAE,CAAC;gBACP,KAAK,EAAE,CAAC;gBACR,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,wEAAwE;YACxE,gCAAgC;YAChC,SAAS;QACX,CAAC;QAED,sEAAsE;QACtE,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACZ,SAAS;QACX,CAAC;QAED,uEAAuE;QACvE,KAAK,EAAE,CAAC;QAER,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,EAAE,EAAE,CAAC;YACP,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;gBAC/B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC;aAC5E,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,GAAG,CAAC,CAAC,CAAC;gBACT,MAAM,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACpD,IAAI,MAAM,CAAC,MAAM;oBAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC/C,MAAM;YACR,CAAC;YACD,KAAK,SAAS,CAAC;YACf,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,GAAG,CAAC,IAAI,CAAC;wBACP,IAAI,EAAE,SAAS;wBACf,KAAK,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE;wBACpC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;qBAC7C,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,oEAAoE;oBACpE,GAAG,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;gBAClD,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC,CAAC,CAAC;gBACV,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;qBAC7B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC;qBACxE,GAAG,CAAC,CAAC,EAAE,EAAY,EAAE;oBACpB,oEAAoE;oBACpE,0EAA0E;oBAC1E,MAAM,QAAQ,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CACxE,CAAC;oBACF,MAAM,QAAQ,GAAe,QAAQ;wBACnC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;wBACrC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBACvE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;gBACzC,CAAC,CAAC,CAAC;gBACL,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,gBAAgB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;gBACvF,MAAM;YACR,CAAC;YACD,KAAK,IAAI,CAAC,CAAC,CAAC;gBACV,MAAM,QAAQ,GAAe,EAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBACjC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;oBACxC,iEAAiE;oBACjE,oEAAoE;oBACpE,IAAI,EAAE,KAAK,IAAI;wBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;yBACxF,IAAI,EAAE,KAAK,IAAI;wBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;gBACpG,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACxC,MAAM;YACR,CAAC;YACD,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACR,CAAC;YACD,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBAChC,MAAM;YACR,CAAC;YACD,KAAK,QAAQ,CAAC;YACd,KAAK,KAAK,CAAC;YACX;gBACE,GAAG,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,YAAY,CAAC,CAAU,EAAE,IAAmB;IACnD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAe,EAAE,CAAC;IAEhC,MAAM,OAAO,GAAG,CAAC,IAAuB,EAAmB,EAAE;QAC3D,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAAG,CACvC,CAAC,EAAE,EAAY,EAAE,CAAC,CAAC;YACjB,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC7E,MAAM,IAAI,GAAa;oBACrB,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;oBACpC,sEAAsE;oBACtE,sEAAsE;oBACtE,QAAQ,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC;iBACnC,CAAC;gBACF,uEAAuE;gBACvE,mEAAmE;gBACnE,yDAAyD;gBACzD,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC9B,IAAI,KAAK;oBAAE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;gBAC9B,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CACH,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,IAAI,KAAK;QAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,KAAK;QAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEhC,MAAM,IAAI,GAAa,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACnD,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACjC,IAAI,GAAG;YAAE,IAAI,CAAC,KAAK,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2DAA2D;AAE3D,yEAAyE;AACzE,SAAS,SAAS,CAAC,IAAa;IAC9B,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACrD,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;QAAE,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC;IACtD,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;QAAE,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC;IACtD,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AAED,4FAA4F;AAC5F,SAAS,YAAY,CAAC,IAAa,EAAE,IAAmB;IACtD,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CACxE,CAAC;IACF,OAAO,QAAQ;QACb,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,SAAS,CAAC,MAA6C;IAC9D,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AACtE,CAAC;AACD,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,iDAAiD,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AACxD,CAAC;AACD,SAAS,QAAQ,CAAC,GAAW;IAC3B,MAAM,CAAC,GAAG,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9B,CAAC;AACD,2DAA2D;AAC3D,SAAS,OAAO,CAAC,CAAU;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,MAAM,GAAG,GAAG,CAAC,KAAgB,EAAE,EAAE;QAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;gBAAE,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;iBACpC,IAAI,CAAC,CAAC,QAAQ;gBAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC,CAAC;IACF,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACtB,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AACrC,CAAC;AACD,wDAAwD;AACxD,SAAS,SAAS,CAAC,CAAU,EAAE,IAAY;IACzC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI;gBAAE,OAAO,CAAC,CAAC;YACpD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YAChC,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AACD,4FAA4F;AAC5F,SAAS,aAAa,CAAC,CAAU,EAAE,IAAY,EAAE,IAAI,GAAG,KAAK;IAC3D,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAE,EAAE;QAChC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK;gBAAE,SAAS;YAC/B,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAClD,IAAI,IAAI,IAAI,CAAC,CAAC,QAAQ;gBAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACvB,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/importer.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Fluid Topics REST importer.
+ *
+ * Harvests a live FT portal via the khub API and reverse-engineers each topic's
+ * DITA-OT HTML5 into Dogsbay TreeNode[] — producing ExportPage[] + NavItem[].
+ * Works on any *public* FT portal (ServiceNow, Johnson Controls /bas, FT's own
+ * docs); gated portals throw KhubAuthError.
+ */
+import type { ExportPage, NavItem } from "@dogsbay/types";
+import { KhubClient } from "./khub.js";
+export interface RestImportOptions {
+    /** Portal root or `…/api/khub` URL (ignored if `client` is given). */
+    portalBase?: string;
+    /** Inject a client (tests). */
+    client?: KhubClient;
+    /** Limit to these map ids; else import every map the portal lists. */
+    maps?: string[];
+    /** Filter listed maps by case-insensitive title substring. */
+    titleFilter?: string;
+    lang?: string;
+    hrefPrefix?: string;
+    throttleMs?: number;
+    fetchImpl?: typeof fetch;
+    taxonomyNames?: readonly string[];
+    onProgress?: (msg: string) => void;
+    /**
+     * Roll each map into ONE page (whole book) instead of one page per topic.
+     * TOC nodes become nested headings; internal xrefs become in-page anchors.
+     */
+    singlePage?: boolean;
+    /**
+     * Group the (otherwise flat) per-map nav under one or more metadata facets —
+     * the khub `/maps` list carries faceted metadata (`Category`, `Product`,
+     * `audience`, …). Comma-separated for multi-level (e.g. "Product,Category").
+     * - undefined → AUTO: group by `Category` (then `Product`) when it's present
+     *   on most maps with >1 distinct value; else stay flat.
+     * - "none" / "flat" / "" → force flat (one top-level item per map).
+     */
+    groupBy?: string;
+    /**
+     * Mark book-type maps (metadata `ft:editorialType === "book"`) with the
+     * granularity `single-page: yes` nav annotation, so the build emits an
+     * additive "Read as single page" rollup for each multi-topic guide while
+     * single-topic articles (Screencasts, At a Glance) stay per-topic. The
+     * build-time size cap skips any rollup that's too large. Composes with
+     * `groupBy`. (This is the additive-view annotation, NOT the import-time
+     * whole-book `singlePage` mode above.)
+     */
+    singlePageBooks?: boolean;
+    /**
+     * FT khubs often carry stale duplicate publications: the same guide published
+     * more than once with the same title + version, distinguished only by
+     * `ft:lastEdition`. They slug identically, so their pages collapse (last-wins)
+     * but the nav gets one entry per map → visible duplicate guides. By default we
+     * keep only the freshest edition per (title, Version_FT). Set this to keep all
+     * maps (no dedup).
+     */
+    keepDuplicateMaps?: boolean;
+    /**
+     * Extra FT topic-metadata keys to lift into page frontmatter verbatim, beyond
+     * the curated set (subject/product/author/category/audience/version/…). The
+     * importer fetches per-topic metadata from `/maps/{id}/topics` (one request
+     * per map) regardless; this only widens which keys are kept.
+     */
+    metadataAllow?: readonly string[];
+    /**
+     * Fetch FULL per-topic metadata (one request PER TOPIC) so each page gets its
+     * own `ft:wordCount` / `ft:outputSize` and a per-topic `updated`
+     * (`ft:lastTechChange`). Off by default — the cheap path (one `/topics` call
+     * per map) only carries the facet keys, so without this the technical
+     * numerics are emitted at book level only. Roughly doubles request count.
+     */
+    fullTopicMetadata?: boolean;
+}
+export interface RestImportResult {
+    pages: ExportPage[];
+    nav: NavItem[];
+    unresolved: Record<string, string[]>;
+    mapsProcessed: number;
+    /** Extracted `data:` images, as content-relative paths + bytes to write. */
+    assets: {
+        path: string;
+        data: Buffer;
+    }[];
+}
+export declare function importFluidTopicsRest(opts: RestImportOptions): Promise<RestImportResult>;
+//# sourceMappingURL=importer.d.ts.map

package/dist/importer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAE1D,OAAO,EAAE,UAAU,EAA0D,MAAM,WAAW,CAAC;AAU/F,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,GAAG,EAAE,OAAO,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC1C;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAqP9F"}