@iwonz/ywtc 3.2.0 → 3.2.1

package/dist/content-model.js

@@ -30,11 +30,97 @@ function absoluteResourceUrl(page, resource) {
     }
     return page.url;
 }
+function normalizedResourceAlias(value) {
+    const trimmed = value.trim();
+    if (!trimmed)
+        return undefined;
+    let normalized = trimmed;
+    try {
+        normalized = decodeURIComponent(trimmed);
+    }
+    catch {
+        // Keep provider text when percent encoding is malformed.
+    }
+    return normalized.replace(/\\/g, '/').replace(/^\.\//, '').toLocaleLowerCase();
+}
+function resourceAliasCandidates(reference) {
+    const aliases = new Set();
+    const add = (value) => {
+        if (!value)
+            return;
+        const normalized = normalizedResourceAlias(value);
+        if (!normalized)
+            return;
+        aliases.add(normalized);
+        aliases.add(normalized.replace(/^attachment:/, ''));
+        aliases.add(normalized.replace(/^\/+/, ''));
+        const marker = normalized.match(/(?:^|\/)((?:\.files|_files|attachments)\/.*)$/i)?.[1];
+        if (marker)
+            aliases.add(marker);
+    };
+    const withoutFragment = reference.split('#', 1)[0] ?? reference;
+    add(withoutFragment);
+    try {
+        const url = new URL(withoutFragment);
+        add(url.pathname);
+    }
+    catch {
+        // Relative references are already covered above.
+    }
+    const snapshot = [...aliases];
+    for (const alias of snapshot)
+        add(alias.split('/').at(-1));
+    return [...aliases];
+}
+export function preferredResourceName(resource) {
+    return resource.preferredName ?? resource.name;
+}
+export function sourceResourceAliases(page, resource) {
+    const aliases = new Set();
+    const add = (value) => {
+        for (const alias of value ? resourceAliasCandidates(value) : [])
+            aliases.add(alias);
+    };
+    add(resource.name);
+    add(resource.preferredName);
+    add(resource.downloadUrl);
+    for (const alias of resource.referenceAliases ?? [])
+        add(alias);
+    add(absoluteResourceUrl(page, resource));
+    return [...aliases];
+}
 export function sourceResourceBindings(page, resources) {
-    return new Map(resources.filter((resource) => resource.kind !== 'grid').map((resource) => [
-        resource.name.toLocaleLowerCase(),
-        { kind: 'source', resource, url: absoluteResourceUrl(page, resource) },
-    ]));
+    const bindings = new Map();
+    const aliases = new Map();
+    for (const resource of resources.filter((item) => item.kind !== 'grid')) {
+        for (const alias of sourceResourceAliases(page, resource)) {
+            const matched = aliases.get(alias) ?? [];
+            if (!matched.some((candidate) => candidate.id === resource.id))
+                matched.push(resource);
+            aliases.set(alias, matched);
+        }
+    }
+    for (const [alias, matched] of aliases) {
+        if (matched.length !== 1)
+            continue;
+        const resource = matched[0];
+        bindings.set(alias, { kind: 'source', resource, url: absoluteResourceUrl(page, resource) });
+    }
+    return bindings;
+}
+export function setSourceResourceBinding(bindings, page, resources, resource, binding) {
+    const aliasOwners = new Map();
+    for (const candidate of resources.filter((item) => item.kind !== 'grid')) {
+        for (const alias of sourceResourceAliases(page, candidate)) {
+            const owners = aliasOwners.get(alias) ?? new Set();
+            owners.add(candidate.id);
+            aliasOwners.set(alias, owners);
+        }
+    }
+    for (const alias of sourceResourceAliases(page, resource)) {
+        if (aliasOwners.get(alias)?.size === 1)
+            bindings.set(alias, binding);
+    }
 }
 function bindingTarget(binding) {
     return binding.kind === 'uploaded' ? binding.resource.url ?? binding.resource.name
@@ -42,13 +128,12 @@ function bindingTarget(binding) {
             : binding.path;
 }
 function resourceBinding(reference, context) {
-    const withoutFragment = reference.split(/[?#]/, 1)[0] ?? reference;
-    const normalized = withoutFragment.replace(/^attachment:/, '').replace(/^\.\//, '').replace(/^\/+/, '').toLocaleLowerCase();
-    const direct = context.resources.get(normalized);
-    if (direct)
-        return direct;
-    const basename = normalized.replace(/\\/g, '/').split('/').at(-1);
-    return basename ? context.resources.get(basename) : undefined;
+    for (const alias of resourceAliasCandidates(reference)) {
+        const binding = context.resources.get(alias);
+        if (binding)
+            return binding;
+    }
+    return undefined;
 }
 function uniqueReferences(references) {
     const seen = new Set();
@@ -172,7 +257,69 @@ function storageToSemanticHtml(storage, warnings) {
     });
     return $('main[data-ywtc-root]').html() ?? '';
 }
-function yfmToHtml(content, warnings) {
+export function stripAnsi(value) {
+    let output = '';
+    for (let index = 0; index < value.length;) {
+        const code = value.charCodeAt(index);
+        const controlSequence = code === 0x9b || (code === 0x1b && value[index + 1] === '[');
+        if (controlSequence) {
+            index += code === 0x9b ? 1 : 2;
+            while (index < value.length) {
+                const current = value.charCodeAt(index++);
+                if (current >= 0x40 && current <= 0x7e)
+                    break;
+            }
+            continue;
+        }
+        if (code === 0x1b && value[index + 1] === ']') {
+            index += 2;
+            while (index < value.length) {
+                if (value.charCodeAt(index) === 0x07) {
+                    index += 1;
+                    break;
+                }
+                if (value.charCodeAt(index) === 0x1b && value[index + 1] === '\\') {
+                    index += 2;
+                    break;
+                }
+                index += 1;
+            }
+            continue;
+        }
+        if (code === 0x1b) {
+            index += Math.min(2, value.length - index);
+            continue;
+        }
+        output += value[index];
+        index += 1;
+    }
+    return output;
+}
+function extractTreeDirectives(content) {
+    const directives = [];
+    const next = content.replace(/{%\s*tree\b([^%]*?)%}/gi, (raw, attributes) => {
+        const values = new Map();
+        for (const match of attributes.matchAll(/([\w-]+)\s*=\s*["']([^"']*)["']/g)) {
+            if (match[1] && match[2] !== undefined)
+                values.set(match[1].toLocaleLowerCase(), match[2].toLocaleLowerCase());
+        }
+        const depthValue = Number(values.get('depth'));
+        const sort = values.get('sort');
+        const sortBy = values.get('sort_by');
+        directives.push({
+            raw,
+            ...(Number.isInteger(depthValue) && depthValue > 0 ? { depth: depthValue } : {}),
+            ...(sort === 'asc' || sort === 'desc' ? { sort } : {}),
+            ...(sortBy === 'title' ? { sortBy } : {}),
+        });
+        return '';
+    });
+    return { content: next, directives };
+}
+function containsEmptyYfmHeading(content) {
+    return /^\s{0,3}#{1,6}\s*$/m.test(content);
+}
+function yfmToHtml(content, warnings, normalizations = []) {
     try {
         const result = transform(content || '', {
             allowHTML: true,
@@ -181,11 +328,36 @@ function yfmToHtml(content, warnings) {
             lang: 'ru',
             linkify: true,
         });
-        warnings.push(...(result.logs.warn ?? []), ...(result.logs.error ?? []));
-        return result.result.html || '<p></p>';
+        const $ = load(result.result.html || '<p></p>', null, false);
+        let removedHeadings = 0;
+        $('h1,h2,h3,h4,h5,h6').each((_index, element) => {
+            const heading = $(element);
+            const meaningful = heading.text().replace(/\s+/g, '').length > 0
+                || heading.find('img,table,pre,code').length > 0;
+            if (meaningful)
+                return;
+            heading.find('[id],a[name]').each((_anchorIndex, anchor) => { heading.before($(anchor).clone()); });
+            heading.remove();
+            removedHeadings += 1;
+        });
+        let ignoredHeaderDiagnostics = removedHeadings;
+        for (const diagnostic of [...(result.logs.warn ?? []), ...(result.logs.error ?? [])]) {
+            const clean = stripAnsi(String(diagnostic)).trim();
+            if (/Header without title/i.test(clean)) {
+                if (ignoredHeaderDiagnostics > 0)
+                    ignoredHeaderDiagnostics -= 1;
+                if (removedHeadings > 0 || !containsEmptyYfmHeading(content))
+                    continue;
+            }
+            if (clean)
+                warnings.push(clean);
+        }
+        if (removedHeadings > 0)
+            normalizations.push(`Removed ${removedHeadings} empty heading${removedHeadings === 1 ? '' : 's'} emitted by source conversion`);
+        return $.html() || '<p></p>';
     }
     catch (error) {
-        warnings.push(`YFM transformation failed: ${error instanceof Error ? error.message : String(error)}`);
+        warnings.push(`YFM transformation failed: ${stripAnsi(error instanceof Error ? error.message : String(error))}`);
         return `<pre data-ywtc-fallback><code>${escapeHtml(content)}</code></pre>`;
     }
 }
@@ -195,7 +367,9 @@ function metrics(html) {
 }
 export function normalizePageContent(page) {
     const warnings = [];
-    const html = page.rawFormat === 'yfm' ? yfmToHtml(page.rawContent, warnings) : storageToSemanticHtml(page.rawContent, warnings);
+    const normalizations = [];
+    const tree = page.rawFormat === 'yfm' ? extractTreeDirectives(page.rawContent) : { content: page.rawContent, directives: [] };
+    const html = page.rawFormat === 'yfm' ? yfmToHtml(tree.content, warnings, normalizations) : storageToSemanticHtml(page.rawContent, warnings);
     const references = extractPageReferences(page.rawContent, page.rawFormat).map((reference, index) => ({
         token: `ref-${index}`,
         kind: reference.kind,
@@ -205,6 +379,15 @@ export function normalizePageContent(page) {
         ...(reference.anchor ? { anchor: reference.anchor } : {}),
     }));
     const children = [{ type: 'html', value: html }];
+    children.push(...tree.directives.map((directive) => ({
+        type: 'tree-directive',
+        value: directive.raw,
+        tree: {
+            ...(directive.depth ? { depth: directive.depth } : {}),
+            ...(directive.sort ? { sort: directive.sort } : {}),
+            ...(directive.sortBy ? { sortBy: directive.sortBy } : {}),
+        },
+    })));
     if (warnings.length > 0)
         children.push({ type: 'vendor-extension', vendor: page.provider, value: page.rawContent });
     return {
@@ -213,6 +396,7 @@ export function normalizePageContent(page) {
         rawFormat: page.rawFormat,
         references,
         warnings,
+        normalizations,
         degraded: warnings.length > 0,
         metrics: metrics(html),
     };
@@ -304,6 +488,7 @@ export function rawContentFallback(page, provider, reason) {
         return {
             body: `<p><strong>Source content fallback</strong> · <a href="${escapeHtml(page.url)}">original page</a></p>${raw}`,
             warnings: [warning],
+            normalizations: [],
             degraded: true,
             representedReferences: page.references.length,
         };
@@ -314,35 +499,88 @@ export function rawContentFallback(page, provider, reason) {
     return {
         body: `**Source content fallback** · [original page](${page.url})\n\n${raw}\n`,
         warnings: [warning],
+        normalizations: [],
         degraded: true,
         representedReferences: page.references.length,
     };
 }
-export function semanticContentFingerprint(body, provider) {
+function semanticText(value) {
+    return value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim();
+}
+function semanticContent(body, provider) {
     const html = renderedPreviewHtml(body, provider);
     const $ = load(html, null, false);
-    $('script, style').remove();
-    const tokens = $('*').toArray().map((element) => {
+    $('script,style').remove();
+    $('h1,h2,h3,h4,h5,h6,p,div,span').each((_index, element) => {
         const node = $(element);
-        const directText = node.clone().children().remove().end().text().replace(/\s+/g, ' ').trim();
-        const attributes = ['href', 'src', 'id', 'class'].map((name) => node.attr(name) ? `${name}=${node.attr(name)}` : '').filter(Boolean);
-        const tagName = 'name' in element ? String(element.name) : element.type;
-        return `${tagName.toLocaleLowerCase()}[${attributes.join(',')}](${directText})`;
+        if (!semanticText(node.text()) && node.find('img,table,pre,code,a[href]').length === 0)
+            node.remove();
     });
-    const text = $.root().text().replace(/\s+/g, ' ').trim();
-    return sha256(JSON.stringify({ text, tokens }));
+    const textValues = (selector) => $(selector).toArray().map((element) => semanticText($(element).text()));
+    const macroNames = [];
+    if (provider === 'confluence') {
+        const storage = load(`<root>${body}</root>`, { xmlMode: true }, false);
+        storage('ac\\:structured-macro').each((_index, element) => {
+            const macro = storage(element);
+            const name = macro.attr('ac:name') ?? 'unknown';
+            const parameters = macro.children('ac\\:parameter').toArray().map((parameter) => (`${storage(parameter).attr('ac:name') ?? ''}=${semanticText(storage(parameter).text())}`)).sort();
+            macroNames.push(`${name}[${parameters.join(',')}]`);
+        });
+    }
+    return {
+        text: semanticText($.root().text()),
+        headings: $('h1,h2,h3,h4,h5,h6').toArray().map((element) => `${element.tagName}:${semanticText($(element).text())}`),
+        listItems: textValues('li'),
+        tableCells: $('th,td').toArray().map((element) => `${element.tagName}:${semanticText($(element).text())}`),
+        code: textValues('pre,code'),
+        links: $('a[href]').toArray().map((element) => `${$(element).attr('href') ?? ''}|${semanticText($(element).text())}`).sort(),
+        images: $('img').toArray().map((element) => `${$(element).attr('src') ?? ''}|${$(element).attr('alt') ?? ''}`).sort(),
+        anchors: $('[id],a[name]').toArray().map((element) => $(element).attr('id') ?? $(element).attr('name') ?? '').filter(Boolean).sort(),
+        structure: {
+            lists: $('ul,ol').length,
+            rows: $('tr').length,
+            details: $('details').length,
+            asides: $('aside').length,
+            codeBlocks: $('pre').length,
+        },
+        macros: macroNames.sort(),
+    };
 }
-export function assertDestinationContent(expectedBody, actualBody, provider, page) {
+export function semanticContentFingerprint(body, provider) {
+    return sha256(JSON.stringify(semanticContent(body, provider)));
+}
+export function verifyDestinationContent(expectedBody, actualBody, provider, page) {
     if (actualBody === undefined)
-        throw new Error(`Verification could not read destination content for ${page.title}`);
+        return { status: 'failed', differences: [`Verification could not read destination content for ${page.title}`] };
     if (actualBody.trim().length === 0)
-        throw new Error(`Verification found empty destination content for ${page.title}`);
+        return { status: 'failed', differences: [`Verification found empty destination content for ${page.title}`] };
     if (isLegacyTransferPlaceholder(actualBody, provider, page)) {
-        throw new Error(`Verification found a transfer placeholder instead of source content for ${page.title}`);
+        return { status: 'failed', differences: [`Verification found a transfer placeholder instead of source content for ${page.title}`] };
     }
-    if (semanticContentFingerprint(expectedBody, provider) !== semanticContentFingerprint(actualBody, provider)) {
-        throw new Error(`Verification found destination content that differs from the rendered source for ${page.title}`);
+    const expected = semanticContent(expectedBody, provider);
+    const actual = semanticContent(actualBody, provider);
+    const differs = (key) => JSON.stringify(expected[key]) !== JSON.stringify(actual[key]);
+    const missingData = ['text', 'tableCells', 'code', 'links', 'images', 'anchors', 'macros'].filter(differs);
+    if (missingData.length > 0) {
+        return {
+            status: 'failed',
+            differences: [`Verification found destination content that differs from the rendered source for ${page.title}: ${missingData.join(', ')}`],
+        };
+    }
+    const simplifiedFormatting = ['headings', 'listItems', 'structure'].filter(differs);
+    if (simplifiedFormatting.length > 0) {
+        return {
+            status: 'partial',
+            differences: [`Destination formatting was simplified for ${page.title}: ${simplifiedFormatting.join(', ')}`],
+        };
     }
+    return { status: 'complete', differences: [] };
+}
+export function assertDestinationContent(expectedBody, actualBody, provider, page) {
+    const result = verifyDestinationContent(expectedBody, actualBody, provider, page);
+    if (result.status === 'failed')
+        throw new Error(result.differences.join('; '));
+    return result;
 }
 function relativeTarget(currentPath, targetPath) {
     if (!currentPath)
@@ -390,15 +628,57 @@ function rewritePortableHtml(html, context) {
     });
     return $.html();
 }
+function treeDirectiveNodes(document) {
+    return document.root.children?.filter((node) => node.type === 'tree-directive') ?? [];
+}
+function treeChildren(context, parentId, directive, depth = 1) {
+    if (directive.depth !== undefined && depth > directive.depth)
+        return [];
+    return [...context.sourcePages.values()]
+        .filter((page) => page.projectedParentId === parentId)
+        .sort((left, right) => {
+        const order = directive.sortBy === 'title' ? left.title.localeCompare(right.title) : left.canonicalId.localeCompare(right.canonicalId);
+        return directive.sort === 'desc' ? -order : order;
+    });
+}
+function renderTreeMarkdown(context, directive) {
+    const render = (parentId, depth) => treeChildren(context, parentId, directive, depth).map((page) => {
+        const destination = context.pages.get(page.canonicalId);
+        if (!destination)
+            return '';
+        const href = 'path' in destination ? relativeTarget(context.currentPath, destination.path) : destination.url;
+        const nested = render(page.canonicalId, depth + 1);
+        return `${'  '.repeat(depth - 1)}- [${page.title}](${href})${nested ? `\n${nested}` : ''}`;
+    }).filter(Boolean).join('\n');
+    return render(context.currentPageId, 1);
+}
+function renderTreeHtml(context, directive) {
+    const render = (parentId, depth) => {
+        const items = treeChildren(context, parentId, directive, depth).map((page) => {
+            const destination = context.pages.get(page.canonicalId);
+            if (!destination)
+                return '';
+            const href = 'path' in destination ? relativeTarget(context.currentPath, destination.path) : destination.url;
+            return `<li><a href="${escapeHtml(href)}">${escapeHtml(page.title)}</a>${render(page.canonicalId, depth + 1)}</li>`;
+        }).filter(Boolean).join('');
+        return items ? `<ul>${items}</ul>` : '';
+    };
+    const tree = render(context.currentPageId, 1);
+    return tree ? `<nav class="ywtc-tree" aria-label="Child pages">${tree}</nav>` : '';
+}
 export async function renderMarkdown(document, context) {
     const html = rewritePortableHtml(htmlFromDocument(document), context);
-    const body = await htmlToMarkdown(html);
-    return { body, warnings: [...document.warnings], degraded: document.degraded, representedReferences: document.references.length };
+    const tree = treeDirectiveNodes(document).map((node) => renderTreeMarkdown(context, node.tree ?? {})).filter(Boolean).join('\n\n');
+    const content = await htmlToMarkdown(html);
+    const body = tree ? `${content.trimEnd()}\n\n${tree}\n` : content;
+    return { body, warnings: [...document.warnings], normalizations: [...(document.normalizations ?? [])], degraded: document.degraded, representedReferences: document.references.length };
 }
 export async function renderStaticHtml(document, context) {
     const rewritten = rewritePortableHtml(htmlFromDocument(document), context);
-    const body = await sanitizeHtml(rewritten);
-    return { body, warnings: [...document.warnings], degraded: document.degraded, representedReferences: document.references.length };
+    const content = await sanitizeHtml(rewritten);
+    const tree = treeDirectiveNodes(document).map((node) => renderTreeHtml(context, node.tree ?? {})).filter(Boolean).join('');
+    const body = `${content}${tree}`;
+    return { body, warnings: [...document.warnings], normalizations: [...(document.normalizations ?? [])], degraded: document.degraded, representedReferences: document.references.length };
 }
 export async function renderYfm(document, context) {
     if (document.rawFormat === 'yfm') {
@@ -440,12 +720,13 @@ export async function renderYfm(document, context) {
         return {
             body,
             warnings: [...document.warnings, ...resourceWarnings],
+            normalizations: [...(document.normalizations ?? [])],
             degraded: document.degraded || resourceWarnings.size > 0,
             representedReferences: document.references.length,
         };
     }
     const body = await htmlToMarkdown(rewritePortableHtml(htmlFromDocument(document), context));
-    return { body, warnings: [...document.warnings, 'Confluence Storage Format was converted to YFM-compatible Markdown'], degraded: document.degraded, representedReferences: document.references.length };
+    return { body, warnings: [...document.warnings, 'Confluence Storage Format was converted to YFM-compatible Markdown'], normalizations: [...(document.normalizations ?? [])], degraded: document.degraded, representedReferences: document.references.length };
 }
 export async function renderConfluenceStorage(document, page, context, spaceKey) {
     if (document.rawFormat === 'confluence-storage') {
@@ -525,6 +806,7 @@ export async function renderConfluenceStorage(document, page, context, spaceKey)
         return {
             body: $('main[data-root]').html() ?? '<p></p>',
             warnings,
+            normalizations: [...(document.normalizations ?? [])],
             degraded: document.degraded || warnings.length > document.warnings.length,
             representedReferences: document.references.length,
         };
@@ -549,7 +831,8 @@ export async function renderConfluenceStorage(document, page, context, spaceKey)
         else if (binding.kind === 'source' && binding.url)
             resourceFallbacks.set(name, binding.url);
     }
-    const converted = convertYfmToStorage(document.raw, {
+    const tree = extractTreeDirectives(document.raw);
+    const converted = convertYfmToStorage(tree.content, {
         currentSlug: page.locator,
         spaceKey,
         wikiWebUrl: new URL(page.url).origin,
@@ -557,12 +840,45 @@ export async function renderConfluenceStorage(document, page, context, spaceKey)
         attachmentsByReference: attachments,
         resourceFallbacksByReference: resourceFallbacks,
     });
+    const normalizedStorage = load(`<main data-root>${converted.storage}</main>`, { xmlMode: true }, false);
+    let removedHeadings = 0;
+    normalizedStorage('h1,h2,h3,h4,h5,h6').each((_index, element) => {
+        const heading = normalizedStorage(element);
+        if (semanticText(heading.text()) || heading.find('img,table,pre,code').length > 0)
+            return;
+        heading.find('[id],a[name]').each((_anchorIndex, anchor) => { heading.before(normalizedStorage(anchor).clone()); });
+        heading.remove();
+        removedHeadings += 1;
+    });
+    let ignorableHeaderWarnings = removedHeadings;
+    const conversionWarnings = converted.warnings.map(stripAnsi).filter((warning) => {
+        if (/Header without title/i.test(warning)) {
+            if (ignorableHeaderWarnings > 0)
+                ignorableHeaderWarnings -= 1;
+            if (removedHeadings > 0 || !containsEmptyYfmHeading(tree.content))
+                return false;
+        }
+        return Boolean(warning.trim());
+    });
+    const normalizations = [...(document.normalizations ?? [])];
+    if (removedHeadings > 0 && !normalizations.some((item) => /empty heading/i.test(item))) {
+        normalizations.push(`Removed ${removedHeadings} empty heading${removedHeadings === 1 ? '' : 's'} emitted by source conversion`);
+    }
+    const treeStorage = tree.directives.map((directive) => {
+        const parameters = [
+            directive.depth ? `<ac:parameter ac:name="depth">${directive.depth}</ac:parameter>` : '',
+            directive.sortBy ? `<ac:parameter ac:name="sort">${directive.sortBy}</ac:parameter>` : '',
+            directive.sort ? `<ac:parameter ac:name="reverse">${directive.sort === 'desc' ? 'true' : 'false'}</ac:parameter>` : '',
+        ].join('');
+        return `<ac:structured-macro ac:name="children">${parameters}</ac:structured-macro>`;
+    }).join('');
     const $ = load(htmlFromDocument(document), null, false);
     const gridStorage = $('.ywtc-grid').toArray().map((element) => $(element).html() ?? '').join('');
     return {
-        body: `${converted.storage}${gridStorage}`,
-        warnings: [...document.warnings, ...converted.warnings],
-        degraded: document.degraded || converted.warnings.length > 0,
+        body: `${normalizedStorage('main[data-root]').html() ?? '<p></p>'}${treeStorage}${gridStorage}`,
+        warnings: [...document.warnings, ...conversionWarnings],
+        normalizations,
+        degraded: document.degraded || conversionWarnings.length > 0,
         representedReferences: document.references.length,
     };
 }