@iwonz/ywtc 3.6.0 → 4.0.1

package/dist/content-model.js

@@ -17,6 +17,23 @@ const linkifier = new LinkifyIt();
 function sha256(value) {
     return createHash('sha256').update(value).digest('hex');
 }
+function transformationNotices(input) {
+    const make = (kind, message) => ({
+        code: `${kind}:${sha256(message).slice(0, 12)}`,
+        kind,
+        message,
+        verification: kind === 'normalization'
+            ? 'semantic-content-equivalence'
+            : kind === 'adaptation'
+                ? 'payload-preserved'
+                : 'full-representation-unverified',
+    });
+    return [
+        ...(input.normalizations ?? []).map((message) => make('normalization', message)),
+        ...(input.adaptations ?? []).map((message) => make('adaptation', message)),
+        ...(input.degradations ?? []).map((message) => make('degradation', message)),
+    ];
+}
 function escapeHtml(value) {
     return String(value ?? '').replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;');
 }
@@ -210,7 +227,7 @@ const confluenceLayoutCellCounts = new Map([
     ['three_equal', 3],
     ['three_with_sidebars', 3],
 ]);
-function storageToSemanticHtml(storage, warnings) {
+function storageToSemanticHtml(storage, warnings, adaptations = []) {
     const $ = load(`<main data-ywtc-root>${storage}</main>`, { xmlMode: false }, false);
     $('script, iframe, object, embed, form').each((_index, element) => {
         warnings.push(`Active or embedded element <${element.tagName}> was replaced by a safe fallback`);
@@ -219,7 +236,7 @@ function storageToSemanticHtml(storage, warnings) {
     $('ac\\:layout').toArray().reverse().forEach((element) => {
         const layout = $(element);
         if (layout.parents('ac\\:layout').length > 0)
-            warnings.push('Nested Confluence layout was preserved as a readable layout fallback');
+            adaptations.push('Nested Confluence layout was preserved as readable nested sections');
         const sections = layout.children('ac\\:layout-section').toArray().map((section) => {
             const item = $(section);
             const requestedType = item.attr('ac:type') ?? 'unsupported';
@@ -227,7 +244,7 @@ function storageToSemanticHtml(storage, warnings) {
             const cells = item.children('ac\\:layout-cell').toArray();
             const sectionType = expectedCells === cells.length ? requestedType : 'unsupported';
             if (sectionType === 'unsupported') {
-                warnings.push(`Confluence layout section "${requestedType}" with ${cells.length} cells was preserved as an unsupported-width layout fallback`);
+                adaptations.push(`Confluence layout section "${requestedType}" with ${cells.length} cells was preserved with equivalent cell content and adaptive widths`);
             }
             const body = cells.map((cell) => `<div class="ywtc-layout-cell">${$(cell).html() ?? ''}</div>`).join('');
             return `<section class="ywtc-layout-section ywtc-layout-${escapeHtml(sectionType)} ywtc-layout-columns-${cells.length}" data-ywtc-layout-type="${escapeHtml(sectionType)}">${body}</section>`;
@@ -277,7 +294,11 @@ function storageToSemanticHtml(storage, warnings) {
         const attachment = image.children('ri\\:attachment').attr('ri:filename');
         const external = image.children('ri\\:url').attr('ri:value');
         const source = attachment ? `attachment:${attachment}` : external;
-        image.replaceWith(source ? `<img src="${escapeHtml(source)}" alt="${escapeHtml(image.attr('ac:alt') ?? attachment ?? '')}">` : '<span>[image]</span>');
+        const dimensions = [
+            image.attr('ac:width') ? ` width="${escapeHtml(image.attr('ac:width'))}"` : '',
+            image.attr('ac:height') ? ` height="${escapeHtml(image.attr('ac:height'))}"` : '',
+        ].join('');
+        image.replaceWith(source ? `<img src="${escapeHtml(source)}" alt="${escapeHtml(image.attr('ac:alt') ?? attachment ?? '')}"${dimensions}>` : '<span>[image]</span>');
     });
     $('ac\\:link').each((_index, element) => {
         const link = $(element);
@@ -422,7 +443,7 @@ function yfmToHtml(content, warnings, normalizations = []) {
 }
 function metrics(html) {
     const $ = load(html, null, false);
-    return { links: $('a').length, images: $('img').length, tables: $('table').length, codeBlocks: $('pre').length };
+    return { links: $('a').length, images: $('img').length, tables: $('table').length, codeBlocks: $('pre').length, preservedExternalReferences: 0 };
 }
 function normalizedNodesFromHtml(html, provider) {
     const $ = load(`<main data-ywtc-nodes>${html}</main>`, null, false);
@@ -469,8 +490,9 @@ function semanticallyEmptyHtml(html) {
 export function normalizePageContent(page) {
     const warnings = [];
     const normalizations = [];
+    const adaptations = [];
     const tree = page.rawFormat === 'yfm' ? extractTreeDirectives(page.rawContent) : { content: page.rawContent, directives: [] };
-    const html = page.rawFormat === 'yfm' ? yfmToHtml(tree.content, warnings, normalizations) : storageToSemanticHtml(page.rawContent, warnings);
+    const html = page.rawFormat === 'yfm' ? yfmToHtml(tree.content, warnings, normalizations) : storageToSemanticHtml(page.rawContent, warnings, adaptations);
     const references = extractPageReferences(page.rawContent, page.rawFormat).map((reference, index) => ({
         token: `ref-${index}`,
         kind: reference.kind,
@@ -498,6 +520,8 @@ export function normalizePageContent(page) {
         references,
         warnings,
         normalizations,
+        adaptations,
+        notices: transformationNotices({ normalizations, adaptations, degradations: warnings }),
         degraded: warnings.length > 0,
         semanticallyEmpty: tree.directives.length === 0 && semanticallyEmptyHtml(html),
         metrics: metrics(html),
@@ -533,10 +557,11 @@ export function appendGridSnapshots(document, resources) {
         htmlNode.value = `${htmlNode.value ?? ''}${html}`;
     else
         document.root.children?.push({ type: 'html', value: html });
+    const adaptations = [...document.adaptations, ...grids.map((resource) => `Dynamic table "${resource.name}" was converted to a static table`)];
     return {
         ...document,
-        warnings: [...document.warnings, ...grids.map((resource) => `Dynamic table "${resource.name}" was converted to a static table`)],
-        degraded: true,
+        adaptations,
+        notices: transformationNotices({ normalizations: document.normalizations, adaptations, degradations: document.warnings }),
         semanticallyEmpty: false,
         metrics: { ...document.metrics, tables: document.metrics.tables + grids.length },
     };
@@ -568,7 +593,7 @@ function linearizeLayoutsForMarkdown(html) {
     return { html: $.html(), layouts };
 }
 async function semanticHtmlToYfm(html) {
-    const warnings = new Set();
+    const adaptations = new Set();
     let tokenSequence = 0;
     const convert = async (fragment) => {
         const $ = load(fragment, null, false);
@@ -584,7 +609,7 @@ async function semanticHtmlToYfm(html) {
                     const sectionItem = $(section);
                     const sectionType = sectionItem.attr('data-ywtc-layout-type') ?? 'unsupported';
                     if (['two_left_sidebar', 'two_right_sidebar', 'three_with_sidebars', 'unsupported'].includes(sectionType)) {
-                        warnings.add(`Confluence ${sectionType} layout widths were converted to equal YFM columns`);
+                        adaptations.add(`Confluence ${sectionType} layout widths were converted to equal YFM columns`);
                     }
                     const blocks = [];
                     for (const cell of sectionItem.children('.ywtc-layout-cell').toArray()) {
@@ -640,7 +665,7 @@ async function semanticHtmlToYfm(html) {
             markdown = markdown.replaceAll(token, rendered);
         return markdown.trim();
     };
-    return { body: `${await convert(html)}\n`, warnings: [...warnings] };
+    return { body: `${await convert(html)}\n`, adaptations: [...adaptations] };
 }
 async function sanitizeHtml(html) {
     const parsed = load(html, null, false);
@@ -704,6 +729,19 @@ function markdownFence(value) {
 }
 export function rawContentFallback(page, provider, reason) {
     const warning = `Preferred rendering failed; complete raw source is shown instead: ${reason}`;
+    if (!page.rawContent.trim()) {
+        const normalizations = ['Preserved an intentionally empty source page'];
+        return {
+            body: provider === 'confluence' ? '<p></p>' : '',
+            warnings: [],
+            normalizations,
+            adaptations: [],
+            notices: transformationNotices({ normalizations }),
+            degraded: false,
+            representedReferences: page.references.length,
+            preservedExternalReferences: 0,
+        };
+    }
     if (provider === 'confluence') {
         const raw = page.rawContent
             ? `<pre><code>${escapeHtml(page.rawContent)}</code></pre>`
@@ -712,8 +750,11 @@ export function rawContentFallback(page, provider, reason) {
             body: `<p><strong>Source content fallback</strong> · <a href="${escapeHtml(page.url)}">original page</a></p>${raw}`,
             warnings: [warning],
             normalizations: [],
+            adaptations: [],
+            notices: transformationNotices({ degradations: [warning] }),
             degraded: true,
             representedReferences: page.references.length,
+            preservedExternalReferences: 0,
         };
     }
     const raw = page.rawContent
@@ -723,8 +764,11 @@ export function rawContentFallback(page, provider, reason) {
         body: `**Source content fallback** · [original page](${page.url})\n\n${raw}\n`,
         warnings: [warning],
         normalizations: [],
+        adaptations: [],
+        notices: transformationNotices({ degradations: [warning] }),
         degraded: true,
         representedReferences: page.references.length,
+        preservedExternalReferences: 0,
     };
 }
 export function emptyRendererFallbackReason(document) {
@@ -790,6 +834,12 @@ function semanticContent(body, provider) {
         links: $('a[href]').toArray().map((element) => `${$(element).attr('href') ?? ''}|${semanticText($(element).text())}`).sort(),
         images: $('img').toArray().map((element) => $(element).attr('src') ?? '').sort(),
         imageLabels: $('img').toArray().map((element) => $(element).attr('alt') ?? '').sort(),
+        imageAttributes: $('img').toArray().map((element) => [
+            $(element).attr('src') ?? '',
+            $(element).attr('alt') ?? '',
+            $(element).attr('width') ?? '',
+            $(element).attr('height') ?? '',
+        ].join('|')).sort(),
         anchors: $('[id],a[name]').toArray().map((element) => $(element).attr('id') ?? $(element).attr('name') ?? '').filter(Boolean).sort(),
         structure: {
             lists: $('ul,ol').length,
@@ -810,6 +860,7 @@ function semanticContentIsEmpty(content) {
         && content.code.length === 0
         && content.links.length === 0
         && content.images.length === 0
+        && content.imageAttributes.length === 0
         && content.anchors.length === 0
         && content.macros.length === 0
         && content.layouts.length === 0
@@ -820,20 +871,20 @@ export function semanticContentFingerprint(body, provider) {
 }
 export function verifyDestinationContent(expectedBody, actualBody, provider, page) {
     if (actualBody === undefined)
-        return { status: 'failed', differences: [`Verification could not read destination content for ${page.title}`] };
+        return { status: 'failed', differences: [`Verification could not read destination content for ${page.title}`], adaptations: [] };
     if (isLegacyTransferPlaceholder(actualBody, provider, page)) {
-        return { status: 'failed', differences: [`Verification found a transfer placeholder instead of source content for ${page.title}`] };
+        return { status: 'failed', differences: [`Verification found a transfer placeholder instead of source content for ${page.title}`], adaptations: [] };
     }
     const expected = semanticContent(expectedBody, provider);
     const actual = semanticContent(actualBody, provider);
     const expectedEmpty = semanticContentIsEmpty(expected);
     const actualEmpty = semanticContentIsEmpty(actual);
     if (expectedEmpty && actualEmpty)
-        return { status: 'complete', differences: [] };
+        return { status: 'complete', differences: [], adaptations: [] };
     if (!expectedEmpty && actualEmpty)
-        return { status: 'failed', differences: [`Verification found empty destination content for ${page.title}`] };
+        return { status: 'failed', differences: [`Verification found empty destination content for ${page.title}`], adaptations: [] };
     if (expectedEmpty && !actualEmpty) {
-        return { status: 'failed', differences: [`Verification found unexpected destination content for semantically empty source page ${page.title}`] };
+        return { status: 'failed', differences: [`Verification found unexpected destination content for semantically empty source page ${page.title}`], adaptations: [] };
     }
     const differs = (key) => JSON.stringify(expected[key]) !== JSON.stringify(actual[key]);
     const missingData = ['text', 'tableCells', 'code', 'links', 'images', 'anchors', 'macros'].filter(differs);
@@ -841,16 +892,20 @@ export function verifyDestinationContent(expectedBody, actualBody, provider, pag
         return {
             status: 'failed',
             differences: [`Verification found destination content that differs from the rendered source for ${page.title}: ${missingData.join(', ')}`],
+            adaptations: [],
         };
     }
-    const simplifiedFormatting = ['headings', 'listItems', 'imageLabels', 'structure', 'layouts'].filter(differs);
-    if (simplifiedFormatting.length > 0) {
+    const missingStructure = ['headings', 'listItems', 'imageLabels', 'imageAttributes', 'structure'].filter(differs);
+    if (missingStructure.length > 0) {
         return {
             status: 'partial',
-            differences: [`Destination formatting was simplified for ${page.title}: ${simplifiedFormatting.join(', ')}`],
+            differences: [`Destination lost or could not verify meaningful structure for ${page.title}: ${missingStructure.join(', ')}`],
+            adaptations: [],
         };
     }
-    return { status: 'complete', differences: [] };
+    if (differs('layouts'))
+        return { status: 'complete', differences: [], adaptations: [`Destination adapted visual layout geometry for ${page.title} while preserving its payload`] };
+    return { status: 'complete', differences: [], adaptations: [] };
 }
 export function assertDestinationContent(expectedBody, actualBody, provider, page) {
     const result = verifyDestinationContent(expectedBody, actualBody, provider, page);
@@ -904,6 +959,13 @@ function rewritePortableHtml(html, context) {
     });
     return $.html();
 }
+function preservedExternalImageCount(document, context) {
+    const $ = load(htmlFromDocument(document), null, false);
+    return $('img[src]').toArray().filter((element) => {
+        const source = $(element).attr('src') ?? '';
+        return /^https?:\/\//i.test(source) && !resourceBinding(source, context);
+    }).length;
+}
 function treeDirectiveNodes(document) {
     return document.root.children?.filter((node) => node.type === 'tree-directive') ?? [];
 }
@@ -944,19 +1006,20 @@ function renderTreeHtml(context, directive) {
 }
 export async function renderMarkdown(document, context) {
     if (document.semanticallyEmpty) {
-        return { body: '', warnings: [...document.warnings], normalizations: [...document.normalizations], degraded: document.degraded, representedReferences: document.references.length };
+        return { body: '', warnings: [...document.warnings], normalizations: [...document.normalizations], adaptations: [...document.adaptations], notices: [...document.notices], degraded: document.degraded, representedReferences: document.references.length, preservedExternalReferences: 0 };
     }
     const portable = rewritePortableHtml(htmlFromDocument(document), context);
     const linearized = linearizeLayoutsForMarkdown(portable);
     const tree = treeDirectiveNodes(document).map((node) => renderTreeMarkdown(context, node.tree ?? {})).filter(Boolean).join('\n\n');
     const content = await htmlToMarkdown(linearized.html);
     const body = tree ? `${content.trimEnd()}\n\n${tree}\n` : content;
-    const layoutWarnings = linearized.layouts > 0 ? [`${linearized.layouts} column layout${linearized.layouts === 1 ? ' was' : 's were'} linearized for portable Markdown`] : [];
-    return { body, warnings: [...document.warnings, ...layoutWarnings], normalizations: [...(document.normalizations ?? [])], degraded: document.degraded || layoutWarnings.length > 0, representedReferences: document.references.length };
+    const layoutAdaptations = linearized.layouts > 0 ? [`${linearized.layouts} column layout${linearized.layouts === 1 ? ' was' : 's were'} linearized for portable Markdown`] : [];
+    const adaptations = [...document.adaptations, ...layoutAdaptations];
+    return { body, warnings: [...document.warnings], normalizations: [...document.normalizations], adaptations, notices: transformationNotices({ normalizations: document.normalizations, adaptations, degradations: document.warnings }), degraded: document.degraded, representedReferences: document.references.length, preservedExternalReferences: preservedExternalImageCount(document, context) };
 }
 export async function renderStaticHtml(document, context) {
     if (document.semanticallyEmpty) {
-        return { body: '', warnings: [...document.warnings], normalizations: [...document.normalizations], degraded: document.degraded, representedReferences: document.references.length };
+        return { body: '', warnings: [...document.warnings], normalizations: [...document.normalizations], adaptations: [...document.adaptations], notices: [...document.notices], degraded: document.degraded, representedReferences: document.references.length, preservedExternalReferences: 0 };
     }
     const rewritten = rewritePortableHtml(htmlFromDocument(document), context);
     const content = await sanitizeHtml(rewritten);
@@ -970,12 +1033,13 @@ export async function renderStaticHtml(document, context) {
             visit(child);
     };
     visit(document.root);
-    const layoutWarnings = unsupportedLayouts.map((cells) => `Unsupported ${cells}-cell layout was stacked in static HTML without losing cell content`);
-    return { body, warnings: [...document.warnings, ...layoutWarnings], normalizations: [...(document.normalizations ?? [])], degraded: document.degraded || layoutWarnings.length > 0, representedReferences: document.references.length };
+    const layoutAdaptations = unsupportedLayouts.map((cells) => `Unsupported ${cells}-cell layout was stacked in static HTML without losing cell content`);
+    const adaptations = [...document.adaptations, ...layoutAdaptations];
+    return { body, warnings: [...document.warnings], normalizations: [...document.normalizations], adaptations, notices: transformationNotices({ normalizations: document.normalizations, adaptations, degradations: document.warnings }), degraded: document.degraded, representedReferences: document.references.length, preservedExternalReferences: preservedExternalImageCount(document, context) };
 }
 export async function renderYfm(document, context) {
     if (document.semanticallyEmpty) {
-        return { body: '', warnings: [...document.warnings], normalizations: [...document.normalizations], degraded: document.degraded, representedReferences: document.references.length };
+        return { body: '', warnings: [...document.warnings], normalizations: [...document.normalizations], adaptations: [...document.adaptations], notices: [...document.notices], degraded: document.degraded, representedReferences: document.references.length, preservedExternalReferences: 0 };
     }
     if (document.rawFormat === 'yfm') {
         let body = document.raw;
@@ -1015,28 +1079,39 @@ export async function renderYfm(document, context) {
             body = `${body.trimEnd()}\n\n${(await htmlToMarkdown(gridHtml)).trimStart()}`;
         return {
             body,
-            warnings: [...document.warnings, ...resourceWarnings],
-            normalizations: [...(document.normalizations ?? [])],
-            degraded: document.degraded || resourceWarnings.size > 0,
+            warnings: [...resourceWarnings],
+            normalizations: [...document.normalizations],
+            adaptations: [...document.adaptations],
+            notices: transformationNotices({ normalizations: document.normalizations, adaptations: document.adaptations, degradations: [...resourceWarnings] }),
+            degraded: resourceWarnings.size > 0,
             representedReferences: document.references.length,
+            preservedExternalReferences: preservedExternalImageCount(document, context),
         };
     }
     const converted = await semanticHtmlToYfm(rewritePortableHtml(htmlFromDocument(document), context));
     return {
         body: converted.body,
-        warnings: [...document.warnings, ...converted.warnings],
-        normalizations: [...(document.normalizations ?? []), 'Converted supported Confluence Storage Format constructs to native YFM equivalents'],
-        degraded: document.degraded || converted.warnings.length > 0,
+        warnings: [...document.warnings],
+        normalizations: [...document.normalizations, 'Converted supported Confluence Storage Format constructs to native YFM equivalents'],
+        adaptations: [...document.adaptations, ...converted.adaptations],
+        notices: transformationNotices({ normalizations: [...document.normalizations, 'Converted supported Confluence Storage Format constructs to native YFM equivalents'], adaptations: [...document.adaptations, ...converted.adaptations], degradations: document.warnings }),
+        degraded: document.degraded,
         representedReferences: document.references.length,
+        preservedExternalReferences: preservedExternalImageCount(document, context),
     };
 }
 export async function renderConfluenceStorage(document, page, context, spaceKey) {
     if (document.semanticallyEmpty) {
-        return { body: '<p></p>', warnings: [...document.warnings], normalizations: [...document.normalizations], degraded: document.degraded, representedReferences: document.references.length };
+        return { body: '<p></p>', warnings: [...document.warnings], normalizations: [...document.normalizations], adaptations: [...document.adaptations], notices: [...document.notices], degraded: document.degraded, representedReferences: document.references.length, preservedExternalReferences: 0 };
     }
     if (document.rawFormat === 'confluence-storage') {
         const $ = load(`<main data-root>${document.raw}</main>`, { xmlMode: true }, false);
-        const warnings = [...document.warnings];
+        const warnings = [];
+        $('script, iframe, object, embed, form').each((_index, element) => {
+            const raw = $.html(element);
+            warnings.push(`Active or embedded element <${element.tagName}> was preserved as inert source text`);
+            $(element).replaceWith(`<pre><code>${escapeHtml(raw)}</code></pre>`);
+        });
         $('ac\\:image').each((_index, element) => {
             const image = $(element);
             const attachment = image.children('ri\\:attachment');
@@ -1111,9 +1186,12 @@ export async function renderConfluenceStorage(document, page, context, spaceKey)
         return {
             body: $('main[data-root]').html() ?? '<p></p>',
             warnings,
-            normalizations: [...(document.normalizations ?? [])],
-            degraded: document.degraded || warnings.length > document.warnings.length,
+            normalizations: [...document.normalizations],
+            adaptations: [...document.adaptations],
+            notices: transformationNotices({ normalizations: document.normalizations, adaptations: document.adaptations, degradations: warnings }),
+            degraded: warnings.length > 0,
             representedReferences: document.references.length,
+            preservedExternalReferences: preservedExternalImageCount(document, context),
         };
     }
     const pagesBySlug = new Map();
@@ -1183,8 +1261,11 @@ export async function renderConfluenceStorage(document, page, context, spaceKey)
         body: `${normalizedStorage('main[data-root]').html() ?? '<p></p>'}${treeStorage}${gridStorage}`,
         warnings: [...document.warnings, ...conversionWarnings],
         normalizations,
+        adaptations: [...document.adaptations, ...converted.adaptations],
+        notices: transformationNotices({ normalizations, adaptations: [...document.adaptations, ...converted.adaptations], degradations: [...document.warnings, ...conversionWarnings] }),
         degraded: document.degraded || conversionWarnings.length > 0,
         representedReferences: document.references.length,
+        preservedExternalReferences: converted.stats.preservedExternalReferences,
     };
 }
 export function resolveYandexReferenceSlug(page, href) {