@adobe/helix-importer 3.0.0 → 3.1.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [3.1.0](https://github.com/adobe/helix-importer/compare/v3.0.0...v3.1.0) (2023-11-13)
+
+
+### Features
+
+* improve default import ([#261](https://github.com/adobe/helix-importer/issues/261)) ([251cfcd](https://github.com/adobe/helix-importer/commit/251cfcdde7ac54525eef49341a9ede52d368cf71))
+
 # [3.0.0](https://github.com/adobe/helix-importer/compare/v2.9.41...v3.0.0) (2023-11-13)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-importer",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Helix Importer tool: create md / docx from html",
   "main": "src/index.js",
   "type": "module",

package/src/importer/HTML2x.js

@@ -17,6 +17,8 @@ import PageImporterResource from './PageImporterResource.js';
 import MemoryHandler from '../storage/MemoryHandler.js';
 import Utils from '../utils/Utils.js';
 import BrowserUtils from '../utils/BrowserUtils.js';
+import defaultTransformDOM from './defaults/transformDOM.js';
+import defaultGenerateDocumentPath from './defaults/generateDocumentPath.js';
 
 // import docxStylesXML from '../resources/styles.xml';
 
@@ -35,27 +37,6 @@ function setBackgroundImagesFromCSS(document) {
   }
 }
 
-async function defaultTransformDOM({
-  // eslint-disable-next-line no-unused-vars
-  url, document, html, params,
-}) {
-  return document.body;
-}
-
-async function defaultGenerateDocumentPath({
-  // eslint-disable-next-line no-unused-vars
-  url, document, html, params,
-}) {
-  let p = new URL(url).pathname;
-  if (p.endsWith('/')) {
-    p = `${p}index`;
-  }
-  return decodeURIComponent(p)
-    .toLowerCase()
-    .replace(/\.html$/, '')
-    .replace(/[^a-z0-9/]/gm, '-');
-}
-
 async function html2x(
   url,
   doc,

package/src/importer/defaults/generateDocumentPath.js

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+export default async function generateDocumentPath({
+  // eslint-disable-next-line no-unused-vars
+  url, document, html, params,
+}) {
+  let p = new URL(url).pathname;
+  if (p.endsWith('/')) {
+    p = `${p}index`;
+  }
+  return decodeURIComponent(p)
+    .toLowerCase()
+    .replace(/\.html$/, '')
+    .replace(/[^a-z0-9/]/gm, '-');
+}

package/src/importer/defaults/rules/adjustImageUrls.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default function adjustImageUrls(main, url) {
+  [...main.querySelectorAll('img')].forEach((img) => {
+    const src = img.getAttribute('src');
+    if (src && (src.startsWith('./') || src.startsWith('/') || src.startsWith('../'))) {
+      try {
+        const u = new URL(src, url);
+        // eslint-disable-next-line no-param-reassign
+        img.src = u.toString();
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.log(`Unable to adjust image URL ${img.src} - removing image`);
+        img.remove();
+      }
+    }
+  });
+}

package/src/importer/defaults/rules/convertIcons.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default function convertIcons(main, document) {
+  [...main.querySelectorAll('img')].forEach((img) => {
+    const src = img.getAttribute('src');
+    if (src && src.endsWith('.svg')) {
+      const span = document.createElement('span');
+      const name = src.split('/').pop().split('.')[0].toLowerCase().trim().replace(/[^a-z0-9]/g, '-');
+      if (name) {
+        span.innerHTML = `:${name}:`;
+        img.replaceWith(span);
+      }
+    }
+  });
+}

package/src/importer/defaults/rules/createMetadata.js

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import Blocks from '../../../utils/Blocks.js';
+
+function getMetadata(name, document) {
+  const attr = name && name.includes(':') ? 'property' : 'name';
+  const meta = [...document.head.querySelectorAll(`meta[${attr}="${name}"]`)]
+    .map((m) => m.content)
+    .join(', ');
+  return meta || '';
+}
+
+export default function createMetadata(main, document) {
+  const meta = {};
+
+  const title = document.querySelector('title');
+  if (title) {
+    meta.Title = title.textContent.replace(/[\n\t]/gm, '');
+  }
+
+  const desc = getMetadata('description', document);
+  if (desc) {
+    meta.Description = desc;
+  }
+
+  const img = getMetadata('og:image', document);
+  if (img) {
+    const el = document.createElement('img');
+    el.src = img;
+    meta.Image = el;
+
+    const imgAlt = getMetadata('og:image:alt', document);
+    if (imgAlt) {
+      el.alt = imgAlt;
+    }
+  }
+
+  const ogtitle = getMetadata('og:title', document);
+  if (ogtitle && ogtitle !== meta.Title) {
+    if (meta.Title) {
+      meta['og:title'] = ogtitle;
+    } else {
+      meta.Title = ogtitle;
+    }
+  }
+
+  const ogdesc = getMetadata('og:description', document);
+  if (ogdesc && ogdesc !== meta.Description) {
+    if (meta.Description) {
+      meta['og:description'] = ogdesc;
+    } else {
+      meta.Description = ogdesc;
+    }
+  }
+
+  const ttitle = getMetadata('twitter:title', document);
+  if (ttitle && ttitle !== meta.Title) {
+    if (meta.Title) {
+      meta['twitter:title'] = ttitle;
+    } else {
+      meta.Title = ttitle;
+    }
+  }
+
+  const tdesc = getMetadata('twitter:description', document);
+  if (tdesc && tdesc !== meta.Description) {
+    if (meta.Description) {
+      meta['twitter:description'] = tdesc;
+    } else {
+      meta.Description = tdesc;
+    }
+  }
+
+  const timg = getMetadata('twitter:image', document);
+  if (timg && timg !== img) {
+    const el = document.createElement('img');
+    el.src = timg;
+    meta['twitter:image'] = el;
+
+    const imgAlt = getMetadata('twitter:image:alt', document);
+    if (imgAlt) {
+      el.alt = imgAlt;
+    }
+  }
+
+  if (Object.keys(meta).length > 0) {
+    const block = Blocks.getMetadataBlock(document, meta);
+    main.append(block);
+  }
+
+  return meta;
+}

package/src/importer/defaults/rules/transformBackgroundImages.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import DOMUtils from '../../../utils/DOMUtils.js';
+
+export default function transformBackgroundImages(main, document) {
+  [...main.querySelectorAll('[style*="background-image: url"]')].forEach((element) => {
+    const img = DOMUtils.getImgFromBackground(element, document);
+    element.prepend(img);
+    element.style.removeProperty('background-image');
+  });
+}

package/src/importer/defaults/transformDOM.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import DOMUtils from '../../utils/DOMUtils.js';
+import createMetadata from './rules/createMetadata.js';
+import adjustImageUrls from './rules/adjustImageUrls.js';
+import convertIcons from './rules/convertIcons.js';
+import transformBackgroundImages from './rules/transformBackgroundImages.js';
+
+export default async function transformDOM({
+  // eslint-disable-next-line no-unused-vars
+  url, document, html, params,
+}) {
+  const main = document.body;
+
+  // attempt to remove non-content elements
+  DOMUtils.remove(main, [
+    'header',
+    '.header',
+    'nav',
+    '.nav',
+    'footer',
+    '.footer',
+    'iframe',
+    'noscript',
+  ]);
+
+  createMetadata(main, document);
+  transformBackgroundImages(main, document);
+  adjustImageUrls(main, url);
+  convertIcons(main, document);
+
+  return main;
+}

package/src/index.js

@@ -27,6 +27,18 @@ import WPUtils from './wp/WPUtils.js';
 
 import { html2md, html2docx } from './importer/HTML2x.js';
 
+import createMetadata from './importer/defaults/rules/createMetadata.js';
+import adjustImageUrls from './importer/defaults/rules/adjustImageUrls.js';
+import convertIcons from './importer/defaults/rules/convertIcons.js';
+import transformBackgroundImages from './importer/defaults/rules/transformBackgroundImages.js';
+
+const rules = {
+  createMetadata,
+  adjustImageUrls,
+  convertIcons,
+  transformBackgroundImages,
+};
+
 export {
   PageImporter,
   PageImporterParams,
@@ -42,4 +54,5 @@ export {
   WPUtils,
   html2md,
   html2docx,
+  rules,
 };

package/src/utils/DOMUtils.js

@@ -258,7 +258,9 @@ export default class DOMUtils {
     const styleAttr = element?.getAttribute('style')?.split(';');
     if (styleAttr) {
       styleAttr.forEach((style) => {
-        const [prop, value] = style.split(':');
+        const split = style.split(':');
+        const prop = split.shift();
+        const value = split.join(':').trim();
         if (prop === 'background-image') {
           const trimmedValue = value.replace(/\s/g, '');
           const elStyle = element.style;
@@ -267,7 +269,7 @@ export default class DOMUtils {
       });
       const url = element.style.backgroundImage;
       if (url && url.toLowerCase() !== 'none') {
-        const src = url.replace(/url\(/gm, '').replace(/'/gm, '').replace(/\)/gm, '');
+        const src = url.replace(/url\(/gm, '').replace(/'/gm, '').replace(/"/gm, '').replace(/\)/gm, '');
         const img = document.createElement('img');
         img.src = src;
         return img;

package/test/TestUtils.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { JSDOM } from 'jsdom';
+
+export default class TestUtils {
+  // test environment createDocumentFromString version using JSDOM
+  static createDocumentFromString(html) {
+    const { document } = new JSDOM(html, { runScripts: undefined }).window;
+    return document;
+  }
+}

package/test/browser/DOMUtils.test.js

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-env mocha */
+/* eslint-disable no-unused-expressions */
+
+import { expect } from '@esm-bundle/chai';
+import BrowserUtils from '../../src/utils/BrowserUtils.js';
+import DOMUtils from '../../src/utils/DOMUtils.js';
+
+const createElement = (document, tag, attrs, styles, innerHTML) => {
+  const element = document.createElement(tag);
+  // eslint-disable-next-line no-restricted-syntax, guard-for-in
+  for (const a in attrs) {
+    element.setAttribute(a, attrs[a]);
+  }
+  // eslint-disable-next-line no-restricted-syntax, guard-for-in
+  for (const p in styles) {
+    element.style[p] = styles[p];
+  }
+  element.innerHTML = innerHTML;
+  return element;
+};
+
+describe('DOMUtils#element', () => {
+  const test = (tag, attrs, styles, innerHTML, expected) => {
+    const document = BrowserUtils.createDocumentFromString('<html><body></body></html>');
+    const element = createElement(document, tag, attrs, styles, innerHTML);
+    const ret = DOMUtils.getImgFromBackground(element, document);
+    if (expected) {
+      expect(ret).to.not.be.null;
+      expect(ret.outerHTML).to.equal(expected);
+    } else {
+      expect(ret).to.be.null;
+    }
+  };
+
+  it('no background-image style', () => {
+    test('p', {}, {}, 'Some content', null);
+    test('img', { src: 'https://www.server.com/image.jpg', title: 'Some title' }, {}, '', null);
+    test('p', {}, { 'background-image': 'none' }, 'Some content', null);
+  });
+
+  it('with background-image style', () => {
+    test('p', {}, { 'background-image': 'url(https://www.server.com/image.jpg)' }, 'Some content', '<img src="https://www.server.com/image.jpg">');
+    test('p', {}, { 'background-image': 'url("https://www.server.com/image.jpg")' }, 'Some content', '<img src="https://www.server.com/image.jpg">');
+    test('p', {}, { 'background-image': 'url(\'https://www.server.com/image.jpg\')' }, 'Some content', '<img src="https://www.server.com/image.jpg">');
+    test('p', {}, { 'background-image': 'url(http://localhost:3001/image.jpg)' }, 'Some content', '<img src="http://localhost:3001/image.jpg">');
+  });
+
+  // `createElement` uses JSDOM to create the test-DOM
+  // the workaround in DOMUtils#getImgFromBackground exists _precisely_
+  // because of a potential bug in JSDOM due to which it doesn't
+  // parse `url()` with whitespaces correctly
+  // browser specific version of the test
+  it('with background-image style containing whitespace in url()', () => {
+    test('p', {}, { 'background-image': 'url( /image.jpg )' }, 'Some content', '<img src="/image.jpg">');
+  });
+});

package/test/importers/HTML2x.spec.js

@@ -14,7 +14,6 @@ import {
   deepStrictEqual, ok, strictEqual, fail,
 } from 'assert';
 import { describe, it } from 'mocha';
-import { JSDOM } from 'jsdom';
 import { docx2md } from '@adobe/helix-docx2md';
 import MockMediaHandler from '../mocks/MockMediaHandler.js';
 
@@ -22,39 +21,11 @@ import DOMUtils from '../../src/utils/DOMUtils.js';
 import {
   html2md,
   html2docx,
-  defaultGenerateDocumentPath,
-  defaultTransformDOM,
 } from '../../src/importer/HTML2x.js';
 
-// test environment createDocumentFromString version using JSDOM
-const createDocumentFromString = (html) => {
-  const { document } = new JSDOM(html, { runScripts: undefined }).window;
-  return document;
-};
-
-describe('defaultTransformDOM tests', () => {
-  it('default transformation', async () => {
-    const document = createDocumentFromString('<html><body><h1>Hello World</h1></body></html>');
-    const out = await defaultTransformDOM({ document });
-    strictEqual(out.outerHTML, '<body><h1>Hello World</h1></body>');
-  });
-});
+import TestUtils from '../TestUtils.js';
 
-describe('defaultGenerateDocumentPath tests', () => {
-  it('default paths', async () => {
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com' }), '/index');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/' }), '/index');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/index.html' }), '/index');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/index' }), '/index');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/page' }), '/page');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/page.html' }), '/page');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page' }), '/folder/page');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page.html' }), '/folder/page');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page/' }), '/folder/page/index');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page with spaces.html' }), '/folder/page-with-spaces');
-    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/PagE_with_3xtr4_charactére.html' }), '/folder/page-with-3xtr4-charact-re');
-  });
-});
+const { createDocumentFromString } = TestUtils;
 
 describe('html2x parameters', () => {
   const URL = 'https://www.sample.com/page.html';
@@ -408,8 +379,8 @@ describe('html2md tests', () => {
     const out = await html2md('https://www.sample.com/page.html', '<html><body><img src="data:abc" data-src="./image.jpg"></body></html>', null, {
       createDocumentFromString,
     });
-    strictEqual(out.html.trim(), '<body><img src="./image.jpg" data-src="./image.jpg"></body>');
-    strictEqual(out.md.trim(), '![][image0]\n\n[image0]: ./image.jpg');
+    strictEqual(out.html.trim(), '<body><img src="https://www.sample.com/image.jpg" data-src="./image.jpg"></body>');
+    strictEqual(out.md.trim(), '![][image0]\n\n[image0]: https://www.sample.com/image.jpg');
   });
 
   it('html2md allows to preprocess the document', async () => {
@@ -422,8 +393,8 @@ describe('html2md tests', () => {
     }, {
       createDocumentFromString,
     });
-    strictEqual(out.html.trim(), '<body><img src="./image.jpg"></body>');
-    strictEqual(out.md.trim(), '![][image0]\n\n[image0]: ./image.jpg');
+    strictEqual(out.html.trim(), '<body><img src="https://www.sample.com/image.jpg"></body>');
+    strictEqual(out.md.trim(), '![][image0]\n\n[image0]: https://www.sample.com/image.jpg');
   });
 
   it('html2md removes original hrs but keeps md section breaks', async () => {

package/test/importers/PageImporter.spec.js

@@ -237,4 +237,8 @@ describe('PageImporter tests - fixtures', () => {
   it('import - sub and sup', async () => {
     await featureTest('subsup');
   });
+
+  it('import - video', async () => {
+    await featureTest('video');
+  });
 });

package/test/importers/defaults/fixtures/adjust-image-urls.expected.html

@@ -0,0 +1,7 @@
+<body>
+  <h1>Hello World</h1>
+  <img src="https://wwww.sample.com/image1.png">
+  <img src="https://wwww.sample.com/path/image2.png">
+  <img src="https://wwww.sample.com/image3.png">
+  <img src="https://wwww.anotherhost.com/image4.png">
+</body>

package/test/importers/defaults/fixtures/adjust-image-urls.input.html

@@ -0,0 +1,10 @@
+<html>
+  <body>
+    <h1>Hello World</h1>
+    <img src="/image1.png">
+    <img src="./image2.png">
+    <img src="https://wwww.sample.com/image3.png">
+    <img src="https://wwww.anotherhost.com/image4.png">
+    <img src="/\/: #brokenlink#?!$$">
+  </body>
+</html>

package/test/importers/defaults/fixtures/background-image.expected.html

@@ -0,0 +1,13 @@
+<body>
+  <h1>Hello World</h1>
+  <div style="">
+    <img src="https://wwww.sample.com/image1.png">
+    <span>some content here</span>
+  </div>
+  <div style="">
+    <img src="https://wwww.sample.com/path/image2.png">
+  </div>
+  <div style="width: 100%;">
+    <img src="https://www.anotherhost.com/image3.png">
+  </div>
+</body>

package/test/importers/defaults/fixtures/background-image.input.html

@@ -0,0 +1,10 @@
+<html>
+  <body>
+    <h1>Hello World</h1>
+    <div style="background-image: url('/image1.png')">
+      <span>some content here</span>
+    </div>
+    <div style='background-image: url("./image2.png")'></div>
+    <div style="width: 100%; background-image: url('https://www.anotherhost.com/image3.png')"></div>
+  </body>
+</html>

package/test/importers/defaults/fixtures/cleanup.expected.html

@@ -0,0 +1,5 @@
+<body>
+  <main>
+    <h1>Hello World</h1>
+  </main>
+</body>

package/test/importers/defaults/fixtures/cleanup.input.html

@@ -0,0 +1,11 @@
+<html>
+  <body>
+    <header>Top header</header>
+    <nav>Nav might be here</nav>
+    <main>
+      <noscript>Some no script here</noscript>
+      <h1>Hello World</h1><iframe src="iframe.html"></iframe>
+    </main>
+    <footer>Bottom footer</footer>
+  </body>
+</html>

package/test/importers/defaults/fixtures/default.expected.html

@@ -0,0 +1,4 @@
+<body>
+  <h1>Hello World</h1>
+  <p>Some text with a <span>span</span>, a link <a href="./anotherpage.html">anotherpage</a> and a <sub>sub</sub>.</p>
+</body>

package/test/importers/defaults/fixtures/default.input.html

@@ -0,0 +1,6 @@
+<html>
+  <body>
+    <h1>Hello World</h1>
+    <p>Some text with a <span>span</span>, a link <a href="./anotherpage.html">anotherpage</a> and a <sub>sub</sub>.</p>
+  </body>
+</html>

package/test/importers/defaults/fixtures/icons.expected.html

@@ -0,0 +1,4 @@
+<body>
+  <h1>Hello World</h1><span>:icon1:</span>
+  <p>This is text with an icon <span>:icon2:</span></p>
+</body>

package/test/importers/defaults/fixtures/icons.input.html

@@ -0,0 +1,6 @@
+<html>
+  <body>
+    <h1>Hello World</h1><img src="/icon1.svg">
+    <p>This is text with an icon <img src="./icon2.svg"></p>
+  </body>
+</html>

package/test/importers/defaults/fixtures/metadata.all.diff.expected.html

@@ -0,0 +1,40 @@
+<body>
+  <h1>Hello World</h1>
+  <table>
+    <tr>
+      <th colspan="2">Metadata</th>
+    </tr>
+    <tr>
+      <td>Title</td>
+      <td>Page title - tite element</td>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>Page description - description meta</td>
+    </tr>
+    <tr>
+      <td>Image</td>
+      <td><img src="https://wwww.sample.com/path/img-og.png" alt="This is the image alt text - og:image:alt meta"></td>
+    </tr>
+    <tr>
+      <td>og:title</td>
+      <td>Page title - og:title meta</td>
+    </tr>
+    <tr>
+      <td>og:description</td>
+      <td>Page description - og:description meta</td>
+    </tr>
+    <tr>
+      <td>twitter:title</td>
+      <td>Page title - twitter:title meta</td>
+    </tr>
+    <tr>
+      <td>twitter:description</td>
+      <td>Page description - twitter:description meta</td>
+    </tr>
+    <tr>
+      <td>twitter:image</td>
+      <td><img src="https://wwww.sample.com/path/img-twitter.png" alt="This is the image alt text - twitter:image:alt meta"></td>
+    </tr>
+  </table>
+</body>

package/test/importers/defaults/fixtures/metadata.all.diff.input.html

@@ -0,0 +1,17 @@
+<html>
+  <head>
+    <title>Page title - tite element</title>
+    <meta name="description" content="Page description - description meta">
+    <meta property="og:title" content="Page title - og:title meta">
+    <meta property="og:description" content="Page description - og:description meta">
+    <meta property="og:image" content="./img-og.png">
+    <meta property="og:image:alt" content="This is the image alt text - og:image:alt meta">
+    <meta property="twitter:title" content="Page title - twitter:title meta">
+    <meta property="twitter:description" content="Page description - twitter:description meta">
+    <meta property="twitter:image" content="./img-twitter.png">
+    <meta property="twitter:image:alt" content="This is the image alt text - twitter:image:alt meta">
+  </head>
+  <body>
+    <h1>Hello World</h1>
+  </body>
+</html>

package/test/importers/defaults/fixtures/metadata.all.same.expected.html

@@ -0,0 +1,20 @@
+<body>
+  <h1>Hello World</h1>
+  <table>
+    <tr>
+      <th colspan="2">Metadata</th>
+    </tr>
+    <tr>
+      <td>Title</td>
+      <td>Page title</td>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>Page description</td>
+    </tr>
+    <tr>
+      <td>Image</td>
+      <td><img src="https://wwww.sample.com/path/img.png" alt="This is the image alt text"></td>
+    </tr>
+  </table>
+</body>

package/test/importers/defaults/fixtures/metadata.all.same.input.html

@@ -0,0 +1,17 @@
+<html>
+  <head>
+    <title>Page title</title>
+    <meta name="description" content="Page description">
+    <meta property="og:title" content="Page title">
+    <meta property="og:description" content="Page description">
+    <meta property="og:image" content="./img.png">
+    <meta property="og:image:alt" content="This is the image alt text">
+    <meta property="twitter:title" content="Page title">
+    <meta property="twitter:description" content="Page description">
+    <meta property="twitter:image" content="./img.png">
+    <meta property="twitter:image:alt" content="This is the image alt text">
+  </head>
+  <body>
+    <h1>Hello World</h1>
+  </body>
+</html>

package/test/importers/defaults/fixtures/metadata.basic.expected.html

@@ -0,0 +1,16 @@
+<body>
+  <h1>Hello World</h1>
+  <table>
+    <tr>
+      <th colspan="2">Metadata</th>
+    </tr>
+    <tr>
+      <td>Title</td>
+      <td>Page title</td>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>Page description</td>
+    </tr>
+  </table>
+</body>

package/test/importers/defaults/fixtures/metadata.basic.input.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <title>Page title</title>
+    <meta name="description" content="Page description">
+  </head>
+  <body>
+    <h1>Hello World</h1>
+  </body>
+</html

package/test/importers/defaults/fixtures/metadata.image.expected.html

@@ -0,0 +1,12 @@
+<body>
+  <h1>Hello World</h1>
+  <table>
+    <tr>
+      <th colspan="2">Metadata</th>
+    </tr>
+    <tr>
+      <td>Image</td>
+      <td><img src="https://wwww.sample.com/img.png" alt="This is the image alt text"></td>
+    </tr>
+  </table>
+</body>

package/test/importers/defaults/fixtures/metadata.image.input.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <meta property="og:image" content="/img.png">
+    <meta property="og:image:alt" content="This is the image alt text">
+  </head>
+  <body>
+    <h1>Hello World</h1>
+  </body>
+</html>

package/test/importers/defaults/fixtures/metadata.og.expected.html

@@ -0,0 +1,16 @@
+<body>
+  <h1>Hello World</h1>
+  <table>
+    <tr>
+      <th colspan="2">Metadata</th>
+    </tr>
+    <tr>
+      <td>Title</td>
+      <td>Page title - og:title meta</td>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>Page description - og:description meta</td>
+    </tr>
+  </table>
+</body>

package/test/importers/defaults/fixtures/metadata.og.input.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <meta property="og:title" content="Page title - og:title meta">
+    <meta property="og:description" content="Page description - og:description meta">
+  </head>
+  <body>
+    <h1>Hello World</h1>
+  </body>
+</html>

package/test/importers/defaults/fixtures/metadata.twitter.expected.html

@@ -0,0 +1,16 @@
+<body>
+  <h1>Hello World</h1>
+  <table>
+    <tr>
+      <th colspan="2">Metadata</th>
+    </tr>
+    <tr>
+      <td>Title</td>
+      <td>Page title - twitter:title meta</td>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>Page description - twitter:description meta</td>
+    </tr>
+  </table>
+</body>

package/test/importers/defaults/fixtures/metadata.twitter.input.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <meta property="twitter:title" content="Page title - twitter:title meta">
+    <meta property="twitter:description" content="Page description - twitter:description meta">
+  </head>
+  <body>
+    <h1>Hello World</h1>
+  </body>
+</html>

package/test/importers/defaults/generateDocumentPath.spec.js

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { strictEqual } from 'assert';
+import { describe, it } from 'mocha';
+
+import defaultGenerateDocumentPath from '../../../src/importer/defaults/generateDocumentPath.js';
+
+describe('defaultGenerateDocumentPath tests', () => {
+  it('default paths', async () => {
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com' }), '/index');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/' }), '/index');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/index.html' }), '/index');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/index' }), '/index');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/page' }), '/page');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/page.html' }), '/page');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page' }), '/folder/page');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page.html' }), '/folder/page');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page/' }), '/folder/page/index');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/page with spaces.html' }), '/folder/page-with-spaces');
+    strictEqual(await defaultGenerateDocumentPath({ url: 'https://wwww.sample.com/folder/PagE_with_3xtr4_charactére.html' }), '/folder/page-with-3xtr4-charact-re');
+  });
+});

package/test/importers/defaults/transformDOM.spec.js

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import path from 'path';
+import fs from 'fs-extra';
+import { dirname } from 'dirname-filename-esm';
+import { strictEqual } from 'assert';
+import { describe, it } from 'mocha';
+
+import defaultTransformDOM from '../../../src/importer/defaults/transformDOM.js';
+import TestUtils from '../../TestUtils.js';
+
+// eslint-disable-next-line no-underscore-dangle
+const __dirname = dirname(import.meta);
+
+const { createDocumentFromString } = TestUtils;
+
+describe('defaultTransformDOM tests', () => {
+  const runTest = async (feature, config) => {
+    const spec = await fs.readFile(path.resolve(__dirname, 'fixtures', `${feature}.input.html`), 'utf-8');
+    const document = createDocumentFromString(spec);
+    const out = await defaultTransformDOM({ document, ...config });
+    const expected = await fs.readFile(path.resolve(__dirname, 'fixtures', `${feature}.expected.html`), 'utf-8');
+    strictEqual(out.outerHTML.replace(/(?:\r\n|\r|\n|\s\s)/g, ''), expected.replace(/(?:\r\n|\r|\n|\s\s)/g, ''));
+  };
+
+  it('default transformation', async () => {
+    await runTest('default');
+  });
+
+  it('default transformation handles basic metadata', async () => {
+    await runTest('metadata.basic');
+  });
+
+  it('default transformation handles img and alt metadata', async () => {
+    await runTest('metadata.image', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation handles identical metadata', async () => {
+    await runTest('metadata.all.same', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation handles different metadata', async () => {
+    await runTest('metadata.all.diff', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation handles falls back to og metadata', async () => {
+    await runTest('metadata.og', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation handles falls back to twitter metadata', async () => {
+    await runTest('metadata.twitter', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation removes non content elements', async () => {
+    await runTest('cleanup');
+  });
+
+  it('default transformation adjusts image urls', async () => {
+    await runTest('adjust-image-urls', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation converts icons', async () => {
+    await runTest('icons', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+
+  it('default transformation converts background-image styles into image element', async () => {
+    await runTest('background-image', {
+      url: 'https://wwww.sample.com/path/page.html',
+    });
+  });
+});

package/test/importers/fixtures/video.spec.html

@@ -0,0 +1,11 @@
+<html>
+  <body>
+    <h1>videos</h1>
+    <p><a href="./video.mp4">video link</a></p>
+    <video src="./video.mp4" poster="./poster.png">
+      <source src="./video.mp4" type="video/mp4">
+      <source src="./video.webm" type="video/webm">
+      <source src="./video.ogg" type="video/ogg">
+    </video>
+  </body>
+</html>

package/test/importers/fixtures/video.spec.md

@@ -0,0 +1,7 @@
+# videos
+
+[video link](https://www.sample.com/video.mp4)
+
+[![][image0]](https://www.sample.com/video.mp4)
+
+[image0]: ./poster.png

package/test/utils/DOMUtils.spec.js

@@ -408,15 +408,15 @@ describe('DOMUtils#getImgFromBackground', () => {
 
   it('no background-image style', () => {
     test(createElement('p', {}, {}, 'Some content'), null);
-
     test(createElement('img', { src: 'https://www.server.com/image.jpg', title: 'Some title' }, {}, ''), null);
-
     test(createElement('p', {}, { 'background-image': 'none' }, 'Some content'), null);
   });
 
   it('with background-image style', () => {
     test(createElement('p', {}, { 'background-image': 'url(https://www.server.com/image.jpg)' }, 'Some content'), '<img src="https://www.server.com/image.jpg">');
-    test(createElement('div', { class: 'someclass' }, { 'background-image': 'url("https://www.server.com/image.jpg")', background: 'rgb(0, 0, 0) none repeat scroll 0% 0% / auto padding-box border-box' }, '<div><div>Some divs</div><div>More divs</div></div>'), '<img src="https://www.server.com/image.jpg">');
+    test(createElement('p', {}, { 'background-image': 'url("https://www.server.com/image.jpg")' }, 'Some content'), '<img src="https://www.server.com/image.jpg">');
+    test(createElement('p', {}, { 'background-image': 'url(\'https://www.server.com/image.jpg\')' }, 'Some content'), '<img src="https://www.server.com/image.jpg">');
+    test(createElement('p', {}, { 'background-image': 'url(http://localhost:3001/image.jpg)' }, 'Some content'), '<img src="http://localhost:3001/image.jpg">');
   });
 
   // `createElement` uses JSDOM to create the test-DOM
@@ -426,6 +426,5 @@ describe('DOMUtils#getImgFromBackground', () => {
   // disabling the test, keeping it as a reference
   xit('with background-image style containing whitespace in url()', () => {
     test(createElement('p', {}, { 'background-image': 'url( /image.jpg )' }, 'Some content'), '<img src="/image.jpg">');
-    test(createElement('div', { class: 'someclass' }, { 'background-image': 'url( https://www.server.com/image.jpg )', background: 'rgb(0, 0, 0) none repeat scroll 0% 0% / auto padding-box border-box' }, '<div><div>Some divs</div><div>More divs</div></div>'), '<img src="https://www.server.com/image.jpg">');
   });
 });