@adobe/helix-importer 2.9.41 → 3.0.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+# [3.0.0](https://github.com/adobe/helix-importer/compare/v2.9.41...v3.0.0) (2023-11-13)
+
+
+### Features
+
+* get rid of JSDOM ([#260](https://github.com/adobe/helix-importer/issues/260)) ([2530363](https://github.com/adobe/helix-importer/commit/2530363c328958b2363f0f178b9b1ef7d7bb73b8))
+
+
+### BREAKING CHANGES
+
+* removing JSDOM
+
 ## [2.9.41](https://github.com/adobe/helix-importer/compare/v2.9.40...v2.9.41) (2023-11-05)
 
 

package/README.md

@@ -2,43 +2,7 @@
 
 Foundation tools for importing website content into that can be consumed in an Helix project.
 
-helix-importer is composed of 2 main building blocks:
-
-1. explorer: crawl a website to construct a list of urls to be importer
-2. importer: construct an importer - for an input url, transform the DOM and convert it into a Markdown file
-
-The folder [./src/wp](./src/wp) contains WordPress specific utils and explorer methods.
-
-## Explorer
-
-Idea of an explorer is to crawl the site in order to collect a list of urls. This list of urls can then be imported.
-
-Here is a basic sample:
-
-```js
-
-import { WPContentPager, FSHandler, CSV } from '@adobe/helix-importer';
-
-async function main() {
-  const pager = new WPContentPager({
-    nbMaxPages: 1000,
-    url: 'url to a WordPress site'
-  });
-
-  const entries = await pager.explore();
-
-  const csv = CSV.toCSV(entries);
-
-  const handler = new FSHandler('output', console);
-  await handler.put('explorer_results.csv', csv);
-}
-```
-
-In this example, the [WPContentPager](./src/wp/explorers/WPContentPager.ts) extends the [PagingExplorer](src/explorer/PagingExplorer.ts) which implements the 2 methods:
-- `fetch` which defines how to fetch one page on results
-- `explore` which extracts the list of urls present on that page
-
-The final result is a list of urls that could be found on list of paged results given by the WordPress API `/page/${page_number}`.
+Basic concept of the importer: for an input url, transform the DOM and convert it into a Markdown / docx file.
 
 ## Importer
 
@@ -50,10 +14,13 @@ Goal of the importer is to get rid of the generic DOM elements like the header /
 
 [HTML2x](src/importer/HTML2x.js) methods (`HTML2md` and `HTML2docx`) are convienence methods to run an import. As input, they take:
 - `URL`: URL of the page to import
-- `document`: the DOM element to import
+- `document`: the DOM element to import - a Document object or a string (see `createDocumentFromString` for the string case)
 - `transformerCfg`: object with the transformation "rules". Object can be either:
   - `{ transformDOM: ({ url, document, html, params }) => { ... return element-to-convert  }, generateDocumentPath: ({ url, document, html, params }) => { ... return path-to-target; }}` for a single mapping between one input document / one output file
   - `{ transform: ({ url, document, html, params }) => { ... return [{ element: first-element-to-convert, path: first-path-to-target }, ...]  }` for a mapping one input document / multiple output files (useful to generate multiple docx from a single web page)
+- `config`: object with several config properties
+  - `createDocumentFromString`: this config is required if you use the methods in a non-browser context and want to pass `document` param as string. This method receives the HTML to parse as a string and must return a Document object.
+  - `setBackgroundImagesFromCSS`: set to false to disable the `background-image` inlining in the DOM.
 
 ### Importer UI
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-importer",
-  "version": "2.9.41",
+  "version": "3.0.0",
   "description": "Helix Importer tool: create md / docx from html",
   "main": "src/index.js",
   "type": "module",
@@ -10,8 +10,10 @@
   },
   "scripts": {
     "lint": "eslint .",
-    "test": "c8 mocha",
-    "test-ci": "c8 mocha",
+    "test:web": "web-test-runner test/browser/*.test.js --node-resolve",
+    "test:web:watch": "web-test-runner test/browser/*.test.js --node-resolve --watch",
+    "test": "c8 mocha && npm run test:web",
+    "test-ci": "c8 mocha && npm run test:web",
     "semantic-release": "semantic-release",
     "prepare": "npx husky install"
   },
@@ -27,15 +29,21 @@
   "devDependencies": {
     "@adobe/eslint-config-helix": "2.0.4",
     "@adobe/helix-docx2md": "1.5.0",
-    "@adobe/helix-mediahandler": "2.3.1",
+    "@adobe/helix-mediahandler": "2.3.3",
+    "@esm-bundle/chai": "4.3.4-fix.0",
     "@semantic-release/changelog": "6.0.3",
     "@semantic-release/exec": "6.0.3",
     "@semantic-release/git": "10.0.1",
+    "@web/test-runner": "0.15.1",
+    "@web/test-runner-commands": "0.6.5",
+    "@web/test-runner-mocha": "0.8.1",
     "c8": "8.0.1",
+    "chai": "4.3.7",
     "dirname-filename-esm": "1.1.1",
     "eslint": "8.53.0",
     "husky": "8.0.3",
-    "lint-staged": "15.0.2",
+    "jsdom": "22.1.0",
+    "lint-staged": "15.1.0",
     "mocha": "10.2.0",
     "mocha-multi-reporters": "1.5.1",
     "mock-fs": "5.2.0",
@@ -52,7 +60,6 @@
     "form-data": "4.0.0",
     "fs-extra": "11.1.1",
     "hast-util-to-mdast": "10.1.0",
-    "jsdom": "22.1.0",
     "node-fetch": "3.3.2",
     "rehype-parse": "9.0.0",
     "rehype-remark": "10.0.0",

package/src/importer/HTML2x.js

@@ -12,12 +12,11 @@
 /* eslint-disable class-methods-use-this, no-console */
 
 import path from 'path';
-import { Response } from 'node-fetch';
-import { JSDOM } from 'jsdom';
 import PageImporter from './PageImporter.js';
 import PageImporterResource from './PageImporterResource.js';
 import MemoryHandler from '../storage/MemoryHandler.js';
 import Utils from '../utils/Utils.js';
+import BrowserUtils from '../utils/BrowserUtils.js';
 
 // import docxStylesXML from '../resources/styles.xml';
 
@@ -93,8 +92,8 @@ async function html2x(
 
   const html = doc.documentElement.outerHTML;
   class InternalImporter extends PageImporter {
-    async fetch() {
-      return new Response(html);
+    async get() {
+      return { document: doc, html };
     }
 
     async process(document) {
@@ -181,6 +180,7 @@ async function html2x(
       stylesXML: config.docxStylesXML,
       image2png: config.image2png,
     },
+    createDocumentFromString: config.createDocumentFromString,
   });
 
   const pirs = await importer.import(url);
@@ -224,10 +224,18 @@ async function html2x(
   }
 }
 
+const parseStringDocument = (html, config) => {
+  if (config?.createDocumentFromString) {
+    return config.createDocumentFromString(html);
+  } else {
+    return BrowserUtils.createDocumentFromString(html);
+  }
+};
+
 /**
  * Returns the result of the conversion from html to md.
  * @param {string} url URL of the document to convert
- * @param {HTMLElement|string} document Document to convert
+ * @param {Document} document Document to convert
  * @param {Object} transformCfg Conversion configuration
  * @param {Object} config Conversion configuration.
  * @param {Object} params Conversion params. Object will be pass to the transformer functions.
@@ -235,8 +243,8 @@ async function html2x(
  */
 async function html2md(url, document, transformCfg, config, params = {}) {
   let doc = document;
-  if (typeof document === 'string') {
-    doc = new JSDOM(document, { runScripts: undefined }).window.document;
+  if (typeof doc === 'string') {
+    doc = parseStringDocument(document, config);
   }
   return html2x(url, doc, transformCfg, { ...config, toMd: true, toDocx: false }, params);
 }
@@ -252,8 +260,8 @@ async function html2md(url, document, transformCfg, config, params = {}) {
  */
 async function html2docx(url, document, transformCfg, config, params = {}) {
   let doc = document;
-  if (typeof document === 'string') {
-    doc = new JSDOM(document, { runScripts: undefined }).window.document;
+  if (typeof doc === 'string') {
+    doc = parseStringDocument(document, config);
   }
   return html2x(url, doc, transformCfg, { ...config, toMd: true, toDocx: true }, params);
 }

package/src/importer/PageImporter.js

@@ -12,8 +12,6 @@
 
 /* eslint-disable class-methods-use-this */
 
-import { JSDOM } from 'jsdom';
-
 import path from 'path';
 import { unified } from 'unified';
 import parse from 'rehype-parse';
@@ -36,6 +34,7 @@ import DOMUtils from '../utils/DOMUtils.js';
 import FileUtils from '../utils/FileUtils.js';
 import MDUtils from '../utils/MDUtils.js';
 import formatPlugin from './mdast-to-md-format-plugin.js';
+import BrowserUtils from '../utils/BrowserUtils.js';
 
 function formatNode(type, state, node) {
   const result = {
@@ -55,6 +54,12 @@ export default class PageImporter {
 
   constructor(params) {
     this.params = params;
+
+    if (!this.params.createDocumentFromString) {
+      // default the string parsing using the browser DOMParser
+      this.params.createDocumentFromString = BrowserUtils.createDocumentFromString;
+    }
+
     this.logger = params.logger || console;
 
     this.useCache = !!params.cache;
@@ -297,8 +302,9 @@ export default class PageImporter {
     const html = await this.download(url);
 
     if (html) {
-      const { document } = new JSDOM(DOMUtils.removeNoscripts(html.toString())).window;
-      this.preProcess(document);
+      const cleanedHTML = DOMUtils.removeNoscripts(html.toString());
+
+      const document = this.params.createDocumentFromString(cleanedHTML);
       return {
         document,
         html,
@@ -315,6 +321,8 @@ export default class PageImporter {
 
     const results = [];
     if (document) {
+      this.preProcess(document);
+
       const entries = await this.process(document, url, entryParams, html);
 
       this.postProcess(document);

package/src/importer/PageImporterParams.js

@@ -22,4 +22,6 @@ export default class PageImporterParams {
   logger;
 
   mdast2docxOptions;
+
+  createDocumentFromString;
 }

package/src/index.js

@@ -9,9 +9,6 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import PagingExplorer from './explorer/PagingExplorer.js';
-import PagingExplorerParams from './explorer/PagingExplorerParams.js';
-
 import PageImporter from './importer/PageImporter.js';
 import PageImporterParams from './importer/PageImporterParams.js';
 import PageImporterResource from './importer/PageImporterResource.js';
@@ -27,15 +24,10 @@ import Loader from './utils/Loader.js';
 import Utils from './utils/Utils.js';
 
 import WPUtils from './wp/WPUtils.js';
-import WPAdminAjaxPager from './wp/explorers/WPAdminAjaxPager.js';
-import WPContentPager from './wp/explorers/WPContentPager.js';
-import WPPostWrapPager from './wp/explorers/WPPostWrapPager.js';
 
 import { html2md, html2docx } from './importer/HTML2x.js';
 
 export {
-  PagingExplorer,
-  PagingExplorerParams,
   PageImporter,
   PageImporterParams,
   PageImporterResource,
@@ -48,9 +40,6 @@ export {
   Loader,
   Utils,
   WPUtils,
-  WPAdminAjaxPager,
-  WPContentPager,
-  WPPostWrapPager,
   html2md,
   html2docx,
 };

package/src/utils/BrowserUtils.js

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default class BrowserUtils {
+  /**
+   * Creates a document from a html string. This function use DOMParser
+   * which should be available in execution context, i.e. a browser.
+   * @param {String} html The html to parse
+   * @returns Document The parsed document
+   */
+  static createDocumentFromString(html) {
+    try {
+      // eslint-disable-next-line no-undef
+      const parser = new DOMParser();
+      return parser.parseFromString(html, 'text/html');
+    } catch (e) {
+      throw new Error('Unable to parse HTML using default createDocumentFromString function and global DOMParser. Please provide a custom createDocumentFromString.');
+    }
+  }
+}

package/src/utils/DOMUtils.js

@@ -10,11 +10,15 @@
  * governing permissions and limitations under the License.
  */
 
-import { JSDOM } from 'jsdom';
-
 export default class DOMUtils {
   static EMPTY_TAGS_TO_PRESERVE = ['img', 'video', 'iframe', 'div', 'picture'];
 
+  static fragment(document, string) {
+    const tpl = document.createElement('template');
+    tpl.innerHTML = string;
+    return tpl.content;
+  }
+
   static reviewInlineElement(document, tagName) {
     let tags = [...document.querySelectorAll(tagName)];
     // first pass, remove empty nodes
@@ -48,7 +52,7 @@ export default class DOMUtils {
     for (let i = tags.length - 1; i >= 0; i -= 1) {
       const tag = tags[i];
       if (tag.innerHTML === '.' || tag.innerHTML === '. ' || tag.innerHTML === ':' || tag.innerHTML === ': ') {
-        tag.replaceWith(JSDOM.fragment(tag.innerHTML));
+        tag.replaceWith(DOMUtils.fragment(document, tag.innerHTML));
       } else {
         const { innerHTML } = tag;
         if (tag.previousSibling) {
@@ -82,13 +86,13 @@ export default class DOMUtils {
           // move trailing space to a new text node outside of current element
           tag.innerHTML = innerHTML.slice(0, innerHTML.length - 1);
           ({ innerHTML } = tag);
-          tag.after(JSDOM.fragment('<span> </span>'));
+          tag.after(DOMUtils.fragment(document, '<span> </span>'));
         }
 
         if (innerHTML.indexOf(' ') === 0) {
           // move leading space to a new text node outside of current element
           tag.innerHTML = innerHTML.slice(1);
-          tag.before(JSDOM.fragment('<span> </span>'));
+          tag.before(DOMUtils.fragment(document, '<span> </span>'));
         }
       }
     }
@@ -146,7 +150,7 @@ export default class DOMUtils {
         if (span.textContent === '') {
           span.remove();
         } else {
-          span.replaceWith(JSDOM.fragment(span.innerHTML));
+          span.replaceWith(DOMUtils.fragment(document, span.innerHTML));
         }
       }
     });
@@ -156,7 +160,7 @@ export default class DOMUtils {
     selectors.forEach((selector) => {
       document.querySelectorAll(selector).forEach((elem) => {
         const captionText = elem.textContent.trim();
-        elem.parentNode.insertBefore(JSDOM.fragment(`<p><em>${captionText}</em></p>`), elem);
+        elem.parentNode.insertBefore(DOMUtils.fragment(document, `<p><em>${captionText}</em></p>`), elem);
         elem.remove();
       });
     });
@@ -198,8 +202,8 @@ export default class DOMUtils {
     return table;
   }
 
-  static generateEmbed(url) {
-    return JSDOM.fragment(`<table><tr><th>Embed</th></tr><tr><td><a href="${url}">${url}</a></td></tr></table>`);
+  static generateEmbed(document, url) {
+    return DOMUtils.fragment(document, `<table><tr><th>Embed</th></tr><tr><td><a href="${url}">${url}</a></td></tr></table>`);
   }
 
   static replaceEmbeds(document) {
@@ -208,7 +212,7 @@ export default class DOMUtils {
       const dataSrc = iframe.getAttribute('data-src');
       const url = dataSrc || src;
       if (url) {
-        iframe.after(DOMUtils.generateEmbed(url));
+        iframe.after(DOMUtils.generateEmbed(document, url));
       }
       iframe.remove();
     });
@@ -218,7 +222,7 @@ export default class DOMUtils {
       if (video.autoplay) {
         blockType = 'Animation';
       }
-      const anim = JSDOM.fragment(`<table><tr><th>${blockType}</th></tr><tr><td>${video.outerHTML}</td></tr></table>`);
+      const anim = DOMUtils.fragment(document, `<table><tr><th>${blockType}</th></tr><tr><td>${video.outerHTML}</td></tr></table>`);
       video.replaceWith(anim);
     });
   }

package/src/wp/WPUtils.js

@@ -9,8 +9,6 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { JSDOM } from 'jsdom';
-
 import DOMUtils from '../utils/DOMUtils.js';
 
 export default class WPUtils {
@@ -27,7 +25,7 @@ export default class WPUtils {
           ? item.parentNode.nextElementSibling
           : item.nextElementSibling;
         const captionText = elem.textContent.trim();
-        elem.parentNode.insertBefore(JSDOM.fragment(`<p><em>${captionText}</em><p>`), elem);
+        elem.parentNode.insertBefore(DOMUtils.fragment(document, `<p><em>${captionText}</em><p>`), elem);
         elem.remove();
       }
     });

package/test/browser/BrowserUtils.test.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-env mocha */
+/* global Document, HTMLElement */
+
+import { expect } from '@esm-bundle/chai';
+import BrowserUtils from '../../src/utils/BrowserUtils.js';
+
+describe('BrowserUtils#createDocumentFromString', () => {
+  it('createDocumentFromString can parse a simple string', () => {
+    const document = BrowserUtils.createDocumentFromString('<html><head><title>Test</title></head><body><h1>Hello World</h1></body></html>');
+    expect(document).to.be.an.instanceof(Document);
+    expect(document.documentElement).to.be.an.instanceof(HTMLElement);
+
+    const title = document.querySelector('title');
+    expect(title).to.be.an.instanceof(HTMLElement);
+    expect(title.textContent).to.equal('Test');
+
+    const h1 = document.querySelector('h1');
+    expect(h1).to.be.an.instanceof(HTMLElement);
+    expect(h1.textContent).to.equal('Hello World');
+  });
+
+  it('createDocumentFromString can parse a non document string', () => {
+    const document = BrowserUtils.createDocumentFromString('<h1>Hello World</h1>');
+    expect(document).to.be.an.instanceof(Document);
+    expect(document.documentElement).to.be.an.instanceof(HTMLElement);
+
+    const h1 = document.querySelector('h1');
+    expect(h1).to.be.an.instanceof(HTMLElement);
+    expect(h1.textContent).to.equal('Hello World');
+  });
+});