@adobe/helix-importer 2.9.41 → 3.1.0

package/CHANGELOG.md

@@ -1,3 +1,22 @@
+# [3.1.0](https://github.com/adobe/helix-importer/compare/v3.0.0...v3.1.0) (2023-11-13)
+
+
+### Features
+
+* improve default import ([#261](https://github.com/adobe/helix-importer/issues/261)) ([251cfcd](https://github.com/adobe/helix-importer/commit/251cfcdde7ac54525eef49341a9ede52d368cf71))
+
+# [3.0.0](https://github.com/adobe/helix-importer/compare/v2.9.41...v3.0.0) (2023-11-13)
+
+
+### Features
+
+* get rid of JSDOM ([#260](https://github.com/adobe/helix-importer/issues/260)) ([2530363](https://github.com/adobe/helix-importer/commit/2530363c328958b2363f0f178b9b1ef7d7bb73b8))
+
+
+### BREAKING CHANGES
+
+* removing JSDOM
+
 ## [2.9.41](https://github.com/adobe/helix-importer/compare/v2.9.40...v2.9.41) (2023-11-05)
 
 

package/README.md

@@ -2,43 +2,7 @@
 
 Foundation tools for importing website content into that can be consumed in an Helix project.
 
-helix-importer is composed of 2 main building blocks:
-
-1. explorer: crawl a website to construct a list of urls to be importer
-2. importer: construct an importer - for an input url, transform the DOM and convert it into a Markdown file
-
-The folder [./src/wp](./src/wp) contains WordPress specific utils and explorer methods.
-
-## Explorer
-
-Idea of an explorer is to crawl the site in order to collect a list of urls. This list of urls can then be imported.
-
-Here is a basic sample:
-
-```js
-
-import { WPContentPager, FSHandler, CSV } from '@adobe/helix-importer';
-
-async function main() {
-  const pager = new WPContentPager({
-    nbMaxPages: 1000,
-    url: 'url to a WordPress site'
-  });
-
-  const entries = await pager.explore();
-
-  const csv = CSV.toCSV(entries);
-
-  const handler = new FSHandler('output', console);
-  await handler.put('explorer_results.csv', csv);
-}
-```
-
-In this example, the [WPContentPager](./src/wp/explorers/WPContentPager.ts) extends the [PagingExplorer](src/explorer/PagingExplorer.ts) which implements the 2 methods:
-- `fetch` which defines how to fetch one page on results
-- `explore` which extracts the list of urls present on that page
-
-The final result is a list of urls that could be found on list of paged results given by the WordPress API `/page/${page_number}`.
+Basic concept of the importer: for an input url, transform the DOM and convert it into a Markdown / docx file.
 
 ## Importer
 
@@ -50,10 +14,13 @@ Goal of the importer is to get rid of the generic DOM elements like the header /
 
 [HTML2x](src/importer/HTML2x.js) methods (`HTML2md` and `HTML2docx`) are convienence methods to run an import. As input, they take:
 - `URL`: URL of the page to import
-- `document`: the DOM element to import
+- `document`: the DOM element to import - a Document object or a string (see `createDocumentFromString` for the string case)
 - `transformerCfg`: object with the transformation "rules". Object can be either:
   - `{ transformDOM: ({ url, document, html, params }) => { ... return element-to-convert  }, generateDocumentPath: ({ url, document, html, params }) => { ... return path-to-target; }}` for a single mapping between one input document / one output file
   - `{ transform: ({ url, document, html, params }) => { ... return [{ element: first-element-to-convert, path: first-path-to-target }, ...]  }` for a mapping one input document / multiple output files (useful to generate multiple docx from a single web page)
+- `config`: object with several config properties
+  - `createDocumentFromString`: this config is required if you use the methods in a non-browser context and want to pass `document` param as string. This method receives the HTML to parse as a string and must return a Document object.
+  - `setBackgroundImagesFromCSS`: set to false to disable the `background-image` inlining in the DOM.
 
 ### Importer UI
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-importer",
-  "version": "2.9.41",
+  "version": "3.1.0",
   "description": "Helix Importer tool: create md / docx from html",
   "main": "src/index.js",
   "type": "module",
@@ -10,8 +10,10 @@
   },
   "scripts": {
     "lint": "eslint .",
-    "test": "c8 mocha",
-    "test-ci": "c8 mocha",
+    "test:web": "web-test-runner test/browser/*.test.js --node-resolve",
+    "test:web:watch": "web-test-runner test/browser/*.test.js --node-resolve --watch",
+    "test": "c8 mocha && npm run test:web",
+    "test-ci": "c8 mocha && npm run test:web",
     "semantic-release": "semantic-release",
     "prepare": "npx husky install"
   },
@@ -27,15 +29,21 @@
   "devDependencies": {
     "@adobe/eslint-config-helix": "2.0.4",
     "@adobe/helix-docx2md": "1.5.0",
-    "@adobe/helix-mediahandler": "2.3.1",
+    "@adobe/helix-mediahandler": "2.3.3",
+    "@esm-bundle/chai": "4.3.4-fix.0",
     "@semantic-release/changelog": "6.0.3",
     "@semantic-release/exec": "6.0.3",
     "@semantic-release/git": "10.0.1",
+    "@web/test-runner": "0.15.1",
+    "@web/test-runner-commands": "0.6.5",
+    "@web/test-runner-mocha": "0.8.1",
     "c8": "8.0.1",
+    "chai": "4.3.7",
     "dirname-filename-esm": "1.1.1",
     "eslint": "8.53.0",
     "husky": "8.0.3",
-    "lint-staged": "15.0.2",
+    "jsdom": "22.1.0",
+    "lint-staged": "15.1.0",
     "mocha": "10.2.0",
     "mocha-multi-reporters": "1.5.1",
     "mock-fs": "5.2.0",
@@ -52,7 +60,6 @@
     "form-data": "4.0.0",
     "fs-extra": "11.1.1",
     "hast-util-to-mdast": "10.1.0",
-    "jsdom": "22.1.0",
     "node-fetch": "3.3.2",
     "rehype-parse": "9.0.0",
     "rehype-remark": "10.0.0",

package/src/importer/HTML2x.js

@@ -12,12 +12,13 @@
 /* eslint-disable class-methods-use-this, no-console */
 
 import path from 'path';
-import { Response } from 'node-fetch';
-import { JSDOM } from 'jsdom';
 import PageImporter from './PageImporter.js';
 import PageImporterResource from './PageImporterResource.js';
 import MemoryHandler from '../storage/MemoryHandler.js';
 import Utils from '../utils/Utils.js';
+import BrowserUtils from '../utils/BrowserUtils.js';
+import defaultTransformDOM from './defaults/transformDOM.js';
+import defaultGenerateDocumentPath from './defaults/generateDocumentPath.js';
 
 // import docxStylesXML from '../resources/styles.xml';
 
@@ -36,27 +37,6 @@ function setBackgroundImagesFromCSS(document) {
   }
 }
 
-async function defaultTransformDOM({
-  // eslint-disable-next-line no-unused-vars
-  url, document, html, params,
-}) {
-  return document.body;
-}
-
-async function defaultGenerateDocumentPath({
-  // eslint-disable-next-line no-unused-vars
-  url, document, html, params,
-}) {
-  let p = new URL(url).pathname;
-  if (p.endsWith('/')) {
-    p = `${p}index`;
-  }
-  return decodeURIComponent(p)
-    .toLowerCase()
-    .replace(/\.html$/, '')
-    .replace(/[^a-z0-9/]/gm, '-');
-}
-
 async function html2x(
   url,
   doc,
@@ -93,8 +73,8 @@ async function html2x(
 
   const html = doc.documentElement.outerHTML;
   class InternalImporter extends PageImporter {
-    async fetch() {
-      return new Response(html);
+    async get() {
+      return { document: doc, html };
     }
 
     async process(document) {
@@ -181,6 +161,7 @@ async function html2x(
       stylesXML: config.docxStylesXML,
       image2png: config.image2png,
     },
+    createDocumentFromString: config.createDocumentFromString,
   });
 
   const pirs = await importer.import(url);
@@ -224,10 +205,18 @@ async function html2x(
   }
 }
 
+const parseStringDocument = (html, config) => {
+  if (config?.createDocumentFromString) {
+    return config.createDocumentFromString(html);
+  } else {
+    return BrowserUtils.createDocumentFromString(html);
+  }
+};
+
 /**
  * Returns the result of the conversion from html to md.
  * @param {string} url URL of the document to convert
- * @param {HTMLElement|string} document Document to convert
+ * @param {Document} document Document to convert
  * @param {Object} transformCfg Conversion configuration
  * @param {Object} config Conversion configuration.
  * @param {Object} params Conversion params. Object will be pass to the transformer functions.
@@ -235,8 +224,8 @@ async function html2x(
  */
 async function html2md(url, document, transformCfg, config, params = {}) {
   let doc = document;
-  if (typeof document === 'string') {
-    doc = new JSDOM(document, { runScripts: undefined }).window.document;
+  if (typeof doc === 'string') {
+    doc = parseStringDocument(document, config);
   }
   return html2x(url, doc, transformCfg, { ...config, toMd: true, toDocx: false }, params);
 }
@@ -252,8 +241,8 @@ async function html2md(url, document, transformCfg, config, params = {}) {
  */
 async function html2docx(url, document, transformCfg, config, params = {}) {
   let doc = document;
-  if (typeof document === 'string') {
-    doc = new JSDOM(document, { runScripts: undefined }).window.document;
+  if (typeof doc === 'string') {
+    doc = parseStringDocument(document, config);
   }
   return html2x(url, doc, transformCfg, { ...config, toMd: true, toDocx: true }, params);
 }

package/src/importer/PageImporter.js

@@ -12,8 +12,6 @@
 
 /* eslint-disable class-methods-use-this */
 
-import { JSDOM } from 'jsdom';
-
 import path from 'path';
 import { unified } from 'unified';
 import parse from 'rehype-parse';
@@ -36,6 +34,7 @@ import DOMUtils from '../utils/DOMUtils.js';
 import FileUtils from '../utils/FileUtils.js';
 import MDUtils from '../utils/MDUtils.js';
 import formatPlugin from './mdast-to-md-format-plugin.js';
+import BrowserUtils from '../utils/BrowserUtils.js';
 
 function formatNode(type, state, node) {
   const result = {
@@ -55,6 +54,12 @@ export default class PageImporter {
 
   constructor(params) {
     this.params = params;
+
+    if (!this.params.createDocumentFromString) {
+      // default the string parsing using the browser DOMParser
+      this.params.createDocumentFromString = BrowserUtils.createDocumentFromString;
+    }
+
     this.logger = params.logger || console;
 
     this.useCache = !!params.cache;
@@ -297,8 +302,9 @@ export default class PageImporter {
     const html = await this.download(url);
 
     if (html) {
-      const { document } = new JSDOM(DOMUtils.removeNoscripts(html.toString())).window;
-      this.preProcess(document);
+      const cleanedHTML = DOMUtils.removeNoscripts(html.toString());
+
+      const document = this.params.createDocumentFromString(cleanedHTML);
       return {
         document,
         html,
@@ -315,6 +321,8 @@ export default class PageImporter {
 
     const results = [];
     if (document) {
+      this.preProcess(document);
+
       const entries = await this.process(document, url, entryParams, html);
 
       this.postProcess(document);

package/src/importer/PageImporterParams.js

@@ -22,4 +22,6 @@ export default class PageImporterParams {
   logger;
 
   mdast2docxOptions;
+
+  createDocumentFromString;
 }

package/src/importer/defaults/generateDocumentPath.js

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+export default async function generateDocumentPath({
+  // eslint-disable-next-line no-unused-vars
+  url, document, html, params,
+}) {
+  let p = new URL(url).pathname;
+  if (p.endsWith('/')) {
+    p = `${p}index`;
+  }
+  return decodeURIComponent(p)
+    .toLowerCase()
+    .replace(/\.html$/, '')
+    .replace(/[^a-z0-9/]/gm, '-');
+}

package/src/importer/defaults/rules/adjustImageUrls.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default function adjustImageUrls(main, url) {
+  [...main.querySelectorAll('img')].forEach((img) => {
+    const src = img.getAttribute('src');
+    if (src && (src.startsWith('./') || src.startsWith('/') || src.startsWith('../'))) {
+      try {
+        const u = new URL(src, url);
+        // eslint-disable-next-line no-param-reassign
+        img.src = u.toString();
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.log(`Unable to adjust image URL ${img.src} - removing image`);
+        img.remove();
+      }
+    }
+  });
+}

package/src/importer/defaults/rules/convertIcons.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default function convertIcons(main, document) {
+  [...main.querySelectorAll('img')].forEach((img) => {
+    const src = img.getAttribute('src');
+    if (src && src.endsWith('.svg')) {
+      const span = document.createElement('span');
+      const name = src.split('/').pop().split('.')[0].toLowerCase().trim().replace(/[^a-z0-9]/g, '-');
+      if (name) {
+        span.innerHTML = `:${name}:`;
+        img.replaceWith(span);
+      }
+    }
+  });
+}

package/src/importer/defaults/rules/createMetadata.js

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import Blocks from '../../../utils/Blocks.js';
+
+function getMetadata(name, document) {
+  const attr = name && name.includes(':') ? 'property' : 'name';
+  const meta = [...document.head.querySelectorAll(`meta[${attr}="${name}"]`)]
+    .map((m) => m.content)
+    .join(', ');
+  return meta || '';
+}
+
+export default function createMetadata(main, document) {
+  const meta = {};
+
+  const title = document.querySelector('title');
+  if (title) {
+    meta.Title = title.textContent.replace(/[\n\t]/gm, '');
+  }
+
+  const desc = getMetadata('description', document);
+  if (desc) {
+    meta.Description = desc;
+  }
+
+  const img = getMetadata('og:image', document);
+  if (img) {
+    const el = document.createElement('img');
+    el.src = img;
+    meta.Image = el;
+
+    const imgAlt = getMetadata('og:image:alt', document);
+    if (imgAlt) {
+      el.alt = imgAlt;
+    }
+  }
+
+  const ogtitle = getMetadata('og:title', document);
+  if (ogtitle && ogtitle !== meta.Title) {
+    if (meta.Title) {
+      meta['og:title'] = ogtitle;
+    } else {
+      meta.Title = ogtitle;
+    }
+  }
+
+  const ogdesc = getMetadata('og:description', document);
+  if (ogdesc && ogdesc !== meta.Description) {
+    if (meta.Description) {
+      meta['og:description'] = ogdesc;
+    } else {
+      meta.Description = ogdesc;
+    }
+  }
+
+  const ttitle = getMetadata('twitter:title', document);
+  if (ttitle && ttitle !== meta.Title) {
+    if (meta.Title) {
+      meta['twitter:title'] = ttitle;
+    } else {
+      meta.Title = ttitle;
+    }
+  }
+
+  const tdesc = getMetadata('twitter:description', document);
+  if (tdesc && tdesc !== meta.Description) {
+    if (meta.Description) {
+      meta['twitter:description'] = tdesc;
+    } else {
+      meta.Description = tdesc;
+    }
+  }
+
+  const timg = getMetadata('twitter:image', document);
+  if (timg && timg !== img) {
+    const el = document.createElement('img');
+    el.src = timg;
+    meta['twitter:image'] = el;
+
+    const imgAlt = getMetadata('twitter:image:alt', document);
+    if (imgAlt) {
+      el.alt = imgAlt;
+    }
+  }
+
+  if (Object.keys(meta).length > 0) {
+    const block = Blocks.getMetadataBlock(document, meta);
+    main.append(block);
+  }
+
+  return meta;
+}

package/src/importer/defaults/rules/transformBackgroundImages.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import DOMUtils from '../../../utils/DOMUtils.js';
+
+export default function transformBackgroundImages(main, document) {
+  [...main.querySelectorAll('[style*="background-image: url"]')].forEach((element) => {
+    const img = DOMUtils.getImgFromBackground(element, document);
+    element.prepend(img);
+    element.style.removeProperty('background-image');
+  });
+}

package/src/importer/defaults/transformDOM.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import DOMUtils from '../../utils/DOMUtils.js';
+import createMetadata from './rules/createMetadata.js';
+import adjustImageUrls from './rules/adjustImageUrls.js';
+import convertIcons from './rules/convertIcons.js';
+import transformBackgroundImages from './rules/transformBackgroundImages.js';
+
+export default async function transformDOM({
+  // eslint-disable-next-line no-unused-vars
+  url, document, html, params,
+}) {
+  const main = document.body;
+
+  // attempt to remove non-content elements
+  DOMUtils.remove(main, [
+    'header',
+    '.header',
+    'nav',
+    '.nav',
+    'footer',
+    '.footer',
+    'iframe',
+    'noscript',
+  ]);
+
+  createMetadata(main, document);
+  transformBackgroundImages(main, document);
+  adjustImageUrls(main, url);
+  convertIcons(main, document);
+
+  return main;
+}

package/src/index.js

@@ -9,9 +9,6 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import PagingExplorer from './explorer/PagingExplorer.js';
-import PagingExplorerParams from './explorer/PagingExplorerParams.js';
-
 import PageImporter from './importer/PageImporter.js';
 import PageImporterParams from './importer/PageImporterParams.js';
 import PageImporterResource from './importer/PageImporterResource.js';
@@ -27,15 +24,22 @@ import Loader from './utils/Loader.js';
 import Utils from './utils/Utils.js';
 
 import WPUtils from './wp/WPUtils.js';
-import WPAdminAjaxPager from './wp/explorers/WPAdminAjaxPager.js';
-import WPContentPager from './wp/explorers/WPContentPager.js';
-import WPPostWrapPager from './wp/explorers/WPPostWrapPager.js';
 
 import { html2md, html2docx } from './importer/HTML2x.js';
 
+import createMetadata from './importer/defaults/rules/createMetadata.js';
+import adjustImageUrls from './importer/defaults/rules/adjustImageUrls.js';
+import convertIcons from './importer/defaults/rules/convertIcons.js';
+import transformBackgroundImages from './importer/defaults/rules/transformBackgroundImages.js';
+
+const rules = {
+  createMetadata,
+  adjustImageUrls,
+  convertIcons,
+  transformBackgroundImages,
+};
+
 export {
-  PagingExplorer,
-  PagingExplorerParams,
   PageImporter,
   PageImporterParams,
   PageImporterResource,
@@ -48,9 +52,7 @@ export {
   Loader,
   Utils,
   WPUtils,
-  WPAdminAjaxPager,
-  WPContentPager,
-  WPPostWrapPager,
   html2md,
   html2docx,
+  rules,
 };

package/src/utils/BrowserUtils.js

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default class BrowserUtils {
+  /**
+   * Creates a document from a html string. This function use DOMParser
+   * which should be available in execution context, i.e. a browser.
+   * @param {String} html The html to parse
+   * @returns Document The parsed document
+   */
+  static createDocumentFromString(html) {
+    try {
+      // eslint-disable-next-line no-undef
+      const parser = new DOMParser();
+      return parser.parseFromString(html, 'text/html');
+    } catch (e) {
+      throw new Error('Unable to parse HTML using default createDocumentFromString function and global DOMParser. Please provide a custom createDocumentFromString.');
+    }
+  }
+}