@adobe/helix-importer 1.0.0

package/src/utils/DOMUtils.js

@@ -0,0 +1,207 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { JSDOM } from 'jsdom';
+
+export default class DOMUtils {
+  static EMPTY_TAGS_TO_PRESERVE = ['img', 'video', 'iframe', 'div', 'picture'];
+
+  static reviewInlineElement(document, tagName) {
+    let tags = [...document.querySelectorAll(tagName)];
+    // first pass, remove empty nodes
+    for (let i = tags.length - 1; i >= 0; i -= 1) {
+      const tag = tags[i];
+      if (tag.textContent === '' && !tag.querySelector(DOMUtils.EMPTY_TAGS_TO_PRESERVE.join(','))) {
+        tag.remove();
+      } else {
+        tag.innerHTML = tag.innerHTML.replace(/ /gm, ' ');
+      }
+    }
+
+    tags = [...document.querySelectorAll(tagName)];
+    // make a first pass to find <tag>x</tag> <tag>y</tag> and move the space
+    for (let i = tags.length - 1; i >= 0; i -= 1) {
+      const tag = tags[i];
+      if (tag.nextSibling && tag.nextSibling.textContent === ' ') {
+        // next is a space, check next next
+        const nextNext = tag.nextSibling.nextSibling;
+        if (nextNext && nextNext.tagName && nextNext.tagName.toLowerCase() === tagName) {
+          // same tag
+          tag.nextSibling.remove();
+          tag.innerHTML = `${tag.innerHTML} `;
+        }
+      }
+    }
+
+    tags = [...document.querySelectorAll(tagName)];
+    // collaspe consecutive <tag>
+    // and make sure element does not start ends with spaces while it is before / after some text
+    for (let i = tags.length - 1; i >= 0; i -= 1) {
+      const tag = tags[i];
+      if (tag.innerHTML === '.' || tag.innerHTML === '. ' || tag.innerHTML === ':' || tag.innerHTML === ': ') {
+        tag.replaceWith(JSDOM.fragment(tag.innerHTML));
+      } else {
+        const { innerHTML } = tag;
+        if (tag.previousSibling) {
+          const previous = tag.previousSibling;
+          if (
+            previous.tagName
+            && previous.tagName.toLowerCase() === tagName
+            && (!previous.href || previous.href === tag.href)
+          ) {
+            if (tag.hasChildNodes()) {
+              [...tag.childNodes].forEach((child) => {
+                previous.append(child);
+              });
+            } else {
+              // previous sibling is an <tag>, merge current one inside the previous one
+              previous.append(innerHTML);
+            }
+            tag.remove();
+          }
+        }
+      }
+    }
+
+    tags = [...document.querySelectorAll(tagName)];
+    // extra leading and trailing spaces into a dedicated span
+    for (let i = tags.length - 1; i >= 0; i -= 1) {
+      const tag = tags[i];
+      let { innerHTML } = tag;
+      if (innerHTML) {
+        if (innerHTML.lastIndexOf(' ') === innerHTML.length - 1) {
+          // move trailing space to a new text node outside of current element
+          tag.innerHTML = innerHTML.slice(0, innerHTML.length - 1);
+          ({ innerHTML } = tag);
+          tag.after(JSDOM.fragment('<span> </span>'));
+        }
+
+        if (innerHTML.indexOf(' ') === 0) {
+          // move leading space to a new text node outside of current element
+          tag.innerHTML = innerHTML.slice(1);
+          tag.before(JSDOM.fragment('<span> </span>'));
+        }
+      }
+    }
+  }
+
+  static reviewParagraphs(document) {
+    const tags = [...document.querySelectorAll('p')];
+    for (let i = tags.length - 1; i >= 0; i -= 1) {
+      const tag = tags[i];
+      // remove useless paragraphs
+      if (
+        (tag.textContent === ''
+          || tag.textContent === ' '
+          || tag.textContent === '&nbsp;'
+          || tag.textContent.charCodeAt(0) === 160)
+        && !tag.querySelector(DOMUtils.EMPTY_TAGS_TO_PRESERVE.join(','))
+      ) {
+        tag.remove();
+      }
+    }
+  }
+
+  static escapeSpecialCharacters(document) {
+    // eslint-disable-next-line no-param-reassign
+    document.body.innerHTML = document.body.innerHTML.replace(/~/gm, '\\~');
+  }
+
+  static reviewHeadings(document) {
+    const tags = [...document.querySelectorAll('h1, h2, h3, h4, h5, h6')];
+    for (let i = tags.length - 1; i >= 0; i -= 1) {
+      const tag = tags[i];
+      // remove useless strong tags
+      tag.innerHTML = tag.innerHTML.replace(/<strong>|<\\strong>/gm, '');
+      if (tag.innerHTML === '') {
+        tag.remove();
+      }
+    }
+  }
+
+  static remove(document, selectors) {
+    selectors.forEach((s) => {
+      document.querySelectorAll(s).forEach((n) => n.remove());
+    });
+  }
+
+  static removeComments(document) {
+    // eslint-disable-next-line no-param-reassign
+    document.body.innerHTML = document.body.innerHTML
+      // remove html comments
+      .replace(/<!--(?!>)[\S\s]*?-->/gm, '');
+  }
+
+  static removeSpans(document) {
+    // remove spans
+    document.querySelectorAll('span').forEach((span) => {
+      if (span.textContent === '') {
+        span.remove();
+      } else {
+        span.replaceWith(JSDOM.fragment(span.innerHTML));
+      }
+    });
+  }
+
+  static replaceByCaptions(document, selectors) {
+    selectors.forEach((selector) => {
+      document.querySelectorAll(selector).forEach((elem) => {
+        const captionText = elem.textContent.trim();
+        elem.parentNode.insertBefore(JSDOM.fragment(`<p><em>${captionText}</em></p>`), elem);
+        elem.remove();
+      });
+    });
+  }
+
+  static generateEmbed(url) {
+    return JSDOM.fragment(`<table><tr><th>Embed</th></tr><tr><td><a href="${url}">${url}</a></td></tr></table>`);
+  }
+
+  static replaceEmbeds(document) {
+    document.querySelectorAll('iframe').forEach((iframe) => {
+      const src = iframe.getAttribute('src');
+      const dataSrc = iframe.getAttribute('data-src');
+      const url = dataSrc || src;
+      if (url) {
+        iframe.after(DOMUtils.generateEmbed(url));
+      }
+      iframe.remove();
+    });
+
+    document.querySelectorAll('video').forEach((video) => {
+      let blockType = 'Video';
+      if (video.autoplay) {
+        blockType = 'Animation';
+      }
+      const anim = JSDOM.fragment(`<table><tr><th>${blockType}</th></tr><tr><td>${video.outerHTML}</td></tr></table>`);
+      video.replaceWith(anim);
+    });
+  }
+
+  static removeNoscripts(html) {
+    return html.replace(/<noscript>((.|\n)*?)<\/noscript>/gm, '');
+  }
+
+  static encodeImagesForTable(document) {
+    const imgs = document.querySelectorAll('img');
+    imgs.forEach((img) => {
+      if (img.closest('table')) {
+        // if image is in a table
+        if (img.title && img.title.indexOf('|') !== -1) {
+          // pipes in title do not get encoded
+          // eslint-disable-next-line no-param-reassign
+          img.title = img.title.replace(/\|/gm, '\\|');
+        }
+      }
+    });
+  }
+}

package/src/utils/FileUtils.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import sanitize from 'sanitize-filename';
+
+export default class FileUtils {
+  static sanitizeFilename(name) {
+    return sanitize(decodeURIComponent(name))
+      .trim()
+      .toLowerCase()
+      .replace(/\./gm, '')
+      .replace(/&/gm, '')
+      .replace(/\s/g, '-')
+      .replace(/-{2,}/g, '-');
+  }
+}

package/src/utils/Utils.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+export default class Utils {
+  static async asyncForEach(array, callback) {
+    for (let index = 0; index < array.length; index += 1) {
+      // eslint-disable-next-line no-await-in-loop
+      await callback(array[index], index, array);
+    }
+  }
+}

package/src/wp/WPUtils.js

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { JSDOM } from 'jsdom';
+
+import DOMUtils from '../utils/DOMUtils.js';
+
+export default class WPUtils {
+  static handleCaptions(document) {
+    DOMUtils.replaceByCaptions(document, ['.wp-caption-text', 'figcaption']);
+
+    // an h5 following an image / video is a caption
+    document.querySelectorAll('p img, video').forEach((item) => {
+      if (
+        (item.parentNode.nextElementSibling && item.parentNode.nextElementSibling.tagName === 'H5')
+        || (item.nextElementSibling && item.nextElementSibling.tagName === 'H5')
+      ) {
+        const elem = item.parentNode.nextElementSibling && item.parentNode.nextElementSibling.tagName === 'H5'
+          ? item.parentNode.nextElementSibling
+          : item.nextElementSibling;
+        const captionText = elem.textContent.trim();
+        elem.parentNode.insertBefore(JSDOM.fragment(`<p><em>${captionText}</em><p>`), elem);
+        elem.remove();
+      }
+    });
+  }
+
+  static genericDOMCleanup(document) {
+    // extract "emphasis" from links
+    // see https://github.com/adobe/helix-pipeline/issues/895
+    document.querySelectorAll('a strong').forEach((elem) => {
+      const parent = elem.parentNode;
+      if (parent.childNodes.length === 1) {
+        // only cover case with 1 child
+        const txt = elem.textContent;
+        // only treat links
+        if (txt && (txt.indexOf('.') !== -1 || txt.indexOf(':') !== -1)) {
+          // eslint-disable-next-line no-param-reassign
+          elem.innerHTML = '';
+          // take out of parent
+          parent.parentNode.insertBefore(elem, parent.nextSibling);
+          elem.appendChild(parent);
+          parent.innerHTML = txt;
+        }
+      }
+    });
+
+    // some images are in headings...
+    document.querySelectorAll('h1 img, h2 img, h3 img, h4 img, h5 img, h6 img').forEach((img) => {
+      // move image after its parent heading
+      img.parentNode.parentNode.insertBefore(img, img.parentNode.nextSibling);
+    });
+
+    // heading could be full of tags
+    document.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h) => {
+      // eslint-disable-next-line no-param-reassign
+      h.innerHTML = h.textContent;
+    });
+  }
+}

package/src/wp/explorers/WPAdminAjaxPager.js

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import FormData from 'form-data';
+import fetch from 'node-fetch';
+import PagingExplorer from '../../explorer/PagingExplorer.js';
+
+const API = 'wp-admin/admin-ajax.php';
+
+export default class WPAdminAjaxPager extends PagingExplorer {
+  async fetch(page) {
+    const api = `${this.params.url}${API}`;
+    const form = new FormData();
+    form.append('action', 'cardsFilter');
+    form.append('filterBy', 'latest');
+    form.append('paged', `${page}`);
+    return fetch(api, {
+      method: 'POST',
+      body: form,
+    });
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  process(document, all) {
+    const entries = [];
+    document.querySelectorAll('.card-item').forEach((el) => {
+      const link = el.querySelector('h4 a');
+      const url = link.getAttribute('href');
+
+      const entryDate = el.querySelector('.date');
+      const date = entryDate.textContent.trim();
+
+      if (all.findIndex((entry) => entry.url === url) === -1) {
+        entries.push({
+          date,
+          url,
+        });
+      }
+    });
+    return entries;
+  }
+}

package/src/wp/explorers/WPContentPager.js

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import fetch from 'node-fetch';
+import PagingExplorer from '../../explorer/PagingExplorer.js';
+
+const API = 'page/';
+
+export default class WPContentPager extends PagingExplorer {
+  async fetch(page) {
+    const api = `${this.params.url}${API}${page}`;
+    return fetch(api);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  process(document, all) {
+    const entries = [];
+    document.querySelectorAll('main .content .entry, main .entries .entry, article .entries .entry').forEach((el) => {
+      const link = el.querySelector('h2 a');
+      if (link) {
+        const url = link.getAttribute('href');
+
+        const entryDate = el.querySelector('.date') || el.querySelector('.entry_footer');
+        let date = '';
+        if (entryDate) {
+          date = entryDate.textContent.trim();
+        }
+
+        if (all.findIndex((entry) => entry.url === url) === -1) {
+          entries.push({
+            date,
+            url,
+          });
+        }
+      }
+    });
+    return entries;
+  }
+}

package/src/wp/explorers/WPPostWrapPager.js

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import fetch from 'node-fetch';
+import PagingExplorer from '../../explorer/PagingExplorer.js';
+
+const API = 'page/';
+
+export default class WPPostWrapPager extends PagingExplorer {
+  async fetch(page) {
+    const api = `${this.params.url}${API}${page}`;
+    return fetch(api);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  process(document, all) {
+    const entries = [];
+    document.querySelectorAll('.post-meta-wrap').forEach((el) => {
+      const link = el.querySelector('.post-item > a');
+      const url = link.getAttribute('href');
+
+      const entryDate = el.querySelector('.post-date');
+      const date = entryDate.textContent.trim();
+
+      if (all.findIndex((entry) => entry.url === url) === -1) {
+        entries.push({
+          date,
+          url,
+        });
+      }
+    });
+    return entries;
+  }
+}

package/test/explorers/PagingExplorer.spec.js

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2020 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* eslint-disable max-classes-per-file, class-methods-use-this */
+
+import { deepStrictEqual, strictEqual } from 'assert';
+import { describe, it } from 'mocha';
+
+import { Response } from 'node-fetch';
+
+import PagingExplorer from '../../src/explorer/PagingExplorer.js';
+
+describe('PagingExplorer tests', () => {
+  const params = {
+    url: 'testdest',
+    nbMaxPages: 3,
+  };
+
+  it('fetch and process are called 3 times if end not reached', async () => {
+    let fetchCalled = 0;
+    let processCalled = 0;
+
+    class Test extends PagingExplorer {
+      async fetch() {
+        fetchCalled += 1;
+        return new Response('test');
+      }
+
+      process() {
+        processCalled += 1;
+        return [{
+          a: 1,
+        }];
+      }
+    }
+
+    const se = new Test(params);
+    await se.explore();
+
+    strictEqual(fetchCalled, 3, 'process is called 3 times');
+    strictEqual(processCalled, 3, 'process is called 3 times');
+  });
+
+  it('fetch stops the process when reaches the end', async () => {
+    let fetchCalled = 0;
+    let processCalled = 0;
+
+    class Test extends PagingExplorer {
+      async fetch() {
+        fetchCalled += 1;
+        if (fetchCalled > 1) {
+          return new Response('reached the end', { status: 404 });
+        }
+        return new Response('test');
+      }
+
+      process() {
+        processCalled += 1;
+        return [{
+          a: 1,
+        }];
+      }
+    }
+
+    const se = new Test(params);
+    await se.explore();
+
+    strictEqual(fetchCalled, 2, 'fetch is called 2 times');
+    strictEqual(processCalled, 1, 'process is called 1 time');
+  });
+
+  it('explore returns the expected result set', async () => {
+    let processCalled = 0;
+    class Test extends PagingExplorer {
+      async fetch() {
+        return new Response('test');
+      }
+
+      process() {
+        processCalled += 1;
+        return [{
+          a: processCalled,
+        }];
+      }
+    }
+
+    const se = new Test(params);
+    const results = await se.explore();
+
+    deepStrictEqual(results, [{ a: 1 }, { a: 2 }, { a: 3 }], 'result is correct');
+  });
+
+  it('explore returns the expected result set when number of pages is not the max one', async () => {
+    let fetchCalled = 0;
+    let processCalled = 0;
+    class Test extends PagingExplorer {
+      async fetch() {
+        fetchCalled += 1;
+        if (fetchCalled > 2) {
+          return new Response('reached the end', { status: 404 });
+        }
+        return new Response('test');
+      }
+
+      process() {
+        processCalled += 1;
+        return [{
+          a: processCalled,
+        }];
+      }
+    }
+
+    const se = new Test(params);
+    const results = await se.explore();
+
+    deepStrictEqual(results, [{ a: 1 }, { a: 2 }], 'result is correct');
+  });
+
+  it('explore, fetch and process can be used to retrieve multipage results', async () => {
+    let fetchCalled = 0;
+    class Test extends PagingExplorer {
+      async fetch() {
+        fetchCalled += 1;
+        if (fetchCalled > 2) {
+          return new Response('reached the end', { status: 404 });
+        }
+        return new Response(`<html>
+          <body>
+            <a href="a${fetchCalled}.html">a${fetchCalled}</a>
+            <a href="b${fetchCalled}.html">b${fetchCalled}</a>
+            <a href="c${fetchCalled}.html">c${fetchCalled}</a>
+          </body
+        </html>`);
+      }
+
+      process(document) {
+        const entries = [];
+        document.querySelectorAll('a').forEach((el) => {
+          entries.push({
+            link: el.getAttribute('href'),
+          });
+        });
+        return entries;
+      }
+    }
+
+    const se = new Test(params);
+    const results = await se.explore();
+
+    deepStrictEqual(results, [
+      { link: 'a1.html' },
+      { link: 'b1.html' },
+      { link: 'c1.html' },
+      { link: 'a2.html' },
+      { link: 'b2.html' },
+      { link: 'c2.html' },
+    ], 'result is correct');
+  });
+
+  it('process receives the entry set from previous pages', async () => {
+    let fetchCalled = 0;
+    class Test extends PagingExplorer {
+      async fetch() {
+        fetchCalled += 1;
+        return new Response(`<html>
+          <body>
+            <a href="a${fetchCalled}.html">a${fetchCalled}</a>
+            <a href="b${fetchCalled}.html">b${fetchCalled}</a>
+            <a href="c${fetchCalled}.html">c${fetchCalled}</a>
+          </body
+        </html>`);
+      }
+
+      process(document, all) {
+        const testResult = [];
+        for (let i = 1; i < fetchCalled; i += 1) {
+          testResult.push({ link: `a${i}.html` });
+          testResult.push({ link: `b${i}.html` });
+          testResult.push({ link: `c${i}.html` });
+        }
+        deepStrictEqual(all, testResult, 'all entries argument contains previous entries from previous pages');
+
+        const entries = [];
+        document.querySelectorAll('a').forEach((el) => {
+          entries.push({
+            link: el.getAttribute('href'),
+          });
+        });
+        return entries;
+      }
+    }
+
+    const se = new Test(params);
+    await se.explore();
+  });
+});