npm - @adobe/helix-importer - Versions diffs - 2.9.40 → 3.0.0 - Mend

@adobe/helix-importer 2.9.40 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

package/CHANGELOG.md +19 -0
package/README.md +5 -38
package/package.json +17 -10
package/src/importer/HTML2x.js +17 -9
package/src/importer/PageImporter.js +12 -4
package/src/importer/PageImporterParams.js +2 -0
package/src/index.js +0 -11
package/src/utils/BrowserUtils.js +29 -0
package/src/utils/DOMUtils.js +15 -11
package/src/wp/WPUtils.js +1 -3
package/test/browser/BrowserUtils.test.js +42 -0
package/test/importers/HTML2x.spec.js +123 -10
package/test/importers/PageImporter.spec.js +33 -2
package/test/utils/DOMUtils.spec.js +20 -0
package/src/explorer/PagingExplorer.js +0 -81
package/src/explorer/PagingExplorerParams.js +0 -17
package/src/wp/explorers/WPAdminAjaxPager.js +0 -51
package/src/wp/explorers/WPContentPager.js +0 -48
package/src/wp/explorers/WPPostWrapPager.js +0 -43
package/test/explorers/PagingExplorer.spec.js +0 -280

package/test/importers/HTML2x.spec.js CHANGED Viewed

@@ -10,7 +10,9 @@
  * governing permissions and limitations under the License.
  */
-import { deepStrictEqual, ok, strictEqual } from 'assert';
+import {
+  deepStrictEqual, ok, strictEqual, fail,
+} from 'assert';
 import { describe, it } from 'mocha';
 import { JSDOM } from 'jsdom';
 import { docx2md } from '@adobe/helix-docx2md';
@@ -24,9 +26,15 @@ import {
   defaultTransformDOM,
 } from '../../src/importer/HTML2x.js';
+// test environment createDocumentFromString version using JSDOM
+const createDocumentFromString = (html) => {
+  const { document } = new JSDOM(html, { runScripts: undefined }).window;
+  return document;
+};
 describe('defaultTransformDOM tests', () => {
   it('default transformation', async () => {
-    const { document } = new JSDOM('<html><body><h1>Hello World</h1></body></html>', { runScripts: undefined }).window;
+    const document = createDocumentFromString('<html><body><h1>Hello World</h1></body></html>');
     const out = await defaultTransformDOM({ document });
     strictEqual(out.outerHTML, '<body><h1>Hello World</h1></body>');
   });
@@ -73,7 +81,9 @@ describe('html2x parameters', () => {
       transformDOM: testParams,
       generateDocumentPath: testParams,
       preprocess: testParams,
-    }, null, {
+    }, {
+      createDocumentFromString,
+    }, {
       originalURL: ORIGNAL_URL,
     });
@@ -81,7 +91,9 @@ describe('html2x parameters', () => {
       transformDOM: testParams,
       generateDocumentPath: testParams,
       preprocess: testParams,
-    }, null, {
+    }, {
+      createDocumentFromString,
+    }, {
       originalURL: ORIGNAL_URL,
     });
   });
@@ -90,22 +102,94 @@ describe('html2x parameters', () => {
     await html2md(URL, HTML, {
       transform: testParams,
       preprocess: testParams,
-    }, null, {
+    }, {
+      createDocumentFromString,
+    }, {
       originalURL: ORIGNAL_URL,
     });
     await html2docx(URL, HTML, {
       transform: testParams,
       preprocess: testParams,
-    }, null, {
+    }, {
+      createDocumentFromString,
+    }, {
+      originalURL: ORIGNAL_URL,
+    });
+  });
+  it('document can be a Document', async () => {
+    const doc = createDocumentFromString(HTML);
+    await html2md(URL, doc, {
+      transformDOM: testParams,
+      generateDocumentPath: testParams,
+      preprocess: testParams,
+    }, {
+      createDocumentFromString,
+    }, {
+      originalURL: ORIGNAL_URL,
+    });
+    await html2docx(URL, doc, {
+      transformDOM: testParams,
+      generateDocumentPath: testParams,
+      preprocess: testParams,
+    }, {
+      createDocumentFromString,
+    }, {
       originalURL: ORIGNAL_URL,
     });
   });
+  it('document cannot be a string in the testing context', async () => {
+    // we need JSDOM to create a document
+    // because importer default implementation relies on DOMParser
+    try {
+      await html2md(URL, HTML, {
+        transformDOM: testParams,
+        generateDocumentPath: testParams,
+        preprocess: testParams,
+      }, {
+        createDocumentFromString: null,
+      }, {
+        originalURL: ORIGNAL_URL,
+      });
+      fail('should have thrown an error: default createDocumentFromString works only in browser context');
+    } catch (e) {
+      ok(true);
+    }
+    try {
+      await html2docx(URL, HTML, {
+        transformDOM: testParams,
+        generateDocumentPath: testParams,
+        preprocess: testParams,
+      }, {
+        createDocumentFromString: null,
+      }, {
+        originalURL: ORIGNAL_URL,
+      });
+      fail('should have thrown an error: default createDocumentFromString works only in browser context');
+    } catch (e) {
+      ok(true);
+    }
+  });
 });
 describe('html2md tests', () => {
   it('html2md provides a default transformation', async () => {
-    const out = await html2md('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>');
+    const out = await html2md('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>', null, {
+      createDocumentFromString,
+    });
+    strictEqual(out.html.trim(), '<body><h1>Hello World</h1></body>');
+    strictEqual(out.md.trim(), '# Hello World');
+    strictEqual(out.path, '/page');
+  });
+  it('html2md accepts a string', async () => {
+    const out = await html2md('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>', null, {
+      createDocumentFromString,
+    });
     strictEqual(out.html.trim(), '<body><h1>Hello World</h1></body>');
     strictEqual(out.md.trim(), '# Hello World');
     strictEqual(out.path, '/page');
@@ -119,6 +203,8 @@ describe('html2md tests', () => {
         return p;
       },
       generateDocumentPath: () => '/folder/my-custom-path',
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.html.trim(), '<p>My Hello to the World</p>');
     strictEqual(out.md.trim(), 'My Hello to the World');
@@ -142,6 +228,8 @@ describe('html2md tests', () => {
           path: '/folder/my-custom-path-p2',
         }];
       },
+    }, {
+      createDocumentFromString,
     });
     const out1 = out[0];
@@ -169,6 +257,8 @@ describe('html2md tests', () => {
           path: '/my-custom-path-p1',
         };
       },
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.html.trim(), '<p>My Hello to the World 1</p>');
@@ -209,6 +299,8 @@ describe('html2md tests', () => {
           },
         }];
       },
+    }, {
+      createDocumentFromString,
     });
     const out1 = out[0];
@@ -247,6 +339,8 @@ describe('html2md tests', () => {
           },
         },
       }],
+    }, {
+      createDocumentFromString,
     });
     // if no element provided, no creation of html, md or docx
@@ -264,6 +358,8 @@ describe('html2md tests', () => {
   it('html2md does not crash if transform returns null', async () => {
     const out = await html2md('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>', {
       transform: () => null,
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.length, 0);
@@ -273,6 +369,8 @@ describe('html2md tests', () => {
     const out = await html2md('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>', {
       transformDOM: () => null,
       generateDocumentPath: () => null,
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.html.trim(), '<body><h1>Hello World</h1></body>');
     strictEqual(out.md.trim(), '# Hello World');
@@ -291,18 +389,25 @@ describe('html2md tests', () => {
           return document.body;
         },
       },
+      {
+        createDocumentFromString,
+      },
     );
     strictEqual(out.html.trim(), '<body><img src="./image.png"></body>');
   });
   it('html2md removes images with src attributes', async () => {
-    const out = await html2md('https://www.sample.com/page.html', '<html><body><img src="data:abc"></body></html>');
+    const out = await html2md('https://www.sample.com/page.html', '<html><body><img src="data:abc"></body></html>', null, {
+      createDocumentFromString,
+    });
     strictEqual(out.html.trim(), '<body></body>');
     strictEqual(out.md.trim(), '');
   });
   it('html2md set image src with data-src attribute value', async () => {
-    const out = await html2md('https://www.sample.com/page.html', '<html><body><img src="data:abc" data-src="./image.jpg"></body></html>');
+    const out = await html2md('https://www.sample.com/page.html', '<html><body><img src="data:abc" data-src="./image.jpg"></body></html>', null, {
+      createDocumentFromString,
+    });
     strictEqual(out.html.trim(), '<body><img src="./image.jpg" data-src="./image.jpg"></body>');
     strictEqual(out.md.trim(), '![][image0]\n\n[image0]: ./image.jpg');
   });
@@ -314,6 +419,8 @@ describe('html2md tests', () => {
         img.setAttribute('src', img.getAttribute('data-fancy-src'));
         img.removeAttribute('data-fancy-src');
       },
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.html.trim(), '<body><img src="./image.jpg"></body>');
     strictEqual(out.md.trim(), '![][image0]\n\n[image0]: ./image.jpg');
@@ -330,6 +437,8 @@ describe('html2md tests', () => {
         p.after(hr);
         return document.body;
       },
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.html.trim(), '<body><p>text 1</p><p>text 2</p><p>text 3</p><p>text 4</p><hr></body>');
     strictEqual(out.md.trim(), 'text 1\n\ntext 2\n\ntext 3\n\ntext 4\n\n---');
@@ -338,7 +447,9 @@ describe('html2md tests', () => {
 describe('html2docx tests', () => {
   it('html2docx provides a default transformation', async () => {
-    const out = await html2docx('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>');
+    const out = await html2docx('https://www.sample.com/page.html', '<html><body><h1>Hello World</h1></body></html>', null, {
+      createDocumentFromString,
+    });
     strictEqual(out.html.trim(), '<body><h1>Hello World</h1></body>');
     strictEqual(out.md.trim(), '# Hello World');
     strictEqual(out.path, '/page');
@@ -357,6 +468,8 @@ describe('html2docx tests', () => {
         return p;
       },
       generateDocumentPath: () => '/folder1/folder2/my-custom-path',
+    }, {
+      createDocumentFromString,
     });
     strictEqual(out.html.trim(), '<p>My Hello to the World</p>');
     strictEqual(out.md.trim(), 'My Hello to the World');

package/test/importers/PageImporter.spec.js CHANGED Viewed

@@ -14,13 +14,15 @@
 import path from 'path';
 import fs from 'fs-extra';
-import { strictEqual, ok } from 'assert';
+import { strictEqual, ok, fail } from 'assert';
 import { describe, it } from 'mocha';
 import { Response } from 'node-fetch';
 import { dirname } from 'dirname-filename-esm';
 import { docx2md } from '@adobe/helix-docx2md';
+import { JSDOM } from 'jsdom';
 import { unified } from 'unified';
 import remarkParse from 'remark-parse';
 import remarkGridTable from '@adobe/remark-gridtables';
@@ -38,6 +40,12 @@ const __dirname = dirname(import.meta);
 const logger = new NoopLogger();
+// test environment createDocumentFromString version using JSDOM
+const createDocumentFromString = (html) => {
+  const { document } = new JSDOM(html, { runScripts: undefined }).window;
+  return document;
+};
 describe('PageImporter tests', () => {
   const storageHandler = new MemoryHandler(logger);
   const config = {
@@ -52,11 +60,31 @@ describe('PageImporter tests', () => {
       }
     }
-    const se = new TestImporter(config);
+    const se = new TestImporter({
+      createDocumentFromString,
+      ...config,
+    });
     const results = await se.import('someurl');
     strictEqual(results.length, 0, 'expect no result');
   });
+  it('import - not providing createDocumentFromString should fail in the test enviroment only', async () => {
+    class TestImporter extends PageImporter {
+      async fetch() {
+        return new Response('test');
+      }
+    }
+    const se = new TestImporter(config);
+    try {
+      await se.import('someurl');
+      fail('should have thrown an error: default createDocumentFromString works only in browser context');
+    } catch (e) {
+      ok(true);
+    }
+  });
 });
 describe('PageImporter tests - various options', () => {
@@ -76,6 +104,7 @@ describe('PageImporter tests - various options', () => {
     const config = {
       storageHandler,
       logger,
+      createDocumentFromString,
     };
     const se = new Test(config);
     const results = await se.import('/someurl');
@@ -105,6 +134,7 @@ describe('PageImporter tests - various options', () => {
       mdast2Docx2Options: {
         stylesXML,
       },
+      createDocumentFromString,
     };
     const se = new Test(config);
     const results = await se.import('/someurl');
@@ -144,6 +174,7 @@ describe('PageImporter tests - fixtures', () => {
       storageHandler,
       skipDocxConversion: true,
       logger,
+      createDocumentFromString,
     };
     const se = new Test(config);
     const results = await se.import(`https://www.sample.com/${feature}`);

package/test/utils/DOMUtils.spec.js CHANGED Viewed

@@ -19,6 +19,26 @@ import { JSDOM } from 'jsdom';
 import DOMUtils from '../../src/utils/DOMUtils.js';
+describe('DOMUtils#fragment tests', () => {
+  const test = (input) => {
+    const { document } = (new JSDOM()).window;
+    const output = DOMUtils.fragment(document, input);
+    const div = document.createElement('div');
+    div.append(output);
+    const expected = document.createElement('div');
+    expected.innerHTML = input;
+    strictEqual(div.outerHTML, expected.outerHTML);
+  };
+  it('can create fragments from string', () => {
+    test('some text');
+    test(' some text with spaces ');
+    test('<a href="linkhref">linkcontent</a>');
+    test('<p><em>Caption Text</em></p>');
+    test('some text: <a href="linkhref">linkcontent</a> <a href="linkhref">another link</a>');
+  });
+});
 describe('DOMUtils#reviewInlineElement tests', () => {
   const test = (input, tag, expected) => {
     const { document } = (new JSDOM(input)).window;

package/src/explorer/PagingExplorer.js DELETED Viewed

@@ -1,81 +0,0 @@
-/*
- * Copyright 2021 Adobe. All rights reserved.
- * This file is licensed to you under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under
- * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
- * OF ANY KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-/* eslint-disable class-methods-use-this, no-console */
-import { JSDOM } from 'jsdom';
-export default class PagingExplorer {
-  params;
-  constructor(params) {
-    this.params = params;
-  }
-  async explore(
-    // eslint-disable-next-line default-param-last
-    page = 1,
-    pageCallback,
-  ) {
-    const startTime = new Date().getTime();
-    let results = [];
-    while (page <= this.params.nbMaxPages) {
-      console.log(`${this.params.url}: Requesting page ${page}/${this.params.nbMaxPages}.`);
-      // eslint-disable-next-line no-await-in-loop
-      const res = await this.fetch(page);
-      if (!res.ok) {
-        console.log(`${this.params.url}: Invalid response, considering no more results`);
-        break;
-      } else {
-        // eslint-disable-next-line no-await-in-loop
-        const text = await res.text();
-        if (text) {
-          const { document } = new JSDOM(text).window;
-          const entries = this.process(document, results);
-          if (entries && entries.length > 0) {
-            results = results.concat(entries);
-            if (pageCallback) {
-              // eslint-disable-next-line no-await-in-loop
-              await pageCallback(entries, page, results);
-            }
-          } else {
-            console.log(`${this.params.url}: No entries found on page ${page}`);
-            break;
-          }
-        } else {
-          console.log(`${this.params.url}: No more results`);
-          break;
-        }
-        // eslint-disable-next-line no-param-reassign
-        page += 1;
-      }
-    }
-    console.log();
-    console.log(`${this.params.url}: Process stopped at page ${page - 1} on ${this.params.nbMaxPages}.`);
-    console.log(`${this.params.url}: Imported ${results.length} post entries.`);
-    console.log(`${this.params.url}: Process took ${(new Date().getTime() - startTime) / 1000}s.`);
-    return results;
-  }
-  fetch() {}
-  process() {}
-}

package/src/explorer/PagingExplorerParams.js DELETED Viewed

@@ -1,17 +0,0 @@
-/*
- * Copyright 2021 Adobe. All rights reserved.
- * This file is licensed to you under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under
- * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
- * OF ANY KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-export default class PagingExplorerParams {
-  nbMaxPages;
-  url;
-}

package/src/wp/explorers/WPAdminAjaxPager.js DELETED Viewed

@@ -1,51 +0,0 @@
-/*
- * Copyright 2021 Adobe. All rights reserved.
- * This file is licensed to you under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under
- * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
- * OF ANY KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-import FormData from 'form-data';
-import fetch from 'node-fetch';
-import PagingExplorer from '../../explorer/PagingExplorer.js';
-const API = 'wp-admin/admin-ajax.php';
-export default class WPAdminAjaxPager extends PagingExplorer {
-  async fetch(page) {
-    const api = `${this.params.url}${API}`;
-    const form = new FormData();
-    form.append('action', 'cardsFilter');
-    form.append('filterBy', 'latest');
-    form.append('paged', `${page}`);
-    return fetch(api, {
-      method: 'POST',
-      body: form,
-    });
-  }
-  // eslint-disable-next-line class-methods-use-this
-  process(document, all) {
-    const entries = [];
-    document.querySelectorAll('.card-item').forEach((el) => {
-      const link = el.querySelector('h4 a');
-      const url = link.getAttribute('href');
-      const entryDate = el.querySelector('.date');
-      const date = entryDate.textContent.trim();
-      if (all.findIndex((entry) => entry.url === url) === -1) {
-        entries.push({
-          date,
-          url,
-        });
-      }
-    });
-    return entries;
-  }
-}

package/src/wp/explorers/WPContentPager.js DELETED Viewed

@@ -1,48 +0,0 @@
-/*
- * Copyright 2021 Adobe. All rights reserved.
- * This file is licensed to you under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under
- * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
- * OF ANY KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-import fetch from 'node-fetch';
-import PagingExplorer from '../../explorer/PagingExplorer.js';
-const API = 'page/';
-export default class WPContentPager extends PagingExplorer {
-  async fetch(page) {
-    const api = `${this.params.url}${API}${page}`;
-    return fetch(api);
-  }
-  // eslint-disable-next-line class-methods-use-this
-  process(document, all) {
-    const entries = [];
-    document.querySelectorAll('main .content .entry, main .entries .entry, article .entries .entry').forEach((el) => {
-      const link = el.querySelector('h2 a');
-      if (link) {
-        const url = link.getAttribute('href');
-        const entryDate = el.querySelector('.date') || el.querySelector('.entry_footer');
-        let date = '';
-        if (entryDate) {
-          date = entryDate.textContent.trim();
-        }
-        if (all.findIndex((entry) => entry.url === url) === -1) {
-          entries.push({
-            date,
-            url,
-          });
-        }
-      }
-    });
-    return entries;
-  }
-}

package/src/wp/explorers/WPPostWrapPager.js DELETED Viewed

@@ -1,43 +0,0 @@
-/*
- * Copyright 2021 Adobe. All rights reserved.
- * This file is licensed to you under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License. You may obtain a copy
- * of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under
- * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
- * OF ANY KIND, either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-import fetch from 'node-fetch';
-import PagingExplorer from '../../explorer/PagingExplorer.js';
-const API = 'page/';
-export default class WPPostWrapPager extends PagingExplorer {
-  async fetch(page) {
-    const api = `${this.params.url}${API}${page}`;
-    return fetch(api);
-  }
-  // eslint-disable-next-line class-methods-use-this
-  process(document, all) {
-    const entries = [];
-    document.querySelectorAll('.post-meta-wrap').forEach((el) => {
-      const link = el.querySelector('.post-item > a');
-      const url = link.getAttribute('href');
-      const entryDate = el.querySelector('.post-date');
-      const date = entryDate.textContent.trim();
-      if (all.findIndex((entry) => entry.url === url) === -1) {
-        entries.push({
-          date,
-          url,
-        });
-      }
-    });
-    return entries;
-  }
-}