@adobe/helix-importer 2.9.28 → 2.9.30

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [2.9.30](https://github.com/adobe/helix-importer/compare/v2.9.29...v2.9.30) (2023-09-13)
+
+
+### Bug Fixes
+
+* **deps:** update adobe fixes ([#226](https://github.com/adobe/helix-importer/issues/226)) ([bf68111](https://github.com/adobe/helix-importer/commit/bf68111284ac5bfe25879022c25f55bcee94386e))
+
+## [2.9.29](https://github.com/adobe/helix-importer/compare/v2.9.28...v2.9.29) (2023-09-07)
+
+
+### Bug Fixes
+
+* cleanup mdast tree before writing as md ([#223](https://github.com/adobe/helix-importer/issues/223)) ([49f88ce](https://github.com/adobe/helix-importer/commit/49f88cec0bbc30cbf8e5806b1467a21f05ab9814)), closes [#217](https://github.com/adobe/helix-importer/issues/217) [#214](https://github.com/adobe/helix-importer/issues/214) [#166](https://github.com/adobe/helix-importer/issues/166) [#213](https://github.com/adobe/helix-importer/issues/213)
+
 ## [2.9.28](https://github.com/adobe/helix-importer/compare/v2.9.27...v2.9.28) (2023-09-02)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-importer",
-  "version": "2.9.28",
+  "version": "2.9.30",
   "description": "Helix Importer tool: create md / docx from html",
   "main": "src/index.js",
   "type": "module",
@@ -27,24 +27,26 @@
   "devDependencies": {
     "@adobe/eslint-config-helix": "2.0.3",
     "@adobe/helix-docx2md": "1.4.15",
-    "@adobe/helix-mediahandler": "2.2.12",
+    "@adobe/helix-mediahandler": "2.2.14",
     "@semantic-release/changelog": "6.0.3",
     "@semantic-release/exec": "6.0.3",
     "@semantic-release/git": "10.0.1",
     "c8": "8.0.1",
     "dirname-filename-esm": "1.1.1",
-    "eslint": "8.48.0",
+    "eslint": "8.49.0",
     "husky": "8.0.3",
     "lint-staged": "14.0.1",
     "mocha": "10.2.0",
     "mocha-multi-reporters": "1.5.1",
     "mock-fs": "5.2.0",
-    "semantic-release": "21.1.1"
+    "remark-parse": "10.0.2",
+    "semantic-release": "21.1.1",
+    "unist-util-inspect": "8.0.0"
   },
   "license": "Apache-2.0",
   "dependencies": {
     "@adobe/helix-markdown-support": "6.3.1",
-    "@adobe/helix-md2docx": "2.1.24",
+    "@adobe/helix-md2docx": "2.1.25",
     "@adobe/mdast-util-gridtables": "2.0.2",
     "@adobe/remark-gridtables": "1.0.4",
     "form-data": "4.0.0",

package/src/importer/PageImporter.js

@@ -17,35 +17,33 @@ import { JSDOM } from 'jsdom';
 import path from 'path';
 import { unified } from 'unified';
 import parse from 'rehype-parse';
-import rehype2remark from 'rehype-remark';
+import { defaultHandlers, toMdast } from 'hast-util-to-mdast';
 import stringify from 'remark-stringify';
 import fs from 'fs-extra';
 import { md2docx } from '@adobe/helix-md2docx';
 import remarkGridTable from '@adobe/remark-gridtables';
-import { imageReferences, remarkGfmNoLink } from '@adobe/helix-markdown-support';
+import {
+  imageReferences,
+  remarkGfmNoLink,
+  sanitizeHeading,
+  sanitizeLinks,
+  sanitizeTextAndFormats,
+  suppressSpaceCode,
+} from '@adobe/helix-markdown-support';
 import gridtableHandlers from './hast-to-mdast-gridtable-handlers.js';
 import Utils from '../utils/Utils.js';
 import DOMUtils from '../utils/DOMUtils.js';
 import FileUtils from '../utils/FileUtils.js';
 import MDUtils from '../utils/MDUtils.js';
-
-function remarkImageReferences() {
-  return imageReferences;
-}
-
-function htmlElementNode(element, state, node) {
-  if (node.children && node.children.length > 0) {
-    return [{
-      type: 'html',
-      value: `<${element}>`,
-    },
-    ...state.all(node),
-    {
-      type: 'html',
-      value: `</${element}>`,
-    }];
-  }
-  return '';
+import formatPlugin from './mdast-to-md-format-plugin.js';
+
+function formatNode(type, state, node) {
+  const result = {
+    type,
+    children: state.all(node),
+  };
+  state.patch(node, result);
+  return result;
 }
 
 export default class PageImporter {
@@ -76,20 +74,32 @@ export default class PageImporter {
     const sanitizedName = FileUtils.sanitizeFilename(name);
     this.logger.log(`Computing Markdown for ${directory}/${sanitizedName}`);
 
-    const processor = unified()
+    const html = resource.document.innerHTML;
+    const hast = await unified()
       .use(parse, { emitParseErrors: true })
-      .use(rehype2remark, {
-        handlers: {
-          u: (state, node) => htmlElementNode('u', state, node),
-          sub: (state, node) => htmlElementNode('sub', state, node),
-          sup: (state, node) => htmlElementNode('sup', state, node),
-          ...gridtableHandlers,
-        },
-      })
-      .use(remarkImageReferences)
-      .use(remarkGridTable)
-      .use(remarkGfmNoLink)
+      .parse(html);
+
+    const mdast = toMdast(hast, {
+      handlers: {
+        ...defaultHandlers,
+        u: (state, node) => formatNode('underline', state, node),
+        sub: (state, node) => formatNode('subscript', state, node),
+        sup: (state, node) => formatNode('superscript', state, node),
+        ...gridtableHandlers,
+      },
+    });
+
+    // cleanup mdast similar to docx2md
+    await sanitizeHeading(mdast);
+    await sanitizeLinks(mdast);
+    await sanitizeTextAndFormats(mdast);
+    await suppressSpaceCode(mdast);
+    await imageReferences(mdast);
+
+    let md = await unified()
       .use(stringify, {
+        strong: '*',
+        emphasis: '_',
         bullet: '-',
         fence: '`',
         fences: true,
@@ -97,20 +107,21 @@ export default class PageImporter {
         rule: '-',
         ruleRepetition: 3,
         ruleSpaces: false,
-      });
-
-    const html = resource.document.innerHTML;
-    const file = await processor.process(html);
-    let contents = String(file);
+      })
+      .use(remarkGridTable)
+      .use(remarkGfmNoLink)
+      .use(formatPlugin) // this converts the `underline` and `subscript` back to tags in the md.
+      .stringify(mdast);
 
     // process image links
+    // TODO: this can be done easier in the MDAST tree
     const { document } = resource;
     const assets = [];
     const imgs = document.querySelectorAll('img');
     imgs.forEach((img) => {
       const { src } = img;
       const isEmbed = img.classList.contains('hlx-embed');
-      if (!isEmbed && src && src !== '' && (contents.indexOf(src) !== -1 || contents.indexOf(decodeURI(src)) !== -1)) {
+      if (!isEmbed && src && src !== '' && (md.indexOf(src) !== -1 || md.indexOf(decodeURI(src)) !== -1)) {
         assets.push({
           url: src,
           append: '#image.png',
@@ -122,7 +133,7 @@ export default class PageImporter {
     as.forEach((a) => {
       const { href } = a;
       try {
-        if ((href && href !== '' && contents.indexOf(href) !== -1) || contents.indexOf(decodeURI(href)) !== -1) {
+        if ((href && href !== '' && md.indexOf(href) !== -1) || md.indexOf(decodeURI(href)) !== -1) {
           const u = new URL(href, url);
           const ext = path.extname(u.href);
           if (ext === '.mp4') {
@@ -141,7 +152,7 @@ export default class PageImporter {
     const vs = document.querySelectorAll('video source');
     vs.forEach((s) => {
       const { src } = s;
-      if ((src && src !== '' && contents.indexOf(src) !== -1) || contents.indexOf(decodeURI(src)) !== -1) {
+      if ((src && src !== '' && md.indexOf(src) !== -1) || md.indexOf(decodeURI(src)) !== -1) {
         try {
           const u = new URL(src, url);
           const ext = path.extname(u.href);
@@ -167,18 +178,18 @@ export default class PageImporter {
     // adjust assets url (from relative to absolute)
     assets.forEach((asset) => {
       const u = new URL(decodeURI(asset.url), url);
-      contents = MDUtils.replaceSrcInMarkdown(contents, asset.url, u.toString());
+      md = MDUtils.replaceSrcInMarkdown(md, asset.url, u.toString());
     });
 
     if (resource.prepend) {
-      contents = resource.prepend + contents;
+      md = resource.prepend + md;
     }
 
-    contents = this.postProcessMD(contents);
+    md = this.postProcessMD(md);
 
     return {
       path: path.join(directory, sanitizedName),
-      content: contents,
+      content: md,
     };
   }
 

package/src/importer/mdast-to-md-format-plugin.js

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/**
+ * Renders special html only format.
+ */
+function format(tagName) {
+  const tagOpen = `<${tagName}>`;
+  const tagClose = `</${tagName}>`;
+
+  /**
+   * @param {Node} node
+   * @param {Parents | undefined} _
+   * @param {State} state
+   * @param {Info} info
+   * @returns {string}
+   */
+  return (node, _, state, info) => {
+    const exit = state.enter('html');
+    const tracker = state.createTracker(info);
+    let value = tracker.move(tagOpen);
+    value += tracker.move(
+      state.containerPhrasing(node, {
+        before: value,
+        after: tagOpen,
+        ...tracker.current(),
+      }),
+    );
+    value += tracker.move(tagClose);
+    exit();
+    return value;
+  };
+}
+
+function toMarkdown() {
+  return {
+    handlers: {
+      subscript: format('sub'),
+      superscript: format('sup'),
+      underline: format('u'),
+    },
+  };
+}
+
+export default function formatPlugin(options) {
+  const data = this.data();
+
+  function add(field, value) {
+    /* c8 ignore next 2 */
+    if (data[field]) {
+      data[field].push(value);
+    } else {
+      data[field] = [value];
+    }
+  }
+
+  // add('micromarkExtensions', syntax(options));
+  // add('fromMarkdownExtensions', fromMarkdown(options));
+  add('toMarkdownExtensions', toMarkdown(options));
+}

package/test/importers/PageImporter.spec.js

@@ -21,6 +21,11 @@ import { dirname } from 'dirname-filename-esm';
 
 import { docx2md } from '@adobe/helix-docx2md';
 
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGridTable from '@adobe/remark-gridtables';
+// eslint-disable-next-line no-unused-vars
+import { inspect, inspectNoColor } from 'unist-util-inspect';
 import PageImporter from '../../src/importer/PageImporter.js';
 import PageImporterResource from '../../src/importer/PageImporterResource.js';
 import MemoryHandler from '../../src/storage/MemoryHandler.js';
@@ -148,6 +153,22 @@ describe('PageImporter tests - fixtures', () => {
     const md = await storageHandler.get(results[0].md);
     const expectedMD = await fs.readFile(path.resolve(__dirname, 'fixtures', `${feature}.spec.md`), 'utf-8');
     strictEqual(md.trim(), expectedMD.trim(), 'imported md is expected one');
+
+    // parse md to verify mdast
+    const mdast = unified()
+      .use(remarkParse)
+      .use(remarkGridTable)
+      .use()
+      .parse(md);
+
+    // process.stdout.write(inspect(mdast, { showPositions: false }));
+    // process.stdout.write('\n');
+
+    if (await fs.pathExistsSync(path.resolve(__dirname, 'fixtures', `${feature}.spec.mdast`))) {
+      const actualMdast = inspectNoColor(mdast, { showPositions: false });
+      const expectedMdast = await fs.readFile(path.resolve(__dirname, 'fixtures', `${feature}.spec.mdast`), 'utf-8');
+      strictEqual(actualMdast.trim(), expectedMdast.trim(), 'imported mdast is expected one');
+    }
   };
 
   it('import - tables', async () => {

package/test/importers/fixtures/em.spec.html

@@ -4,5 +4,6 @@
     <p><strong><em>&nbsp;</em></strong></p><p>usefull</p>
     <p><strong><em>emphasis</em></strong><strong><em> space </em></strong><strong><em>another emphasis</em></strong> <strong><em>last emphasis</em></strong></p>
     <p><a href="https://www.sample.com">linkcontent</a><i>. </i></p>
+    <p><strong><em>Side shuffle </em></strong><em>– </em>Quadriceps, glutes, hamstrings, calves</p>
   </body>
-</html>
+</html>

package/test/importers/fixtures/em.spec.md

@@ -2,6 +2,8 @@
 
 usefull
 
-***emphasis* *space* *another emphasis* *last emphasis***
+**_emphasis_ _space_ _another emphasis_ _last emphasis_**
 
-[linkcontent](https://www.sample.com).
+[linkcontent](https://www.sample.com).
+
+**_Side shuffle_** _–_ Quadriceps, glutes, hamstrings, calves

package/test/importers/fixtures/em.spec.mdast

@@ -0,0 +1,33 @@
+root[5]
+├─0 heading[1]
+│   │ depth: 1
+│   └─0 text "EM sample"
+├─1 paragraph[1]
+│   └─0 text "usefull"
+├─2 paragraph[1]
+│   └─0 strong[7]
+│       ├─0 emphasis[1]
+│       │   └─0 text "emphasis"
+│       ├─1 text " "
+│       ├─2 emphasis[1]
+│       │   └─0 text "space"
+│       ├─3 text " "
+│       ├─4 emphasis[1]
+│       │   └─0 text "another emphasis"
+│       ├─5 text " "
+│       └─6 emphasis[1]
+│           └─0 text "last emphasis"
+├─3 paragraph[2]
+│   ├─0 link[1]
+│   │   │ title: null
+│   │   │ url: "https://www.sample.com"
+│   │   └─0 text "linkcontent"
+│   └─1 text "."
+└─4 paragraph[4]
+    ├─0 strong[1]
+    │   └─0 emphasis[1]
+    │       └─0 text "Side shuffle"
+    ├─1 text " "
+    ├─2 emphasis[1]
+    │   └─0 text "–"
+    └─3 text " Quadriceps, glutes, hamstrings, calves"

package/test/importers/fixtures/space.spec.md

@@ -11,9 +11,6 @@ A paragraph with a br at the end and \  " ".
 
 A paragraph followed by a br
 
-\
-
-
 A paragraph after the br
 
-A paragraph after the nbsp;
+A paragraph after the nbsp;

package/test/importers/fixtures/u.spec.md

@@ -8,20 +8,14 @@ Some normal text with random <u>underline</u> or <u>span with underline</u> or <
 
 **<u>Underline 3</u>**
 
--   <u>
-    li underline 1
-    </u>
--   <u>
-    li underline 2
-    </u>
+-   <u>li underline 1</u>
+-   <u>li underline 2</u>
     also may have text here
--   <u>
-    li underline 3
-    </u>
+-   <u>li underline 3</u>
 
 [Unlined link](https:/www.sample.com/a) or [<u>Linked underline</u>](https:/www.sample.com/b) ?
 
-<u>**Some underline and strong text**</u>
+**<u>Some underline and strong text</u>**
 
 [U and A are not friends](https://www.austinparks.org/)[Boys & Girls Clubs of the Austin Area](http://www.bgcaustin.org/)[The First Tee of Greater Austin](http://www.thefirstteeaustin.org/club/scripts/public/public.asp)
 

package/test/importers/fixtures/u.spec.mdast

@@ -0,0 +1,102 @@
+root[10]
+├─0 heading[1]
+│   │ depth: 1
+│   └─0 text "Underline combo sample"
+├─1 paragraph[3]
+│   ├─0 html "<u>"
+│   ├─1 text "Underline 1"
+│   └─2 html "</u>"
+├─2 paragraph[13]
+│   ├─0  text "Some normal text with random "
+│   ├─1  html "<u>"
+│   ├─2  text "underline"
+│   ├─3  html "</u>"
+│   ├─4  text " or "
+│   ├─5  html "<u>"
+│   ├─6  text "span with underline"
+│   ├─7  html "</u>"
+│   ├─8  text " or "
+│   ├─9  html "<u>"
+│   ├─10 text "underline with span"
+│   ├─11 html "</u>"
+│   └─12 text "..."
+├─3 paragraph[1]
+│   └─0 strong[3]
+│       ├─0 html "<u>"
+│       ├─1 text "Underline 2"
+│       └─2 html "</u>"
+├─4 paragraph[1]
+│   └─0 strong[3]
+│       ├─0 html "<u>"
+│       ├─1 text "Underline 3"
+│       └─2 html "</u>"
+├─5 list[3]
+│   │ ordered: false
+│   │ start: null
+│   │ spread: false
+│   ├─0 listItem[1]
+│   │   │ spread: false
+│   │   │ checked: null
+│   │   └─0 paragraph[3]
+│   │       ├─0 html "<u>"
+│   │       ├─1 text "li underline 1"
+│   │       └─2 html "</u>"
+│   ├─1 listItem[1]
+│   │   │ spread: false
+│   │   │ checked: null
+│   │   └─0 paragraph[4]
+│   │       ├─0 html "<u>"
+│   │       ├─1 text "li underline 2"
+│   │       ├─2 html "</u>"
+│   │       └─3 text "\nalso may have text here"
+│   └─2 listItem[1]
+│       │ spread: false
+│       │ checked: null
+│       └─0 paragraph[3]
+│           ├─0 html "<u>"
+│           ├─1 text "li underline 3"
+│           └─2 html "</u>"
+├─6 paragraph[4]
+│   ├─0 link[1]
+│   │   │ title: null
+│   │   │ url: "https:/www.sample.com/a"
+│   │   └─0 text "Unlined link"
+│   ├─1 text " or "
+│   ├─2 link[3]
+│   │   │ title: null
+│   │   │ url: "https:/www.sample.com/b"
+│   │   ├─0 html "<u>"
+│   │   ├─1 text "Linked underline"
+│   │   └─2 html "</u>"
+│   └─3 text " ?"
+├─7 paragraph[1]
+│   └─0 strong[3]
+│       ├─0 html "<u>"
+│       ├─1 text "Some underline and strong text"
+│       └─2 html "</u>"
+├─8 paragraph[3]
+│   ├─0 link[1]
+│   │   │ title: null
+│   │   │ url: "https://www.austinparks.org/"
+│   │   └─0 text "U and A are not friends"
+│   ├─1 link[1]
+│   │   │ title: null
+│   │   │ url: "http://www.bgcaustin.org/"
+│   │   └─0 text "Boys & Girls Clubs of the Austin Area"
+│   └─2 link[1]
+│       │ title: null
+│       │ url: "http://www.thefirstteeaustin.org/club/scripts/public/public.asp"
+│       └─0 text "The First Tee of Greater Austin"
+└─9 paragraph[3]
+    ├─0 link[1]
+    │   │ title: null
+    │   │ url: "https://www.austinparks.org/"
+    │   └─0 text "U and A are not friends"
+    ├─1 link[1]
+    │   │ title: null
+    │   │ url: "http://www.bgcaustin.org/"
+    │   └─0 text "Boys & Girls Clubs of the Austin Area"
+    └─2 link[1]
+        │ title: null
+        │ url: "http://www.thefirstteeaustin.org/club/scripts/public/public.asp"
+        └─0 text "The First Tee of Greater Austin"