@adobe/helix-html-pipeline 1.1.1 → 1.2.0

package/CHANGELOG.md

@@ -1,3 +1,24 @@
+# [1.2.0](https://github.com/adobe/helix-html-pipeline/compare/v1.1.3...v1.2.0) (2022-03-16)
+
+
+### Features
+
+* use hast instead of jsdom ([#12](https://github.com/adobe/helix-html-pipeline/issues/12)) ([bee0a0b](https://github.com/adobe/helix-html-pipeline/commit/bee0a0b3309919f896520bc700dd2d867be19a1c)), closes [#11](https://github.com/adobe/helix-html-pipeline/issues/11)
+
+## [1.1.3](https://github.com/adobe/helix-html-pipeline/compare/v1.1.2...v1.1.3) (2022-03-12)
+
+
+### Bug Fixes
+
+* **deps:** update dependency @adobe/helix-shared-utils to v2.0.5 ([4ea15f9](https://github.com/adobe/helix-html-pipeline/commit/4ea15f9888486ba0e81e92c7796236726da5b74c))
+
+## [1.1.2](https://github.com/adobe/helix-html-pipeline/compare/v1.1.1...v1.1.2) (2022-03-11)
+
+
+### Bug Fixes
+
+* handling invalid input url with 400 ([#16](https://github.com/adobe/helix-html-pipeline/issues/16)) ([4491691](https://github.com/adobe/helix-html-pipeline/commit/449169107cc1d6a3b7b5fd211b39174d59fb2a8e)), closes [#15](https://github.com/adobe/helix-html-pipeline/issues/15)
+
 ## [1.1.1](https://github.com/adobe/helix-html-pipeline/compare/v1.1.0...v1.1.1) (2022-03-10)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-html-pipeline",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "description": "Helix HTML Pipeline",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -33,10 +33,13 @@
   },
   "dependencies": {
     "@adobe/helix-markdown-support": "3.1.2",
-    "@adobe/helix-shared-utils": "2.0.4",
+    "@adobe/helix-shared-utils": "2.0.5",
     "github-slugger": "1.4.0",
+    "hast-util-raw": "7.2.1",
+    "hast-util-select": "5.0.1",
     "hast-util-to-html": "8.0.3",
-    "jsdom": "19.0.0",
+    "hast-util-to-string": "2.0.0",
+    "hastscript": "7.0.2",
     "mdast-util-gfm-footnote": "1.0.1",
     "mdast-util-gfm-strikethrough": "1.0.1",
     "mdast-util-gfm-table": "1.0.3",
@@ -50,19 +53,21 @@
     "micromark-extension-gfm-task-list-item": "1.0.3",
     "micromark-util-combine-extensions": "1.0.0",
     "mime": "3.0.0",
-    "property-information": "6.1.1",
+    "rehype-format": "4.0.1",
+    "rehype-minify-whitespace": "5.0.0",
+    "rehype-parse": "8.0.4",
     "remark-parse": "10.0.1",
     "strip-markdown": "5.0.0",
-    "unified": "10.1.1",
+    "unified": "10.1.2",
     "unist-util-map": "3.0.0",
+    "unist-util-remove": "3.1.0",
     "unist-util-remove-position": "4.0.1",
     "unist-util-select": "4.0.1",
-    "unist-util-visit": "4.1.0",
-    "uri-js": "4.4.1"
+    "unist-util-visit": "4.1.0"
   },
   "devDependencies": {
     "@adobe/eslint-config-helix": "1.3.2",
-    "@markedjs/html-differ": "4.0.0",
+    "@markedjs/html-differ": "4.0.1",
     "@semantic-release/changelog": "6.0.1",
     "@semantic-release/git": "10.0.1",
     "@semantic-release/npm": "9.0.1",
@@ -71,23 +76,20 @@
     "codecov": "3.8.3",
     "commitizen": "4.2.4",
     "cz-conventional-changelog": "3.3.0",
-    "eslint": "8.10.0",
+    "eslint": "8.11.0",
     "eslint-plugin-header": "3.1.1",
     "eslint-plugin-import": "2.25.4",
     "esmock": "1.7.4",
-    "hastscript": "7.0.2",
     "husky": "7.0.4",
-    "hyperscript": "2.0.2",
     "js-yaml": "4.1.0",
     "jsdoc-to-markdown": "7.1.1",
+    "jsdom": "19.0.0",
     "junit-report-builder": "3.0.0",
     "lint-staged": "12.3.5",
-    "mocha": "9.2.1",
+    "mocha": "9.2.2",
     "mocha-multi-reporters": "1.5.1",
     "remark-gfm": "3.0.1",
-    "semantic-release": "19.0.2",
-    "sinon": "13.0.1",
-    "unist-builder": "3.0.0"
+    "semantic-release": "19.0.2"
   },
   "lint-staged": {
     "*.js": "eslint",

package/src/PipelineContent.d.ts

@@ -11,6 +11,7 @@
  */
 import {Node} from "unist";
 import GithubSlugger from 'github-slugger';
+import { Root } from 'hast';
 
 declare enum SourceType {
   CONTENT = 'content',
@@ -50,12 +51,9 @@ declare class PipelineContent {
   mdast: Node;
 
   /**
-   * document specific metadata
+   * The transformed document (hast) representation
    */
-  meta: object;
-  title: string;
-  intro: string;
-  image: string;
+  hast: Root;
 
   /**
    * slugger to use for heading id calculations
@@ -63,7 +61,10 @@ declare class PipelineContent {
   slugger: GithubSlugger;
 
   /**
-   * The transformed document (jsom) representation
+   * document specific metadata
    */
-  document: Document;
+  meta: object;
+  title: string;
+  intro: string;
+  image: string;
 }

package/src/PipelineContent.js

@@ -9,6 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import GithubSlugger from 'github-slugger';
 
 /**
  * State of the pipeline
@@ -21,6 +22,7 @@ export class PipelineContent {
   constructor() {
     Object.assign(this, {
       sourceBus: 'content',
+      slugger: new GithubSlugger(),
     });
   }
 }

package/src/PipelineResponse.d.ts

@@ -9,6 +9,8 @@
  * OF ANY KIND; either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { Element } from 'hast';
+
 declare interface PipelineResponseInit {
   status?: number;
   headers: Map<string, string> | object;
@@ -17,7 +19,10 @@ declare interface PipelineResponseInit {
 declare class PipelineResponse {
   constructor(body?:string, init?:PipelineResponseInit);
   status: number;
-  document?: Document;
+  /**
+   * The transformed document (hast) representation
+   */
+  document: Element;
   body: string;
   headers: Map<string, string>;
   error: any;

package/src/html-pipe.js

@@ -62,11 +62,14 @@ export async function htmlPipe(state, req) {
   });
 
   try { // fetch config first, since we need to compute the content-bus-id from the fstab ...
+    state.timer?.update('config-fetch');
     await fetchConfig(state, req, res);
+
     // ...and apply the folder mapping
     await folderMapping(state, req, res);
 
     // load metadata and content in parallel
+    state.timer?.update('content-fetch');
     await Promise.all([
       fetchMetadata(state, req, res),
       fetchContent(state, req, res),
@@ -80,9 +83,12 @@ export async function htmlPipe(state, req) {
     }
 
     if (state.content.sourceBus === 'code') {
+      state.timer?.update('serialize');
       await renderCode(state, req, res);
     } else {
+      state.timer?.update('parse');
       await parseMarkdown(state);
+      state.timer?.update('render');
       await splitSections(state);
       await getMetadata(state); // this one extracts the metadata from the mdast
       await unwrapSoleImages(state);
@@ -96,6 +102,7 @@ export async function htmlPipe(state, req) {
       await addHeadingIds(state);
       await render(state, req, res);
       await removeHlxProps(state, req, res);
+      state.timer?.update('serialize');
       await tohtml(state, req, res);
     }
 
@@ -110,6 +117,13 @@ export async function htmlPipe(state, req) {
     }
     log.error(`error running pipeline: ${res.status} ${res.error}`, e);
     res.headers.set('x-error', cleanupHeaderValue(res.error));
+
+    // turn any URL errors into a 400, since they are user input
+    // see https://github.com/adobe/helix-pipeline-service/issues/346
+    if (e.code === 'ERR_INVALID_URL') {
+      res.status = 400;
+      res.headers.set('x-error', cleanupHeaderValue(`invalid url: ${e.input}`));
+    }
   }
 
   return res;

package/src/steps/add-heading-ids.js

@@ -9,6 +9,8 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { toString } from 'hast-util-to-string';
+import { visit } from 'unist-util-visit';
 
 /**
  * Adds missing `id` attributes to the headings
@@ -16,17 +18,16 @@
  * @param {PipelineContent } content The current context of processing pipeline
  */
 export default async function fixSections({ content }) {
-  const { slugger, document } = content;
-  ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
-    .forEach((tagName) => {
-      document.querySelectorAll(tagName)
-        .forEach(($h) => {
-          if (!$h.id) {
-            const text = $h.textContent.trim();
-            if (text) {
-              $h.setAttribute('id', slugger.slug(text));
-            }
-          }
-        });
-    });
+  const { slugger, hast } = content;
+  visit(hast, (node) => {
+    if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(node.tagName)) {
+      const { properties } = node;
+      if (!properties.id) {
+        const text = toString(node).trim();
+        if (text) {
+          properties.id = slugger.slug(text);
+        }
+      }
+    }
+  });
 }

package/src/steps/create-page-blocks.js

@@ -9,57 +9,57 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { h } from 'hastscript';
+import { selectAll, select } from 'hast-util-select';
+import { toString } from 'hast-util-to-string';
 import { toClassName } from './utils.js';
+import { replace, childNodes } from '../utils/hast-utils.js';
 
 /**
  * Creates a "DIV representation" of a table.
  * @type PipelineStep
- * @param {Document} document
- * @param {HTMLTableElement} $table the table element
+ * @param {HTMLTTableElement} $table the table element
  * @returns {HTMLDivElement} the resulting div
  */
-function tableToDivs(document, $table) {
-  const $cards = document.createElement('div');
-
-  // iterate over the table to avoid problem with query selector and nested tables
+function tableToDivs($table) {
+  const $cards = h('div');
   const $rows = [];
-  if ($table.tHead) {
-    $rows.push(...$table.tHead.rows);
-  }
-  for (const $tbody of $table.tBodies) {
-    $rows.push(...$tbody.rows);
+  for (const child of $table.children) {
+    if (child.tagName === 'thead' || child.tagName === 'tbody') {
+      $rows.push(...childNodes(child));
+    }
   }
+
   if ($rows.length === 0) {
     return $cards;
   }
-  const $headerRow = $rows.shift();
+  const $headerCols = childNodes($rows.shift());
 
   // special case, only 1 row and 1 column with a nested table
-  if ($rows.length === 0 && $headerRow.cells.length === 1) {
-    const $nestedTable = $headerRow.cells[0].querySelector(':scope table');
+  if ($rows.length === 0 && $headerCols.length === 1) {
+    const $nestedTable = select(':scope table', $headerCols[0]);
     if ($nestedTable) {
       return $nestedTable;
     }
   }
 
   // get columns names
-  const clazz = Array.from($headerRow.cells)
-    .map((e) => toClassName(e.textContent))
+  const clazz = $headerCols
+    .map((e) => toClassName(toString(e)))
     .filter((c) => !!c)
     .join('-');
   if (clazz) {
-    $cards.classList.add(clazz);
+    $cards.properties.className = [clazz];
   }
 
   // construct page block
   for (const $row of $rows) {
-    const $card = document.createElement('div');
-    for (const $cell of $row.cells) {
-      const $div = document.createElement('div');
-      $div.append(...$cell.childNodes);
-      $card.append($div);
+    const $card = h('div');
+    for (const $cell of childNodes($row)) {
+      // convert to div
+      $card.children.push(h('div', $cell.children));
     }
-    $cards.append($card);
+    $cards.children.push($card);
   }
   return $cards;
 }
@@ -70,9 +70,10 @@ function tableToDivs(document, $table) {
  * @param context The current context of processing pipeline
  */
 export default function createPageBlocks({ content }) {
-  const { document } = content;
-  document.querySelectorAll('body > div > table').forEach(($table) => {
-    const $div = tableToDivs(document, $table);
-    $table.parentNode.replaceChild($div, $table);
+  const { hast } = content;
+  selectAll('div > table', hast).forEach(($table) => {
+    const $div = tableToDivs($table);
+    // replace child in parent
+    replace(hast, $table, $div);
   });
 }

package/src/steps/create-pictures.js

@@ -9,6 +9,9 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { h } from 'hastscript';
+import { selectAll } from 'hast-util-select';
+import { replace } from '../utils/hast-utils.js';
 import { optimizeImageURL } from './utils.js';
 
 /**
@@ -17,19 +20,20 @@ import { optimizeImageURL } from './utils.js';
  * @param context The current context of processing pipeline
  */
 export default async function createPictures({ content }) {
-  const { document } = content;
+  const { hast } = content;
 
   // transform <img> to <picture>
-  document.querySelectorAll('img[src^="./media_"]').forEach((img, i) => {
-    const picture = document.createElement('picture');
-    const source = document.createElement('source');
-    const src = img.getAttribute('src');
-    source.setAttribute('media', '(max-width: 400px)');
-    source.setAttribute('srcset', optimizeImageURL(src, 750));
-    picture.appendChild(source);
-    img.setAttribute('loading', i > 0 ? 'lazy' : 'eager'); // load all but first image lazy
-    img.setAttribute('src', optimizeImageURL(src, 2000));
-    img.parentNode.insertBefore(picture, img);
-    picture.appendChild(img);
+  selectAll('img[src^="./media_"]', hast).forEach((img, i) => {
+    const { src } = img.properties;
+    const source = h('source');
+    source.properties.media = '(max-width: 400px)';
+    source.properties.srcset = optimizeImageURL(src, 750);
+
+    const picture = h('picture', source);
+    img.properties.loading = i > 0 ? 'lazy' : 'eager';
+    img.properties.src = optimizeImageURL(src, 2000);
+
+    replace(hast, img, picture);
+    picture.children.push(img);
   });
 }

package/src/steps/extract-metadata.js

@@ -9,10 +9,15 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { selectAll, select } from 'hast-util-select';
+import { toString } from 'hast-util-to-string';
+import { remove } from 'unist-util-remove';
+import { visit, EXIT, CONTINUE } from 'unist-util-visit';
 import {
   getAbsoluteUrl, makeCanonicalHtmlUrl, optimizeImageURL, resolveUrl,
 } from './utils.js';
 import { filterGlobalMetadata, toMetaName, ALLOWED_RESPONSE_HEADERS } from '../utils/metadata.js';
+import { childNodes } from '../utils/hast-utils.js';
 
 /**
  * Cleans up comma-separated string lists and returns an array.
@@ -28,55 +33,53 @@ function toList(list) {
 
 /**
  * Returns the config from a block element as object with key/value pairs.
- * @param {HTMLDivElement} $block The block element
+ * @param {Element} $block The block element
  * @returns {object} The block config
  */
 function readBlockConfig($block) {
   const config = {};
-  $block.querySelectorAll(':scope>div').forEach(($row) => {
-    if ($row.children && $row.children[1]) {
-      const name = toMetaName($row.children[0].textContent);
+  selectAll(':scope>div', $block).forEach(($row) => {
+    if ($row?.children[1]) {
+      const [$name, $value] = $row.children;
+      const name = toMetaName(toString($name));
       if (name) {
         let value;
-        if ($row.children[1].hasChildNodes() && $row.children[1].firstElementChild) {
+        const $firstChild = childNodes($value)[0];
+        if ($firstChild) {
           // check for multiple paragraph or a list
-          let childNodes;
-          const { tagName } = $row.children[1].firstElementChild;
-          if (tagName === 'P') {
+          let list;
+          const { tagName } = $firstChild;
+          if (tagName === 'p') {
             // contains a list of <p> paragraphs
-            childNodes = $row.children[1].childNodes;
-          } else if (tagName === 'UL' || tagName === 'OL') {
+            list = childNodes($value);
+          } else if (tagName === 'ul' || tagName === 'ol') {
             // contains a list
-            childNodes = $row.children[1].children[0].childNodes;
+            list = childNodes($firstChild);
           }
 
-          if (childNodes) {
-            value = '';
-            childNodes.forEach((child) => {
-              value += `${child.textContent}, `;
-            });
-            value = value.substring(0, value.length - 2);
+          if (list) {
+            value = list.map((child) => toString(child)).join(', ');
           }
         }
 
         if (!value) {
           // for text content only
-          value = $row.children[1].textContent.trim().replace(/ {3}/g, ',');
+          value = toString($value).trim().replace(/ {3}/g, ',');
         }
 
         if (!value) {
           // check for value inside link
-          const $a = $row.children[1].querySelector('a');
+          const $a = select('a', $value);
           if ($a) {
-            value = $a.getAttribute('href');
+            value = $a.properties.href;
           }
         }
         if (!value) {
           // check for value inside img
-          const $img = $row.children[1].querySelector('img');
+          const $img = select('img', $value);
           if ($img) {
             // strip query string
-            value = $img.getAttribute('src');
+            value = $img.properties.src;
           }
         }
         if (value) {
@@ -91,15 +94,17 @@ function readBlockConfig($block) {
 
 /**
  * Looks for metadata in the document.
- * @param {HTMLDocument} document The document
+ * @param {Root} document The hast document
  * @return {object} The metadata
  */
 function getLocalMetadata(document) {
   let metaConfig = {};
-  const metaBlock = document.querySelector('body div.metadata');
+  const metaBlock = select('div.metadata', document);
   if (metaBlock) {
     metaConfig = readBlockConfig(metaBlock);
-    metaBlock.remove();
+    // TODO: here we should also remove the parent div of the former table, otherwise it results
+    // TODO: in an empty <div></div>
+    remove(document, { cascade: false }, metaBlock);
   }
   return metaConfig;
 }
@@ -118,6 +123,27 @@ function optimizeMetaImage(pagePath, imgUrl) {
   return src;
 }
 
+/**
+ * Extracts the description from the document. note, that the selectAll('div > p') used in
+ * jsdom doesn't work as expected in hast
+ * @param {Root} hast
+ * @see https://github.com/syntax-tree/unist/discussions/66
+ */
+function extractDescription(hast) {
+  let desc = '';
+  visit(hast, (node, idx, parent) => {
+    if (parent?.tagName === 'div' && node.tagName === 'p') {
+      const words = toString(node).trim().split(/\s+/);
+      if (words.length >= 10 || words.some((w) => w.length > 25 && !w.startsWith('http'))) {
+        desc = `${words.slice(0, 25).join(' ')}${words.length > 25 ? ' ...' : ''}`;
+        return EXIT;
+      }
+    }
+    return CONTINUE;
+  });
+  return desc;
+}
+
 /**
  * Extracts the metadata and stores it in the content meta
  * @type PipelineStep
@@ -126,13 +152,13 @@ function optimizeMetaImage(pagePath, imgUrl) {
  */
 export default function extractMetaData(state, req) {
   const { content } = state;
-  const { meta, document } = content;
+  const { meta, hast } = content;
 
   // extract global metadata from spreadsheet, and overlay
   // with local metadata from document
   const metaConfig = Object.assign(
     filterGlobalMetadata(state.metadata, state.info.path),
-    getLocalMetadata(document),
+    getLocalMetadata(hast),
   );
 
   // first process supported metadata properties
@@ -173,25 +199,16 @@ export default function extractMetaData(state, req) {
   if (!meta.title) {
     // content.title is not correct if the h1 is in a page-block since the pipeline
     // only respects the heading nodes in the mdast
-    const $title = document.querySelector('body > div h1');
+    const $title = select('div h1', hast);
     if ($title) {
-      content.title = $title.textContent;
+      content.title = toString($title);
     }
     meta.title = content.title;
   }
   if (!meta.description) {
-    // description: text from paragraphs with 10 or more words
-    let desc = [];
-    document.querySelectorAll('div > p').forEach((p) => {
-      if (desc.length === 0) {
-        const words = p.textContent.trim().split(/\s+/);
-        if (words.length >= 10 || words.some((w) => w.length > 25 && !w.startsWith('http'))) {
-          desc = desc.concat(words);
-        }
-      }
-    });
-    meta.description = `${desc.slice(0, 25).join(' ')}${desc.length > 25 ? ' ...' : ''}`;
+    meta.description = extractDescription(hast);
   }
+
   // use the req.url and not the state.info.path in case of folder mapping
   meta.url = makeCanonicalHtmlUrl(getAbsoluteUrl(req.headers, req.url.pathname));
   if (!meta.canonical) {
@@ -200,11 +217,11 @@ export default function extractMetaData(state, req) {
 
   // content.image is not correct if the first image is in a page-block. since the pipeline
   // only respects the image nodes in the mdast
-  const $hero = document.querySelector('body > div img');
+  const $hero = select('div img', hast);
   if ($hero) {
-    content.image = $hero.src;
-    if ($hero.alt) {
-      content.imageAlt = $hero.alt;
+    content.image = $hero.properties.src;
+    if ($hero.properties.alt) {
+      content.imageAlt = $hero.properties.alt;
     }
   }
 

package/src/steps/fix-sections.js

@@ -9,6 +9,8 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { selectAll } from 'hast-util-select';
+import { h } from 'hastscript';
 import { wrapContent } from './utils.js';
 
 /**
@@ -17,20 +19,17 @@ import { wrapContent } from './utils.js';
  * @param {PipelineContent} content
  */
 export default async function fixSections({ content }) {
-  const { document } = content;
-  const $sections = document.querySelectorAll('body > div');
+  const { hast } = content;
+  const $sections = selectAll('div', hast);
 
   // if there are no sections wrap everything in a div with appropriate class names from meta
   if ($sections.length === 0) {
-    const $outerDiv = document.createElement('div');
+    const $outerDiv = h('div');
     if (content.meta && content.meta.class) {
-      content.meta.class.split(/[ ,]/)
+      $outerDiv.properties.className = content.meta.class.split(/[ ,]/)
         .map((c) => c.trim())
-        .filter((c) => !!c)
-        .forEach((c) => {
-          $outerDiv.classList.add(c);
-        });
+        .filter((c) => !!c);
     }
-    wrapContent($outerDiv, document.body);
+    wrapContent($outerDiv, hast);
   }
 }