@adobe/helix-html-pipeline 1.1.3 → 1.3.0

package/CHANGELOG.md

@@ -1,3 +1,24 @@
+# [1.3.0](https://github.com/adobe/helix-html-pipeline/compare/v1.2.1...v1.3.0) (2022-03-17)
+
+
+### Features
+
+* provide pipes for OPTIONS and POSTs ([#24](https://github.com/adobe/helix-html-pipeline/issues/24)) ([1dfc47e](https://github.com/adobe/helix-html-pipeline/commit/1dfc47e764a0b1d8acee80b51b845be2e54a16f5))
+
+## [1.2.1](https://github.com/adobe/helix-html-pipeline/compare/v1.2.0...v1.2.1) (2022-03-16)
+
+
+### Bug Fixes
+
+* reject double-slashes ([#22](https://github.com/adobe/helix-html-pipeline/issues/22)) ([5aee75d](https://github.com/adobe/helix-html-pipeline/commit/5aee75d4109550525d971c64d87e4f2420863c30)), closes [#20](https://github.com/adobe/helix-html-pipeline/issues/20)
+
+# [1.2.0](https://github.com/adobe/helix-html-pipeline/compare/v1.1.3...v1.2.0) (2022-03-16)
+
+
+### Features
+
+* use hast instead of jsdom ([#12](https://github.com/adobe/helix-html-pipeline/issues/12)) ([bee0a0b](https://github.com/adobe/helix-html-pipeline/commit/bee0a0b3309919f896520bc700dd2d867be19a1c)), closes [#11](https://github.com/adobe/helix-html-pipeline/issues/11)
+
 ## [1.1.3](https://github.com/adobe/helix-html-pipeline/compare/v1.1.2...v1.1.3) (2022-03-12)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-html-pipeline",
-  "version": "1.1.3",
+  "version": "1.3.0",
   "description": "Helix HTML Pipeline",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -35,8 +35,11 @@
     "@adobe/helix-markdown-support": "3.1.2",
     "@adobe/helix-shared-utils": "2.0.5",
     "github-slugger": "1.4.0",
+    "hast-util-raw": "7.2.1",
+    "hast-util-select": "5.0.1",
     "hast-util-to-html": "8.0.3",
-    "jsdom": "19.0.0",
+    "hast-util-to-string": "2.0.0",
+    "hastscript": "7.0.2",
     "mdast-util-gfm-footnote": "1.0.1",
     "mdast-util-gfm-strikethrough": "1.0.1",
     "mdast-util-gfm-table": "1.0.3",
@@ -50,15 +53,17 @@
     "micromark-extension-gfm-task-list-item": "1.0.3",
     "micromark-util-combine-extensions": "1.0.0",
     "mime": "3.0.0",
-    "property-information": "6.1.1",
+    "rehype-format": "4.0.1",
+    "rehype-minify-whitespace": "5.0.0",
+    "rehype-parse": "8.0.4",
     "remark-parse": "10.0.1",
     "strip-markdown": "5.0.0",
     "unified": "10.1.2",
     "unist-util-map": "3.0.0",
+    "unist-util-remove": "3.1.0",
     "unist-util-remove-position": "4.0.1",
     "unist-util-select": "4.0.1",
-    "unist-util-visit": "4.1.0",
-    "uri-js": "4.4.1"
+    "unist-util-visit": "4.1.0"
   },
   "devDependencies": {
     "@adobe/eslint-config-helix": "1.3.2",
@@ -75,19 +80,16 @@
     "eslint-plugin-header": "3.1.1",
     "eslint-plugin-import": "2.25.4",
     "esmock": "1.7.4",
-    "hastscript": "7.0.2",
     "husky": "7.0.4",
-    "hyperscript": "2.0.2",
     "js-yaml": "4.1.0",
     "jsdoc-to-markdown": "7.1.1",
+    "jsdom": "19.0.0",
     "junit-report-builder": "3.0.0",
     "lint-staged": "12.3.5",
     "mocha": "9.2.2",
     "mocha-multi-reporters": "1.5.1",
     "remark-gfm": "3.0.1",
-    "semantic-release": "19.0.2",
-    "sinon": "13.0.1",
-    "unist-builder": "3.0.0"
+    "semantic-release": "19.0.2"
   },
   "lint-staged": {
     "*.js": "eslint",

package/src/PipelineContent.d.ts

@@ -11,6 +11,7 @@
  */
 import {Node} from "unist";
 import GithubSlugger from 'github-slugger';
+import { Root } from 'hast';
 
 declare enum SourceType {
   CONTENT = 'content',
@@ -50,12 +51,9 @@ declare class PipelineContent {
   mdast: Node;
 
   /**
-   * document specific metadata
+   * The transformed document (hast) representation
    */
-  meta: object;
-  title: string;
-  intro: string;
-  image: string;
+  hast: Root;
 
   /**
    * slugger to use for heading id calculations
@@ -63,7 +61,10 @@ declare class PipelineContent {
   slugger: GithubSlugger;
 
   /**
-   * The transformed document (jsom) representation
+   * document specific metadata
    */
-  document: Document;
+  meta: object;
+  title: string;
+  intro: string;
+  image: string;
 }

package/src/PipelineContent.js

@@ -9,6 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import GithubSlugger from 'github-slugger';
 
 /**
  * State of the pipeline
@@ -21,6 +22,7 @@ export class PipelineContent {
   constructor() {
     Object.assign(this, {
       sourceBus: 'content',
+      slugger: new GithubSlugger(),
     });
   }
 }

package/src/PipelineResponse.d.ts

@@ -9,6 +9,8 @@
  * OF ANY KIND; either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { Element } from 'hast';
+
 declare interface PipelineResponseInit {
   status?: number;
   headers: Map<string, string> | object;
@@ -17,7 +19,10 @@ declare interface PipelineResponseInit {
 declare class PipelineResponse {
   constructor(body?:string, init?:PipelineResponseInit);
   status: number;
-  document?: Document;
+  /**
+   * The transformed document (hast) representation
+   */
+  document: Element;
   body: string;
   headers: Map<string, string>;
   error: any;

package/src/PipelineState.d.ts

@@ -9,12 +9,13 @@
  * OF ANY KIND; either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import {PathInfo, S3Loader, PipelineTimer} from "./index";
+import {PathInfo, S3Loader, FormsMessageDispatcher, PipelineTimer} from "./index";
 import {PipelineContent} from "./PipelineContent";
 
 declare interface PipelineOptions {
   log: Console;
   s3Loader: S3Loader;
+  messageDispatcher: FormsMessageDispatcher;
   owner: string;
   repo: string;
   ref: string;
@@ -31,6 +32,7 @@ declare class PipelineState {
   content: PipelineContent;
   contentBusId: string;
   s3Loader: S3Loader;
+  messageDispatcher: FormsMessageDispatcher;
 
   /**
    * Content bus partition

package/src/PipelineState.js

@@ -36,6 +36,7 @@ export class PipelineState {
       helixConfig: undefined,
       metadata: undefined,
       s3Loader: opts.s3Loader,
+      messageDispatcher: opts.messageDispatcher,
       timer: opts.timer,
     });
   }

package/src/forms-pipe.js

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { cleanupHeaderValue } from '@adobe/helix-shared-utils';
+import { PipelineResponse } from './PipelineResponse.js';
+import fetchMetadata from './steps/fetch-metadata.js';
+import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+import { getOriginalHost } from './steps/utils.js';
+
+function error(log, msg, status, response) {
+  log.error(msg);
+  response.status = status;
+  response.headers.set('x-error', cleanupHeaderValue(msg));
+  return response;
+}
+
+/**
+ * Converts URLSearchParams to an object
+ * @param {URLSearchParams} searchParams the search params object
+ * @returns {Object} The converted object
+ */
+function searchParamsToObject(searchParams) {
+  const result = {};
+
+  for (const key of searchParams.keys()) {
+    // get all values association with the key
+    const values = searchParams.getAll(key);
+
+    // if multiple values, convert to array
+    result[key] = (values.length === 1) ? values[0] : values;
+  }
+
+  return result;
+}
+
+/**
+ * Extracts and parses the body data from the request
+ * @param {PipelineRequest} request the request object (see fetch api)
+ * @returns {Object} The body data
+ * @throws {Error} If an error occurs parsing the body
+ */
+export async function extractBodyData(request) {
+  let { body } = request;
+  if (!body) {
+    throw Error('missing body');
+  }
+  const type = request.headers.get('content-type');
+
+  // if content is form-urlencoded we need place the object in the body
+  // in a "data" property in the body as this is what forms-service expects.
+  if (/^application\/x-www-form-urlencoded/.test(type)) {
+    // did they pass an object in the body when a form-urlencoded body was expected?
+    if (body === '[object Object]') {
+      throw Error('invalid form-urlencoded body');
+    }
+
+    body = {
+      data: searchParamsToObject(new URLSearchParams(body)),
+    };
+
+    // else treat the body as json
+  } else if (/^application\/json/.test(type)) {
+    body = JSON.parse(body);
+    // verify the body data is as expected
+    if (!body.data) {
+      throw Error('missing body.data');
+    }
+  } else {
+    throw Error(`post body content-type not supported: ${type}`);
+  }
+  return body;
+}
+
+/**
+ * Handle a pipeline POST request.
+ * At this point POST's only apply to json files that are backed by a workbook.
+ * @param {PipelineState} state pipeline options
+ * @param {PipelineRequest} request
+ * @returns {Promise<PipelineResponse>} a response
+ */
+export async function formsPipe(state, request) {
+  const { log } = state;
+
+  // todo: improve
+  const response = new PipelineResponse('', {
+    headers: {
+      'content-type': 'text/plain; charset=utf-8',
+    },
+  });
+  await fetchMetadata(state, request, response);
+  await setCustomResponseHeaders(state, request, response);
+
+  const {
+    owner, repo, ref, contentBusId, partition, s3Loader,
+  } = state;
+  const { path } = state.info;
+  const resourcePath = `${path}.json`;
+
+  // block all POSTs to resources with extensions
+  if (state.info.originalExtension !== '') {
+    return error(log, 'POST to URL with extension not allowed', 405, response);
+  }
+
+  // head workbook in content bus
+  const resourceFetchResponse = await s3Loader.headObject('helix-content-bus', `${contentBusId}/${partition}${resourcePath}`);
+  if (resourceFetchResponse.status !== 200) {
+    return resourceFetchResponse;
+  }
+
+  let body;
+  try {
+    body = await extractBodyData(request);
+  } catch (err) {
+    return error(log, err.message, 400, response);
+  }
+
+  const sheets = resourceFetchResponse.headers.get('x-amz-meta-x-sheet-names');
+  if (!sheets) {
+    return error(log, `Target workbook at ${resourcePath} missing x-sheet-names header.`, 403, response);
+  }
+
+  const sourceLocation = resourceFetchResponse.headers.get('x-amz-meta-x-source-location');
+  const referer = request.headers.get('referer') || 'unknown';
+  const sheetNames = sheets.split(',');
+
+  if (!sourceLocation || !sheetNames.includes('incoming')) {
+    return error(log, `Target workbook at ${resourcePath} is not setup to intake data.`, 403, response);
+  }
+
+  // Send message to SQS if workbook contains and incoming
+  // sheet and the source location is not null
+  const host = getOriginalHost(request.headers);
+
+  const message = {
+    url: `https://${ref}--${repo}--${owner}.hlx.live${resourcePath}`,
+    body,
+    host,
+    sourceLocation,
+    referer,
+  };
+
+  try {
+    // Send message to forms queue
+    const { requestId, messageId } = await state.messageDispatcher.dispatch(message);
+    response.status = 201;
+    response.headers.set('x-request-id', requestId);
+    response.headers.set('x-message-id', messageId);
+    return response;
+  } catch (err) {
+    return error(log, `Failed to send message to forms queue: ${err}`, 500, response);
+  }
+}

package/src/html-pipe.js

@@ -62,11 +62,14 @@ export async function htmlPipe(state, req) {
   });
 
   try { // fetch config first, since we need to compute the content-bus-id from the fstab ...
+    state.timer?.update('config-fetch');
     await fetchConfig(state, req, res);
+
     // ...and apply the folder mapping
     await folderMapping(state, req, res);
 
     // load metadata and content in parallel
+    state.timer?.update('content-fetch');
     await Promise.all([
       fetchMetadata(state, req, res),
       fetchContent(state, req, res),
@@ -80,9 +83,12 @@ export async function htmlPipe(state, req) {
     }
 
     if (state.content.sourceBus === 'code') {
+      state.timer?.update('serialize');
       await renderCode(state, req, res);
     } else {
+      state.timer?.update('parse');
       await parseMarkdown(state);
+      state.timer?.update('render');
       await splitSections(state);
       await getMetadata(state); // this one extracts the metadata from the mdast
       await unwrapSoleImages(state);
@@ -96,6 +102,7 @@ export async function htmlPipe(state, req) {
       await addHeadingIds(state);
       await render(state, req, res);
       await removeHlxProps(state, req, res);
+      state.timer?.update('serialize');
       await tohtml(state, req, res);
     }
 

package/src/index.d.ts

@@ -83,6 +83,19 @@ declare interface S3Loader {
   headObject(bucketId, key): Promise<PipelineResponse>;
 }
 
+declare interface DispatchMessageResponse {
+  messageId:string,
+  requestId:string,
+}
+
+declare interface FormsMessageDispatcher {
+  /**
+   * Dispatches the message to the forms queue
+   * @param {object} message
+   */
+  dispatch(message:object): Promise<DispatchMessageResponse>;
+}
+
 /**
  * Timer
  */

package/src/index.js

@@ -11,6 +11,8 @@
  */
 export * from './html-pipe.js';
 export * from './json-pipe.js';
+export * from './options-pipe.js';
+export * from './forms-pipe.js';
 export * from './PipelineContent.js';
 export * from './PipelineRequest.js';
 export * from './PipelineResponse.js';

package/src/options-pipe.js

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { PipelineResponse } from './PipelineResponse.js';
+import fetchMetadata from './steps/fetch-metadata.js';
+import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+
+/**
+ * Handles options requests
+ * @param {PipelineState} state pipeline options
+ * @param {PipelineRequest} request
+ * @returns {Response} a response
+ */
+export async function optionsPipe(state, request) {
+  // todo: improve
+  const response = new PipelineResponse('', {
+    status: 204,
+    headers: {
+      // Set preflight cache duration
+      'access-control-max-age': '86400',
+      // Allow content type header
+      'access-control-allow-headers': 'content-type',
+    },
+  });
+  await fetchMetadata(state, request, response);
+  await setCustomResponseHeaders(state, request, response);
+
+  return response;
+}

package/src/steps/add-heading-ids.js

@@ -9,6 +9,8 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { toString } from 'hast-util-to-string';
+import { visit } from 'unist-util-visit';
 
 /**
  * Adds missing `id` attributes to the headings
@@ -16,17 +18,16 @@
  * @param {PipelineContent } content The current context of processing pipeline
  */
 export default async function fixSections({ content }) {
-  const { slugger, document } = content;
-  ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
-    .forEach((tagName) => {
-      document.querySelectorAll(tagName)
-        .forEach(($h) => {
-          if (!$h.id) {
-            const text = $h.textContent.trim();
-            if (text) {
-              $h.setAttribute('id', slugger.slug(text));
-            }
-          }
-        });
-    });
+  const { slugger, hast } = content;
+  visit(hast, (node) => {
+    if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(node.tagName)) {
+      const { properties } = node;
+      if (!properties.id) {
+        const text = toString(node).trim();
+        if (text) {
+          properties.id = slugger.slug(text);
+        }
+      }
+    }
+  });
 }

package/src/steps/create-page-blocks.js

@@ -9,57 +9,57 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { h } from 'hastscript';
+import { selectAll, select } from 'hast-util-select';
+import { toString } from 'hast-util-to-string';
 import { toClassName } from './utils.js';
+import { replace, childNodes } from '../utils/hast-utils.js';
 
 /**
  * Creates a "DIV representation" of a table.
  * @type PipelineStep
- * @param {Document} document
- * @param {HTMLTableElement} $table the table element
+ * @param {HTMLTTableElement} $table the table element
  * @returns {HTMLDivElement} the resulting div
  */
-function tableToDivs(document, $table) {
-  const $cards = document.createElement('div');
-
-  // iterate over the table to avoid problem with query selector and nested tables
+function tableToDivs($table) {
+  const $cards = h('div');
   const $rows = [];
-  if ($table.tHead) {
-    $rows.push(...$table.tHead.rows);
-  }
-  for (const $tbody of $table.tBodies) {
-    $rows.push(...$tbody.rows);
+  for (const child of $table.children) {
+    if (child.tagName === 'thead' || child.tagName === 'tbody') {
+      $rows.push(...childNodes(child));
+    }
   }
+
   if ($rows.length === 0) {
     return $cards;
   }
-  const $headerRow = $rows.shift();
+  const $headerCols = childNodes($rows.shift());
 
   // special case, only 1 row and 1 column with a nested table
-  if ($rows.length === 0 && $headerRow.cells.length === 1) {
-    const $nestedTable = $headerRow.cells[0].querySelector(':scope table');
+  if ($rows.length === 0 && $headerCols.length === 1) {
+    const $nestedTable = select(':scope table', $headerCols[0]);
     if ($nestedTable) {
       return $nestedTable;
     }
   }
 
   // get columns names
-  const clazz = Array.from($headerRow.cells)
-    .map((e) => toClassName(e.textContent))
+  const clazz = $headerCols
+    .map((e) => toClassName(toString(e)))
     .filter((c) => !!c)
     .join('-');
   if (clazz) {
-    $cards.classList.add(clazz);
+    $cards.properties.className = [clazz];
   }
 
   // construct page block
   for (const $row of $rows) {
-    const $card = document.createElement('div');
-    for (const $cell of $row.cells) {
-      const $div = document.createElement('div');
-      $div.append(...$cell.childNodes);
-      $card.append($div);
+    const $card = h('div');
+    for (const $cell of childNodes($row)) {
+      // convert to div
+      $card.children.push(h('div', $cell.children));
     }
-    $cards.append($card);
+    $cards.children.push($card);
   }
   return $cards;
 }
@@ -70,9 +70,10 @@ function tableToDivs(document, $table) {
  * @param context The current context of processing pipeline
  */
 export default function createPageBlocks({ content }) {
-  const { document } = content;
-  document.querySelectorAll('body > div > table').forEach(($table) => {
-    const $div = tableToDivs(document, $table);
-    $table.parentNode.replaceChild($div, $table);
+  const { hast } = content;
+  selectAll('div > table', hast).forEach(($table) => {
+    const $div = tableToDivs($table);
+    // replace child in parent
+    replace(hast, $table, $div);
   });
 }

package/src/steps/create-pictures.js

@@ -9,6 +9,9 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
+import { h } from 'hastscript';
+import { selectAll } from 'hast-util-select';
+import { replace } from '../utils/hast-utils.js';
 import { optimizeImageURL } from './utils.js';
 
 /**
@@ -17,19 +20,20 @@ import { optimizeImageURL } from './utils.js';
  * @param context The current context of processing pipeline
  */
 export default async function createPictures({ content }) {
-  const { document } = content;
+  const { hast } = content;
 
   // transform <img> to <picture>
-  document.querySelectorAll('img[src^="./media_"]').forEach((img, i) => {
-    const picture = document.createElement('picture');
-    const source = document.createElement('source');
-    const src = img.getAttribute('src');
-    source.setAttribute('media', '(max-width: 400px)');
-    source.setAttribute('srcset', optimizeImageURL(src, 750));
-    picture.appendChild(source);
-    img.setAttribute('loading', i > 0 ? 'lazy' : 'eager'); // load all but first image lazy
-    img.setAttribute('src', optimizeImageURL(src, 2000));
-    img.parentNode.insertBefore(picture, img);
-    picture.appendChild(img);
+  selectAll('img[src^="./media_"]', hast).forEach((img, i) => {
+    const { src } = img.properties;
+    const source = h('source');
+    source.properties.media = '(max-width: 400px)';
+    source.properties.srcset = optimizeImageURL(src, 750);
+
+    const picture = h('picture', source);
+    img.properties.loading = i > 0 ? 'lazy' : 'eager';
+    img.properties.src = optimizeImageURL(src, 2000);
+
+    replace(hast, img, picture);
+    picture.children.push(img);
   });
 }