@adobe/helix-html-pipeline 1.0.0

package/src/utils/json-filter.js

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { cleanupHeaderValue } from '@adobe/helix-shared-utils';
+import { PipelineResponse } from '../PipelineResponse.js';
+
+const TYPE_KEY = ':type';
+
+const VERSION_KEY = ':version';
+
+const NAMES_KEY = ':names';
+
+/**
+ * Creates a json response from the given data and query
+ * @param {PipelineState} state
+ * @param {object} data
+ * @param {object} query
+ * @returns {PipelineResponse}
+ */
+export default function jsonFilter(state, data, query) {
+  const {
+    limit = 1000,
+    offset = 0,
+    sheet = [],
+    raw,
+  } = query;
+  const { log } = state;
+
+  function filter(dataStruct) {
+    const len = Math.min(limit, dataStruct.data.length - offset);
+    const filtered = dataStruct.data.slice(offset, offset + len);
+    return {
+      total: dataStruct.total,
+      offset,
+      limit: filtered.length,
+      data: filtered,
+    };
+  }
+
+  let json;
+  try {
+    state.timer?.update('json-parse');
+    json = JSON.parse(data);
+  } catch (e) {
+    const msg = `failed to parse json: ${e.message}`;
+    if (raw) {
+      log.warn(msg);
+      return new PipelineResponse(data, {
+        headers: {
+          'content-type': 'text/plain',
+        },
+      });
+    }
+
+    log.error(msg);
+    return new PipelineResponse('', {
+      status: 502,
+      headers: {
+        'x-error': cleanupHeaderValue(msg),
+      },
+    });
+  }
+
+  // when raw request, only handle multisheets.
+  if (raw && !(NAMES_KEY in json)) {
+    return new PipelineResponse(data, {
+      headers: {
+        'content-type': 'application/json',
+      },
+    });
+  }
+
+  // if single sheet, convert it to multisheet
+  if (Array.isArray(json.data)) {
+    json = {
+      [NAMES_KEY]: ['default'],
+      default: json,
+    };
+  }
+
+  if (!json[NAMES_KEY]) {
+    const msg = 'multisheet data invalid. missing ":names" property.';
+    log.error(msg);
+    return new PipelineResponse('', {
+      status: 502,
+      headers: {
+        'x-error': cleanupHeaderValue(msg),
+      },
+    });
+  }
+
+  state.timer?.update('json-filter');
+  const requestedSheets = Array.isArray(sheet) ? sheet : [sheet];
+  if (requestedSheets.length === 0 && 'default' in json) {
+    requestedSheets.push('default');
+  }
+  const sheetNames = [];
+  const sheets = {};
+  json[NAMES_KEY]
+    .filter((name) => requestedSheets.indexOf(name) >= 0 || requestedSheets.length === 0)
+    .forEach((name) => {
+      sheets[name] = filter(json[name]);
+      sheetNames.push(name);
+    });
+  if (sheetNames.length === 0) {
+    const msg = `filtered result does not contain selected sheet(s): ${requestedSheets.join(',')}`;
+    log.info(msg);
+    return new PipelineResponse('', {
+      status: 404,
+      headers: {
+        'x-error': cleanupHeaderValue(msg),
+      },
+    });
+  }
+
+  let body;
+  let type = 'sheet';
+  if (sheetNames.length === 1) {
+    body = sheets[sheetNames[0]];
+  } else {
+    type = 'multi-sheet';
+    body = {
+      ...sheets,
+      [VERSION_KEY]: 3,
+      [NAMES_KEY]: sheetNames,
+    };
+  }
+  body[TYPE_KEY] = type;
+  state.timer?.update('json-stringify');
+  return new PipelineResponse(JSON.stringify(body), {
+    headers: {
+      'content-type': 'application/json',
+    },
+  });
+}

package/src/utils/last-modified.js

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Updates the context.content.lastModified if the time in `timeString` is newer than the existing
+ * one if none exists yet. please note that it generates helper property `lastModifiedTime` in
+ * unix epoch format.
+ *
+ * the date string will be a "http-date": https://httpwg.org/specs/rfc7231.html#http.date
+ *
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res the pipeline context
+ * @param {string} httpDate http-date string
+ */
+export function updateLastModified(state, res, httpDate) {
+  if (!httpDate) {
+    return;
+  }
+  const { log } = state;
+  const time = new Date(httpDate).getTime();
+  if (Number.isNaN(time)) {
+    log.warn(`updateLastModified date is invalid: ${httpDate}`);
+    return;
+  }
+
+  if (time > (res.lastModifiedTime ?? 0)) {
+    res.lastModifiedTime = time;
+    res.headers.set('last-modified', httpDate);
+  }
+}
+
+/**
+ * Returns the last modified date from response headers, giving 'x-amz-meta-x-source-last-modified'
+ * preference.
+ * @param {object} headers
+ */
+export function extractLastModified(headers) {
+  return headers.get('x-amz-meta-x-source-last-modified') ?? headers.get('last-modified');
+}

package/src/utils/link-handler.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2018 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { link as fallback } from 'mdast-util-to-hast/lib/handlers/link.js';
+import { parse, serialize } from 'uri-js';
+
+export default function link({ extension = 'html' } = {}) {
+  return function handler(h, node) {
+    const n = { ...node };
+    const uriParts = parse(n.url);
+    if (!uriParts.scheme && uriParts.path) {
+      uriParts.path = uriParts.path.replace(/\.md$/, `.${extension}`);
+      n.url = serialize(uriParts, { absolutePath: true });
+    }
+    return fallback(h, n);
+  };
+}

package/src/utils/mdast-to-vdom.js

@@ -0,0 +1,323 @@
+/*
+ * Copyright 2018 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { selectAll } from 'unist-util-select';
+import { toHast as mdast2hast, defaultHandlers } from 'mdast-util-to-hast';
+import { toHtml as hast2html } from 'hast-util-to-html';
+import { JSDOM } from 'jsdom';
+
+import toDOM from './hast-util-to-dom.js';
+import HeadingHandler from './heading-handler.js';
+import link from './link-handler.js';
+import table from './table-handler.js';
+import icon from './icon-handler.js';
+import section from './section-handler.js';
+
+const types = [
+  'root',
+  'paragraph',
+  'text',
+  'heading',
+  'thematicBreak',
+  'blockquote',
+  'list',
+  'listItem',
+  'table',
+  'tableRow',
+  'tableCell',
+  'html',
+  'code',
+  'yaml',
+  'definition',
+  'footnoteDefinition',
+  'emphasis',
+  'strong',
+  'delete',
+  'inlineCode',
+  'break',
+  'link',
+  'image',
+  'linkReference',
+  'imageReference',
+  'footnote',
+  'footnoteReference',
+  'embed',
+  'dataEmbed',
+  'section',
+  'icon',
+];
+
+/**
+ * @typedef {function(parent, tagName, attributes, children)} handlerFunction
+ * @param {Node} parent the root node to append the new dom node to
+ * @param {string} tagName name of the new tag
+ * @param {object} attributes HTML attributes as key-value pairs
+ * @param {Node[]} children list of children
+ */
+
+/**
+ * Utility class that transforms an MDAST (Markdown) node into a (virtual) DOM
+ * representation of the same content.
+ */
+export default class VDOMTransformer {
+  /**
+   * Initializes the transformer with a Markdown document or fragment of a document
+   * @param {Node} mdast the markdown AST node to start the transformation from.
+   * @param {object} options options for custom transformers
+   */
+  constructor(mdast, options) {
+    this._matchers = [];
+    this._handlers = {};
+    this._options = {};
+
+    // go over all handlers that have been defined
+
+    const that = this;
+    // use our own handle function for every known node type
+    types.map((type) => {
+      this._handlers[type] = (cb, node, parent) => VDOMTransformer.handle(cb, node, parent, that);
+      return true;
+    });
+
+    if (mdast) {
+      this.withMdast(mdast);
+    }
+    this.withOptions(options);
+  }
+
+  withOptions(options = {}) {
+    this._options = Object.assign(this._options, options);
+
+    this._headingHandler = new HeadingHandler(options.slugger);
+    this.match('heading', this._headingHandler.handler());
+    this.match('link', link(this._options));
+    this.match('icon', icon());
+    this.match('table', table());
+    this.match('section', section());
+    this.match('html', (h, node) => {
+      if (node.value.startsWith('<!--')) {
+        return h.augment(node, {
+          type: 'comment',
+          value: node.value.substring(4, node.value.length - 3),
+        });
+      }
+      this._hasRaw = true;
+      const frag = JSDOM.fragment(node.value);
+      return h.augment(node, {
+        type: 'raw',
+        value: '', // we ignore the value here and treat it later in hast-util-to-dom
+        frag,
+        html: node.value,
+      });
+    });
+
+    return this;
+  }
+
+  withMdast(mdast) {
+    this._root = mdast;
+
+    return this;
+  }
+
+  /**
+   * A mdast-util-to-hast handler function that applies matchers and
+   * falls back to the default mdast-util-to-hast handlers if no matchers
+   * apply
+   * @private
+   * @param {handlerFunction} cb
+   * @param {Node} node the MDAST node to transofrm
+   * @param {Node} parent the MDAST parent or root node for select expressions
+   * @param {VDOMTransformer} that the MDAST to VDOM transformer
+   * @returns {Node} a HTAST representation of the `node` input
+   */
+  static handle(cb, node, parent, that) {
+    // get the function that handles this node type
+    // this will fall back to the default if none matches
+    const handlefn = that.matches(node);
+
+    // process the node
+
+    const result = handlefn(cb, node, parent);
+    if (result && typeof result === 'string') {
+      throw new Error('returning string from a handler is not supported yet.');
+    } else if (result && typeof result === 'object' && result.outerHTML) {
+      throw new Error('returning a DOM element from a handler is not supported yet.');
+    }
+    return result;
+  }
+
+  /**
+   * Returns the default handler for a given node type
+   * @param {Node} node an MDAST node
+   * @returns {handlerFunction} the default handler function
+   */
+  static default(node) {
+    // use the default handler from mdast-util-to-hast
+    return defaultHandlers[node.type];
+  }
+
+  /**
+   * A predicate function that filters MDAST nodes
+   * @typedef {function(node)} matcherFunction
+   * @param {Node} node an MDAST node
+   * @returns {boolean} true for matching nodes
+   */
+
+  /**
+   * Registers a handler function for nodes that match either a select expression
+   * or a matcher predicate function. The `matcher` will be evaluated against every
+   * node in the MDAST. In cases where the `matcher` matches (returns true), the
+   * processor will be called with the current node.
+   * @param {(string|matcherFunction)} matcher either an unist-util-select expression
+   * or a predicate function
+   * @param {handlerFunction} processor the appropriate handler function to handle matching types.
+   * @returns {VDOMTransformer} this, enabling chaining
+   */
+  match(matcher, processor) {
+    const matchfn = typeof matcher === 'function' ? matcher : VDOMTransformer.matchfn(this._root, matcher);
+    this._matchers.push([matchfn, processor]);
+    return this;
+  }
+
+  /**
+   * Finds an appropriate handler for a given MDAST node
+   * @private
+   * @param {Node} node an MDAST node
+   * @returns {handlerFunction} a handler function to process the node with
+   */
+  matches(node) {
+    // go through all matchers to find processors where matchfn matches
+    // start with most recently added processors
+    for (let i = this._matchers.length - 1; i >= 0; i -= 1) {
+      const [matchfn, processor] = this._matchers[i];
+      if (matchfn(node, this._root)) {
+        // return the first processor that matches
+        return processor;
+      }
+    }
+    // add the fallback processors
+    return VDOMTransformer.default(node);
+  }
+
+  /**
+   * Turns an unist-util-select expression into a matcher predicate function
+   * @private
+   * @param {Node} ast the MDAST root node to evaluated expressions against
+   * @param {string} pattern a CSS-like unist-util-select expression
+   * @returns {matcherFunction} a corresponding matcher function that returns true
+   * for nodes matching the pattern
+   */
+  static matchfn(ast, pattern) {
+    // evaluating selectAll on a large tree for each handler is very expensive.
+    // use node name for simple element selectors
+    if (/^\w+$/.test(pattern)) {
+      return function match(node) {
+        return node.type === pattern;
+      };
+    }
+
+    return function match(node, myast = ast) {
+      return selectAll(pattern, myast).indexOf(node) >= 0;
+    };
+  }
+
+  /**
+   * Tries to sanitize inline HTML elements. The remark parser creates `raw` nodes for html.
+   * In case of inline html, those are not closed elements. since we generated the DOM already
+   * in the HTML handler, we get incomplete fragments, missing the inner HTML. This method tries
+   * to fix this. It doesn't support inner markdown-elements, though.
+   *
+   * @param {object} node HAST node
+   */
+  static sanitizeInlineHTML(node) {
+    const stack = [];
+    for (let i = 0; i < node.children.length; i += 1) {
+      const child = node.children[i];
+      if (child.type === 'raw') {
+        if (child.frag.firstElementChild === null) {
+          if (stack.length === 0) {
+            // ignore unmatched inline elements
+          } else {
+            const last = stack.pop();
+            let html = '';
+            for (let j = last; j <= i; j += 1) {
+              const innerChild = node.children[j];
+              if (innerChild.type === 'raw') {
+                html += innerChild.html;
+              } else {
+                html += hast2html(innerChild);
+              }
+            }
+            node.children[last].frag = JSDOM.fragment(html);
+            node.children[last].html = html;
+            node.children.splice(last + 1, i - last);
+            i = last;
+          }
+        } else {
+          stack.push(i);
+        }
+      } else if (child.children && child.children.length) {
+        VDOMTransformer.sanitizeInlineHTML(child);
+      }
+    }
+  }
+
+  /**
+   * Turns the MDAST into a full DOM-like structure using JSDOM
+   * @returns {Document} a full DOM document
+   */
+  getDocument() {
+    // mdast -> hast; hast -> DOM using JSDOM
+    const hast = mdast2hast(this._root, {
+      handlers: this._handlers,
+      allowDangerousHtml: true,
+    });
+
+    if (this._hasRaw) {
+      VDOMTransformer.sanitizeInlineHTML(hast);
+    }
+
+    const dom = new JSDOM();
+    const doc = dom.window.document;
+    const frag = toDOM(doc, hast, { fragment: true });
+
+    if (frag.nodeName === '#document') {
+      // this only happens if it's an empty markdown document, so just ignore
+    } else {
+      doc.body.appendChild(frag);
+    }
+
+    // add convenience function to serialize entire document. this is to make it similar to the
+    // document created in html-to-vdom.
+    doc.serialize = dom.serialize.bind(dom);
+
+    // this is a bit a hack to pass the JSDOM instance along, so that other module can use it.
+    // this ensures that other modules can parse documents and fragments that are compatible
+    // with this document
+    Object.defineProperty(doc, 'JSDOM', {
+      enumerable: false,
+      writable: false,
+      value: JSDOM,
+    });
+
+    // this is another hack to pass the respective window instance along. this is to ensure that
+    // the shared prototypes can be used to check node instances (see jsdom 16.x release)
+    Object.defineProperty(doc, 'window', {
+      enumerable: false,
+      writable: false,
+      value: dom.window,
+    });
+
+    return doc;
+  }
+}

package/src/utils/mdast-util-gfm-nolink.js

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/*
+copied from
+https://github.com/micromark/micromark-extension-gfm/blob/7bc6b6f3baf941f877d7b2111e9257b21b13b37e/index.js
+
+(The MIT License)
+
+Copyright (c) 2020 Titus Wormer <tituswormer@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @typedef {import('mdast-util-from-markdown').Extension} FromMarkdownExtension
+ * @typedef {import('mdast-util-to-markdown').Options} ToMarkdownExtension
+ *
+ * @typedef {import('mdast-util-gfm-table').Options} Options
+ */
+
+// import {
+//   gfmAutolinkLiteralFromMarkdown,
+//   gfmAutolinkLiteralToMarkdown
+// } from 'mdast-util-gfm-autolink-literal'
+import {
+  gfmFootnoteFromMarkdown,
+  gfmFootnoteToMarkdown,
+} from 'mdast-util-gfm-footnote';
+import {
+  gfmStrikethroughFromMarkdown,
+  gfmStrikethroughToMarkdown,
+} from 'mdast-util-gfm-strikethrough';
+import { gfmTableFromMarkdown, gfmTableToMarkdown } from 'mdast-util-gfm-table';
+import {
+  gfmTaskListItemFromMarkdown,
+  gfmTaskListItemToMarkdown,
+} from 'mdast-util-gfm-task-list-item';
+
+/**
+ * @returns {Array.<FromMarkdownExtension>}
+ */
+export function gfmFromMarkdown() {
+  return [
+    gfmFootnoteFromMarkdown(),
+    gfmStrikethroughFromMarkdown,
+    gfmTableFromMarkdown,
+    // gfmAutolinkLiteralFromMarkdown,
+    gfmTaskListItemFromMarkdown,
+  ];
+}
+
+/**
+ * @param {Options} [options]
+ * @returns {ToMarkdownExtension}
+ */
+export function gfmToMarkdown(options) {
+  return {
+    extensions: [
+      // gfmAutolinkLiteralToMarkdown,
+      gfmFootnoteToMarkdown(),
+      gfmStrikethroughToMarkdown,
+      gfmTableToMarkdown(options),
+      gfmTaskListItemToMarkdown,
+    ],
+  };
+}