@adobe/helix-html-pipeline 1.0.0

package/src/json-pipe.js

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import fetchMetadata from './steps/fetch-metadata.js';
+import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+import { PipelineResponse } from './PipelineResponse.js';
+import jsonFilter from './utils/json-filter.js';
+
+/**
+ * Runs the default pipeline and returns the response.
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ * @returns {PipelineResponse}
+ */
+export async function jsonPipe(state, req) {
+  const { log } = state;
+  const {
+    owner, repo, ref, contentBusId, partition, s3Loader,
+  } = state;
+  const { path } = state.info;
+  const { searchParams } = req.url;
+  const params = Object.fromEntries(searchParams.entries());
+  if (params.sheet) {
+    params.sheet = searchParams.getAll('sheet');
+  }
+  const {
+    limit,
+    offset,
+    sheet,
+  } = params;
+
+  if (!path.endsWith('.json')) {
+    log.error('only json resources supported.');
+    return new PipelineResponse('', {
+      status: 400,
+      headers: {
+        'x-error': 'only json resources supported.',
+      },
+    });
+  }
+
+  // fetch data from content bus
+  state.timer?.update('json-fetch');
+  let dataResponse = await s3Loader.getObject('helix-content-bus', `${contentBusId}/${partition}${path}`);
+
+  // if not found, fall back to code bus
+  if (dataResponse.status === 404) {
+    dataResponse = await s3Loader.getObject('helix-code-bus', `${owner}/${repo}/${ref}${path}`);
+  }
+
+  // if still not found, return status
+  if (dataResponse.status !== 200) {
+    return dataResponse;
+  }
+  const data = dataResponse.body;
+
+  // filter data
+  const response = jsonFilter(state, data, {
+    limit: limit ? Number.parseInt(limit, 10) : undefined,
+    offset: offset ? Number.parseInt(offset, 10) : undefined,
+    sheet,
+    raw: limit === undefined && offset === undefined && sheet === undefined,
+  });
+
+  // set last-modified (note, that it is not influenced by metadata or helix-config.json)
+  const lastModified = dataResponse.headers.get('last-modified');
+  if (lastModified) {
+    response.headers.set('last-modified', lastModified);
+  }
+
+  // set surrogate key
+  response.headers.set('x-surrogate-key', `${contentBusId}${path}`.replace(/\//g, '_'));
+
+  // Load metadata from metadata.json
+  await fetchMetadata(state, req, response);
+  await setCustomResponseHeaders(state, req, response);
+
+  return response;
+}

package/src/steps/add-heading-ids.js

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Adds missing `id` attributes to the headings
+ * @type PipelineStep
+ * @param {PipelineContent } content The current context of processing pipeline
+ */
+export default async function fixSections({ content }) {
+  const { slugger, document } = content;
+  ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+    .forEach((tagName) => {
+      document.querySelectorAll(tagName)
+        .forEach(($h) => {
+          if (!$h.id) {
+            const text = $h.textContent.trim();
+            if (text) {
+              $h.setAttribute('id', slugger.slug(text));
+            }
+          }
+        });
+    });
+}

package/src/steps/create-page-blocks.js

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { toClassName } from './utils.js';
+
+/**
+ * Creates a "DIV representation" of a table.
+ * @type PipelineStep
+ * @param {Document} document
+ * @param {HTMLTableElement} $table the table element
+ * @returns {HTMLDivElement} the resulting div
+ */
+function tableToDivs(document, $table) {
+  const $cards = document.createElement('div');
+
+  // iterate over the table to avoid problem with query selector and nested tables
+  const $rows = [];
+  if ($table.tHead) {
+    $rows.push(...$table.tHead.rows);
+  }
+  for (const $tbody of $table.tBodies) {
+    $rows.push(...$tbody.rows);
+  }
+  if ($rows.length === 0) {
+    return $cards;
+  }
+  const $headerRow = $rows.shift();
+
+  // special case, only 1 row and 1 column with a nested table
+  if ($rows.length === 0 && $headerRow.cells.length === 1) {
+    const $nestedTable = $headerRow.cells[0].querySelector(':scope table');
+    if ($nestedTable) {
+      return $nestedTable;
+    }
+  }
+
+  // get columns names
+  const clazz = Array.from($headerRow.cells)
+    .map((e) => toClassName(e.textContent))
+    .filter((c) => !!c)
+    .join('-');
+  if (clazz) {
+    $cards.classList.add(clazz);
+  }
+
+  // construct page block
+  for (const $row of $rows) {
+    const $card = document.createElement('div');
+    for (const $cell of $row.cells) {
+      const $div = document.createElement('div');
+      $div.append(...$cell.childNodes);
+      $card.append($div);
+    }
+    $cards.append($card);
+  }
+  return $cards;
+}
+
+/**
+ * Converts tables into page blocks.
+ * see https://github.com/adobe/helix-pages/issues/638
+ * @param context The current context of processing pipeline
+ */
+export default function createPageBlocks({ content }) {
+  const { document } = content;
+  document.querySelectorAll('body > div > table').forEach(($table) => {
+    const $div = tableToDivs(document, $table);
+    $table.parentNode.replaceChild($div, $table);
+  });
+}

package/src/steps/create-pictures.js

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { optimizeImageURL } from './utils.js';
+
+/**
+ * Converts imgs to pictures
+ * @type PipelineStep
+ * @param context The current context of processing pipeline
+ */
+export default async function createPictures({ content }) {
+  const { document } = content;
+
+  // transform <img> to <picture>
+  document.querySelectorAll('img[src^="./media_"]').forEach((img, i) => {
+    const picture = document.createElement('picture');
+    const source = document.createElement('source');
+    const src = img.getAttribute('src');
+    source.setAttribute('media', '(max-width: 400px)');
+    source.setAttribute('srcset', optimizeImageURL(src, 750));
+    picture.appendChild(source);
+    img.setAttribute('loading', i > 0 ? 'lazy' : 'eager'); // load all but first image lazy
+    img.setAttribute('src', optimizeImageURL(src, 2000));
+    img.parentNode.insertBefore(picture, img);
+    picture.appendChild(img);
+  });
+}

package/src/steps/extract-metadata.js

@@ -0,0 +1,257 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { resolve } from 'url';
+import { getAbsoluteUrl, makeCanonicalHtmlUrl, optimizeImageURL } from './utils.js';
+
+/**
+ * Converts all non-valid characters to `-`.
+ * @param {string} text input text
+ * @returns {string} the meta name
+ */
+function toMetaName(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^0-9a-z:_]/gi, '-');
+}
+
+/**
+ * Cleans up comma-separated string lists and returns an array.
+ * @param {string} list A comma-separated list
+ * @returns {string[]} The clean list
+ */
+function toList(list) {
+  return list
+    .split(',')
+    .map((key) => key.trim())
+    .filter((key) => !!key);
+}
+
+/**
+ * Returns the config from a block element as object with key/value pairs.
+ * @param {HTMLDivElement} $block The block element
+ * @returns {object} The block config
+ */
+function readBlockConfig($block) {
+  const config = {};
+  $block.querySelectorAll(':scope>div').forEach(($row) => {
+    if ($row.children && $row.children[1]) {
+      const name = toMetaName($row.children[0].textContent);
+      if (name) {
+        let value;
+        if ($row.children[1].hasChildNodes() && $row.children[1].firstElementChild) {
+          // check for multiple paragraph or a list
+          let childNodes;
+          const { tagName } = $row.children[1].firstElementChild;
+          if (tagName === 'P') {
+            // contains a list of <p> paragraphs
+            childNodes = $row.children[1].childNodes;
+          } else if (tagName === 'UL' || tagName === 'OL') {
+            // contains a list
+            childNodes = $row.children[1].children[0].childNodes;
+          }
+
+          if (childNodes) {
+            value = '';
+            childNodes.forEach((child) => {
+              value += `${child.textContent}, `;
+            });
+            value = value.substring(0, value.length - 2);
+          }
+        }
+
+        if (!value) {
+          // for text content only
+          value = $row.children[1].textContent.trim().replace(/ {3}/g, ',');
+        }
+
+        if (!value) {
+          // check for value inside link
+          const $a = $row.children[1].querySelector('a');
+          if ($a) {
+            value = $a.getAttribute('href');
+          }
+        }
+        if (!value) {
+          // check for value inside img
+          const $img = $row.children[1].querySelector('img');
+          if ($img) {
+            // strip query string
+            value = $img.getAttribute('src');
+          }
+        }
+        if (value) {
+          // only keep non-empty value
+          config[name] = value;
+        }
+      }
+    }
+  });
+  return config;
+}
+
+function applyMetaRule(target, obj) {
+  Object.keys(obj).forEach((key) => {
+    const metaKey = toMetaName(key);
+    if (metaKey !== 'url' && obj[key]) {
+      target[metaKey] = obj[key];
+    }
+  });
+}
+
+function globToRegExp(glob) {
+  const reString = glob
+    .replace(/\*\*/g, '_')
+    .replace(/\*/g, '[0-9a-z-.]*')
+    .replace(/_/g, '.*');
+  return new RegExp(`^${reString}$`);
+}
+
+export function filterGlobalMetadata(metaRules, path) {
+  const metaConfig = {};
+  metaRules.forEach((rule) => {
+    const glob = rule.url || rule.URL || rule.Url;
+    if (glob && typeof glob === 'string' && /[0-9a-z-/*]/.test(glob)) {
+      if (glob.indexOf('*') >= 0) {
+        if (globToRegExp(glob).test(path)) {
+          applyMetaRule(metaConfig, rule);
+        }
+      } else if (glob === path) {
+        applyMetaRule(metaConfig, rule);
+      }
+    }
+  });
+  return metaConfig;
+}
+
+/**
+ * Looks for metadata in the document.
+ * @param {HTMLDocument} document The document
+ * @return {object} The metadata
+ */
+function getLocalMetadata(document) {
+  let metaConfig = {};
+  const metaBlock = document.querySelector('body div.metadata');
+  if (metaBlock) {
+    metaConfig = readBlockConfig(metaBlock);
+    metaBlock.remove();
+  }
+  return metaConfig;
+}
+
+/**
+ * Adds image optimization parameters suitable for meta images to a URL.
+ * @param {string} pagePath The path of the requested page
+ * @param {string} imgUrl The image URL
+ * @returns The optimized image URL
+ */
+function optimizeMetaImage(pagePath, imgUrl) {
+  const src = resolve(pagePath, imgUrl);
+  if (src.startsWith('/')) {
+    return optimizeImageURL(src, 1200, 'pjpg');
+  }
+  return src;
+}
+
+/**
+ * Extracts the metadata and stores it in the content meta
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ */
+export default function extractMetaData(state, req) {
+  const { content } = state;
+  const { meta, document } = content;
+
+  // extract global metadata from spreadsheet, and overlay
+  // with local metadata from document
+  const metaConfig = Object.assign(
+    filterGlobalMetadata(state.metadata, state.info.path),
+    getLocalMetadata(document),
+  );
+
+  // first process supported metadata properties
+  [
+    'title',
+    'description',
+    'keywords',
+    'tags',
+    'image',
+    'image-alt',
+    'canonical',
+    'feed',
+  ].forEach((name) => {
+    if (metaConfig[name]) {
+      meta[name] = metaConfig[name];
+      delete metaConfig[name];
+    }
+  });
+  if (Object.keys(metaConfig).length > 0) {
+    // add rest to meta.custom
+    meta.custom = Object.keys(metaConfig).map((name) => ({
+      name,
+      value: metaConfig[name],
+      property: name.includes(':'),
+    }));
+  }
+
+  if (meta.keywords) {
+    meta.keywords = toList(meta.keywords).join(', ');
+  }
+  if (meta.tags) {
+    meta.tags = toList(meta.tags);
+  }
+
+  // complete metadata with insights from content
+  if (!meta.title) {
+    // content.title is not correct if the h1 is in a page-block since the pipeline
+    // only respects the heading nodes in the mdast
+    const $title = document.querySelector('body > div h1');
+    if ($title) {
+      content.title = $title.textContent;
+    }
+    meta.title = content.title;
+  }
+  if (!meta.description) {
+    // description: text from paragraphs with 10 or more words
+    let desc = [];
+    document.querySelectorAll('div > p').forEach((p) => {
+      if (desc.length === 0) {
+        const words = p.textContent.trim().split(/\s+/);
+        if (words.length >= 10 || words.some((w) => w.length > 25 && !w.startsWith('http'))) {
+          desc = desc.concat(words);
+        }
+      }
+    });
+    meta.description = `${desc.slice(0, 25).join(' ')}${desc.length > 25 ? ' ...' : ''}`;
+  }
+  meta.url = makeCanonicalHtmlUrl(getAbsoluteUrl(req.headers, req.url.href));
+  if (!meta.canonical) {
+    meta.canonical = meta.url;
+  }
+
+  // content.image is not correct if the first image is in a page-block. since the pipeline
+  // only respects the image nodes in the mdast
+  const $hero = document.querySelector('body > div img');
+  if ($hero) {
+    content.image = $hero.src;
+    if ($hero.alt) {
+      content.imageAlt = $hero.alt;
+    }
+  }
+
+  meta.image = getAbsoluteUrl(
+    req.headers,
+    optimizeMetaImage(state.info.path, meta.image || content.image || '/default-meta-image.png'),
+  );
+
+  meta.imageAlt = meta['image-alt'] || content.imageAlt;
+}

package/src/steps/fetch-config.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+import { PipelineStatusError } from '../PipelineStatusError.js';
+
+/**
+ * Fetches the helix-config.json from the code-bus and stores it in `state.helixConfig`
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ * @param {PipelineResponse} res
+ * @returns {Promise<void>}
+ */
+export default async function fetchConfig(state, req, res) {
+  const {
+    log, owner, repo, ref,
+  } = state;
+
+  const key = `${owner}/${repo}/${ref}/helix-config.json`;
+  const ret = await state.s3Loader.getObject('helix-code-bus', key);
+  if (ret.status !== 200) {
+    throw new PipelineStatusError(ret.status === 404 ? 404 : 502, `unable to load /helix-config.json: ${ret.status}`);
+  }
+  try {
+    state.helixConfig = JSON.parse(ret.body);
+  } catch (e) {
+    log.info('failed to parse helix-config.json', e);
+    throw new PipelineStatusError(400, `Failed parsing of /helix-config.json: ${e.message}`);
+  }
+
+  // also update last-modified
+  updateLastModified(state, res, extractLastModified(ret.headers));
+}

package/src/steps/fetch-content.js

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+
+/**
+ * Loads the content from either the content-bus or code-bus and stores it in `state.content`
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ * @param {PipelineResponse} res
+ * @returns {Promise<void>}
+ */
+export default async function fetchContent(state, req, res) {
+  const {
+    log, contentBusId, info, partition, owner, repo, ref,
+  } = state;
+
+  const isCode = state.content.sourceBus === 'code';
+  const key = isCode
+    ? `${owner}/${repo}/${ref}/${info.resourcePath}`
+    : `${contentBusId}/${partition}${info.resourcePath}`;
+  const bucketId = isCode ? 'helix-code-bus' : 'helix-content-bus';
+
+  const ret = await state.s3Loader.getObject(bucketId, key);
+
+  // check for redirect
+  const redirectLocation = ret.headers.get('x-amz-meta-redirect-location');
+  if (redirectLocation) {
+    res.status = 301;
+    res.body = '';
+    res.headers.set('location', redirectLocation);
+    res.error = 'moved';
+    return;
+  }
+
+  if (ret.status === 200) {
+    state.content.data = ret.body;
+
+    // store extra source location if present
+    state.content.sourceLocation = ret.headers.get('x-amz-meta-x-source-location');
+    log.info(`source-location: ${state.content.sourceLocation}`);
+
+    updateLastModified(state, res, extractLastModified(ret.headers));
+
+    // reject requests to /index *after* checking for redirects
+    // (https://github.com/adobe/helix-pipeline-service/issues/290)
+    if (state.info.originalFilename === 'index') {
+      res.status = 404;
+      res.error = `request to ${info.path} not allowed (no-index).`;
+    }
+  } else {
+    // keep 404, but propagate others as 502
+    res.status = ret.status === 404 ? 404 : 502;
+    res.error = `failed to load ${info.path} from ${state.content.sourceBus}-bus: ${ret.status}`;
+  }
+
+  if (res.status === 404) {
+    // try to load 404.html from code-bus
+    const ret404 = await state.s3Loader.getObject('helix-code-bus', `${owner}/${repo}/${ref}/404.html`);
+    if (ret404.status === 200) {
+      // override last-modified if source-last-modified is set
+      const lastModified = extractLastModified(ret404.headers);
+      if (lastModified) {
+        ret404.headers.set('last-modified', lastModified);
+      }
+
+      // keep 404 response status
+      res.body = ret.body;
+      res.headers.set('last-modified', ret404.headers.get('last-modified'));
+      res.headers.set('content-type', 'text/html; charset=utf-8');
+      res.headers.set('x-surrogate-key', `${ref}--${repo}--${owner}_404`);
+    }
+  }
+}

package/src/steps/fetch-metadata.js

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { PipelineStatusError } from '../PipelineStatusError.js';
+import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+
+/**
+ * Loads the metadata.json from the content-bus and stores it in `state.metadata`
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ * @param {PipelineResponse} res
+ * @returns {Promise<void>}
+ */
+export default async function fetchMetadata(state, req, res) {
+  const { contentBusId, partition } = state;
+  const key = `${contentBusId}/${partition}/metadata.json`;
+  const ret = await state.s3Loader.getObject('helix-content-bus', key);
+  if (ret.status === 200) {
+    let json;
+    try {
+      json = JSON.parse(ret.body);
+    } catch (e) {
+      throw new PipelineStatusError(400, `failed parsing of /metadata.json: ${e.message}`);
+    }
+
+    const { data } = json.default ?? json;
+    if (!Array.isArray(data)) {
+      throw new PipelineStatusError(400, 'failed loading of /metadata.json: data must be an array');
+    }
+    state.metadata = data;
+
+    // also update last-modified
+    updateLastModified(state, res, extractLastModified(ret.headers));
+    return;
+  }
+
+  if (ret.status !== 404) {
+    throw new PipelineStatusError(502, `failed to load /metadata.json: ${ret.status}`);
+  }
+
+  // ignore 404
+  state.metadata = [];
+}