@adobe/helix-html-pipeline 1.2.1 → 1.3.2

package/CHANGELOG.md

@@ -1,3 +1,25 @@
+## [1.3.2](https://github.com/adobe/helix-html-pipeline/compare/v1.3.1...v1.3.2) (2022-03-19)
+
+
+### Bug Fixes
+
+* expect spec-compliant URL ([d65428a](https://github.com/adobe/helix-html-pipeline/commit/d65428a2f0e68471f8eed785706766744a7f168b))
+
+## [1.3.1](https://github.com/adobe/helix-html-pipeline/compare/v1.3.0...v1.3.1) (2022-03-18)
+
+
+### Bug Fixes
+
+* **deps:** update dependency @adobe/helix-shared-utils to v2.0.6 ([#26](https://github.com/adobe/helix-html-pipeline/issues/26)) ([186c376](https://github.com/adobe/helix-html-pipeline/commit/186c376d0252b0c96ee461670cf45a711aa93f4f))
+* preserve formatting of script tags ([#25](https://github.com/adobe/helix-html-pipeline/issues/25)) ([7009f20](https://github.com/adobe/helix-html-pipeline/commit/7009f20d37190f5704b7f9363c59912b4272c0bf)), closes [#23](https://github.com/adobe/helix-html-pipeline/issues/23)
+
+# [1.3.0](https://github.com/adobe/helix-html-pipeline/compare/v1.2.1...v1.3.0) (2022-03-17)
+
+
+### Features
+
+* provide pipes for OPTIONS and POSTs ([#24](https://github.com/adobe/helix-html-pipeline/issues/24)) ([1dfc47e](https://github.com/adobe/helix-html-pipeline/commit/1dfc47e764a0b1d8acee80b51b845be2e54a16f5))
+
 ## [1.2.1](https://github.com/adobe/helix-html-pipeline/compare/v1.2.0...v1.2.1) (2022-03-16)
 
 

package/README.md

@@ -3,8 +3,8 @@
 This package contains the common code for `helix-pipeline-service` and `helix-cloudflare-pipeline` for rendering the html response for helix3. it has the following design goals:
 
 - be platform neutral, i.e. not using node or browser specific modules or dependencies.
-- +/-0 runtime dependencies
-- offer extension interfaces where platform abstraction is required (e.g. reading from s3)
+- +/-0 runtime dependencies (eg. node [crypto](https://nodejs.org/api/crypto.html))
+- offer extension interfaces where platform abstraction is required (e.g. reading from S3, sending to SQS)
 
 ## Status
 [![codecov](https://img.shields.io/codecov/c/github/adobe/helix-html-pipeline.svg)](https://codecov.io/gh/adobe/helix-html-pipeline)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-html-pipeline",
-  "version": "1.2.1",
+  "version": "1.3.2",
   "description": "Helix HTML Pipeline",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "@adobe/helix-markdown-support": "3.1.2",
-    "@adobe/helix-shared-utils": "2.0.5",
+    "@adobe/helix-shared-utils": "2.0.6",
     "github-slugger": "1.4.0",
     "hast-util-raw": "7.2.1",
     "hast-util-select": "5.0.1",

package/src/PipelineState.d.ts

@@ -9,12 +9,13 @@
  * OF ANY KIND; either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import {PathInfo, S3Loader, PipelineTimer} from "./index";
+import {PathInfo, S3Loader, FormsMessageDispatcher, PipelineTimer} from "./index";
 import {PipelineContent} from "./PipelineContent";
 
 declare interface PipelineOptions {
   log: Console;
   s3Loader: S3Loader;
+  messageDispatcher: FormsMessageDispatcher;
   owner: string;
   repo: string;
   ref: string;
@@ -31,6 +32,7 @@ declare class PipelineState {
   content: PipelineContent;
   contentBusId: string;
   s3Loader: S3Loader;
+  messageDispatcher: FormsMessageDispatcher;
 
   /**
    * Content bus partition

package/src/PipelineState.js

@@ -36,6 +36,7 @@ export class PipelineState {
       helixConfig: undefined,
       metadata: undefined,
       s3Loader: opts.s3Loader,
+      messageDispatcher: opts.messageDispatcher,
       timer: opts.timer,
     });
   }

package/src/forms-pipe.js

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { cleanupHeaderValue } from '@adobe/helix-shared-utils';
+import { PipelineResponse } from './PipelineResponse.js';
+import fetchMetadata from './steps/fetch-metadata.js';
+import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+import { getOriginalHost } from './steps/utils.js';
+
+function error(log, msg, status, response) {
+  log.error(msg);
+  response.status = status;
+  response.headers.set('x-error', cleanupHeaderValue(msg));
+  return response;
+}
+
+/**
+ * Converts URLSearchParams to an object
+ * @param {URLSearchParams} searchParams the search params object
+ * @returns {Object} The converted object
+ */
+function searchParamsToObject(searchParams) {
+  const result = {};
+
+  for (const key of searchParams.keys()) {
+    // get all values association with the key
+    const values = searchParams.getAll(key);
+
+    // if multiple values, convert to array
+    result[key] = (values.length === 1) ? values[0] : values;
+  }
+
+  return result;
+}
+
+/**
+ * Extracts and parses the body data from the request
+ * @param {PipelineRequest} request the request object (see fetch api)
+ * @returns {Object} The body data
+ * @throws {Error} If an error occurs parsing the body
+ */
+export async function extractBodyData(request) {
+  let { body } = request;
+  if (!body) {
+    throw Error('missing body');
+  }
+  const type = request.headers.get('content-type');
+
+  // if content is form-urlencoded we need place the object in the body
+  // in a "data" property in the body as this is what forms-service expects.
+  if (/^application\/x-www-form-urlencoded/.test(type)) {
+    // did they pass an object in the body when a form-urlencoded body was expected?
+    if (body === '[object Object]') {
+      throw Error('invalid form-urlencoded body');
+    }
+
+    body = {
+      data: searchParamsToObject(new URLSearchParams(body)),
+    };
+
+    // else treat the body as json
+  } else if (/^application\/json/.test(type)) {
+    body = JSON.parse(body);
+    // verify the body data is as expected
+    if (!body.data) {
+      throw Error('missing body.data');
+    }
+  } else {
+    throw Error(`post body content-type not supported: ${type}`);
+  }
+  return body;
+}
+
+/**
+ * Handle a pipeline POST request.
+ * At this point POST's only apply to json files that are backed by a workbook.
+ * @param {PipelineState} state pipeline options
+ * @param {PipelineRequest} request
+ * @returns {Promise<PipelineResponse>} a response
+ */
+export async function formsPipe(state, request) {
+  const { log } = state;
+
+  // todo: improve
+  const response = new PipelineResponse('', {
+    headers: {
+      'content-type': 'text/plain; charset=utf-8',
+    },
+  });
+  await fetchMetadata(state, request, response);
+  await setCustomResponseHeaders(state, request, response);
+
+  const {
+    owner, repo, ref, contentBusId, partition, s3Loader,
+  } = state;
+  const { path } = state.info;
+  const resourcePath = `${path}.json`;
+
+  // block all POSTs to resources with extensions
+  if (state.info.originalExtension !== '') {
+    return error(log, 'POST to URL with extension not allowed', 405, response);
+  }
+
+  // head workbook in content bus
+  const resourceFetchResponse = await s3Loader.headObject('helix-content-bus', `${contentBusId}/${partition}${resourcePath}`);
+  if (resourceFetchResponse.status !== 200) {
+    return resourceFetchResponse;
+  }
+
+  let body;
+  try {
+    body = await extractBodyData(request);
+  } catch (err) {
+    return error(log, err.message, 400, response);
+  }
+
+  const sheets = resourceFetchResponse.headers.get('x-amz-meta-x-sheet-names');
+  if (!sheets) {
+    return error(log, `Target workbook at ${resourcePath} missing x-sheet-names header.`, 403, response);
+  }
+
+  const sourceLocation = resourceFetchResponse.headers.get('x-amz-meta-x-source-location');
+  const referer = request.headers.get('referer') || 'unknown';
+  const sheetNames = sheets.split(',');
+
+  if (!sourceLocation || !sheetNames.includes('incoming')) {
+    return error(log, `Target workbook at ${resourcePath} is not setup to intake data.`, 403, response);
+  }
+
+  // Send message to SQS if workbook contains and incoming
+  // sheet and the source location is not null
+  const host = getOriginalHost(request.headers);
+
+  const message = {
+    url: `https://${ref}--${repo}--${owner}.hlx.live${resourcePath}`,
+    body,
+    host,
+    sourceLocation,
+    referer,
+  };
+
+  try {
+    // Send message to forms queue
+    const { requestId, messageId } = await state.messageDispatcher.dispatch(message);
+    response.status = 201;
+    response.headers.set('x-request-id', requestId);
+    response.headers.set('x-message-id', messageId);
+    return response;
+  } catch (err) {
+    return error(log, `Failed to send message to forms queue: ${err}`, 500, response);
+  }
+}

package/src/index.d.ts

@@ -83,6 +83,19 @@ declare interface S3Loader {
   headObject(bucketId, key): Promise<PipelineResponse>;
 }
 
+declare interface DispatchMessageResponse {
+  messageId:string,
+  requestId:string,
+}
+
+declare interface FormsMessageDispatcher {
+  /**
+   * Dispatches the message to the forms queue
+   * @param {object} message
+   */
+  dispatch(message:object): Promise<DispatchMessageResponse>;
+}
+
 /**
  * Timer
  */

package/src/index.js

@@ -11,6 +11,8 @@
  */
 export * from './html-pipe.js';
 export * from './json-pipe.js';
+export * from './options-pipe.js';
+export * from './forms-pipe.js';
 export * from './PipelineContent.js';
 export * from './PipelineRequest.js';
 export * from './PipelineResponse.js';

package/src/options-pipe.js

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { PipelineResponse } from './PipelineResponse.js';
+import fetchMetadata from './steps/fetch-metadata.js';
+import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+
+/**
+ * Handles options requests
+ * @param {PipelineState} state pipeline options
+ * @param {PipelineRequest} request
+ * @returns {Response} a response
+ */
+export async function optionsPipe(state, request) {
+  // todo: improve
+  const response = new PipelineResponse('', {
+    status: 204,
+    headers: {
+      // Set preflight cache duration
+      'access-control-max-age': '86400',
+      // Allow content type header
+      'access-control-allow-headers': 'content-type',
+    },
+  });
+  await fetchMetadata(state, request, response);
+  await setCustomResponseHeaders(state, request, response);
+
+  return response;
+}

package/src/steps/stringify-response.js

@@ -12,6 +12,8 @@
 import { toHtml } from 'hast-util-to-html';
 // import rehypeFormat from 'rehype-format';
 import rehypeMinifyWhitespace from 'rehype-minify-whitespace';
+import { visit } from 'unist-util-visit';
+
 /**
  * Serializes the response document to HTML
  * @param {PipelineState} state
@@ -32,7 +34,25 @@ export default function stringify(state, req, res) {
   // TODO: for the next breaking release, pretty print the HTML with rehypeFormat.
   // TODO: but for backward compatibility, output all on 1 line.
   // rehypeFormat()(doc);
+
+  // due to a bug in rehype-minify-whitespace, script content is also minified to 1 line, which
+  // can result in errors https://github.com/rehypejs/rehype-minify/issues/44
+  // so we 'save' all text first and revert it afterwards
+  visit(doc, (node) => {
+    if (node.tagName === 'script' && node.children[0]?.type === 'text') {
+      node.children[0].savedValue = node.children[0].value;
+    }
+  });
+
   rehypeMinifyWhitespace()(doc);
+
+  visit(doc, (node) => {
+    if (node.tagName === 'script' && node.children[0]?.type === 'text') {
+      node.children[0].value = node.children[0].savedValue;
+      delete node.children[0].savedValue;
+    }
+  });
+
   res.body = toHtml(doc, {
     upperDoctype: true,
   });

package/src/steps/utils.js

@@ -125,8 +125,9 @@ export function optimizeImageURL(src, width, format = 'webply', optimize = 'medi
  * @returns {string} resolved url
  */
 export function resolveUrl(from, to) {
-  const resolvedUrl = new URL(to, new URL(from, 'resolve://'));
-  if (resolvedUrl.protocol === 'resolve:') {
+  const DUMMY_BASE = 'https://__dummmy__';
+  const resolvedUrl = new URL(to, new URL(from, DUMMY_BASE));
+  if (resolvedUrl.origin === DUMMY_BASE) {
     // `from` is a relative URL.
     const { pathname, search, hash } = resolvedUrl;
     return pathname + search + hash;