@adobe/helix-html-pipeline 3.8.11 → 3.9.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [3.9.0](https://github.com/adobe/helix-html-pipeline/compare/v3.8.12...v3.9.0) (2023-03-23)
+
+
+### Features
+
+* restrict repositories ([#281](https://github.com/adobe/helix-html-pipeline/issues/281)) ([ba7e670](https://github.com/adobe/helix-html-pipeline/commit/ba7e670a0c2fe5e331c37000d0c023cfb79961ce)), closes [#277](https://github.com/adobe/helix-html-pipeline/issues/277)
+
+## [3.8.12](https://github.com/adobe/helix-html-pipeline/compare/v3.8.11...v3.8.12) (2023-03-10)
+
+
+### Bug Fixes
+
+* ensure the .md is delivered ([#275](https://github.com/adobe/helix-html-pipeline/issues/275)) ([c22ab6b](https://github.com/adobe/helix-html-pipeline/commit/c22ab6b23817fa5d19a6563e2fc0b4a1f201a04f))
+
 ## [3.8.11](https://github.com/adobe/helix-html-pipeline/compare/v3.8.10...v3.8.11) (2023-03-02)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-html-pipeline",
-  "version": "3.8.11",
+  "version": "3.9.0",
   "description": "Helix HTML Pipeline",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -50,7 +50,7 @@
     "hast-util-to-html": "8.0.4",
     "hast-util-to-string": "2.0.0",
     "hastscript": "7.2.0",
-    "jose": "4.12.0",
+    "jose": "4.13.1",
     "mdast-util-gfm-footnote": "1.0.2",
     "mdast-util-gfm-strikethrough": "1.0.3",
     "mdast-util-gfm-table": "1.0.7",
@@ -83,20 +83,20 @@
     "@semantic-release/git": "10.0.1",
     "@semantic-release/npm": "9.0.2",
     "c8": "7.13.0",
-    "eslint": "8.34.0",
+    "eslint": "8.36.0",
     "eslint-import-resolver-exports": "1.0.0-beta.5",
     "eslint-plugin-header": "3.1.1",
     "eslint-plugin-import": "2.27.5",
     "esmock": "2.1.0",
     "husky": "8.0.3",
     "js-yaml": "4.1.0",
-    "jsdom": "21.1.0",
+    "jsdom": "21.1.1",
     "junit-report-builder": "3.0.1",
-    "lint-staged": "13.1.2",
+    "lint-staged": "13.2.0",
     "mocha": "10.2.0",
     "mocha-multi-reporters": "1.5.1",
     "remark-gfm": "3.0.1",
-    "semantic-release": "20.1.0"
+    "semantic-release": "20.1.3"
   },
   "lint-staged": {
     "*.js": "eslint",

package/src/PipelineState.d.ts

@@ -23,6 +23,10 @@ type Fetch = (url: string|Request, options?: RequestOptions) => Promise<Response
 
 declare interface AccessConfig {
   allow:(string|string[]);
+
+  require: {
+    repository:(string|string[]);
+  };
 }
 
 declare interface HelixConfigAll {

package/src/html-pipe.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 import { cleanupHeaderValue } from '@adobe/helix-shared-utils';
-import { authenticate } from './steps/authenticate.js';
+import { authenticate, requireProject } from './steps/authenticate.js';
 import addHeadingIds from './steps/add-heading-ids.js';
 import createPageBlocks from './steps/create-page-blocks.js';
 import createPictures from './steps/create-pictures.js';
@@ -91,7 +91,10 @@ export async function htmlPipe(state, req) {
       fetchContent(state, req, res),
     ]);
 
-    await authenticate(state, req, res);
+    await requireProject(state, req, res);
+    if (!res.error) {
+      await authenticate(state, req, res);
+    }
 
     if (res.error) {
       // if content loading produced an error, we're done.
@@ -104,7 +107,7 @@ export async function htmlPipe(state, req) {
       return res;
     }
 
-    if (state.content.sourceBus === 'code') {
+    if (state.content.sourceBus === 'code' || state.info.originalExtension === '.md') {
       state.timer?.update('serialize');
       await renderCode(state, req, res);
     } else {

package/src/json-pipe.js

@@ -42,6 +42,31 @@ export default function folderMapping(state) {
   }
 }
 
+async function fetchJsonContent(state, req, res) {
+  const {
+    owner, repo, ref, contentBusId, partition, s3Loader, log,
+  } = state;
+  const { path } = state.info;
+  let ret = await s3Loader.getObject('helix-content-bus', `${contentBusId}/${partition}${path}`);
+
+  // if not found, fall back to code bus
+  if (ret.status === 404) {
+    ret = await s3Loader.getObject('helix-code-bus', `${owner}/${repo}/${ref}${path}`);
+  }
+  if (ret.status === 200) {
+    state.content.data = ret.body;
+
+    // store extra source location if present
+    state.content.sourceLocation = ret.headers.get('x-amz-meta-x-source-location');
+    log.info(`source-location: ${state.content.sourceLocation}`);
+
+    updateLastModified(state, res, extractLastModified(ret.headers));
+  } else {
+    res.status = ret.status === 404 ? 404 : 502;
+    res.error = `failed to load ${state.info.resourcePath}: ${ret.status}`;
+  }
+}
+
 /**
  * Runs the default pipeline and returns the response.
  * @param {PipelineState} state
@@ -51,9 +76,7 @@ export default function folderMapping(state) {
 export async function jsonPipe(state, req) {
   const { log } = state;
   state.type = 'json';
-  const {
-    owner, repo, ref, contentBusId, partition, s3Loader,
-  } = state;
+  const { contentBusId } = state;
   const { extension } = state.info;
   const { searchParams } = req.url;
   const params = Object.fromEntries(searchParams.entries());
@@ -81,56 +104,54 @@ export async function jsonPipe(state, req) {
     await fetchConfig(state, req);
     await folderMapping(state);
 
-    // fetch data from content bus
-    const { path } = state.info;
+    /** @type PipelineResponse */
+    const res = new PipelineResponse('', {
+      headers: {
+        'content-type': 'application/json',
+      },
+    });
+
     state.timer?.update('json-fetch');
-    let dataResponse = await s3Loader.getObject('helix-content-bus', `${contentBusId}/${partition}${path}`);
+    await Promise.all([
+      fetchConfigAll(state, req, res),
+      fetchJsonContent(state, req, res),
+    ]);
 
-    // if not found, fall back to code bus
-    if (dataResponse.status === 404) {
-      dataResponse = await s3Loader.getObject('helix-code-bus', `${owner}/${repo}/${ref}${path}`);
-    }
+    await authenticate(state, req, res);
 
-    // if still not found, return status
-    if (dataResponse.status !== 200) {
-      if (dataResponse.status < 500) {
-        await fetchConfigAll(state, req, dataResponse);
-        await setCustomResponseHeaders(state, req, dataResponse);
-      }
-      return dataResponse;
+    if (res.error) {
+      throw new PipelineStatusError(res.status, res.error);
     }
-    const data = dataResponse.body;
 
     // filter data
-    const response = jsonFilter(state, data, {
+    jsonFilter(state, res, {
       limit: limit ? Number.parseInt(limit, 10) : undefined,
       offset: offset ? Number.parseInt(offset, 10) : undefined,
       sheet,
       raw: limit === undefined && offset === undefined && sheet === undefined,
     });
 
-    // set last-modified
-    updateLastModified(state, response, extractLastModified(dataResponse.headers));
-
     // set surrogate keys
     const keys = [];
+    const { path } = state.info;
     keys.push(`${contentBusId}${path}`.replace(/\//g, '_')); // TODO: remove
     keys.push(await computeSurrogateKey(`${contentBusId}${path}`));
-    response.headers.set('x-surrogate-key', keys.join(' '));
-
-    // Load config-all and set response headers
-    await fetchConfigAll(state, req, response);
-    await authenticate(state, req, response);
-    await setCustomResponseHeaders(state, req, response);
+    res.headers.set('x-surrogate-key', keys.join(' '));
 
-    return response;
+    await setCustomResponseHeaders(state, req, res);
+    return res;
   } catch (e) {
     const res = new PipelineResponse('', {
       status: e instanceof PipelineStatusError ? e.code : 500,
     });
     const level = res.status >= 500 ? 'error' : 'info';
     log[level](`pipeline status: ${res.status} ${e.message}`, e);
+    res.body = '';
     res.headers.set('x-error', cleanupHeaderValue(e.message));
+    if (res.status < 500) {
+      await setCustomResponseHeaders(state, req, res);
+    }
+
     return res;
   }
 }

package/src/steps/authenticate.js

@@ -82,3 +82,42 @@ export async function authenticate(state, req, res) {
     res.headers.set('x-hlx-auth-key', authInfo.profile.pem);
   }
 }
+
+/**
+ * Checks if the given owner repo is alloed
+ * @param {string} owner
+ * @param {string} repo
+ * @param {string[]} allows
+ * @returns {boolean}
+ */
+export function isOwnerRepoAllowed(owner, repo, allows = []) {
+  if (allows.length === 0) {
+    return true;
+  }
+  return allows
+    .map((ownerRepo) => ownerRepo.split('/'))
+    .findIndex(([o, r]) => owner === o && (repo === r || r === '*')) >= 0;
+}
+
+/**
+ * Checks if the
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ * @param {PipelineResponse} res
+ * @returns {Promise<void>}
+ */
+export async function requireProject(state, req, res) {
+  // if not restricted, do nothing
+  const ownerRepo = state.config?.access?.require?.repository;
+  if (!ownerRepo) {
+    return;
+  }
+  const ownerRepos = Array.isArray(ownerRepo) ? ownerRepo : [ownerRepo];
+  const { log, owner, repo } = state;
+  if (!isOwnerRepoAllowed(owner, repo, ownerRepos)) {
+    log.warn(`${owner}/${repo} not allowed for ${ownerRepos}`);
+    res.status = 403;
+    res.error = 'forbidden.';
+  }
+}

package/src/utils/json-filter.js

@@ -9,8 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { cleanupHeaderValue } from '@adobe/helix-shared-utils';
-import { PipelineResponse } from '../PipelineResponse.js';
+import { PipelineStatusError } from '../PipelineStatusError.js';
 
 const TYPE_KEY = ':type';
 
@@ -21,11 +20,10 @@ const NAMES_KEY = ':names';
 /**
  * Creates a json response from the given data and query
  * @param {PipelineState} state
- * @param {object} data
+ * @param {PipelineResponse} res
  * @param {object} query
- * @returns {PipelineResponse}
  */
-export default function jsonFilter(state, data, query) {
+export default function jsonFilter(state, res, query) {
   const {
     limit = 1000,
     offset = 0,
@@ -45,37 +43,26 @@ export default function jsonFilter(state, data, query) {
     };
   }
 
+  const { data } = state.content;
   let json;
   try {
     state.timer?.update('json-parse');
     json = JSON.parse(data);
   } catch (e) {
     const msg = `failed to parse json: ${e.message}`;
-    if (raw) {
-      log.warn(msg);
-      return new PipelineResponse(data, {
-        headers: {
-          'content-type': 'text/plain',
-        },
-      });
+    if (!raw) {
+      throw new PipelineStatusError(502, msg);
     }
-
-    log.error(msg);
-    return new PipelineResponse('', {
-      status: 502,
-      headers: {
-        'x-error': cleanupHeaderValue(msg),
-      },
-    });
+    log.warn(msg);
+    res.body = data;
+    res.headers.set('content-type', 'text/plain; charset=utf-8');
+    return;
   }
 
   // when raw request, only handle multisheets.
   if (raw && !(NAMES_KEY in json)) {
-    return new PipelineResponse(data, {
-      headers: {
-        'content-type': 'application/json',
-      },
-    });
+    res.body = data;
+    return;
   }
 
   // if single sheet, convert it to multisheet
@@ -87,14 +74,7 @@ export default function jsonFilter(state, data, query) {
   }
 
   if (!json[NAMES_KEY]) {
-    const msg = 'multisheet data invalid. missing ":names" property.';
-    log.error(msg);
-    return new PipelineResponse('', {
-      status: 502,
-      headers: {
-        'x-error': cleanupHeaderValue(msg),
-      },
-    });
+    throw new PipelineStatusError(502, 'multisheet data invalid. missing ":names" property.');
   }
 
   state.timer?.update('json-filter');
@@ -111,14 +91,7 @@ export default function jsonFilter(state, data, query) {
       sheetNames.push(name);
     });
   if (sheetNames.length === 0 && requestedSheets.length > 0) {
-    const msg = `filtered result does not contain selected sheet(s): ${requestedSheets.join(',')}`;
-    log.info(msg);
-    return new PipelineResponse('', {
-      status: 404,
-      headers: {
-        'x-error': cleanupHeaderValue(msg),
-      },
-    });
+    throw new PipelineStatusError(404, `filtered result does not contain selected sheet(s): ${requestedSheets.join(',')}`);
   }
 
   let body;
@@ -135,9 +108,5 @@ export default function jsonFilter(state, data, query) {
   }
   body[TYPE_KEY] = type;
   state.timer?.update('json-stringify');
-  return new PipelineResponse(JSON.stringify(body), {
-    headers: {
-      'content-type': 'application/json',
-    },
-  });
+  res.body = JSON.stringify(body);
 }