@adobe/helix-html-pipeline 1.0.0

package/src/PipelineContent.d.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License; Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing; software distributed under
+ * the License is distributed on an "AS IS" BASIS; WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND; either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import {Node} from "unist";
+import GithubSlugger from 'github-slugger';
+
+declare enum SourceType {
+  CONTENT = 'content',
+  CODE = 'code',
+}
+
+declare class PipelineContent {
+  /**
+   * source of content: `content` or `code`
+   * @default 'content'
+   */
+  sourceBus: SourceType;
+
+  /**
+   * http status of the content fetch response
+   */
+  status: number;
+
+  /**
+   * raw data of the content
+   */
+  data: string;
+
+  /**
+   * http headers of the content fetch response
+   */
+  headers: object;
+
+  /**
+   * the source location of the loaded content
+   */
+  sourceLocation: string;
+
+  /**
+   * Markdown AST of the parsed content
+   */
+  mdast: Node;
+
+  /**
+   * document specific metadata
+   */
+  meta: object;
+  title: string;
+  intro: string;
+  image: string;
+
+  /**
+   * slugger to use for heading id calculations
+   */
+  slugger: GithubSlugger;
+
+  /**
+   * The transformed document (jsom) representation
+   */
+  document: Document;
+}

package/src/PipelineContent.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * State of the pipeline
+ * @class PipelineState
+ */
+export class PipelineContent {
+  /**
+   * Creates the pipeline content
+   */
+  constructor() {
+    Object.assign(this, {
+      sourceBus: 'content',
+    });
+  }
+}

package/src/PipelineRequest.d.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License; Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing; software distributed under
+ * the License is distributed on an "AS IS" BASIS; WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND; either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import {PathInfo, PipelineContent, S3Loader} from "./index";
+
+declare interface PipelineRequestInit {
+  method?:string,
+  headers?:Map<string, string> | object;
+  body?:string;
+}
+
+declare class PipelineRequest {
+  constructor(url:URL|string, opts?:RequestInit);
+  url: URL;
+  method: string;
+  headers: Map<string, string>;
+  body: string;
+}

package/src/PipelineRequest.js

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Request of a pipeline
+ * @class PipelineRequest
+ */
+export class PipelineRequest {
+  /**
+   * Creates the pipeline request
+   * @param {URL|string} url
+   * @param {PipelineRequestInit} [init]
+   */
+  constructor(url, init = {}) {
+    let headers = init.headers ?? new Map();
+    if (typeof headers.get !== 'function') {
+      headers = new Map(Object.entries(init.headers));
+    }
+
+    Object.assign(this, {
+      url: url instanceof URL ? url : new URL(url),
+      method: init.method ?? 'GET',
+      body: init.body,
+      headers,
+    });
+  }
+}

package/src/PipelineResponse.d.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License; Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing; software distributed under
+ * the License is distributed on an "AS IS" BASIS; WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND; either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+declare interface PipelineResponseInit {
+  status?: number;
+  headers: Map<string, string> | object;
+}
+
+declare class PipelineResponse {
+  constructor(body?:string, init?:PipelineResponseInit);
+  status: number;
+  document?: Document;
+  body: string;
+  headers: Map<string, string>;
+  error: any;
+
+  /**
+   * the last modified time of the response. this is the max of the last-modified times of the
+   * various source. e.g. if the `head.html` is newer than the `content`, then the last-modified
+   * header will be the one of the `head.html`
+   */
+  lastModifiedTime: number;
+}
+

package/src/PipelineResponse.js

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Response of a pipeline
+ * @class PipelineResponse
+ */
+export class PipelineResponse {
+  /**
+   * Creates the pipeline response
+   */
+  constructor(body = undefined, init = {}) {
+    let headers = init.headers ?? new Map([['content-type', 'text/html; charset=utf-8']]);
+    if (typeof headers.get !== 'function') {
+      headers = new Map(Object.entries(init.headers));
+    }
+
+    Object.assign(this, {
+      status: init.status ?? 200,
+      body,
+      document: undefined,
+      headers,
+      error: undefined,
+      lastModifiedTime: 0,
+    });
+  }
+
+  /**
+   * Returns the json parsed object of `this.body`.
+   * @returns {object}
+   */
+  json() {
+    return JSON.parse(this.body);
+  }
+}

package/src/PipelineState.d.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License; Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing; software distributed under
+ * the License is distributed on an "AS IS" BASIS; WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND; either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import {PathInfo, S3Loader, PipelineTimer} from "./index";
+import {PipelineContent} from "./PipelineContent";
+
+declare interface PipelineOptions {
+  log: Console;
+  s3Loader: S3Loader;
+  owner: string;
+  repo: string;
+  ref: string;
+  partition: string;
+  path: string;
+  contentBusId: string;
+  timer: PipelineTimer;
+}
+
+declare class PipelineState {
+  constructor(opts: PipelineOptions);
+  log: Console;
+  info: PathInfo;
+  content: PipelineContent;
+  contentBusId: string;
+  s3Loader: S3Loader;
+
+  /**
+   * Content bus partition
+   * @example 'live'
+   * @example 'preview'
+   */
+  partition: string;
+
+  /**
+   * Repository owner
+   */
+  owner: string;
+
+  /**
+   * Repository name
+   */
+  repo: string;
+
+  /**
+   * Repository ref
+   */
+  ref: string;
+
+  /**
+   * helix-config.json once loaded (contains fstab, head.html, etc)
+   */
+  helixConfig?: object;
+
+  /**
+   * metadata.json once loaded
+   */
+  metadata?: object;
+
+  /**
+   * optional timer that is used to measure the timing
+   */
+  timer?: PipelineTimer;
+}
+

package/src/PipelineState.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { getPathInfo } from './utils/path.js';
+import { PipelineContent } from './PipelineContent.js';
+
+/**
+ * State of the pipeline
+ * @class PipelineState
+ */
+export class PipelineState {
+  /**
+   * Creates the pipeline state
+   * @param {PipelineOptions} opts
+   */
+  constructor(opts) {
+    Object.assign(this, {
+      log: opts.log ?? console,
+      info: getPathInfo(opts.path),
+      content: new PipelineContent(),
+      // todo: compute content-bus id from fstab
+      contentBusId: opts.contentBusId,
+      owner: opts.owner,
+      repo: opts.repo,
+      ref: opts.ref,
+      partition: opts.partition,
+      helixConfig: undefined,
+      metadata: undefined,
+      s3Loader: opts.s3Loader,
+      timer: opts.timer,
+    });
+  }
+}

package/src/PipelineStatusError.d.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+declare class PipelineStatusError extends Error {
+  code:number;
+}

package/src/PipelineStatusError.js

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+export class PipelineStatusError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.code = code;
+  }
+}

package/src/html-pipe.js

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { cleanupHeaderValue } from '@adobe/helix-shared-utils';
+import addHeadingIds from './steps/add-heading-ids.js';
+import createPageBlocks from './steps/create-page-blocks.js';
+import createPictures from './steps/create-pictures.js';
+import extractMetaData from './steps/extract-metadata.js';
+import fetchConfig from './steps/fetch-config.js';
+import fetchContent from './steps/fetch-content.js';
+import fetchMetadata from './steps/fetch-metadata.js';
+import fixSections from './steps/fix-sections.js';
+import folderMapping from './steps/folder-mapping.js';
+import getMetadata from './steps/get-metadata.js';
+import html from './steps/make-html.js';
+import parseMarkdown from './steps/parse-markdown.js';
+import removeHlxProps from './steps/removeHlxProps.js';
+import render from './steps/render.js';
+import renderCode from './steps/render-code.js';
+import rewriteBlobImages from './steps/rewrite-blob-images.js';
+import rewriteIcons from './steps/rewrite-icons.js';
+import setXSurrogateKeyHeader from './steps/set-x-surrogate-key-header.js';
+import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
+import splitSections from './steps/split-sections.js';
+import tohtml from './steps/stringify-response.js';
+import { PipelineStatusError } from './PipelineStatusError.js';
+import { PipelineResponse } from './PipelineResponse.js';
+
+/**
+ * Runs the default pipeline and returns the response.
+ * @param {PipelineState} state
+ * @param {PipelineRequest} req
+ * @returns {PipelineResponse}
+ */
+export async function htmlPipe(state, req) {
+  const { log } = state;
+
+  /** @type PipelineResponse */
+  const res = new PipelineResponse();
+
+  try { // fetch config first, since we need to compute the content-bus-id from the fstab ...
+    await fetchConfig(state, req, res);
+    // ...and apply the folder mapping
+    await folderMapping(state, req, res);
+
+    // load metadata and content in parallel
+    await Promise.all([
+      fetchMetadata(state, req, res),
+      fetchContent(state, req, res),
+    ]);
+
+    if (res.error) {
+      // if content loading produced an error, we're done.
+      log.error(`error running pipeline: ${res.status} ${res.error}`);
+      res.headers.set('x-error', cleanupHeaderValue(res.error));
+      return res;
+    }
+
+    if (state.content.sourceBus === 'code') {
+      await renderCode(state, req, res);
+    } else {
+      await parseMarkdown(state);
+      await splitSections(state);
+      await getMetadata(state); // this one extracts the metadata from the mdast
+      await html(state);
+      await rewriteBlobImages(state);
+      await rewriteIcons(state);
+      await fixSections(state);
+      await createPageBlocks(state);
+      await createPictures(state);
+      await extractMetaData(state, req);
+      await addHeadingIds(state);
+      await render(state, req, res);
+      await removeHlxProps(state, req, res);
+      await tohtml(state, req, res);
+    }
+
+    await setCustomResponseHeaders(state, req, res);
+    await setXSurrogateKeyHeader(state, req, res);
+  } catch (e) {
+    res.error = e.message;
+    if (e instanceof PipelineStatusError) {
+      res.status = e.code;
+    } else {
+      res.status = 500;
+    }
+    log.error(`error running pipeline: ${res.status} ${res.error}`, e);
+    res.headers.set('x-error', cleanupHeaderValue(res.error));
+  }
+
+  return res;
+}

package/src/index.d.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License; Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing; software distributed under
+ * the License is distributed on an "AS IS" BASIS; WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND; either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import {PipelineState} from "./PipelineState";
+import {PipelineRequest} from "./PipelineRequest";
+import {PipelineResponse} from "./PipelineResponse";
+
+export * from './PipelineContent';
+export * from './PipelineRequest';
+export * from './PipelineResponse';
+export * from './PipelineState';
+export * from './PipelineStatusError';
+
+/**
+ * Path Info
+ */
+declare interface PathInfo {
+  /**
+   * Resource path
+   * @example '/express/index.md'
+   * @example '/en/query-index.json'
+   */
+  resourcePath: string;
+
+  /**
+   * Web path
+   * @example '/express/'
+   * @example '/blogs/may-21'
+   * @example '/en/query-index.json'
+   */
+  path: string;
+
+  /**
+   * extension / type of resource
+   * @example '.html'
+   * @example '.md'
+   */
+  extension: string;
+
+  /**
+   * selector
+   * @example 'plain'
+   */
+  selector?: string;
+
+  /**
+   * original path as passed via request
+   */
+  originalPath: string;
+
+  /**
+   * original filename as passed via request
+   */
+  originalFilename: string;
+
+  /**
+   * original extension as passed via request
+   */
+  originalExtension: string;
+}
+
+declare interface S3Loader {
+  /**
+   * Loads a S3 object from the given bucket and key.
+   * @param {string} bucketId
+   * @param {string} key
+   */
+  getObject(bucketId, key): Promise<PipelineResponse>;
+
+  /**
+   * Retrieves the head metadata of a S3 object from the given bucket and key.
+   * @param {string} bucketId
+   * @param {string} key
+   */
+  headObject(bucketId, key): Promise<PipelineResponse>;
+}
+
+/**
+ * Timer
+ */
+declare interface PipelineTimer {
+  /**
+   * Records the timestamp of the given `step`
+   */
+  update(step:string): void;
+}
+
+declare interface PipelineStep {
+  async(state: PipelineState, req: PipelineRequest, resp: PipelineResponse): Promise<void>;
+}

package/src/index.js

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+export * from './html-pipe.js';
+export * from './json-pipe.js';
+export * from './PipelineContent.js';
+export * from './PipelineRequest.js';
+export * from './PipelineResponse.js';
+export * from './PipelineState.js';
+export * from './PipelineStatusError.js';