@adobe/helix-html-pipeline 6.24.3 → 6.25.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [6.25.0](https://github.com/adobe/helix-html-pipeline/compare/v6.24.3...v6.25.0) (2025-04-09)
+
+
+### Features
+
+* read metadata directly from source if required ([#853](https://github.com/adobe/helix-html-pipeline/issues/853)) ([3160ce7](https://github.com/adobe/helix-html-pipeline/commit/3160ce7b4ce61204666026f4c4d46f295ef6b17f))
+
 ## [6.24.3](https://github.com/adobe/helix-html-pipeline/compare/v6.24.2...v6.24.3) (2025-04-07)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-html-pipeline",
-  "version": "6.24.3",
+  "version": "6.25.0",
   "description": "Helix HTML Pipeline",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/html-pipe.js

@@ -36,6 +36,7 @@ import { PipelineResponse } from './PipelineResponse.js';
 import { validatePathInfo } from './utils/path.js';
 import fetchMappedMetadata from './steps/fetch-mapped-metadata.js';
 import { applyMetaLastModified, setLastModified } from './utils/last-modified.js';
+import fetchSourcedMetadata from './steps/fetch-sourced-metadata.js';
 
 /**
  * Fetches the content and if not found, fetches the 404.html
@@ -132,6 +133,7 @@ export async function htmlPipe(state, req) {
     state.timer?.update('metadata-fetch');
     await Promise.all([
       contentPromise,
+      fetchSourcedMetadata(state, res),
       fetchMappedMetadata(state, res),
     ]);
 

package/src/steps/fetch-sourced-metadata.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2021 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { PipelineStatusError } from '../PipelineStatusError.js';
+import { Modifiers } from '../utils/modifiers.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
+
+/**
+ * Loads metadata from the metadata sources if required. this happens when the amount of metadata
+ * was too large to include in the config response. the metadata is loaded if
+ * `state.metadata` is empty and `config.metadata.source` is not.
+ *
+ * @type PipelineStep
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res
+ * @returns {Promise<void>}
+ */
+// eslint-disable-next-line no-unused-vars
+export default async function fetchSourcedMetadata(state, res) {
+  if (!state.metadata.isEmpty()) {
+    return;
+  }
+  const sources = state.config.metadata?.source || [];
+  if (sources.length === 0) {
+    return;
+  }
+
+  const { contentBusId, partition } = state;
+  const metadatas = {};
+  await Promise.all(sources.map(async (src) => {
+    metadatas[src] = [];
+    const key = `${contentBusId}/${partition}/${src}`;
+    const ret = await state.s3Loader.getObject('helix-content-bus', key);
+    if (ret.status === 200) {
+      let json;
+      try {
+        json = JSON.parse(ret.body);
+      } catch (e) {
+        throw new PipelineStatusError(500, `failed parsing of ${key}: ${e.message}`);
+      }
+      const { data } = json.default ?? json;
+      if (!data) {
+        state.log.info(`default sheet missing in ${key}`);
+        return;
+      }
+
+      if (!Array.isArray(data)) {
+        throw new PipelineStatusError(500, `failed loading of ${key}: data must be an array`);
+      }
+      metadatas[src] = data;
+      recordLastModified(state, res, 'content', extractLastModified(ret.headers));
+    } else if (ret.status !== 404) {
+      throw new PipelineStatusError(502, `failed to load ${key}: ${ret.status}`);
+    }
+  }));
+  // aggregate the metadata in the same order as specified
+  const metadata = [];
+  for (const src of sources) {
+    metadata.push(...metadatas[src]);
+  }
+  state.metadata = Modifiers.fromModifierSheet(metadata);
+}

package/src/utils/modifiers.js

@@ -124,6 +124,14 @@ export class Modifiers {
     return new Modifiers(res);
   }
 
+  /**
+   * Returns true if there are no modifiers.
+   * @returns {boolean}
+   */
+  isEmpty() {
+    return this.modifiers.length === 0;
+  }
+
   constructor(config) {
     this.modifiers = Object.entries(config).map(([url, mods]) => {
       const pat = url.indexOf('*') >= 0 ? globToRegExp(url) : url;