@adobe/helix-html-pipeline 6.15.11 → 6.16.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [6.16.0](https://github.com/adobe/helix-html-pipeline/compare/v6.15.11...v6.16.0) (2024-11-18)
+
+
+### Features
+
+* respect last-modified from metadata sheet ([#744](https://github.com/adobe/helix-html-pipeline/issues/744)) ([50b3e07](https://github.com/adobe/helix-html-pipeline/commit/50b3e07d909651368e4b5e2c473391b63aa05913)), closes [#743](https://github.com/adobe/helix-html-pipeline/issues/743)
+
 ## [6.15.11](https://github.com/adobe/helix-html-pipeline/compare/v6.15.10...v6.15.11) (2024-11-18)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-html-pipeline",
-  "version": "6.15.11",
+  "version": "6.16.0",
   "description": "Helix HTML Pipeline",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -55,6 +55,7 @@
     "lodash.escape": "4.0.1",
     "mdast-util-to-hast": "13.2.0",
     "mdast-util-to-string": "4.0.0",
+    "micromark-util-subtokenize": "2.0.1",
     "mime": "4.0.4",
     "rehype-format": "5.0.1",
     "rehype-parse": "9.0.1",

package/src/PipelineResponse.js

@@ -30,7 +30,7 @@ export class PipelineResponse {
       document: undefined,
       headers,
       error: undefined,
-      lastModifiedTime: 0,
+      lastModifiedSources: {},
     });
   }
 

package/src/html-pipe.js

@@ -35,6 +35,7 @@ import { PipelineStatusError } from './PipelineStatusError.js';
 import { PipelineResponse } from './PipelineResponse.js';
 import { validatePathInfo } from './utils/path.js';
 import fetchMappedMetadata from './steps/fetch-mapped-metadata.js';
+import { applyMetaLastModified, setLastModified } from './utils/last-modified.js';
 
 /**
  * Fetches the content and if not found, fetches the 404.html
@@ -140,6 +141,7 @@ export async function htmlPipe(state, req) {
       log[level](`pipeline status: ${res.status} ${res.error}`);
       res.headers.set('x-error', cleanupHeaderValue(res.error));
       if (res.status < 500) {
+        setLastModified(state, res);
         await setCustomResponseHeaders(state, req, res);
       }
       return res;
@@ -166,8 +168,10 @@ export async function htmlPipe(state, req) {
       await render(state, req, res);
       state.timer?.update('serialize');
       await tohtml(state, req, res);
+      await applyMetaLastModified(state, res);
     }
 
+    setLastModified(state, res);
     await setCustomResponseHeaders(state, req, res);
     await setXSurrogateKeyHeader(state, req, res);
   } catch (e) {

package/src/json-pipe.js

@@ -14,7 +14,7 @@ import initConfig from './steps/init-config.js';
 import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
 import { PipelineResponse } from './PipelineResponse.js';
 import jsonFilter from './utils/json-filter.js';
-import { extractLastModified, updateLastModified } from './utils/last-modified.js';
+import { extractLastModified, recordLastModified, setLastModified } from './utils/last-modified.js';
 import { getPathInfo } from './utils/path.js';
 import { PipelineStatusError } from './PipelineStatusError.js';
 
@@ -85,7 +85,7 @@ async function fetchJsonContent(state, req, res) {
     state.content.sourceLocation = ret.headers.get('x-amz-meta-x-source-location');
     log.info(`source-location: ${state.content.sourceLocation}`);
 
-    updateLastModified(state, res, extractLastModified(ret.headers));
+    recordLastModified(state, res, 'content', extractLastModified(ret.headers));
   } else {
     // also add code surrogate key in case json is later added to code bus (#688)
     state.content.sourceBus = 'code|content';
@@ -191,6 +191,7 @@ export async function jsonPipe(state, req) {
     const keys = await computeSurrogateKeys(state);
     res.headers.set('x-surrogate-key', keys.join(' '));
 
+    setLastModified(state, res);
     await setCustomResponseHeaders(state, req, res);
     return res;
   } catch (e) {

package/src/sitemap-pipe.js

@@ -18,7 +18,7 @@ import setCustomResponseHeaders from './steps/set-custom-response-headers.js';
 import { PipelineStatusError } from './PipelineStatusError.js';
 import { PipelineResponse } from './PipelineResponse.js';
 import initConfig from './steps/init-config.js';
-import { extractLastModified, updateLastModified } from './utils/last-modified.js';
+import { extractLastModified, recordLastModified, setLastModified } from './utils/last-modified.js';
 
 async function generateSitemap(state) {
   const {
@@ -103,7 +103,7 @@ export async function sitemapPipe(state, req) {
       const ret = await generateSitemap(state);
       if (ret.status === 200) {
         res.status = 200;
-        updateLastModified(state, res, extractLastModified(ret.headers));
+        recordLastModified(state, res, 'content', extractLastModified(ret.headers));
         delete res.error;
         state.content.data = ret.body;
       }
@@ -115,6 +115,7 @@ export async function sitemapPipe(state, req) {
 
     state.timer?.update('serialize');
     await renderCode(state, req, res);
+    setLastModified(state, res);
     await setCustomResponseHeaders(state, req, res);
     await setXSurrogateKeyHeader(state, req, res);
   } catch (e) {

package/src/steps/fetch-404.js

@@ -9,7 +9,7 @@
  * OF ANY KIND, either express or implied. See the License for the specific language
  * governing permissions and limitations under the License.
  */
-import { extractLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 import { getPathKey } from './set-x-surrogate-key-header.js';
 
 /**
@@ -29,7 +29,7 @@ export default async function fetch404(state, req, res) {
     // override last-modified if source-last-modified is set
     const lastModified = extractLastModified(ret.headers);
     if (lastModified) {
-      ret.headers.set('last-modified', lastModified);
+      recordLastModified(state, res, 'content', lastModified);
     }
 
     // keep 404 response status

package/src/steps/fetch-content.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 import { computeSurrogateKey } from '@adobe/helix-shared-utils';
-import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 
 /**
  * Loads the content from either the content-bus or code-bus and stores it in `state.content`
@@ -65,7 +65,7 @@ export default async function fetchContent(state, req, res) {
     state.content.sourceLocation = ret.headers.get('x-amz-meta-x-source-location');
     log.info(`source-location: ${state.content.sourceLocation}`);
 
-    updateLastModified(state, res, extractLastModified(ret.headers));
+    recordLastModified(state, res, 'content', extractLastModified(ret.headers));
 
     // reject requests to /index *after* checking for redirects
     // (https://github.com/adobe/helix-pipeline-service/issues/290)

package/src/steps/fetch-mapped-metadata.js

@@ -12,7 +12,7 @@
 
 import { PipelineStatusError } from '../PipelineStatusError.js';
 import { Modifiers } from '../utils/modifiers.js';
-import { extractLastModified, updateLastModified } from '../utils/last-modified.js';
+import { extractLastModified, recordLastModified } from '../utils/last-modified.js';
 
 /**
  * Loads metadata for a mapped path and puts it into `state.mappedMetadata`. If path is not
@@ -53,7 +53,7 @@ export default async function fetchMappedMetadata(state, res) {
     state.mappedMetadata = Modifiers.fromModifierSheet(
       data,
     );
-    updateLastModified(state, res, extractLastModified(ret.headers));
+    recordLastModified(state, res, 'metadata', extractLastModified(ret.headers));
     return;
   }
   if (ret.status !== 404) {

package/src/steps/init-config.js

@@ -11,7 +11,7 @@
  */
 import { Modifiers } from '../utils/modifiers.js';
 import { getOriginalHost } from './utils.js';
-import { updateLastModified } from '../utils/last-modified.js';
+import { recordLastModified } from '../utils/last-modified.js';
 
 function replaceParams(str, info) {
   if (!str) {
@@ -44,5 +44,5 @@ export default function initConfig(state, req, res) {
   state.previewHost = replaceParams(config.cdn?.preview?.host, state);
   state.liveHost = replaceParams(config.cdn?.live?.host, state);
   state.prodHost = config.cdn?.prod?.host || getOriginalHost(req.headers);
-  updateLastModified(state, res, state.config.lastModified);
+  recordLastModified(state, res, 'config', state.config.lastModified);
 }

package/src/utils/last-modified.js

@@ -11,30 +11,43 @@
  */
 
 /**
- * Updates the context.content.lastModified if the time in `timeString` is newer than the existing
- * one if none exists yet. please note that it generates helper property `lastModifiedTime` in
- * unix epoch format.
- *
- * the date string will be a "http-date": https://httpwg.org/specs/rfc7231.html#http.date
+ * Records the last modified for the given source.
  *
  * @param {PipelineState} state
- * @param {PipelineResponse} res the pipeline context
+ * @param {PipelineResponse} res the pipeline response
+ * @param {string} source the source providing a last-modified date
  * @param {string} httpDate http-date string
  */
-export function updateLastModified(state, res, httpDate) {
+export function recordLastModified(state, res, source, httpDate) {
   if (!httpDate) {
     return;
   }
   const { log } = state;
-  const time = new Date(httpDate).getTime();
-  if (Number.isNaN(time)) {
-    log.warn(`updateLastModified date is invalid: ${httpDate}`);
+  const date = new Date(httpDate);
+  if (Number.isNaN(date.valueOf())) {
+    log.warn(`last-modified date is invalid: ${httpDate} for ${source}`);
     return;
   }
+  res.lastModifiedSources[source] = {
+    time: date.valueOf(),
+    date: date.toUTCString(),
+  };
+}
 
-  if (time > (res.lastModifiedTime ?? 0)) {
-    res.lastModifiedTime = time;
-    res.headers.set('last-modified', httpDate);
+/**
+ * Calculates the last modified by using the latest date from all the recorded sources
+ * and sets it on the `last-modified` header.
+ *
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res the pipeline response
+ */
+export function setLastModified(state, res) {
+  let latestTime = 0;
+  for (const { time, date } of Object.values(res.lastModifiedSources)) {
+    if (time > latestTime) {
+      latestTime = time;
+      res.headers.set('last-modified', date);
+    }
   }
 }
 
@@ -55,3 +68,13 @@ export function extractLastModified(headers) {
   }
   return headers.get('last-modified');
 }
+
+/**
+ * Sets the metadata last modified entry to the one define in the page specific metadata if
+ * it exists. this allows to control the last-modified per metadata record.
+ * @param {PipelineState} state
+ * @param {PipelineResponse} res the pipeline response
+ */
+export function applyMetaLastModified(state, res) {
+  recordLastModified(state, res, 'metadata', state.content.meta.page['last-modified']);
+}