@adobe/helix-config 2.7.1 → 2.8.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [2.8.0](https://github.com/adobe/helix-config/compare/v2.7.1...v2.8.0) (2024-04-24)
+
+
+### Features
+
+* add support for robots.txt ([#57](https://github.com/adobe/helix-config/issues/57)) ([dedfec3](https://github.com/adobe/helix-config/commit/dedfec38c0ff97953b71ea3d71a4547f981eb56d)), closes [#56](https://github.com/adobe/helix-config/issues/56)
+
 ## [2.7.1](https://github.com/adobe/helix-config/compare/v2.7.0...v2.7.1) (2024-04-24)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/helix-config",
-  "version": "2.7.1",
+  "version": "2.8.0",
   "description": "Helix Config",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/config-legacy.js

@@ -66,6 +66,19 @@ async function fetchConfigAll(ctx, contentBusId, partition) {
   return res.json();
 }
 
+/**
+ * Retrieves the robots.txt from the code-bus
+ * @param {ConfigContext} ctx the context
+ * @param {string} owner
+ * @param {string} repo
+ * @returns {Promise<string|null>} the robots.txt
+ */
+export async function fetchRobotsTxt(ctx, owner, repo) {
+  const key = `${owner}/${repo}/main/robots.txt`;
+  const res = await ctx.loader.getObject(HELIX_CODE_BUS, key);
+  return res.body;
+}
+
 /**
  * Loads the content from a helix 4 project.
  * @param {ConfigContext} ctx
@@ -131,6 +144,12 @@ export async function resolveLegacyConfig(ctx, rso, scope) {
     if (configAllLive?.metadata) {
       config.metadata.live = configAllLive.metadata;
     }
+    const robots = await fetchRobotsTxt(ctx, rso.org, rso.site);
+    if (robots) {
+      config.robots = {
+        txt: robots,
+      };
+    }
   }
   return config;
 }

package/src/config-view.js

@@ -18,7 +18,7 @@ import {
   SCOPE_DELIVERY,
   SCOPE_RAW,
 } from './ConfigContext.js';
-import { resolveLegacyConfig } from './config-legacy.js';
+import { resolveLegacyConfig, fetchRobotsTxt } from './config-legacy.js';
 
 /**
  * @typedef Config
@@ -167,6 +167,12 @@ async function resolveConfig(ctx, rso, scope) {
       preview: await loadMetadata(ctx, config, 'preview'),
       live: await loadMetadata(ctx, config, 'live'),
     };
+    if (!config.robots) {
+      const txt = await fetchRobotsTxt(ctx, config.code.owner, config.code.repo);
+      if (txt) {
+        config.robots = { txt };
+      }
+    }
   }
   if (scope === SCOPE_PIPELINE || scope === SCOPE_DELIVERY) {
     config.head = await loadHeadHtml(ctx, config, rso.ref);
@@ -254,6 +260,7 @@ export async function getConfigResponse(ctx, opts) {
       },
     };
     delete adminConfig.public;
+    delete adminConfig.robots;
     return new PipelineResponse(JSON.stringify(adminConfig, null, 2), {
       headers: {
         'content-type': 'application/json',
@@ -288,6 +295,7 @@ export async function getConfigResponse(ctx, opts) {
       cdn: config.cdn,
       folders: config.folders,
       public: config.public,
+      robots: config.robots,
     };
     return new PipelineResponse(JSON.stringify(pipelineConfig, null, 2), {
       headers: {

package/src/schemas/profile.schema.json

@@ -46,6 +46,9 @@
     },
     "metadata": {
       "$ref": "https://ns.adobe.com/helix/config/metadata-source"
+    },
+    "robots": {
+      "$ref": "https://ns.adobe.com/helix/config/robots"
     }
   },
   "required": [

package/src/schemas/robots.schema.cjs

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+module.exports = require('./robots.schema.json');

package/src/schemas/robots.schema.json

@@ -0,0 +1,23 @@
+{
+  "meta:license": [
+    "Copyright 2024 Adobe. All rights reserved.",
+    "This file is licensed to you under the Apache License, Version 2.0 (the \"License\");",
+    "you may not use this file except in compliance with the License. You may obtain a copy",
+    "of the License at http://www.apache.org/licenses/LICENSE-2.0",
+    "",
+    "Unless required by applicable law or agreed to in writing, software distributed under",
+    "the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS",
+    "OF ANY KIND, either express or implied. See the License for the specific language",
+    "governing permissions and limitations under the License."
+  ],
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://ns.adobe.com/helix/config/robots",
+  "type": "object",
+  "title": "robots",
+  "properties": {
+    "txt": {
+      "type": "string"
+    }
+  },
+  "additionalProperties": false
+}

package/src/schemas/site.schema.json

@@ -51,6 +51,9 @@
     },
     "metadata": {
       "$ref": "https://ns.adobe.com/helix/config/metadata-source"
+    },
+    "robots": {
+      "$ref": "https://ns.adobe.com/helix/config/robots"
     }
   },
   "required": [

package/src/storage/config-store.js

@@ -36,6 +36,7 @@ const FRAGMENTS = {
     'access/preview': 'object',
     'access/live': 'object',
     public: 'object',
+    robots: 'object',
   },
 };
 

package/src/storage/config-validator.js

@@ -26,6 +26,7 @@ import markupSchema from '../schemas/content-source-markup.schema.cjs';
 import metadataSchema from '../schemas/metadata-source.schema.cjs';
 import onedriveSchema from '../schemas/content-source-onedrive.schema.cjs';
 import profileSchema from '../schemas/profile.schema.cjs';
+import robotsSchema from '../schemas/robots.schema.cjs';
 import sidekickSchema from '../schemas/sidekick.schema.cjs';
 import siteSchema from '../schemas/site.schema.cjs';
 
@@ -43,6 +44,7 @@ const SCHEMAS = [
   metadataSchema,
   onedriveSchema,
   profileSchema,
+  robotsSchema,
   sidekickSchema,
   siteSchema,
 ];

package/types/profile-config.d.ts

@@ -31,6 +31,7 @@ export interface HelixProfileConfig {
   access?: Access;
   sidekick?: SidekickConfig;
   metadata?: Metadata;
+  robots?: Robots;
 }
 /**
  * Defines the content bus location and source.
@@ -247,3 +248,6 @@ export interface SidekickPlugin {
 export interface Metadata {
   source?: [] | [string];
 }
+export interface Robots {
+  txt?: string;
+}

package/types/site-config.d.ts

@@ -34,6 +34,7 @@ export interface HelixSiteConfig {
   access?: Access;
   sidekick?: SidekickConfig;
   metadata?: Metadata;
+  robots?: Robots;
 }
 /**
  * Defines the content bus location and source.
@@ -250,3 +251,6 @@ export interface SidekickPlugin {
 export interface Metadata {
   source?: [] | [string];
 }
+export interface Robots {
+  txt?: string;
+}