@adobe/helix-config-storage 1.0.0

package/src/config-merge.js

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-disable no-continue, no-param-reassign */
+
+/**
+ * @typedef RootProperty
+ * @property {boolean} atomic whether the property should be overwritten atomically
+ * @property {boolean} isModified whether the property need special merging
+ */
+
+const ROOT_PROPERTIES = {
+  name: { atomic: true },
+  title: { atomic: true },
+  description: { atomic: true },
+  content: { atomic: true },
+  code: { atomic: true },
+  folders: {},
+  headers: { isModifier: true },
+  cdn: {},
+  access: {},
+  sidekick: {},
+  metadata: { isModifier: true },
+  robots: {},
+  extends: {},
+  tokens: {},
+  public: {},
+  groups: {},
+};
+
+const FORCED_TYPES = {
+  '.cdn.prod.route': 'array',
+};
+
+/**
+ * Merges the `src` object into the `dst` object.
+ * - if the dst property is a scalar, it will be overwritten by the dst property
+ * - if the src property is an empty string, it will delete the dst property
+ * - if the dst property is an array, it will be expanded by the src property uniquely
+ * - some well known paths are forced to be arrays
+ * @param {object} dst
+ * @param {object} src
+ * @param {boolean} isModifier specifies that the objects are modifiers sheets and need special
+ *                             merging
+ */
+function merge(dst, src, path, isModifier) {
+  for (const [key, value] of Object.entries(src)) {
+    const itemPath = `${path}.${key}`;
+    if (value === '') {
+      // remove key if value is empty string (keep false, 0)
+      delete dst[key];
+    } else if (typeof value !== 'object') {
+      // handle non object
+      if (key in dst) {
+        let dstValue = dst[key];
+        if (!Array.isArray(dstValue) && FORCED_TYPES[itemPath] === 'array') {
+          dstValue = [dst[key]];
+          dst[key] = dstValue;
+        }
+        if (!Array.isArray(dstValue)) {
+          dst[key] = value;
+        } else if (!dstValue.includes(value)) {
+          // only add to array if not exist yet
+          dstValue.push(value);
+        }
+      } else {
+        dst[key] = value;
+      }
+    } else if (Array.isArray(value)) {
+      // handle array
+      if (key in dst) {
+        if (!Array.isArray(dst[key])) {
+          dst[key] = [dst[key]];
+        }
+      } else {
+        dst[key] = [];
+      }
+      const dstArray = dst[key];
+      for (const item of value) {
+        if (typeof item === 'object') {
+          // todo: handle arrays in arrays eventually
+          // special case for modifier sheets
+          if (isModifier && item.key) {
+            const idx = dstArray.findIndex((i) => i.key === item.key);
+            if (idx >= 0) {
+              dstArray.splice(idx, 1);
+            }
+            if (!item.value) {
+              // don't set empty modifier
+              continue;
+            }
+          }
+          const dstObj = Object.create(null);
+          dstArray.push(dstObj);
+          merge(dstObj, item, itemPath, isModifier);
+        } else if (!dstArray.includes(item)) {
+          dstArray.push(item);
+        }
+      }
+    } else {
+      // handle plain object
+      // eslint-disable-next-line no-use-before-define
+      mergeObject(dst, key, value, `${path}.${key}`, isModifier);
+    }
+  }
+}
+
+function mergeObject(dst, key, value, path, isModifier) {
+  const dstObj = dst[key] ?? Object.create(null);
+  if (!dst[key]) {
+    dst[key] = dstObj;
+  }
+  merge(dstObj, value, path, isModifier);
+}
+
+function mergeConfig(dst, src) {
+  Object.entries(ROOT_PROPERTIES).forEach(([key, { atomic, isModifier }]) => {
+    if (src[key]) {
+      if (atomic) {
+        dst[key] = src[key];
+      } else {
+        mergeObject(dst, key, src[key], `.${key}`, isModifier);
+      }
+    }
+  });
+}
+
+export function getMergedConfig(site, profile) {
+  if (!profile) {
+    return site;
+  }
+  const ret = Object.create(null);
+  ret.version = 1;
+  mergeConfig(ret, profile);
+  mergeConfig(ret, site);
+  return ret;
+}

package/src/config-store.js

@@ -0,0 +1,359 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-disable no-param-reassign */
+import { isDeepStrictEqual } from 'util';
+import { HelixStorage } from '@adobe/helix-shared-storage';
+import { StatusCodeError } from './status-code-error.js';
+import {
+  createToken, createUser,
+  migrateToken,
+  updateCodeSource,
+  updateContentSource,
+  deepGetOrCreate, deepPut,
+} from './utils.js';
+import { validate as validateSchema } from './config-validator.js';
+
+const FRAGMENTS_COMMON = {
+  content: 'object',
+  code: 'object',
+  folders: 'object',
+  headers: 'object',
+  metadata: 'object',
+  sidekick: 'object',
+  cdn: {
+    '.': 'object',
+    prod: 'object',
+    preview: 'object',
+    live: 'object',
+  },
+  access: {
+    '.': 'object',
+    admin: 'object',
+    preview: 'object',
+    live: 'object',
+  },
+  tokens: {
+    '.': 'tokens',
+    '.param': {
+      name: 'id',
+      '.': 'token',
+    },
+  },
+  groups: {
+    '.': 'object',
+    '.param': {
+      '.': 'object',
+      members: 'object',
+    },
+  },
+  public: 'object',
+  robots: 'object',
+};
+
+const FRAGMENTS = {
+  sites: {
+    ...FRAGMENTS_COMMON,
+    extends: 'object',
+  },
+  profiles: FRAGMENTS_COMMON,
+  org: {
+    users: {
+      '.': 'users',
+      '.param': {
+        name: 'id',
+        '.': 'user',
+      },
+    },
+    groups: {
+      '.': 'object',
+      '.param': {
+        '.': 'object',
+        members: 'object',
+      },
+    },
+  },
+};
+
+export function getFragmentInfo(type, relPath) {
+  if (!relPath) {
+    return null;
+  }
+  const parts = relPath.split('/');
+  let fragment = FRAGMENTS[type];
+  const info = {
+    relPath,
+  };
+  for (const part of parts) {
+    let next = fragment[part];
+    if (!next) {
+      next = fragment['.param'];
+      if (!next) {
+        throw new StatusCodeError(400, 'invalid object path.');
+      }
+      info[next.name] = part;
+    }
+    fragment = next;
+  }
+  if (typeof fragment === 'string') {
+    info.type = fragment;
+  } else {
+    info.type = fragment['.'];
+  }
+  return info;
+}
+
+/**
+ * Redact / transform the token config
+ * @param token
+ */
+function redactToken(token) {
+  return {
+    id: token.id,
+    created: token.created,
+  };
+}
+
+/**
+ * Redact / transform the tokens config
+ * @param tokens
+ */
+function redactTokens(tokens) {
+  return Object.values(tokens).map(redactToken);
+}
+
+/**
+ * redact information from the config
+ * @param {object} config
+ * @param {object} frag
+ */
+function redact(config, frag) {
+  if (!config) {
+    return config;
+  }
+  let ret = config;
+  if (frag?.type === 'tokens') {
+    ret = redactTokens(config);
+  }
+  if (frag?.type === 'token') {
+    ret = redactToken(config);
+  }
+  if (ret.tokens) {
+    // eslint-disable-next-line no-param-reassign
+    ret.tokens = redactTokens(ret.tokens);
+  }
+  return ret;
+}
+
+/**
+ * General purpose config store.
+ */
+export class ConfigStore {
+  /**
+   * @param {string} org the org id
+   * @param {string} type store type (org, sites, profiles, secrets, users)
+   * @param {string} name config name
+   */
+  constructor(org, type = 'org', name = '') {
+    if (!org) {
+      throw new Error('org required');
+    }
+    if (org.includes('/') || type.includes('/') || name.includes('/')) {
+      throw new Error('orgId, type and name must not contain slashes');
+    }
+    this.org = org;
+    this.type = type;
+    this.name = name;
+    this.key = type === 'org'
+      ? `/orgs/${this.org}/${this.name || 'config'}.json`
+      : `/orgs/${this.org}/${this.type}/${this.name}.json`;
+  }
+
+  /**
+   * Validates the config and throws an error if not valid.
+   * @param {AdminContext} ctx
+   * @param {HelixSiteConfig|HelixProfileConfig} data the data of the config to be updated
+   * @returns {Promise<boolean|undefined>}
+   */
+  async validate(ctx, data) {
+    return validateSchema(data, this.type);
+  }
+
+  async create(ctx, data, relPath = '') {
+    if (relPath) {
+      throw new StatusCodeError(409, 'create not supported on substructures.');
+    }
+    const storage = HelixStorage.fromContext(ctx).configBus();
+    if (await storage.head(this.key)) {
+      throw new StatusCodeError(409, 'config already exists.');
+    }
+    if (data.tokens) {
+      throw new StatusCodeError(400, 'creating config with tokens not supported yet.');
+    }
+    if (this.type === 'org' && data.users) {
+      throw new StatusCodeError(400, 'creating org config with users is not supported yet.');
+    }
+    if (this.type !== 'org') {
+      updateContentSource(ctx, data.content);
+      updateCodeSource(ctx, data.code);
+    }
+    await this.validate(ctx, data);
+    await storage.put(this.key, JSON.stringify(data), 'application/json');
+    await this.purge(ctx, null, data);
+  }
+
+  async #list(ctx) {
+    const storage = HelixStorage.fromContext(ctx).configBus();
+    const key = `orgs/${this.org}/${this.type}/`;
+    const list = await storage.list(key);
+    return {
+      [this.type]: list.map((entry) => {
+        const siteKey = entry.key;
+        if (siteKey.endsWith('.json')) {
+          const name = siteKey.split('/').pop();
+          return {
+            path: `/config/${this.org}/${this.type}/${name}`,
+            name: name.substring(0, name.length - 5),
+          };
+        }
+        return null;
+      }).filter((entry) => entry),
+    };
+  }
+
+  async read(ctx, relPath = '') {
+    if (this.name === '' && (this.type === 'sites' || this.type === 'profiles')) {
+      return this.#list(ctx);
+    }
+
+    const storage = HelixStorage.fromContext(ctx).configBus();
+    const buf = await storage.get(this.key);
+    if (!buf) {
+      return null;
+    }
+    let obj = JSON.parse(buf);
+    const frag = getFragmentInfo(this.type, relPath);
+    if (frag) {
+      obj = deepGetOrCreate(obj, frag.relPath);
+    }
+    return redact(obj, frag) ?? null;
+  }
+
+  async update(ctx, data, relPath = '') {
+    const storage = HelixStorage.fromContext(ctx).configBus();
+    const buf = await storage.get(this.key);
+    let old = buf ? JSON.parse(buf) : null;
+    const frag = getFragmentInfo(this.type, relPath);
+    let config = data;
+    // set config to null if empty object
+    if (isDeepStrictEqual(config, {})) {
+      config = null;
+    }
+
+    let ret = null;
+    if (!config && frag?.type !== 'tokens') {
+      throw new StatusCodeError(400, 'no config in body.');
+    }
+    if (frag) {
+      if (!old) {
+        if (this.type === 'profiles' && frag.type === 'tokens') {
+          old = {};
+        } else {
+          throw new StatusCodeError(404, 'config not found.');
+        }
+      }
+      if (relPath === 'code') {
+        updateCodeSource(ctx, data);
+      } else if (relPath === 'content') {
+        updateContentSource(ctx, data);
+      } else if (frag.type === 'token') {
+        // don't allow to update individual token
+        throw new StatusCodeError(400, 'invalid object path.');
+      } else if (frag.type === 'tokens') {
+        // TODO: remove support after all helix4 projects are migrated
+        let token;
+        if (data.jwt) {
+          token = await migrateToken(this.org, data.jwt);
+        } else {
+          // create new token
+          token = createToken(this.org);
+        }
+
+        data = {
+          ...token,
+        };
+        // don't store token value in the config
+        delete data.value;
+        relPath = ['tokens', token.id];
+        frag.type = 'token';
+        ret = token;
+        // don't expose hash in return value
+        delete ret.hash;
+      } else if (frag.type === 'users') {
+        const user = createUser();
+        data = {
+          ...data,
+          ...user,
+        };
+        relPath = ['users', user.id];
+        frag.type = 'user';
+        // todo: define via "schema"
+        if (!old.users) {
+          old.users = [];
+        }
+      } else if (frag.type === 'user') {
+        const user = deepGetOrCreate(old, relPath);
+        if (!user) {
+          throw new StatusCodeError(404, 'object not found.');
+        }
+        if (data.id) {
+          if (data.id !== user.id) {
+            throw new StatusCodeError(400, 'object id mismatch.');
+          }
+        } else {
+          data.id = user.id;
+        }
+      }
+      config = deepPut(old, relPath, data);
+    } else if (this.type !== 'org') {
+      updateContentSource(ctx, config.content);
+      updateCodeSource(ctx, config.code);
+    }
+    await this.validate(ctx, config);
+    await storage.put(this.key, JSON.stringify(config), 'application/json');
+    await this.purge(ctx, buf ? JSON.parse(buf) : null, config);
+    return ret ?? redact(data, frag);
+  }
+
+  async remove(ctx, relPath) {
+    const storage = HelixStorage.fromContext(ctx).configBus();
+    const buf = await storage.get(this.key);
+    if (!buf) {
+      throw new StatusCodeError(404, 'config not found.');
+    }
+    const frag = getFragmentInfo(this.type, relPath);
+    if (frag) {
+      const data = deepPut(JSON.parse(buf), relPath, null);
+      await this.validate(ctx, data);
+      await storage.put(this.key, JSON.stringify(data), 'application/json');
+      await this.purge(ctx, JSON.parse(buf), data);
+      return;
+    }
+
+    await storage.remove(this.key);
+    await this.purge(ctx, JSON.parse(buf), null);
+  }
+
+  // eslint-disable-next-line class-methods-use-this,no-unused-vars
+  async purge(ctx, oldConfig, newConfig) {
+    // override in subclass
+  }
+}

package/src/config-validator.js

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import Ajv2019 from 'ajv/dist/2019.js';
+import ajvFormats from 'ajv-formats';
+import { ValidationError } from './ValidationError.js';
+
+import accessAdminSchema from './schemas/access-admin.schema.cjs';
+import accessSchema from './schemas/access.schema.cjs';
+import accessSiteSchema from './schemas/access-site.schema.cjs';
+import cdnSchema from './schemas/cdn.schema.cjs';
+import cdnProdFastlySchema from './schemas/cdn-prod-fastly.schema.cjs';
+import cdnProdCloudflareSchema from './schemas/cdn-prod-cloudflare.schema.cjs';
+import cdnProdAkamaiSchema from './schemas/cdn-prod-akamai.schema.cjs';
+import cdnProdManagedSchema from './schemas/cdn-prod-managed.schema.cjs';
+import cdnProdCloudfrontSchema from './schemas/cdn-prod-cloudfront.schema.cjs';
+import commonSchema from './schemas/common.schema.cjs';
+import codeSchema from './schemas/code.schema.cjs';
+import contentSchema from './schemas/content.schema.cjs';
+import foldersSchema from './schemas/folders.schema.cjs';
+import groupsSchema from './schemas/groups.schema.cjs';
+import googleSchema from './schemas/content-source-google.schema.cjs';
+import headersSchema from './schemas/headers.schema.cjs';
+import markupSchema from './schemas/content-source-markup.schema.cjs';
+import metadataSchema from './schemas/metadata-source.schema.cjs';
+import orgSchema from './schemas/org.schema.cjs';
+import onedriveSchema from './schemas/content-source-onedrive.schema.cjs';
+import publicSchema from './schemas/public.schema.cjs';
+import profileSchema from './schemas/profile.schema.cjs';
+import profilesSchema from './schemas/profiles.schema.cjs';
+import robotsSchema from './schemas/robots.schema.cjs';
+import sidekickSchema from './schemas/sidekick.schema.cjs';
+import siteSchema from './schemas/site.schema.cjs';
+import sitesSchema from './schemas/sites.schema.cjs';
+import tokensSchema from './schemas/tokens.schema.cjs';
+import userSchema from './schemas/user.schema.cjs';
+import usersSchema from './schemas/users.schema.cjs';
+
+export const SCHEMAS = [
+  accessSchema,
+  accessAdminSchema,
+  accessSiteSchema,
+  cdnSchema,
+  commonSchema,
+  contentSchema,
+  codeSchema,
+  foldersSchema,
+  googleSchema,
+  groupsSchema,
+  headersSchema,
+  markupSchema,
+  metadataSchema,
+  orgSchema,
+  onedriveSchema,
+  publicSchema,
+  profileSchema,
+  profilesSchema,
+  robotsSchema,
+  sidekickSchema,
+  siteSchema,
+  sitesSchema,
+  tokensSchema,
+  userSchema,
+  usersSchema,
+  cdnProdFastlySchema,
+  cdnProdCloudflareSchema,
+  cdnProdAkamaiSchema,
+  cdnProdManagedSchema,
+  cdnProdCloudfrontSchema,
+];
+
+const SCHEMA_TYPES = {
+  profiles: profileSchema,
+  sites: siteSchema,
+  org: orgSchema,
+};
+
+/**
+ * Validates the loaded configuration and coerces types and sets defaults
+ * @param {object} config The configuration to validate
+ * @param {string} type the config type
+ * @returns {object} The validated configuration
+ */
+export async function validate(config, type) {
+  const schema = SCHEMA_TYPES[type];
+  if (!schema) {
+    throw new Error(`no such type: ${type}`);
+  }
+  const ajv = new Ajv2019({
+    allErrors: true,
+    verbose: true,
+    useDefaults: true,
+    coerceTypes: 'array',
+    strict: false,
+  });
+  ajvFormats(ajv);
+
+  ajv.addSchema(SCHEMAS);
+  const res = ajv.validate(schema, config);
+  if (res) {
+    return res;
+  }
+  throw new ValidationError(ajv.errorsText(), ajv.errors);
+}

package/src/index.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+export { ConfigStore } from './config-store.js';
+export { SCHEMAS } from './config-validator.js';
+export * from './ValidationError.js';
+export * from './config-merge.js';

package/src/schemas/access-admin.schema.cjs

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+module.exports = require('./access-admin.schema.json');

package/src/schemas/access-admin.schema.json

@@ -0,0 +1,57 @@
+{
+  "$comment": "https://github.com/adobe/helix-config/blob/main/LICENSE.txt",
+  "$id": "https://ns.adobe.com/helix/config/access/admin",
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+  "title": "Admin Access Config",
+  "type": "object",
+  "properties": {
+    "role": {
+      "title": "Role",
+      "type": "object",
+      "patternProperties": {
+        "^[a-z-_]+$": {
+          "description": "The email glob of the users or a group reference for the respective role.",
+          "type": "array",
+          "items": {"type": "string"}
+        }
+      },
+      "additionalProperties": false
+    },
+    "requireAuth": {
+      "description": "Enforce authentication if set to true. If set to 'auto' it will enforce authentication if a role mapping is defined. defaults to 'auto'.",
+      "default": "auto",
+      "oneOf": [
+        {
+          "type": "boolean"
+        },
+        {
+          "type": "string",
+          "enum": ["auto"]
+        }
+      ]
+    },
+    "defaultRole": {
+      "description": "the default roles assigned to the users. defaults to `basic_publish` for unauthenticated setups.",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": [
+          "admin",
+          "author",
+          "publish",
+          "develop",
+          "basic_author",
+          "basic_publish",
+          "config",
+          "config_admin"
+        ]
+      }
+    },
+    "apiKeyId": {
+      "description": "the id of the API key(s). this is used to validate the API KEYS and allows to invalidate them.",
+      "type": "array",
+      "items": {"type": "string"}
+    }
+  },
+  "additionalProperties": false
+}