@backstage/config-loader 0.6.10 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,58 @@
 # @backstage/config-loader
 
+## 0.8.0
+
+### Minor Changes
+
+- 1e99c73c75: Update `loadConfig` to return `LoadConfigResult` instead of an array of `AppConfig`.
+
+  This function is primarily used internally by other config loaders like `loadBackendConfig` which means no changes are required for most users.
+
+  If you use `loadConfig` directly you will need to update your usage from:
+
+  ```diff
+  - const appConfigs = await loadConfig(options)
+  + const { appConfigs } = await loadConfig(options)
+  ```
+
+### Patch Changes
+
+- 8809b6c0dd: Update the json-schema dependency version.
+- Updated dependencies
+  - @backstage/cli-common@0.1.6
+
+## 0.7.2
+
+### Patch Changes
+
+- 0611f3b3e2: Reading app config from a remote server
+- 26c5659c97: Bump msw to the same version as the rest
+
+## 0.7.1
+
+### Patch Changes
+
+- 10615525f3: Switch to use the json and observable types from `@backstage/types`
+- ea21f7f567: bump `typescript-json-schema` from 0.50.1 to 0.51.0
+- Updated dependencies
+  - @backstage/config@0.1.11
+  - @backstage/cli-common@0.1.5
+  - @backstage/errors@0.1.4
+
+## 0.7.0
+
+### Minor Changes
+
+- 7e97d0b8c1: Removed the `EnvFunc` public export. Its only usage was to be passed in to `LoadConfigOptions.experimentalEnvFunc`. If you were using this type, add a definition in your own project instead with the signature `(name: string) => Promise<string | undefined>`.
+
+### Patch Changes
+
+- 223e8de6b4: Configuration schema errors are now filtered using the provided visibility option. This means that schema errors due to missing backend configuration will no longer break frontend builds.
+- 7e97d0b8c1: Add public tags and documentation
+- 36e67d2f24: Internal updates to apply more strict checks to throw errors.
+- Updated dependencies
+  - @backstage/errors@0.1.3
+
 ## 0.6.10
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -2,22 +2,27 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
+var errors = require('@backstage/errors');
 var yaml = require('yaml');
 var path = require('path');
 var Ajv = require('ajv');
 var mergeAllOf = require('json-schema-merge-allof');
+var traverse = require('json-schema-traverse');
 var config = require('@backstage/config');
 var fs = require('fs-extra');
 var typescriptJsonSchema = require('typescript-json-schema');
 var chokidar = require('chokidar');
+var fetch = require('node-fetch');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
 var yaml__default = /*#__PURE__*/_interopDefaultLegacy(yaml);
 var Ajv__default = /*#__PURE__*/_interopDefaultLegacy(Ajv);
 var mergeAllOf__default = /*#__PURE__*/_interopDefaultLegacy(mergeAllOf);
+var traverse__default = /*#__PURE__*/_interopDefaultLegacy(traverse);
 var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs);
 var chokidar__default = /*#__PURE__*/_interopDefaultLegacy(chokidar);
+var fetch__default = /*#__PURE__*/_interopDefaultLegacy(fetch);
 
 const ENV_PREFIX = "APP_CONFIG_";
 const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
@@ -65,6 +70,7 @@ function safeJsonParse(str) {
   try {
     return [null, JSON.parse(str)];
   } catch (err) {
+    errors.assertError(err);
     return [err, str];
   }
 }
@@ -95,6 +101,7 @@ async function applyConfigTransforms(initialDir, input, transforms) {
           break;
         }
       } catch (error) {
+        errors.assertError(error);
         throw new Error(`error at ${path}, ${error.message}`);
       }
     }
@@ -232,7 +239,7 @@ const CONFIG_VISIBILITIES = ["frontend", "backend", "secret"];
 const DEFAULT_CONFIG_VISIBILITY = "backend";
 
 function compileConfigSchemas(schemas) {
-  const visibilityByPath = new Map();
+  const visibilityByDataPath = new Map();
   const ajv = new Ajv__default['default']({
     allErrors: true,
     allowUnionTypes: true,
@@ -252,7 +259,7 @@ function compileConfigSchemas(schemas) {
         }
         if (visibility && visibility !== "backend") {
           const normalizedPath = context.dataPath.replace(/\['?(.*?)'?\]/g, (_, segment) => `/${segment}`);
-          visibilityByPath.set(normalizedPath, visibility);
+          visibilityByDataPath.set(normalizedPath, visibility);
         }
         return true;
       };
@@ -267,23 +274,27 @@ function compileConfigSchemas(schemas) {
   }
   const merged = mergeConfigSchemas(schemas.map((_) => _.value));
   const validate = ajv.compile(merged);
+  const visibilityBySchemaPath = new Map();
+  traverse__default['default'](merged, (schema, path) => {
+    if (schema.visibility && schema.visibility !== "backend") {
+      visibilityBySchemaPath.set(path, schema.visibility);
+    }
+  });
   return (configs) => {
     var _a;
     const config$1 = config.ConfigReader.fromConfigs(configs).get();
-    visibilityByPath.clear();
+    visibilityByDataPath.clear();
     const valid = validate(config$1);
     if (!valid) {
-      const errors = (_a = validate.errors) != null ? _a : [];
       return {
-        errors: errors.map(({dataPath, message, params}) => {
-          const paramStr = Object.entries(params).map(([name, value]) => `${name}=${value}`).join(" ");
-          return `Config ${message || ""} { ${paramStr} } at ${dataPath}`;
-        }),
-        visibilityByPath: new Map()
+        errors: (_a = validate.errors) != null ? _a : [],
+        visibilityByDataPath: new Map(visibilityByDataPath),
+        visibilityBySchemaPath
       };
     }
     return {
-      visibilityByPath: new Map(visibilityByPath)
+      visibilityByDataPath: new Map(visibilityByDataPath),
+      visibilityBySchemaPath
     };
   };
 }
@@ -412,6 +423,7 @@ function compileTsSchemas(paths) {
         validationKeywords: ["visibility"]
       }, [path$1.split(path.sep).join("/")]);
     } catch (error) {
+      errors.assertError(error);
       if (error.message !== "type Config not found") {
         throw error;
       }
@@ -424,12 +436,12 @@ function compileTsSchemas(paths) {
   return tsSchemas;
 }
 
-function filterByVisibility(data, includeVisibilities, visibilityByPath, transformFunc, withFilteredKeys) {
+function filterByVisibility(data, includeVisibilities, visibilityByDataPath, transformFunc, withFilteredKeys) {
   var _a;
   const filteredKeys = new Array();
   function transform(jsonVal, visibilityPath, filterPath) {
     var _a2;
-    const visibility = (_a2 = visibilityByPath.get(visibilityPath)) != null ? _a2 : DEFAULT_CONFIG_VISIBILITY;
+    const visibility = (_a2 = visibilityByDataPath.get(visibilityPath)) != null ? _a2 : DEFAULT_CONFIG_VISIBILITY;
     const isVisible = includeVisibilities.includes(visibility);
     if (typeof jsonVal !== "object") {
       if (isVisible) {
@@ -479,7 +491,40 @@ function filterByVisibility(data, includeVisibilities, visibilityByPath, transfo
     data: (_a = transform(data, "", "")) != null ? _a : {}
   };
 }
+function filterErrorsByVisibility(errors, includeVisibilities, visibilityByDataPath, visibilityBySchemaPath) {
+  if (!errors) {
+    return [];
+  }
+  if (!includeVisibilities) {
+    return errors;
+  }
+  const visibleSchemaPaths = Array.from(visibilityBySchemaPath).filter(([, v]) => includeVisibilities.includes(v)).map(([k]) => k);
+  return errors.filter((error) => {
+    var _a;
+    if (error.keyword === "type" && ["object", "array"].includes(error.params.type)) {
+      return true;
+    }
+    if (error.keyword === "required") {
+      const trimmedPath = error.schemaPath.slice(1, -"/required".length);
+      const fullPath = `${trimmedPath}/properties/${error.params.missingProperty}`;
+      if (visibleSchemaPaths.some((visiblePath) => visiblePath.startsWith(fullPath))) {
+        return true;
+      }
+    }
+    const vis = (_a = visibilityByDataPath.get(error.dataPath)) != null ? _a : DEFAULT_CONFIG_VISIBILITY;
+    return vis && includeVisibilities.includes(vis);
+  });
+}
 
+function errorsToError(errors) {
+  const messages = errors.map(({dataPath, message, params}) => {
+    const paramStr = Object.entries(params).map(([name, value]) => `${name}=${value}`).join(" ");
+    return `Config ${message || ""} { ${paramStr} } at ${dataPath}`;
+  });
+  const error = new Error(`Config validation failed, ${messages.join("; ")}`);
+  error.messages = messages;
+  return error;
+}
 async function loadConfigSchema(options) {
   var _a;
   let schemas;
@@ -496,21 +541,20 @@ async function loadConfigSchema(options) {
   return {
     process(configs, {visibility, valueTransform, withFilteredKeys} = {}) {
       const result = validate(configs);
-      if (result.errors) {
-        const error = new Error(`Config validation failed, ${result.errors.join("; ")}`);
-        error.messages = result.errors;
-        throw error;
+      const visibleErrors = filterErrorsByVisibility(result.errors, visibility, result.visibilityByDataPath, result.visibilityBySchemaPath);
+      if (visibleErrors.length > 0) {
+        throw errorsToError(visibleErrors);
       }
       let processedConfigs = configs;
       if (visibility) {
         processedConfigs = processedConfigs.map(({data, context}) => ({
           context,
-          ...filterByVisibility(data, visibility, result.visibilityByPath, valueTransform, withFilteredKeys)
+          ...filterByVisibility(data, visibility, result.visibilityByDataPath, valueTransform, withFilteredKeys)
         }));
       } else if (valueTransform) {
         processedConfigs = processedConfigs.map(({data, context}) => ({
           context,
-          ...filterByVisibility(data, Array.from(CONFIG_VISIBILITIES), result.visibilityByPath, valueTransform, withFilteredKeys)
+          ...filterByVisibility(data, Array.from(CONFIG_VISIBILITIES), result.visibilityByDataPath, valueTransform, withFilteredKeys)
         }));
       }
       return processedConfigs;
@@ -524,10 +568,28 @@ async function loadConfigSchema(options) {
   };
 }
 
+function isValidUrl(url) {
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 async function loadConfig(options) {
-  const {configRoot, experimentalEnvFunc: envFunc, watch} = options;
-  const configPaths = options.configPaths.slice();
-  if (configPaths.length === 0) {
+  const {configRoot, experimentalEnvFunc: envFunc, watch, remote} = options;
+  const configPaths = options.configTargets.slice().filter((e) => e.hasOwnProperty("path")).map((configTarget) => configTarget.path);
+  options.configPaths.forEach((cp) => {
+    if (!configPaths.includes(cp)) {
+      configPaths.push(cp);
+    }
+  });
+  const configUrls = options.configTargets.slice().filter((e) => e.hasOwnProperty("url")).map((configTarget) => configTarget.url);
+  if (remote === void 0 && configUrls.length > 0) {
+    throw new Error(`Remote config detected but this feature is turned off`);
+  }
+  if (configPaths.length === 0 && configUrls.length === 0) {
     configPaths.push(path.resolve(configRoot, "app-config.yaml"));
     const localConfig = path.resolve(configRoot, "app-config.local.yaml");
     if (await fs__default['default'].pathExists(localConfig)) {
@@ -553,18 +615,53 @@ async function loadConfig(options) {
     }
     return configs;
   };
+  const loadRemoteConfigFiles = async () => {
+    const configs = [];
+    const readConfigFromUrl = async (url) => {
+      const response = await fetch__default['default'](url);
+      if (!response.ok) {
+        throw new Error(`Could not read config file at ${url}`);
+      }
+      return await response.text();
+    };
+    for (let i = 0; i < configUrls.length; i++) {
+      const configUrl = configUrls[i];
+      if (!isValidUrl(configUrl)) {
+        throw new Error(`Config load path is not valid: '${configUrl}'`);
+      }
+      const remoteConfigContent = await readConfigFromUrl(configUrl);
+      if (!remoteConfigContent) {
+        throw new Error(`Config is not valid`);
+      }
+      const configYaml = yaml__default['default'].parse(remoteConfigContent);
+      const substitutionTransform = createSubstitutionTransform(env);
+      const data = await applyConfigTransforms(configRoot, configYaml, [
+        substitutionTransform
+      ]);
+      configs.push({data, context: configUrl});
+    }
+    return configs;
+  };
   let fileConfigs;
   try {
     fileConfigs = await loadConfigFiles();
   } catch (error) {
-    throw new Error(`Failed to read static configuration file, ${error.message}`);
+    throw new errors.ForwardedError("Failed to read static configuration file", error);
+  }
+  let remoteConfigs = [];
+  if (remote) {
+    try {
+      remoteConfigs = await loadRemoteConfigFiles();
+    } catch (error) {
+      throw new errors.ForwardedError(`Failed to read remote configuration file`, error);
+    }
   }
   const envConfigs = await readEnvConfig(process.env);
-  if (watch) {
-    let currentSerializedConfig = JSON.stringify(fileConfigs);
+  const watchConfigFile = (watchProp) => {
     const watcher = chokidar__default['default'].watch(configPaths, {
       usePolling: process.env.NODE_ENV === "test"
     });
+    let currentSerializedConfig = JSON.stringify(fileConfigs);
     watcher.on("change", async () => {
       try {
         const newConfigs = await loadConfigFiles();
@@ -573,18 +670,55 @@ async function loadConfig(options) {
           return;
         }
         currentSerializedConfig = newSerializedConfig;
-        watch.onChange([...newConfigs, ...envConfigs]);
+        watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]);
       } catch (error) {
         console.error(`Failed to reload configuration files, ${error}`);
       }
     });
-    if (watch.stopSignal) {
-      watch.stopSignal.then(() => {
+    if (watchProp.stopSignal) {
+      watchProp.stopSignal.then(() => {
         watcher.close();
       });
     }
+  };
+  const watchRemoteConfig = (watchProp, remoteProp) => {
+    const hasConfigChanged = async (oldRemoteConfigs, newRemoteConfigs) => {
+      return JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs);
+    };
+    let handle;
+    try {
+      handle = setInterval(async () => {
+        console.info(`Checking for config update`);
+        const newRemoteConfigs = await loadRemoteConfigFiles();
+        if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) {
+          remoteConfigs = newRemoteConfigs;
+          console.info(`Remote config change, reloading config ...`);
+          watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]);
+          console.info(`Remote config reloaded`);
+        }
+      }, remoteProp.reloadIntervalSeconds * 1e3);
+    } catch (error) {
+      console.error(`Failed to reload configuration files, ${error}`);
+    }
+    if (watchProp.stopSignal) {
+      watchProp.stopSignal.then(() => {
+        if (handle !== void 0) {
+          console.info(`Stopping remote config watch`);
+          clearInterval(handle);
+          handle = void 0;
+        }
+      });
+    }
+  };
+  if (watch) {
+    watchConfigFile(watch);
   }
-  return [...fileConfigs, ...envConfigs];
+  if (watch && remote) {
+    watchRemoteConfig(watch, remote);
+  }
+  return {
+    appConfigs: remote ? [...remoteConfigs, ...fileConfigs, ...envConfigs] : [...fileConfigs, ...envConfigs]
+  };
 }
 
 exports.loadConfig = loadConfig;