@backstage/config-loader 0.7.1 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,51 @@
 # @backstage/config-loader
 
+## 0.9.0
+
+### Minor Changes
+
+- f6722d2458: Removed deprecated option `env` from `LoadConfigOptions` and associated tests
+- 67d6cb3c7e: Removed deprecated option `configPaths` as it has been superseded by `configTargets`
+
+### Patch Changes
+
+- 1e7070443d: In case remote.reloadIntervalSeconds is passed, it must be a valid positive value
+
+## 0.8.1
+
+### Patch Changes
+
+- b055a6addc: Align on usage of `cross-fetch` vs `node-fetch` in frontend vs backend packages, and remove some unnecessary imports of either one of them
+- 4bea7b81d3: Uses key visibility as fallback in non-object arrays
+
+## 0.8.0
+
+### Minor Changes
+
+- 1e99c73c75: Update `loadConfig` to return `LoadConfigResult` instead of an array of `AppConfig`.
+
+  This function is primarily used internally by other config loaders like `loadBackendConfig` which means no changes are required for most users.
+
+  If you use `loadConfig` directly you will need to update your usage from:
+
+  ```diff
+  - const appConfigs = await loadConfig(options)
+  + const { appConfigs } = await loadConfig(options)
+  ```
+
+### Patch Changes
+
+- 8809b6c0dd: Update the json-schema dependency version.
+- Updated dependencies
+  - @backstage/cli-common@0.1.6
+
+## 0.7.2
+
+### Patch Changes
+
+- 0611f3b3e2: Reading app config from a remote server
+- 26c5659c97: Bump msw to the same version as the rest
+
 ## 0.7.1
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -12,6 +12,7 @@ var config = require('@backstage/config');
 var fs = require('fs-extra');
 var typescriptJsonSchema = require('typescript-json-schema');
 var chokidar = require('chokidar');
+var fetch = require('node-fetch');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
@@ -21,6 +22,7 @@ var mergeAllOf__default = /*#__PURE__*/_interopDefaultLegacy(mergeAllOf);
 var traverse__default = /*#__PURE__*/_interopDefaultLegacy(traverse);
 var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs);
 var chokidar__default = /*#__PURE__*/_interopDefaultLegacy(chokidar);
+var fetch__default = /*#__PURE__*/_interopDefaultLegacy(fetch);
 
 const ENV_PREFIX = "APP_CONFIG_";
 const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
@@ -62,7 +64,7 @@ function readEnvConfig(env) {
       }
     }
   }
-  return data ? [{data, context: "env"}] : [];
+  return data ? [{ data, context: "env" }] : [];
 }
 function safeJsonParse(str) {
   try {
@@ -137,13 +139,13 @@ async function applyConfigTransforms(initialDir, input, transforms) {
 
 const includeFileParser = {
   ".json": async (content) => JSON.parse(content),
-  ".yaml": async (content) => yaml__default['default'].parse(content),
-  ".yml": async (content) => yaml__default['default'].parse(content)
+  ".yaml": async (content) => yaml__default["default"].parse(content),
+  ".yml": async (content) => yaml__default["default"].parse(content)
 };
 function createIncludeTransform(env, readFile, substitute) {
   return async (input, baseDir) => {
     if (!isObject(input)) {
-      return {applied: false};
+      return { applied: false };
     }
     const [includeKey] = Object.keys(input).filter((key) => key.startsWith("$"));
     if (includeKey) {
@@ -151,7 +153,7 @@ function createIncludeTransform(env, readFile, substitute) {
         throw new Error(`include key ${includeKey} should not have adjacent keys`);
       }
     } else {
-      return {applied: false};
+      return { applied: false };
     }
     const rawIncludedValue = input[includeKey];
     if (typeof rawIncludedValue !== "string") {
@@ -166,13 +168,13 @@ function createIncludeTransform(env, readFile, substitute) {
       case "$file":
         try {
           const value = await readFile(path.resolve(baseDir, includeValue));
-          return {applied: true, value};
+          return { applied: true, value };
         } catch (error) {
           throw new Error(`failed to read file ${includeValue}, ${error}`);
         }
       case "$env":
         try {
-          return {applied: true, value: await env(includeValue)};
+          return { applied: true, value: await env(includeValue) };
         } catch (error) {
           throw new Error(`failed to read env ${includeValue}, ${error}`);
         }
@@ -215,7 +217,7 @@ function createIncludeTransform(env, readFile, substitute) {
 function createSubstitutionTransform(env) {
   return async (input) => {
     if (typeof input !== "string") {
-      return {applied: false};
+      return { applied: false };
     }
     const parts = input.split(/(\$?\$\{[^{}]*\})/);
     for (let i = 1; i < parts.length; i += 2) {
@@ -227,9 +229,9 @@ function createSubstitutionTransform(env) {
       }
     }
     if (parts.some((part) => part === void 0)) {
-      return {applied: true, value: void 0};
+      return { applied: true, value: void 0 };
     }
-    return {applied: true, value: parts.join("")};
+    return { applied: true, value: parts.join("") };
   };
 }
 
@@ -237,8 +239,8 @@ const CONFIG_VISIBILITIES = ["frontend", "backend", "secret"];
 const DEFAULT_CONFIG_VISIBILITY = "backend";
 
 function compileConfigSchemas(schemas) {
-  const visibilityByDataPath = new Map();
-  const ajv = new Ajv__default['default']({
+  const visibilityByDataPath = /* @__PURE__ */ new Map();
+  const ajv = new Ajv__default["default"]({
     allErrors: true,
     allowUnionTypes: true,
     schemas: {
@@ -272,8 +274,8 @@ function compileConfigSchemas(schemas) {
   }
   const merged = mergeConfigSchemas(schemas.map((_) => _.value));
   const validate = ajv.compile(merged);
-  const visibilityBySchemaPath = new Map();
-  traverse__default['default'](merged, (schema, path) => {
+  const visibilityBySchemaPath = /* @__PURE__ */ new Map();
+  traverse__default["default"](merged, (schema, path) => {
     if (schema.visibility && schema.visibility !== "backend") {
       visibilityBySchemaPath.set(path, schema.visibility);
     }
@@ -297,7 +299,7 @@ function compileConfigSchemas(schemas) {
   };
 }
 function mergeConfigSchemas(schemas) {
-  const merged = mergeAllOf__default['default']({allOf: schemas}, {
+  const merged = mergeAllOf__default["default"]({ allOf: schemas }, {
     ignoreAdditionalProperties: true,
     resolvers: {
       visibility(values, path) {
@@ -321,18 +323,18 @@ const req = typeof __non_webpack_require__ === "undefined" ? require : __non_web
 async function collectConfigSchemas(packageNames, packagePaths) {
   const schemas = new Array();
   const tsSchemaPaths = new Array();
-  const visitedPackageVersions = new Map();
-  const currentDir = await fs__default['default'].realpath(process.cwd());
+  const visitedPackageVersions = /* @__PURE__ */ new Map();
+  const currentDir = await fs__default["default"].realpath(process.cwd());
   async function processItem(item) {
     var _a, _b, _c, _d;
     let pkgPath = item.packagePath;
     if (pkgPath) {
-      const pkgExists = await fs__default['default'].pathExists(pkgPath);
+      const pkgExists = await fs__default["default"].pathExists(pkgPath);
       if (!pkgExists) {
         return;
       }
     } else if (item.name) {
-      const {name, parentPath} = item;
+      const { name, parentPath } = item;
       try {
         pkgPath = req.resolve(`${name}/package.json`, parentPath && {
           paths: [parentPath]
@@ -343,13 +345,13 @@ async function collectConfigSchemas(packageNames, packagePaths) {
     if (!pkgPath) {
       return;
     }
-    const pkg = await fs__default['default'].readJson(pkgPath);
+    const pkg = await fs__default["default"].readJson(pkgPath);
     let versions = visitedPackageVersions.get(pkg.name);
     if (versions == null ? void 0 : versions.has(pkg.version)) {
       return;
     }
     if (!versions) {
-      versions = new Set();
+      versions = /* @__PURE__ */ new Set();
       visitedPackageVersions.set(pkg.name, versions);
     }
     versions.add(pkg.version);
@@ -375,7 +377,7 @@ async function collectConfigSchemas(packageNames, packagePaths) {
           tsSchemaPaths.push(path.relative(currentDir, path.resolve(path.dirname(pkgPath), pkg.configSchema)));
         } else {
           const path$1 = path.resolve(path.dirname(pkgPath), pkg.configSchema);
-          const value = await fs__default['default'].readJson(path$1);
+          const value = await fs__default["default"].readJson(path$1);
           schemas.push({
             value,
             path: path.relative(currentDir, path$1)
@@ -388,11 +390,11 @@ async function collectConfigSchemas(packageNames, packagePaths) {
         });
       }
     }
-    await Promise.all(depNames.map((depName) => processItem({name: depName, parentPath: pkgPath})));
+    await Promise.all(depNames.map((depName) => processItem({ name: depName, parentPath: pkgPath })));
   }
   await Promise.all([
-    ...packageNames.map((name) => processItem({name, parentPath: currentDir})),
-    ...packagePaths.map((path) => processItem({name: path, packagePath: path}))
+    ...packageNames.map((name) => processItem({ name, parentPath: currentDir })),
+    ...packagePaths.map((path) => processItem({ name: path, packagePath: path }))
   ]);
   const tsSchemas = compileTsSchemas(tsSchemaPaths);
   return schemas.concat(tsSchemas);
@@ -429,7 +431,7 @@ function compileTsSchemas(paths) {
     if (!value) {
       throw new Error(`Invalid schema in ${path$1}, missing Config export`);
     }
-    return {path: path$1, value};
+    return { path: path$1, value };
   });
   return tsSchemas;
 }
@@ -444,7 +446,7 @@ function filterByVisibility(data, includeVisibilities, visibilityByDataPath, tra
     if (typeof jsonVal !== "object") {
       if (isVisible) {
         if (transformFunc) {
-          return transformFunc(jsonVal, {visibility});
+          return transformFunc(jsonVal, { visibility });
         }
         return jsonVal;
       }
@@ -457,7 +459,12 @@ function filterByVisibility(data, includeVisibilities, visibilityByDataPath, tra
     } else if (Array.isArray(jsonVal)) {
       const arr = new Array();
       for (const [index, value] of jsonVal.entries()) {
-        const out = transform(value, `${visibilityPath}/${index}`, `${filterPath}[${index}]`);
+        let path = visibilityPath;
+        const hasVisibilityInIndex = visibilityByDataPath.get(`${visibilityPath}/${index}`);
+        if (hasVisibilityInIndex || typeof value === "object") {
+          path = `${visibilityPath}/${index}`;
+        }
+        const out = transform(value, path, `${filterPath}[${index}]`);
         if (out !== void 0) {
           arr.push(out);
         }
@@ -515,7 +522,7 @@ function filterErrorsByVisibility(errors, includeVisibilities, visibilityByDataP
 }
 
 function errorsToError(errors) {
-  const messages = errors.map(({dataPath, message, params}) => {
+  const messages = errors.map(({ dataPath, message, params }) => {
     const paramStr = Object.entries(params).map(([name, value]) => `${name}=${value}`).join(" ");
     return `Config ${message || ""} { ${paramStr} } at ${dataPath}`;
   });
@@ -529,7 +536,7 @@ async function loadConfigSchema(options) {
   if ("dependencies" in options) {
     schemas = await collectConfigSchemas(options.dependencies, (_a = options.packagePaths) != null ? _a : []);
   } else {
-    const {serialized} = options;
+    const { serialized } = options;
     if ((serialized == null ? void 0 : serialized.backstageConfigSchemaVersion) !== 1) {
       throw new Error("Serialized configuration schema is invalid or has an invalid version number");
     }
@@ -537,7 +544,7 @@ async function loadConfigSchema(options) {
   }
   const validate = compileConfigSchemas(schemas);
   return {
-    process(configs, {visibility, valueTransform, withFilteredKeys} = {}) {
+    process(configs, { visibility, valueTransform, withFilteredKeys } = {}) {
       const result = validate(configs);
       const visibleErrors = filterErrorsByVisibility(result.errors, visibility, result.visibilityByDataPath, result.visibilityBySchemaPath);
       if (visibleErrors.length > 0) {
@@ -545,12 +552,12 @@ async function loadConfigSchema(options) {
       }
       let processedConfigs = configs;
       if (visibility) {
-        processedConfigs = processedConfigs.map(({data, context}) => ({
+        processedConfigs = processedConfigs.map(({ data, context }) => ({
           context,
           ...filterByVisibility(data, visibility, result.visibilityByDataPath, valueTransform, withFilteredKeys)
         }));
       } else if (valueTransform) {
-        processedConfigs = processedConfigs.map(({data, context}) => ({
+        processedConfigs = processedConfigs.map(({ data, context }) => ({
           context,
           ...filterByVisibility(data, Array.from(CONFIG_VISIBILITIES), result.visibilityByDataPath, valueTransform, withFilteredKeys)
         }));
@@ -566,13 +573,30 @@ async function loadConfigSchema(options) {
   };
 }
 
+function isValidUrl(url) {
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 async function loadConfig(options) {
-  const {configRoot, experimentalEnvFunc: envFunc, watch} = options;
-  const configPaths = options.configPaths.slice();
-  if (configPaths.length === 0) {
+  const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options;
+  const configPaths = options.configTargets.slice().filter((e) => e.hasOwnProperty("path")).map((configTarget) => configTarget.path);
+  const configUrls = options.configTargets.slice().filter((e) => e.hasOwnProperty("url")).map((configTarget) => configTarget.url);
+  if (remote === void 0) {
+    if (configUrls.length > 0) {
+      throw new Error(`Please make sure you are passing the remote option when loading remote configurations. See https://backstage.io/docs/conf/writing#configuration-files for detailed info.`);
+    }
+  } else if (remote.reloadIntervalSeconds <= 0) {
+    throw new Error(`Remote config must be contain a non zero reloadIntervalSeconds: <seconds> value`);
+  }
+  if (configPaths.length === 0 && configUrls.length === 0) {
     configPaths.push(path.resolve(configRoot, "app-config.yaml"));
     const localConfig = path.resolve(configRoot, "app-config.local.yaml");
-    if (await fs__default['default'].pathExists(localConfig)) {
+    if (await fs__default["default"].pathExists(localConfig)) {
       configPaths.push(localConfig);
     }
   }
@@ -584,14 +608,41 @@ async function loadConfig(options) {
         throw new Error(`Config load path is not absolute: '${configPath}'`);
       }
       const dir = path.dirname(configPath);
-      const readFile = (path$1) => fs__default['default'].readFile(path.resolve(dir, path$1), "utf8");
-      const input = yaml__default['default'].parse(await readFile(configPath));
+      const readFile = (path$1) => fs__default["default"].readFile(path.resolve(dir, path$1), "utf8");
+      const input = yaml__default["default"].parse(await readFile(configPath));
       const substitutionTransform = createSubstitutionTransform(env);
       const data = await applyConfigTransforms(dir, input, [
         createIncludeTransform(env, readFile, substitutionTransform),
         substitutionTransform
       ]);
-      configs.push({data, context: path.basename(configPath)});
+      configs.push({ data, context: path.basename(configPath) });
+    }
+    return configs;
+  };
+  const loadRemoteConfigFiles = async () => {
+    const configs = [];
+    const readConfigFromUrl = async (url) => {
+      const response = await fetch__default["default"](url);
+      if (!response.ok) {
+        throw new Error(`Could not read config file at ${url}`);
+      }
+      return await response.text();
+    };
+    for (let i = 0; i < configUrls.length; i++) {
+      const configUrl = configUrls[i];
+      if (!isValidUrl(configUrl)) {
+        throw new Error(`Config load path is not valid: '${configUrl}'`);
+      }
+      const remoteConfigContent = await readConfigFromUrl(configUrl);
+      if (!remoteConfigContent) {
+        throw new Error(`Config is not valid`);
+      }
+      const configYaml = yaml__default["default"].parse(remoteConfigContent);
+      const substitutionTransform = createSubstitutionTransform(env);
+      const data = await applyConfigTransforms(configRoot, configYaml, [
+        substitutionTransform
+      ]);
+      configs.push({ data, context: configUrl });
     }
     return configs;
   };
@@ -601,12 +652,20 @@ async function loadConfig(options) {
   } catch (error) {
     throw new errors.ForwardedError("Failed to read static configuration file", error);
   }
+  let remoteConfigs = [];
+  if (remote) {
+    try {
+      remoteConfigs = await loadRemoteConfigFiles();
+    } catch (error) {
+      throw new errors.ForwardedError(`Failed to read remote configuration file`, error);
+    }
+  }
   const envConfigs = await readEnvConfig(process.env);
-  if (watch) {
-    let currentSerializedConfig = JSON.stringify(fileConfigs);
-    const watcher = chokidar__default['default'].watch(configPaths, {
+  const watchConfigFile = (watchProp) => {
+    const watcher = chokidar__default["default"].watch(configPaths, {
       usePolling: process.env.NODE_ENV === "test"
     });
+    let currentSerializedConfig = JSON.stringify(fileConfigs);
     watcher.on("change", async () => {
       try {
         const newConfigs = await loadConfigFiles();
@@ -615,18 +674,55 @@ async function loadConfig(options) {
           return;
         }
         currentSerializedConfig = newSerializedConfig;
-        watch.onChange([...newConfigs, ...envConfigs]);
+        watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]);
       } catch (error) {
         console.error(`Failed to reload configuration files, ${error}`);
       }
     });
-    if (watch.stopSignal) {
-      watch.stopSignal.then(() => {
+    if (watchProp.stopSignal) {
+      watchProp.stopSignal.then(() => {
         watcher.close();
       });
     }
+  };
+  const watchRemoteConfig = (watchProp, remoteProp) => {
+    const hasConfigChanged = async (oldRemoteConfigs, newRemoteConfigs) => {
+      return JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs);
+    };
+    let handle;
+    try {
+      handle = setInterval(async () => {
+        console.info(`Checking for config update`);
+        const newRemoteConfigs = await loadRemoteConfigFiles();
+        if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) {
+          remoteConfigs = newRemoteConfigs;
+          console.info(`Remote config change, reloading config ...`);
+          watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]);
+          console.info(`Remote config reloaded`);
+        }
+      }, remoteProp.reloadIntervalSeconds * 1e3);
+    } catch (error) {
+      console.error(`Failed to reload configuration files, ${error}`);
+    }
+    if (watchProp.stopSignal) {
+      watchProp.stopSignal.then(() => {
+        if (handle !== void 0) {
+          console.info(`Stopping remote config watch`);
+          clearInterval(handle);
+          handle = void 0;
+        }
+      });
+    }
+  };
+  if (watch) {
+    watchConfigFile(watch);
+  }
+  if (watch && remote) {
+    watchRemoteConfig(watch, remote);
   }
-  return [...fileConfigs, ...envConfigs];
+  return {
+    appConfigs: remote ? [...remoteConfigs, ...fileConfigs, ...envConfigs] : [...fileConfigs, ...envConfigs]
+  };
 }
 
 exports.loadConfig = loadConfig;