@travetto/config 3.4.0-rc.4 → 3.4.0-rc.6

package/README.md

@@ -23,9 +23,14 @@ Config loading follows a defined resolution path, below is the order in increasi
    1. `resources/application.<ext>` - Priority `100` - Load the default `application.<ext>` if available.
    1. `resources/{env}.<ext>` - Priority `200` - Load environment specific profile configurations as defined by the values of `process.env.TRV_ENV`.
    1. `resources/*.<ext>` - Priority `300` - Load profile specific configurations as defined by the values in `process.env.TRV_PROFILES`
+   1. [@Injectable](https://github.com/travetto/travetto/tree/main/module/di/src/decorator.ts#L31) [ConfigSource](https://github.com/travetto/travetto/tree/main/module/config/src/source/types.ts#L11) - Priority `???` - These are custom config sources provided by the module, and are able to define their own priorities
+   1. [OverrideConfigSource](https://github.com/travetto/travetto/tree/main/module/config/src/source/override.ts#L11) - Priority `999` - This is for [EnvVar](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L34) overrides, and is at the top priority for all built-in config sources.
 By default all configuration data is inert, and will only be applied when constructing an instance of a configuration class.
 
-**Note**: When working in a monorepo, the parent resources folder will also be searched with a lower priority than the the module's specific resources.  This allows for shared-global configuration that can be overridden at the module level.  The general rule is that the longest path has the highest priority.
+**Note**: When working in a monorepo, the parent resources folder will also be searched with a lower priority than the the module's specific resources.  This allows for shared-global configuration that can be overridden at the module level. The general priority is:
+   1. Mono-repo root
+   1. Module root
+   1. Folders for `TRV_RESOURCES`, in order
 
 ### A Complete Example
 A more complete example setup would look like:
@@ -91,24 +96,20 @@ The framework provides two simple base classes that assist with existing pattern
 **Code: Memory Provider**
 ```typescript
 import { ConfigData } from '../parser/types';
-import { ConfigSource } from './types';
+import { ConfigSource, ConfigSpec } from './types';
 
 /**
  * Meant to be instantiated and provided as a unique config source
  */
 export class MemoryConfigSource implements ConfigSource {
-  priority: number;
-  data: ConfigData;
-  source: string;
+  #spec: ConfigSpec;
 
   constructor(key: string, data: ConfigData, priority: number = 500) {
-    this.data = data;
-    this.priority = priority;
-    this.source = `memory://${key}`;
+    this.#spec = { data, priority, source: `memory://${key}` };
   }
 
-  getData(): ConfigData {
-    return this.data;
+  get(): ConfigSpec {
+    return this.#spec;
   }
 }
 ```
@@ -117,27 +118,24 @@ export class MemoryConfigSource implements ConfigSource {
 ```typescript
 import { Env } from '@travetto/base';
 
-import { ConfigSource } from './types';
-import { ConfigData } from '../parser/types';
+import { ConfigSource, ConfigSpec } from './types';
 
 /**
  * Represents the environment mapped data as a JSON blob
  */
 export class EnvConfigSource implements ConfigSource {
-  priority: number;
-  source: string;
   #envKey: string;
+  #priority: number;
 
   constructor(key: string, priority: number) {
     this.#envKey = key;
-    this.priority = priority;
-    this.source = `env://${this.#envKey}`;
+    this.#priority = priority;
   }
 
-  getData(): ConfigData | undefined {
+  get(): ConfigSpec | undefined {
     try {
       const data = JSON.parse(Env.get(this.#envKey, '{}'));
-      return data;
+      return { data, priority: this.#priority, source: `env://${this.#envKey}` };
     } catch (e) {
       console.error(`env.${this.#envKey} is an invalid format`, { text: Env.get(this.#envKey) });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/config",
-  "version": "3.4.0-rc.4",
+  "version": "3.4.0-rc.6",
   "description": "Configuration support",
   "keywords": [
     "yaml",
@@ -26,9 +26,9 @@
     "directory": "module/config"
   },
   "dependencies": {
-    "@travetto/di": "^3.4.0-rc.4",
-    "@travetto/schema": "^3.4.0-rc.4",
-    "@travetto/yaml": "^3.4.0-rc.4"
+    "@travetto/di": "^3.4.0-rc.6",
+    "@travetto/schema": "^3.4.0-rc.6",
+    "@travetto/yaml": "^3.4.0-rc.6"
   },
   "travetto": {
     "displayName": "Configuration"

package/src/parser/parser.ts

@@ -19,10 +19,12 @@ export class ParserManager {
 
     // Register parsers
     this.#parsers = Object.fromEntries(parsers.flatMap(p => p.ext.map(e => [e, p])));
-
     this.#extMatch = parsers.length ? new RegExp(`(${Object.keys(this.#parsers).join('|').replaceAll('.', '[.]')})`) : /^$/;
   }
 
+  /**
+   * Attempt ot parse a file, based on file's extension
+   */
   async parse(file: string): Promise<ConfigData> {
     const ext = path.extname(file);
     if (!this.#parsers[ext]) {
@@ -31,6 +33,9 @@ export class ParserManager {
     return fs.readFile(file, 'utf8').then(content => this.#parsers[ext].parse(content));
   }
 
+  /**
+   * Determine if file matches
+   */
   matches(file: string): boolean {
     return this.#extMatch.test(file);
   }

package/src/service.ts

@@ -1,6 +1,6 @@
 import util from 'util';
 
-import { AppError, Class, ClassInstance, GlobalEnv, DataUtil, Env } from '@travetto/base';
+import { AppError, Class, ClassInstance, GlobalEnv, DataUtil } from '@travetto/base';
 import { DependencyRegistry, Injectable } from '@travetto/di';
 import { RootIndex } from '@travetto/manifest';
 import { BindUtil, SchemaRegistry, SchemaValidator, ValidationResultError } from '@travetto/schema';
@@ -8,11 +8,11 @@ import { BindUtil, SchemaRegistry, SchemaValidator, ValidationResultError } from
 import { ConfigSourceTarget, ConfigTarget } from './internal/types';
 import { ParserManager } from './parser/parser';
 import { ConfigData } from './parser/types';
-import { ConfigSource } from './source/types';
+import { ConfigSource, ConfigSpec } from './source/types';
 import { FileConfigSource } from './source/file';
 import { OverrideConfigSource } from './source/override';
 
-type ConfigSpec = { source: string, priority: number, detail?: string };
+type ConfigSpecSimple = Omit<ConfigSpec, 'data'>;
 
 /**
  * Manager for application configuration
@@ -21,22 +21,9 @@ type ConfigSpec = { source: string, priority: number, detail?: string };
 export class ConfigurationService {
 
   #storage: Record<string, unknown> = {};   // Lowered, and flattened
-  #specs: ConfigSpec[] = [];
+  #specs: ConfigSpecSimple[] = [];
   #secrets: (RegExp | string)[] = [/secure(-|_|[a-z])|password|private|secret|salt|(api(-|_)?key)/i];
 
-  async #toSpecPairs(cfg: ConfigSource): Promise<[ConfigSpec, ConfigData][]> {
-    const data = await cfg.getData();
-    if (!data) {
-      return [];
-    }
-    const arr = Array.isArray(data) ? data : [data];
-    return arr.map((d, i) => [{
-      priority: cfg.priority + i,
-      source: cfg.source,
-      ...(d.__ID__ ? { detail: d.__ID__?.toString() } : {})
-    }, d]);
-  }
-
   /**
    * Get a sub tree of the config, or everything if namespace is not passed
    * @param ns The namespace of the config to search for, can be dotted for accessing sub namespaces
@@ -69,22 +56,23 @@ export class ConfigurationService {
 
     const parser = await DependencyRegistry.getInstance(ParserManager);
 
-    const specPairs = await Promise.all([
-      new FileConfigSource(parser, 'application', 100),
-      new FileConfigSource(parser, GlobalEnv.envName, 200),
-      ...(Env.getList('TRV_PROFILES') ?? []).map((p, i) => new FileConfigSource(parser, p, 300 + i * 10)),
+    const possible = await Promise.all([
+      new FileConfigSource(parser),
       ...configs,
       new OverrideConfigSource()
-    ].map(src => this.#toSpecPairs(src)));
+    ].map(src => src.get()));
 
-    const specs = specPairs.flat().sort(([a], [b]) => a.priority - b.priority);
+    const specs = possible
+      .flat()
+      .filter((x): x is Exclude<typeof x, undefined> => !!x)
+      .sort((a, b) => a.priority - b.priority);
 
-    this.#specs = specs.map(([v]) => v);
-
-    for (const [, element] of specs) {
-      DataUtil.deepAssign(this.#storage, BindUtil.expandPaths(element), 'coerce');
+    for (const spec of specs) {
+      DataUtil.deepAssign(this.#storage, BindUtil.expandPaths(spec.data), 'coerce');
     }
 
+    this.#specs = specs.map(({ data, ...v }) => v);
+
     // Initialize Secrets
     const userSpecified = (this.#get('config')?.secrets ?? []);
     for (const el of Array.isArray(userSpecified) ? userSpecified : [userSpecified]) {
@@ -102,7 +90,7 @@ export class ConfigurationService {
    * Export all active configuration, useful for displaying active state
    *   - Will not show fields marked as secret
    */
-  async exportActive(): Promise<{ sources: ConfigSpec[], active: ConfigData }> {
+  async exportActive(): Promise<{ sources: ConfigSpecSimple[], active: ConfigData }> {
     const configTargets = await DependencyRegistry.getCandidateTypes(ConfigTarget);
     const configs = await Promise.all(
       configTargets

package/src/source/env.ts

@@ -1,26 +1,23 @@
 import { Env } from '@travetto/base';
 
-import { ConfigSource } from './types';
-import { ConfigData } from '../parser/types';
+import { ConfigSource, ConfigSpec } from './types';
 
 /**
  * Represents the environment mapped data as a JSON blob
  */
 export class EnvConfigSource implements ConfigSource {
-  priority: number;
-  source: string;
   #envKey: string;
+  #priority: number;
 
   constructor(key: string, priority: number) {
     this.#envKey = key;
-    this.priority = priority;
-    this.source = `env://${this.#envKey}`;
+    this.#priority = priority;
   }
 
-  getData(): ConfigData | undefined {
+  get(): ConfigSpec | undefined {
     try {
       const data = JSON.parse(Env.get(this.#envKey, '{}'));
-      return data;
+      return { data, priority: this.#priority, source: `env://${this.#envKey}` };
     } catch (e) {
       console.error(`env.${this.#envKey} is an invalid format`, { text: Env.get(this.#envKey) });
     }

package/src/source/file.ts

@@ -1,44 +1,54 @@
-import { FileQueryProvider } from '@travetto/base';
+import fs from 'fs/promises';
+
+import { Env, FileResourceProvider, GlobalEnv } from '@travetto/base';
 import { RootIndex, path } from '@travetto/manifest';
 
-import { ConfigSource } from './types';
+import { ConfigSource, ConfigSpec } from './types';
 import { ParserManager } from '../parser/parser';
-import { ConfigData } from '../parser/types';
+
+type Profile = [string, number] | readonly [string, number];
 
 /**
  * File-base config source, builds on common file resource provider
  */
-export class FileConfigSource extends FileQueryProvider implements ConfigSource {
+export class FileConfigSource implements ConfigSource {
 
-  priority = 10;
-  source: string;
-  profile: string;
-  parser: ParserManager;
+  #profiles: Profile[];
+  #searchPaths: string[];
+  #parser: ParserManager;
 
-  constructor(parser: ParserManager, profile: string, priority: number, paths: string[] = []) {
-    super({ includeCommon: true, paths });
-    this.priority = priority;
-    this.profile = profile;
-    this.parser = parser;
-    this.source = `file://${profile}`;
+  constructor(parser: ParserManager, paths?: string[], profiles?: Profile[]) {
+    this.#parser = parser;
+    this.#searchPaths = FileResourceProvider.resolveSearchPaths({ includeCommon: true, paths }).reverse();
+    this.#profiles = profiles ?? [
+      ['application', 100],
+      [GlobalEnv.envName, 200],
+      ...(Env.getList('TRV_PROFILES') ?? [])
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        .map((p, i) => [p, 300 + i * 10] as [string, number])
+    ];
   }
 
-  async getData(): Promise<ConfigData[]> {
-    const out: { file: string, data: ConfigData }[] = [];
-    for await (const file of this.query(f => this.parser.matches(f))) {
-      const ext = path.extname(file);
-      const base = path.basename(file, ext);
-      if (base === this.profile && !file.includes('/')) { // Ensures no nesting
-        for (const resolved of await this.resolveAll(file)) {
-          out.push({ file: resolved, data: await this.parser.parse(resolved) });
+  async get(): Promise<ConfigSpec[]> {
+    const cache: Record<string, Promise<string[]>> = {};
+    const configs: Promise<ConfigSpec>[] = [];
+    for (const [profile, priority] of this.#profiles) {
+      let i = 0;
+      for (const folder of this.#searchPaths) {
+        const files = await (cache[folder] ??= fs.readdir(folder).catch(() => []));
+        for (const file of files) {
+          if (this.#parser.matches(file) && path.basename(file, path.extname(file)) === profile) {
+            const full = path.resolve(folder, file);
+            configs.push(this.#parser.parse(full).then(data => ({
+              data,
+              priority: priority + i++,
+              source: `file://${profile}`,
+              detail: full.replace(`${RootIndex.manifest.workspacePath}/`, '')
+            })));
+          }
         }
       }
     }
-
-    // Ensure more specific files are processed later
-    return out
-      .sort((a, b) => (a.file.length - b.file.length) || a.file.localeCompare(b.file))
-      // eslint-disable-next-line @typescript-eslint/naming-convention
-      .map(a => ({ ...a.data, __ID__: a.file.replace(`${RootIndex.manifest.workspacePath}/`, '') }));
+    return Promise.all(configs);
   }
 }

package/src/source/memory.ts

@@ -1,21 +1,17 @@
 import { ConfigData } from '../parser/types';
-import { ConfigSource } from './types';
+import { ConfigSource, ConfigSpec } from './types';
 
 /**
  * Meant to be instantiated and provided as a unique config source
  */
 export class MemoryConfigSource implements ConfigSource {
-  priority: number;
-  data: ConfigData;
-  source: string;
+  #spec: ConfigSpec;
 
   constructor(key: string, data: ConfigData, priority: number = 500) {
-    this.data = data;
-    this.priority = priority;
-    this.source = `memory://${key}`;
+    this.#spec = { data, priority, source: `memory://${key}` };
   }
 
-  getData(): ConfigData {
-    return this.data;
+  get(): ConfigSpec {
+    return this.#spec;
   }
 }

package/src/source/override.ts

@@ -2,16 +2,13 @@ import { SchemaRegistry } from '@travetto/schema';
 
 import { ConfigOverrides, CONFIG_OVERRIDES } from '../internal/types';
 import { ConfigData } from '../parser/types';
-import { ConfigSource } from './types';
+import { ConfigSource, ConfigSpec } from './types';
 
 /**
  * Overridable config source, provides ability to override field level values, currently used by
  * - @EnvVar as a means to allow environment specific overrides
  */
 export class OverrideConfigSource implements ConfigSource {
-  priority = 999;
-  source = 'memory://override';
-
   #build(): ConfigData {
     const out: ConfigData = {};
     for (const cls of SchemaRegistry.getClasses()) {
@@ -26,7 +23,7 @@ export class OverrideConfigSource implements ConfigSource {
     return out;
   }
 
-  getData(): ConfigData {
-    return this.#build();
+  get(): ConfigSpec {
+    return { data: this.#build(), priority: 999, source: 'memory://override' };
   }
 }

package/src/source/types.ts

@@ -1,10 +1,13 @@
 import { ConfigData } from '../parser/types';
 
+type OrProm<T> = T | Promise<T>;
+type OneOf<T> = T[] | T | undefined;
+
+export type ConfigSpec = { data: ConfigData, priority: number, source: string, detail?: string };
+
 /**
  * @concrete ../internal/types:ConfigSourceTarget
  */
 export interface ConfigSource {
-  priority: number;
-  source: string;
-  getData(): Promise<ConfigData[] | ConfigData | undefined> | ConfigData[] | ConfigData | undefined;
+  get(): OrProm<OneOf<ConfigSpec>>;
 }