@travetto/config 3.4.0-rc.3 → 3.4.0-rc.5

package/README.md

@@ -20,11 +20,18 @@ The configuration information is comprised of:
    *  configuration files - [YAML](https://en.wikipedia.org/wiki/YAML), [JSON](https://www.json.org), and basic properties file
    *  configuration classes
 Config loading follows a defined resolution path, below is the order in increasing specificity (`ext` can be `yaml`, `yml`, `json`, `properties`):
-   1. `resources/application.<ext>` - Load the default `application.<ext>` if available.
-   1. `resources/*.<ext>` - Load profile specific configurations as defined by the values in `process.env.TRV_PROFILES`
-   1. `resources/{env}.<ext>` - Load environment specific profile configurations as defined by the values of `process.env.TRV_ENV`.
+   1. `resources/application.<ext>` - Priority `100` - Load the default `application.<ext>` if available.
+   1. `resources/{env}.<ext>` - Priority `200` - Load environment specific profile configurations as defined by the values of `process.env.TRV_ENV`.
+   1. `resources/*.<ext>` - Priority `300` - Load profile specific configurations as defined by the values in `process.env.TRV_PROFILES`
+   1. [@Injectable](https://github.com/travetto/travetto/tree/main/module/di/src/decorator.ts#L31) [ConfigSource](https://github.com/travetto/travetto/tree/main/module/config/src/source/types.ts#L6) - Priority `???` - These are custom config sources provided by the module, and are able to define their own priorities
+   1. [OverrideConfigSource](https://github.com/travetto/travetto/tree/main/module/config/src/source/override.ts#L11) - Priority `999` - This is for [EnvVar](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L34) overrides, and is at the top priority for all built-in config sources.
 By default all configuration data is inert, and will only be applied when constructing an instance of a configuration class.
 
+**Note**: When working in a monorepo, the parent resources folder will also be searched with a lower priority than the the module's specific resources.  This allows for shared-global configuration that can be overridden at the module level. The general priority is:
+   1. Mono-repo root
+   1. Module root
+   1. Folders for `TRV_RESOURCES`, in order
+
 ### A Complete Example
 A more complete example setup would look like:
 
@@ -65,9 +72,17 @@ $ trv main doc/resolve.ts
 
 Config {
   sources: [
-    'application.1 - file://./doc/resources/application.yml',
-    'prod.1 - file://./doc/resources/prod.json',
-    'override.3 - memory://override'
+    {
+      priority: 100,
+      source: 'file://application',
+      detail: 'module/config/doc/resources/application.yml'
+    },
+    {
+      priority: 300,
+      source: 'file://prod',
+      detail: 'module/config/doc/resources/prod.json'
+    },
+    { priority: 999, source: 'memory://override' }
   ],
   active: {
     DBConfig: { host: 'prod-host-db', port: 2000, creds: { user: 'admin-user' } }
@@ -81,95 +96,84 @@ The framework provides two simple base classes that assist with existing pattern
 **Code: Memory Provider**
 ```typescript
 import { ConfigData } from '../parser/types';
-import { ConfigSource, ConfigValue } from './types';
+import { ConfigSource } from './types';
 
 /**
  * Meant to be instantiated and provided as a unique config source
  */
 export class MemoryConfigSource implements ConfigSource {
-  priority = 1;
-  data: Record<string, ConfigData>;
-  name = 'memory';
+  priority: number;
+  data: ConfigData;
+  source: string;
 
-  constructor(data: Record<string, ConfigData>, priority: number = 1) {
+  constructor(key: string, data: ConfigData, priority: number = 500) {
     this.data = data;
     this.priority = priority;
+    this.source = `memory://${key}`;
   }
 
-  getValues(profiles: string[]): ConfigValue[] {
-    const out: ConfigValue[] = [];
-    for (const profile of profiles) {
-      if (this.data[profile]) {
-        out.push({ profile, config: this.data[profile], source: `${this.name}://${profile}`, priority: this.priority });
-      }
-    }
-    return out;
+  getData(): ConfigData {
+    return this.data;
   }
 }
 ```
 
 **Code: Environment JSON Provider**
 ```typescript
-import { Env, GlobalEnv } from '@travetto/base';
+import { Env } from '@travetto/base';
 
-import { ConfigSource, ConfigValue } from './types';
+import { ConfigSource } from './types';
+import { ConfigData } from '../parser/types';
 
 /**
  * Represents the environment mapped data as a JSON blob
  */
 export class EnvConfigSource implements ConfigSource {
   priority: number;
-  name = 'env';
+  source: string;
   #envKey: string;
 
   constructor(key: string, priority: number) {
     this.#envKey = key;
     this.priority = priority;
+    this.source = `env://${this.#envKey}`;
   }
 
-  getValues(profiles: string[]): ConfigValue[] {
+  getData(): ConfigData | undefined {
     try {
       const data = JSON.parse(Env.get(this.#envKey, '{}'));
-      return [{ profile: GlobalEnv.envName, config: data, source: `${this.name}://${this.#envKey}`, priority: this.priority }];
+      return data;
     } catch (e) {
       console.error(`env.${this.#envKey} is an invalid format`, { text: Env.get(this.#envKey) });
-      return [];
     }
   }
 }
 ```
 
 ### Custom Configuration Provider
-In addition to files and environment variables, configuration sources can also be provided via the class itself.  This is useful for reading remote configurations, or dealing with complex configuration normalization.  The only caveat to this pattern, is that the these configuration sources cannot rely on the [ConfigurationService](https://github.com/travetto/travetto/tree/main/module/config/src/service.ts#L16) service for input.  This means any needed configuration will need to be accessed via specific patterns.
+In addition to files and environment variables, configuration sources can also be provided via the class itself.  This is useful for reading remote configurations, or dealing with complex configuration normalization.  The only caveat to this pattern, is that the these configuration sources cannot rely on the [ConfigurationService](https://github.com/travetto/travetto/tree/main/module/config/src/service.ts#L21) service for input.  This means any needed configuration will need to be accessed via specific patterns.
 
 **Code: Custom Configuration Source**
 ```typescript
-import { ConfigSource, ConfigValue } from '@travetto/config';
+import { ConfigData, ConfigSource } from '@travetto/config';
 import { Injectable } from '@travetto/di';
 
 @Injectable()
 export class CustomConfigSource implements ConfigSource {
-  priority = 1000;
-  name = 'custom';
-
-  async getValues(): Promise<ConfigValue[]> {
-    return [
-      {
-        config: { user: { name: 'bob' } },
-        priority: this.priority,
-        profile: 'override',
-        source: `custom://${CustomConfigSource.name}`
-      }
-    ];
+  priority = 2000;
+  source = 'custom://override';
+
+  async getData(): Promise<ConfigData> {
+    return { user: { name: 'bob' } };
   }
 }
 ```
 
 ## Startup
-At startup, the [ConfigurationService](https://github.com/travetto/travetto/tree/main/module/config/src/service.ts#L16) service will log out all the registered configuration objects.  The configuration state output is useful to determine if everything is configured properly when diagnosing runtime errors.  This service will find all configurations, and output a redacted version with all secrets removed.  The default pattern for secrets is `/password|private|secret/i`.  More values can be added in your configuration under the path `config.secrets`.  These values can either be simple strings (for exact match), or `/pattern/` to create a regular expression.
+At startup, the [ConfigurationService](https://github.com/travetto/travetto/tree/main/module/config/src/service.ts#L21) service will log out all the registered configuration objects.  The configuration state output is useful to determine if everything is configured properly when diagnosing runtime errors.  This service will find all configurations, and output a redacted version with all secrets removed.  The default pattern for secrets is `/password|private|secret/i`.  More values can be added in your configuration under the path `config.secrets`.  These values can either be simple strings (for exact match), or `/pattern/` to create a regular expression.
 
 ## Consuming
-The [ConfigurationService](https://github.com/travetto/travetto/tree/main/module/config/src/service.ts#L16) service provides injectable access to all of the loaded configuration. For simplicity, a decorator, [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L13) allows for classes to automatically be bound with config information on post construction via the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module. The decorator will install a `postConstruct` method if not already defined, that performs the binding of configuration.  This is due to the fact that we cannot rewrite the constructor, and order of operation matters.
+The [ConfigurationService](https://github.com/travetto/travetto/tree/main/module/config/src/service.ts#L21) service provides injectable access to all of the loaded configuration. For simplicity, a decorator, [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L13) allows for classes to automatically be bound with config information on post construction via the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module. The decorator will install a `postConstruct` method if not already defined, that performs the binding of configuration.  This is due to the fact that we cannot rewrite the constructor, and order of operation matters.
 
 ### Environment Variables
 Additionally there are times in which you may want to also support configuration via environment variables.  [EnvVar](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L34) supports override configuration values when environment variables are present. 
@@ -228,9 +232,17 @@ $ DATABASE_PORT=200 trv main doc/dbconfig-run.ts
 
 Config {
   sources: [
-    'application.1 - file://./doc/resources/application.yml',
-    'prod.1 - file://./doc/resources/prod.json',
-    'override.3 - memory://override'
+    {
+      priority: 100,
+      source: 'file://application',
+      detail: 'module/config/doc/resources/application.yml'
+    },
+    {
+      priority: 300,
+      source: 'file://prod',
+      detail: 'module/config/doc/resources/prod.json'
+    },
+    { priority: 999, source: 'memory://override' }
   ],
   active: {
     DBConfig: { host: 'prod-host-db', port: 200, creds: { user: 'admin-user' } }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/config",
-  "version": "3.4.0-rc.3",
+  "version": "3.4.0-rc.5",
   "description": "Configuration support",
   "keywords": [
     "yaml",
@@ -26,9 +26,9 @@
     "directory": "module/config"
   },
   "dependencies": {
-    "@travetto/di": "^3.4.0-rc.3",
-    "@travetto/schema": "^3.4.0-rc.3",
-    "@travetto/yaml": "^3.4.0-rc.3"
+    "@travetto/di": "^3.4.0-rc.5",
+    "@travetto/schema": "^3.4.0-rc.5",
+    "@travetto/yaml": "^3.4.0-rc.5"
   },
   "travetto": {
     "displayName": "Configuration"

package/src/parser/parser.ts

@@ -0,0 +1,37 @@
+import fs from 'fs/promises';
+
+import { DependencyRegistry, Injectable } from '@travetto/di';
+import { AppError } from '@travetto/base';
+import { path } from '@travetto/manifest';
+
+import { ConfigParserTarget } from '../internal/types';
+import { ConfigData, ConfigParser } from './types';
+
+@Injectable()
+export class ParserManager {
+
+  #extMatch: RegExp;
+  #parsers: Record<string, ConfigParser>;
+
+  async postConstruct(): Promise<void> {
+    const parserClasses = await DependencyRegistry.getCandidateTypes(ConfigParserTarget);
+    const parsers = await Promise.all(parserClasses.map(x => DependencyRegistry.getInstance<ConfigParser>(x.class, x.qualifier)));
+
+    // Register parsers
+    this.#parsers = Object.fromEntries(parsers.flatMap(p => p.ext.map(e => [e, p])));
+
+    this.#extMatch = parsers.length ? new RegExp(`(${Object.keys(this.#parsers).join('|').replaceAll('.', '[.]')})`) : /^$/;
+  }
+
+  async parse(file: string): Promise<ConfigData> {
+    const ext = path.extname(file);
+    if (!this.#parsers[ext]) {
+      throw new AppError(`Unknown config format: ${ext}`, 'data');
+    }
+    return fs.readFile(file, 'utf8').then(content => this.#parsers[ext].parse(content));
+  }
+
+  matches(file: string): boolean {
+    return this.#extMatch.test(file);
+  }
+}

package/src/service.ts

@@ -6,8 +6,13 @@ import { RootIndex } from '@travetto/manifest';
 import { BindUtil, SchemaRegistry, SchemaValidator, ValidationResultError } from '@travetto/schema';
 
 import { ConfigSourceTarget, ConfigTarget } from './internal/types';
+import { ParserManager } from './parser/parser';
 import { ConfigData } from './parser/types';
-import { ConfigSource, ConfigValue } from './source/types';
+import { ConfigSource } from './source/types';
+import { FileConfigSource } from './source/file';
+import { OverrideConfigSource } from './source/override';
+
+type ConfigSpec = { source: string, priority: number, detail?: string };
 
 /**
  * Manager for application configuration
@@ -15,21 +20,23 @@ import { ConfigSource, ConfigValue } from './source/types';
 @Injectable()
 export class ConfigurationService {
 
-  private static getSorted(configs: ConfigValue[], profiles: string[]): ConfigValue[] {
-    const order = Object.fromEntries(Object.entries(profiles).map(([k, v]) => [v, +k] as const));
-
-    return configs.sort((left, right) =>
-      (order[left.profile] - order[right.profile]) ||
-      left.priority - right.priority ||
-      left.source.localeCompare(right.source)
-    );
-  }
-
   #storage: Record<string, unknown> = {};   // Lowered, and flattened
-  #profiles: string[] = ['application', GlobalEnv.envName, ...Env.getList('TRV_PROFILES') ?? [], 'override'];
-  #sources: string[] = [];
+  #specs: ConfigSpec[] = [];
   #secrets: (RegExp | string)[] = [/secure(-|_|[a-z])|password|private|secret|salt|(api(-|_)?key)/i];
 
+  async #toSpecPairs(cfg: ConfigSource): Promise<[ConfigSpec, ConfigData][]> {
+    const data = await cfg.getData();
+    if (!data) {
+      return [];
+    }
+    const arr = Array.isArray(data) ? data : [data];
+    return arr.map((d, i) => [{
+      priority: cfg.priority + i,
+      source: cfg.source,
+      ...(d.__ID__ ? { detail: d.__ID__?.toString() } : {})
+    }, d]);
+  }
+
   /**
    * Get a sub tree of the config, or everything if namespace is not passed
    * @param ns The namespace of the config to search for, can be dotted for accessing sub namespaces
@@ -57,17 +64,24 @@ export class ConfigurationService {
     const providers = await DependencyRegistry.getCandidateTypes(ConfigSourceTarget);
 
     const configs = await Promise.all(
-      providers.map(async (el) => {
-        const inst = await DependencyRegistry.getInstance<ConfigSource>(el.class, el.qualifier);
-        return inst.getValues(this.#profiles);
-      })
+      providers.map(async (el) => await DependencyRegistry.getInstance<ConfigSource>(el.class, el.qualifier))
     );
 
-    const sorted = ConfigurationService.getSorted(configs.flat(), this.#profiles);
+    const parser = await DependencyRegistry.getInstance(ParserManager);
+
+    const specPairs = await Promise.all([
+      new FileConfigSource(parser, 'application', 100),
+      new FileConfigSource(parser, GlobalEnv.envName, 200),
+      ...(Env.getList('TRV_PROFILES') ?? []).map((p, i) => new FileConfigSource(parser, p, 300 + i * 10)),
+      ...configs,
+      new OverrideConfigSource()
+    ].map(src => this.#toSpecPairs(src)));
+
+    const specs = specPairs.flat().sort(([a], [b]) => a.priority - b.priority);
 
-    this.#sources = sorted.map(x => `${x.profile}.${x.priority} - ${x.source}`);
+    this.#specs = specs.map(([v]) => v);
 
-    for (const { config: element } of sorted) {
+    for (const [, element] of specs) {
       DataUtil.deepAssign(this.#storage, BindUtil.expandPaths(element), 'coerce');
     }
 
@@ -88,7 +102,7 @@ export class ConfigurationService {
    * Export all active configuration, useful for displaying active state
    *   - Will not show fields marked as secret
    */
-  async exportActive(): Promise<{ sources: string[], active: ConfigData }> {
+  async exportActive(): Promise<{ sources: ConfigSpec[], active: ConfigData }> {
     const configTargets = await DependencyRegistry.getCandidateTypes(ConfigTarget);
     const configs = await Promise.all(
       configTargets
@@ -106,7 +120,7 @@ export class ConfigurationService {
       );
       out[el.class.name] = DataUtil.filterByKeys(data, this.#secrets);
     }
-    return { sources: this.#sources, active: out };
+    return { sources: this.#specs, active: out };
   }
 
   /**

package/src/source/env.ts

@@ -1,27 +1,28 @@
-import { Env, GlobalEnv } from '@travetto/base';
+import { Env } from '@travetto/base';
 
-import { ConfigSource, ConfigValue } from './types';
+import { ConfigSource } from './types';
+import { ConfigData } from '../parser/types';
 
 /**
  * Represents the environment mapped data as a JSON blob
  */
 export class EnvConfigSource implements ConfigSource {
   priority: number;
-  name = 'env';
+  source: string;
   #envKey: string;
 
   constructor(key: string, priority: number) {
     this.#envKey = key;
     this.priority = priority;
+    this.source = `env://${this.#envKey}`;
   }
 
-  getValues(profiles: string[]): ConfigValue[] {
+  getData(): ConfigData | undefined {
     try {
       const data = JSON.parse(Env.get(this.#envKey, '{}'));
-      return [{ profile: GlobalEnv.envName, config: data, source: `${this.name}://${this.#envKey}`, priority: this.priority }];
+      return data;
     } catch (e) {
       console.error(`env.${this.#envKey} is an invalid format`, { text: Env.get(this.#envKey) });
-      return [];
     }
   }
 }

package/src/source/file.ts

@@ -1,59 +1,42 @@
-import { path } from '@travetto/manifest';
 import { FileQueryProvider } from '@travetto/base';
-import { DependencyRegistry, InjectableFactory } from '@travetto/di';
+import { RootIndex, path } from '@travetto/manifest';
 
-import { ConfigParserTarget } from '../internal/types';
-import { ConfigParser } from '../parser/types';
-import { ConfigSource, ConfigValue } from './types';
+import { ConfigSource } from './types';
+import { ParserManager } from '../parser/parser';
+import { ConfigData } from '../parser/types';
 
 /**
  * File-base config source, builds on common file resource provider
  */
 export class FileConfigSource extends FileQueryProvider implements ConfigSource {
 
-  @InjectableFactory()
-  static getInstance(): ConfigSource {
-    return new FileConfigSource();
-  }
-
-  depth = 1;
-  extMatch: RegExp;
-  parsers: Record<string, ConfigParser>;
-  priority = 1;
+  priority = 10;
+  source: string;
+  profile: string;
+  parser: ParserManager;
 
-  constructor(paths: string[] = []) {
+  constructor(parser: ParserManager, profile: string, priority: number, paths: string[] = []) {
     super({ includeCommon: true, paths });
+    this.priority = priority;
+    this.profile = profile;
+    this.parser = parser;
+    this.source = `file://${profile}`;
   }
 
-  async postConstruct(): Promise<void> {
-    const parserClasses = await DependencyRegistry.getCandidateTypes(ConfigParserTarget);
-    const parsers = await Promise.all(parserClasses.map(x => DependencyRegistry.getInstance<ConfigParser>(x.class, x.qualifier)));
-
-    // Register parsers
-    this.parsers = Object.fromEntries(parsers.flatMap(p => p.ext.map(e => [e, p])));
-
-    this.extMatch = parsers.length ? new RegExp(`(${Object.keys(this.parsers).join('|').replaceAll('.', '[.]')})`) : /^$/;
-  }
-
-  async getValues(profiles: string[]): Promise<ConfigValue[]> {
-    const out: ConfigValue[] = [];
-
-    for await (const file of this.query(f => this.extMatch.test(f))) {
+  async getData(): Promise<ConfigData[]> {
+    const out: ConfigData[] = [];
+    for await (const file of this.query(f => this.parser.matches(f))) {
       const ext = path.extname(file);
-      const profile = path.basename(file, ext);
-      if (!profiles.includes(profile) || !this.parsers[ext]) {
-        continue;
+      const base = path.basename(file, ext);
+      if (base === this.profile && !file.includes('/')) { // Ensures no nesting
+        for (const resolved of await this.resolveAll(file)) {
+          // Ensure more specific files are processed later, .resolveAll returns with most specific first
+          const data = await this.parser.parse(resolved);
+          // eslint-disable-next-line @typescript-eslint/naming-convention    
+          out.push({ ...data, __ID__: resolved.replace(`${RootIndex.manifest.workspacePath}/`, '') });
+        }
       }
-      const content = await this.read(file);
-      const desc = await this.describe(file);
-      out.push({
-        profile,
-        config: await this.parsers[ext].parse(content),
-        source: `file://${desc.path}`,
-        priority: this.priority
-      });
     }
-    // Ensure more specific files are processed later
-    return out.sort((a, b) => (a.source.length - b.source.length) || a.source.localeCompare(b.source));
+    return out.reverse();
   }
 }

package/src/source/memory.ts

@@ -1,26 +1,21 @@
 import { ConfigData } from '../parser/types';
-import { ConfigSource, ConfigValue } from './types';
+import { ConfigSource } from './types';
 
 /**
  * Meant to be instantiated and provided as a unique config source
  */
 export class MemoryConfigSource implements ConfigSource {
-  priority = 1;
-  data: Record<string, ConfigData>;
-  name = 'memory';
+  priority: number;
+  data: ConfigData;
+  source: string;
 
-  constructor(data: Record<string, ConfigData>, priority: number = 1) {
+  constructor(key: string, data: ConfigData, priority: number = 500) {
     this.data = data;
     this.priority = priority;
+    this.source = `memory://${key}`;
   }
 
-  getValues(profiles: string[]): ConfigValue[] {
-    const out: ConfigValue[] = [];
-    for (const profile of profiles) {
-      if (this.data[profile]) {
-        out.push({ profile, config: this.data[profile], source: `${this.name}://${profile}`, priority: this.priority });
-      }
-    }
-    return out;
+  getData(): ConfigData {
+    return this.data;
   }
 }

package/src/source/override.ts

@@ -1,18 +1,16 @@
-import { Injectable } from '@travetto/di';
 import { SchemaRegistry } from '@travetto/schema';
 
 import { ConfigOverrides, CONFIG_OVERRIDES } from '../internal/types';
 import { ConfigData } from '../parser/types';
-import { ConfigSource, ConfigValue } from './types';
+import { ConfigSource } from './types';
 
 /**
  * Overridable config source, provides ability to override field level values, currently used by
  * - @EnvVar as a means to allow environment specific overrides
  */
-@Injectable()
 export class OverrideConfigSource implements ConfigSource {
-  priority = 3;
-  name = 'override';
+  priority = 999;
+  source = 'memory://override';
 
   #build(): ConfigData {
     const out: ConfigData = {};
@@ -28,12 +26,7 @@ export class OverrideConfigSource implements ConfigSource {
     return out;
   }
 
-  getValues(profiles: string[]): [ConfigValue] {
-    return [{
-      config: this.#build(),
-      profile: 'override',
-      source: 'memory://override',
-      priority: this.priority
-    }];
+  getData(): ConfigData {
+    return this.#build();
   }
 }

package/src/source/types.ts

@@ -1,11 +1,10 @@
 import { ConfigData } from '../parser/types';
 
-export type ConfigValue = { config: ConfigData, source: string, profile: string, priority: number };
-
 /**
  * @concrete ../internal/types:ConfigSourceTarget
  */
 export interface ConfigSource {
   priority: number;
-  getValues(profiles: string[]): Promise<ConfigValue[]> | ConfigValue[];
-}
+  source: string;
+  getData(): Promise<ConfigData[] | ConfigData | undefined> | ConfigData[] | ConfigData | undefined;
+}