@geekmidas/envkit 0.0.7 → 0.1.0

package/docs/async-secrets-design.md

@@ -0,0 +1,355 @@
+# Async Secrets Resolution Design
+
+## Problem Statement
+
+Applications often need to fetch sensitive configuration values (secrets) from external providers like HashiCorp Vault, AWS Secrets Manager, or other secret management systems. The current `EnvironmentParser` only supports synchronous parsing of environment variables, which doesn't accommodate async secret fetching.
+
+Additionally, environment variables for secrets typically contain **references** to the actual secret (e.g., a Vault path), not the secret value itself:
+
+```bash
+# Environment variables
+DB_HOST=localhost           # Actual value
+DB_PASSWORD=/vault/prod/db  # Reference to secret, not the actual password
+API_KEY=/vault/prod/api     # Reference to secret
+```
+
+## Proposed Solution
+
+Extend `EnvironmentParser` with:
+1. A separate `getSecret()` getter to distinguish secrets from regular env vars
+2. A configurable `secretsResolver` that fetches actual values from refs
+3. A cache to avoid redundant fetches for the same ref
+4. An `echoSecretsResolver` for testing that returns refs as values
+
+## API Design
+
+### Constructor Options
+
+```typescript
+interface EnvironmentParserOptions {
+  /**
+   * Function to resolve secret references to actual values.
+   * Receives an array of refs (values from env vars marked as secrets).
+   * Returns a Map of ref → resolved value.
+   */
+  secretsResolver?: SecretsResolver;
+}
+
+type SecretsResolver = (refs: string[]) => Promise<Map<string, string>>;
+```
+
+### Getters
+
+```typescript
+parser.create((get, getSecret) => ({
+  // Regular env var - resolved synchronously
+  host: get('DB_HOST').string(),
+  port: get('PORT').string().transform(Number),
+
+  // Secret env var - resolved asynchronously via secretsResolver
+  // The env var value is treated as a ref, not the actual value
+  password: getSecret('DB_PASSWORD').string(),
+  apiKey: getSecret('API_KEY').string(),
+}));
+```
+
+### Parsed Config Types
+
+```typescript
+// Regular values are their actual types
+config.host     // string
+config.port     // number
+
+// Secret values are Promises of their types
+config.password // Promise<string>
+config.apiKey   // Promise<string>
+
+// Usage
+const password = await config.password;
+```
+
+### Built-in Resolvers
+
+```typescript
+import { echoSecretsResolver } from '@geekmidas/envkit';
+
+/**
+ * Echo resolver returns the ref as the value.
+ * Useful for testing where the "ref" IS the actual test value.
+ */
+export const echoSecretsResolver: SecretsResolver = async (refs) =>
+  new Map(refs.map((ref) => [ref, ref]));
+```
+
+## Resolution Flow
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                         parse() called                               │
+└─────────────────────────────────────────────────────────────────────┘
+                                   │
+                                   ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│  1. Regular env vars (get) are parsed synchronously as normal        │
+│     config.host = "localhost"                                        │
+│     config.port = 3000                                               │
+└─────────────────────────────────────────────────────────────────────┘
+                                   │
+                                   ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│  2. Secret env vars (getSecret) return Promises                      │
+│     config.password = Promise<string>                                │
+│     config.apiKey = Promise<string>                                  │
+└─────────────────────────────────────────────────────────────────────┘
+                                   │
+                                   ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│  3. When Promise is awaited:                                         │
+│     a. Read ref from env var (e.g., "/vault/prod/db")               │
+│     b. Check cache - if cached, return cached value                  │
+│     c. If not cached, call secretsResolver([ref])                    │
+│     d. Cache the resolved value                                      │
+│     e. Apply Zod validation/transformation                           │
+│     f. Return validated value                                        │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+## Caching Strategy
+
+A resolved secrets cache prevents redundant API calls:
+
+```typescript
+// Internal cache (per EnvironmentParser instance)
+private resolvedCache = new Map<string, string>();
+
+async resolveSecret(ref: string): Promise<string> {
+  // Return cached value if available
+  if (this.resolvedCache.has(ref)) {
+    return this.resolvedCache.get(ref)!;
+  }
+
+  // Fetch from resolver
+  const resolved = await this.secretsResolver([ref]);
+  const value = resolved.get(ref);
+
+  if (value === undefined) {
+    throw new Error(`Secret resolver did not return value for ref: ${ref}`);
+  }
+
+  // Cache for future use
+  this.resolvedCache.set(ref, value);
+  return value;
+}
+```
+
+Benefits:
+- Same ref accessed multiple times → single resolver call
+- Consistent values within a parser instance
+- Reduces load on secret providers
+
+## Usage Examples
+
+### Production with Vault
+
+```typescript
+import { EnvironmentParser } from '@geekmidas/envkit';
+
+// Vault resolver implementation
+const vaultResolver: SecretsResolver = async (refs) => {
+  const secrets = await vaultClient.batchRead(refs);
+  return new Map(refs.map((ref, i) => [ref, secrets[i].value]));
+};
+
+const parser = new EnvironmentParser(process.env, {
+  secretsResolver: vaultResolver,
+});
+
+const config = parser.create((get, getSecret) => ({
+  database: {
+    host: get('DB_HOST').string(),
+    port: get('DB_PORT').coerce.number(),
+    password: getSecret('DB_PASSWORD').string(),
+  },
+  api: {
+    baseUrl: get('API_BASE_URL').string().url(),
+    key: getSecret('API_KEY').string().min(32),
+  },
+})).parse();
+
+// Use in service registration
+const databaseService = {
+  serviceName: 'database' as const,
+  async register(envParser: EnvironmentParser<{}>) {
+    const config = envParser.create((get, getSecret) => ({
+      host: get('DB_HOST').string(),
+      password: getSecret('DB_PASSWORD').string(),
+    })).parse();
+
+    // Await the secret
+    const password = await config.password;
+
+    return new Database({
+      host: config.host,
+      password,
+    });
+  },
+};
+```
+
+### Testing with Echo Resolver
+
+```typescript
+import { EnvironmentParser, echoSecretsResolver } from '@geekmidas/envkit';
+
+describe('DatabaseService', () => {
+  it('should connect with credentials', async () => {
+    // In tests, the "ref" IS the actual test value
+    const env = {
+      DB_HOST: 'localhost',
+      DB_PORT: '5432',
+      DB_PASSWORD: 'test-password-123', // This IS the password for tests
+    };
+
+    const parser = new EnvironmentParser(env, {
+      secretsResolver: echoSecretsResolver,
+    });
+
+    const service = await databaseService.register(parser);
+
+    expect(service.isConnected()).toBe(true);
+  });
+});
+```
+
+### AWS Secrets Manager
+
+```typescript
+import { SecretsManagerClient, BatchGetSecretValueCommand } from '@aws-sdk/client-secrets-manager';
+
+const awsResolver: SecretsResolver = async (refs) => {
+  const client = new SecretsManagerClient({});
+  const command = new BatchGetSecretValueCommand({
+    SecretIdList: refs,
+  });
+
+  const response = await client.send(command);
+  const result = new Map<string, string>();
+
+  for (const secret of response.SecretValues ?? []) {
+    if (secret.ARN && secret.SecretString) {
+      result.set(secret.ARN, secret.SecretString);
+    }
+  }
+
+  return result;
+};
+```
+
+## Type Definitions
+
+```typescript
+/**
+ * Function type for resolving secret references to actual values.
+ */
+export type SecretsResolver = (refs: string[]) => Promise<Map<string, string>>;
+
+/**
+ * Extended getter that includes secret() method.
+ */
+export type SecretEnvFetcher<TPath extends string = string> = (
+  name: TPath,
+) => typeof z;
+
+/**
+ * Builder function signature with both getters.
+ */
+export type EnvironmentBuilderWithSecrets<TResponse extends EmptyObject> = (
+  get: EnvFetcher,
+  getSecret: SecretEnvFetcher,
+) => TResponse;
+
+/**
+ * Infers config type, wrapping secret values in Promise.
+ */
+export type InferConfigWithSecrets<T extends EmptyObject> = {
+  [K in keyof T]: T[K] extends SecretSchema<infer U>
+    ? Promise<U>
+    : T[K] extends z.ZodSchema
+      ? z.infer<T[K]>
+      : T[K] extends Record<string, unknown>
+        ? InferConfigWithSecrets<T[K]>
+        : T[K];
+};
+```
+
+## Error Handling
+
+### Missing Resolver
+
+```typescript
+// If getSecret() is used but no resolver provided
+const parser = new EnvironmentParser(env); // no resolver
+
+const config = parser.create((get, getSecret) => ({
+  password: getSecret('DB_PASSWORD').string(),
+})).parse();
+
+await config.password;
+// Error: SecretsResolver is required when using getSecret().
+// Configure it via EnvironmentParser options.
+```
+
+### Missing Ref in Environment
+
+```typescript
+// If env var doesn't exist
+const env = { DB_HOST: 'localhost' }; // DB_PASSWORD not set
+
+const config = parser.create((get, getSecret) => ({
+  password: getSecret('DB_PASSWORD').string(),
+})).parse();
+
+await config.password;
+// Error: Environment variable "DB_PASSWORD" is not defined.
+// Expected a secret reference.
+```
+
+### Resolver Doesn't Return Value
+
+```typescript
+// If resolver doesn't return value for a ref
+const brokenResolver: SecretsResolver = async (refs) => new Map();
+
+await config.password;
+// Error: Secret resolver did not return value for ref: /vault/prod/db
+```
+
+## Migration Path
+
+Existing code using `EnvironmentParser` continues to work unchanged:
+
+```typescript
+// Before (still works)
+const config = parser.create((get) => ({
+  port: get('PORT').string().transform(Number),
+})).parse();
+
+// After (opt-in to secrets)
+const config = parser.create((get, getSecret) => ({
+  port: get('PORT').string().transform(Number),
+  password: getSecret('DB_PASSWORD').string(),
+})).parse();
+```
+
+## Open Questions
+
+1. **Batch resolution timing**: Should we batch all secret resolutions when `parse()` is called, or resolve lazily when each Promise is awaited?
+   - **Lazy (proposed)**: Each secret resolved on first await, cached for subsequent access
+   - **Eager**: All secrets resolved upfront in parse(), requires parseAsync()
+
+2. **Cache scope**: Should the cache be per-parser instance or global?
+   - **Per-instance (proposed)**: Isolated, predictable behavior
+   - **Global**: More efficient for multiple parsers with same refs
+
+3. **Cache invalidation**: Should there be a way to clear the cache?
+   - Could add `parser.clearSecretCache()` method if needed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geekmidas/envkit",
-  "version": "0.0.7",
+  "version": "0.1.0",
   "private": false,
   "type": "module",
   "exports": {
@@ -13,8 +13,17 @@
       "types": "./dist/sst.d.ts",
       "import": "./dist/sst.mjs",
       "require": "./dist/sst.cjs"
+    },
+    "./sniffer": {
+      "types": "./dist/SnifferEnvironmentParser.d.ts",
+      "import": "./dist/SnifferEnvironmentParser.mjs",
+      "require": "./dist/SnifferEnvironmentParser.cjs"
     }
   },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/geekmidas/toolbox"
+  },
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -25,7 +34,7 @@
     "lodash.snakecase": "~4.1.1"
   },
   "peerDependencies": {
-    "zod": "~3.25.67"
+    "zod": "~4.1.13"
   },
   "devDependencies": {
     "@types/lodash.set": "~4.3.9",

package/src/EnvironmentBuilder.ts

@@ -0,0 +1,196 @@
+import snakecase from 'lodash.snakecase';
+
+/**
+ * Converts a string to environment variable case format (UPPER_SNAKE_CASE).
+ * Numbers following underscores are preserved without the underscore.
+ *
+ * @param name - The string to convert
+ * @returns The converted string in environment variable format
+ *
+ * @example
+ * environmentCase('myVariable') // 'MY_VARIABLE'
+ * environmentCase('apiV2') // 'APIV2'
+ */
+export function environmentCase(name: string): string {
+  return snakecase(name)
+    .toUpperCase()
+    .replace(/_\d+/g, (r) => {
+      return r.replace('_', '');
+    });
+}
+
+/**
+ * A record of environment variable names to their values.
+ * Values can be primitives or nested records.
+ */
+export interface EnvRecord {
+  [key: string]: EnvValue;
+}
+
+/**
+ * Represents a value that can be stored in an environment record.
+ * Can be a primitive value or a nested record of environment values.
+ */
+export type EnvValue = string | number | boolean | EnvRecord;
+
+/**
+ * A resolver function that converts a typed value into environment variables.
+ *
+ * @template T - The type of value this resolver handles (without the `type` key)
+ * @param key - The key name from the input record
+ * @param value - The value to resolve (without the `type` key)
+ * @returns A record of environment variable names to their values
+ */
+export type EnvironmentResolver<T = any> = (key: string, value: T) => EnvRecord;
+
+/**
+ * A map of type discriminator strings to their resolver functions.
+ */
+export type Resolvers = Record<string, EnvironmentResolver<any>>;
+
+/**
+ * Options for configuring the EnvironmentBuilder.
+ */
+export interface EnvironmentBuilderOptions {
+  /**
+   * Handler called when a value's type doesn't match any registered resolver.
+   * Defaults to console.warn.
+   */
+  onUnmatchedValue?: (key: string, value: unknown) => void;
+}
+
+/**
+ * Input value type - either a string or an object with a `type` discriminator.
+ */
+export type InputValue = string | { type: string; [key: string]: unknown };
+
+/**
+ * Base type for typed input values with a specific type discriminator.
+ */
+export type TypedInputValue<TType extends string = string> = {
+  type: TType;
+  [key: string]: unknown;
+};
+
+/**
+ * Extracts the `type` string value from an input value.
+ */
+type ExtractType<T> = T extends { type: infer U extends string } ? U : never;
+
+/**
+ * Removes the `type` key from an object type.
+ */
+type OmitType<T> = T extends { type: string } ? Omit<T, 'type'> : never;
+
+/**
+ * Extracts all unique `type` values from a record (excluding plain strings).
+ */
+type AllTypeValues<TRecord extends Record<string, InputValue>> = {
+  [K in keyof TRecord]: ExtractType<TRecord[K]>;
+}[keyof TRecord];
+
+/**
+ * For a given type value, finds the corresponding value type (without `type` key).
+ */
+type ValueForType<
+  TRecord extends Record<string, InputValue>,
+  TType extends string,
+> = {
+  [K in keyof TRecord]: TRecord[K] extends { type: TType }
+    ? OmitType<TRecord[K]>
+    : never;
+}[keyof TRecord];
+
+/**
+ * Generates typed resolvers based on the input record.
+ * Keys are the `type` values, values are resolver functions receiving the value without `type`.
+ */
+export type TypedResolvers<TRecord extends Record<string, InputValue>> = {
+  [TType in AllTypeValues<TRecord>]: EnvironmentResolver<
+    ValueForType<TRecord, TType>
+  >;
+};
+
+/**
+ * A generic, extensible class for building environment variables from
+ * objects with type-discriminated values.
+ *
+ * @template TRecord - The input record type for type inference
+ * @template TResolvers - The resolvers type (defaults to TypedResolvers<TRecord>)
+ *
+ * @example
+ * ```typescript
+ * const env = new EnvironmentBuilder(
+ *   {
+ *     apiKey: { type: 'secret', value: 'xyz' },
+ *     appName: 'my-app'
+ *   },
+ *   {
+ *     // `value` is typed as { value: string } (without `type`)
+ *     secret: (key, value) => ({ [key]: value.value }),
+ *   }
+ * ).build();
+ * // { API_KEY: 'xyz', APP_NAME: 'my-app' }
+ * ```
+ */
+export class EnvironmentBuilder<
+  TRecord extends Record<string, InputValue> = Record<string, InputValue>,
+  TResolvers extends Resolvers = TypedResolvers<TRecord>,
+> {
+  private readonly record: TRecord;
+  private readonly resolvers: TResolvers;
+  private readonly options: Required<EnvironmentBuilderOptions>;
+
+  constructor(
+    record: TRecord,
+    resolvers: TResolvers,
+    options: EnvironmentBuilderOptions = {},
+  ) {
+    this.record = record;
+    this.resolvers = resolvers;
+    this.options = {
+      onUnmatchedValue:
+        options.onUnmatchedValue ??
+        ((key, value) => {
+          console.warn(`No resolver found for key "${key}":`, { value });
+        }),
+    };
+  }
+
+  /**
+   * Build environment variables from the input record.
+   *
+   * - Plain string values are passed through with key transformation
+   * - Object values with a `type` property are matched against resolvers
+   * - Resolvers receive values without the `type` key
+   * - Only root-level keys are transformed to UPPER_SNAKE_CASE
+   *
+   * @returns A record of environment variables
+   */
+  build(): EnvRecord {
+    const env: EnvRecord = {};
+
+    for (const [key, value] of Object.entries(this.record)) {
+      // Handle plain string values
+      if (typeof value === 'string') {
+        env[environmentCase(key)] = value;
+        continue;
+      }
+
+      // Handle objects with type discriminator
+      const { type, ...rest } = value;
+      const resolver = this.resolvers[type];
+      if (resolver) {
+        const resolved = resolver(key, rest);
+        // Transform only root-level keys from resolver output
+        for (const [resolvedKey, resolvedValue] of Object.entries(resolved)) {
+          env[environmentCase(resolvedKey)] = resolvedValue;
+        }
+      } else {
+        this.options.onUnmatchedValue(key, value);
+      }
+    }
+
+    return env;
+  }
+}

package/src/EnvironmentParser.ts

@@ -13,8 +13,12 @@ export class ConfigParser<TResponse extends EmptyObject> {
    * Creates a new ConfigParser instance.
    *
    * @param config - The configuration object to parse
+   * @param envVars - Set of environment variable names that were accessed
    */
-  constructor(private readonly config: TResponse) {}
+  constructor(
+    private readonly config: TResponse,
+    private readonly envVars: Set<string> = new Set(),
+  ) {}
   /**
    * Parses the config object and validates it against the Zod schemas
    * @returns The parsed config object
@@ -65,6 +69,26 @@ export class ConfigParser<TResponse extends EmptyObject> {
 
     return parsedConfig;
   }
+
+  /**
+   * Returns an array of environment variable names that were accessed during config creation.
+   * This is useful for deployment and configuration management to know which env vars are required.
+   *
+   * @returns Array of environment variable names, sorted alphabetically
+   *
+   * @example
+   * ```typescript
+   * const config = envParser.create((get) => ({
+   *   dbUrl: get('DATABASE_URL').string(),
+   *   port: get('PORT').number()
+   * }));
+   *
+   * config.getEnvironmentVariables(); // ['DATABASE_URL', 'PORT']
+   * ```
+   */
+  getEnvironmentVariables(): string[] {
+    return Array.from(this.envVars).sort();
+  }
 }
 
 /**
@@ -87,6 +111,11 @@ export class ConfigParser<TResponse extends EmptyObject> {
  * ```
  */
 export class EnvironmentParser<T extends EmptyObject> {
+  /**
+   * Set to track which environment variable names have been accessed
+   */
+  private readonly accessedVars: Set<string> = new Set();
+
   /**
    * Creates a new EnvironmentParser instance.
    *
@@ -177,6 +206,9 @@ export class EnvironmentParser<T extends EmptyObject> {
    * @returns A proxied Zod object with wrapped schema creators
    */
   private getZodGetter = (name: string) => {
+    // Track that this environment variable was accessed
+    this.accessedVars.add(name);
+
     // Return an object that has all Zod schemas but with our wrapper
     return new Proxy(
       { ...z },
@@ -227,7 +259,24 @@ export class EnvironmentParser<T extends EmptyObject> {
     builder: (get: EnvFetcher) => TReturn,
   ): ConfigParser<TReturn> {
     const config = builder(this.getZodGetter);
-    return new ConfigParser(config);
+    return new ConfigParser(config, this.accessedVars);
+  }
+
+  /**
+   * Returns an array of environment variable names that were accessed via the getter.
+   * This is useful for build-time analysis to determine which env vars a service needs.
+   *
+   * @returns Array of environment variable names, sorted alphabetically
+   *
+   * @example
+   * ```typescript
+   * const sniffer = new EnvironmentParser({});
+   * service.register(sniffer);
+   * const envVars = sniffer.getEnvironmentVariables(); // ['DATABASE_URL', 'PORT']
+   * ```
+   */
+  getEnvironmentVariables(): string[] {
+    return Array.from(this.accessedVars).sort();
   }
 }