@push.rocks/smartconfig 6.0.0

package/ts/classes.appdata.ts

@@ -0,0 +1,548 @@
+import * as plugins from './plugins.js';
+import { KeyValueStore } from './classes.keyvaluestore.js';
+
+// ============================================================================
+// Singleton Qenv Provider
+// ============================================================================
+let sharedQenv: plugins.qenv.Qenv | undefined;
+
+function getQenv(): plugins.qenv.Qenv {
+  if (!sharedQenv) {
+    sharedQenv = new plugins.qenv.Qenv(
+      process.cwd(),
+      plugins.path.join(process.cwd(), '.nogit')
+    );
+  }
+  return sharedQenv;
+}
+
+// ============================================================================
+// Security - Redaction for sensitive data
+// ============================================================================
+/**
+ * Redacts sensitive values in logs to prevent exposure of secrets
+ */
+function redactSensitiveValue(key: string, value: unknown): string {
+  // List of patterns that indicate sensitive data
+  const sensitivePatterns = [
+    /secret/i, /token/i, /key/i, /password/i, /pass/i,
+    /api/i, /credential/i, /auth/i, /private/i, /jwt/i,
+    /cert/i, /signature/i, /bearer/i
+  ];
+
+  // Check if key contains sensitive pattern
+  const isSensitive = sensitivePatterns.some(pattern => pattern.test(key));
+
+  if (isSensitive) {
+    if (typeof value === 'string') {
+      // Show first 3 chars and length for debugging
+      return value.length > 3
+        ? `${value.substring(0, 3)}...[${value.length} chars]`
+        : '[redacted]';
+    }
+    return '[redacted]';
+  }
+
+  // Check if value looks like a JWT token or base64 secret
+  if (typeof value === 'string') {
+    // JWT tokens start with eyJ
+    if (value.startsWith('eyJ')) {
+      return `eyJ...[${value.length} chars]`;
+    }
+    // Very long strings might be encoded secrets
+    if (value.length > 100) {
+      return `${value.substring(0, 50)}...[${value.length} chars total]`;
+    }
+  }
+
+  return JSON.stringify(value);
+}
+
+// ============================================================================
+// Type Converters - Centralized conversion logic
+// ============================================================================
+function toBoolean(value: unknown): boolean {
+  // If already boolean, return as-is
+  if (typeof value === 'boolean') {
+    console.log(`    🔹 toBoolean: value is already boolean: ${value}`);
+    return value;
+  }
+
+  // Handle null/undefined
+  if (value == null) {
+    console.log(`    🔹 toBoolean: value is null/undefined, returning false`);
+    return false;
+  }
+
+  // Handle string representations
+  const s = String(value).toLowerCase().trim();
+
+  // True values: "true", "1", "yes", "y", "on"
+  if (['true', '1', 'yes', 'y', 'on'].includes(s)) {
+    console.log(`    🔹 toBoolean: converting "${value}" to true`);
+    return true;
+  }
+
+  // False values: "false", "0", "no", "n", "off"
+  if (['false', '0', 'no', 'n', 'off'].includes(s)) {
+    console.log(`    🔹 toBoolean: converting "${value}" to false`);
+    return false;
+  }
+
+  // Default: non-empty string = true, empty = false
+  const result = s.length > 0;
+  console.log(`    🔹 toBoolean: defaulting "${value}" to ${result}`);
+  return result;
+}
+
+function toJson<T = any>(value: unknown): T | undefined {
+  if (value == null) return undefined;
+  if (typeof value === 'string') {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return undefined;
+    }
+  }
+  return value as T;
+}
+
+function fromBase64(value: unknown): string | undefined {
+  if (value == null) return undefined;
+  try {
+    return Buffer.from(String(value), 'base64').toString('utf8');
+  } catch {
+    return String(value);
+  }
+}
+
+function toNumber(value: unknown): number | undefined {
+  if (value == null) return undefined;
+  const num = Number(value);
+  return Number.isNaN(num) ? undefined : num;
+}
+
+function toString(value: unknown): string | undefined {
+  if (value == null) return undefined;
+  return String(value);
+}
+
+// ============================================================================
+// Declarative Pipeline Architecture
+// ============================================================================
+type Transform = 'boolean' | 'json' | 'base64' | 'number';
+
+type MappingSpec = {
+  source:
+    | { type: 'env'; key: string }
+    | { type: 'hard'; value: string };
+  transforms: Transform[];
+};
+
+// Transform registry for extensibility
+const transformRegistry: Record<string, (v: unknown) => unknown> = {
+  boolean: toBoolean,
+  json: toJson,
+  base64: fromBase64,
+  number: toNumber,
+};
+
+/**
+ * Parse a mapping string into a declarative spec
+ */
+function parseMappingSpec(input: string): MappingSpec {
+  const transforms: Transform[] = [];
+  let remaining = input;
+
+  // Check for hardcoded prefixes with type conversion
+  if (remaining.startsWith('hard_boolean:')) {
+    return {
+      source: { type: 'hard', value: remaining.slice(13) },
+      transforms: ['boolean']
+    };
+  }
+
+  if (remaining.startsWith('hard_json:')) {
+    return {
+      source: { type: 'hard', value: remaining.slice(10) },
+      transforms: ['json']
+    };
+  }
+
+  if (remaining.startsWith('hard_base64:')) {
+    return {
+      source: { type: 'hard', value: remaining.slice(12) },
+      transforms: ['base64']
+    };
+  }
+
+  // Check for generic hard: prefix
+  if (remaining.startsWith('hard:')) {
+    remaining = remaining.slice(5);
+    // Check for legacy suffixes on hardcoded values
+    if (remaining.endsWith('_JSON')) {
+      transforms.push('json');
+      remaining = remaining.slice(0, -5);
+    } else if (remaining.endsWith('_BASE64')) {
+      transforms.push('base64');
+      remaining = remaining.slice(0, -7);
+    }
+    return {
+      source: { type: 'hard', value: remaining },
+      transforms
+    };
+  }
+
+  // Check for env var prefixes
+  if (remaining.startsWith('boolean:')) {
+    transforms.push('boolean');
+    remaining = remaining.slice(8);
+  } else if (remaining.startsWith('json:')) {
+    transforms.push('json');
+    remaining = remaining.slice(5);
+  } else if (remaining.startsWith('base64:')) {
+    transforms.push('base64');
+    remaining = remaining.slice(7);
+  }
+
+  // Check for legacy suffixes on env vars
+  if (remaining.endsWith('_JSON')) {
+    transforms.push('json');
+    remaining = remaining.slice(0, -5);
+  } else if (remaining.endsWith('_BASE64')) {
+    transforms.push('base64');
+    remaining = remaining.slice(0, -7);
+  }
+
+  return {
+    source: { type: 'env', key: remaining },
+    transforms
+  };
+}
+
+/**
+ * Resolve the source value (env var or hardcoded)
+ */
+async function resolveSource(source: MappingSpec['source']): Promise<unknown> {
+  if (source.type === 'hard') {
+    return source.value;
+  }
+  // source.type === 'env'
+  // Workaround for Qenv bug where empty strings are treated as undefined
+  // Check process.env directly first to preserve empty strings
+  if (Object.prototype.hasOwnProperty.call(process.env, source.key)) {
+    return process.env[source.key];
+  }
+  // Fall back to Qenv for other sources (env.json, docker secrets, etc.)
+  return await getQenv().getEnvVarOnDemand(source.key);
+}
+
+/**
+ * Apply transformations in sequence
+ */
+function applyTransforms(value: unknown, transforms: Transform[]): unknown {
+  return transforms.reduce((acc, transform) => {
+    const fn = transformRegistry[transform];
+    return fn ? fn(acc) : acc;
+  }, value);
+}
+
+/**
+ * Process a mapping value through the complete pipeline
+ */
+async function processMappingValue(mappingString: string): Promise<unknown> {
+  const spec = parseMappingSpec(mappingString);
+  const keyName = spec.source.type === 'env' ? spec.source.key : 'hardcoded';
+
+  console.log(`    🔍 Processing mapping: "${mappingString}"`);
+  console.log(`       Source: ${spec.source.type === 'env' ? `env:${spec.source.key}` : `hard:${spec.source.value}`}`);
+  console.log(`       Transforms: ${spec.transforms.length > 0 ? spec.transforms.join(', ') : 'none'}`);
+
+  const rawValue = await resolveSource(spec.source);
+  console.log(`       Raw value: ${redactSensitiveValue(keyName, rawValue)} (type: ${typeof rawValue})`);
+
+  if (rawValue === undefined || rawValue === null) {
+    console.log(`       ⚠️ Raw value is undefined/null, returning undefined`);
+    return undefined;
+  }
+
+  const result = applyTransforms(rawValue, spec.transforms);
+  console.log(`       Final value: ${redactSensitiveValue(keyName, result)} (type: ${typeof result})`);
+  return result;
+}
+
+/**
+ * Recursively evaluate mapping values (strings or nested objects)
+ */
+async function evaluateMappingValue(mappingValue: any): Promise<any> {
+  // Handle null explicitly - it should return null, not be treated as object
+  if (mappingValue === null) {
+    console.log(`    📌 Value is null, returning null`);
+    return null;
+  }
+
+  // Handle strings (mapping specs)
+  if (typeof mappingValue === 'string') {
+    return processMappingValue(mappingValue);
+  }
+
+  // Handle objects (but not arrays or null)
+  if (mappingValue && typeof mappingValue === 'object' && !Array.isArray(mappingValue)) {
+    console.log(`    📂 Processing nested object with ${Object.keys(mappingValue).length} keys`);
+    const result: any = {};
+    for (const [key, value] of Object.entries(mappingValue)) {
+      console.log(`      → Processing nested key "${key}"`);
+      const evaluated = await evaluateMappingValue(value);
+      // Important: Don't filter out false or other falsy values!
+      // Only skip if explicitly undefined
+      if (evaluated !== undefined) {
+        result[key] = evaluated;
+        console.log(`      ✓ Nested key "${key}" = ${redactSensitiveValue(key, evaluated)} (type: ${typeof evaluated})`);
+      } else {
+        console.log(`      ⚠️ Nested key "${key}" evaluated to undefined, skipping`);
+      }
+    }
+    return result;
+  }
+
+  // For any other type (numbers, booleans, etc.), return as-is
+  // Note: We don't have key context here, so we'll just indicate the type
+  console.log(`    📎 Returning value as-is: [value] (type: ${typeof mappingValue})`);
+  return mappingValue;
+}
+
+// ============================================================================
+// AppData Interface and Class
+// ============================================================================
+export interface IAppDataOptions<T = any> {
+  dirPath?: string;
+  requiredKeys?: Array<keyof T>;
+
+  /**
+   * Whether keys should be persisted on disk or not
+   */
+  ephemeral?: boolean;
+
+  /**
+   * @deprecated Use 'ephemeral' instead
+   */
+  ephermal?: boolean;
+
+  /**
+   * kvStoreKey: 'MY_ENV_VAR'
+   */
+  envMapping?: plugins.tsclass.typeFest.PartialDeep<T>;
+  overwriteObject?: plugins.tsclass.typeFest.PartialDeep<T>;
+}
+
+export class AppData<T = any> {
+  /**
+   * creates appdata. If no pathArg is given, data will be stored here:
+   * ${PWD}/.nogit/appdata
+   * @param pathArg
+   * @returns
+   */
+  public static async createAndInit<T = any>(
+    optionsArg: IAppDataOptions<T> = {},
+  ): Promise<AppData<T>> {
+    const appData = new AppData<T>(optionsArg);
+    await appData.readyDeferred.promise;
+    return appData;
+  }
+
+  /**
+   * Static helper to get an environment variable as a boolean
+   * @param envVarName The name of the environment variable
+   * @returns boolean value (true if env var is "true", false otherwise)
+   */
+  public static async valueAsBoolean(envVarName: string): Promise<boolean> {
+    const value = await getQenv().getEnvVarOnDemand(envVarName);
+    return toBoolean(value);
+  }
+
+  /**
+   * Static helper to get an environment variable as parsed JSON
+   * @param envVarName The name of the environment variable
+   * @returns Parsed JSON object/array
+   */
+  public static async valueAsJson<R = any>(envVarName: string): Promise<R | undefined> {
+    const value = await getQenv().getEnvVarOnDemand(envVarName);
+    return toJson<R>(value);
+  }
+
+  /**
+   * Static helper to get an environment variable as base64 decoded string
+   * @param envVarName The name of the environment variable
+   * @returns Decoded string
+   */
+  public static async valueAsBase64(envVarName: string): Promise<string | undefined> {
+    const value = await getQenv().getEnvVarOnDemand(envVarName);
+    return fromBase64(value);
+  }
+
+  /**
+   * Static helper to get an environment variable as a string
+   * @param envVarName The name of the environment variable
+   * @returns String value
+   */
+  public static async valueAsString(envVarName: string): Promise<string | undefined> {
+    const value = await getQenv().getEnvVarOnDemand(envVarName);
+    return toString(value);
+  }
+
+  /**
+   * Static helper to get an environment variable as a number
+   * @param envVarName The name of the environment variable
+   * @returns Number value
+   */
+  public static async valueAsNumber(envVarName: string): Promise<number | undefined> {
+    const value = await getQenv().getEnvVarOnDemand(envVarName);
+    return toNumber(value);
+  }
+
+  // instance
+  public readyDeferred = plugins.smartpromise.defer<void>();
+  public options: IAppDataOptions<T>;
+  private kvStore: KeyValueStore<T>;
+
+  constructor(optionsArg: IAppDataOptions<T> = {}) {
+    this.options = optionsArg;
+    this.init();
+  }
+
+  /**
+   * inits app data
+   */
+  private async init() {
+    console.log('🚀 Initializing AppData...');
+
+    // Handle backward compatibility for typo
+    const isEphemeral = this.options.ephemeral ?? this.options.ephermal ?? false;
+    if (this.options.ephermal && !this.options.ephemeral) {
+      console.warn('⚠️ Option "ephermal" is deprecated, use "ephemeral" instead.');
+    }
+
+    if (this.options.dirPath) {
+      console.log(`  📁 Using custom directory: ${this.options.dirPath}`);
+    } else if (isEphemeral) {
+      console.log(`  💨 Using ephemeral storage (in-memory only)`);
+    } else {
+      const appDataDir = '/app/data';
+      const dataDir = '/data';
+      const nogitAppData = '.nogit/appdata';
+      const appDataExists = plugins.smartfile.fs.isDirectory(appDataDir);
+      const dataExists = plugins.smartfile.fs.isDirectory(dataDir);
+      if (appDataExists) {
+        this.options.dirPath = appDataDir;
+        console.log(`  📁 Auto-selected container directory: ${appDataDir}`);
+      } else if (dataExists) {
+        this.options.dirPath = dataDir;
+        console.log(`  📁 Auto-selected data directory: ${dataDir}`);
+      } else {
+        await plugins.smartfile.fs.ensureDir(nogitAppData);
+        this.options.dirPath = nogitAppData;
+        console.log(`  📁 Auto-selected local directory: ${nogitAppData}`);
+      }
+    }
+
+    this.kvStore = new KeyValueStore<T>({
+      typeArg: isEphemeral ? 'ephemeral' : 'custom',
+      identityArg: 'appkv',
+      customPath: this.options.dirPath,
+      mandatoryKeys: this.options.requiredKeys as Array<keyof T>,
+    });
+
+    if (this.options.envMapping) {
+      console.log(`📦 Processing envMapping for AppData...`);
+      const totalKeys = Object.keys(this.options.envMapping).length;
+      let processedCount = 0;
+
+      // Process each top-level key in envMapping
+      for (const key in this.options.envMapping) {
+        try {
+          const mappingSpec = this.options.envMapping[key];
+          const specType = mappingSpec === null ? 'null' :
+                          typeof mappingSpec === 'string' ? mappingSpec :
+                          typeof mappingSpec === 'object' ? 'nested object' :
+                          typeof mappingSpec;
+          console.log(`  → Processing key "${key}" with spec: ${specType}`);
+
+          const evaluated = await evaluateMappingValue(mappingSpec);
+          // Important: Don't skip false, 0, empty string, or null values!
+          // Only skip if explicitly undefined
+          if (evaluated !== undefined) {
+            await this.kvStore.writeKey(key as keyof T, evaluated);
+            processedCount++;
+            const valueType = evaluated === null ? 'null' :
+                            Array.isArray(evaluated) ? 'array' :
+                            typeof evaluated;
+            const valuePreview = evaluated === null ? 'null' :
+                               typeof evaluated === 'object' ?
+                                 (Array.isArray(evaluated) ? `[${evaluated.length} items]` : `{${Object.keys(evaluated).length} keys}`) :
+                                 redactSensitiveValue(key, evaluated);
+            console.log(`  ✅ Successfully processed key "${key}" = ${valuePreview} (type: ${valueType})`);
+          } else {
+            console.log(`  ⚠️ Key "${key}" evaluated to undefined, skipping`);
+          }
+        } catch (err) {
+          console.error(`  ❌ Failed to evaluate envMapping for key "${key}":`, err);
+        }
+      }
+
+      console.log(`📊 EnvMapping complete: ${processedCount}/${totalKeys} keys successfully processed`);
+    }
+
+    // Apply overwrite object after env mapping
+    if (this.options.overwriteObject) {
+      const overwriteKeys = Object.keys(this.options.overwriteObject);
+      console.log(`🔄 Applying overwriteObject with ${overwriteKeys.length} key(s)...`);
+
+      for (const key of overwriteKeys) {
+        const value = this.options.overwriteObject[key];
+        const valueType = Array.isArray(value) ? 'array' : typeof value;
+        console.log(`  🔧 Overwriting key "${key}" with ${valueType} value`);
+
+        await this.kvStore.writeKey(
+          key as keyof T,
+          value,
+        );
+      }
+
+      console.log(`✅ OverwriteObject complete: ${overwriteKeys.length} key(s) overwritten`);
+    }
+
+    this.readyDeferred.resolve();
+    console.log('✨ AppData initialization complete!');
+  }
+
+  /**
+   * returns a kvstore that resides in appdata
+   */
+  public async getKvStore(): Promise<KeyValueStore<T>> {
+    await this.readyDeferred.promise;
+    return this.kvStore;
+  }
+
+  public async logMissingKeys(): Promise<Array<keyof T>> {
+    const kvStore = await this.getKvStore();
+    const missingMandatoryKeys = await kvStore.getMissingMandatoryKeys();
+    if (missingMandatoryKeys.length > 0) {
+      console.log(
+        `The following mandatory keys are missing in the appdata:\n   -> ${missingMandatoryKeys.join(
+          ',\n   -> ',
+        )}`,
+      );
+    } else {
+      console.log('All mandatory keys are present in the appdata');
+    }
+    return missingMandatoryKeys;
+  }
+
+  public async waitForAndGetKey<K extends keyof T>(
+    keyArg: K,
+  ): Promise<T[K] | undefined> {
+    await this.readyDeferred.promise;
+    await this.kvStore.waitForKeysPresent([keyArg]);
+    return this.kvStore.readKey(keyArg);
+  }
+}