@smythos/sre 1.5.13 → 1.5.16

package/dist/types/subsystems/IO/Storage.service/connectors/S3Storage.class.d.ts

@@ -15,7 +15,9 @@ export declare class S3Storage extends StorageConnector {
     private client;
     private bucket;
     private isInitialized;
+    private initializationPromise;
     constructor(_settings: S3Config);
+    private ensureInitialized;
     private initialize;
     /**
      * Reads an object from the S3 bucket.

package/dist/types/subsystems/Security/Vault.service/connectors/JSONFileVault.class.d.ts

@@ -16,6 +16,13 @@ export declare class JSONFileVault extends VaultConnector {
     constructor(_settings: JSONFileVaultConfig);
     private findVaultFile;
     private getMasterKeyInteractive;
+    /**
+     * Resolves environment variable references in vault values.
+     * Supports syntax: $env(VARIABLE_NAME)
+     * @param value The value to process
+     * @returns The value with environment variables resolved
+     */
+    private resolveEnvironmentVariables;
     protected get(acRequest: AccessRequest, keyId: string): Promise<any>;
     protected exists(acRequest: AccessRequest, keyId: string): Promise<boolean>;
     protected listKeys(acRequest: AccessRequest): Promise<string[]>;

package/dist/types/utils/lazy-client.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Configuration for lazy client error handling
+ */
+export interface LazyClientConfig {
+    /** Custom package name if different from import path */
+    packageName?: string;
+    /** Custom display name for the package */
+    displayName?: string;
+    /** Additional installation notes */
+    installNotes?: string;
+    /** Documentation URL */
+    docsUrl?: string;
+    /** Disable auto-installation messages */
+    silent?: boolean;
+}
+/**
+ * Generic lazy loading utility for client libraries
+ * Preserves strong typing and requires minimal code changes
+ */
+export declare class LazyClient<T = any> {
+    private clientFactory;
+    private config;
+    private _client;
+    private _clientPromise;
+    constructor(clientFactory: () => Promise<T>, config?: LazyClientConfig);
+    /**
+     * Creates a proxy that records method calls and executes them when needed
+     */
+    private createProxy;
+    /**
+     * Executes the recorded method chain on the actual client
+     */
+    private executeMethodChain;
+    /**
+     * Ensures the client is loaded and returns it
+     */
+    private ensureClient;
+    /**
+     * Returns the proxy that behaves like the actual client
+     */
+    get client(): T;
+    /**
+     * Get the actual client instance (async)
+     */
+    getClient(): Promise<T>;
+    /**
+     * Check if the client is already loaded
+     */
+    get isLoaded(): boolean;
+}
+/**
+ * Helper function to create a lazy client with dynamic import
+ */
+export declare function createLazyClient<T>(importFn: () => Promise<any>, clientFactory: (module: any) => T, config?: LazyClientConfig): LazyClient<T>;
+/**
+ * Specific helper for common client patterns
+ */
+export declare function createLazyClientFromConstructor<T>(packageName: string, constructorName: string, ...args: any[]): LazyClient<T>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smythos/sre",
-  "version": "1.5.13",
+  "version": "1.5.16",
   "description": "Smyth Runtime Environment",
   "author": "Alaa-eddine KADDOURI",
   "license": "MIT",

package/src/helpers/S3Cache.helper.ts

@@ -1,8 +1,9 @@
 import { GetBucketLifecycleConfigurationCommandOutput, PutBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3';
 
 import { GetBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3';
-
 import { S3Client } from '@aws-sdk/client-s3';
+import { Logger } from '@sre/helpers/Log.helper';
+const console = Logger('S3Cache');
 
 export function generateLifecycleRules() {
     const rules = [];
@@ -58,7 +59,6 @@ export function generateLifecycleRules() {
     return rules;
 }
 
-
 export function generateExpiryMetadata(expiryDays) {
     let metadataValue;
 
@@ -90,6 +90,17 @@ export function ttlToExpiryDays(ttl: number) {
 }
 
 export async function checkAndInstallLifecycleRules(bucketName: string, s3Client: S3Client) {
+    // Validate inputs
+    if (!bucketName || bucketName.trim() === '') {
+        throw new Error('Bucket name is required and cannot be empty');
+    }
+
+    if (!s3Client) {
+        throw new Error('S3Client is required');
+    }
+
+    console.log(`Checking lifecycle rules for bucket: ${bucketName}`);
+
     try {
         // Check existing lifecycle configuration
         const getLifecycleCommand = new GetBucketLifecycleConfigurationCommand({ Bucket: bucketName });
@@ -105,6 +116,7 @@ export async function checkAndInstallLifecycleRules(bucketName: string, s3Client
             const putLifecycleCommand = new PutBucketLifecycleConfigurationCommand(params);
             // Put the new lifecycle configuration
             await s3Client.send(putLifecycleCommand);
+            console.log(`Added ${nonExistingNewRules.length} new lifecycle rules to bucket: ${bucketName}`);
         } else {
             console.log('Lifecycle configuration already exists');
         }
@@ -124,6 +136,12 @@ export async function checkAndInstallLifecycleRules(bucketName: string, s3Client
             console.log('Lifecycle configuration created successfully.');
         } else {
             console.error('Error checking lifecycle configuration:', error);
+            console.error('Bucket name provided:', bucketName);
+            console.error('Error details:', {
+                name: error.name,
+                message: error.message,
+                code: error.code,
+            });
         }
     }
 }

package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts

@@ -58,10 +58,18 @@ export class S3Storage extends StorageConnector {
     private client: S3Client;
     private bucket: string;
     private isInitialized: boolean = false;
+    private initializationPromise: Promise<void> | null = null;
 
     constructor(protected _settings: S3Config) {
         super(_settings);
         //if (!SmythRuntime.Instance) throw new Error('SRE not initialized');
+
+        // Validate required configuration
+        if (!_settings.bucket || _settings.bucket.trim() === '') {
+            console.warn('S3 bucket name is required and cannot be empty, connector not initialized');
+            return;
+        }
+
         this.bucket = _settings.bucket;
         const clientConfig: any = {};
         if (_settings.region) clientConfig.region = _settings.region;
@@ -73,12 +81,41 @@ export class S3Storage extends StorageConnector {
         }
 
         this.client = new S3Client(clientConfig);
-        this.initialize();
+        // Don't call initialize() synchronously in constructor
+        // It will be called when needed by methods that require initialization
     }
 
-    private async initialize() {
-        await checkAndInstallLifecycleRules(this.bucket, this.client);
-        this.isInitialized = true;
+    private async ensureInitialized(): Promise<void> {
+        if (this.isInitialized) {
+            return;
+        }
+
+        if (this.initializationPromise) {
+            return this.initializationPromise;
+        }
+
+        this.initializationPromise = this.initialize();
+        return this.initializationPromise;
+    }
+
+    private async initialize(): Promise<void> {
+        if (!this.client) {
+            console.warn('S3 client not initialized');
+            return;
+        }
+        if (this.isInitialized) {
+            return;
+        }
+
+        try {
+            await checkAndInstallLifecycleRules(this.bucket, this.client);
+            this.isInitialized = true;
+        } catch (error) {
+            console.error('Failed to initialize S3Storage:', error);
+            // Reset the initialization promise so it can be retried
+            this.initializationPromise = null;
+            throw error;
+        }
     }
 
     /**
@@ -90,6 +127,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     public async read(acRequest: AccessRequest, resourceId: string) {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
         const params = {
@@ -133,6 +172,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     async getMetadata(acRequest: AccessRequest, resourceId: string): Promise<StorageMetadata | undefined> {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
 
@@ -147,6 +188,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     async setMetadata(acRequest: AccessRequest, resourceId: string, metadata: StorageMetadata) {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
 
@@ -171,9 +214,10 @@ export class S3Storage extends StorageConnector {
      */
     @SecureConnector.AccessControl
     async write(acRequest: AccessRequest, resourceId: string, value: StorageData, acl?: IACL, metadata?: StorageMetadata): Promise<void> {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
-        if (!this.isInitialized) await this.initialize();
         const accessCandidate = acRequest.candidate;
 
         let amzACL = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
@@ -207,6 +251,8 @@ export class S3Storage extends StorageConnector {
      */
     @SecureConnector.AccessControl
     async delete(acRequest: AccessRequest, resourceId: string): Promise<void> {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
 
@@ -225,6 +271,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     async exists(acRequest: AccessRequest, resourceId: string): Promise<boolean> {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
         const command = new HeadObjectCommand({
@@ -250,6 +298,8 @@ export class S3Storage extends StorageConnector {
     //if the resource exists we read it's ACL and return it
     //if the resource does not exist we return an write access ACL for the candidate
     public async getResourceACL(resourceId: string, candidate: IAccessCandidate) {
+        await this.ensureInitialized();
+
         const s3Metadata = await this.getS3Metadata(resourceId);
         const exists = s3Metadata !== undefined; //undefined metadata means the resource does not exist
         //let acl: ACL = ACL.from(s3Metadata?.['x-amz-meta-acl'] as IACL);
@@ -263,6 +313,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     async getACL(acRequest: AccessRequest, resourceId: string): Promise<ACL | undefined> {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
 
@@ -277,6 +329,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     async setACL(acRequest: AccessRequest, resourceId: string, acl: IACL) {
+        await this.ensureInitialized();
+
         // const accessTicket = await this.getAccessTicket(resourceId, acRequest);
         // if (accessTicket.access !== TAccessResult.Granted) throw new Error('Access Denied');
 
@@ -294,6 +348,8 @@ export class S3Storage extends StorageConnector {
 
     @SecureConnector.AccessControl
     async expire(acRequest: AccessRequest, resourceId: string, ttl: number) {
+        await this.ensureInitialized();
+
         const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
         const s3PutObjectTaggingCommand = new PutObjectTaggingCommand({
             Bucket: this.bucket,

package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts

@@ -23,7 +23,7 @@ export type JSONFileVaultConfig = {
     shared?: string;
 };
 
-export class JSONFileVault extends VaultConnector { 
+export class JSONFileVault extends VaultConnector {
     public name: string = 'JSONFileVault';
     private vaultData: any;
     private index: any;
@@ -119,12 +119,39 @@ export class JSONFileVault extends VaultConnector {
         return masterKey;
     }
 
+    /**
+     * Resolves environment variable references in vault values.
+     * Supports syntax: $env(VARIABLE_NAME)
+     * @param value The value to process
+     * @returns The value with environment variables resolved
+     */
+    private resolveEnvironmentVariables(value: any): any {
+        if (typeof value !== 'string') {
+            return value;
+        }
+
+        // Match $env(VARIABLE_NAME) pattern
+        const envVarPattern = /\$env\(([^)]+)\)/g;
+
+        return value.replace(envVarPattern, (match, envVarName) => {
+            const envValue = process.env[envVarName];
+            if (envValue === undefined) {
+                console.warn(`Environment variable ${envVarName} not found, keeping original value: ${match}`);
+                return match;
+            }
+            return envValue;
+        });
+    }
+
     @SecureConnector.AccessControl
     protected async get(acRequest: AccessRequest, keyId: string) {
         const accountConnector = ConnectorService.getAccountConnector();
         const teamId = await accountConnector.getCandidateTeam(acRequest.candidate);
 
-        return this.vaultData?.[teamId]?.[keyId] || this.vaultData?.[this.shared]?.[keyId];
+        const rawValue = this.vaultData?.[teamId]?.[keyId] || this.vaultData?.[this.shared]?.[keyId];
+
+        // Resolve environment variables if the value contains $env() references
+        return this.resolveEnvironmentVariables(rawValue);
     }
 
     @SecureConnector.AccessControl

package/src/utils/lazy-client.ts

@@ -0,0 +1,261 @@
+/**
+ * Configuration for lazy client error handling
+ */
+export interface LazyClientConfig {
+    /** Custom package name if different from import path */
+    packageName?: string;
+    /** Custom display name for the package */
+    displayName?: string;
+    /** Additional installation notes */
+    installNotes?: string;
+    /** Documentation URL */
+    docsUrl?: string;
+    /** Disable auto-installation messages */
+    silent?: boolean;
+}
+
+/**
+ * Detects the package manager being used
+ */
+function detectPackageManager(): string {
+    // Check npm_config_user_agent environment variable
+    const userAgent = process.env.npm_config_user_agent;
+    if (userAgent?.includes('pnpm')) return 'pnpm';
+    if (userAgent?.includes('yarn')) return 'yarn';
+
+    // Fallback: check for lock files in cwd (if accessible)
+    try {
+        const fs = require('fs');
+        if (fs.existsSync('pnpm-lock.yaml')) return 'pnpm';
+        if (fs.existsSync('yarn.lock')) return 'yarn';
+    } catch {
+        // Ignore file system errors
+    }
+
+    return 'npm';
+}
+
+/**
+ * Extracts package name from import path
+ */
+function extractPackageName(importPath: string, config: LazyClientConfig): string {
+    if (config.packageName) {
+        return config.packageName;
+    }
+
+    // Handle scoped packages: @scope/package or @scope/package/subpath
+    if (importPath.startsWith('@')) {
+        const parts = importPath.split('/');
+        return `${parts[0]}/${parts[1]}`;
+    }
+
+    // Handle regular packages: package or package/subpath
+    return importPath.split('/')[0];
+}
+
+/**
+ * Creates a helpful error message for missing packages
+ */
+function createInstallationMessage(importPath: string, originalError: Error, config: LazyClientConfig): Error {
+    if (config.silent) {
+        return originalError;
+    }
+
+    const packageName = extractPackageName(importPath, config);
+    const displayName = config.displayName || packageName;
+    const packageManager = detectPackageManager();
+
+    let message = `\n🔌 ${displayName} is required but not installed\n\n`;
+
+    message += `Quick install:\n`;
+    message += `  ${packageManager} add ${packageName}\n\n`;
+
+    // Add alternative package managers
+    if (packageManager !== 'npm') {
+        message += `Or with npm:\n  npm install ${packageName}\n\n`;
+    }
+    if (packageManager !== 'pnpm') {
+        message += `Or with pnpm:\n  pnpm add ${packageName}\n\n`;
+    }
+
+    if (config.installNotes) {
+        message += `📝 Note: ${config.installNotes}\n\n`;
+    }
+
+    if (config.docsUrl) {
+        message += `📚 Documentation: ${config.docsUrl}\n\n`;
+    }
+
+    message += `Original error: ${originalError.message}`;
+
+    const enhancedError = new Error(message);
+    enhancedError.name = 'LazyClientImportError';
+    enhancedError.stack = originalError.stack;
+    return enhancedError;
+}
+
+/**
+ * Generic lazy loading utility for client libraries
+ * Preserves strong typing and requires minimal code changes
+ */
+export class LazyClient<T = any> {
+    private _client: T | null = null;
+    private _clientPromise: Promise<T> | null = null;
+
+    constructor(private clientFactory: () => Promise<T>, private config: LazyClientConfig = {}) {}
+
+    /**
+     * Creates a proxy that records method calls and executes them when needed
+     */
+    private createProxy(methodPath: Array<{ method: string; args: any[] }> = []): any {
+        const self = this;
+
+        // Create a proxy that can be both called and have properties accessed
+        return new Proxy(() => {}, {
+            get(target, prop, receiver) {
+                if (prop === 'then') {
+                    // This is being awaited - execute the method chain and return a thenable
+                    const promise = self.executeMethodChain(methodPath);
+                    return promise.then.bind(promise);
+                }
+
+                if (prop === 'catch') {
+                    // Handle .catch() calls
+                    const promise = self.executeMethodChain(methodPath);
+                    return promise.catch.bind(promise);
+                }
+
+                if (prop === 'finally') {
+                    // Handle .finally() calls
+                    const promise = self.executeMethodChain(methodPath);
+                    return promise.finally.bind(promise);
+                }
+
+                if (prop === Symbol.toStringTag) {
+                    return 'LazyClient';
+                }
+
+                if (typeof prop !== 'string') {
+                    return undefined;
+                }
+
+                // Return a function that extends the method chain
+                return (...args: any[]) => {
+                    const newPath = [...methodPath, { method: prop, args }];
+                    return self.createProxy(newPath);
+                };
+            },
+
+            apply(target, thisArg, argumentsList) {
+                // If called as a function, extend the method chain
+                return self.createProxy(methodPath);
+            },
+
+            has(target, prop) {
+                return true;
+            },
+        });
+    }
+
+    /**
+     * Executes the recorded method chain on the actual client
+     */
+    private async executeMethodChain(methodPath: Array<{ method: string; args: any[] }>): Promise<any> {
+        const client = await this.ensureClient();
+
+        let current: any = client;
+
+        // Execute each method call in sequence
+        for (const { method, args } of methodPath) {
+            if (current && typeof current[method] === 'function') {
+                current = current[method].apply(current, args);
+            } else if (current && current[method] !== undefined) {
+                current = current[method];
+            } else {
+                throw new Error(`Method or property '${method}' not found`);
+            }
+        }
+
+        return current;
+    }
+
+    /**
+     * Ensures the client is loaded and returns it
+     */
+    private async ensureClient(): Promise<T> {
+        if (this._client) {
+            return this._client;
+        }
+
+        if (!this._clientPromise) {
+            this._clientPromise = this.clientFactory().catch((error) => {
+                // Reset promise so next attempt will retry
+                this._clientPromise = null;
+                throw error;
+            });
+        }
+
+        this._client = await this._clientPromise;
+        return this._client;
+    }
+
+    /**
+     * Returns the proxy that behaves like the actual client
+     */
+    get client(): T {
+        return this.createProxy() as T;
+    }
+
+    /**
+     * Get the actual client instance (async)
+     */
+    async getClient(): Promise<T> {
+        return this.ensureClient();
+    }
+
+    /**
+     * Check if the client is already loaded
+     */
+    get isLoaded(): boolean {
+        return this._client !== null;
+    }
+}
+
+/**
+ * Helper function to create a lazy client with dynamic import
+ */
+export function createLazyClient<T>(importFn: () => Promise<any>, clientFactory: (module: any) => T, config: LazyClientConfig = {}): LazyClient<T> {
+    return new LazyClient<T>(async () => {
+        try {
+            const module = await importFn();
+            return clientFactory(module);
+        } catch (error) {
+            // Try to extract import path from the error or use the configured package name
+            const importPath = config.packageName || 'unknown-package';
+            throw createInstallationMessage(importPath, error as Error, config);
+        }
+    }, config);
+}
+
+/**
+ * Specific helper for common client patterns
+ */
+// export function createLazyClientFromConstructor<T>(
+//     importFn: () => Promise<any>,
+//     constructorName: string,
+//     config: LazyClientConfig = {},
+//     ...args: any[]
+// ): LazyClient<T> {
+//     return createLazyClient<T>(importFn, (module) => new module[constructorName](...args), config);
+// }
+export function createLazyClientFromConstructor<T>(packageName: string, constructorName: string, ...args: any[]): LazyClient<T> {
+    const importFn = () => import(packageName);
+    const config = {
+        packageName,
+        displayName: constructorName,
+        //installNotes: `Install ${packageName} with your package manager`,
+        //docsUrl: `https://www.npmjs.com/package/${packageName}`,
+    };
+    console.log('LazyClient', packageName, constructorName);
+    return createLazyClient<T>(importFn, (module) => new module[constructorName](...args), config);
+}