@soulcraft/brainy 0.54.6 → 0.55.0

package/dist/cortex/config.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Cortex Configuration Management
+ *
+ * Handles encrypted configuration storage, environment loading,
+ * and distributed coordination through Brainy.
+ */
+import { BrainyData } from '../brainyData.js';
+import { StorageConfig } from '../coreTypes.js';
+export interface CortexConfigData {
+    version: number;
+    storage: StorageConfig;
+    encryption?: {
+        enabled: boolean;
+        keyDerivation: 'pbkdf2' | 'scrypt';
+        iterations?: number;
+    };
+    coordination?: {
+        enabled: boolean;
+        realtime: boolean;
+        pollInterval: number;
+    };
+    environments?: {
+        current: string;
+        available: string[];
+    };
+}
+export interface EncryptedValue {
+    encrypted: true;
+    algorithm: string;
+    iv: string;
+    authTag: string;
+    data: string;
+}
+export declare class CortexConfig {
+    private static instance;
+    private brainy?;
+    private config?;
+    private configPath;
+    private masterKey?;
+    private constructor();
+    static getInstance(): CortexConfig;
+    /**
+     * Initialize Cortex configuration
+     */
+    init(options?: Partial<CortexConfigData>): Promise<void>;
+    /**
+     * Load existing configuration
+     */
+    load(): Promise<void>;
+    /**
+     * Save configuration to disk
+     */
+    private saveConfig;
+    /**
+     * Initialize Brainy instance
+     */
+    private initBrainy;
+    /**
+     * Initialize or load master encryption key
+     */
+    private initMasterKey;
+    /**
+     * Get Brainy instance
+     */
+    getBrainy(): BrainyData;
+    /**
+     * Encrypt a value
+     */
+    encrypt(value: string): EncryptedValue;
+    /**
+     * Decrypt a value
+     */
+    decrypt(encryptedValue: EncryptedValue): string;
+    /**
+     * Set a configuration value in Brainy
+     */
+    set(key: string, value: any, options?: {
+        encrypt?: boolean;
+    }): Promise<void>;
+    /**
+     * Get a configuration value from Brainy
+     */
+    get(key: string): Promise<any>;
+    /**
+     * List all configuration keys
+     */
+    list(): Promise<Array<{
+        key: string;
+        encrypted: boolean;
+        environment: string;
+    }>>;
+    /**
+     * Load all configurations as environment variables
+     */
+    loadEnvironment(): Promise<Record<string, string>>;
+    /**
+     * Import configuration from .env file
+     */
+    importEnv(filePath: string): Promise<void>;
+    /**
+     * Get storage configuration
+     */
+    getStorageConfig(): StorageConfig | undefined;
+    /**
+     * Get current environment
+     */
+    getCurrentEnvironment(): string;
+    /**
+     * Switch environment
+     */
+    switchEnvironment(environment: string): Promise<void>;
+}

package/dist/cortex/config.js

@@ -0,0 +1,312 @@
+/**
+ * Cortex Configuration Management
+ *
+ * Handles encrypted configuration storage, environment loading,
+ * and distributed coordination through Brainy.
+ */
+import { BrainyData } from '../brainyData.js';
+import * as crypto from 'crypto';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+export class CortexConfig {
+    constructor() {
+        this.configPath = path.join(process.cwd(), '.brainy', 'cortex.json');
+    }
+    static getInstance() {
+        if (!CortexConfig.instance) {
+            CortexConfig.instance = new CortexConfig();
+        }
+        return CortexConfig.instance;
+    }
+    /**
+     * Initialize Cortex configuration
+     */
+    async init(options = {}) {
+        // Create .brainy directory if it doesn't exist
+        const brainyDir = path.dirname(this.configPath);
+        await fs.mkdir(brainyDir, { recursive: true });
+        // Default configuration
+        const defaultConfig = {
+            version: 1,
+            storage: options.storage || { type: 'memory' },
+            encryption: {
+                enabled: true,
+                keyDerivation: 'pbkdf2',
+                iterations: 100000
+            },
+            coordination: {
+                enabled: true,
+                realtime: false,
+                pollInterval: 30000 // 30 seconds
+            },
+            environments: {
+                current: process.env.NODE_ENV || 'development',
+                available: ['development', 'staging', 'production']
+            }
+        };
+        this.config = { ...defaultConfig, ...options };
+        // Save configuration
+        await this.saveConfig();
+        // Initialize Brainy with the configuration
+        await this.initBrainy();
+        // Generate or load master key
+        await this.initMasterKey();
+    }
+    /**
+     * Load existing configuration
+     */
+    async load() {
+        try {
+            const configData = await fs.readFile(this.configPath, 'utf-8');
+            this.config = JSON.parse(configData);
+            await this.initBrainy();
+            await this.initMasterKey();
+        }
+        catch (error) {
+            throw new Error(`No Cortex configuration found. Run 'cortex init' first.`);
+        }
+    }
+    /**
+     * Save configuration to disk
+     */
+    async saveConfig() {
+        await fs.writeFile(this.configPath, JSON.stringify(this.config, null, 2), 'utf-8');
+    }
+    /**
+     * Initialize Brainy instance
+     */
+    async initBrainy() {
+        if (!this.config) {
+            throw new Error('Configuration not loaded');
+        }
+        this.brainy = new BrainyData({
+            storage: this.config.storage,
+            writeOnlyMode: false,
+            enableMetadataIndexing: true
+        });
+        await this.brainy.init();
+    }
+    /**
+     * Initialize or load master encryption key
+     */
+    async initMasterKey() {
+        const keyPath = path.join(process.cwd(), '.brainy', 'cortex.key');
+        // Try to load from environment first
+        if (process.env.CORTEX_MASTER_KEY) {
+            this.masterKey = Buffer.from(process.env.CORTEX_MASTER_KEY, 'base64');
+            return;
+        }
+        // Try to load from file
+        try {
+            const keyData = await fs.readFile(keyPath, 'utf-8');
+            this.masterKey = Buffer.from(keyData, 'base64');
+        }
+        catch (error) {
+            // Generate new key
+            this.masterKey = crypto.randomBytes(32);
+            await fs.writeFile(keyPath, this.masterKey.toString('base64'), 'utf-8');
+            // Set restrictive permissions (Unix-like systems)
+            try {
+                await fs.chmod(keyPath, 0o600);
+            }
+            catch (e) {
+                // Windows doesn't support chmod, ignore
+            }
+            console.log('🔐 Generated new master key at .brainy/cortex.key');
+            console.log('⚠️  Keep this key safe! You\'ll need it to decrypt your configs.');
+        }
+    }
+    /**
+     * Get Brainy instance
+     */
+    getBrainy() {
+        if (!this.brainy) {
+            throw new Error('Brainy not initialized. Run load() first.');
+        }
+        return this.brainy;
+    }
+    /**
+     * Encrypt a value
+     */
+    encrypt(value) {
+        if (!this.masterKey) {
+            throw new Error('Master key not initialized');
+        }
+        const algorithm = 'aes-256-gcm';
+        const iv = crypto.randomBytes(16);
+        const cipher = crypto.createCipheriv(algorithm, this.masterKey, iv);
+        let encrypted = cipher.update(value, 'utf8', 'hex');
+        encrypted += cipher.final('hex');
+        const authTag = cipher.getAuthTag();
+        return {
+            encrypted: true,
+            algorithm,
+            iv: iv.toString('hex'),
+            authTag: authTag.toString('hex'),
+            data: encrypted
+        };
+    }
+    /**
+     * Decrypt a value
+     */
+    decrypt(encryptedValue) {
+        if (!this.masterKey) {
+            throw new Error('Master key not initialized');
+        }
+        const decipher = crypto.createDecipheriv(encryptedValue.algorithm, this.masterKey, Buffer.from(encryptedValue.iv, 'hex'));
+        decipher.setAuthTag(Buffer.from(encryptedValue.authTag, 'hex'));
+        let decrypted = decipher.update(encryptedValue.data, 'hex', 'utf8');
+        decrypted += decipher.final('utf8');
+        return decrypted;
+    }
+    /**
+     * Set a configuration value in Brainy
+     */
+    async set(key, value, options = {}) {
+        if (!this.brainy) {
+            await this.load();
+        }
+        const configKey = `_cortex/config/${key}`;
+        const configValue = options.encrypt && typeof value === 'string'
+            ? this.encrypt(value)
+            : value;
+        await this.brainy.addNoun({
+            id: configKey,
+            type: 'cortex_config',
+            metadata: {
+                key,
+                value: configValue,
+                encrypted: options.encrypt || false,
+                environment: this.config?.environments?.current,
+                updatedAt: new Date().toISOString()
+            }
+        });
+    }
+    /**
+     * Get a configuration value from Brainy
+     */
+    async get(key) {
+        if (!this.brainy) {
+            await this.load();
+        }
+        const configKey = `_cortex/config/${key}`;
+        try {
+            const noun = await this.brainy.getNoun(configKey);
+            if (!noun?.metadata?.value) {
+                return undefined;
+            }
+            const value = noun.metadata.value;
+            // Decrypt if needed
+            if (value.encrypted === true) {
+                return this.decrypt(value);
+            }
+            return value;
+        }
+        catch (error) {
+            return undefined;
+        }
+    }
+    /**
+     * List all configuration keys
+     */
+    async list() {
+        if (!this.brainy) {
+            await this.load();
+        }
+        const result = await this.brainy.getNouns({
+            filter: { type: 'cortex_config' },
+            pagination: { limit: 1000 }
+        });
+        return result.items.map(noun => ({
+            key: noun.metadata?.key || noun.id.replace('_cortex/config/', ''),
+            encrypted: noun.metadata?.encrypted || false,
+            environment: noun.metadata?.environment || 'default'
+        }));
+    }
+    /**
+     * Load all configurations as environment variables
+     */
+    async loadEnvironment() {
+        if (!this.brainy) {
+            await this.load();
+        }
+        const configs = await this.brainy.getNouns({
+            filter: {
+                type: 'cortex_config',
+                'metadata.environment': this.config?.environments?.current
+            },
+            pagination: { limit: 1000 }
+        });
+        const env = {};
+        for (const config of configs.items) {
+            const key = config.metadata?.key || config.id.replace('_cortex/config/', '');
+            let value = config.metadata?.value;
+            // Decrypt if needed
+            if (value?.encrypted === true) {
+                value = this.decrypt(value);
+            }
+            // Convert to string if needed
+            if (typeof value !== 'string') {
+                value = JSON.stringify(value);
+            }
+            // Use the key as-is (could be nested like 'database.url')
+            // Or convert to UPPER_SNAKE_CASE
+            const envKey = key.toUpperCase().replace(/\./g, '_').replace(/-/g, '_');
+            env[envKey] = value;
+        }
+        return env;
+    }
+    /**
+     * Import configuration from .env file
+     */
+    async importEnv(filePath) {
+        const content = await fs.readFile(filePath, 'utf-8');
+        const lines = content.split('\n');
+        for (const line of lines) {
+            // Skip comments and empty lines
+            if (!line.trim() || line.startsWith('#')) {
+                continue;
+            }
+            const [key, ...valueParts] = line.split('=');
+            const value = valueParts.join('=').trim();
+            if (key && value) {
+                // Detect if it looks like a secret
+                const isSecret = key.includes('KEY') ||
+                    key.includes('SECRET') ||
+                    key.includes('PASSWORD') ||
+                    key.includes('TOKEN');
+                await this.set(key.trim(), value, { encrypt: isSecret });
+            }
+        }
+    }
+    /**
+     * Get storage configuration
+     */
+    getStorageConfig() {
+        return this.config?.storage;
+    }
+    /**
+     * Get current environment
+     */
+    getCurrentEnvironment() {
+        return this.config?.environments?.current || 'development';
+    }
+    /**
+     * Switch environment
+     */
+    async switchEnvironment(environment) {
+        if (!this.config) {
+            await this.load();
+        }
+        if (!this.config.environments?.available.includes(environment)) {
+            throw new Error(`Unknown environment: ${environment}`);
+        }
+        this.config.environments.current = environment;
+        await this.saveConfig();
+    }
+}
+//# sourceMappingURL=config.js.map

package/dist/cortex/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/cortex/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAE7C,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAA;AAChC,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAE9B,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;AA6BrC,MAAM,OAAO,YAAY;IAOvB;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAA;IACtE,CAAC;IAED,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;YAC3B,YAAY,CAAC,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAA;QAC5C,CAAC;QACD,OAAO,YAAY,CAAC,QAAQ,CAAA;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,UAAqC,EAAE;QAChD,+CAA+C;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAC/C,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAE9C,wBAAwB;QACxB,MAAM,aAAa,GAAqB;YACtC,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9C,UAAU,EAAE;gBACV,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,QAAQ;gBACvB,UAAU,EAAE,MAAM;aACnB;YACD,YAAY,EAAE;gBACZ,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,KAAK;gBACf,YAAY,EAAE,KAAK,CAAC,aAAa;aAClC;YACD,YAAY,EAAE;gBACZ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,aAAa;gBAC9C,SAAS,EAAE,CAAC,aAAa,EAAE,SAAS,EAAE,YAAY,CAAC;aACpD;SACF,CAAA;QAED,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,EAAE,CAAA;QAE9C,qBAAqB;QACrB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QAEvB,2CAA2C;QAC3C,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QAEvB,8BAA8B;QAC9B,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;YAC9D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;YACpC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;YACvB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAA;QAC5E,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EACpC,OAAO,CACR,CAAA;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC7C,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC;YAC3B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,aAAa,EAAE,KAAK;YACpB,sBAAsB,EAAE,IAAI;SAC7B,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,CAAA;QAEjE,qCAAqC;QACrC,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YAClC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAA;YACrE,OAAM;QACR,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YACnD,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mBAAmB;YACnB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;YACvC,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAA;YAEvE,kDAAkD;YAClD,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,wCAAwC;YAC1C,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAA;YAChE,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAA;QACjF,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;QAC9D,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAA;IACpB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,KAAa;QACnB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QAED,MAAM,SAAS,GAAG,aAAa,CAAA;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;QAEnE,IAAI,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;QACnD,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAEhC,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAA;QAEnC,OAAO;YACL,SAAS,EAAE,IAAI;YACf,SAAS;YACT,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;YACtB,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YAChC,IAAI,EAAE,SAAS;SAChB,CAAA;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,cAA8B;QACpC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,CACtC,cAAc,CAAC,SAAS,EACxB,IAAI,CAAC,SAAS,EACd,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CACtC,CAAA;QAED,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAA;QAE/D,IAAI,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QACnE,SAAS,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAEnC,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAU,EAAE,UAAiC,EAAE;QACpE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,MAAM,SAAS,GAAG,kBAAkB,GAAG,EAAE,CAAA;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,KAAK,KAAK,QAAQ;YAC9D,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;YACrB,CAAC,CAAC,KAAK,CAAA;QAET,MAAM,IAAI,CAAC,MAAO,CAAC,OAAO,CAAC;YACzB,EAAE,EAAE,SAAS;YACb,IAAI,EAAE,eAAe;YACrB,QAAQ,EAAE;gBACR,GAAG;gBACH,KAAK,EAAE,WAAW;gBAClB,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;gBACnC,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO;gBAC/C,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;SACF,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,MAAM,SAAS,GAAG,kBAAkB,GAAG,EAAE,CAAA;QAEzC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAClD,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;gBAC3B,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAA;YAEjC,oBAAoB;YACpB,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,KAAuB,CAAC,CAAA;YAC9C,CAAC;YAED,OAAO,KAAK,CAAA;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,SAAS,CAAA;QAClB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,QAAQ,CAAC;YACzC,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE;YACjC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC5B,CAAC,CAAA;QAEF,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;YACjE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,KAAK;YAC5C,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,IAAI,SAAS;SACrD,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,QAAQ,CAAC;YAC1C,MAAM,EAAE;gBACN,IAAI,EAAE,eAAe;gBACrB,sBAAsB,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO;aAC3D;YACD,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SAC5B,CAAC,CAAA;QAEF,MAAM,GAAG,GAA2B,EAAE,CAAA;QAEtC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAA;YAC5E,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAA;YAElC,oBAAoB;YACpB,IAAI,KAAK,EAAE,SAAS,KAAK,IAAI,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAuB,CAAC,CAAA;YAC/C,CAAC;YAED,8BAA8B;YAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAC/B,CAAC;YAED,0DAA0D;YAC1D,iCAAiC;YACjC,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;YACvE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAA;QACrB,CAAC;QAED,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,QAAgB;QAC9B,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,gCAAgC;YAChC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,SAAQ;YACV,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YAEzC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACjB,mCAAmC;gBACnC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;oBACpB,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACtB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;oBACxB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;gBAErC,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC1D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,CAAA;IAC7B,CAAC;IAED;;OAEG;IACH,qBAAqB;QACnB,OAAO,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,IAAI,aAAa,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;QACnB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAO,CAAC,YAAY,EAAE,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CAAC,wBAAwB,WAAW,EAAE,CAAC,CAAA;QACxD,CAAC;QAED,IAAI,CAAC,MAAO,CAAC,YAAa,CAAC,OAAO,GAAG,WAAW,CAAA;QAChD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;IACzB,CAAC;CACF"}

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -57,6 +57,11 @@ export declare class S3CompatibleStorage extends BaseStorage {
     private systemPrefix;
     private useDualWrite;
     protected statisticsCache: StatisticsData | null;
+    private throttlingDetected;
+    private throttlingBackoffMs;
+    private maxBackoffMs;
+    private consecutiveThrottleErrors;
+    private lastThrottleTime;
     private lockPrefix;
     private activeLocks;
     private changeLogPrefix;
@@ -106,6 +111,18 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Initialize the storage adapter
      */
     init(): Promise<void>;
+    /**
+     * Detect and handle storage throttling (429, 503, timeouts)
+     */
+    private isThrottlingError;
+    /**
+     * Handle throttling by implementing exponential backoff
+     */
+    private handleThrottling;
+    /**
+     * Smart delay based on current throttling status
+     */
+    private smartDelay;
     /**
      * Auto-cleanup legacy /index folder during initialization
      * This removes old index data that has been migrated to _system

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -48,6 +48,12 @@ export class S3CompatibleStorage extends BaseStorage {
         this.useDualWrite = true; // Write to both locations during migration
         // Statistics caching for better performance
         this.statisticsCache = null;
+        // Throttling detection and adaptation
+        this.throttlingDetected = false;
+        this.throttlingBackoffMs = 1000; // Start with 1 second
+        this.maxBackoffMs = 30000; // Max 30 seconds
+        this.consecutiveThrottleErrors = 0;
+        this.lastThrottleTime = 0;
         // Distributed locking for concurrent access control
         this.lockPrefix = 'locks/';
         this.activeLocks = new Set();
@@ -240,6 +246,60 @@ export class S3CompatibleStorage extends BaseStorage {
             throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
         }
     }
+    /**
+     * Detect and handle storage throttling (429, 503, timeouts)
+     */
+    isThrottlingError(error) {
+        const statusCode = error.$metadata?.httpStatusCode || error.statusCode;
+        const message = error.message?.toLowerCase() || '';
+        return (statusCode === 429 || // Too Many Requests
+            statusCode === 503 || // Service Unavailable / Slow Down
+            message.includes('throttl') ||
+            message.includes('slow down') ||
+            message.includes('rate limit') ||
+            message.includes('too many requests'));
+    }
+    /**
+     * Handle throttling by implementing exponential backoff
+     */
+    async handleThrottling(error) {
+        if (this.isThrottlingError(error)) {
+            this.throttlingDetected = true;
+            this.consecutiveThrottleErrors++;
+            this.lastThrottleTime = Date.now();
+            // Exponential backoff: 1s, 2s, 4s, 8s, etc. up to maxBackoffMs
+            this.throttlingBackoffMs = Math.min(this.throttlingBackoffMs * 2, this.maxBackoffMs);
+            prodLog.warn(`🐌 Storage throttling detected (${error.$metadata?.httpStatusCode || 'timeout'}). Backing off for ${this.throttlingBackoffMs}ms`);
+            await new Promise(resolve => setTimeout(resolve, this.throttlingBackoffMs));
+        }
+        else {
+            // Reset throttling detection on successful non-throttling operation
+            if (this.consecutiveThrottleErrors > 0) {
+                this.consecutiveThrottleErrors = 0;
+                this.throttlingBackoffMs = 1000; // Reset to initial backoff
+                if (this.throttlingDetected) {
+                    prodLog.info('✅ Storage throttling cleared');
+                    this.throttlingDetected = false;
+                }
+            }
+        }
+    }
+    /**
+     * Smart delay based on current throttling status
+     */
+    async smartDelay() {
+        if (this.throttlingDetected) {
+            // If currently throttled, add a preventive delay
+            const timeSinceThrottle = Date.now() - this.lastThrottleTime;
+            if (timeSinceThrottle < 60000) { // Within 1 minute of throttling
+                await new Promise(resolve => setTimeout(resolve, Math.min(this.throttlingBackoffMs / 2, 5000)));
+            }
+        }
+        else {
+            // Normal yield
+            await new Promise(resolve => setImmediate(resolve));
+        }
+    }
     /**
      * Auto-cleanup legacy /index folder during initialization
      * This removes old index data that has been migrated to _system
@@ -1493,9 +1553,13 @@ export class S3CompatibleStorage extends BaseStorage {
                     return { id, metadata };
                 }
                 catch (error) {
-                    // Enhanced error handling for minor socket issues
+                    // Handle throttling and enhanced error handling
+                    await this.handleThrottling(error);
                     const errorMessage = error instanceof Error ? error.message : String(error);
-                    if (errorMessage.includes('timeout') || errorMessage.includes('ECONNRESET')) {
+                    if (this.isThrottlingError(error)) {
+                        // Throttling errors are already logged in handleThrottling
+                    }
+                    else if (errorMessage.includes('timeout') || errorMessage.includes('ECONNRESET')) {
                         this.logger.debug(`⏰ Metadata timeout for ${id} (normal during initial indexing):`, errorMessage);
                     }
                     else {
@@ -1515,21 +1579,23 @@ export class S3CompatibleStorage extends BaseStorage {
                     errorCount++;
                 }
             }
-            // Adaptive delay based on error rates (helps with S3 rate limiting)
+            // Smart delay based on error rates and throttling status
             const errorRate = errorCount / batch.length;
             if (errorRate > 0.5) {
-                // High error rate - add extra delay
-                await new Promise(resolve => setTimeout(resolve, 2000)); // 2 second delay
-                prodLog.debug(`🐌 High error rate (${(errorRate * 100).toFixed(1)}%) - adding extra delay`);
+                // High error rate - use smart delay with throttling awareness
+                await this.smartDelay();
+                await new Promise(resolve => setTimeout(resolve, 2000)); // Extra delay for high error rates
+                prodLog.debug(`🐌 High error rate (${(errorRate * 100).toFixed(1)}%) - adding smart delay`);
             }
             else if (errorRate > 0.2) {
-                // Moderate error rate - add modest delay  
-                await new Promise(resolve => setTimeout(resolve, 500)); // 500ms delay
-                prodLog.debug(`⚡ Moderate error rate (${(errorRate * 100).toFixed(1)}%) - adding modest delay`);
+                // Moderate error rate - smart delay
+                await this.smartDelay();
+                await new Promise(resolve => setTimeout(resolve, 500)); // Modest extra delay
+                prodLog.debug(`⚡ Moderate error rate (${(errorRate * 100).toFixed(1)}%) - adding smart delay`);
             }
             else {
-                // Low error rate - normal yield
-                await new Promise(resolve => setImmediate(resolve));
+                // Low error rate - just smart delay (respects throttling status)
+                await this.smartDelay();
             }
         }
         return results;