bunsane 0.2.3 → 0.2.7

package/core/cache/CacheWarmer.ts

@@ -1,5 +1,7 @@
 import { CacheManager } from './CacheManager.js';
 import { SchedulerManager } from '../SchedulerManager.js';
+import { Entity } from '../Entity.js';
+import { logger } from '../Logger.js';
 
 /**
  * CacheWarmer preloads frequently accessed data into the cache to improve
@@ -34,7 +36,7 @@ export class CacheWarmer {
     let warmed = 0;
     let failed = 0;
 
-    console.log(`Starting entity cache warming for ${entityIds.length} ${entityType} entities`);
+    logger.info({ msg: `Starting entity cache warming`, count: entityIds.length, entityType });
 
     // Process entities in batches to avoid overwhelming the database
     const batchSize = 10;
@@ -46,7 +48,7 @@ export class CacheWarmer {
         const entities = await this.loadEntitiesBatch(batch, entityType);
         warmed += entities.length;
       } catch (error) {
-        console.warn(`Failed to warm batch of entities:`, error);
+        logger.warn({ msg: 'Failed to warm batch of entities', error });
         failed += batch.length;
       }
 
@@ -55,7 +57,7 @@ export class CacheWarmer {
     }
 
     const duration = Date.now() - startTime;
-    console.log(`Entity cache warming completed: ${warmed} warmed, ${failed} failed in ${duration}ms`);
+    logger.info({ msg: 'Entity cache warming completed', warmed, failed, duration });
 
     return { success: failed === 0, warmed, failed, duration };
   }
@@ -71,7 +73,7 @@ export class CacheWarmer {
     enabled?: boolean;
   }): void {
     if (!config.enabled) {
-      console.log(`Cache warming job "${config.name}" is disabled`);
+      logger.debug({ msg: 'Cache warming job disabled', name: config.name });
       return;
     }
 
@@ -80,18 +82,18 @@ export class CacheWarmer {
 
     const job = this.scheduler.scheduleJob(config.name, config.cronExpression, async () => {
       try {
-        console.log(`Running scheduled cache warming: ${config.name}`);
+        logger.info({ msg: 'Running scheduled cache warming', name: config.name });
 
         if (config.type === 'entity') {
           await this.warmEntityCache(config.config.entityIds, config.config.entityType);
         }
       } catch (error) {
-        console.error(`Scheduled cache warming failed for "${config.name}":`, error);
+        logger.error({ msg: 'Scheduled cache warming failed', name: config.name, error });
       }
     });
 
     this.warmingJobs.set(config.name, job);
-    console.log(`Scheduled cache warming job "${config.name}" with cron: ${config.cronExpression}`);
+    logger.info({ msg: 'Scheduled cache warming job', name: config.name, cron: config.cronExpression });
   }
 
   /**
@@ -102,7 +104,7 @@ export class CacheWarmer {
     if (job) {
       job.cancel();
       this.warmingJobs.delete(name);
-      console.log(`Cancelled cache warming job: ${name}`);
+      logger.info({ msg: 'Cancelled cache warming job', name });
       return true;
     }
     return false;
@@ -126,11 +128,17 @@ export class CacheWarmer {
   }> {
     const startTime = Date.now();
 
-    // Warm entities
-    const firstEntity = config.entities?.[0];
-    const entityResults = firstEntity
-      ? await this.warmEntityCache(firstEntity.entityIds, firstEntity.entityType)
-      : { success: true, warmed: 0, failed: 0, duration: 0 };
+    // Warm all entity groups
+    let entityResults = { success: true, warmed: 0, failed: 0, duration: 0 };
+    if (config.entities) {
+      for (const entry of config.entities) {
+        const result = await this.warmEntityCache(entry.entityIds, entry.entityType);
+        entityResults.warmed += result.warmed;
+        entityResults.failed += result.failed;
+        entityResults.duration += result.duration;
+        if (!result.success) entityResults.success = false;
+      }
+    }
 
     const totalDuration = Date.now() - startTime;
 
@@ -141,17 +149,31 @@ export class CacheWarmer {
   }
 
   /**
-   * Loads a batch of entities (placeholder - would need actual entity loading logic)
+   * Loads a batch of entities from the database and populates the cache.
+   * Uses Entity.FindById to load each entity with all its components,
+   * then writes the entity and its components into cache via CacheManager.
    */
-  private async loadEntitiesBatch(entityIds: string[], entityType: string): Promise<any[]> {
-    // This is a placeholder - in a real implementation, this would load entities
-    // from the database using the appropriate entity manager or query system
-    console.log(`Loading batch of ${entityIds.length} ${entityType} entities: ${entityIds.slice(0, 3).join(', ')}...`);
-
-    // Simulate loading delay
-    await new Promise(resolve => setTimeout(resolve, 10));
+  private async loadEntitiesBatch(entityIds: string[], entityType: string): Promise<Entity[]> {
+    const loaded: Entity[] = [];
+
+    const results = await Promise.allSettled(
+      entityIds.map(id => Entity.FindById(id))
+    );
+
+    for (const result of results) {
+      if (result.status === 'fulfilled' && result.value) {
+        const entity = result.value;
+        loaded.push(entity);
+
+        await this.cacheManager.setEntityWriteThrough(entity);
+        const components = entity.componentList();
+        if (components.length > 0) {
+          await this.cacheManager.setComponentWriteThrough(entity.id, components);
+        }
+      }
+    }
 
-    // Return mock entities - in real implementation, this would be actual entity data
-    return entityIds.map(id => ({ id, type: entityType, loaded: true }));
+    logger.debug({ msg: 'Loaded entity batch', entityType, requested: entityIds.length, loaded: loaded.length });
+    return loaded;
   }
 }

package/core/cache/MemoryCache.ts

@@ -1,6 +1,14 @@
 import { type CacheProvider, type CacheStats } from './CacheProvider';
 import { logger } from '../Logger';
 
+function formatBytes(bytes: number): string {
+    if (bytes === 0) return '0 B';
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(1024));
+    const value = bytes / Math.pow(1024, i);
+    return `${value.toFixed(2)} ${units[i]}`;
+}
+
 interface CacheEntry<T> {
     value: T;
     expiresAt?: number;
@@ -158,7 +166,8 @@ export class MemoryCache implements CacheProvider {
             misses: this.stats.misses,
             hitRate,
             size: this.stats.size,
-            memoryUsage: this.stats.memoryUsage
+            memoryUsage: this.stats.memoryUsage,
+            memoryUsageHuman: formatBytes(this.stats.memoryUsage)
         };
     }
 

package/core/cache/RedisCache.ts

@@ -13,10 +13,22 @@ export interface HealthStatus {
     connected: boolean;
     latency: number;
     memoryUsage?: number;
+    memoryUsageHuman?: string;
     connections?: number;
     version?: string;
 }
 
+/**
+ * Format bytes into human-readable string
+ */
+function formatBytes(bytes: number): string {
+    if (bytes === 0) return '0 B';
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(1024));
+    const value = bytes / Math.pow(1024, i);
+    return `${value.toFixed(2)} ${units[i]}`;
+}
+
 export interface RedisCacheConfig {
     host: string;
     port: number;
@@ -27,6 +39,8 @@ export interface RedisCacheConfig {
     maxRetriesPerRequest?: number;
     lazyConnect?: boolean;
     enableReadyCheck?: boolean;
+    connectTimeout?: number;
+    commandTimeout?: number;
 }
 
 /**
@@ -60,7 +74,8 @@ export class RedisCache implements CacheProvider {
             maxRetriesPerRequest: config.maxRetriesPerRequest || 3,
             lazyConnect: config.lazyConnect || false,
             enableReadyCheck: config.enableReadyCheck || false,
-            // Connection pooling settings
+            connectTimeout: config.connectTimeout ?? 5000,
+            commandTimeout: config.commandTimeout ?? 3000,
             enableOfflineQueue: true,
         };
 
@@ -105,7 +120,7 @@ export class RedisCache implements CacheProvider {
                 const memoryMatch = info.match(/used_memory:(\d+)/);
                 if (memoryMatch && memoryMatch[1]) {
                     const memoryUsage = parseInt(memoryMatch[1], 10);
-                    logger.debug({ msg: 'Redis memory usage', memoryUsage });
+                    logger.debug({ msg: 'Redis memory usage', memoryUsage: formatBytes(memoryUsage), memoryUsageBytes: memoryUsage });
                 }
             } catch (error) {
                 logger.error({ error, msg: 'Failed to get Redis memory info' });
@@ -194,19 +209,20 @@ export class RedisCache implements CacheProvider {
             const prefixedKeys = keys.map(k => this.prefixKey(k));
             const values = await this.client.mget(...prefixedKeys);
 
-            return values.map((value, index) => {
+            return await Promise.all(values.map(async (value, index) => {
                 if (value === null) {
                     this.stats.misses++;
                     return null;
                 }
                 this.stats.hits++;
                 try {
-                    return JSON.parse(value) as T;
+                    const parsed = JSON.parse(value);
+                    return await CompressionUtils.decompress(parsed) as T;
                 } catch (parseError) {
                     logger.error({ error: parseError, key: keys[index], msg: 'Failed to parse cached value' });
                     return null;
                 }
-            });
+            }));
         } catch (error) {
             logger.error({ error, msg: 'Redis getMany error' });
             return new Array(keys.length).fill(null);
@@ -222,7 +238,8 @@ export class RedisCache implements CacheProvider {
 
             for (const entry of entries) {
                 const prefixedKey = this.prefixKey(entry.key);
-                const serializedValue = JSON.stringify(entry.value);
+                const compressedValue = await CompressionUtils.compress(entry.value);
+                const serializedValue = JSON.stringify(compressedValue);
 
                 if (entry.ttl) {
                     pipeline.setex(prefixedKey, Math.floor(entry.ttl / 1000), serializedValue);
@@ -306,7 +323,8 @@ export class RedisCache implements CacheProvider {
                 misses: this.stats.misses,
                 hitRate: this.stats.hits / (this.stats.hits + this.stats.misses) || 0,
                 size,
-                memoryUsage
+                memoryUsage,
+                memoryUsageHuman: memoryUsage !== undefined ? formatBytes(memoryUsage) : undefined
             };
         } catch (error) {
             logger.error({ error, msg: 'Redis getStats error' });
@@ -351,6 +369,7 @@ export class RedisCache implements CacheProvider {
                 connected: true,
                 latency,
                 memoryUsage,
+                memoryUsageHuman: memoryUsage !== undefined ? formatBytes(memoryUsage) : undefined,
                 connections,
                 version
             };

package/core/validateEnv.ts

@@ -52,6 +52,14 @@ const envSchema = z
             .string()
             .regex(/^\d+$/, "DB_STATEMENT_TIMEOUT must be numeric")
             .optional(),
+        DB_QUERY_TIMEOUT: z
+            .string()
+            .regex(/^\d+$/, "DB_QUERY_TIMEOUT must be numeric (milliseconds)")
+            .optional(),
+        DB_CONNECTION_TIMEOUT: z
+            .string()
+            .regex(/^\d+$/, "DB_CONNECTION_TIMEOUT must be numeric (seconds)")
+            .optional(),
     })
     .refine(
         (env) => {

package/database/DatabaseHelper.ts

@@ -108,6 +108,14 @@ export const CreateEntityTable = async () => {
         updated_at TIMESTAMP DEFAULT NOW(),
         deleted_at TIMESTAMP
     );`;
+
+    // Add partial index for soft-delete queries - critical for 1M+ scale
+    // This allows efficient filtering of non-deleted entities
+    await db.unsafe(`
+        CREATE INDEX IF NOT EXISTS idx_entities_deleted_null
+        ON entities (id)
+        WHERE deleted_at IS NULL
+    `);
 }
 
 export const CreateComponentTable = async () => {
@@ -638,4 +646,108 @@ export const BenchmarkPartitionCounts = async (partitionCounts: number[] = [8, 1
     return results;
 }
 
-export const GenerateTableName = (name: string) => `components_${name.toLowerCase().replace(/\s+/g, '_')}`;
+export const GenerateTableName = (name: string) => `components_${name.toLowerCase().replace(/\s+/g, '_')}`;
+
+/**
+ * Creates a GIN index on a JSONB foreign key field for optimized relation queries.
+ * This significantly improves @HasMany and @BelongsTo relation resolution performance.
+ *
+ * @param tableName The component table name (e.g., 'components_userprofile')
+ * @param foreignKeyField The JSONB field name that holds the foreign key (e.g., 'user_id')
+ * @returns Promise<boolean> - true if index was created, false if it already exists
+ *
+ * @example
+ * // Create index for user_id foreign key
+ * await CreateForeignKeyIndex('components_userprofile', 'user_id');
+ */
+export const CreateForeignKeyIndex = async (tableName: string, foreignKeyField: string): Promise<boolean> => {
+    tableName = validateIdentifier(tableName);
+    foreignKeyField = validateIdentifier(foreignKeyField);
+
+    const indexName = `idx_${tableName}_fk_${foreignKeyField}`;
+
+    // Check if index already exists
+    const existingIndex = await db.unsafe(`
+        SELECT 1 FROM pg_indexes
+        WHERE tablename = '${tableName}' AND indexname = '${indexName}'
+    `);
+
+    if (existingIndex.length > 0) {
+        logger.trace(`Foreign key index ${indexName} already exists`);
+        return false;
+    }
+
+    // Check partition strategy
+    const partitionStrategy = await GetPartitionStrategy();
+    const useConcurrently = partitionStrategy !== 'hash' && !process.env.USE_PGLITE;
+
+    try {
+        await retryWithBackoff(async () => {
+            // Use btree index on the extracted text value for equality lookups (faster than GIN for FK)
+            await db.unsafe(`
+                CREATE INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF NOT EXISTS ${indexName}
+                ON ${tableName} ((data->>'${foreignKeyField}'))
+                WHERE deleted_at IS NULL
+            `);
+        });
+        logger.info(`Created foreign key index ${indexName} on ${tableName}.data->>'${foreignKeyField}'`);
+        return true;
+    } catch (error: any) {
+        if (error.message?.includes('duplicate key value violates unique constraint')) {
+            logger.trace(`Foreign key index ${indexName} already exists (concurrent creation)`);
+            return false;
+        }
+        throw error;
+    }
+};
+
+/**
+ * Creates foreign key indexes for all relation fields defined in archetypes.
+ * Should be called during database initialization for optimal relation query performance.
+ */
+export const CreateRelationIndexes = async (): Promise<void> => {
+    const storage = getMetadataStorage();
+    const createdIndexes: string[] = [];
+
+    for (const [archetypeId, relations] of storage.archetypes_relations_map) {
+        for (const relation of relations) {
+            if (!relation.options?.foreignKey) continue;
+
+            const foreignKey = relation.options.foreignKey;
+            // Skip nested foreign keys (handled differently)
+            if (foreignKey.includes('.')) continue;
+
+            // Find the component that has this foreign key
+            const archetypeMetadata = storage.archetypes.find(a =>
+                storage.getComponentId(a.name) === archetypeId || a.typeId === archetypeId
+            );
+
+            if (!archetypeMetadata) continue;
+
+            // Get the component fields for this archetype
+            const archetypeFields = storage.archetypes_field_map.get(archetypeId) || [];
+
+            for (const field of archetypeFields) {
+                const componentId = storage.getComponentId(field.component.name);
+                const componentProps = storage.getComponentProperties(componentId);
+                const hasForeignKey = componentProps.some(prop => prop.propertyKey === foreignKey);
+
+                if (hasForeignKey) {
+                    const tableName = GenerateTableName(field.component.name);
+                    try {
+                        const created = await CreateForeignKeyIndex(tableName, foreignKey);
+                        if (created) {
+                            createdIndexes.push(`${tableName}.${foreignKey}`);
+                        }
+                    } catch (error) {
+                        logger.warn(`Failed to create FK index for ${tableName}.${foreignKey}: ${error}`);
+                    }
+                }
+            }
+        }
+    }
+
+    if (createdIndexes.length > 0) {
+        logger.info(`Created ${createdIndexes.length} relation foreign key indexes`);
+    }
+};

package/database/index.ts

@@ -1,56 +1,89 @@
 import {SQL} from "bun";
 import { logger } from "../core/Logger";
 
-let connectionUrl = `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT ?? "5432"}/${process.env.POSTGRES_DB}`;
-if(process.env.DB_CONNECTION_URL) {
-    connectionUrl = process.env.DB_CONNECTION_URL;
-}
+// Query timeout in milliseconds (default 30s, configurable via env)
+// This is used by Query.exec(), Entity.save(), etc.
+export const QUERY_TIMEOUT_MS = parseInt(process.env.DB_QUERY_TIMEOUT ?? '30000', 10);
+
+// Module-level state for the database connection
+let _db: SQL | null = null;
 
-// Add statement_timeout only when explicitly configured (opt-in)
-// Note: PgBouncer rejects statement_timeout as a startup parameter — use PostgreSQL config or connect_query instead
-if (process.env.USE_PGLITE !== 'true' && process.env.DB_STATEMENT_TIMEOUT) {
-    try {
-        const urlObj = new URL(connectionUrl);
-        urlObj.searchParams.set('options', `-c statement_timeout=${process.env.DB_STATEMENT_TIMEOUT}`);
-        connectionUrl = urlObj.toString();
-    } catch {
-        // Non-standard URL format, skip statement_timeout
+function createDatabase(): SQL {
+    let url = `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT ?? "5432"}/${process.env.POSTGRES_DB}`;
+    if(process.env.DB_CONNECTION_URL) {
+        url = process.env.DB_CONNECTION_URL;
     }
-}
 
-// Log connection URL with credentials redacted
-const redactedUrl = connectionUrl.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:****@');
-logger.info(`Database connection URL: ${redactedUrl}`);
-
-// OPTIMIZED: Reduced from 20 to 10 to prevent overwhelming PGBouncer
-// With 5 app instances: 5 × 10 = 50 connections (well under PGBouncer's limit)
-const maxConnections = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '10', 10);
-logger.info(`Connection pool size: ${maxConnections} connections`);
-
-const db = new SQL({
-    url: connectionUrl,
-    // Connection pool settings - OPTIMIZED for PGBouncer
-    max: maxConnections,
-    idleTimeout: 30000, // Close idle connections after 30s
-    maxLifetime: 600000, // Connection lifetime 10 minutes
-    connectionTimeout: 30, // Timeout when establishing new connections
-    onclose: (err) => {
-        if (err) {
-            if((err as unknown as { code: string }).code === "ERR_POSTGRES_IDLE_TIMEOUT") {
-                logger.trace("Closing connection. Idle");
+    // Add statement_timeout only when explicitly configured (opt-in)
+    // Note: PgBouncer rejects statement_timeout as a startup parameter
+    if (process.env.USE_PGLITE !== 'true' && process.env.DB_STATEMENT_TIMEOUT) {
+        try {
+            const urlObj = new URL(url);
+            urlObj.searchParams.set('options', `-c statement_timeout=${process.env.DB_STATEMENT_TIMEOUT}`);
+            url = urlObj.toString();
+        } catch {
+            // Non-standard URL format, skip statement_timeout
+        }
+    }
+
+    const redactedUrl = url.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:****@');
+    logger.info(`Database connection URL: ${redactedUrl}`);
+
+    const max = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '10', 10);
+    logger.info(`Connection pool size: ${max} connections`);
+    logger.info(`Query timeout: ${QUERY_TIMEOUT_MS}ms`);
+
+    const connTimeout = parseInt(process.env.DB_CONNECTION_TIMEOUT ?? '30', 10);
+
+    return new SQL({
+        url,
+        max,
+        idleTimeout: 30000,
+        maxLifetime: 600000,
+        connectionTimeout: connTimeout,
+        onclose: (err) => {
+            if (err) {
+                const errCode = (err as unknown as { code: string }).code;
+                if(errCode === "ERR_POSTGRES_IDLE_TIMEOUT") {
+                    logger.trace("Closing connection. Idle");
+                } else if (errCode === "ERR_POSTGRES_CONNECTION_CLOSED") {
+                    logger.warn("Database connection closed unexpectedly");
+                } else {
+                    logger.error("Database connection closed with error:");
+                    logger.error(err);
+                }
             } else {
-                logger.error("Database connection closed with error:");
-                logger.error(err);
+                logger.trace("Database connection closed gracefully.");
             }
-        } else {
-            logger.trace("Database connection closed gracefully.");
-        }   
-    },
-    onconnect: () => {
-        // Log when new connections are created
-        logger.trace("New database connection established");
+        },
+        onconnect: () => {
+            logger.trace("New database connection established");
+        }
+    });
+}
+
+/**
+ * Get the database connection. Lazily initializes on first access.
+ * This allows env vars to be set before the first database usage.
+ */
+export function getDb(): SQL {
+    if (!_db) {
+        _db = createDatabase();
     }
-});
+    return _db;
+}
+
+/**
+ * Reinitialize the database connection with current env vars.
+ * Used by benchmark tests that set env vars after module load.
+ */
+export function resetDatabase(): void {
+    _db = createDatabase();
+}
 
+// For backward compatibility, initialize eagerly on import
+// This ensures existing code using `import db from './database'` continues to work
+// Note: For benchmarks that need delayed initialization, use getDb() or resetDatabase()
+const db = getDb();
 
-export default db;
+export default db;