bunsane 0.1.4 → 0.2.0

package/core/BatchLoader.ts

@@ -1,100 +1,210 @@
-import { Entity } from "core/Entity";
-import { BaseComponent } from "core/Components";
+import { Entity } from "./Entity";
+import { BaseComponent } from "./components";
 import { timed } from "./Decorators";
 import db from "../database";
 import { sql } from "bun";
 
-// Phase 2A: Memory Pooling for Entity Objects
-class EntityPool {
-    private static instance: EntityPool;
-    private pool: Map<string, Entity[]> = new Map();
-    private maxPoolSize = 1000;
-
-    static getInstance(): EntityPool {
-        if (!EntityPool.instance) {
-            EntityPool.instance = new EntityPool();
-        }
-        return EntityPool.instance;
-    }
-
-    get(entityId: string): Entity | null {
-        const entities = this.pool.get(entityId);
-        if (entities && entities.length > 0) {
-            return entities.pop()!;
-        }
-        return null;
-    }
-
-    put(entity: Entity): void {
-        const entityId = entity.id;
-        let entities = this.pool.get(entityId);
-        if (!entities) {
-            entities = [];
-            this.pool.set(entityId, entities);
-        }
-        if (entities.length < this.maxPoolSize) {
-            entities.push(entity);
-        }
-    }
+interface CachedRelation {
+    ids: string[];
+    expiresAt: number;
+}
 
-    clear(): void {
-        this.pool.clear();
-    }
+interface BatchLoaderOptions {
+    fieldName?: string;
+    batchSize?: number;
+    maxConcurrency?: number;
+    cacheTTL?: number;
 }
 
 export class BatchLoader {
-    private static entityPool = EntityPool.getInstance();
-
-    @timed("BatchLoader.loadRelatedEntities")
-    static async loadRelatedEntities<C extends BaseComponent & { value: string }>(
-        entities: Entity[],
-        component: new () => C,
-        loader: (ids: string[]) => Promise<Entity[]>
-    ): Promise<Map<string, Entity>> {
-        const ids: string[] = [];
-        for (const entity of entities) {
-            const data = await entity.get(component) as any;
-            if (data?.value) {
-                ids.push(data.value);
-            }
-        }
-        const uniqueIds = [...new Set(ids)];
-        const relatedEntities = await loader(uniqueIds);
-        const map = new Map<string, Entity>();
-        for (const related of relatedEntities) {
-            map.set(related.id, related);
-        }
-        return map;
-    }
+    private static cache = new Map<string, Map<string, CachedRelation>>();
+    private static readonly DEFAULT_BATCH_SIZE = 1000;
+    private static readonly DEFAULT_MAX_CONCURRENCY = 5;
+    private static readonly DEFAULT_CACHE_TTL = 300_000; // 5 minutes
 
+    /**
+     * Load related entities efficiently with caching and batching
+     */
     @timed("BatchLoader.loadRelatedEntitiesBatched")
     static async loadRelatedEntitiesBatched<C extends BaseComponent>(
         entities: Entity[],
         component: new () => C,
-        loader: (ids: string[]) => Promise<Entity[]>
+        loader: (ids: string[]) => Promise<Entity[]>,
+        options: BatchLoaderOptions = {}
     ): Promise<Map<string, Entity>> {
         if (entities.length === 0) return new Map();
 
+        const {
+            fieldName = 'value',
+            batchSize = this.DEFAULT_BATCH_SIZE,
+            maxConcurrency = this.DEFAULT_MAX_CONCURRENCY,
+            cacheTTL = this.DEFAULT_CACHE_TTL
+        } = options;
+
         const comp = new component();
         const typeId = comp.getTypeID();
         const parentIds = entities.map(e => e.id);
 
-        const rows = await db`
-            SELECT c.entity_id, (c.data->>'value') AS related_id
-            FROM components c
-            WHERE c.entity_id IN ${sql(parentIds)}
-              AND c.type_id = ${typeId}
-              AND c.deleted_at IS NULL
-        `;
+        // Check cache first
+        const cacheKey = `${typeId}:${fieldName}`;
+        let cachedResults = this.cache.get(cacheKey);
+        if (!cachedResults) {
+            cachedResults = new Map();
+            this.cache.set(cacheKey, cachedResults);
+        }
+
+        const now = Date.now();
+
+        // Get uncached or expired parent IDs
+        const uncachedParentIds = parentIds.filter(id => {
+            const entry = cachedResults.get(id);
+            if (!entry) return true;
+            if (now > entry.expiresAt) {
+                cachedResults.delete(id);
+                return true;
+            }
+            return false;
+        });
+
+        if (uncachedParentIds.length > 0) {
+            // Batch the parent IDs to avoid huge IN clauses
+            const batches = this.chunkArray(uncachedParentIds, batchSize);
+
+            for (const batch of batches) {
+                const rows = await db`
+                    SELECT c.entity_id, (c.data->>${sql(fieldName)}) AS related_id
+                    FROM components c
+                    WHERE c.entity_id IN ${sql(batch)}
+                      AND c.type_id = ${typeId}
+                      AND c.deleted_at IS NULL
+                      AND c.data->>${sql(fieldName)} IS NOT NULL
+                `;
 
-        const uniqueIds = [...new Set(rows.map((r: any) => r.related_id).filter(Boolean))] as string[];
-        if (uniqueIds.length === 0) return new Map();
+                // Group by parent entity for caching
+                const parentGroups = new Map<string, string[]>();
+                for (const row of rows) {
+                    const parentId = row.entity_id;
+                    const relatedId = row.related_id;
+                    if (!parentGroups.has(parentId)) {
+                        parentGroups.set(parentId, []);
+                    }
+                    parentGroups.get(parentId)!.push(relatedId);
+                }
+
+                const expiresAt = Date.now() + cacheTTL;
+
+                // Cache the related IDs for each parent
+                for (const [parentId, relatedIds] of parentGroups) {
+                    cachedResults.set(parentId, { ids: relatedIds, expiresAt });
+                }
+
+                // Cache empty arrays for parents with no relations
+                for (const parentId of batch) {
+                    if (!parentGroups.has(parentId)) {
+                        cachedResults.set(parentId, { ids: [], expiresAt });
+                    }
+                }
+            }
+        }
+
+        // Collect all unique related IDs from cache
+        const allRelatedIds = new Set<string>();
+        for (const parentId of parentIds) {
+            const entry = cachedResults.get(parentId);
+            const relatedIds = entry?.ids || [];
+            relatedIds.forEach((id: string) => allRelatedIds.add(id));
+        }
+
+        if (allRelatedIds.size === 0) return new Map();
+
+        // Load related entities with concurrency control
+        const relatedIdArray = Array.from(allRelatedIds);
+        const entityBatches = this.chunkArray(relatedIdArray, batchSize);
+
+        const relatedEntities: Entity[] = [];
+        const semaphore = new Semaphore(maxConcurrency);
+
+        await Promise.all(
+            entityBatches.map(async (batch) => {
+                await semaphore.acquire();
+                try {
+                    const entities = await loader(batch);
+                    relatedEntities.push(...entities);
+                } finally {
+                    semaphore.release();
+                }
+            })
+        );
 
-        const relatedEntities = await loader(uniqueIds);
         const map = new Map<string, Entity>();
         for (const related of relatedEntities) {
             map.set(related.id, related);
         }
         return map;
     }
+
+    /**
+     * Clear the cache (useful for testing or memory management)
+     */
+    static clearCache(): void {
+        this.cache.clear();
+    }
+
+    /**
+     * Get cache statistics including expired entry count
+     */
+    static getCacheStats(): { size: number; entries: number; expired: number } {
+        let totalEntries = 0;
+        let expiredEntries = 0;
+        const now = Date.now();
+        for (const [, parentMap] of this.cache) {
+            for (const [, entry] of parentMap) {
+                totalEntries++;
+                if (now > entry.expiresAt) {
+                    expiredEntries++;
+                }
+            }
+        }
+        return { size: this.cache.size, entries: totalEntries, expired: expiredEntries };
+    }
+
+    private static chunkArray<T>(array: T[], size: number): T[][] {
+        const chunks: T[][] = [];
+        for (let i = 0; i < array.length; i += size) {
+            chunks.push(array.slice(i, i + size));
+        }
+        return chunks;
+    }
+}
+
+/**
+ * Simple semaphore for concurrency control
+ */
+class Semaphore {
+    private permits: number;
+    private waitQueue: (() => void)[] = [];
+
+    constructor(permits: number) {
+        this.permits = permits;
+    }
+
+    async acquire(): Promise<void> {
+        if (this.permits > 0) {
+            this.permits--;
+            return;
+        }
+
+        return new Promise((resolve) => {
+            this.waitQueue.push(resolve);
+        });
+    }
+
+    release(): void {
+        this.permits++;
+        if (this.waitQueue.length > 0) {
+            const resolve = this.waitQueue.shift()!;
+            this.permits--;
+            resolve();
+        }
+    }
 }

package/core/Config.ts

@@ -0,0 +1,153 @@
+/**
+ * Configuration module for Bunsane Framework
+ * Handles environment variables and application settings
+ */
+
+export interface BunsaneConfig {
+    // Query optimization settings
+    useLateralJoins: boolean;
+    partitionStrategy: 'list' | 'hash';
+    useDirectPartition: boolean;
+
+    // Database settings
+    databaseUrl?: string;
+    databasePoolSize?: number;
+
+    // Application settings
+    appPort: number;
+    nodeEnv: string;
+
+    // Debug settings
+    debugMode: boolean;
+    logLevel: string;
+}
+
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG: BunsaneConfig = {
+    useLateralJoins: true, // Default to true for PG12+
+    partitionStrategy: 'list', // LIST partitioning - one partition per component type
+    useDirectPartition: true,  // Direct partition access - queries go directly to partition tables
+    appPort: 3000,
+    nodeEnv: process.env.NODE_ENV || 'development',
+    debugMode: false,
+    logLevel: 'info'
+};
+
+/**
+ * Configuration singleton class
+ */
+class ConfigManager {
+    private static instance: ConfigManager;
+    private config: BunsaneConfig;
+
+    private constructor() {
+        this.config = this.loadConfig();
+    }
+
+    public static getInstance(): ConfigManager {
+        if (!ConfigManager.instance) {
+            ConfigManager.instance = new ConfigManager();
+        }
+        return ConfigManager.instance;
+    }
+
+    /**
+     * Load configuration from environment variables
+     */
+    private loadConfig(): BunsaneConfig {
+        return {
+            useLateralJoins: this.parseBoolean(process.env.BUNSANE_USE_LATERAL_JOINS, DEFAULT_CONFIG.useLateralJoins),
+            partitionStrategy: this.parsePartitionStrategy(process.env.BUNSANE_PARTITION_STRATEGY, DEFAULT_CONFIG.partitionStrategy),
+            useDirectPartition: this.parseBoolean(process.env.BUNSANE_USE_DIRECT_PARTITION, DEFAULT_CONFIG.useDirectPartition),
+            databaseUrl: process.env.DATABASE_URL,
+            databasePoolSize: this.parseNumber(process.env.DATABASE_POOL_SIZE, 10),
+            appPort: this.parseNumber(process.env.APP_PORT, DEFAULT_CONFIG.appPort),
+            nodeEnv: process.env.NODE_ENV || DEFAULT_CONFIG.nodeEnv,
+            debugMode: this.parseBoolean(process.env.DEBUG, DEFAULT_CONFIG.debugMode),
+            logLevel: process.env.LOG_LEVEL || DEFAULT_CONFIG.logLevel
+        };
+    }
+
+    private parseBoolean(value: string | undefined, defaultValue: boolean): boolean {
+        if (!value) return defaultValue;
+        return value.toLowerCase() === 'true' || value === '1';
+    }
+
+    private parseNumber(value: string | undefined, defaultValue: number): number {
+        if (!value) return defaultValue;
+        const parsed = parseInt(value, 10);
+        return isNaN(parsed) ? defaultValue : parsed;
+    }
+
+    private parsePartitionStrategy(value: string | undefined, defaultValue: 'list' | 'hash'): 'list' | 'hash' {
+        if (!value) return defaultValue;
+        const lowerValue = value.toLowerCase();
+        return lowerValue === 'hash' ? 'hash' : 'list';
+    }
+
+    /**
+     * Get the current configuration
+     */
+    public getConfig(): BunsaneConfig {
+        return { ...this.config };
+    }
+
+    /**
+     * Get a specific configuration value
+     */
+    public get<K extends keyof BunsaneConfig>(key: K): BunsaneConfig[K] {
+        return this.config[key];
+    }
+
+    /**
+     * Check if LATERAL joins should be used
+     */
+    public shouldUseLateralJoins(): boolean {
+        return this.config.useLateralJoins;
+    }
+
+    /**
+     * Check if debug mode is enabled
+     */
+    public isDebugMode(): boolean {
+        return this.config.debugMode;
+    }
+
+    /**
+     * Get partition strategy
+     */
+    public getPartitionStrategy(): 'list' | 'hash' {
+        return this.config.partitionStrategy;
+    }
+
+    /**
+     * Check if direct partition table access should be used
+     */
+    public shouldUseDirectPartition(): boolean {
+        return this.config.useDirectPartition;
+    }
+
+    /**
+     * Reload configuration from environment variables
+     */
+    public reloadConfig(): void {
+        this.config = this.loadConfig();
+    }
+}
+
+/**
+ * Global configuration instance
+ */
+export const config = ConfigManager.getInstance();
+
+/**
+ * Convenience functions for common config checks
+ */
+export const shouldUseLateralJoins = () => config.shouldUseLateralJoins();
+export const isDebugMode = () => config.isDebugMode();
+export const getPartitionStrategy = () => config.getPartitionStrategy();
+export const shouldUseDirectPartition = () => config.shouldUseDirectPartition();
+
+export default config;

package/core/Decorators.ts

@@ -16,7 +16,10 @@ export function timed(hint?: string) {
         const start = performance.now();
         const result = await originalMethod.apply(this, args);
         const end = performance.now();
-        logger.trace(`Execution time for ${propertyKey}${hint ? ` (${hint})` : ''}: ${end - start} ms`);
+        if(end - start > 100) {
+          logger.warn(`Execution time for ${propertyKey}${hint ? ` (${hint})` : ''}: ${end - start} ms`);
+        }
+        // logger.trace(`Execution time for ${propertyKey}${hint ? ` (${hint})` : ''}: ${end - start} ms`);
         return result;
       } else {
         return await originalMethod.apply(this, args);