bunsane 0.2.4 → 0.2.8

package/database/DatabaseHelper.ts

@@ -108,6 +108,14 @@ export const CreateEntityTable = async () => {
         updated_at TIMESTAMP DEFAULT NOW(),
         deleted_at TIMESTAMP
     );`;
+
+    // Add partial index for soft-delete queries - critical for 1M+ scale
+    // This allows efficient filtering of non-deleted entities
+    await db.unsafe(`
+        CREATE INDEX IF NOT EXISTS idx_entities_deleted_null
+        ON entities (id)
+        WHERE deleted_at IS NULL
+    `);
 }
 
 export const CreateComponentTable = async () => {
@@ -638,4 +646,108 @@ export const BenchmarkPartitionCounts = async (partitionCounts: number[] = [8, 1
     return results;
 }
 
-export const GenerateTableName = (name: string) => `components_${name.toLowerCase().replace(/\s+/g, '_')}`;
+export const GenerateTableName = (name: string) => `components_${name.toLowerCase().replace(/\s+/g, '_')}`;
+
+/**
+ * Creates a GIN index on a JSONB foreign key field for optimized relation queries.
+ * This significantly improves @HasMany and @BelongsTo relation resolution performance.
+ *
+ * @param tableName The component table name (e.g., 'components_userprofile')
+ * @param foreignKeyField The JSONB field name that holds the foreign key (e.g., 'user_id')
+ * @returns Promise<boolean> - true if index was created, false if it already exists
+ *
+ * @example
+ * // Create index for user_id foreign key
+ * await CreateForeignKeyIndex('components_userprofile', 'user_id');
+ */
+export const CreateForeignKeyIndex = async (tableName: string, foreignKeyField: string): Promise<boolean> => {
+    tableName = validateIdentifier(tableName);
+    foreignKeyField = validateIdentifier(foreignKeyField);
+
+    const indexName = `idx_${tableName}_fk_${foreignKeyField}`;
+
+    // Check if index already exists
+    const existingIndex = await db.unsafe(`
+        SELECT 1 FROM pg_indexes
+        WHERE tablename = '${tableName}' AND indexname = '${indexName}'
+    `);
+
+    if (existingIndex.length > 0) {
+        logger.trace(`Foreign key index ${indexName} already exists`);
+        return false;
+    }
+
+    // Check partition strategy
+    const partitionStrategy = await GetPartitionStrategy();
+    const useConcurrently = partitionStrategy !== 'hash' && !process.env.USE_PGLITE;
+
+    try {
+        await retryWithBackoff(async () => {
+            // Use btree index on the extracted text value for equality lookups (faster than GIN for FK)
+            await db.unsafe(`
+                CREATE INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF NOT EXISTS ${indexName}
+                ON ${tableName} ((data->>'${foreignKeyField}'))
+                WHERE deleted_at IS NULL
+            `);
+        });
+        logger.info(`Created foreign key index ${indexName} on ${tableName}.data->>'${foreignKeyField}'`);
+        return true;
+    } catch (error: any) {
+        if (error.message?.includes('duplicate key value violates unique constraint')) {
+            logger.trace(`Foreign key index ${indexName} already exists (concurrent creation)`);
+            return false;
+        }
+        throw error;
+    }
+};
+
+/**
+ * Creates foreign key indexes for all relation fields defined in archetypes.
+ * Should be called during database initialization for optimal relation query performance.
+ */
+export const CreateRelationIndexes = async (): Promise<void> => {
+    const storage = getMetadataStorage();
+    const createdIndexes: string[] = [];
+
+    for (const [archetypeId, relations] of storage.archetypes_relations_map) {
+        for (const relation of relations) {
+            if (!relation.options?.foreignKey) continue;
+
+            const foreignKey = relation.options.foreignKey;
+            // Skip nested foreign keys (handled differently)
+            if (foreignKey.includes('.')) continue;
+
+            // Find the component that has this foreign key
+            const archetypeMetadata = storage.archetypes.find(a =>
+                storage.getComponentId(a.name) === archetypeId || a.typeId === archetypeId
+            );
+
+            if (!archetypeMetadata) continue;
+
+            // Get the component fields for this archetype
+            const archetypeFields = storage.archetypes_field_map.get(archetypeId) || [];
+
+            for (const field of archetypeFields) {
+                const componentId = storage.getComponentId(field.component.name);
+                const componentProps = storage.getComponentProperties(componentId);
+                const hasForeignKey = componentProps.some(prop => prop.propertyKey === foreignKey);
+
+                if (hasForeignKey) {
+                    const tableName = GenerateTableName(field.component.name);
+                    try {
+                        const created = await CreateForeignKeyIndex(tableName, foreignKey);
+                        if (created) {
+                            createdIndexes.push(`${tableName}.${foreignKey}`);
+                        }
+                    } catch (error) {
+                        logger.warn(`Failed to create FK index for ${tableName}.${foreignKey}: ${error}`);
+                    }
+                }
+            }
+        }
+    }
+
+    if (createdIndexes.length > 0) {
+        logger.info(`Created ${createdIndexes.length} relation foreign key indexes`);
+    }
+};

package/database/index.ts

@@ -1,56 +1,89 @@
 import {SQL} from "bun";
 import { logger } from "../core/Logger";
 
-let connectionUrl = `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT ?? "5432"}/${process.env.POSTGRES_DB}`;
-if(process.env.DB_CONNECTION_URL) {
-    connectionUrl = process.env.DB_CONNECTION_URL;
-}
+// Query timeout in milliseconds (default 30s, configurable via env)
+// This is used by Query.exec(), Entity.save(), etc.
+export const QUERY_TIMEOUT_MS = parseInt(process.env.DB_QUERY_TIMEOUT ?? '30000', 10);
+
+// Module-level state for the database connection
+let _db: SQL | null = null;
 
-// Add statement_timeout only when explicitly configured (opt-in)
-// Note: PgBouncer rejects statement_timeout as a startup parameter — use PostgreSQL config or connect_query instead
-if (process.env.USE_PGLITE !== 'true' && process.env.DB_STATEMENT_TIMEOUT) {
-    try {
-        const urlObj = new URL(connectionUrl);
-        urlObj.searchParams.set('options', `-c statement_timeout=${process.env.DB_STATEMENT_TIMEOUT}`);
-        connectionUrl = urlObj.toString();
-    } catch {
-        // Non-standard URL format, skip statement_timeout
+function createDatabase(): SQL {
+    let url = `postgres://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT ?? "5432"}/${process.env.POSTGRES_DB}`;
+    if(process.env.DB_CONNECTION_URL) {
+        url = process.env.DB_CONNECTION_URL;
     }
-}
 
-// Log connection URL with credentials redacted
-const redactedUrl = connectionUrl.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:****@');
-logger.info(`Database connection URL: ${redactedUrl}`);
-
-// OPTIMIZED: Reduced from 20 to 10 to prevent overwhelming PGBouncer
-// With 5 app instances: 5 × 10 = 50 connections (well under PGBouncer's limit)
-const maxConnections = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '10', 10);
-logger.info(`Connection pool size: ${maxConnections} connections`);
-
-const db = new SQL({
-    url: connectionUrl,
-    // Connection pool settings - OPTIMIZED for PGBouncer
-    max: maxConnections,
-    idleTimeout: 30000, // Close idle connections after 30s
-    maxLifetime: 600000, // Connection lifetime 10 minutes
-    connectionTimeout: 30, // Timeout when establishing new connections
-    onclose: (err) => {
-        if (err) {
-            if((err as unknown as { code: string }).code === "ERR_POSTGRES_IDLE_TIMEOUT") {
-                logger.trace("Closing connection. Idle");
+    // Add statement_timeout only when explicitly configured (opt-in)
+    // Note: PgBouncer rejects statement_timeout as a startup parameter
+    if (process.env.USE_PGLITE !== 'true' && process.env.DB_STATEMENT_TIMEOUT) {
+        try {
+            const urlObj = new URL(url);
+            urlObj.searchParams.set('options', `-c statement_timeout=${process.env.DB_STATEMENT_TIMEOUT}`);
+            url = urlObj.toString();
+        } catch {
+            // Non-standard URL format, skip statement_timeout
+        }
+    }
+
+    const redactedUrl = url.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:****@');
+    logger.info(`Database connection URL: ${redactedUrl}`);
+
+    const max = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '10', 10);
+    logger.info(`Connection pool size: ${max} connections`);
+    logger.info(`Query timeout: ${QUERY_TIMEOUT_MS}ms`);
+
+    const connTimeout = parseInt(process.env.DB_CONNECTION_TIMEOUT ?? '30', 10);
+
+    return new SQL({
+        url,
+        max,
+        idleTimeout: 30000,
+        maxLifetime: 600000,
+        connectionTimeout: connTimeout,
+        onclose: (err) => {
+            if (err) {
+                const errCode = (err as unknown as { code: string }).code;
+                if(errCode === "ERR_POSTGRES_IDLE_TIMEOUT") {
+                    logger.trace("Closing connection. Idle");
+                } else if (errCode === "ERR_POSTGRES_CONNECTION_CLOSED") {
+                    logger.warn("Database connection closed unexpectedly");
+                } else {
+                    logger.error("Database connection closed with error:");
+                    logger.error(err);
+                }
             } else {
-                logger.error("Database connection closed with error:");
-                logger.error(err);
+                logger.trace("Database connection closed gracefully.");
             }
-        } else {
-            logger.trace("Database connection closed gracefully.");
-        }   
-    },
-    onconnect: () => {
-        // Log when new connections are created
-        logger.trace("New database connection established");
+        },
+        onconnect: () => {
+            logger.trace("New database connection established");
+        }
+    });
+}
+
+/**
+ * Get the database connection. Lazily initializes on first access.
+ * This allows env vars to be set before the first database usage.
+ */
+export function getDb(): SQL {
+    if (!_db) {
+        _db = createDatabase();
     }
-});
+    return _db;
+}
+
+/**
+ * Reinitialize the database connection with current env vars.
+ * Used by benchmark tests that set env vars after module load.
+ */
+export function resetDatabase(): void {
+    _db = createDatabase();
+}
 
+// For backward compatibility, initialize eagerly on import
+// This ensures existing code using `import db from './database'` continues to work
+// Note: For benchmarks that need delayed initialization, use getDb() or resetDatabase()
+const db = getDb();
 
-export default db;
+export default db;

package/docs/SCALABILITY_PLAN.md

@@ -0,0 +1,175 @@
+# BunSane Scalability Plan: 1M+ Entities
+
+## Problem Statement
+
+At 50k entities, complex multi-component queries with sorting degrade catastrophically:
+- 10k entities: 20ms
+- 50k entities: 7,880ms (394x slower)
+- Projected 1M: minutes to hours
+
+Root cause: Cartesian product explosion in nested loop joins when sorting on JSONB fields.
+
+## Bottleneck Analysis
+
+### 1. Multi-Component Query Pattern (Critical)
+
+Current SQL for 2-component query:
+```sql
+SELECT DISTINCT ec.entity_id
+FROM entity_components ec
+WHERE ec.type_id IN ($1, $2) AND ec.deleted_at IS NULL
+GROUP BY ec.entity_id
+HAVING COUNT(DISTINCT ec.type_id) = 2
+```
+
+**Problem**: Scans ALL entity_components for ALL matching types, then aggregates.
+At 1M entities × 3 components = 3M rows scanned before filtering.
+
+**Solution**: Use INTERSECT or EXISTS pattern:
+```sql
+-- Option A: INTERSECT (better for 2-3 components)
+SELECT entity_id FROM entity_components WHERE type_id = $1 AND deleted_at IS NULL
+INTERSECT
+SELECT entity_id FROM entity_components WHERE type_id = $2 AND deleted_at IS NULL
+
+-- Option B: EXISTS (better for many components)
+SELECT DISTINCT e.entity_id
+FROM entity_components e
+WHERE e.type_id = $1 AND e.deleted_at IS NULL
+AND EXISTS (SELECT 1 FROM entity_components e2
+            WHERE e2.entity_id = e.entity_id AND e2.type_id = $2 AND e2.deleted_at IS NULL)
+```
+
+### 2. Sorting on JSONB Fields (Critical)
+
+Current pattern:
+```sql
+ORDER BY c.data->>'age' DESC
+```
+
+**Problem**: Can't use B-tree indexes, falls to sequential scan + in-memory sort.
+
+**Solutions**:
+
+A. **Expression Index** (per-field, must exist):
+```sql
+CREATE INDEX idx_testuser_age_btree ON components_testuser ((data->>'age'));
+-- For numeric sorting:
+CREATE INDEX idx_testuser_age_numeric ON components_testuser (((data->>'age')::numeric));
+```
+
+B. **Query must cast for numeric sort**:
+```sql
+ORDER BY (c.data->>'age')::numeric DESC  -- Uses numeric index
+```
+
+C. **Covering Index** (include entity_id for index-only scan):
+```sql
+CREATE INDEX idx_testuser_age_covering
+ON components_testuser ((data->>'age'), entity_id)
+WHERE deleted_at IS NULL;
+```
+
+### 3. Missing Index on entities.deleted_at
+
+Every query does `WHERE deleted_at IS NULL` on entities table.
+
+**Fix**:
+```sql
+CREATE INDEX idx_entities_deleted_null ON entities (id) WHERE deleted_at IS NULL;
+```
+
+### 4. OFFSET Pagination Scaling
+
+`OFFSET 900000` requires scanning 900k rows to skip them.
+
+**Already implemented**: `cursor(entityId)` pagination in Query.ts.
+**Action**: Document as required pattern for large datasets.
+
+## Implementation Plan
+
+### Phase 1: Quick Wins (Immediate)
+
+1. **Add missing index on entities**
+   - File: `database/DatabaseHelper.ts`
+   - Add: `idx_entities_deleted_null`
+
+2. **Numeric cast in ORDER BY**
+   - File: `query/ComponentInclusionNode.ts`
+   - Detect numeric fields and add `::numeric` cast
+
+3. **Use INTERSECT for 2-3 component queries**
+   - File: `query/ComponentInclusionNode.ts`
+   - Threshold: Use INTERSECT when componentIds.size <= 3
+
+### Phase 2: Index Strategy (Short-term)
+
+4. **Auto-create expression indexes for sortable fields**
+   - File: `database/IndexingStrategy.ts`
+   - Add: `createSortIndex(table, field, type: 'text' | 'numeric' | 'date')`
+
+5. **Query hints for sort fields**
+   - New decorator: `@SortableField(type)`
+   - Creates appropriate expression index at registration
+
+### Phase 3: Query Restructuring (Medium-term)
+
+6. **EXISTS pattern for multi-component with filters**
+   - Rewrite CTE to use correlated EXISTS
+   - Push filters into EXISTS subqueries
+
+7. **Batch entity lookup optimization**
+   - Use `= ANY($1::uuid[])` instead of `IN (...)` for large ID lists
+   - Better plan caching with array parameter
+
+### Phase 4: Denormalization Options (Long-term)
+
+8. **entity_component_summary table**
+```sql
+CREATE TABLE entity_component_summary (
+    entity_id UUID PRIMARY KEY,
+    component_types TEXT[],  -- Array of type_ids
+    updated_at TIMESTAMP
+);
+CREATE INDEX idx_ecs_types_gin ON entity_component_summary USING GIN (component_types);
+```
+
+Query pattern:
+```sql
+SELECT entity_id FROM entity_component_summary
+WHERE component_types @> ARRAY[$1, $2]::text[]
+```
+
+9. **Materialized views for hot paths**
+   - Pre-join common component combinations
+   - Refresh on schedule or trigger
+
+## Benchmarks Required
+
+| Scenario | Target (1M entities) |
+|----------|---------------------|
+| Single component, no filter | < 50ms |
+| Single component, indexed filter | < 20ms |
+| 2-component intersection | < 100ms |
+| 3-component intersection | < 200ms |
+| Sort on indexed field, limit 100 | < 50ms |
+| Complex (2-comp + filter + sort) | < 500ms |
+| Count | < 100ms |
+| Cursor pagination (any page) | < 50ms |
+
+## Migration Strategy
+
+1. New indexes are additive (no breaking changes)
+2. Query changes behind feature flag: `BUNSANE_QUERY_V2=true`
+3. Gradual rollout with A/B testing on query performance
+4. Deprecate old patterns after validation
+
+## Files to Modify
+
+- `database/DatabaseHelper.ts` - Add entity index
+- `database/IndexingStrategy.ts` - Sort index creation
+- `query/ComponentInclusionNode.ts` - INTERSECT pattern, numeric cast
+- `query/QueryDAG.ts` - Component count threshold for strategy selection
+- `core/components/Decorators.ts` - @SortableField decorator
+- New: `query/strategies/IntersectStrategy.ts`
+- New: `query/strategies/ExistsStrategy.ts`

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "bunsane",
-    "version": "0.2.4",
+    "version": "0.2.8",
     "author": {
         "name": "yaaruu"
     },
@@ -29,7 +29,18 @@
         "test:stress": "bun test tests/stress --timeout 600000",
         "test:stress:smoke": "STRESS_RECORD_COUNT=10000 bun test tests/stress --timeout 300000",
         "test:stress:standard": "STRESS_RECORD_COUNT=100000 bun test tests/stress --timeout 600000",
-        "test:stress:full": "STRESS_RECORD_COUNT=1000000 bun test tests/stress --timeout 1800000"
+        "test:stress:full": "STRESS_RECORD_COUNT=1000000 bun test tests/stress --timeout 1800000",
+        "bench:generate:xs": "bun tests/benchmark/scripts/generate-db.ts xs",
+        "bench:generate:sm": "bun tests/benchmark/scripts/generate-db.ts sm",
+        "bench:generate:md": "bun tests/benchmark/scripts/generate-db.ts md",
+        "bench:generate:lg": "bun tests/benchmark/scripts/generate-db.ts lg",
+        "bench:generate:xl": "bun tests/benchmark/scripts/generate-db.ts xl",
+        "bench:generate:all": "bun tests/benchmark/scripts/generate-db.ts --all",
+        "bench:run:xs": "bun tests/benchmark/scripts/run-benchmarks.ts xs",
+        "bench:run:sm": "bun tests/benchmark/scripts/run-benchmarks.ts sm",
+        "bench:run:md": "bun tests/benchmark/scripts/run-benchmarks.ts md",
+        "bench:run:lg": "bun tests/benchmark/scripts/run-benchmarks.ts lg",
+        "bench:run:xl": "bun tests/benchmark/scripts/run-benchmarks.ts xl"
     },
     "devDependencies": {
         "@electric-sql/pglite": "^0.3.15",

package/query/CTENode.ts

@@ -13,49 +13,69 @@ export class CTENode extends QueryNode {
         }
 
         let cteSql = "WITH base_entities AS (\n";
-        cteSql += "    SELECT DISTINCT ec.entity_id\n";
-        cteSql += "    FROM entity_components ec\n";
-        cteSql += "    WHERE ec.type_id IN (";
 
-        // Add component type placeholders
-        const typePlaceholders = componentIds.map((_, index) => `$${context.addParam(componentIds[index])}`).join(', ');
-        cteSql += typePlaceholders + ")\n";
-        cteSql += "    AND ec.deleted_at IS NULL\n";
-
-        // Add cursor-based pagination filter in CTE (more efficient than OFFSET)
+        // Build cursor condition for reuse across INTERSECT queries
+        let cursorCondition = "";
         if (context.cursorId !== null) {
             const operator = context.cursorDirection === 'after' ? '>' : '<';
-            cteSql += `    AND ec.entity_id ${operator} $${context.addParam(context.cursorId)}\n`;
+            cursorCondition = ` AND ec.entity_id ${operator} $${context.addParam(context.cursorId)}`;
         }
 
-        // Add exclusions if any
+        // Build exclusion condition for reuse across INTERSECT queries
+        let exclusionCondition = "";
         if (excludedIds.length > 0) {
             const excludedPlaceholders = excludedIds.map((id) => `$${context.addParam(id)}`).join(', ');
-            cteSql += `    AND NOT EXISTS (\n`;
-            cteSql += `        SELECT 1 FROM entity_components ec_ex\n`;
-            cteSql += `        WHERE ec_ex.entity_id = ec.entity_id\n`;
-            cteSql += `        AND ec_ex.type_id IN (${excludedPlaceholders})\n`;
-            cteSql += `        AND ec_ex.deleted_at IS NULL\n`;
-            cteSql += `    )\n`;
+            exclusionCondition = ` AND NOT EXISTS (
+                SELECT 1 FROM entity_components ec_ex
+                WHERE ec_ex.entity_id = ec.entity_id
+                AND ec_ex.type_id IN (${excludedPlaceholders})
+                AND ec_ex.deleted_at IS NULL
+            )`;
         }
 
-        // Add entity exclusions if any
+        // Build entity exclusion condition for reuse
+        let entityExclusionCondition = "";
         if (context.excludedEntityIds.size > 0) {
             const entityExcludedIds = Array.from(context.excludedEntityIds);
             const entityPlaceholders = entityExcludedIds.map((id) => `$${context.addParam(id)}`).join(', ');
-            cteSql += `    AND ec.entity_id NOT IN (${entityPlaceholders})\n`;
+            entityExclusionCondition = ` AND ec.entity_id NOT IN (${entityPlaceholders})`;
         }
 
-        // Group by entity_id to count distinct component types
-        // This ensures entities have ALL required components
-        cteSql += `    GROUP BY ec.entity_id\n`;
-        cteSql += `    HAVING COUNT(DISTINCT ec.type_id) >= $${context.addParam(componentIds.length)}\n`;
+        if (componentIds.length === 1) {
+            // Single component - simple query, no INTERSECT needed
+            const paramIdx = context.addParam(componentIds[0]);
+            cteSql += `    SELECT DISTINCT ec.entity_id\n`;
+            cteSql += `    FROM entity_components ec\n`;
+            cteSql += `    WHERE ec.type_id = $${paramIdx}::text\n`;
+            cteSql += `    AND ec.deleted_at IS NULL\n`;
+            if (cursorCondition) cteSql += `    ${cursorCondition.trim()}\n`;
+            if (exclusionCondition) cteSql += `    ${exclusionCondition.trim()}\n`;
+            if (entityExclusionCondition) cteSql += `    ${entityExclusionCondition.trim()}\n`;
+        } else {
+            // Multiple components - use INTERSECT for much faster queries
+            // INTERSECT allows PostgreSQL to use index scans independently per component
+            // then efficiently merge results, avoiding Cartesian product explosion
+            const intersectQueries = componentIds.map((compId) => {
+                const paramIdx = context.addParam(compId);
+                let subquery = `SELECT ec.entity_id FROM entity_components ec WHERE ec.type_id = $${paramIdx}::text AND ec.deleted_at IS NULL`;
+                // Add cursor/exclusion conditions to each subquery for efficiency
+                if (cursorCondition) subquery += cursorCondition;
+                if (exclusionCondition) subquery += exclusionCondition;
+                if (entityExclusionCondition) subquery += entityExclusionCondition;
+                return `(${subquery})`;
+            });
+            cteSql += `    SELECT entity_id FROM (\n`;
+            cteSql += `        ${intersectQueries.join('\n        INTERSECT\n        ')}\n`;
+            cteSql += `    ) AS intersected\n`;
+        }
 
         // Add ORDER BY for deterministic pagination results
         // Must be before LIMIT/OFFSET for consistent page results
         // Reverse order for 'before' cursor direction
         const orderDirection = context.cursorDirection === 'before' ? 'DESC' : 'ASC';
-        cteSql += `    ORDER BY ec.entity_id ${orderDirection}\n`;
+        // Use correct column reference based on query structure
+        const orderColumn = componentIds.length === 1 ? 'ec.entity_id' : 'entity_id';
+        cteSql += `    ORDER BY ${orderColumn} ${orderDirection}\n`;
 
         // Check if there are component filters - if so, pagination must happen AFTER filtering
         // Otherwise we'd limit results before applying filters, causing incorrect results