bunsane 0.3.2 → 0.4.0

package/database/DatabaseHelper.ts

@@ -2,6 +2,7 @@ import db from "./index";
 import { logger as MainLogger } from "../core/Logger";
 import { getMetadataStorage } from "../core/metadata";
 import { ensureMultipleJSONBPathIndexes } from "./IndexingStrategy";
+import { getMembershipTable } from "../query/membershipSource";
 const logger = MainLogger.child({ scope: "DatabaseHelper" });
 
 const BUNSANE_RELATION_TYPED_COLUMN = process.env.BUNSANE_RELATION_TYPED_COLUMN === 'true' || false;
@@ -49,7 +50,7 @@ export const GetSchema = async () => {
 
 export const HasValidBaseTable = async (): Promise<boolean> => {
     const tables = await GetSchema();
-    const neededTables = ["entities", "components", "entity_components"];
+    const neededTables = ["entities", "components"];
     return neededTables.every(t => tables.includes(t));
 }
 
@@ -73,13 +74,9 @@ export const PrepareDatabase = async () => {
         logger.error(`Failed to create component table: ${error}`);
         throw error;
     }
-    try {
-        await CreateEntityComponentTable();
-        await PopulateComponentIds();
-    } catch (error) {
-        logger.error(`Failed to create entity component table: ${error}`);
-        throw error;
-    }
+    // `entity_components` is no longer created or written. `components`
+    // (UNIQUE(entity_id, type_id)) is the single source of membership truth
+    // as of Phase 3 of docs/ENTITY_COMPONENTS_REMOVAL_PLAN.md.
 }
 
 export const GetDatabaseDataSize = async () => {
@@ -131,27 +128,17 @@ export const CreateComponentTable = async () => {
 
     // If the table exists but has a different partitioning strategy, we need to recreate it
     if (tableExists.length > 0 && existingStrategy !== partitionStrategy) {
-        logger.info(`Partitioning strategy changed from ${existingStrategy} to ${partitionStrategy}. Recreating components table...`);
+        await assertComponentDataSafeToDrop(existingStrategy, partitionStrategy);
+        logger.warn(`Partitioning strategy changed from ${existingStrategy} to ${partitionStrategy}. Recreating components table...`);
 
         // Drop the existing table and all its partitions
         await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
 
         // Also clean up any orphaned partition tables
-        const orphanedPartitions = await db.unsafe(`
-            SELECT tablename
-            FROM pg_tables
-            WHERE tablename LIKE 'components_%'
-            AND schemaname = 'public'
-        `);
-
-        for (const partition of orphanedPartitions) {
-            await db.unsafe(`DROP TABLE IF EXISTS ${partition.tablename} CASCADE`);
-        }
+        await dropOrphanedPartitionTables();
     }
 
     if (partitionStrategy === 'hash') {
-        // Clean up any existing LIST partition tables before creating HASH partitions
-        await cleanupOldListPartitions();
         await CreateHashPartitionedComponentTable();
     } else {
         // Original LIST partitioning
@@ -178,41 +165,64 @@ export const CreateComponentTable = async () => {
     }
 }
 
-const cleanupOldListPartitions = async () => {
+/**
+ * Pure decision: may the components table be dropped for a partition strategy
+ * switch? Returns null when safe, otherwise the refusal error message.
+ */
+export const partitionRecreateRefusal = (
+    hasData: boolean,
+    forceFlag: string | undefined,
+    existingStrategy: string | null,
+    requestedStrategy: string
+): string | null => {
+    if (!hasData) return null;
+    if (forceFlag === 'true') return null;
+    return (
+        `Refusing to recreate 'components' table: partition strategy changed from '${existingStrategy}' to '${requestedStrategy}' but the table contains data. ` +
+        `Recreating would permanently delete all component data. Options: ` +
+        `(1) set BUNSANE_PARTITION_STRATEGY back to '${existingStrategy}' to keep the current layout; ` +
+        `(2) back up component data, then restart once with BUNSANE_FORCE_PARTITION_RECREATE=true to drop and recreate (DESTRUCTIVE); ` +
+        `(3) migrate the data manually to the new layout before switching.`
+    );
+}
+
+export const assertComponentDataSafeToDrop = async (existingStrategy: string | null, requestedStrategy: string) => {
+    let hasData = false;
     try {
-        logger.info(`Cleaning up old LIST partition tables before creating HASH partitions`);
-        
-        // Get all existing LIST partition tables
-        const existingPartitions = await db.unsafe(`
-            SELECT tablename 
-            FROM pg_tables 
-            WHERE tablename LIKE 'components_%' 
-            AND schemaname = 'public'
-            AND tablename != 'components'
-        `);
-        
-        for (const row of existingPartitions) {
-            const tableName = row.tablename;
-            logger.trace(`Dropping old LIST partition table: ${tableName}`);
-            await db.unsafe(`DROP TABLE IF EXISTS ${tableName} CASCADE`);
-        }
-        
-        // Drop the main components table if it exists (to recreate with HASH partitioning)
-        const mainTableExists = await db.unsafe(`
-            SELECT 1 FROM information_schema.tables 
-            WHERE table_name = 'components' 
-            AND table_schema = 'public'
-        `);
-        
-        if (mainTableExists.length > 0) {
-            logger.trace(`Dropping existing components table for HASH partitioning`);
-            await db.unsafe(`DROP TABLE IF EXISTS components CASCADE`);
-        }
-        
-        logger.info(`Cleaned up ${existingPartitions.length} old partition tables`);
+        const rows = await db.unsafe(`SELECT 1 FROM components LIMIT 1`);
+        hasData = rows.length > 0;
     } catch (error) {
-        logger.warn(`Could not clean up old LIST partitions: ${error}`);
-        // Continue anyway - the table creation might still work
+        logger.warn(`Could not check components table for data before recreate: ${error}`);
+    }
+
+    const refusal = partitionRecreateRefusal(
+        hasData,
+        process.env.BUNSANE_FORCE_PARTITION_RECREATE,
+        existingStrategy,
+        requestedStrategy
+    );
+    if (refusal) throw new Error(refusal);
+
+    if (hasData) {
+        logger.warn(`BUNSANE_FORCE_PARTITION_RECREATE=true — dropping components table WITH DATA to switch partition strategy from '${existingStrategy}' to '${requestedStrategy}'. This permanently deletes all component data.`);
+    }
+}
+
+const dropOrphanedPartitionTables = async () => {
+    const orphanedPartitions = await db.unsafe(`
+        SELECT tablename
+        FROM pg_tables
+        WHERE tablename LIKE 'components_%'
+        AND schemaname = 'public'
+        AND tablename != 'components'
+    `);
+
+    for (const partition of orphanedPartitions) {
+        await db.unsafe(`DROP TABLE IF EXISTS ${partition.tablename} CASCADE`);
+    }
+
+    if (orphanedPartitions.length > 0) {
+        logger.info(`Cleaned up ${orphanedPartitions.length} orphaned partition tables`);
     }
 }
 
@@ -471,58 +481,75 @@ export const CreateEntityComponentTable = async () => {
     }
 }
 
+/**
+ * Rollback/repair tool. Backfills the legacy `entity_components` mirror from
+ * `components` (the single source of truth as of Phase 3). Only needed if you
+ * intend to downgrade to a build that still reads `entity_components`
+ * (BUNSANE_MEMBERSHIP_SOURCE=legacy).
+ *
+ * The framework no longer creates `entity_components` on boot. If the table is
+ * absent this throws a clear error: create it first via
+ * `CreateEntityComponentTable()`, then re-run this.
+ *
+ * Intended for a freshly-created/empty table: ON CONFLICT DO NOTHING skips
+ * pre-existing rows, so `deleted_at` drift on them is not reconciled.
+ */
 export const PopulateComponentIds = async () => {
+    const tableExists = await db.unsafe(`
+        SELECT 1 FROM information_schema.tables
+        WHERE table_name = 'entity_components'
+        AND table_schema = 'public'
+    `);
+    if (tableExists.length === 0) {
+        throw new Error(
+            `Cannot populate entity_components: the table does not exist. ` +
+            `It is no longer created since Phase 3 of the entity_components removal. ` +
+            `If you need the legacy mirror (e.g. for a downgrade), run ` +
+            `CreateEntityComponentTable() first, then re-run PopulateComponentIds().`
+        );
+    }
     try {
-        // Populate component_id for existing rows that don't have it set
-        await db`UPDATE entity_components 
-                 SET component_id = c.id 
-                 FROM components c 
-                 WHERE entity_components.entity_id = c.entity_id 
-                 AND entity_components.type_id = c.type_id 
+        // Backfill membership rows from components, then set component_id for
+        // any rows still missing it.
+        await db`INSERT INTO entity_components (entity_id, type_id, component_id, deleted_at)
+                 SELECT c.entity_id, c.type_id, c.id, c.deleted_at
+                 FROM components c
+                 ON CONFLICT (entity_id, type_id) DO NOTHING`;
+        await db`UPDATE entity_components
+                 SET component_id = c.id
+                 FROM components c
+                 WHERE entity_components.entity_id = c.entity_id
+                 AND entity_components.type_id = c.type_id
                  AND entity_components.component_id IS NULL`;
-        
-        logger.info(`Populated component_id for existing entity_components rows`);
+
+        logger.info(`Backfilled entity_components from components`);
     } catch (error) {
-        logger.warn(`Could not populate component_id for existing rows: ${error}`);
+        logger.warn(`Could not backfill entity_components: ${error}`);
+        throw error;
     }
 }
 
 export const EnsureDatabaseMigrations = async () => {
     logger.trace(`Checking for database migrations...`);
-    
-    try {
-        // First, ensure the table exists and has the basic structure
-        await CreateEntityComponentTable();
-        
-        // Check if entity_components table has component_id column
-        const columnCheck = await db`SELECT column_name FROM information_schema.columns 
-                                     WHERE table_name = 'entity_components' 
-                                     AND column_name = 'component_id' 
-                                     AND table_schema = 'public'`;
-        
-        if (columnCheck.length === 0) {
-            logger.info(`entity_components table missing component_id column, adding it...`);
-            // Add the column
-            await db`ALTER TABLE entity_components ADD COLUMN component_id UUID`;
-            logger.info(`Added component_id column to entity_components table`);
-            
-            // Wait a bit for the column to be available
-            await new Promise(resolve => setTimeout(resolve, 500));
-            
-            // Populate existing data
-            await PopulateComponentIds();
-        } else {
-            logger.trace(`entity_components table already has component_id column`);
-        }
-    } catch (error) {
-        logger.error(`Failed during database migration: ${error}`);
-        // Try to add the column anyway in case the check failed
-        try {
-            await db`ALTER TABLE entity_components ADD COLUMN IF NOT EXISTS component_id UUID`;
-            logger.info(`Attempted to add component_id column as fallback`);
-        } catch (fallbackError) {
-            logger.error(`Fallback column addition also failed: ${fallbackError}`);
-        }
+
+    // `entity_components` is no longer created, migrated, or written (Phase 3
+    // of docs/ENTITY_COMPONENTS_REMOVAL_PLAN.md). Any pre-existing table is
+    // left in place, untouched — never auto-dropped. Membership now lives
+    // solely in `components` (UNIQUE(entity_id, type_id)). Run
+    // `PopulateComponentIds()` manually to backfill the legacy table if a
+    // downgrade is ever required.
+    const orphanCheck = await db.unsafe(`
+        SELECT 1 FROM information_schema.tables
+        WHERE table_name = 'entity_components'
+        AND table_schema = 'public'
+    `);
+    if (orphanCheck.length > 0) {
+        logger.info(
+            `[entity_components] Orphaned table detected. ` +
+            `This table is no longer used by the framework (see docs/ENTITY_COMPONENTS_REMOVAL_PLAN.md). ` +
+            `Verify your upgrade succeeded (run a smoke-test query against the 'components' table), ` +
+            `then drop the orphan manually: DROP TABLE entity_components;`
+        );
     }
 }
 
@@ -621,10 +648,10 @@ export const BenchmarkPartitionCounts = async (partitionCounts: number[] = [8, 1
         await db.unsafe(`ANALYZE ${tempTableName}`);
         
         // Run benchmark query
-        const explainResult = await db.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) 
-            SELECT DISTINCT ec.entity_id as id 
-            FROM entity_components ec 
-            WHERE ec.type_id = (SELECT type_id FROM ${tempTableName} LIMIT 1) 
+        const explainResult = await db.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
+            SELECT DISTINCT ec.entity_id as id
+            FROM ${getMembershipTable()} ec
+            WHERE ec.type_id = (SELECT type_id FROM ${tempTableName} LIMIT 1)
             AND ec.deleted_at IS NULL`);
         
         const plan = explainResult[0]['QUERY PLAN'] ? JSON.parse(explainResult[0]['QUERY PLAN']) : explainResult[0];

package/database/IndexingStrategy.ts

@@ -11,7 +11,7 @@ const validateIdentifier = (str: string, maxLength: number = 64): string => {
     return str;
 };
 
-export type IndexType = 'gin' | 'btree' | 'hash' | 'numeric';
+export type IndexType = 'gin' | 'btree' | 'hash' | 'numeric' | 'fulltext';
 
 export interface IndexDefinition {
     tableName: string;
@@ -140,8 +140,9 @@ export const ensureMultipleJSONBPathIndexes = async (
 ): Promise<void> => {
     for (const def of indexDefinitions) {
         if (def.indexType === 'numeric') {
-            // Use numeric index for range queries
             await ensureNumericIndex(def.tableName, def.field);
+        } else if (def.indexType === 'fulltext') {
+            await ensureFullTextIndex(def.tableName, def.field);
         } else {
             await ensureJSONBPathIndex(
                 def.tableName,
@@ -241,6 +242,75 @@ export const ensureNumericIndex = async (
     }
 };
 
+/**
+ * Creates a GIN index on a JSONB field for full-text search using to_tsvector.
+ *
+ * The index expression MUST match FullTextSearchBuilder's vectorSql exactly:
+ *   to_tsvector('<language>', <alias>.data->'<field>')
+ * so PostgreSQL can use this index for those predicates.
+ *
+ * @param tableName The table name to create index on
+ * @param field The JSONB field path containing text for full-text search
+ * @param language PostgreSQL text search language (default 'english')
+ */
+export const ensureFullTextIndex = async (
+    tableName: string,
+    field: string,
+    language: string = 'english'
+): Promise<void> => {
+    tableName = validateIdentifier(tableName);
+    field = validateIdentifier(field);
+
+    const indexName = `idx_${tableName}_${field}_fts`;
+
+    try {
+        logger.trace(`Ensuring full-text GIN index ${indexName} on ${tableName} for field ${field} (language: ${language})`);
+
+        const existingIndexes = await db.unsafe(`
+            SELECT indexname
+            FROM pg_indexes
+            WHERE tablename = '${tableName}' AND indexname = '${indexName}'
+        `);
+
+        if (existingIndexes.length > 0) {
+            logger.trace(`Index ${indexName} already exists`);
+            return;
+        }
+
+        const partitionCheck = await db.unsafe(`
+            SELECT relkind
+            FROM pg_class
+            WHERE relname = '${tableName}' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')
+        `);
+
+        const isPartitioned = partitionCheck.length > 0 && partitionCheck[0].relkind === 'p';
+        const useConcurrently = !isPartitioned && !process.env.USE_PGLITE;
+
+        // Expression matches FullTextSearchBuilder.vectorSql: to_tsvector('<lang>', data->'<field>')
+        // The -> operator (not ->>) returns jsonb; PostgreSQL casts jsonb text to tsvector input.
+        const indexSQL = `CREATE INDEX${useConcurrently ? ' CONCURRENTLY' : ''} IF NOT EXISTS ${indexName} ON ${tableName} USING GIN (to_tsvector('${language}', data->'${field}'))`;
+
+        logger.trace(`Creating full-text index with SQL: ${indexSQL}`);
+        await db.unsafe(indexSQL);
+        logger.info(`Created full-text GIN index ${indexName} on ${tableName}${useConcurrently ? ' (concurrently)' : ' (blocking)'}`);
+
+    } catch (error: any) {
+        if (error.message && (
+            error.message.includes('already exists') ||
+            error.code === '42P07'
+        )) {
+            logger.trace(`Index ${indexName} already exists (confirmed by error), skipping creation`);
+            return;
+        }
+        if (error.code === '40P01' || (error.message && error.message.includes('deadlock'))) {
+            logger.warn(`Deadlock detected while creating index ${indexName}, skipping`);
+            return;
+        }
+        logger.error(`Failed to create full-text index on ${tableName} for field ${field}: ${error}`);
+        throw error;
+    }
+};
+
 /**
  * Creates a composite index on multiple JSONB fields for efficient combined filter queries.
  * Useful for queries like `WHERE status = 'active' AND age >= 21`

package/database/PreparedStatementCache.ts

@@ -20,8 +20,14 @@ export interface CacheStats {
 }
 
 /**
- * LRU Cache for prepared statements to eliminate PostgreSQL planning overhead
- * for repeated query patterns in the Bunsane Query system.
+ * @deprecated No longer used on the query hot path. This cache never called
+ * a prepare API — it stored `{ sql, _isPrepared: true }` placeholders and
+ * every "hit" still executed `db.unsafe(sql, params)`, so PostgreSQL
+ * re-planned regardless. Bun SQL auto-prepares parameterized statements per
+ * connection (prepare:true default), providing real server-side plan reuse
+ * at the driver layer. The class is kept so `Query.getCacheStats()`,
+ * `/metrics`, and registry invalidation call sites remain stable; it now
+ * reports an idle cache.
  */
 export class PreparedStatementCache {
     private cache: Map<string, CacheEntry> = new Map();

package/database/cancellable.ts

@@ -1,22 +1,35 @@
-/**
- * Wraps a Bun SQL Query so an AbortSignal can cancel the in-flight query
- * via the underlying `query.cancel()` method. When the signal fires the
- * server-side query receives a cancel request, the awaited promise rejects,
- * any enclosing transaction triggers ROLLBACK, and the pooled backend
- * connection is released. Without this, a wall-clock timeout leaks the
- * backend into `idle in transaction` under pgbouncer transaction-mode.
- */
-export async function runWithSignal<T>(q: any, signal?: AbortSignal): Promise<T> {
-    if (!signal) return await q;
-    if (signal.aborted) {
-        try { q.cancel?.(); } catch { /* ignore */ }
-        throw signal.reason ?? new Error('Query aborted');
-    }
-    const onAbort = () => { try { q.cancel?.(); } catch { /* ignore */ } };
-    signal.addEventListener('abort', onAbort, { once: true });
-    try {
-        return await q;
-    } finally {
-        signal.removeEventListener('abort', onAbort);
-    }
-}
+/**
+ * Wraps a Bun SQL Query so an AbortSignal can cancel the in-flight query
+ * via the underlying `query.cancel()` method. When the signal fires the
+ * server-side query receives a cancel request, the awaited promise rejects,
+ * any enclosing transaction triggers ROLLBACK, and the pooled backend
+ * connection is released. Without this, a wall-clock timeout leaks the
+ * backend into `idle in transaction` under pgbouncer transaction-mode.
+ *
+ * Rejection on abort is immediate (raced) rather than waiting for the
+ * driver to honor the cancel — some drivers (PGlite socket bridge) ignore
+ * `cancel()` entirely and would otherwise hang the caller until the query
+ * finishes on its own. The query's eventual settle is swallowed so it can't
+ * surface as an unhandled rejection after the race is lost.
+ */
+export async function runWithSignal<T>(q: any, signal?: AbortSignal): Promise<T> {
+    if (!signal) return await q;
+    if (signal.aborted) {
+        try { q.cancel?.(); } catch { /* ignore */ }
+        throw signal.reason ?? new Error('Query aborted');
+    }
+    let onAbort: (() => void) | undefined;
+    const abortPromise = new Promise<never>((_, reject) => {
+        onAbort = () => {
+            try { q.cancel?.(); } catch { /* ignore */ }
+            Promise.resolve(q).catch(() => { /* swallow post-abort settle */ });
+            reject(signal.reason ?? new Error('Query aborted'));
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
+    });
+    try {
+        return await Promise.race([q, abortPromise]);
+    } finally {
+        if (onAbort) signal.removeEventListener('abort', onAbort);
+    }
+}

package/database/index.ts

@@ -14,8 +14,14 @@ function createDatabase(): SQL {
         url = process.env.DB_CONNECTION_URL;
     }
 
-    // Add statement_timeout only when explicitly configured (opt-in)
-    // Note: PgBouncer rejects statement_timeout as a startup parameter
+    // DB_STATEMENT_TIMEOUT (opt-in, server-side query cancellation):
+    //   - Skipped under PgBouncer because it rejects startup parameters.
+    //   - DB_QUERY_TIMEOUT (default 30 s) is JS-side only: it raises a client error
+    //     after the deadline but does NOT cancel the in-flight PostgreSQL query,
+    //     which continues running and holds locks until the server decides to stop it.
+    //   - Production deployments NOT behind PgBouncer should set DB_STATEMENT_TIMEOUT
+    //     (e.g. DB_STATEMENT_TIMEOUT=30000) so the server itself kills runaway queries
+    //     and releases locks promptly.
     if (process.env.USE_PGLITE !== 'true' && process.env.DB_STATEMENT_TIMEOUT) {
         try {
             const urlObj = new URL(url);
@@ -29,10 +35,16 @@ function createDatabase(): SQL {
     const redactedUrl = url.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:****@');
     logger.info(`Database connection URL: ${redactedUrl}`);
 
-    const max = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '10', 10);
+    const max = parseInt(process.env.POSTGRES_MAX_CONNECTIONS ?? '20', 10);
     logger.info(`Connection pool size: ${max} connections`);
     logger.info(`Query timeout: ${QUERY_TIMEOUT_MS}ms`);
 
+    // DB_CONNECTION_TIMEOUT (default 30 s): the pool waits this long for a free
+    // slot before rejecting the caller. At 30 s, pool exhaustion queues HTTP
+    // requests for up to 30 s each, holding sockets and consuming memory.
+    // User-facing services should consider 5 s for fast-fail so clients get
+    // an error quickly rather than a slow timeout. Long-running background
+    // workers (schedulers, outbox, migrations) can keep higher values.
     const connTimeout = parseInt(process.env.DB_CONNECTION_TIMEOUT ?? '30', 10);
 
     return new SQL({