@stonyx/orm 0.2.5-alpha.0 → 0.3.1

package/dist/postgres/migration-generator.js

@@ -0,0 +1,280 @@
+import { introspectModels, introspectViews, buildTableDDL, buildViewDDL, buildVectorIndexDDL, schemasToSnapshot, viewSchemasToSnapshot, getTopologicalOrder } from './schema-introspector.js';
+import { buildCreateHypertable, buildEnableCompression, buildCompressionPolicy } from '../timescale/query-builder.js';
+import { readFile, createFile, createDirectory, fileExists } from '@stonyx/utils/file';
+import path from 'path';
+import config from 'stonyx/config';
+import log from 'stonyx/log';
+export async function generateMigration(description = 'migration', configKey = 'postgres') {
+    const { migrationsDir } = config.orm[configKey];
+    const rootPath = config.rootPath;
+    const migrationsPath = path.resolve(rootPath, migrationsDir);
+    await createDirectory(migrationsPath);
+    const schemas = introspectModels();
+    const currentSnapshot = schemasToSnapshot(schemas);
+    const previousSnapshot = await loadLatestSnapshot(migrationsPath);
+    const diff = diffSnapshots(previousSnapshot, currentSnapshot);
+    // Don't return early -- check view changes too before deciding
+    if (!diff.hasChanges) {
+        const viewSchemasPrelim = introspectViews();
+        const currentViewSnapshotPrelim = viewSchemasToSnapshot(viewSchemasPrelim);
+        const previousViewSnapshotPrelim = extractViewsFromSnapshot(previousSnapshot);
+        const viewDiffPrelim = diffViewSnapshots(previousViewSnapshotPrelim, currentViewSnapshotPrelim);
+        if (!viewDiffPrelim.hasChanges) {
+            log.db?.('No schema changes detected.');
+            return null;
+        }
+    }
+    const upStatements = [];
+    const downStatements = [];
+    // pgvector extension (include in initial migration)
+    if (Object.keys(previousSnapshot).length === 0) {
+        upStatements.push('CREATE EXTENSION IF NOT EXISTS vector;');
+    }
+    // New tables -- in topological order (parents before children)
+    const allOrder = getTopologicalOrder(schemas);
+    const addedOrdered = allOrder.filter(name => diff.addedModels.includes(name));
+    for (const name of addedOrdered) {
+        upStatements.push(buildTableDDL(name, schemas[name], schemas) + ';');
+        // HNSW indexes for vector columns
+        const indexStatements = buildVectorIndexDDL(name, schemas[name]);
+        for (const stmt of indexStatements) {
+            upStatements.push(stmt + ';');
+        }
+        downStatements.unshift(`DROP TABLE IF EXISTS "${schemas[name].table}" CASCADE;`);
+    }
+    // Hypertable conversion + compression (TimescaleDB only)
+    if (configKey === 'timescale') {
+        for (const name of addedOrdered) {
+            const schema = schemas[name];
+            if (!schema.hypertable)
+                continue;
+            const { timeColumn, chunkInterval } = schema.hypertable;
+            upStatements.push(buildCreateHypertable(schema.table, timeColumn, { chunkInterval }).sql + ';');
+            if (schema.hypertable.compress) {
+                const { segmentBy, orderBy, after } = schema.hypertable.compress;
+                upStatements.push(buildEnableCompression(schema.table, segmentBy, orderBy).sql + ';');
+                if (after) {
+                    upStatements.push(buildCompressionPolicy(schema.table, after).sql + ';');
+                }
+            }
+            downStatements.unshift('-- Hypertable conversion is not reversible; table drop handles cleanup');
+        }
+    }
+    // Removed tables (warn only, commented out)
+    for (const name of diff.removedModels) {
+        upStatements.push(`-- WARNING: Model '${name}' was removed. Uncomment to drop table:`);
+        upStatements.push(`-- DROP TABLE IF EXISTS "${previousSnapshot[name].table}" CASCADE;`);
+        downStatements.push(`-- Recreate table for removed model '${name}' manually if needed`);
+    }
+    // Added columns
+    for (const { model, column, type } of diff.addedColumns) {
+        const table = currentSnapshot[model].table;
+        upStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type};`);
+        downStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+        // Add HNSW index if it's a vector column
+        if (type.startsWith('vector(')) {
+            upStatements.push(`CREATE INDEX IF NOT EXISTS "idx_${table}_${column}_hnsw" ON "${table}" USING hnsw ("${column}" vector_cosine_ops) WITH (m = 16, ef_construction = 200);`);
+        }
+    }
+    // Removed columns
+    for (const { model, column, type } of diff.removedColumns) {
+        const table = previousSnapshot[model]?.table;
+        if (!table)
+            throw new Error(`Missing table name in snapshot for model "${model}"`);
+        upStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+        downStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type};`);
+    }
+    // Changed column types
+    for (const { model, column, from, to } of diff.changedColumns) {
+        const table = currentSnapshot[model].table;
+        upStatements.push(`ALTER TABLE "${table}" ALTER COLUMN "${column}" TYPE ${to};`);
+        downStatements.push(`ALTER TABLE "${table}" ALTER COLUMN "${column}" TYPE ${from};`);
+    }
+    // Added foreign keys
+    for (const { model, column, references } of diff.addedForeignKeys) {
+        const table = currentSnapshot[model].table;
+        const refModel = Object.entries(currentSnapshot).find(([, s]) => s.table === references.references);
+        const fkType = refModel && refModel[1].idType === 'string' ? 'VARCHAR(255)' : 'INTEGER';
+        const constraintName = `fk_${table}_${column}`;
+        upStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${fkType};`);
+        upStatements.push(`ALTER TABLE "${table}" ADD CONSTRAINT "${constraintName}" FOREIGN KEY ("${column}") REFERENCES "${references.references}"("${references.column}") ON DELETE SET NULL;`);
+        downStatements.push(`ALTER TABLE "${table}" DROP CONSTRAINT "${constraintName}";`);
+        downStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+    }
+    // Removed foreign keys
+    for (const { model, column, references } of diff.removedForeignKeys) {
+        const table = previousSnapshot[model]?.table;
+        if (!table)
+            throw new Error(`Missing table name in snapshot for model "${model}"`);
+        const refModel = Object.entries(previousSnapshot).find(([, s]) => s.table === references.references);
+        const fkType = refModel && refModel[1].idType === 'string' ? 'VARCHAR(255)' : 'INTEGER';
+        const constraintName = `fk_${table}_${column}`;
+        upStatements.push(`ALTER TABLE "${table}" DROP CONSTRAINT "${constraintName}";`);
+        upStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+        downStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${fkType};`);
+        downStatements.push(`ALTER TABLE "${table}" ADD CONSTRAINT "${constraintName}" FOREIGN KEY ("${column}") REFERENCES "${references.references}"("${references.column}") ON DELETE SET NULL;`);
+    }
+    // View migrations -- views are created AFTER tables (dependency order)
+    const viewSchemas = introspectViews();
+    const currentViewSnapshot = viewSchemasToSnapshot(viewSchemas);
+    const previousViewSnapshot = extractViewsFromSnapshot(previousSnapshot);
+    const viewDiff = diffViewSnapshots(previousViewSnapshot, currentViewSnapshot);
+    if (viewDiff.hasChanges) {
+        upStatements.push('');
+        upStatements.push('-- Views');
+        downStatements.push('');
+        downStatements.push('-- Views');
+        // Added views
+        for (const name of viewDiff.addedViews) {
+            try {
+                const ddl = buildViewDDL(name, viewSchemas[name], schemas);
+                upStatements.push(ddl + ';');
+                downStatements.unshift(`DROP VIEW IF EXISTS "${viewSchemas[name].viewName}";`);
+            }
+            catch (error) {
+                upStatements.push(`-- WARNING: Could not generate DDL for view '${name}': ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        // Removed views
+        for (const name of viewDiff.removedViews) {
+            upStatements.push(`-- WARNING: View '${name}' was removed. Uncomment to drop view:`);
+            upStatements.push(`-- DROP VIEW IF EXISTS "${previousViewSnapshot[name].viewName}";`);
+            downStatements.push(`-- Recreate view for removed view '${name}' manually if needed`);
+        }
+        // Changed views (source or aggregates changed)
+        for (const name of viewDiff.changedViews) {
+            try {
+                const ddl = buildViewDDL(name, viewSchemas[name], schemas);
+                upStatements.push(ddl + ';');
+            }
+            catch (error) {
+                upStatements.push(`-- WARNING: Could not generate DDL for changed view '${name}': ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+    }
+    const combinedHasChanges = diff.hasChanges || viewDiff.hasChanges;
+    if (!combinedHasChanges) {
+        log.db?.('No schema changes detected.');
+        return null;
+    }
+    // Merge view snapshot into the main snapshot
+    const combinedSnapshot = { ...currentSnapshot };
+    for (const [name, viewSnap] of Object.entries(currentViewSnapshot)) {
+        combinedSnapshot[name] = viewSnap;
+    }
+    const sanitizedDescription = description.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_]/g, '');
+    const timestamp = Math.floor(Date.now() / 1000);
+    const filename = `${timestamp}_${sanitizedDescription}.sql`;
+    const content = `-- UP\n${upStatements.join('\n')}\n\n-- DOWN\n${downStatements.join('\n')}\n`;
+    await createFile(path.join(migrationsPath, filename), content);
+    await createFile(path.join(migrationsPath, '.snapshot.json'), JSON.stringify(combinedSnapshot, null, 2));
+    log.db?.(`Migration generated: ${filename}`);
+    return { filename, content, snapshot: combinedSnapshot };
+}
+export async function loadLatestSnapshot(migrationsPath) {
+    const snapshotPath = path.join(migrationsPath, '.snapshot.json');
+    const exists = await fileExists(snapshotPath);
+    if (!exists)
+        return {};
+    return readFile(snapshotPath, { json: true });
+}
+export function diffSnapshots(previous, current) {
+    const addedModels = [];
+    const removedModels = [];
+    const addedColumns = [];
+    const removedColumns = [];
+    const changedColumns = [];
+    const addedForeignKeys = [];
+    const removedForeignKeys = [];
+    // Find added models
+    for (const name of Object.keys(current)) {
+        if (!previous[name])
+            addedModels.push(name);
+    }
+    // Find removed models
+    for (const name of Object.keys(previous)) {
+        if (!current[name])
+            removedModels.push(name);
+    }
+    // Find column changes in existing models
+    for (const name of Object.keys(current)) {
+        if (!previous[name])
+            continue;
+        const prevCols = previous[name].columns || {};
+        const currCols = current[name].columns || {};
+        // Added columns
+        for (const [col, type] of Object.entries(currCols)) {
+            if (!prevCols[col]) {
+                addedColumns.push({ model: name, column: col, type });
+            }
+            else if (prevCols[col] !== type) {
+                changedColumns.push({ model: name, column: col, from: prevCols[col], to: type });
+            }
+        }
+        // Removed columns
+        for (const [col, type] of Object.entries(prevCols)) {
+            if (!currCols[col]) {
+                removedColumns.push({ model: name, column: col, type });
+            }
+        }
+        // Foreign key changes
+        const prevFKs = previous[name].foreignKeys || {};
+        const currFKs = current[name].foreignKeys || {};
+        for (const [col, refs] of Object.entries(currFKs)) {
+            if (!prevFKs[col]) {
+                addedForeignKeys.push({ model: name, column: col, references: refs });
+            }
+        }
+        for (const [col, refs] of Object.entries(prevFKs)) {
+            if (!currFKs[col]) {
+                removedForeignKeys.push({ model: name, column: col, references: refs });
+            }
+        }
+    }
+    const hasChanges = addedModels.length > 0 || removedModels.length > 0 ||
+        addedColumns.length > 0 || removedColumns.length > 0 ||
+        changedColumns.length > 0 || addedForeignKeys.length > 0 || removedForeignKeys.length > 0;
+    return {
+        hasChanges,
+        addedModels,
+        removedModels,
+        addedColumns,
+        removedColumns,
+        changedColumns,
+        addedForeignKeys,
+        removedForeignKeys,
+    };
+}
+export function detectSchemaDrift(schemas, snapshot) {
+    const current = schemasToSnapshot(schemas);
+    return diffSnapshots(snapshot, current);
+}
+export function extractViewsFromSnapshot(snapshot) {
+    const views = {};
+    for (const [name, entry] of Object.entries(snapshot)) {
+        if (entry.isView)
+            views[name] = entry;
+    }
+    return views;
+}
+export function diffViewSnapshots(previous, current) {
+    const addedViews = [];
+    const removedViews = [];
+    const changedViews = [];
+    for (const name of Object.keys(current)) {
+        if (!previous[name]) {
+            addedViews.push(name);
+        }
+        else if (current[name].viewQuery !== previous[name].viewQuery ||
+            current[name].source !== previous[name].source) {
+            changedViews.push(name);
+        }
+    }
+    for (const name of Object.keys(previous)) {
+        if (!current[name]) {
+            removedViews.push(name);
+        }
+    }
+    const hasChanges = addedViews.length > 0 || removedViews.length > 0 || changedViews.length > 0;
+    return { hasChanges, addedViews, removedViews, changedViews };
+}

package/dist/postgres/migration-runner.d.ts

@@ -0,0 +1,10 @@
+import type { Pool } from 'pg';
+export declare function ensureMigrationsTable(pool: Pool, tableName?: string): Promise<void>;
+export declare function getAppliedMigrations(pool: Pool, tableName?: string): Promise<string[]>;
+export declare function getMigrationFiles(migrationsDir: string): Promise<string[]>;
+export declare function parseMigrationFile(content: string): {
+    up: string;
+    down: string;
+};
+export declare function applyMigration(pool: Pool, filename: string, upSql: string, tableName?: string): Promise<void>;
+export declare function rollbackMigration(pool: Pool, filename: string, downSql: string, tableName?: string): Promise<void>;

package/dist/postgres/migration-runner.js

@@ -0,0 +1,87 @@
+import { fileExists } from '@stonyx/utils/file';
+import fs from 'fs/promises';
+import { validateIdentifier } from './query-builder.js';
+export async function ensureMigrationsTable(pool, tableName = '__migrations') {
+    validateIdentifier(tableName, 'migration table name');
+    await pool.query(`
+    CREATE TABLE IF NOT EXISTS "${tableName}" (
+      id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+      filename VARCHAR(255) NOT NULL UNIQUE,
+      applied_at TIMESTAMPTZ DEFAULT NOW()
+    )
+  `);
+}
+export async function getAppliedMigrations(pool, tableName = '__migrations') {
+    validateIdentifier(tableName, 'migration table name');
+    const result = await pool.query(`SELECT filename FROM "${tableName}" ORDER BY id ASC`);
+    return result.rows.map(row => row.filename);
+}
+export async function getMigrationFiles(migrationsDir) {
+    const exists = await fileExists(migrationsDir);
+    if (!exists)
+        return [];
+    const entries = await fs.readdir(migrationsDir);
+    return entries
+        .filter(f => f.endsWith('.sql'))
+        .sort();
+}
+export function parseMigrationFile(content) {
+    const upMarker = '-- UP';
+    const downMarker = '-- DOWN';
+    const upIndex = content.indexOf(upMarker);
+    const downIndex = content.indexOf(downMarker);
+    if (upIndex === -1) {
+        return { up: content.trim(), down: '' };
+    }
+    const upStart = upIndex + upMarker.length;
+    const upEnd = downIndex !== -1 ? downIndex : content.length;
+    const up = content.slice(upStart, upEnd).trim();
+    const down = downIndex !== -1 ? content.slice(downIndex + downMarker.length).trim() : '';
+    return { up, down };
+}
+export async function applyMigration(pool, filename, upSql, tableName = '__migrations') {
+    validateIdentifier(tableName, 'migration table name');
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const statements = splitStatements(upSql);
+        for (const stmt of statements) {
+            await client.query(stmt);
+        }
+        await client.query(`INSERT INTO "${tableName}" (filename) VALUES ($1)`, [filename]);
+        await client.query('COMMIT');
+    }
+    catch (error) {
+        await client.query('ROLLBACK');
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+}
+export async function rollbackMigration(pool, filename, downSql, tableName = '__migrations') {
+    validateIdentifier(tableName, 'migration table name');
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        const statements = splitStatements(downSql);
+        for (const stmt of statements) {
+            await client.query(stmt);
+        }
+        await client.query(`DELETE FROM "${tableName}" WHERE filename = $1`, [filename]);
+        await client.query('COMMIT');
+    }
+    catch (error) {
+        await client.query('ROLLBACK');
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+}
+function splitStatements(sql) {
+    return sql
+        .split(';')
+        .map(s => s.trim())
+        .filter(s => s.length > 0 && !s.startsWith('--'));
+}

package/dist/postgres/postgres-db.d.ts

@@ -0,0 +1,119 @@
+import { getPool, closePool } from './connection.js';
+import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
+import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
+import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
+import { buildInsert, buildUpdate, buildDelete, buildSelect, buildVectorSearch, buildHybridSearch } from './query-builder.js';
+import { store } from '@stonyx/orm';
+import { createRecord } from '../manage-record.js';
+import { confirm } from '@stonyx/utils/prompt';
+import { readFile } from '@stonyx/utils/file';
+import { getPluralName } from '../plural-registry.js';
+import config from 'stonyx/config';
+import log from 'stonyx/log';
+import path from 'path';
+import type { Pool } from 'pg';
+import type { OrmRecord } from '../types/orm-types.js';
+interface PersistContext {
+    record?: OrmRecord;
+    recordId?: unknown;
+    oldState?: Record<string, unknown>;
+}
+interface PersistResponse {
+    data?: {
+        id: unknown;
+    };
+}
+interface SearchResult {
+    record: OrmRecord;
+    distance: number;
+}
+interface VectorSearchOptions {
+    limit?: number;
+    where?: Record<string, unknown>;
+}
+interface PostgresDeps {
+    getPool: typeof getPool;
+    closePool: typeof closePool;
+    ensureMigrationsTable: typeof ensureMigrationsTable;
+    getAppliedMigrations: typeof getAppliedMigrations;
+    getMigrationFiles: typeof getMigrationFiles;
+    applyMigration: typeof applyMigration;
+    parseMigrationFile: typeof parseMigrationFile;
+    introspectModels: typeof introspectModels;
+    introspectViews: typeof introspectViews;
+    getTopologicalOrder: typeof getTopologicalOrder;
+    schemasToSnapshot: typeof schemasToSnapshot;
+    loadLatestSnapshot: typeof loadLatestSnapshot;
+    detectSchemaDrift: typeof detectSchemaDrift;
+    buildInsert: typeof buildInsert;
+    buildUpdate: typeof buildUpdate;
+    buildDelete: typeof buildDelete;
+    buildSelect: typeof buildSelect;
+    buildVectorSearch: typeof buildVectorSearch;
+    buildHybridSearch: typeof buildHybridSearch;
+    createRecord: typeof createRecord;
+    store: typeof store;
+    confirm: typeof confirm;
+    readFile: typeof readFile;
+    getPluralName: typeof getPluralName;
+    config: typeof config;
+    log: typeof log;
+    path: typeof path;
+    [key: string]: unknown;
+}
+export default class PostgresDB {
+    /** PostgreSQL extensions to enable on pool init. Subclasses can override. */
+    static extensions: string[];
+    /** Config key under config.orm for this adapter. Subclasses can override. */
+    static configKey: string;
+    /** Singleton instance, set by subclass constructor name. */
+    static instance: PostgresDB | undefined;
+    deps: PostgresDeps;
+    pool: Pool | null;
+    pgConfig: Record<string, unknown>;
+    constructor(deps?: Partial<PostgresDeps>);
+    protected requirePool(): Pool;
+    init(): Promise<void>;
+    startup(): Promise<void>;
+    shutdown(): Promise<void>;
+    save(): Promise<void>;
+    /**
+     * Loads only models with memory: true into the in-memory store on startup.
+     * Models with memory: false are skipped -- accessed on-demand via find()/findAll().
+     */
+    loadMemoryRecords(): Promise<void>;
+    /**
+     * @deprecated Use loadMemoryRecords() instead. Kept for backward compatibility.
+     */
+    loadAllRecords(): Promise<void>;
+    /**
+     * Find a single record by ID from PostgreSQL.
+     * Does NOT cache the result in the store for memory: false models.
+     */
+    findRecord(modelName: string, id: string | number): Promise<OrmRecord | undefined>;
+    /**
+     * Find all records of a model from PostgreSQL, with optional conditions.
+     */
+    findAll(modelName: string, conditions?: Record<string, unknown>): Promise<OrmRecord[]>;
+    /**
+     * Perform a vector similarity search using cosine distance.
+     */
+    vectorSearch(modelName: string, vectorColumn: string, queryVector: number[], options?: VectorSearchOptions): Promise<SearchResult[]>;
+    /**
+     * Perform a hybrid search combining vector similarity with text filtering.
+     */
+    hybridSearch(modelName: string, vectorColumn: string, queryVector: number[], textColumn: string, textQuery: string, options?: VectorSearchOptions): Promise<SearchResult[]>;
+    /**
+     * Remove a record from the in-memory store if its model has memory: false.
+     * The record object itself survives -- the caller retains the reference.
+     * @private
+     */
+    private _evictIfNotMemory;
+    private _rowToRawData;
+    persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void>;
+    private _persistCreate;
+    private _persistUpdate;
+    private _persistDelete;
+    private _recordToRow;
+}
+export {};