@stonyx/orm 0.2.1-beta.80 → 0.2.1-beta.82

package/config/environment.js

@@ -19,6 +19,13 @@ const {
   MYSQL_DATABASE,
   MYSQL_CONNECTION_LIMIT,
   MYSQL_MIGRATIONS_DIR,
+  PG_HOST,
+  PG_PORT,
+  PG_USER,
+  PG_PASSWORD,
+  PG_DATABASE,
+  PG_CONNECTION_LIMIT,
+  PG_MIGRATIONS_DIR,
 } = process.env;
 
 export default {
@@ -50,6 +57,16 @@ export default {
     migrationsDir: MYSQL_MIGRATIONS_DIR ?? 'migrations',
     migrationsTable: '__migrations',
   } : undefined,
+  postgres: PG_HOST ? {
+    host: PG_HOST ?? 'localhost',
+    port: parseInt(PG_PORT ?? '5432'),
+    user: PG_USER ?? 'postgres',
+    password: PG_PASSWORD ?? '',
+    database: PG_DATABASE ?? 'stonyx',
+    connectionLimit: parseInt(PG_CONNECTION_LIMIT ?? '10'),
+    migrationsDir: PG_MIGRATIONS_DIR ?? 'migrations',
+    migrationsTable: '__migrations',
+  } : undefined,
   restServer: {
     enabled: ORM_USE_REST_SERVER ?? 'true', // Whether to load restServer for automatic route setup or
     route: ORM_REST_ROUTE ?? '/',

package/package.json

@@ -4,7 +4,7 @@
     "stonyx-async",
     "stonyx-module"
   ],
-  "version": "0.2.1-beta.80",
+  "version": "0.2.1-beta.82",
   "description": "",
   "main": "src/main.js",
   "type": "module",
@@ -48,12 +48,16 @@
   },
   "peerDependencies": {
     "@stonyx/rest-server": ">=0.2.1-beta.14",
-    "mysql2": "^3.0.0"
+    "mysql2": "^3.0.0",
+    "pg": "^8.0.0"
   },
   "peerDependenciesMeta": {
     "mysql2": {
       "optional": true
     },
+    "pg": {
+      "optional": true
+    },
     "@stonyx/rest-server": {
       "optional": true
     }
@@ -62,6 +66,7 @@
     "@stonyx/rest-server": "0.2.1-beta.30",
     "@stonyx/utils": "0.2.3-beta.7",
     "mysql2": "^3.20.0",
+    "pg": "^8.20.0",
     "qunit": "^2.24.1",
     "sinon": "^21.0.0"
   },

package/src/main.js

@@ -109,11 +109,16 @@ export default class Orm {
 
     setup(eventNames);
 
-    if (config.orm.mysql) {
+    if (config.orm.postgres) {
+      const { default: PostgresDB } = await import('./postgres/postgres-db.js');
+      this.sqlDb = new PostgresDB();
+      this.db = this.sqlDb;
+      promises.push(this.sqlDb.init());
+    } else if (config.orm.mysql) {
       const { default: MysqlDB } = await import('./mysql/mysql-db.js');
-      this.mysqlDb = new MysqlDB();
-      this.db = this.mysqlDb;
-      promises.push(this.mysqlDb.init());
+      this.sqlDb = new MysqlDB();
+      this.db = this.sqlDb;
+      promises.push(this.sqlDb.init());
     } else if (this.options.dbType !== 'none') {
       const db = new DB();
       this.db = db;
@@ -131,9 +136,9 @@ export default class Orm {
       return modelClass?.memory === true;
     };
 
-    // Wire up MySQL reference for on-demand queries from store.find()/findAll()
-    if (this.mysqlDb) {
-      Orm.store._mysqlDb = this.mysqlDb;
+    // Wire up SQL adapter reference for on-demand queries from store.find()/findAll()
+    if (this.sqlDb) {
+      Orm.store._sqlDb = this.sqlDb;
     }
 
     Orm.ready = await Promise.all(promises);
@@ -141,11 +146,11 @@ export default class Orm {
   }
 
   async startup() {
-    if (this.mysqlDb) await this.mysqlDb.startup();
+    if (this.sqlDb) await this.sqlDb.startup();
   }
 
   async shutdown() {
-    if (this.mysqlDb) await this.mysqlDb.shutdown();
+    if (this.sqlDb) await this.sqlDb.shutdown();
   }
 
   static get db() {

package/src/manage-record.js

@@ -110,10 +110,10 @@ export function updateRecord(record, rawData, userOptions={}) {
 function assignRecordId(modelName, rawData) {
   if (rawData.id) return;
 
-  // In MySQL mode with numeric IDs, defer to MySQL auto-increment
-  if (Orm.instance?.mysqlDb && !isStringIdModel(modelName)) {
+  // In SQL mode with numeric IDs, defer to database auto-increment
+  if (Orm.instance?.sqlDb && !isStringIdModel(modelName)) {
     rawData.id = `__pending_${Date.now()}_${Math.random()}`;
-    rawData.__pendingMysqlId = true;
+    rawData.__pendingSqlId = true;
     return;
   }
 

package/src/mysql/mysql-db.js

@@ -352,7 +352,7 @@ export default class MysqlDB {
     const insertData = this._recordToRow(record, schema);
 
     // For auto-increment models, remove the pending ID
-    const isPendingId = record.__data.__pendingMysqlId;
+    const isPendingId = record.__data.__pendingSqlId;
 
     if (isPendingId) {
       delete insertData.id;
@@ -380,7 +380,7 @@ export default class MysqlDB {
         response.data.id = realId;
       }
 
-      delete record.__data.__pendingMysqlId;
+      delete record.__data.__pendingSqlId;
     }
   }
 

package/src/orm-request.js

@@ -385,9 +385,9 @@ export default class OrmRequest extends Request {
       // Execute main handler
       const response = await handler(request, state);
 
-      // Persist to MySQL for write operations
-      if (Orm.instance.mysqlDb && WRITE_OPERATIONS.has(operation)) {
-        await Orm.instance.mysqlDb.persist(operation, this.model, context, response);
+      // Persist to SQL database for write operations
+      if (Orm.instance.sqlDb && WRITE_OPERATIONS.has(operation)) {
+        await Orm.instance.sqlDb.persist(operation, this.model, context, response);
       }
 
       // Add response and relevant records to context

package/src/postgres/connection.js

@@ -0,0 +1,30 @@
+let pool = null;
+
+export async function getPool(pgConfig) {
+  if (pool) return pool;
+
+  const { default: pg } = await import('pg');
+
+  pool = new pg.Pool({
+    host: pgConfig.host,
+    port: pgConfig.port,
+    user: pgConfig.user,
+    password: pgConfig.password,
+    database: pgConfig.database,
+    max: pgConfig.connectionLimit,
+    idleTimeoutMillis: 30000,
+    connectionTimeoutMillis: 10000,
+  });
+
+  // Enable pgvector extension
+  await pool.query('CREATE EXTENSION IF NOT EXISTS vector');
+
+  return pool;
+}
+
+export async function closePool() {
+  if (!pool) return;
+
+  await pool.end();
+  pool = null;
+}

package/src/postgres/migration-generator.js

@@ -0,0 +1,302 @@
+import { introspectModels, introspectViews, buildTableDDL, buildViewDDL, buildVectorIndexDDL, schemasToSnapshot, viewSchemasToSnapshot, getTopologicalOrder } from './schema-introspector.js';
+import { readFile, createFile, createDirectory, fileExists } from '@stonyx/utils/file';
+import path from 'path';
+import config from 'stonyx/config';
+import log from 'stonyx/log';
+
+export async function generateMigration(description = 'migration') {
+  const { migrationsDir } = config.orm.postgres;
+  const rootPath = config.rootPath;
+  const migrationsPath = path.resolve(rootPath, migrationsDir);
+
+  await createDirectory(migrationsPath);
+
+  const schemas = introspectModels();
+  const currentSnapshot = schemasToSnapshot(schemas);
+  const previousSnapshot = await loadLatestSnapshot(migrationsPath);
+  const diff = diffSnapshots(previousSnapshot, currentSnapshot);
+
+  // Don't return early — check view changes too before deciding
+  if (!diff.hasChanges) {
+    const viewSchemasPrelim = introspectViews();
+    const currentViewSnapshotPrelim = viewSchemasToSnapshot(viewSchemasPrelim);
+    const previousViewSnapshotPrelim = extractViewsFromSnapshot(previousSnapshot);
+    const viewDiffPrelim = diffViewSnapshots(previousViewSnapshotPrelim, currentViewSnapshotPrelim);
+
+    if (!viewDiffPrelim.hasChanges) {
+      log.db('No schema changes detected.');
+      return null;
+    }
+  }
+
+  const upStatements = [];
+  const downStatements = [];
+
+  // pgvector extension (include in initial migration)
+  if (Object.keys(previousSnapshot).length === 0) {
+    upStatements.push('CREATE EXTENSION IF NOT EXISTS vector;');
+  }
+
+  // New tables — in topological order (parents before children)
+  const allOrder = getTopologicalOrder(schemas);
+  const addedOrdered = allOrder.filter(name => diff.addedModels.includes(name));
+
+  for (const name of addedOrdered) {
+    upStatements.push(buildTableDDL(name, schemas[name], schemas) + ';');
+
+    // HNSW indexes for vector columns
+    const indexStatements = buildVectorIndexDDL(name, schemas[name]);
+    for (const stmt of indexStatements) {
+      upStatements.push(stmt + ';');
+    }
+
+    downStatements.unshift(`DROP TABLE IF EXISTS "${schemas[name].table}" CASCADE;`);
+  }
+
+  // Removed tables (warn only, commented out)
+  for (const name of diff.removedModels) {
+    upStatements.push(`-- WARNING: Model '${name}' was removed. Uncomment to drop table:`);
+    upStatements.push(`-- DROP TABLE IF EXISTS "${previousSnapshot[name].table}" CASCADE;`);
+    downStatements.push(`-- Recreate table for removed model '${name}' manually if needed`);
+  }
+
+  // Added columns
+  for (const { model, column, type } of diff.addedColumns) {
+    const table = currentSnapshot[model].table;
+    upStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type};`);
+    downStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+
+    // Add HNSW index if it's a vector column
+    if (type.startsWith('vector(')) {
+      upStatements.push(`CREATE INDEX IF NOT EXISTS "idx_${table}_${column}_hnsw" ON "${table}" USING hnsw ("${column}" vector_cosine_ops) WITH (m = 16, ef_construction = 200);`);
+    }
+  }
+
+  // Removed columns
+  for (const { model, column, type } of diff.removedColumns) {
+    const table = previousSnapshot[model].table;
+    upStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+    downStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type};`);
+  }
+
+  // Changed column types
+  for (const { model, column, from, to } of diff.changedColumns) {
+    const table = currentSnapshot[model].table;
+    upStatements.push(`ALTER TABLE "${table}" ALTER COLUMN "${column}" TYPE ${to};`);
+    downStatements.push(`ALTER TABLE "${table}" ALTER COLUMN "${column}" TYPE ${from};`);
+  }
+
+  // Added foreign keys
+  for (const { model, column, references } of diff.addedForeignKeys) {
+    const table = currentSnapshot[model].table;
+    const refModel = Object.entries(currentSnapshot).find(([, s]) => s.table === references.references);
+    const fkType = refModel && refModel[1].idType === 'string' ? 'VARCHAR(255)' : 'INTEGER';
+    const constraintName = `fk_${table}_${column}`;
+    upStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${fkType};`);
+    upStatements.push(`ALTER TABLE "${table}" ADD CONSTRAINT "${constraintName}" FOREIGN KEY ("${column}") REFERENCES "${references.references}"("${references.column}") ON DELETE SET NULL;`);
+    downStatements.push(`ALTER TABLE "${table}" DROP CONSTRAINT "${constraintName}";`);
+    downStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+  }
+
+  // Removed foreign keys
+  for (const { model, column, references } of diff.removedForeignKeys) {
+    const table = previousSnapshot[model].table;
+    const refModel = Object.entries(previousSnapshot).find(([, s]) => s.table === references.references);
+    const fkType = refModel && refModel[1].idType === 'string' ? 'VARCHAR(255)' : 'INTEGER';
+    const constraintName = `fk_${table}_${column}`;
+    upStatements.push(`ALTER TABLE "${table}" DROP CONSTRAINT "${constraintName}";`);
+    upStatements.push(`ALTER TABLE "${table}" DROP COLUMN "${column}";`);
+    downStatements.push(`ALTER TABLE "${table}" ADD COLUMN "${column}" ${fkType};`);
+    downStatements.push(`ALTER TABLE "${table}" ADD CONSTRAINT "${constraintName}" FOREIGN KEY ("${column}") REFERENCES "${references.references}"("${references.column}") ON DELETE SET NULL;`);
+  }
+
+  // View migrations — views are created AFTER tables (dependency order)
+  const viewSchemas = introspectViews();
+  const currentViewSnapshot = viewSchemasToSnapshot(viewSchemas);
+  const previousViewSnapshot = extractViewsFromSnapshot(previousSnapshot);
+  const viewDiff = diffViewSnapshots(previousViewSnapshot, currentViewSnapshot);
+
+  if (viewDiff.hasChanges) {
+    upStatements.push('');
+    upStatements.push('-- Views');
+    downStatements.push('');
+    downStatements.push('-- Views');
+
+    // Added views
+    for (const name of viewDiff.addedViews) {
+      try {
+        const ddl = buildViewDDL(name, viewSchemas[name], schemas);
+        upStatements.push(ddl + ';');
+        downStatements.unshift(`DROP VIEW IF EXISTS "${viewSchemas[name].viewName}";`);
+      } catch (error) {
+        upStatements.push(`-- WARNING: Could not generate DDL for view '${name}': ${error.message}`);
+      }
+    }
+
+    // Removed views
+    for (const name of viewDiff.removedViews) {
+      upStatements.push(`-- WARNING: View '${name}' was removed. Uncomment to drop view:`);
+      upStatements.push(`-- DROP VIEW IF EXISTS "${previousViewSnapshot[name].viewName}";`);
+      downStatements.push(`-- Recreate view for removed view '${name}' manually if needed`);
+    }
+
+    // Changed views (source or aggregates changed)
+    for (const name of viewDiff.changedViews) {
+      try {
+        const ddl = buildViewDDL(name, viewSchemas[name], schemas);
+        upStatements.push(ddl + ';');
+      } catch (error) {
+        upStatements.push(`-- WARNING: Could not generate DDL for changed view '${name}': ${error.message}`);
+      }
+    }
+  }
+
+  const combinedHasChanges = diff.hasChanges || viewDiff.hasChanges;
+
+  if (!combinedHasChanges) {
+    log.db('No schema changes detected.');
+    return null;
+  }
+
+  // Merge view snapshot into the main snapshot
+  const combinedSnapshot = { ...currentSnapshot };
+  for (const [name, viewSnap] of Object.entries(currentViewSnapshot)) {
+    combinedSnapshot[name] = viewSnap;
+  }
+
+  const sanitizedDescription = description.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_]/g, '');
+  const timestamp = Math.floor(Date.now() / 1000);
+  const filename = `${timestamp}_${sanitizedDescription}.sql`;
+  const content = `-- UP\n${upStatements.join('\n')}\n\n-- DOWN\n${downStatements.join('\n')}\n`;
+
+  await createFile(path.join(migrationsPath, filename), content);
+  await createFile(path.join(migrationsPath, '.snapshot.json'), JSON.stringify(combinedSnapshot, null, 2));
+
+  log.db(`Migration generated: ${filename}`);
+
+  return { filename, content, snapshot: combinedSnapshot };
+}
+
+export async function loadLatestSnapshot(migrationsPath) {
+  const snapshotPath = path.join(migrationsPath, '.snapshot.json');
+  const exists = await fileExists(snapshotPath);
+
+  if (!exists) return {};
+
+  return readFile(snapshotPath, { json: true });
+}
+
+export function diffSnapshots(previous, current) {
+  const addedModels = [];
+  const removedModels = [];
+  const addedColumns = [];
+  const removedColumns = [];
+  const changedColumns = [];
+  const addedForeignKeys = [];
+  const removedForeignKeys = [];
+
+  // Find added models
+  for (const name of Object.keys(current)) {
+    if (!previous[name]) addedModels.push(name);
+  }
+
+  // Find removed models
+  for (const name of Object.keys(previous)) {
+    if (!current[name]) removedModels.push(name);
+  }
+
+  // Find column changes in existing models
+  for (const name of Object.keys(current)) {
+    if (!previous[name]) continue;
+
+    const { columns: prevCols = {} } = previous[name];
+    const { columns: currCols = {} } = current[name];
+
+    // Added columns
+    for (const [col, type] of Object.entries(currCols)) {
+      if (!prevCols[col]) {
+        addedColumns.push({ model: name, column: col, type });
+      } else if (prevCols[col] !== type) {
+        changedColumns.push({ model: name, column: col, from: prevCols[col], to: type });
+      }
+    }
+
+    // Removed columns
+    for (const [col, type] of Object.entries(prevCols)) {
+      if (!currCols[col]) {
+        removedColumns.push({ model: name, column: col, type });
+      }
+    }
+
+    // Foreign key changes
+    const prevFKs = previous[name].foreignKeys || {};
+    const currFKs = current[name].foreignKeys || {};
+
+    for (const [col, refs] of Object.entries(currFKs)) {
+      if (!prevFKs[col]) {
+        addedForeignKeys.push({ model: name, column: col, references: refs });
+      }
+    }
+
+    for (const [col, refs] of Object.entries(prevFKs)) {
+      if (!currFKs[col]) {
+        removedForeignKeys.push({ model: name, column: col, references: refs });
+      }
+    }
+  }
+
+  const hasChanges = addedModels.length > 0 || removedModels.length > 0 ||
+    addedColumns.length > 0 || removedColumns.length > 0 ||
+    changedColumns.length > 0 || addedForeignKeys.length > 0 || removedForeignKeys.length > 0;
+
+  return {
+    hasChanges,
+    addedModels,
+    removedModels,
+    addedColumns,
+    removedColumns,
+    changedColumns,
+    addedForeignKeys,
+    removedForeignKeys,
+  };
+}
+
+export function detectSchemaDrift(schemas, snapshot) {
+  const current = schemasToSnapshot(schemas);
+  return diffSnapshots(snapshot, current);
+}
+
+export function extractViewsFromSnapshot(snapshot) {
+  const views = {};
+  for (const [name, entry] of Object.entries(snapshot)) {
+    if (entry.isView) views[name] = entry;
+  }
+  return views;
+}
+
+export function diffViewSnapshots(previous, current) {
+  const addedViews = [];
+  const removedViews = [];
+  const changedViews = [];
+
+  for (const name of Object.keys(current)) {
+    if (!previous[name]) {
+      addedViews.push(name);
+    } else if (
+      current[name].viewQuery !== previous[name].viewQuery ||
+      current[name].source !== previous[name].source
+    ) {
+      changedViews.push(name);
+    }
+  }
+
+  for (const name of Object.keys(previous)) {
+    if (!current[name]) {
+      removedViews.push(name);
+    }
+  }
+
+  const hasChanges = addedViews.length > 0 || removedViews.length > 0 || changedViews.length > 0;
+
+  return { hasChanges, addedViews, removedViews, changedViews };
+}

package/src/postgres/migration-runner.js

@@ -0,0 +1,109 @@
+import { readFile, fileExists } from '@stonyx/utils/file';
+import path from 'path';
+import fs from 'fs/promises';
+
+export async function ensureMigrationsTable(pool, tableName = '__migrations') {
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS "${tableName}" (
+      id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+      filename VARCHAR(255) NOT NULL UNIQUE,
+      applied_at TIMESTAMPTZ DEFAULT NOW()
+    )
+  `);
+}
+
+export async function getAppliedMigrations(pool, tableName = '__migrations') {
+  const result = await pool.query(
+    `SELECT filename FROM "${tableName}" ORDER BY id ASC`
+  );
+
+  return result.rows.map(row => row.filename);
+}
+
+export async function getMigrationFiles(migrationsDir) {
+  const exists = await fileExists(migrationsDir);
+  if (!exists) return [];
+
+  const entries = await fs.readdir(migrationsDir);
+
+  return entries
+    .filter(f => f.endsWith('.sql'))
+    .sort();
+}
+
+export function parseMigrationFile(content) {
+  const upMarker = '-- UP';
+  const downMarker = '-- DOWN';
+  const upIndex = content.indexOf(upMarker);
+  const downIndex = content.indexOf(downMarker);
+
+  if (upIndex === -1) {
+    return { up: content.trim(), down: '' };
+  }
+
+  const upStart = upIndex + upMarker.length;
+  const upEnd = downIndex !== -1 ? downIndex : content.length;
+  const up = content.slice(upStart, upEnd).trim();
+  const down = downIndex !== -1 ? content.slice(downIndex + downMarker.length).trim() : '';
+
+  return { up, down };
+}
+
+export async function applyMigration(pool, filename, upSql, tableName = '__migrations') {
+  const client = await pool.connect();
+
+  try {
+    await client.query('BEGIN');
+
+    const statements = splitStatements(upSql);
+
+    for (const stmt of statements) {
+      await client.query(stmt);
+    }
+
+    await client.query(
+      `INSERT INTO "${tableName}" (filename) VALUES ($1)`,
+      [filename]
+    );
+
+    await client.query('COMMIT');
+  } catch (error) {
+    await client.query('ROLLBACK');
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+
+export async function rollbackMigration(pool, filename, downSql, tableName = '__migrations') {
+  const client = await pool.connect();
+
+  try {
+    await client.query('BEGIN');
+
+    const statements = splitStatements(downSql);
+
+    for (const stmt of statements) {
+      await client.query(stmt);
+    }
+
+    await client.query(
+      `DELETE FROM "${tableName}" WHERE filename = $1`,
+      [filename]
+    );
+
+    await client.query('COMMIT');
+  } catch (error) {
+    await client.query('ROLLBACK');
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+
+function splitStatements(sql) {
+  return sql
+    .split(';')
+    .map(s => s.trim())
+    .filter(s => s.length > 0 && !s.startsWith('--'));
+}