masterrecord 0.1.3 → 0.2.0

package/Entity/entityTrackerModel.js

@@ -1,5 +1,5 @@
 
-// version : 0.0.7
+// version : 0.0.9
 var tools =  require('../Tools');
 class EntityTrackerModel {
 
@@ -29,6 +29,10 @@ class EntityTrackerModel {
                     set: function(value) {
                         modelClass.__state = "modified";
                         modelClass.__dirtyFields.push(modelField);
+                        // ensure this entity is tracked on any modification
+                        if(modelClass.__context && typeof modelClass.__context.__track === 'function'){
+                            modelClass.__context.__track(modelClass);
+                        }
                         if(typeof currentEntity[modelField].set === "function"){
                             this["__proto__"]["_" + modelField] = currentEntity[modelField].set(value);
                         }else{
@@ -157,7 +161,7 @@ class EntityTrackerModel {
                                                 var entityFieldJoinName = currentEntity[entityField].foreignTable === undefined? entityField : currentEntity[entityField].foreignTable;
                                                 var thirdEntity = this.__context[tools.capitalize(entityFieldJoinName)];
                                                 var firstJoiningID = joiningEntity.__entity[this.__entity.__name].foreignTable;
-                                                var secondJoiningID = joiningEntity.__entity[entityField].foreignTable;
+                                                var secondJoiningID = Object.values(joiningEntity.__entity).find(e => e.foreignTable === ent.__name);
                                                 if(firstJoiningID && secondJoiningID )
                                                 {
                                                     var modelValue = ent.include(`p => p.${entityFieldJoinName}.select(j => j.${joiningEntity.__entity[this.__entity.__name].foreignKey})`).include(`p =>p.${this.__entity.__name}`).where(`r =>r.${this.__entity.__name}.${priKey} = ${this[priKey]}`).toList();
@@ -234,4 +238,4 @@ class EntityTrackerModel {
 
 }
 
-module.exports = EntityTrackerModel
+module.exports = EntityTrackerModel;

package/MIGRATIONS.md

@@ -0,0 +1,178 @@
+### Migrations and Server Update Guide
+
+This project ships a CLI, exposed as `masterrecord`, to manage database migrations. Below are the steps to enable migrations, create migrations, apply them, and update your running server.
+
+### 1) Install the CLI (local repo checkout)
+
+- From the project root, install the CLI globally:
+```bash
+npm install -g ./
+```
+
+After install, the `masterrecord` command becomes available in your shell.
+
+### 2) Prepare your Context and Environment
+
+- Ensure your app has a Context class that extends `context` and configures a DB connection (SQLite or MySQL) using either `useSqlite()` or `useMySql()`.
+- Set the environment via the `master` env var when running commands, e.g. `master=development` or `master=production`.
+- Provide environment JSON at `env.<ENV>.json` reachable from your app root, keyed by your Context class name. Example for SQLite:
+```json
+{
+  "AppContext": {
+    "type": "better-sqlite3",
+    "connection": "/db/app.sqlite"
+  }
+}
+```
+Example for MySQL:
+```json
+{
+  "AppContext": {
+    "type": "mysql",
+    "host": "localhost",
+    "user": "root",
+    "password": "secret",
+    "database": "app_db"
+  }
+}
+```
+
+### 3) Enable migrations (one-time per Context)
+
+- Run from the project root where your Context file lives. Use the Context file name (without extension) as the argument.
+```bash
+master=development masterrecord enable-migrations AppContext
+```
+This creates `db/migrations/<context>_contextSnapShot.json` and the `db/migrations` directory.
+
+### 4) Create a migration
+
+- After you change your entity models, generate a migration file:
+```bash
+master=development masterrecord add-migration <MigrationName> AppContext
+```
+This writes a new file to `db/migrations/<timestamp>_<MigrationName>_migration.js`.
+
+### 5) Apply migrations to the database
+
+- Apply only the latest pending migration:
+```bash
+master=development masterrecord update-database AppContext
+```
+- Apply all migrations from the beginning (useful for a clean DB):
+```bash
+master=development masterrecord update-database-restart AppContext
+```
+- List migration files (debug/inspection):
+```bash
+master=development masterrecord get-migrations AppContext
+```
+
+Notes:
+- The CLI searches for `<context>_contextSnapShot.json` under `db/migrations` relative to your current working directory.
+- For MySQL, ensure your credentials allow DDL. For SQLite, the data directory is created if missing.
+
+### 6) Updating the running server
+
+General flow to roll out schema changes:
+- Stop the server or put it into maintenance mode (optional but recommended for non-backward-compatible changes).
+- Pull the latest code (containing updated models and generated migration files).
+- Run migrations against the target environment:
+```bash
+master=production masterrecord update-database AppContext
+```
+- Restart your server/process manager (e.g., `pm2 restart <app>`, `docker compose up -d`, or your platform’s restart command).
+
+Backward-compatible rollout tip:
+- If possible, deploy additive changes first (new tables/columns), release app code that begins using them, then later clean up/removal migrations.
+
+### Troubleshooting
+
+- Cannot find Context file: ensure you run commands from the app root and pass the correct Context file name used when defining your class (case-insensitive in the snapshot, but supply the same name you used).
+- Cannot connect to DB: confirm `master=<env>` is set and `env.<env>.json` exists with correct credentials and paths.
+- MySQL type mismatches: the migration engine maps MasterRecord types to SQL types; verify your entity field `type` values are correct.
+
+### Recent improvements (2025-09)
+
+- Query language and SQL engines:
+  - Correct parsing of multi-char operators (>=, <=, ===, !==) and spaced logical operators.
+  - Support for grouped OR conditions rendered as parenthesized OR in WHERE across SQLite/MySQL.
+  - Resilient fallback for partially parsed expressions.
+- Relationships:
+  - `hasManyThrough` supported in insert and delete cascades.
+- Environment file discovery:
+  - Context now walks up directories to find `config/environments/env.<env>.json`; fixed error throwing.
+- Migrations (DDL generation):
+  - Default values emitted for SQLite/MySQL (including boolean coercion).
+  - `CREATE TABLE IF NOT EXISTS` to avoid failures when rerunning.
+  - Table introspection added; existing tables are synced: missing columns are added, MySQL applies `ALTER ... MODIFY` for NULL/DEFAULT changes, SQLite rebuilds table when necessary.
+- Migration API additions in `schema.js`:
+  - `renameColumn(table)` implemented for SQLite/MySQL.
+  - `seed(tableName, rows)` implemented for bulk/single inserts with safe quoting.
+
+### Using renameColumn and seed in migrations
+
+Basic migration skeleton (generated by CLI):
+```js
+var masterrecord = require('masterrecord');
+
+class AddSettings extends masterrecord.schema { 
+  constructor(context){ super(context); }
+
+  up(table){
+    this.init(table);
+    // Add a new table
+    this.createTable(table.MailSettings);
+
+    // Rename a column on an existing table
+    this.renameColumn({ tableName: 'MailSettings', name: 'from_email', newName: 'reply_to' });
+
+    // Seed initial data (single row)
+    this.seed('MailSettings', {
+      from_name: 'System',
+      reply_to: 'no-reply@example.com',
+      return_path_matches_from: 0,
+      weekly_summary_enabled: 0,
+      created_at: Date.now(),
+      updated_at: Date.now()
+    });
+
+    // Seed multiple rows
+    this.seed('MailSettings', [
+      { from_name: 'Support', reply_to: 'support@example.com', created_at: Date.now(), updated_at: Date.now() },
+      { from_name: 'Marketing', reply_to: 'marketing@example.com', created_at: Date.now(), updated_at: Date.now() }
+    ]);
+  }
+
+  down(table){
+    this.init(table);
+    // Revert the rename
+    this.renameColumn({ tableName: 'MailSettings', name: 'reply_to', newName: 'from_email' });
+
+    // Optionally clean up seeded rows
+    // this.context._execute("DELETE FROM MailSettings WHERE reply_to IN ('no-reply@example.com','support@example.com','marketing@example.com')");
+
+    // Drop table if that was part of up
+    // this.dropTable(table.MailSettings);
+  }
+}
+module.exports = AddSettings;
+```
+
+Notes:
+- `renameColumn` expects an object: `{ tableName, name, newName }` and works in both SQLite and MySQL.
+- `seed(tableName, rows)` accepts:
+  - a single object: `{ col: value, ... }`
+  - or an array of objects: `[{...}, {...}]`
+  Values are auto-quoted; booleans become 1/0.
+- When a table already exists, `update-database` will sync schema:
+  - Add missing columns.
+  - MySQL: adjust default/nullability via `ALTER ... MODIFY`.
+  - SQLite: rebuilds the table when nullability/default/type changes require it.
+
+### Tips
+- Prefer additive changes (add columns) before destructive changes (drops/renames) to minimize downtime.
+- For large SQLite tables, a rebuild copies data; consider maintenance windows.
+- Use `master=development masterrecord get-migrations AppContext` to inspect migration order.
+
+

package/Migrations/cli.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 
-// version 0.0.4
+// version 0.0.5
 // https://docs.microsoft.com/en-us/ef/ef6/modeling/code-first/migrations/
 // how to add environment variables on cli call example - master=development masterrecord add-migration auth authContext
 
-const program = require('commander');
+const { program } = require('commander');
 let fs = require('fs');
 let path = require('path');
 var Migration = require('./migrations');
@@ -16,7 +16,7 @@ const [,, ...args] = process.argv
 
 
 program
-  .version('0.0.3', '-v, --version', '0.0.3')
+  .version('0.0.3')
   .description('A ORM framework that facilitates the creation and use of business objects whose data requires persistent storage to a database');
 
   // Instructions : to run command you must go to main project folder is located and run the command using the context file name.

package/Migrations/migrationMySQLQuery.js

@@ -1,5 +1,5 @@
 
-// verison 0.0.2
+// verison 0.0.4
 class migrationMySQLQuery {
 
     #tempTableName = "_temp_alter_column_update"
@@ -37,14 +37,24 @@ class migrationMySQLQuery {
         var unique = table.unique ? " UNIQUE" : "";
         var type = this.typeManager(table.type);
         var tableName = table.name;
-        var defaultValue  = "";
-        if(table.default != null){
-
-            defaultValue = ` DEFAULT ${this.boolType(table.default)}`
-        }
-        if(table.relationshipType === 'belongsTo'){
+        if(table.relationshipType === 'belongsTo' && table.foreignKey){
             tableName = table.foreignKey;
         }
+        var defaultValue  = "";
+        if(table.default !== undefined && table.default !== null){
+            let def = table.default;
+            if(table.type === 'boolean'){
+                def = this.boolType(def);
+                defaultValue = ` DEFAULT ${def}`;
+            }
+            else if(table.type === 'integer' || table.type === 'float' || table.type === 'decimal'){
+                defaultValue = ` DEFAULT ${def}`;
+            }
+            else{
+                const esc = String(def).replace(/'/g, "''");
+                defaultValue = ` DEFAULT '${esc}'`;
+            }
+        }
 
         return `${tableName} ${type}${nullName}${defaultValue}${unique}${primaryKey}${auto}`;
     }
@@ -176,7 +186,7 @@ class migrationMySQLQuery {
             }
         }
 
-        var completeQuery = `CREATE TABLE ${table.__name} (${queryVar.replace(/,\s*$/, "")});`;
+        var completeQuery = `CREATE TABLE IF NOT EXISTS ${table.__name} (${queryVar.replace(/,\s*$/, "")});`;
         return completeQuery;
 
             /*

package/Migrations/migrationSQLiteQuery.js

@@ -1,5 +1,5 @@
 
-// verison 0.0.5
+// verison 0.0.7
 class migrationSQLiteQuery {
 
     #tempTableName = "_temp_alter_column_update"
@@ -36,8 +36,28 @@ class migrationSQLiteQuery {
         var nullName = table.nullable ? "" : " NOT NULL";
         var unique = table.unique ? " UNIQUE" : "";
         var type = this.#typeManager(table.type);
+        var colName = table.name;
+        if(table.relationshipType === 'belongsTo' && table.foreignKey){
+            colName = table.foreignKey;
+        }
+        // DEFAULT clause
+        var defaultClause = "";
+        if(table.default !== undefined && table.default !== null){
+            let def = table.default;
+            if(table.type === 'boolean'){
+                def = (def === true || def === 'true') ? 1 : 0;
+                defaultClause = ` DEFAULT ${def}`;
+            }
+            else if(table.type === 'integer' || table.type === 'float' || table.type === 'decimal'){
+                defaultClause = ` DEFAULT ${def}`;
+            }
+            else{
+                const esc = String(def).replace(/'/g, "''");
+                defaultClause = ` DEFAULT '${esc}'`;
+            }
+        }
 
-        return `${table.name} ${type}${nullName}${unique}${primaryKey}${auto}`;
+        return `${colName} ${type}${nullName}${defaultClause}${unique}${primaryKey}${auto}`;
     }
 
     #typeManager(type){
@@ -75,6 +95,13 @@ class migrationSQLiteQuery {
 
 
     addColum(table){
+        // If a full column spec is provided, map it to a proper SQLite column definition
+        if(table.column){
+            const def = this.#columnMapping(table.column);
+            return `ALTER TABLE ${table.tableName}
+        ADD COLUMN ${def}`;
+        }
+        // Fallback legacy behavior: raw name provided must include full definition if caller wants type/constraints
         return `ALTER TABLE ${table.tableName}
         ADD COLUMN ${table.name}`;
 
@@ -111,7 +138,7 @@ class migrationSQLiteQuery {
             }
         }
     
-        return `CREATE TABLE ${table.__name} (${queryVar.replace(/,\s*$/, "")});`;
+        return `CREATE TABLE IF NOT EXISTS ${table.__name} (${queryVar.replace(/,\s*$/, "")});`;
 
             /*
                 INTEGER PRIMARY KEY AUTOINCREMENT

package/Migrations/schema.js

@@ -1,4 +1,4 @@
-// version 0.0.4
+// version 0.0.5
 class schema{
 
     constructor(context){
@@ -65,24 +65,175 @@ class schema{
     createTable(table){
 
         if(table){
-            if(this.context.isSQLite){
-                var sqliteQuery = require("./migrationSQLiteQuery");
-                var queryBuilder = new sqliteQuery();
-                var query = queryBuilder.createTable(table);
-                this.context._execute(query);
-            }
+            // If table exists, run sync instead of blind create
+            const tableName = table.__name;
+            if(this.context._SQLEngine.tableExists && this.context._SQLEngine.tableExists(tableName)){
+                this.syncTable(table);
+            } else {
+                if(this.context.isSQLite){
+                    var sqliteQuery = require("./migrationSQLiteQuery");
+                    var queryBuilder = new sqliteQuery();
+                    var query = queryBuilder.createTable(table);
+                    this.context._execute(query);
+                }
 
-            if(this.context.isMySQL){
-                var sqlquery = require("./migrationMySQLQuery");
-                var queryBuilder = new sqlquery();
-                var query = queryBuilder.createTable(table);
-                this.context._execute(query);
+                if(this.context.isMySQL){
+                    var sqlquery = require("./migrationMySQLQuery");
+                    var queryBuilder = new sqlquery();
+                    var query = queryBuilder.createTable(table);
+                    this.context._execute(query);
+                }
             }
         }else{
             console.log("Table that your trying to create is undefined. PLease check if there are any changes that need to be made");
         }
     }
 
+    // Compute diffs and apply minimal changes
+    syncTable(table){
+        const engine = this.context._SQLEngine;
+        const tableName = table.__name;
+        const existing = engine.getTableInfo ? engine.getTableInfo(tableName) : [];
+        // Build a set of existing columns (sqlite: name, mysql: name)
+        const existingNames = new Set((existing || []).map(c => (c.name || c.COLUMN_NAME))); // both engines map to name
+        // Add missing columns only (safe path)
+        for (var key in table) {
+            if(typeof table[key] === 'object'){
+                const col = table[key];
+                // Skip relationships
+                if(col.type === 'hasOne' || col.type === 'hasMany' || col.type === 'hasManyThrough') continue;
+                const colName = (col.relationshipType === 'belongsTo' && col.foreignKey) ? col.foreignKey : col.name;
+                if(!existingNames.has(colName)){
+                    // add column
+                    const newCol = {
+                        tableName: tableName,
+                        name: colName,
+                        type: col.type
+                    };
+                    // MySQL path uses addColum with realDataType
+                    if(this.context.isSQLite){
+                        var sqliteQuery = require("./migrationSQLiteQuery");
+                        var queryBuilder = new sqliteQuery();
+                        // Build a conservative column add (no NOT NULL without default)
+                        const add = queryBuilder.addColum({ tableName, name: colName });
+                        this.context._execute(add);
+                    }
+                    if(this.context.isMySQL){
+                        var sqlquery = require("./migrationMySQLQuery");
+                        var queryBuilder = new sqlquery();
+                        newCol.realDataType = queryBuilder.typeManager(col.type);
+                        const query = queryBuilder.addColum(newCol);
+                        this.context._execute(query);
+                    }
+                }
+            }
+        }
+        // Detect modifications (nullable/default/type)
+        const desiredCols = [];
+        for (var key in table) {
+            if(typeof table[key] === 'object'){
+                const col = table[key];
+                if(col.type === 'hasOne' || col.type === 'hasMany' || col.type === 'hasManyThrough') continue;
+                const colName = (col.relationshipType === 'belongsTo' && col.foreignKey) ? col.foreignKey : col.name;
+                desiredCols.push({ name: colName, col });
+            }
+        }
+
+        const needRebuildSQLite = () => {
+            if(!this.context.isSQLite) return false;
+            const byName = {};
+            for(const row of existing){ byName[row.name] = row; }
+            for(const d of desiredCols){
+                const row = byName[d.name];
+                if(!row) continue;
+                const notnull = row.notnull === 1;
+                const desiredNotNull = d.col.nullable === false;
+                const desiredType = d.col.type;
+                const existingType = (row.type || '').toLowerCase();
+                // compare default (normalize quotes)
+                const exDefRaw = row.dflt_value == null ? null : String(row.dflt_value);
+                let exDef = exDefRaw;
+                if(typeof exDef === 'string' && exDef.length >= 2 && exDef.startsWith("'") && exDef.endsWith("'")){
+                    exDef = exDef.slice(1, -1);
+                }
+                const dsDef = d.col.default == null ? null : String(d.col.default);
+                if(desiredNotNull !== notnull) return true;
+                if(exDef !== dsDef) return true;
+                // rough type differences that require rebuild
+                if((desiredType === 'boolean' && existingType !== 'integer') ||
+                   (desiredType === 'string' && existingType !== 'text') ||
+                   (desiredType === 'integer' && existingType !== 'integer')){
+                    return true;
+                }
+            }
+            return false;
+        };
+
+        if(this.context.isMySQL){
+            // Apply MODIFY for defaults/nullability
+            var sqlquery = require("./migrationMySQLQuery");
+            var queryBuilder = new sqlquery();
+            const byName = {};
+            for(const row of existing){ byName[row.name || row.COLUMN_NAME] = row; }
+            for(const d of desiredCols){
+                const row = byName[d.name];
+                if(!row) continue;
+                const desiredNotNull = d.col.nullable === false;
+                const existingNullable = (row.is_nullable || row.IS_NULLABLE || '').toString().toUpperCase() === 'YES';
+                // default normalize
+                const dsDef = d.col.default;
+                let exDef2 = row.dflt_value || row.COLUMN_DEFAULT;
+                if(typeof exDef2 === 'string' && exDef2.length >= 2 && exDef2.startsWith("'") && exDef2.endsWith("'")){
+                    exDef2 = exDef2.slice(1, -1);
+                }
+                const differsNull = (desiredNotNull === true && existingNullable === true) || (desiredNotNull !== true && existingNullable === false);
+                const differsDef = (dsDef ?? null) !== (exDef2 ?? null);
+                if(differsNull || differsDef){
+                    const type = queryBuilder.typeManager(d.col.type);
+                    const nullPart = desiredNotNull ? 'NOT NULL' : 'NULL';
+                    let defPart = '';
+                    if(dsDef !== undefined && dsDef !== null){
+                        if(d.col.type === 'boolean'){
+                            defPart = ` DEFAULT ${queryBuilder.boolType(dsDef)}`;
+                        } else if(d.col.type === 'integer' || d.col.type === 'float' || d.col.type === 'decimal'){
+                            defPart = ` DEFAULT ${dsDef}`;
+                        } else {
+                            const esc = String(dsDef).replace(/'/g, "''");
+                            defPart = ` DEFAULT '${esc}'`;
+                        }
+                    } else {
+                        defPart = ' DEFAULT NULL';
+                    }
+                    const alter = `ALTER TABLE ${tableName} MODIFY COLUMN ${d.name} ${type} ${nullPart}${defPart}`;
+                    this.context._execute(alter);
+                }
+            }
+        }
+
+        if(needRebuildSQLite()){
+            var sqliteQuery = require("./migrationSQLiteQuery");
+            var queryBuilder = new sqliteQuery();
+            // rename old table
+            const rename = queryBuilder.renameTable({ tableName, newName: "_temp_alter_column_update" });
+            this.context._execute(rename);
+            // create new with desired schema
+            const create = queryBuilder.createTable(table);
+            this.context._execute(create);
+            // compute common columns
+            const oldInfo = engine.getTableInfo(tableName.replace(/.*/, '_temp_alter_column_update')) || engine.getTableInfo("_temp_alter_column_update");
+            const oldNames = new Set((oldInfo || existing).map(r => r.name));
+            const newNames = desiredCols.map(d => d.name);
+            const common = newNames.filter(n => oldNames.has(n));
+            if(common.length > 0){
+                const cols = common.join(',');
+                const insert = `INSERT INTO ${tableName} (${cols}) SELECT ${cols} FROM _temp_alter_column_update`;
+                this.context._execute(insert);
+            }
+            const drop = queryBuilder.dropTable("_temp_alter_column_update");
+            this.context._execute(drop);
+        }
+    }
+
 
     dropTable(table){
         if(table){
@@ -130,12 +281,46 @@ class schema{
         }
     }
 
-    renameColumn(){
-        // TODO
+    renameColumn(table){
+        if(table){
+            if(this.context.isSQLite){
+                var sqliteQuery = require("./migrationSQLiteQuery");
+                var queryBuilder = new sqliteQuery();
+                var query = queryBuilder.renameColumn(table);
+                this.context._execute(query);
+            }
+
+            if(this.context.isMySQL){
+                var sqlquery = require("./migrationMySQLQuery");
+                var queryBuilder = new sqlquery();
+                var query = queryBuilder.renameColumn(table);
+                this.context._execute(query);
+            }
+        }
     }
 
-    seed(){
-        // TODO
+    seed(tableName, rows){
+        if(!tableName || !rows){ return; }
+        const items = Array.isArray(rows) ? rows : [rows];
+        for(const row of items){
+            const cols = Object.keys(row);
+            if(cols.length === 0){ continue; }
+            const colList = cols.map(c => this.context.isSQLite ? `[${c}]` : `${c}`).join(", ");
+            const vals = cols.map(k => {
+                const v = row[k];
+                if(v === null || v === undefined){ return 'NULL'; }
+                if(typeof v === 'boolean'){
+                    return this.context.isSQLite ? (v ? 1 : 0) : (v ? 1 : 0);
+                }
+                if(typeof v === 'number'){
+                    return String(v);
+                }
+                const esc = String(v).replace(/'/g, "''");
+                return `'${esc}'`;
+            }).join(", ");
+            const sql = `INSERT INTO ${tableName} (${colList}) VALUES (${vals})`;
+            this.context._execute(sql);
+        }
     }
     
 }