millas 0.2.12-beta-1 → 0.2.13-beta

package/src/orm/migration/ModelInspector.js

@@ -3,6 +3,8 @@
 const fs = require('fs-extra');
 const path = require('path');
 const MillasLog = require('../../logger/internal');
+const { walkJs, extractClasses, isMillasModel, fieldsEqual } = require('./utils');
+const { normaliseField } = require('./ProjectState');
 
 /**
  * ModelInspector
@@ -32,6 +34,85 @@ class ModelInspector {
         this._snapshotPath = snapshotPath || path.join(process.cwd(), '.millas', 'schema.json');
     }
 
+    /**
+     * The baseline schema that each system migration creates.
+     * Keyed by table name → field definitions (same shape as _extractFields output).
+     *
+     * When makemigrations encounters a system-owned table for the first time
+     * (no snapshot entry yet), it seeds the snapshot from this baseline rather
+     * than from the current model. That ensures any fields the developer added
+     * beyond the baseline are detected as add_column diffs — not silently ignored.
+     *
+     * Keep in sync with src/migrations/system/000*.js.
+     */
+    static get SYSTEM_BASELINES() {
+        // Lazy-load AuthUser so this file has no hard dependency at module load time.
+        // The getter is only called during makeMigrations(), never at require() time.
+        const AuthUser = require('../../auth/AuthUser');
+        const { fields } = require('../fields/index');
+
+        const extractFields = (fieldsMap) => {
+            const result = {};
+            for (const [name, field] of Object.entries(fieldsMap)) {
+                result[name] = {
+                    type:       field.type        ?? 'string',
+                    nullable:   field.nullable     ?? false,
+                    unique:     field.unique        ?? false,
+                    default:    field.default !== undefined ? field.default : null,
+                    max:        field.max          ?? null,
+                    unsigned:   field.unsigned      ?? false,
+                    enumValues: field.enumValues    ?? null,
+                    references: field.references    ?? null,
+                    precision:  field.precision     ?? null,
+                    scale:      field.scale         ?? null,
+                };
+            }
+            return result;
+        };
+
+        return {
+            // system/0001_users.js — mirrors AuthUser.fields exactly
+            users: extractFields(AuthUser.fields),
+
+            // system/0002_admin_log.js
+            millas_admin_log: extractFields({
+                id:         fields.id(),
+                user_id:    fields.integer({ unsigned: true, nullable: true }),
+                user_email: fields.string({ nullable: true }),
+                resource:   fields.string(),
+                record_id:  fields.string({ nullable: true }),
+                action:     fields.enum(['create', 'update', 'delete']),
+                label:      fields.string({ nullable: true }),
+                change_msg: fields.text({ nullable: true }),
+                created_at: fields.timestamp(),
+            }),
+
+            // system/0003_sessions.js
+            millas_sessions: extractFields({
+                session_key: fields.string({ max: 64 }),
+                user_id:     fields.integer({ unsigned: true }),
+                payload:     fields.text({ nullable: true }),
+                ip_address:  fields.string({ max: 45, nullable: true }),
+                user_agent:  fields.string({ max: 512, nullable: true }),
+                expires_at:  fields.timestamp(),
+                created_at:  fields.timestamp(),
+            }),
+
+            // millas_migrations — internal tracking table, not user-accessible
+            millas_migrations: extractFields({
+                id:    fields.id(),
+                name:  fields.string(),
+                pool:  fields.string({ max: 20 }),
+                batch: fields.integer(),
+            }),
+        };
+    }
+
+    /** Convenience: just the set of system table names. */
+    static get SYSTEM_TABLES() {
+        return new Set(Object.keys(ModelInspector.SYSTEM_BASELINES));
+    }
+
     /**
      * Detect changes and generate migration files.
      * Returns { files: string[], message: string }
@@ -75,34 +156,60 @@ class ModelInspector {
      */
     _scanModels() {
         const schema = {};
+        const tableToFile = {}; // track which file owns each table name
 
         if (!fs.existsSync(this._modelsPath)) return schema;
 
-        const files = fs.readdirSync(this._modelsPath)
-            .filter(f => f.endsWith('.js') && !f.startsWith('.') && f !== 'index.js');
+        const files = walkJs(this._modelsPath);
 
-        for (const file of files) {
-            const fullPath = path.join(this._modelsPath, file);
+        for (const fullPath of files) {
+            const file = path.basename(fullPath);
+            const relPath = path.relative(this._modelsPath, fullPath);
 
             // Always bust require cache so the inspector picks up edits made
             // in the same process (e.g. during tests).
             try {
                 delete require.cache[require.resolve(fullPath)];
-            } catch { /* path not yet cached — fine */
-            }
+            } catch { /* path not yet cached — fine */ }
 
             let exported;
-            exported = require(fullPath);
-
+            try {
+                exported = require(fullPath);
+            } catch (err) {
+                // Surface require errors so developers know why a model was skipped.
+                // Common causes: missing dependency, syntax error, bad import path.
+                MillasLog.warn(`[makemigrations] Skipping ${path.relative(this._modelsPath, fullPath)}: ${err.message}`);
+                process.stderr.write(`  ⚠  Could not load model: ${path.relative(this._modelsPath, fullPath)}\n     ${err.message}\n`);
+                continue;
+            }
 
             // Collect every candidate class from the export
-            const candidates = this._extractClasses(exported);
+            const candidates = extractClasses(exported);
 
             for (const ModelClass of candidates) {
-                if (!this._isMillasModel(ModelClass)) continue;
+                if (!isMillasModel(ModelClass)) continue;
 
                 const table = this._resolveTable(ModelClass, file);
-                schema[table] = this._extractFields(ModelClass.fields);
+
+                if (tableToFile[table] && tableToFile[table] !== relPath) {
+                    // Same table claimed by two files.
+                    // This is the inheritance pattern: User extends BaseUser, same table.
+                    // Keep the one with MORE fields — that's always the most derived class,
+                    // which has the complete column set for the table.
+                    const existingFieldCount = Object.keys(schema[table] || {}).length;
+                    const newFieldCount      = Object.keys(ModelClass.fields || {}).length;
+
+                    if (newFieldCount > existingFieldCount) {
+                        // New class is more derived — replace
+                        tableToFile[table] = relPath;
+                        schema[table] = this._extractFields(ModelClass.fields);
+                    }
+                    // Otherwise keep the existing (more derived) definition silently.
+                    // No warning — this is expected when extending a base model.
+                } else if (!tableToFile[table]) {
+                    tableToFile[table] = relPath;
+                    schema[table] = this._extractFields(ModelClass.fields);
+                }
             }
         }
 
@@ -110,49 +217,19 @@ class ModelInspector {
     }
 
     /**
-     * Given a module export (class, plain object, or anything), return an
-     * array of class/function values that might be Model subclasses.
+     * Recursively collect all .js files under a directory,
+     * excluding dotfiles and index.js at any depth.
      */
-    _extractClasses(exported) {
-        if (!exported) return [];
 
-        // Direct class export:  module.exports = MyModel
-        if (typeof exported === 'function') return [exported];
-
-        // Named export object:  module.exports = { MyModel, AnotherModel }
-        if (typeof exported === 'object') {
-            return Object.values(exported).filter(v => typeof v === 'function');
-        }
-
-        return [];
-    }
-
-    /**
-     * A class qualifies as a Millas Model if:
-     *   - It is a function (class)
-     *   - It has a static `fields` property that is a non-null object
-     *
-     * We intentionally do NOT do `instanceof` checks so the inspector
-     * works even when the user imports Model from a different resolution
-     * path than the one this file was loaded from.
-     */
-    _isMillasModel(cls) {
-        if (typeof cls !== 'function') return false;
-        if (!cls.fields || typeof cls.fields !== 'object') return false;
-        // Must have at least one field
-        return Object.keys(cls.fields).length > 0;
-    }
 
     /**
-     * Derive the table name from the model class or fall back to the file name.
+     * Derive the table name from the model class.
+     * Delegates to utils.resolveTable which respects abstract flag and convention.
+     * fileName fallback kept for backward compat with old snapshot entries.
      */
     _resolveTable(ModelClass, fileName) {
-        // Explicitly set static table = '...'
-        if (typeof ModelClass.table === 'string' && ModelClass.table) {
-            return ModelClass.table;
-        }
-        // Convention: file name without extension, pluralised, lowercased
-        return fileName.replace(/\.js$/, '').toLowerCase() + 's';
+        return resolveTable(ModelClass) ||
+          (fileName ? fileName.replace(/\.js$/, '').toLowerCase() + 's' : null);
     }
 
     /**
@@ -161,24 +238,11 @@ class ModelInspector {
      * snapshot storage and deterministic JSON comparison.
      */
     _extractFields(fields) {
+        // Delegate to normaliseField — single source of truth for field shape.
         const result = {};
-
         for (const [name, field] of Object.entries(fields)) {
-            // Normalise — accept both FieldDefinition instances and plain objects
-            result[name] = {
-                type: field.type ?? 'string',
-                nullable: field.nullable ?? false,
-                unique: field.unique ?? false,
-                default: field.default !== undefined ? field.default : null,
-                max: field.max ?? null,
-                unsigned: field.unsigned ?? false,
-                enumValues: field.enumValues ?? null,
-                references: field.references ?? null,
-                precision: field.precision ?? null,
-                scale: field.scale ?? null,
-            };
+            result[name] = normaliseField(field);
         }
-
         return result;
     }
 
@@ -190,13 +254,24 @@ class ModelInspector {
         // New tables (model added / first run)
         for (const table of Object.keys(current)) {
             if (!snapshot[table]) {
-                diffs.push({type: 'create_table', table, fields: current[table]});
+                if (ModelInspector.SYSTEM_TABLES.has(table)) {
+                    // System table — already created by a system migration.
+                    // Seed the snapshot from the KNOWN SYSTEM BASELINE (what the
+                    // migration actually created), NOT from the current model.
+                    // This ensures any extra fields the developer added beyond the
+                    // baseline are detected as add_column diffs below.
+                    snapshot[table] = ModelInspector.SYSTEM_BASELINES[table] || current[table];
+                } else {
+                    diffs.push({type: 'create_table', table, fields: current[table]});
+                }
             }
         }
 
         // Dropped tables (model file removed)
+        // Never generate drop_table for system tables — they are managed by
+        // system migrations, not by user model files.
         for (const table of Object.keys(snapshot)) {
-            if (!current[table]) {
+            if (!current[table] && !ModelInspector.SYSTEM_TABLES.has(table)) {
                 diffs.push({type: 'drop_table', table, fields: snapshot[table]});
             }
         }
@@ -225,7 +300,7 @@ class ModelInspector {
             // Changed columns — compare each attribute individually for stability
             for (const col of Object.keys(curr)) {
                 if (!prev[col]) continue; // new column — already handled above
-                if (!this._fieldsEqual(curr[col], prev[col])) {
+                if (!fieldsEqual(curr[col], prev[col])) {
                     diffs.push({
                         type: 'alter_column',
                         table,
@@ -244,13 +319,7 @@ class ModelInspector {
      * Stable field equality check that ignores key-ordering differences
      * which can appear when objects are reconstituted from JSON.
      */
-    _fieldsEqual(a, b) {
-        const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
-        for (const k of keys) {
-            if (JSON.stringify(a[k]) !== JSON.stringify(b[k])) return false;
-        }
-        return true;
-    }
+
 
     // ─── Migration generation ─────────────────────────────────────────────────
 
@@ -510,4 +579,4 @@ ${this._renderColumn('      ', diff.column, diff.previous, '.alter()')}
     }
 }
 
-module.exports = ModelInspector;
+module.exports = ModelInspector;

package/src/orm/migration/ModelScanner.js

@@ -0,0 +1,225 @@
+'use strict';
+
+const fs   = require('fs-extra');
+const path = require('path');
+const { ProjectState, normaliseField } = require('./ProjectState');
+const { walkJs, extractClasses, isMillasModel, resolveTable } = require('./utils');
+
+/**
+ * ModelScanner
+ *
+ * Scans model files from disk and builds a ProjectState representing
+ * the CURRENT state of the schema as defined in source code.
+ *
+ * This is the "current state" that makemigrations diffs against the
+ * "reconstructed state" from migration files.
+ *
+ * ── Model loading convention ──────────────────────────────────────────────────
+ *
+ * Models are loaded from a single entry point — app/models/index.js —
+ * which must export all models as named exports:
+ *
+ *   // app/models/index.js
+ *   module.exports = { User, Post, Comment, Order };
+ *
+ * This mirrors Django's pattern where each app's models/__init__.py
+ * explicitly imports and exposes every model class. The developer
+ * decides what is registered — ModelScanner never auto-discovers files.
+ *
+ * Sub-folders are supported as long as index.js re-exports everything:
+ *
+ *   // app/models/index.js
+ *   const { User }    = require('./auth/User');
+ *   const { Post }    = require('./content/Post');
+ *   module.exports    = { User, Post };
+ *
+ * ── Inheritance handling ──────────────────────────────────────────────────────
+ *
+ *   Abstract Base Class (static abstract = true):
+ *     - No table created
+ *     - Fields merged into every concrete subclass
+ *
+ *   Multi-table Inheritance (different static table from parent):
+ *     - Parent gets its own table (already in migration history)
+ *     - Child gets its own table
+ *     - A OneToOne FK from child → parent is auto-injected
+ *
+ *   Same-table inheritance (child overrides fields, same static table):
+ *     - Treated as one model — most-derived fields win
+ *
+ *   Proxy model (static proxy = true):
+ *     - No table created, no migration generated
+ */
+class ModelScanner {
+  constructor(modelsPath) {
+    this._modelsPath = modelsPath;
+  }
+
+  /**
+   * Scan all model files and return a ProjectState.
+   * Never touches the database.
+   */
+  scan() {
+    const state   = new ProjectState();
+    const classes = this._loadAllClasses();
+
+    // Two passes: first register concrete tables, then detect relationships
+    const tableToClass = new Map(); // table → most-derived class
+
+    for (const cls of classes) {
+      // Skip abstract and proxy models — no table
+      // Only skip if the class itself declares abstract/proxy (not inherited)
+      if (cls.hasOwnProperty('abstract') && cls.abstract) continue;
+      if (cls.hasOwnProperty('proxy') && cls.proxy) continue;
+
+      const table = resolveTable(cls);
+      if (!table) continue;
+
+      // Keep the most-derived class for each table
+      const existing = tableToClass.get(table);
+      if (!existing || this._fieldCount(cls) >= this._fieldCount(existing)) {
+        tableToClass.set(table, cls);
+      }
+    }
+
+    // Build state from the canonical (most-derived) class per table
+    for (const [table, cls] of tableToClass) {
+      const fields = this._resolveFields(cls, classes, tableToClass);
+      state.createModel(table, fields);
+    }
+
+    return state;
+  }
+
+  // ─── Internal ─────────────────────────────────────────────────────────────
+
+  /**
+   * Load all model classes from app/models/index.js.
+   *
+   * The index must export all models as named (or default) exports.
+   * ModelScanner never auto-discovers individual files — the developer
+   * controls what is registered, exactly like Django's models/__init__.py.
+   *
+   * Falls back to scanning individual .js files in the models directory
+   * only if index.js does not exist, so existing projects without an
+   * index.js keep working. A warning is printed to encourage migration.
+   */
+  _loadAllClasses() {
+    if (!fs.existsSync(this._modelsPath)) return [];
+
+    const indexPath = path.join(this._modelsPath, 'index.js');
+
+    // ── Primary path: load from index.js ────────────────────────────────
+    if (fs.existsSync(indexPath)) {
+      try { delete require.cache[require.resolve(indexPath)]; } catch {}
+      let exported;
+      try {
+        exported = require(indexPath);
+      } catch (err) {
+        process.stderr.write(
+          `  ✖  [makemigrations] Failed to load app/models/index.js: ${err.message}\n`
+        );
+        return [];
+      }
+      const classes = [];
+      const candidates = extractClasses(exported);
+      for (const cls of candidates) {
+        if (isMillasModel(cls)) classes.push(cls);
+      }
+      return classes;
+    }
+
+    // ── Fallback: walk individual files (legacy — no index.js yet) ───────
+    process.stderr.write(
+      `  ⚠  [makemigrations] No app/models/index.js found. ` +
+      `Create one that exports all your models to follow the recommended pattern.\n` +
+      `  Falling back to file scanning (slower, may miss models in subfolders).\n`
+    );
+
+    const classes = [];
+    const files   = walkJs(this._modelsPath);
+
+    for (const fullPath of files) {
+      try { delete require.cache[require.resolve(fullPath)]; } catch {}
+      let exported;
+      try {
+        exported = require(fullPath);
+      } catch (err) {
+        process.stderr.write(
+          `  ⚠  [makemigrations] Could not load ${path.relative(this._modelsPath, fullPath)}: ${err.message}\n`
+        );
+        continue;
+      }
+      const candidates = extractClasses(exported);
+      for (const cls of candidates) {
+        if (isMillasModel(cls)) classes.push(cls);
+      }
+    }
+
+    return classes;
+  }
+
+
+
+  _resolveFields(cls, allClasses, tableToClass) {
+    const fields = {};
+
+    // Walk the prototype chain collecting fields, child overrides parent
+    const chain = this._inheritanceChain(cls);
+
+    for (const ancestor of chain.reverse()) {
+      // Skip ancestors that have their own table (multi-table inheritance)
+      // unless they are the starting class itself
+      if (ancestor !== cls) {
+        const ancestorTable = resolveTable(ancestor);
+        if (ancestorTable && ancestorTable !== resolveTable(cls)) {
+          // Multi-table: inject OneToOne FK instead of merging fields
+          // The FK column name is `${ancestorTable.replace(/s$/, '')}_ptr`
+          const ptrCol = `${ancestorTable.replace(/s$/, '')}_ptr`;
+          fields[ptrCol] = normaliseField({
+            type:      'integer',
+            unsigned:  true,
+            nullable:  false,
+            unique:    true,
+            references: { table: ancestorTable, column: 'id', onDelete: 'CASCADE' },
+          });
+          continue;
+        }
+      }
+
+      // Merge fields from this ancestor (skip abstract flag, just take fields)
+      if (ancestor.fields && typeof ancestor.fields === 'object') {
+        for (const [name, def] of Object.entries(ancestor.fields)) {
+          if (def && typeof def === 'object' && def.type === 'm2m') continue; // skip M2M
+          // null = explicit removal, like Django's title = None — remove from merged set
+          if (def === null) { delete fields[name]; continue; }
+          // Django convention: ForeignKey declared as 'landlord' creates column 'landlord_id'.
+          // If already ends with _id, use as-is to avoid double-appending.
+          const colName = (def && def._isForeignKey && !name.endsWith('_id'))
+            ? name + '_id'
+            : name;
+          fields[colName] = normaliseField(def);
+        }
+      }
+    }
+
+    return fields;
+  }
+
+  _inheritanceChain(cls) {
+    const chain = [];
+    let current = cls;
+    while (current && current.fields !== undefined) {
+      chain.push(current);
+      current = Object.getPrototypeOf(current);
+    }
+    return chain; // child first, ancestors last
+  }
+
+  _fieldCount(cls) {
+    return cls.fields ? Object.keys(cls.fields).length : 0;
+  }
+
+}
+
+module.exports = ModelScanner;