@slingr/cli 0.0.2 → 0.0.4

package/dist/utils/jsonlLoader.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -7,15 +40,19 @@ exports.JsonlDatasetLoader = void 0;
 exports.discoverModels = discoverModels;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const node_path_1 = __importDefault(require("node:path"));
+const framework_backend_1 = require("@slingr/framework-backend");
 require("reflect-metadata");
+const cliProgress = __importStar(require("cli-progress"));
 /**
  * Utility class for loading datasets from JSONL files using Slingr model fromJSON() functionality
  */
 class JsonlDatasetLoader {
+    /** Per-run cache to avoid repeated DB lookups for the same referenced record. */
+    dbLookupCache = new Map();
     /**
      * Analyze model dependencies to determine the correct loading order
      */
-    analyzeDependencies(modelMap, verbose = false) {
+    analyzeDependencies(modelMap, verbose = false, availableModels) {
         const dependencies = {};
         // Initialize dependency tracking for all models
         for (const modelName of Object.keys(modelMap)) {
@@ -27,7 +64,7 @@ class JsonlDatasetLoader {
         }
         // Analyze each model for dependencies
         for (const [modelName, ModelClass] of Object.entries(modelMap)) {
-            const modelDeps = this.extractModelDependencies(ModelClass, verbose);
+            const modelDeps = this.extractModelDependencies(ModelClass, verbose, availableModels);
             dependencies[modelName].dependencies = modelDeps;
             // Update dependents for referenced models
             for (const depName of modelDeps) {
@@ -50,7 +87,7 @@ class JsonlDatasetLoader {
     /**
      * Extract dependencies from a model class by analyzing its decorators/metadata
      */
-    extractModelDependencies(ModelClass, verbose = false) {
+    extractModelDependencies(ModelClass, verbose = false, availableModels) {
         const dependencies = [];
         if (verbose) {
             console.log(`    🔍 Analyzing ${ModelClass.name} for dependencies...`);
@@ -60,9 +97,17 @@ class JsonlDatasetLoader {
             const instance = new ModelClass();
             // Get all property descriptors from the prototype and instance
             const proto = Object.getPrototypeOf(instance);
-            const propertyNames = Object.getOwnPropertyNames(proto).concat(Object.getOwnPropertyNames(instance));
+            const prototypeProps = Object.getOwnPropertyNames(proto).concat(Object.getOwnPropertyNames(instance));
+            // Also get fields from MODEL_FIELDS metadata (stores all decorated field names)
+            // This is critical for fields like @ReferenceField that may not have values assigned
+            const modelFields = Reflect.getMetadata?.('model:fields', ModelClass) || [];
+            // Combine and deduplicate property names
+            const propertyNames = [...new Set([...prototypeProps, ...modelFields])];
             if (verbose) {
                 console.log(`    Properties found:`, propertyNames.filter(p => p !== 'constructor'));
+                if (modelFields.length > 0) {
+                    console.log(`    Model fields from metadata:`, modelFields);
+                }
             }
             for (const propertyName of propertyNames) {
                 if (propertyName === 'constructor') {
@@ -87,46 +132,63 @@ class JsonlDatasetLoader {
                     }
                     // Check for Reference and Composition relationships
                     if (fieldType === 'relationship' && relationshipType && designType) {
-                        // For relationships, designType contains the name of the referenced model
-                        if (typeof designType === 'string') {
-                            dependencies.push(designType);
-                            if (verbose) {
-                                console.log(`    ✅ Found dependency: ${ModelClass.name}.${propertyName} → ${designType} (${relationshipType})`);
-                            }
-                        }
-                        else if (typeof designType === 'function' && designType.name) {
-                            dependencies.push(designType.name);
+                        // Skip regular compositions - in compositions, the child depends on the parent, not the other way around
+                        // The child's @OwnerReferenceField creates the dependency from child to parent
+                        if (relationshipType === 'composition') {
                             if (verbose) {
-                                console.log(`    ✅ Found dependency: ${ModelClass.name}.${propertyName} → ${designType.name} (${relationshipType})`);
+                                console.log(`    ⏭️  Skipping composition: ${ModelClass.name}.${propertyName} → ${designType?.name || designType} (child depends on parent, not parent on child)`);
                             }
+                            continue;
                         }
-                    }
-                    // Also check property getter/setter for reference information
-                    const descriptor = Object.getOwnPropertyDescriptor(instance, propertyName) ||
-                        Object.getOwnPropertyDescriptor(proto, propertyName);
-                    if (descriptor) {
-                        // Check if the property has reference metadata
-                        const refMetadata = Reflect.getMetadata?.('slingr:reference', instance, propertyName);
-                        if (refMetadata && refMetadata.elementType) {
-                            const referencedType = refMetadata.elementType();
-                            if (referencedType && referencedType.name) {
-                                dependencies.push(referencedType.name);
+                        // For references and sharedCompositions, check if the target has a JSONL file
+                        // Only add as dependency if the file exists (meaning it's loaded separately)
+                        const targetName = typeof designType === 'string' ? designType : designType.name;
+                        // If availableModels is provided, only add dependency if the model has a JSONL file
+                        // This allows compositions to be embedded OR loaded from separate files
+                        if (targetName) {
+                            const shouldAddDependency = !availableModels || availableModels.has(targetName);
+                            if (shouldAddDependency) {
+                                dependencies.push(targetName);
                                 if (verbose) {
-                                    console.log(`    Found reference dependency: ${ModelClass.name}.${propertyName} → ${referencedType.name}`);
+                                    console.log(`    ✅ Found dependency: ${ModelClass.name}.${propertyName} → ${targetName} (${relationshipType}, has JSONL file)`);
                                 }
                             }
+                            else if (verbose) {
+                                console.log(`    ⏭️  Skipping dependency: ${ModelClass.name}.${propertyName} → ${targetName} (${relationshipType}, no JSONL file - will use embedded data)`);
+                            }
                         }
-                        // Check composition metadata
-                        const compMetadata = Reflect.getMetadata?.('slingr:composition', instance, propertyName);
-                        if (compMetadata && compMetadata.elementType) {
-                            const composedType = compMetadata.elementType();
-                            if (composedType && composedType.name) {
-                                dependencies.push(composedType.name);
-                                if (verbose) {
-                                    console.log(`    Found composition dependency: ${ModelClass.name}.${propertyName} → ${composedType.name}`);
+                    }
+                    // Check field:type:options for elementType (handles array references and compositions)
+                    // This is the correct way to get the target class for relationships in the Slingr framework
+                    const fieldOptions = Reflect.getMetadata?.('field:type:options', proto, propertyName);
+                    if (fieldOptions?.elementType && typeof fieldOptions.elementType === 'function') {
+                        try {
+                            const targetClass = fieldOptions.elementType();
+                            if (targetClass?.name && !dependencies.includes(targetClass.name)) {
+                                // Skip if this is a regular composition field
+                                if (relationshipType === 'composition') {
+                                    if (verbose) {
+                                        console.log(`    ⏭️  Skipping composition via type: ${ModelClass.name}.${propertyName} → ${targetClass.name}`);
+                                    }
+                                }
+                                else {
+                                    // Only add dependency if the model has a JSONL file
+                                    const shouldAddDependency = !availableModels || availableModels.has(targetClass.name);
+                                    if (shouldAddDependency) {
+                                        dependencies.push(targetClass.name);
+                                        if (verbose) {
+                                            console.log(`    ✅ Found dependency via type: ${ModelClass.name}.${propertyName} → ${targetClass.name} (has JSONL file)`);
+                                        }
+                                    }
+                                    else if (verbose) {
+                                        console.log(`    ⏭️  Skipping dependency via type: ${ModelClass.name}.${propertyName} → ${targetClass.name} (no JSONL file - embedded)`);
+                                    }
                                 }
                             }
                         }
+                        catch {
+                            // elementType() can fail if referenced class is not loaded yet, ignore
+                        }
                     }
                 }
                 catch (error) {
@@ -145,6 +207,29 @@ class JsonlDatasetLoader {
         // Remove duplicates and self-references
         return Array.from(new Set(dependencies)).filter(dep => dep !== ModelClass.name);
     }
+    /**
+     * Get the referenced model name from field metadata
+     */
+    getReferencedModelName(ModelClass, fieldName) {
+        const designType = Reflect.getMetadata('design:type', ModelClass.prototype, fieldName);
+        if (designType && designType.name && designType.name !== 'Array') {
+            return designType.name;
+        }
+        // Try elementType for arrays
+        const fieldOptions = Reflect.getMetadata('field:type:options', ModelClass.prototype, fieldName);
+        if (fieldOptions?.elementType && typeof fieldOptions.elementType === 'function') {
+            try {
+                const targetClass = fieldOptions.elementType();
+                if (targetClass?.name) {
+                    return targetClass.name;
+                }
+            }
+            catch {
+                // Ignore
+            }
+        }
+        return null;
+    }
     /**
      * Perform topological sort to determine loading order
      */
@@ -154,6 +239,9 @@ class JsonlDatasetLoader {
         const result = [];
         const visit = (modelName) => {
             if (visiting.has(modelName)) {
+                console.error(`\n❌ Circular dependency detected!`);
+                console.error(`Currently visiting chain:`, Array.from(visiting));
+                console.error(`Trying to visit again:`, modelName);
                 throw new Error(`Circular dependency detected involving: ${modelName}`);
             }
             if (visited.has(modelName)) {
@@ -183,33 +271,99 @@ class JsonlDatasetLoader {
      * Load all JSONL files from a dataset directory
      */
     async loadDataset(options) {
-        const { datasetPath, modelMap, validateRecords = true, verbose = false } = options;
+        const { datasetPath, modelMap, validateRecords = true, verbose = false, includeModels, excludeModels, dataSource, } = options;
+        // Reset per-run cache so stale entries from a previous call don't bleed through.
+        this.dbLookupCache.clear();
         if (!(await fs_extra_1.default.pathExists(datasetPath))) {
             throw new Error(`Dataset directory not found: ${datasetPath}`);
         }
-        // Analyze dependencies first to determine correct loading order
-        const dependencyAnalysis = this.analyzeDependencies(modelMap, verbose);
-        // Find all JSONL files in the dataset directory
+        // Find all JSONL files in the dataset directory first
         const files = await fs_extra_1.default.readdir(datasetPath);
-        const jsonlFiles = files.filter(f => f.endsWith('.jsonl'));
+        let jsonlFiles = files.filter(f => f.endsWith('.jsonl'));
+        if (verbose) {
+            console.log(`📁 Found ${jsonlFiles.length} JSONL files in dataset directory:`, jsonlFiles.map(f => f.replace(/\.jsonl$/i, '')));
+        }
+        // Apply model filtering before dependency analysis (case-insensitive)
+        if (includeModels && includeModels.length > 0) {
+            // Create a case-insensitive lookup map: lowercase -> original model name
+            const includeLookup = new Map(includeModels.map(name => [name.toLowerCase(), name]));
+            if (verbose) {
+                console.log(`🔍 Filtering to include only:`, includeModels);
+            }
+            jsonlFiles = jsonlFiles.filter(f => {
+                const modelName = f.replace(/\.jsonl$/i, '');
+                const matched = includeLookup.has(modelName.toLowerCase());
+                if (verbose && !matched) {
+                    console.log(`  ⏭️  Skipping ${f} (not in include list)`);
+                }
+                return matched;
+            });
+            if (verbose) {
+                console.log(`✅ Filtered to ${jsonlFiles.length} files:`, jsonlFiles.map(f => f.replace(/\.jsonl$/i, '')));
+            }
+        }
+        else if (excludeModels && excludeModels.length > 0) {
+            // Create a case-insensitive lookup set
+            const excludeLookup = new Set(excludeModels.map(name => name.toLowerCase()));
+            if (verbose) {
+                console.log(`🚫 Filtering to exclude:`, excludeModels);
+            }
+            jsonlFiles = jsonlFiles.filter(f => {
+                const modelName = f.replace(/\.jsonl$/i, '');
+                const excluded = excludeLookup.has(modelName.toLowerCase());
+                if (verbose && excluded) {
+                    console.log(`  ⏭️  Excluding ${f}`);
+                }
+                return !excluded;
+            });
+            if (verbose) {
+                console.log(`✅ After exclusion: ${jsonlFiles.length} files:`, jsonlFiles.map(f => f.replace(/\.jsonl$/i, '')));
+            }
+        }
+        // Create a case-insensitive lookup for available models
+        const availableModels = new Set(jsonlFiles.map(f => f.replace(/\.jsonl$/i, '')));
+        const availableModelsLowercase = new Set(Array.from(availableModels).map(name => name.toLowerCase()));
+        // Filter modelMap to only include models that have JSONL files (case-insensitive)
+        const filteredModelMap = Object.fromEntries(Object.entries(modelMap).filter(([modelName]) => {
+            const matched = availableModelsLowercase.has(modelName.toLowerCase());
+            if (verbose && !matched) {
+                console.log(`  ⏭️  Skipping model ${modelName} (no corresponding JSONL file)`);
+            }
+            return matched;
+        }));
+        if (verbose) {
+            console.log(`📦 Filtered model map contains ${Object.keys(filteredModelMap).length} models:`, Object.keys(filteredModelMap));
+        }
+        // Analyze dependencies to determine correct loading order
+        const dependencyAnalysis = this.analyzeDependencies(filteredModelMap, verbose, availableModels);
+        if (verbose) {
+            console.log('\n📋 Loading order determined:', dependencyAnalysis.loadOrder.join(' → '));
+            console.log('');
+        }
         if (jsonlFiles.length === 0) {
             throw new Error(`No JSONL files found in dataset directory: ${datasetPath}`);
         }
         if (verbose) {
             console.log(`Found ${jsonlFiles.length} JSONL files to process:`, jsonlFiles);
         }
+        // Create a case-insensitive lookup: lowercase model name -> actual filename
+        const filenameLookup = new Map(jsonlFiles.map(filename => {
+            const modelName = filename.replace(/\.jsonl$/i, '');
+            return [modelName.toLowerCase(), filename];
+        }));
         const results = [];
         const loadedEntities = {};
         // Process files in dependency order instead of arbitrary order
         for (const modelName of dependencyAnalysis.loadOrder) {
-            const fileName = `${modelName}.jsonl`;
-            if (!jsonlFiles.includes(fileName)) {
+            // Find the actual filename (case-insensitive lookup)
+            const fileName = filenameLookup.get(modelName.toLowerCase());
+            if (!fileName) {
                 if (verbose) {
-                    console.log(`📄 No JSONL file found for model: ${modelName} (expected: ${fileName}). Skipping...`);
+                    console.log(`📄 No JSONL file found for model: ${modelName} (expected: ${modelName}.jsonl). Skipping...`);
                 }
                 continue;
             }
-            const ModelClass = modelMap[modelName];
+            const ModelClass = filteredModelMap[modelName];
             if (!ModelClass) {
                 if (verbose) {
                     console.warn(`No model mapping found for: ${modelName}. Skipping...`);
@@ -217,7 +371,8 @@ class JsonlDatasetLoader {
                 continue;
             }
             const filePath = node_path_1.default.join(datasetPath, fileName);
-            const result = await this.loadJsonlFile(filePath, ModelClass, modelName, loadedEntities, validateRecords, verbose);
+            const result = await this.loadJsonlFile(filePath, ModelClass, modelName, loadedEntities, validateRecords, verbose, dataSource, modelMap // Pass full modelMap for reference resolution
+            );
             // Store successfully loaded entities for future reference resolution
             if (result.records.length > 0) {
                 loadedEntities[modelName] = {};
@@ -236,7 +391,9 @@ class JsonlDatasetLoader {
      * Load a single JSONL file and convert records using the model's fromJSON method
      */
     /**
-     * Resolves simple references in format {"id": "uuid"} to full objects from loaded entities
+     * Resolves simple references in format {"id": "uuid"} to full objects from loaded entities.
+     * Only resolves compositions and sharedCompositions, NOT regular references (to avoid circular dependencies).
+     * Also handles arrays of compositions.
      */
     async resolveSimpleReferences(data, ModelClass, loadedEntities, verbose = false) {
         if (!data || typeof data !== 'object') {
@@ -249,8 +406,46 @@ class JsonlDatasetLoader {
             const fieldType = Reflect.getMetadata('field:type', ModelClass.prototype, fieldName);
             const relationshipType = Reflect.getMetadata('field:relationship:type', ModelClass.prototype, fieldName);
             const designType = Reflect.getMetadata('design:type', ModelClass.prototype, fieldName);
-            // If it's a reference/composition and has simple format {"id": "uuid"}
-            if ((fieldType === 'reference' || fieldType === 'composition' || relationshipType) &&
+            // Handle arrays of compositions (not references)
+            if (Array.isArray(resolvedData[fieldName])) {
+                // Only resolve if this is a composition/sharedComposition, not a reference
+                if (relationshipType === 'composition' || relationshipType === 'sharedComposition') {
+                    const resolvedArray = [];
+                    for (const item of resolvedData[fieldName]) {
+                        if (item && typeof item === 'object' && item.id && Object.keys(item).length === 1) {
+                            // Try to resolve the composition
+                            const refId = item.id;
+                            let resolved = false;
+                            // Try to find the model name and resolve from loadedEntities
+                            const refModelName = this.getReferencedModelName(ModelClass, fieldName);
+                            if (refModelName && loadedEntities[refModelName] && loadedEntities[refModelName][refId]) {
+                                resolvedArray.push(loadedEntities[refModelName][refId]);
+                                resolved = true;
+                                if (verbose) {
+                                    console.log(`    Resolved composition array item ${fieldName}[].id=${refId} → ${refModelName} object`);
+                                }
+                            }
+                            if (!resolved) {
+                                // Keep the ID-only reference if we can't resolve it
+                                resolvedArray.push(item);
+                                if (verbose && refModelName) {
+                                    console.log(`    Warning: Could not resolve composition ${fieldName}[].id=${refId} (${refModelName} not found)`);
+                                }
+                            }
+                        }
+                        else {
+                            // Item already has full data, keep it as is
+                            resolvedArray.push(item);
+                        }
+                    }
+                    resolvedData[fieldName] = resolvedArray;
+                }
+                // For regular reference arrays, leave them as-is (don't resolve to avoid cycles)
+                continue;
+            }
+            // Only resolve compositions and sharedCompositions, NOT regular references
+            // This prevents circular dependency issues with fromJSON
+            if ((relationshipType === 'composition' || relationshipType === 'sharedComposition') &&
                 resolvedData[fieldName] &&
                 typeof resolvedData[fieldName] === 'object' &&
                 resolvedData[fieldName].id &&
@@ -314,7 +509,315 @@ class JsonlDatasetLoader {
         }
         return resolvedData;
     }
-    async loadJsonlFile(filePath, ModelClass, modelName, loadedEntities, validateRecords = true, verbose = false) {
+    /**
+     * Fetches a referenced entity from the database, using an in-memory cache keyed by
+     * `ClassName:id` to eliminate redundant queries for the same record within a load run.
+     *
+     * When the entity is not found (or an error occurs), returns a minimal object
+     * `{ id: referenceId }` so that TypeORM can still extract the FK on save.
+     */
+    async fetchReferenceFromDb(dataSource, TargetClass, referenceId, label, verbose) {
+        const cacheKey = `${TargetClass.name}:${referenceId}`;
+        if (this.dbLookupCache.has(cacheKey)) {
+            // null in the cache means a previous lookup returned nothing — return normalized stub.
+            return this.dbLookupCache.get(cacheKey) ?? { id: referenceId };
+        }
+        try {
+            const loaded = await dataSource.findOneBy(TargetClass, { id: referenceId });
+            this.dbLookupCache.set(cacheKey, loaded ?? null);
+            if (loaded) {
+                if (verbose) {
+                    console.log(`    ↳ Loaded ${label}="${referenceId}" from database (${TargetClass.name})`);
+                }
+                return loaded;
+            }
+            if (verbose) {
+                console.log(`    ⚠️ Could not resolve ${label}="${referenceId}" (not found in database)`);
+            }
+        }
+        catch (error) {
+            this.dbLookupCache.set(cacheKey, null);
+            if (verbose) {
+                console.log(`    ⚠️ Error loading ${label}="${referenceId}" from database:`, error.message);
+            }
+        }
+        return { id: referenceId };
+    }
+    /**
+     * Resolves unresolved references from the database before saving.
+     * This handles references that couldn't be resolved from loadedEntities (excluded models).
+     * For single references, sets the FK column directly for efficiency.
+     * For array references, loads entities from DB to populate the relationship.
+     */
+    async resolveUnresolvedReferencesFromDb(modelInstance, dataSource, verbose = false) {
+        const proto = Object.getPrototypeOf(modelInstance);
+        const ModelClass = modelInstance.constructor;
+        const modelFields = Reflect.getMetadata?.('model:fields', ModelClass) || [];
+        const instanceProps = Object.getOwnPropertyNames(modelInstance);
+        const propertyNames = [...new Set([...instanceProps, ...modelFields])];
+        for (const propertyName of propertyNames) {
+            if (propertyName === 'constructor') {
+                continue;
+            }
+            try {
+                const fieldType = Reflect.getMetadata?.('field:type', modelInstance, propertyName);
+                const relationshipType = Reflect.getMetadata?.('field:relationship:type', modelInstance, propertyName);
+                // Only process reference fields
+                if (fieldType !== 'relationship' || relationshipType !== 'reference') {
+                    continue;
+                }
+                const currentValue = modelInstance[propertyName];
+                // Skip if value is already resolved (BaseDataModel instance) or truly undefined/null
+                if (currentValue === undefined || currentValue === null) {
+                    continue;
+                }
+                // Also skip if it's already a BaseDataModel instance
+                const { BaseDataModel } = await import('@slingr/framework-backend');
+                if (currentValue instanceof BaseDataModel) {
+                    continue;
+                }
+                // Get the referenced model class
+                const fieldOptions = Reflect.getMetadata?.('field:type:options', proto, propertyName);
+                if (!fieldOptions?.elementType || typeof fieldOptions.elementType !== 'function') {
+                    continue;
+                }
+                const TargetClass = fieldOptions.elementType();
+                if (!TargetClass) {
+                    continue;
+                }
+                // Handle array references
+                if (Array.isArray(currentValue)) {
+                    const resolvedArray = [];
+                    for (const item of currentValue) {
+                        let referenceId;
+                        if (typeof item === 'string') {
+                            referenceId = item;
+                        }
+                        else if (item && typeof item === 'object' && 'id' in item) {
+                            referenceId = item.id;
+                        }
+                        if (!referenceId) {
+                            resolvedArray.push(item);
+                            continue;
+                        }
+                        const entity = await this.fetchReferenceFromDb(dataSource, TargetClass, referenceId, `${propertyName}[${resolvedArray.length}]`, verbose);
+                        resolvedArray.push(entity);
+                    }
+                    modelInstance[propertyName] = resolvedArray;
+                }
+                else {
+                    let referenceId;
+                    if (typeof currentValue === 'string') {
+                        referenceId = currentValue;
+                    }
+                    else if (currentValue && typeof currentValue === 'object' && 'id' in currentValue) {
+                        referenceId = currentValue.id;
+                    }
+                    if (referenceId) {
+                        modelInstance[propertyName] = await this.fetchReferenceFromDb(dataSource, TargetClass, referenceId, propertyName, verbose);
+                    }
+                }
+            }
+            catch (error) {
+                // Ignore errors for individual properties
+                if (verbose) {
+                    console.log(`    ⚠️ Error resolving ${propertyName}:`, error.message);
+                }
+            }
+        }
+    }
+    /**
+     * Resolves string UUID reference fields on a model instance to their loaded BaseDataModel entities.
+     *
+     * During dataset loading, reference fields are stored as plain UUID strings in JSONL files.
+     * After fromJSON(), these remain as strings. However, filter validation (ValidateRelationshipFilter)
+     * requires actual BaseDataModel instances to properly evaluate filter criteria like:
+     *   filter: (company) => ({ state: company.state })
+     *
+     * Without this resolution, company.state would be a plain string, and the filter validator
+     * wouldn't be able to check if the county's state matches. This mirrors what
+     * preprocessReferenceFields does in the GraphQL/UI path (fetching from DB), but uses
+     * already-loaded entities from the dataset instead.
+     *
+     * When dataSource is provided, missing array references will be loaded from the database.
+     * This is necessary for validation when referenced entities were loaded in a previous session.
+     *
+     * @param modelMap Full map of all model classes (not filtered) - needed to look up referenced model classes
+     */
+    async resolveReferenceFieldsToEntities(modelInstance, ModelClass, loadedEntities, verbose = false, dataSource, modelMap) {
+        const proto = Object.getPrototypeOf(modelInstance);
+        const modelFields = Reflect.getMetadata?.('model:fields', ModelClass) || [];
+        const instanceProps = Object.getOwnPropertyNames(modelInstance);
+        const propertyNames = [...new Set([...instanceProps, ...modelFields])];
+        for (const propertyName of propertyNames) {
+            if (propertyName === 'constructor') {
+                continue;
+            }
+            try {
+                const fieldType = Reflect.getMetadata?.('field:type', modelInstance, propertyName);
+                const relationshipType = Reflect.getMetadata?.('field:relationship:type', modelInstance, propertyName);
+                // Only process reference fields (not compositions, which are handled differently)
+                if (fieldType !== 'relationship' || relationshipType !== 'reference') {
+                    continue;
+                }
+                const currentValue = modelInstance[propertyName];
+                // Check if the value is an array (for many-to-many or one-to-many relationships)
+                if (Array.isArray(currentValue)) {
+                    // Process each item in the array
+                    const resolvedArray = [];
+                    let hasUnresolved = false;
+                    for (const item of currentValue) {
+                        // Extract ID from each item
+                        let referenceId;
+                        if (typeof item === 'string') {
+                            referenceId = item;
+                        }
+                        else if (item instanceof framework_backend_1.BaseDataModel && item.id) {
+                            referenceId = item.id;
+                        }
+                        else if (item && typeof item === 'object' && 'id' in item) {
+                            referenceId = item.id;
+                        }
+                        if (!referenceId) {
+                            resolvedArray.push(item);
+                            continue;
+                        }
+                        // Get elementType from field options
+                        const fieldOptions = Reflect.getMetadata?.('field:type:options', proto, propertyName);
+                        let targetClassName = null;
+                        let TargetClass = null;
+                        if (fieldOptions?.elementType && typeof fieldOptions.elementType === 'function') {
+                            try {
+                                const targetClass = fieldOptions.elementType();
+                                if (targetClass?.name) {
+                                    targetClassName = targetClass.name;
+                                    TargetClass = targetClass;
+                                }
+                            }
+                            catch {
+                                // Ignore - elementType() might fail if model not imported in current context
+                            }
+                        }
+                        // If elementType() failed or wasn't available, try to get class from modelMap
+                        if (!TargetClass && targetClassName && modelMap) {
+                            TargetClass = modelMap[targetClassName];
+                            if (verbose && TargetClass) {
+                                console.log(`  ↳ Resolved ${targetClassName} class from modelMap for database query`);
+                            }
+                        }
+                        if (!targetClassName) {
+                            resolvedArray.push(item);
+                            continue;
+                        }
+                        // Try to resolve from loadedEntities
+                        const entityMap = loadedEntities[targetClassName];
+                        if (entityMap && entityMap[referenceId]) {
+                            resolvedArray.push(entityMap[referenceId]);
+                        }
+                        else if (dataSource && TargetClass) {
+                            const entity = await this.fetchReferenceFromDb(dataSource, TargetClass, referenceId, `${propertyName}[${resolvedArray.length}]`, verbose);
+                            resolvedArray.push(entity);
+                            if (!(entity instanceof framework_backend_1.BaseDataModel)) {
+                                hasUnresolved = true;
+                            }
+                        }
+                        else {
+                            // No dataSource — normalize to a minimal stub so TypeORM can derive the FK on save.
+                            resolvedArray.push({ id: referenceId });
+                            hasUnresolved = true;
+                            if (verbose) {
+                                console.log(`  ⚠️ Could not resolve array reference ${propertyName}[${resolvedArray.length - 1}]="${referenceId}" (${targetClassName} not found in loaded entities, no dataSource available)`);
+                            }
+                        }
+                    }
+                    // Always set the array - either fully resolved or with plain objects containing IDs
+                    // Unresolved items will be loaded from the database before saving
+                    modelInstance[propertyName] = resolvedArray;
+                    if (hasUnresolved && verbose) {
+                        console.log(`     → Array ${propertyName} contains unresolved references - will be loaded from database before saving`);
+                    }
+                    continue;
+                }
+                // Handle single reference (non-array)
+                // Extract the ID to look up from loadedEntities.
+                // Handles three formats:
+                //   1. Plain string UUID:  "554aef99-..."
+                //   2. Partial BaseDataModel:  State { id: "554aef99-...", name: undefined }
+                //   3. Object with id:     { id: "554aef99-..." }
+                let referenceId;
+                if (typeof currentValue === 'string') {
+                    referenceId = currentValue;
+                }
+                else if (currentValue instanceof framework_backend_1.BaseDataModel && currentValue.id) {
+                    referenceId = currentValue.id;
+                }
+                else if (currentValue && typeof currentValue === 'object' && 'id' in currentValue) {
+                    referenceId = currentValue.id;
+                }
+                if (!referenceId) {
+                    continue;
+                }
+                // Determine the referenced model class
+                let targetClassName = null;
+                let TargetClass = null;
+                // Try elementType from field options (most reliable for Slingr framework)
+                const fieldOptions = Reflect.getMetadata?.('field:type:options', proto, propertyName);
+                if (fieldOptions?.elementType && typeof fieldOptions.elementType === 'function') {
+                    try {
+                        const targetClass = fieldOptions.elementType();
+                        if (targetClass?.name) {
+                            targetClassName = targetClass.name;
+                            TargetClass = targetClass;
+                        }
+                    }
+                    catch {
+                        // Ignore elementType resolution errors
+                    }
+                }
+                // Fallback to design:type
+                if (!targetClassName) {
+                    const designType = Reflect.getMetadata?.('design:type', modelInstance, propertyName);
+                    if (designType && designType.name && designType.name !== 'Array' && designType.name !== 'String') {
+                        targetClassName = designType.name;
+                        TargetClass = designType;
+                    }
+                }
+                // If elementType/designType didn't give us a class, try modelMap
+                if (!TargetClass && targetClassName && modelMap) {
+                    TargetClass = modelMap[targetClassName];
+                    if (verbose && TargetClass) {
+                        console.log(`  ↳ Resolved ${targetClassName} class from modelMap for database query`);
+                    }
+                }
+                if (!targetClassName) {
+                    continue;
+                }
+                // Look up the entity in loadedEntities
+                const entityMap = loadedEntities[targetClassName];
+                if (entityMap && entityMap[referenceId]) {
+                    modelInstance[propertyName] = entityMap[referenceId];
+                    if (verbose) {
+                        console.log(`  ↳ Resolved reference ${propertyName}="${referenceId}" → ${targetClassName} instance`);
+                    }
+                }
+                else if (dataSource && TargetClass) {
+                    modelInstance[propertyName] = await this.fetchReferenceFromDb(dataSource, TargetClass, referenceId, propertyName, verbose);
+                }
+                else {
+                    // No dataSource — normalize to a minimal stub so TypeORM can derive the FK on save.
+                    // Setting the property to a plain string UUID would be silently ignored by TypeORM.
+                    modelInstance[propertyName] = { id: referenceId };
+                    if (verbose) {
+                        console.log(`  ⚠️ Could not resolve reference ${propertyName}="${referenceId}" (${targetClassName} not found in loaded entities, no dataSource available)`);
+                    }
+                }
+            }
+            catch {
+                // Ignore errors for individual properties
+            }
+        }
+    }
+    async loadJsonlFile(filePath, ModelClass, modelName, loadedEntities, validateRecords = true, verbose = false, dataSource, modelMap) {
         if (verbose) {
             console.log(`Loading JSONL file: ${filePath} for model: ${modelName}`);
         }
@@ -326,6 +829,9 @@ class JsonlDatasetLoader {
             successCount: 0,
             errorCount: 0,
             errors: [],
+            saveSuccessCount: 0,
+            saveErrorCount: 0,
+            saveErrors: [],
         };
         for (let i = 0; i < lines.length; i++) {
             try {
@@ -337,6 +843,24 @@ class JsonlDatasetLoader {
                 const resolvedData = await this.resolveSimpleReferences(rawData, ModelClass, loadedEntities, verbose);
                 // Use the model's fromJSON method to create the instance
                 const modelInstance = ModelClass.fromJSON(resolvedData);
+                // Restore fields that were excluded by class-transformer (e.g., fields with available: false)
+                // During dataset loading, we want to include ALL fields from the JSONL file,
+                // even those marked as excluded for API/JSON serialization
+                for (const key of Object.keys(resolvedData)) {
+                    // If the field exists in resolvedData but is undefined/missing in modelInstance,
+                    // it was likely excluded by @Exclude() or similar decorators
+                    if (resolvedData[key] !== undefined && modelInstance[key] === undefined) {
+                        modelInstance[key] = resolvedData[key];
+                        if (verbose) {
+                            console.log(`  ↳ Restored excluded field '${key}' for ${ModelClass.name}`);
+                        }
+                    }
+                }
+                // Resolve string UUID reference fields to loaded BaseDataModel entities.
+                // This is needed for filter validation: filters like { state: company.state }
+                // need company.state to be a BaseDataModel instance (not a plain UUID string)
+                // so the validator can properly check referenced objects against filter criteria.
+                await this.resolveReferenceFieldsToEntities(modelInstance, ModelClass, loadedEntities, verbose, dataSource, modelMap);
                 // Validate the instance if requested
                 if (validateRecords) {
                     const validationErrors = await modelInstance.validate();
@@ -411,6 +935,30 @@ class JsonlDatasetLoader {
     camelToSnakeCase(str) {
         return str.replace(/([A-Z])/g, '_$1').toLowerCase();
     }
+    /**
+     * Recursively format validation error constraints for display
+     */
+    formatValidationConstraints(validationErrors, prefix = '', isRoot = true) {
+        const lines = [];
+        for (const error of validationErrors) {
+            const propertyPath = prefix ? `${prefix}.${error.property}` : error.property;
+            // Add constraints for this property if they exist
+            if (error.constraints) {
+                const constraintEntries = Object.entries(error.constraints);
+                if (constraintEntries.length > 0) {
+                    lines.push(`${propertyPath}: ${JSON.stringify(error.constraints)}`);
+                }
+            }
+            // Recursively process children
+            if (error.children && error.children.length > 0) {
+                const childLines = this.formatValidationConstraints(error.children, propertyPath, false);
+                if (childLines) {
+                    lines.push(childLines);
+                }
+            }
+        }
+        return lines.join('\n');
+    }
     /**
      * Load dataset into database using TypeORM with dynamic table creation
      * This method creates tables dynamically and uses TypeORM for data insertion
@@ -421,12 +969,26 @@ class JsonlDatasetLoader {
         if (!typeormDataSource || !typeormDataSource.isInitialized) {
             throw new Error('TypeORM DataSource is not initialized');
         }
+        // If verbose, enable logging. If not verbose, disable logging to improve performance
+        if (verbose) {
+            typeormDataSource.setOptions({ logging: ['error', 'warn', 'info'] });
+        }
+        else {
+            typeormDataSource.setOptions({ logging: false });
+        }
         // Process results in the order they were provided (which should be dependency order)
         for (const result of results) {
-            if (result.errorCount > 0 && verbose) {
+            if (result.errorCount > 0) {
                 console.log(`⚠️  Model ${result.modelName} has ${result.errorCount} validation errors:`);
                 for (const error of result.errors) {
-                    console.log(`   Record ${error.recordIndex + 1}: ${JSON.stringify(error.validationErrors)}`);
+                    const formattedErrors = this.formatValidationConstraints(error.validationErrors);
+                    console.log(`   Record ${error.recordIndex + 1}:`);
+                    if (formattedErrors) {
+                        console.log(`     ${formattedErrors.replace(/\n/g, '\n     ')}`);
+                    }
+                    else {
+                        console.log(`     (no constraint details available)`);
+                    }
                 }
             }
             if (result.successCount === 0) {
@@ -438,18 +1000,58 @@ class JsonlDatasetLoader {
             if (verbose) {
                 console.log(`📊 Processing ${result.modelName}: ${result.successCount} valid records`);
             }
-            try {
-                // Save records using the framework's save method which respects relationships
-                for (const record of result.records) {
+            // Create progress bar for this model
+            const progressBar = new cliProgress.SingleBar({
+                format: `   Loading ${result.modelName} [{bar}] {percentage}% | {value}/{total} records | Success: {success} | Failed: {failed}`,
+                barCompleteChar: '█',
+                barIncompleteChar: '░',
+                hideCursor: true,
+            });
+            progressBar.start(result.records.length, 0, { success: 0, failed: 0 });
+            // Save records using the framework's save method which respects relationships
+            // Continue on errors instead of stopping
+            for (let i = 0; i < result.records.length; i++) {
+                const record = result.records[i];
+                try {
+                    // Before saving, resolve any unresolved references from the database
+                    // This handles cases where referenced models were excluded from loading but exist in DB
+                    await this.resolveUnresolvedReferencesFromDb(record, dataSource, verbose);
                     await dataSource.save(record);
+                    result.saveSuccessCount++;
                 }
-                if (verbose) {
-                    console.log(`✅ Successfully loaded ${result.successCount} records for ${result.modelName}`);
+                catch (error) {
+                    result.saveErrorCount++;
+                    result.saveErrors.push({
+                        recordIndex: i,
+                        record: record,
+                        error: error.message,
+                    });
                 }
+                // Update progress bar
+                progressBar.update(i + 1, {
+                    success: result.saveSuccessCount,
+                    failed: result.saveErrorCount,
+                });
             }
-            catch (error) {
-                console.error(`❌ Failed to load ${result.modelName}:`, error);
-                throw error;
+            progressBar.stop();
+            // Show summary for this model
+            if (result.saveSuccessCount > 0) {
+                console.log(`   ✅ Successfully saved ${result.saveSuccessCount} records`);
+            }
+            if (result.saveErrorCount > 0) {
+                console.log(`   ❌ Failed to save ${result.saveErrorCount} records`);
+                // Show first few errors if verbose
+                if (verbose && result.saveErrors.length > 0) {
+                    const errorsToShow = Math.min(3, result.saveErrors.length);
+                    console.log(`   First ${errorsToShow} errors:`);
+                    for (let i = 0; i < errorsToShow; i++) {
+                        const saveError = result.saveErrors[i];
+                        console.log(`      Record ${saveError.recordIndex + 1}: ${saveError.error}`);
+                    }
+                    if (result.saveErrors.length > errorsToShow) {
+                        console.log(`      ... and ${result.saveErrors.length - errorsToShow} more errors`);
+                    }
+                }
             }
         }
     }
@@ -552,15 +1154,20 @@ exports.JsonlDatasetLoader = JsonlDatasetLoader;
 /**
  * Auto-discover models from the compiled JavaScript files
  */
-async function discoverModels(distPath) {
+async function discoverModels(distPath, verbose = false) {
     const modelMap = {};
     if (!(await fs_extra_1.default.pathExists(distPath))) {
         throw new Error(`Dist directory not found: ${distPath}. Please run 'npm run build' first.`);
     }
     // Import glob dynamically since it might be ESM
     const { glob } = await import('glob');
-    // Only look for model files, not test files
-    const entityFiles = await glob(node_path_1.default.join(distPath, 'data', '**', '*.js').replace(/\\/g, '/')).then((files) => files.filter((file) => !file.includes('.test.js')));
+    // Check both dist/data (legacy) and dist/src/data (current TypeScript output)
+    // Try primary location first
+    let entityFiles = await glob(node_path_1.default.join(distPath, 'src', 'data', '**', '*.js').replace(/\\/g, '/')).then((files) => files.filter((file) => !file.includes('.test.js')));
+    // If no files found, try alternative path (dist/src/data)
+    if (entityFiles.length === 0) {
+        entityFiles = await glob(node_path_1.default.join(distPath, 'data', '**', '*.js').replace(/\\/g, '/')).then((files) => files.filter((file) => !file.includes('.test.js')));
+    }
     for (const file of entityFiles) {
         try {
             // Clear the require cache to ensure fresh loading
@@ -569,13 +1176,17 @@ async function discoverModels(distPath) {
             const exports = Object.values(module);
             for (const exp of exports) {
                 if (typeof exp === 'function' && exp.prototype && exp.name && typeof exp.fromJSON === 'function') {
-                    console.log(`✅ Found model: ${exp.name} in ${file}`);
+                    if (verbose) {
+                        console.log(`✅ Found model: ${exp.name} in ${file}`);
+                    }
                     modelMap[exp.name] = exp;
                 }
             }
         }
         catch (error) {
-            console.warn(`Failed to load entity from file: ${file}:`, error);
+            if (verbose) {
+                console.warn(`Failed to load entity from file: ${file}:`, error);
+            }
         }
     }
     return modelMap;