@slingr/cli 0.0.1

package/dist/utils/jsonlLoader.d.ts

@@ -0,0 +1,140 @@
+import { BaseModel, TypeOrmSqlDataSource } from 'slingr-framework';
+import 'reflect-metadata';
+/**
+ * Interface for model constructors that extend BaseModel
+ */
+export interface ModelConstructor<T extends BaseModel> {
+    new (): T;
+    fromJSON(data: any): T;
+    prototype: T;
+}
+/**
+ * Interface for dataset loading options
+ */
+export interface DatasetLoadOptions {
+    /**
+     * Directory containing the JSONL files
+     */
+    datasetPath: string;
+    /**
+     * Map of model name to model constructor
+     * Key: filename without .jsonl extension
+     * Value: Model constructor class
+     */
+    modelMap: Record<string, ModelConstructor<any>>;
+    /**
+     * Whether to validate each record after loading from JSON
+     * @default true
+     */
+    validateRecords?: boolean;
+    /**
+     * Whether to log detailed information during loading
+     * @default false
+     */
+    verbose?: boolean;
+}
+/**
+ * Interface for model dependency information
+ */
+export interface ModelDependency {
+    modelName: string;
+    dependencies: string[];
+    dependents: string[];
+}
+/**
+ * Interface for dependency analysis result
+ */
+export interface DependencyAnalysisResult {
+    dependencies: Record<string, ModelDependency>;
+    loadOrder: string[];
+}
+/**
+ * Result of loading a dataset
+ */
+export interface DatasetLoadResult<T extends BaseModel = BaseModel> {
+    /**
+     * Name of the model/file that was loaded
+     */
+    modelName: string;
+    /**
+     * Array of loaded and converted model instances
+     */
+    records: T[];
+    /**
+     * Number of records successfully loaded
+     */
+    successCount: number;
+    /**
+     * Number of records that failed to load
+     */
+    errorCount: number;
+    /**
+     * Array of validation errors if any
+     */
+    errors: Array<{
+        recordIndex: number;
+        recordData: any;
+        validationErrors: any[];
+    }>;
+}
+/**
+ * Utility class for loading datasets from JSONL files using Slingr model fromJSON() functionality
+ */
+export declare class JsonlDatasetLoader {
+    /**
+     * Analyze model dependencies to determine the correct loading order
+     */
+    analyzeDependencies(modelMap: Record<string, ModelConstructor<any>>, verbose?: boolean): DependencyAnalysisResult;
+    /**
+     * Extract dependencies from a model class by analyzing its decorators/metadata
+     */
+    private extractModelDependencies;
+    /**
+     * Perform topological sort to determine loading order
+     */
+    private topologicalSort;
+    /**
+     * Load all JSONL files from a dataset directory
+     */
+    loadDataset(options: DatasetLoadOptions): Promise<DatasetLoadResult[]>;
+    /**
+     * Load a single JSONL file and convert records using the model's fromJSON method
+     */
+    /**
+     * Resolves simple references in format {"id": "uuid"} to full objects from loaded entities
+     */
+    resolveSimpleReferences<T extends BaseModel>(data: any, ModelClass: ModelConstructor<T>, loadedEntities: Record<string, Record<string, any>>, verbose?: boolean): Promise<any>;
+    loadJsonlFile<T extends BaseModel>(filePath: string, ModelClass: ModelConstructor<T>, modelName: string, loadedEntities: Record<string, Record<string, any>>, validateRecords?: boolean, verbose?: boolean): Promise<DatasetLoadResult<T>>;
+    /**
+     * Convert loaded model instances to database-friendly format
+     * This method respects the model's field availability settings and converts camelCase to snake_case
+     */
+    convertToDbFormat<T extends BaseModel>(records: T[], verbose?: boolean): any[];
+    /**
+     * Convert camelCase string to snake_case
+     */
+    private camelToSnakeCase;
+    /**
+     * Load dataset into database using TypeORM with dynamic table creation
+     * This method creates tables dynamically and uses TypeORM for data insertion
+     * Note: Results should already be in dependency order from loadDataset()
+     */
+    loadDatasetToDatabase(results: DatasetLoadResult[], dataSource: TypeOrmSqlDataSource, verbose?: boolean): Promise<void>;
+    /**
+     * Create table dynamically using TypeORM (database-agnostic)
+     */
+    private createTableDynamically;
+    /**
+     * Insert data dynamically using TypeORM (database-agnostic)
+     */
+    private insertDataDynamically;
+    /**
+     * Get the database column schema based on ALL records to capture all possible fields
+     */
+    inferDbSchemaFromAllRecords(records: any[]): Record<string, string>;
+}
+/**
+ * Auto-discover models from the compiled JavaScript files
+ */
+export declare function discoverModels(distPath: string): Promise<Record<string, ModelConstructor<any>>>;
+//# sourceMappingURL=jsonlLoader.d.ts.map

package/dist/utils/jsonlLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonlLoader.d.ts","sourceRoot":"","sources":["../../src/utils/jsonlLoader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,kBAAkB,CAAC;AAE1B;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC,SAAS,SAAS;IACnD,QAAQ,CAAC,CAAC;IACV,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;IAEhD;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS;IAChE;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,OAAO,EAAE,CAAC,EAAE,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,KAAK,CAAC;QACZ,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,GAAG,CAAC;QAChB,gBAAgB,EAAE,GAAG,EAAE,CAAC;KACzB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,qBAAa,kBAAkB;IAC7B;;OAEG;IACH,mBAAmB,CACjB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAC/C,OAAO,GAAE,OAAe,GACvB,wBAAwB;IAyC3B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAuHhC;;OAEG;IACH,OAAO,CAAC,eAAe;IAuCvB;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAuE5E;;OAEG;IACH;;OAEG;IACG,uBAAuB,CAAC,CAAC,SAAS,SAAS,EAC/C,IAAI,EAAE,GAAG,EACT,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAC/B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,EACnD,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,GAAG,CAAC;IAsFT,aAAa,CAAC,CAAC,SAAS,SAAS,EACrC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAC/B,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,EACnD,eAAe,GAAE,OAAc,EAC/B,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAkFhC;;;OAGG;IACH,iBAAiB,CAAC,CAAC,SAAS,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,OAAO,GAAE,OAAe,GAAG,GAAG,EAAE;IA0BrF;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAIxB;;;;OAIG;IACG,qBAAqB,CACzB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,UAAU,EAAE,oBAAoB,EAChC,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,IAAI,CAAC;IA2ChB;;OAEG;YACW,sBAAsB;IAwDpC;;OAEG;YACW,qBAAqB;IA6BnC;;OAEG;IACH,2BAA2B,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAyBpE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAmCrG"}