@toiroakr/lines-db 0.4.1 → 0.6.0

package/bin/cli.js

@@ -119,6 +119,54 @@ function sanitizeIdentifier(value) {
 	return value.replace(/[^A-Za-z0-9_$]/g, "");
 }
 
+//#endregion
+//#region src/runtime.ts
+function detectRuntime() {
+	if (typeof process !== "undefined" && process.versions && process.versions.node) return "node";
+	return "unknown";
+}
+const RUNTIME = detectRuntime();
+
+//#endregion
+//#region src/sqlite-adapter.ts
+/**
+* Create a SQLite database instance for Node.js
+*/
+function createDatabase(path = ":memory:") {
+	if (RUNTIME === "node") return createNodeDatabase(path);
+	else throw new Error(`Unsupported runtime: ${RUNTIME}`);
+}
+/**
+* Create a Node.js SQLite database
+*/
+function createNodeDatabase(path) {
+	const { DatabaseSync } = __require("node:sqlite");
+	const db = new DatabaseSync(path);
+	db.exec("PRAGMA foreign_keys = ON");
+	return {
+		prepare(sql) {
+			const stmt = db.prepare(sql);
+			return {
+				run(...params) {
+					return stmt.run(...params);
+				},
+				get(...params) {
+					return stmt.get(...params);
+				},
+				all(...params) {
+					return stmt.all(...params);
+				}
+			};
+		},
+		exec(sql) {
+			db.exec(sql);
+		},
+		close() {
+			db.close();
+		}
+	};
+}
+
 //#endregion
 //#region src/jsonl-reader.ts
 var JsonlReader = class {
@@ -199,6 +247,31 @@ var JsonlReader = class {
 	}
 };
 
+//#endregion
+//#region src/jsonl-writer.ts
+var JsonlWriter = class {
+	/**
+	* Write data to JSONL file
+	*/
+	static async write(filePath, data) {
+		await writeFile(filePath, data.map((obj) => JSON.stringify(obj)).join("\n") + "\n", "utf-8");
+	}
+	/**
+	* Append data to JSONL file
+	*/
+	static async append(filePath, data) {
+		const { readFile: readFile$1, writeFile: writeFile$1 } = await import("node:fs/promises");
+		try {
+			const existing = await readFile$1(filePath, "utf-8");
+			const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
+			await writeFile$1(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
+		} catch (error) {
+			if (error.code === "ENOENT") await this.write(filePath, data);
+			else throw error;
+		}
+	}
+};
+
 //#endregion
 //#region src/schema-loader.ts
 var SchemaLoader = class {
@@ -248,79 +321,6 @@ var SchemaLoader = class {
 	}
 };
 
-//#endregion
-//#region src/runtime.ts
-function detectRuntime() {
-	if (typeof process !== "undefined" && process.versions && process.versions.node) return "node";
-	return "unknown";
-}
-const RUNTIME = detectRuntime();
-
-//#endregion
-//#region src/sqlite-adapter.ts
-/**
-* Create a SQLite database instance for Node.js
-*/
-function createDatabase(path = ":memory:") {
-	if (RUNTIME === "node") return createNodeDatabase(path);
-	else throw new Error(`Unsupported runtime: ${RUNTIME}`);
-}
-/**
-* Create a Node.js SQLite database
-*/
-function createNodeDatabase(path) {
-	const { DatabaseSync } = __require("node:sqlite");
-	const db = new DatabaseSync(path);
-	db.exec("PRAGMA foreign_keys = ON");
-	return {
-		prepare(sql) {
-			const stmt = db.prepare(sql);
-			return {
-				run(...params) {
-					return stmt.run(...params);
-				},
-				get(...params) {
-					return stmt.get(...params);
-				},
-				all(...params) {
-					return stmt.all(...params);
-				}
-			};
-		},
-		exec(sql) {
-			db.exec(sql);
-		},
-		close() {
-			db.close();
-		}
-	};
-}
-
-//#endregion
-//#region src/jsonl-writer.ts
-var JsonlWriter = class {
-	/**
-	* Write data to JSONL file
-	*/
-	static async write(filePath, data) {
-		await writeFile(filePath, data.map((obj) => JSON.stringify(obj)).join("\n") + "\n", "utf-8");
-	}
-	/**
-	* Append data to JSONL file
-	*/
-	static async append(filePath, data) {
-		const { readFile: readFile$1, writeFile: writeFile$1 } = await import("node:fs/promises");
-		try {
-			const existing = await readFile$1(filePath, "utf-8");
-			const lines = data.map((obj) => JSON.stringify(obj)).join("\n");
-			await writeFile$1(filePath, existing.trim() + "\n" + lines + "\n", "utf-8");
-		} catch (error) {
-			if (error.code === "ENOENT") await this.write(filePath, data);
-			else throw error;
-		}
-	}
-};
-
 //#endregion
 //#region src/directory-scanner.ts
 var DirectoryScanner = class {
@@ -373,27 +373,49 @@ var LinesDB = class LinesDB {
 		return new LinesDB(config, dbPath);
 	}
 	/**
-	* Initialize database by loading all JSONL files
+	* Initialize database by loading all JSONL files or a specific table
 	* Uses dependency resolution to ensure foreign key references are loaded in correct order
+	* @param options Optional configuration for initialization
+	* @param options.tableName Optional table name to initialize. If not provided, initializes all tables
+	* @param options.detailedValidate If true, performs detailed validation by inserting rows one by one to catch constraint violations
+	* @param options.transform Optional transform function to apply to rows before validation (only applied to the specified tableName)
+	* @returns ValidationResult containing validation status, errors, and warnings
 	*/
-	async initialize() {
+	async initialize(options) {
+		const allErrors = [];
+		const allWarnings = [];
+		const tableName = options?.tableName;
+		const detailedValidate = options?.detailedValidate ?? false;
+		const transform = options?.transform;
 		this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
+		const tablesToLoad = tableName ? [tableName] : Array.from(this.tables.keys());
+		for (const tableNameToLoad of tablesToLoad) if (!this.tables.has(tableNameToLoad)) throw new Error(`Table '${tableNameToLoad}' not found in directory '${this.config.dataDir}'`);
 		const loadedTables = /* @__PURE__ */ new Set();
 		const loadingTables = /* @__PURE__ */ new Set();
-		for (const [tableName] of this.tables) if (!loadedTables.has(tableName)) try {
-			await this.loadTableWithDependencies(tableName, loadedTables, loadingTables);
-		} catch (error) {
-			console.warn(`Warning: Failed to load table '${tableName}':`, error instanceof Error ? error.message : String(error));
-			this.tables.delete(tableName);
-			this.schemas.delete(tableName);
-			this.validationSchemas.delete(tableName);
+		const attemptedTables = /* @__PURE__ */ new Set();
+		for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
+			const tableTransform = tableNameToLoad === tableName ? transform : void 0;
+			const { errors, warnings } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
+			allErrors.push(...errors);
+			allWarnings.push(...warnings);
 		}
+		return {
+			valid: allErrors.length === 0,
+			errors: allErrors,
+			warnings: allWarnings
+		};
 	}
 	/**
 	* Load a table and its dependencies recursively
 	*/
-	async loadTableWithDependencies(tableName, loadedTables, loadingTables) {
-		if (loadedTables.has(tableName)) return;
+	async loadTableWithDependencies(tableName, loadedTables, loadingTables, attemptedTables, detailedValidate, transform) {
+		const errors = [];
+		const warnings = [];
+		if (attemptedTables.has(tableName)) return {
+			errors,
+			warnings
+		};
+		attemptedTables.add(tableName);
 		if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
 		const tableConfig = this.tables.get(tableName);
 		if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
@@ -407,21 +429,35 @@ var LinesDB = class LinesDB {
 			} catch {}
 			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
 				const referencedTable = fk.references.table;
-				if (!loadedTables.has(referencedTable)) if (this.tables.has(referencedTable)) await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables);
-				else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
+				if (referencedTable === tableName) continue;
+				if (!attemptedTables.has(referencedTable)) if (this.tables.has(referencedTable)) {
+					const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
+					errors.push(...depResult.errors);
+					warnings.push(...depResult.warnings);
+				} else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
+			}
+			const { loaded, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform);
+			errors.push(...loadErrors);
+			if (loaded) loadedTables.add(tableName);
+			else {
+				warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
+				this.tables.delete(tableName);
 			}
-			if (await this.loadTable(tableName, tableConfig)) loadedTables.add(tableName);
-			else this.tables.delete(tableName);
 		} finally {
 			loadingTables.delete(tableName);
 		}
+		return {
+			errors,
+			warnings
+		};
 	}
 	/**
 	* Load a single table from JSONL file
-	* @returns true if table was loaded, false if skipped
+	* @returns Object with loaded status and validation errors
 	*/
-	async loadTable(tableName, config) {
-		const data = await JsonlReader.read(config.jsonlPath);
+	async loadTable(tableName, config, detailedValidate, transform) {
+		let data = await JsonlReader.read(config.jsonlPath);
+		if (transform) data = data.map((row) => transform(row));
 		let validationSchema = config.validationSchema;
 		const schemaMetadata = {};
 		if (!validationSchema) try {
@@ -455,18 +491,32 @@ var LinesDB = class LinesDB {
 				else throw error;
 			}
 		}
-		if (validationErrors.length > 0) {
-			const enhancedError = /* @__PURE__ */ new Error(`Validation failed for ${validationErrors.length} row(s) in table ${tableName}`);
-			enhancedError.name = "ValidationError";
-			enhancedError.validationErrors = validationErrors;
-			enhancedError.issues = validationErrors[0].error.issues;
-			throw enhancedError;
-		}
+		const validationErrorDetails = validationErrors.map((ve) => ({
+			file: config.jsonlPath,
+			tableName,
+			rowIndex: ve.rowIndex,
+			issues: ve.error.issues,
+			type: "schema"
+		}));
+		if (validationErrors.length > 0) return {
+			loaded: false,
+			errors: validationErrorDetails
+		};
 		let schema;
-		if (config.schema) schema = config.schema;
-		else if (config.autoInferSchema !== false) {
-			if (validatedData.length === 0) return false;
-			schema = JsonlReader.inferSchema(tableName, validatedData);
+		let inferredSchema;
+		if (validatedData.length > 0) inferredSchema = JsonlReader.inferSchema(tableName, validatedData);
+		if (config.schema) {
+			schema = config.schema;
+			if (inferredSchema) for (const inferredCol of inferredSchema.columns) {
+				const schemaCol = schema.columns.find((c) => c.name === inferredCol.name);
+				if (schemaCol && inferredCol.valueType && !schemaCol.valueType) schemaCol.valueType = inferredCol.valueType;
+			}
+		} else if (config.autoInferSchema !== false) {
+			if (validatedData.length === 0) return {
+				loaded: false,
+				errors: []
+			};
+			schema = inferredSchema;
 		} else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
 		const biSchema = validationSchema;
 		const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
@@ -483,8 +533,17 @@ var LinesDB = class LinesDB {
 		if (indexes) schema.indexes = indexes;
 		this.schemas.set(tableName, schema);
 		this.createTable(schema);
-		this.insertData(tableName, schema, validatedData);
-		return true;
+		if (detailedValidate) {
+			const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
+			if (insertErrors.length > 0) return {
+				loaded: false,
+				errors: insertErrors
+			};
+		} else this.insertData(tableName, schema, validatedData);
+		return {
+			loaded: true,
+			errors: []
+		};
 	}
 	/**
 	* Create table in SQLite with constraints and indexes
@@ -529,18 +588,85 @@ var LinesDB = class LinesDB {
 		return `"${identifier.replace(/"/g, "\"\"")}"`;
 	}
 	/**
-	* Insert data into table
+	* Insert data into table using batch insert (multiple rows per SQL)
+	* SQLite has a parameter limit (default 999), so we batch rows accordingly
+	* Throws exception if any constraint violation occurs
 	*/
 	insertData(tableName, schema, data) {
+		if (data.length === 0) return;
+		const columnNames = schema.columns.map((col) => col.name);
+		const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
+		const columnCount = columnNames.length;
+		const maxBatchSize = Math.floor(900 / columnCount);
+		const batchSize = Math.max(1, Math.min(maxBatchSize, 100));
+		for (let i = 0; i < data.length; i += batchSize) {
+			const batch = data.slice(i, i + batchSize);
+			const rowPlaceholders = columnNames.map(() => "?").join(", ");
+			const valuesPlaceholders = batch.map(() => `(${rowPlaceholders})`).join(", ");
+			const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES ${valuesPlaceholders}`;
+			const values = [];
+			for (const row of batch) for (const col of columnNames) values.push(this.normalizeValue(row[col]));
+			this.db.prepare(sql).run(...values);
+		}
+	}
+	/**
+	* Insert data into table one row at a time with detailed error reporting
+	* This is used for validation to catch constraint violations
+	*/
+	insertDataWithDetailedValidation(tableName, schema, data, filePath) {
+		const errors = [];
 		const columnNames = schema.columns.map((col) => col.name);
 		const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
 		const placeholders = columnNames.map(() => "?").join(", ");
 		const sql = `INSERT INTO ${this.quoteTableName(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
 		const stmt = this.db.prepare(sql);
-		for (const row of data) {
-			const values = columnNames.map((col) => this.normalizeValue(row[col]));
-			stmt.run(...values);
+		for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+			const row = data[rowIndex];
+			try {
+				const values = columnNames.map((col) => this.normalizeValue(row[col]));
+				stmt.run(...values);
+			} catch (error) {
+				const constraintError = this.analyzeConstraintError(error, filePath, tableName, rowIndex, row, schema.foreignKeys || []);
+				if (constraintError) errors.push(constraintError);
+			}
 		}
+		return errors;
+	}
+	/**
+	* Analyze constraint error and extract detailed information
+	*/
+	analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys) {
+		const errorMessage = error instanceof Error ? error.message : String(error);
+		if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
+			const fkValue = row[fk.column];
+			if (fkValue === null || fkValue === void 0) continue;
+			try {
+				const result = this.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [this.normalizeValue(fkValue)]);
+				if (result.length > 0 && result[0].count === 0) return {
+					file,
+					tableName,
+					rowIndex,
+					issues: [],
+					type: "foreignKey",
+					foreignKeyError: {
+						column: fk.column,
+						value: fkValue,
+						referencedTable: fk.references.table,
+						referencedColumn: fk.references.column
+					}
+				};
+			} catch (_) {}
+		}
+		return {
+			file,
+			tableName,
+			rowIndex,
+			issues: [{
+				message: errorMessage,
+				path: []
+			}],
+			type: "schema"
+		};
 	}
 	/**
 	* Execute a raw SQL query
@@ -952,9 +1078,13 @@ var LinesDB = class LinesDB {
 	/**
 	* Sync database changes back to JSONL files
 	* Uses backward transformation when available
+	* @param tableName Optional table name to sync. If not provided, syncs all loaded tables
 	*/
-	async sync() {
-		for (const [tableName] of this.tables) await this.syncTable(tableName);
+	async sync(tableName) {
+		if (tableName) {
+			if (!this.schemas.has(tableName)) throw new Error(`Table '${tableName}' is not loaded`);
+			await this.syncTable(tableName);
+		} else for (const [name] of this.schemas) await this.syncTable(name);
 	}
 	/**
 	* Execute a function within a transaction
@@ -992,241 +1122,6 @@ var LinesDB = class LinesDB {
 	}
 };
 
-//#endregion
-//#region src/validator.ts
-var Validator = class {
-	path;
-	projectRoot;
-	constructor(options) {
-		this.path = options.path;
-		this.projectRoot = options.projectRoot || process.cwd();
-	}
-	/**
-	* Validate JSONL file(s)
-	*/
-	async validate() {
-		const fullPath = this.path.startsWith("/") ? this.path : join(this.projectRoot, this.path);
-		const stats = await stat(fullPath);
-		if (stats.isDirectory()) return this.validateDirectory(fullPath);
-		else if (stats.isFile() && fullPath.endsWith(".jsonl")) return this.validateFile(fullPath);
-		else throw new Error(`Invalid path: ${this.path}. Must be a directory or .jsonl file.`);
-	}
-	/**
-	* Validate all JSONL files in a directory
-	*/
-	async validateDirectory(dirPath) {
-		const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
-		if (jsonlFiles.length === 0) throw new Error(`No JSONL files found in directory: ${dirPath}`);
-		const allErrors = [];
-		const allWarnings = [];
-		const filesWithSchema = [];
-		for (const file of jsonlFiles) if (await SchemaLoader.hasSchema(file)) filesWithSchema.push(file);
-		else {
-			const tableName = basename(file, ".jsonl");
-			allWarnings.push(`Skipping validation for '${tableName}': schema file not found`);
-		}
-		for (const file of filesWithSchema) {
-			const result = await this.validateFile(file);
-			allErrors.push(...result.errors);
-			allWarnings.push(...result.warnings);
-		}
-		if (filesWithSchema.length > 0 && allErrors.length === 0) {
-			const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
-			allErrors.push(...dbErrors);
-		}
-		return {
-			valid: allErrors.length === 0,
-			errors: allErrors,
-			warnings: allWarnings
-		};
-	}
-	/**
-	* Validate by loading data into database one row at a time
-	* This catches constraint violations and extracts detailed error information
-	*/
-	async validateWithDatabase(dirPath, jsonlFiles) {
-		const errors = [];
-		try {
-			const db = LinesDB.create({ dataDir: ":memory:" });
-			for (const file of jsonlFiles) {
-				const tableName = basename(file, ".jsonl");
-				const data = await JsonlReader.read(file);
-				let schema;
-				let foreignKeys = [];
-				let indexes = [];
-				let primaryKey;
-				try {
-					schema = await SchemaLoader.loadSchema(file);
-					const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
-					const schemaModule = await import(`${pathToFileURL$1(file.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
-					const schemaExport = schemaModule.schema || schemaModule.default;
-					if (schemaExport?.foreignKeys) foreignKeys = schemaExport.foreignKeys;
-					if (schemaExport?.indexes) indexes = schemaExport.indexes;
-					if (schemaExport?.primaryKey) primaryKey = schemaExport.primaryKey;
-				} catch (_error) {
-					continue;
-				}
-				try {
-					const tableSchema = this.createTableSchema(tableName, data, schema, foreignKeys, indexes, primaryKey);
-					this.createTableInDb(db, tableSchema);
-					for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
-						const row = data[rowIndex];
-						try {
-							this.insertRowIntoDb(db, tableName, tableSchema, row);
-						} catch (error) {
-							const constraintError = this.analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db);
-							if (constraintError) errors.push(constraintError);
-						}
-					}
-				} catch (_error) {
-					continue;
-				}
-			}
-			await db.close();
-		} catch (error) {
-			errors.push({
-				file: dirPath,
-				tableName: "database",
-				rowIndex: 0,
-				issues: [{
-					message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
-					path: []
-				}],
-				type: "schema"
-			});
-		}
-		return errors;
-	}
-	/**
-	* Create table schema from data and validation schema
-	*/
-	createTableSchema(tableName, data, validationSchema, foreignKeys, indexes, primaryKey) {
-		if (data.length === 0) throw new Error(`No data found in ${tableName}`);
-		const schema = JsonlReader.inferSchema(tableName, data);
-		if (primaryKey) {
-			const pkColumn = schema.columns.find((col) => col.name === primaryKey);
-			if (pkColumn) pkColumn.primaryKey = true;
-		} else if (!schema.columns.some((col) => col.primaryKey)) {
-			const idColumn = schema.columns.find((c) => c.name === "id");
-			if (idColumn) idColumn.primaryKey = true;
-		}
-		if (foreignKeys && foreignKeys.length > 0) schema.foreignKeys = foreignKeys;
-		if (indexes && indexes.length > 0) schema.indexes = indexes;
-		return schema;
-	}
-	/**
-	* Create table in database
-	*/
-	createTableInDb(db, schema) {
-		const columns = schema.columns.map((col) => {
-			let colDef = `${this.quoteIdentifier(col.name)} ${col.type.toUpperCase()}`;
-			if (col.primaryKey) colDef += " PRIMARY KEY";
-			return colDef;
-		});
-		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) columns.push(`FOREIGN KEY (${this.quoteIdentifier(fk.column)}) REFERENCES ${this.quoteIdentifier(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`);
-		const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(schema.name)} (${columns.join(", ")})`;
-		db.execute(sql);
-		if (schema.indexes && schema.indexes.length > 0) for (const index of schema.indexes) {
-			const indexName = index.name || `idx_${schema.name}_${index.columns.join("_")}`;
-			const uniqueKeyword = index.unique ? "UNIQUE" : "";
-			const indexColumns = index.columns.map((col) => this.quoteIdentifier(col)).join(", ");
-			const indexSql = `CREATE ${uniqueKeyword} INDEX IF NOT EXISTS ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(schema.name)} (${indexColumns})`;
-			db.execute(indexSql);
-		}
-	}
-	/**
-	* Insert a row into database
-	*/
-	insertRowIntoDb(db, tableName, schema, row) {
-		const columnNames = schema.columns.map((col) => col.name);
-		const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
-		const placeholders = columnNames.map(() => "?").join(", ");
-		const sql = `INSERT INTO ${this.quoteIdentifier(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
-		const values = columnNames.map((col) => {
-			const value = row[col];
-			if (value === null || value === void 0) return null;
-			if (typeof value === "object") return JSON.stringify(value);
-			if (typeof value === "boolean") return value ? 1 : 0;
-			return value;
-		});
-		db.execute(sql, values);
-	}
-	/**
-	* Analyze constraint error and extract detailed information
-	*/
-	analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db) {
-		const errorMessage = error instanceof Error ? error.message : String(error);
-		if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
-			const fkValue = row[fk.column];
-			if (fkValue === null || fkValue === void 0) continue;
-			try {
-				const result = db.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [fkValue]);
-				if (result.length > 0 && result[0].count === 0) return {
-					file,
-					tableName,
-					rowIndex,
-					issues: [],
-					type: "foreignKey",
-					foreignKeyError: {
-						column: fk.column,
-						value: fkValue,
-						referencedTable: fk.references.table,
-						referencedColumn: fk.references.column
-					}
-				};
-			} catch (_) {}
-		}
-		return {
-			file,
-			tableName,
-			rowIndex,
-			issues: [{
-				message: errorMessage,
-				path: []
-			}],
-			type: "schema"
-		};
-	}
-	/**
-	* Quote SQL identifier
-	*/
-	quoteIdentifier(name) {
-		return `"${name.replace(/"/g, "\"\"")}"`;
-	}
-	/**
-	* Validate a single JSONL file
-	*/
-	async validateFile(filePath) {
-		const tableName = basename(filePath, ".jsonl");
-		const data = await JsonlReader.read(filePath);
-		const schema = await SchemaLoader.loadSchema(filePath);
-		const errors = [];
-		for (let i = 0; i < data.length; i++) {
-			const row = data[i];
-			const result = schema["~standard"].validate(row);
-			if (result instanceof Promise) throw new Error("Asynchronous validation is not supported.");
-			if (result.issues && result.issues.length > 0) errors.push({
-				file: filePath,
-				tableName,
-				rowIndex: i,
-				issues: result.issues,
-				type: "schema"
-			});
-		}
-		if (errors.length === 0) {
-			const dirPath = dirname(filePath);
-			const allJsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
-			const dbErrors = await this.validateWithDatabase(dirPath, allJsonlFiles);
-			errors.push(...dbErrors.filter((e) => e.file === filePath));
-		}
-		return {
-			valid: errors.length === 0,
-			errors,
-			warnings: []
-		};
-	}
-};
-
 //#endregion
 //#region src/error-formatter.ts
 var ErrorFormatter = class {
@@ -1374,7 +1269,24 @@ program.command("generate").description("Generate TypeScript type definitions fr
 });
 program.command("validate").description("Validate JSONL file(s) against schema").argument("<path>", "File or directory path to validate").option("-v, --verbose", "Show verbose error output", false).action(async (path, options) => {
 	try {
-		const result = await new Validator({ path }).validate();
+		const stats = await stat(path);
+		let dataDir;
+		let tableName;
+		if (stats.isDirectory()) dataDir = path;
+		else if (stats.isFile() && path.endsWith(".jsonl")) {
+			dataDir = dirname(path);
+			tableName = basename(path, ".jsonl");
+		} else throw new Error(`Invalid path: ${path}. Must be a directory or .jsonl file.`);
+		const db = LinesDB.create({ dataDir });
+		let result;
+		try {
+			result = await db.initialize({
+				tableName,
+				detailedValidate: true
+			});
+		} finally {
+			await db.close();
+		}
 		if (result.warnings.length > 0) {
 			for (const warning of result.warnings) console.warn(styleText("yellow", `⚠ ${warning}`));
 			console.log("");
@@ -1423,20 +1335,57 @@ program.command("validate").description("Validate JSONL file(s) against schema")
 		process.exit(1);
 	}
 });
-program.command("migrate").description("Migrate data with transformation function").argument("<file>", "JSONL file to migrate").argument("<transform>", "Transform function (e.g., \"(row) => ({ ...row, age: row.age + 1 })\")").option("-f, --filter <expr>", "Filter expression").option("-e, --errorOutput <path>", "Output file path for transformed data when migration fails").option("-v, --verbose", "Show verbose error output", false).action(async (filePath, transformStr, options) => {
-	const tableName = (filePath.split("/").pop() || "").replace(".jsonl", "");
-	if (!tableName) {
-		console.error("Error: Invalid file path. Must be a .jsonl file");
+program.command("migrate").description("Migrate data with transformation function").argument("<path>", "File or directory path to migrate").argument("<transform>", "Transform function (e.g., \"(row) => ({ ...row, age: row.age + 1 })\")").option("-f, --filter <expr>", "Filter expression").option("-e, --errorOutput <path>", "Output file path for transformed data when migration fails").option("-v, --verbose", "Show verbose error output", false).action(async (path, transformStr, options) => {
+	try {
+		const stats = await stat(path);
+		if (stats.isDirectory()) await migrateDirectory(path, transformStr, options);
+		else if (stats.isFile() && path.endsWith(".jsonl")) await migrateFile(path, transformStr, options);
+		else {
+			console.error(`Error: Invalid path: ${path}. Must be a directory or .jsonl file.`);
+			process.exit(1);
+		}
+	} catch (error) {
+		if (error instanceof Error && "code" in error && error.code === "ENOENT") console.error(`Error: Path not found: ${path}`);
+		else console.error(`Error: ${String(error)}`);
 		process.exit(1);
 	}
-	const lastSlashIndex = filePath.lastIndexOf("/");
-	const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : ".";
-	const db = LinesDB.create({ dataDir });
-	await db.initialize();
+});
+/**
+* Migrate all JSONL files in a directory
+*/
+async function migrateDirectory(dirPath, transformStr, options) {
+	const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => entry.name);
+	if (jsonlFiles.length === 0) {
+		console.error(`Error: No JSONL files found in directory: ${dirPath}`);
+		process.exit(1);
+	}
+	console.log(`Found ${jsonlFiles.length} JSONL file(s) in directory`);
+	const db = LinesDB.create({ dataDir: dirPath });
+	const initResult = await db.initialize({ detailedValidate: true });
+	if (initResult.warnings.length > 0) for (const warning of initResult.warnings) console.warn(styleText("yellow", `⚠ ${warning}`));
+	if (!initResult.valid) {
+		console.error(`Error: Failed to initialize database due to validation errors:`);
+		const formatter = new ErrorFormatter({ verbose: options.verbose });
+		for (const error of initResult.errors) console.error(formatter.formatValidationErrors([{
+			file: error.file,
+			rowIndex: error.rowIndex,
+			issues: error.issues
+		}]));
+		await db.close();
+		process.exit(1);
+	}
+	const tableNames = db.getTableNames();
+	if (tableNames.length === 0) {
+		console.error(`Error: No tables could be loaded from directory: ${dirPath}`);
+		await db.close();
+		process.exit(1);
+	}
+	console.log(`Loaded ${tableNames.length} table(s): ${tableNames.join(", ")}\n`);
 	try {
 		const transform = runInSandbox(`(${transformStr})`);
 		if (typeof transform !== "function") {
 			console.error("Error: Transform must be a function");
+			await db.close();
 			process.exit(1);
 		}
 		let filter = void 0;
@@ -1445,49 +1394,33 @@ program.command("migrate").description("Migrate data with transformation functio
 		} catch {
 			filter = runInSandbox(`(${options.filter})`);
 		}
-		let rowsToMigrate;
-		try {
-			rowsToMigrate = filter ? db.find(tableName, filter) : db.find(tableName);
-		} catch (error) {
-			console.error(`Error: Failed to access table '${tableName}'`);
-			console.error(`  ${error instanceof Error ? error.message : String(error)}`);
-			console.error(`\nThe table may have failed to load during initialization.`);
-			console.error(`Check the table's data and schema for any constraint violations.`);
-			await db.close();
-			process.exit(1);
-		}
-		console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
-		if (rowsToMigrate.length === 0) {
-			console.log("No rows to migrate. Exiting.");
-			await db.close();
-			process.exit(0);
-		}
-		const transformedRows = rowsToMigrate.map((row) => transform(row));
-		try {
+		let totalRowsMigrated = 0;
+		let hasErrors = false;
+		for (const tableName of tableNames) try {
+			console.log(`Processing table '${tableName}'...`);
+			const rowsToMigrate = filter ? db.find(tableName, filter) : db.find(tableName);
+			if (rowsToMigrate.length === 0) {
+				console.log(`  No rows to migrate`);
+				continue;
+			}
+			console.log(`  Found ${rowsToMigrate.length} row(s) to migrate`);
+			const transformedRows = rowsToMigrate.map((row) => transform(row));
 			await db.transaction(async () => {
 				db.batchUpdate(tableName, transformedRows, { validate: true });
 			});
-			await db.close();
-			console.log(`\nMigration completed successfully:`);
-			console.log(`  ✓ ${rowsToMigrate.length} row(s) updated`);
-			process.exit(0);
+			console.log(`  ✓ ${rowsToMigrate.length} row(s) updated\n`);
+			totalRowsMigrated += rowsToMigrate.length;
 		} catch (error) {
-			await db.close();
-			if (options.errorOutput) try {
-				const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join("\n");
-				await writeFile(options.errorOutput, jsonlContent, "utf-8");
-				console.error(styleText("yellow", `\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`));
-			} catch (writeError) {
-				console.error(styleText("red", `\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`));
-			}
+			hasErrors = true;
+			console.error(styleText("red", `  ✗ Failed to migrate table '${tableName}'`));
 			const formatter = new ErrorFormatter({ verbose: options.verbose });
-			console.error(formatter.formatMigrationFailureHeader());
 			if (error instanceof Error && error.name === "ValidationError") {
 				const validationError = error;
 				if (validationError.validationErrors) {
-					console.error(`\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`);
+					console.error(`  Found ${validationError.validationErrors.length} validation error(s):\n`);
+					const rowsToMigrate = filter ? db.find(tableName, filter) : db.find(tableName);
 					const errorInfos = validationError.validationErrors.map(({ rowIndex, rowData, error: rowError }) => ({
-						file: filePath,
+						file: `${dirPath}/${tableName}.jsonl`,
 						rowIndex,
 						issues: rowError.issues,
 						data: rowData,
@@ -1495,32 +1428,165 @@ program.command("migrate").description("Migrate data with transformation functio
 					}));
 					const formatted = formatter.formatValidationErrors(errorInfos);
 					console.error(formatted);
-				} else {
-					console.error("\nValidation error:\n");
-					const errorInfo = {
-						file: filePath,
-						rowIndex: 0,
-						issues: validationError.issues
-					};
-					const formatted = formatter.formatValidationErrors([errorInfo]);
-					console.error(formatted);
 				}
-			} else if (error instanceof Error) {
-				console.error(`\n  ${error.message}`);
-				if (options.verbose && error.stack) console.error(`\nStack trace:\n${error.stack}`);
-				if (error.message.includes("UNIQUE constraint failed") || error.message.includes("FOREIGN KEY constraint failed") || error.message.includes("NOT NULL constraint failed") || error.message.includes("CHECK constraint failed")) {
-					console.error("\n  This is a SQLite constraint violation.");
-					console.error("  Please check your data and schema requirements.");
-				}
-			} else console.error(`\n  ${String(error)}`);
+			} else if (error instanceof Error) console.error(`  ${error.message}`);
 			console.error("");
+		}
+		await db.close();
+		if (hasErrors) {
+			console.error(styleText("red", `\n✗ Migration completed with errors for some tables`));
+			console.log(`Total rows migrated: ${totalRowsMigrated}`);
 			process.exit(1);
+		} else {
+			console.log(styleText("green", `\n✓ Migration completed successfully for all tables`));
+			console.log(`Total rows migrated: ${totalRowsMigrated}`);
+			process.exit(0);
 		}
 	} catch (error) {
 		await db.close();
 		throw error;
 	}
-});
+}
+/**
+* Migrate a single JSONL file
+*/
+async function migrateFile(filePath, transformStr, options) {
+	const tableName = (filePath.split("/").pop() || "").replace(".jsonl", "");
+	if (!tableName) {
+		console.error("Error: Invalid file path. Must be a .jsonl file");
+		process.exit(1);
+	}
+	const lastSlashIndex = filePath.lastIndexOf("/");
+	const dataDir = lastSlashIndex > 0 ? filePath.substring(0, lastSlashIndex) : ".";
+	let transform;
+	try {
+		const parsedTransform = runInSandbox(`(${transformStr})`);
+		if (typeof parsedTransform !== "function") {
+			console.error("Error: Transform must be a function");
+			process.exit(1);
+		}
+		transform = parsedTransform;
+	} catch (error) {
+		console.error("Error: Failed to parse transform function");
+		console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+		process.exit(1);
+	}
+	const db = LinesDB.create({ dataDir });
+	const initResult = await db.initialize({
+		tableName,
+		transform,
+		detailedValidate: true
+	});
+	if (initResult.warnings.length > 0) for (const warning of initResult.warnings) console.warn(styleText("yellow", `⚠ ${warning}`));
+	if (!initResult.valid) {
+		console.error(`Error: Failed to initialize database due to validation errors:`);
+		const formatter = new ErrorFormatter({ verbose: options.verbose });
+		for (const error of initResult.errors) console.error(formatter.formatValidationErrors([{
+			file: error.file,
+			rowIndex: error.rowIndex,
+			issues: error.issues
+		}]));
+		await db.close();
+		process.exit(1);
+	}
+	try {
+		let filter = void 0;
+		if (options.filter) try {
+			filter = JSON.parse(options.filter);
+		} catch {
+			filter = runInSandbox(`(${options.filter})`);
+		}
+		if (filter) {
+			let rowsToMigrate;
+			try {
+				rowsToMigrate = db.find(tableName, filter);
+			} catch (error) {
+				console.error(`Error: Failed to access table '${tableName}'`);
+				console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+				console.error(`\nThe table may have failed to load during initialization.`);
+				console.error(`Check the table's data and schema for any constraint violations.`);
+				await db.close();
+				process.exit(1);
+			}
+			console.log(`Found ${rowsToMigrate.length} row(s) to migrate in table '${tableName}'`);
+			if (rowsToMigrate.length === 0) {
+				console.log("No rows to migrate. Exiting.");
+				await db.close();
+				process.exit(0);
+			}
+			const transformedRows = rowsToMigrate.map((row) => transform(row));
+			try {
+				await db.transaction(async () => {
+					db.batchUpdate(tableName, transformedRows, { validate: true });
+				});
+				await db.close();
+				console.log(`\nMigration completed successfully:`);
+				console.log(`  ✓ ${rowsToMigrate.length} row(s) updated`);
+				process.exit(0);
+			} catch (error) {
+				await db.close();
+				if (options.errorOutput) try {
+					const jsonlContent = transformedRows.map((row) => JSON.stringify(row)).join("\n");
+					await writeFile(options.errorOutput, jsonlContent, "utf-8");
+					console.error(styleText("yellow", `\n⚠ Transformed data (${transformedRows.length} rows) written to: ${options.errorOutput}`));
+				} catch (writeError) {
+					console.error(styleText("red", `\n✗ Failed to write error output file: ${writeError instanceof Error ? writeError.message : String(writeError)}`));
+				}
+				const formatter = new ErrorFormatter({ verbose: options.verbose });
+				console.error(formatter.formatMigrationFailureHeader());
+				if (error instanceof Error && error.name === "ValidationError") {
+					const validationError = error;
+					if (validationError.validationErrors) {
+						console.error(`\nFound ${validationError.validationErrors.length} validation error(s) in transformed data:\n`);
+						const errorInfos = validationError.validationErrors.map(({ rowIndex, rowData, error: rowError }) => ({
+							file: filePath,
+							rowIndex,
+							issues: rowError.issues,
+							data: rowData,
+							originalData: rowsToMigrate[rowIndex]
+						}));
+						const formatted = formatter.formatValidationErrors(errorInfos);
+						console.error(formatted);
+					} else {
+						console.error("\nValidation error:\n");
+						const errorInfo = {
+							file: filePath,
+							rowIndex: 0,
+							issues: validationError.issues
+						};
+						const formatted = formatter.formatValidationErrors([errorInfo]);
+						console.error(formatted);
+					}
+				} else if (error instanceof Error) {
+					console.error(`\n  ${error.message}`);
+					if (options.verbose && error.stack) console.error(`\nStack trace:\n${error.stack}`);
+					if (error.message.includes("UNIQUE constraint failed") || error.message.includes("FOREIGN KEY constraint failed") || error.message.includes("NOT NULL constraint failed") || error.message.includes("CHECK constraint failed")) {
+						console.error("\n  This is a SQLite constraint violation.");
+						console.error("  Please check your data and schema requirements.");
+					}
+				} else console.error(`\n  ${String(error)}`);
+				console.error("");
+				process.exit(1);
+			}
+		} else try {
+			const allRows = db.find(tableName);
+			console.log(`Migrated ${allRows.length} row(s) in table '${tableName}'`);
+			await db.sync(tableName);
+			await db.close();
+			console.log(`\nMigration completed successfully:`);
+			console.log(`  ✓ ${allRows.length} row(s) updated`);
+			process.exit(0);
+		} catch (error) {
+			await db.close();
+			console.error("Error: Failed to sync changes to file");
+			console.error(`  ${error instanceof Error ? error.message : String(error)}`);
+			process.exit(1);
+		}
+	} catch (error) {
+		await db.close();
+		throw error;
+	}
+}
 program.parse();
 
 //#endregion