@toiroakr/lines-db 0.4.0 → 0.4.1

package/dist/index.js

@@ -1085,37 +1085,48 @@ var Validator = class {
 		};
 	}
 	/**
-	* Validate by loading data into an actual database
-	* This catches constraint violations (unique, primary key, foreign key, etc.)
+	* Validate by loading data into database one row at a time
+	* This catches constraint violations and extracts detailed error information
 	*/
 	async validateWithDatabase(dirPath, jsonlFiles) {
 		const errors = [];
-		const warnMessages = [];
-		const originalWarn = console.warn;
-		console.warn = (...args) => {
-			const message = args.map((arg) => String(arg)).join(" ");
-			warnMessages.push(message);
-			originalWarn(...args);
-		};
 		try {
-			const db = LinesDB.create({ dataDir: dirPath });
-			await db.initialize();
-			await db.close();
-			for (const message of warnMessages) if (message.includes("Failed to load table")) {
-				const tableNameMatch = message.match(/Failed to load table '([^']+)'/);
-				const tableName = tableNameMatch ? tableNameMatch[1] : "unknown";
-				const file = jsonlFiles.find((f) => basename(f, ".jsonl") === tableName);
-				errors.push({
-					file: file || `${dirPath}/${tableName}.jsonl`,
-					tableName,
-					rowIndex: 0,
-					issues: [{
-						message: message.replace(/^Warning:\s*/, ""),
-						path: []
-					}],
-					type: "schema"
-				});
+			const db = LinesDB.create({ dataDir: ":memory:" });
+			for (const file of jsonlFiles) {
+				const tableName = basename(file, ".jsonl");
+				const data = await JsonlReader.read(file);
+				let schema;
+				let foreignKeys = [];
+				let indexes = [];
+				let primaryKey;
+				try {
+					schema = await SchemaLoader.loadSchema(file);
+					const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+					const schemaModule = await import(`${pathToFileURL$1(file.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+					const schemaExport = schemaModule.schema || schemaModule.default;
+					if (schemaExport?.foreignKeys) foreignKeys = schemaExport.foreignKeys;
+					if (schemaExport?.indexes) indexes = schemaExport.indexes;
+					if (schemaExport?.primaryKey) primaryKey = schemaExport.primaryKey;
+				} catch (_error) {
+					continue;
+				}
+				try {
+					const tableSchema = this.createTableSchema(tableName, data, schema, foreignKeys, indexes, primaryKey);
+					this.createTableInDb(db, tableSchema);
+					for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+						const row = data[rowIndex];
+						try {
+							this.insertRowIntoDb(db, tableName, tableSchema, row);
+						} catch (error) {
+							const constraintError = this.analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db);
+							if (constraintError) errors.push(constraintError);
+						}
+					}
+				} catch (_error) {
+					continue;
+				}
 			}
+			await db.close();
 		} catch (error) {
 			errors.push({
 				file: dirPath,
@@ -1127,12 +1138,106 @@ var Validator = class {
 				}],
 				type: "schema"
 			});
-		} finally {
-			console.warn = originalWarn;
 		}
 		return errors;
 	}
 	/**
+	* Create table schema from data and validation schema
+	*/
+	createTableSchema(tableName, data, validationSchema, foreignKeys, indexes, primaryKey) {
+		if (data.length === 0) throw new Error(`No data found in ${tableName}`);
+		const schema = JsonlReader.inferSchema(tableName, data);
+		if (primaryKey) {
+			const pkColumn = schema.columns.find((col) => col.name === primaryKey);
+			if (pkColumn) pkColumn.primaryKey = true;
+		} else if (!schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
+		}
+		if (foreignKeys && foreignKeys.length > 0) schema.foreignKeys = foreignKeys;
+		if (indexes && indexes.length > 0) schema.indexes = indexes;
+		return schema;
+	}
+	/**
+	* Create table in database
+	*/
+	createTableInDb(db, schema) {
+		const columns = schema.columns.map((col) => {
+			let colDef = `${this.quoteIdentifier(col.name)} ${col.type.toUpperCase()}`;
+			if (col.primaryKey) colDef += " PRIMARY KEY";
+			return colDef;
+		});
+		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) columns.push(`FOREIGN KEY (${this.quoteIdentifier(fk.column)}) REFERENCES ${this.quoteIdentifier(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`);
+		const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(schema.name)} (${columns.join(", ")})`;
+		db.execute(sql);
+		if (schema.indexes && schema.indexes.length > 0) for (const index of schema.indexes) {
+			const indexName = index.name || `idx_${schema.name}_${index.columns.join("_")}`;
+			const uniqueKeyword = index.unique ? "UNIQUE" : "";
+			const indexColumns = index.columns.map((col) => this.quoteIdentifier(col)).join(", ");
+			const indexSql = `CREATE ${uniqueKeyword} INDEX IF NOT EXISTS ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(schema.name)} (${indexColumns})`;
+			db.execute(indexSql);
+		}
+	}
+	/**
+	* Insert a row into database
+	*/
+	insertRowIntoDb(db, tableName, schema, row) {
+		const columnNames = schema.columns.map((col) => col.name);
+		const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
+		const placeholders = columnNames.map(() => "?").join(", ");
+		const sql = `INSERT INTO ${this.quoteIdentifier(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
+		const values = columnNames.map((col) => {
+			const value = row[col];
+			if (value === null || value === void 0) return null;
+			if (typeof value === "object") return JSON.stringify(value);
+			if (typeof value === "boolean") return value ? 1 : 0;
+			return value;
+		});
+		db.execute(sql, values);
+	}
+	/**
+	* Analyze constraint error and extract detailed information
+	*/
+	analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db) {
+		const errorMessage = error instanceof Error ? error.message : String(error);
+		if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
+			const fkValue = row[fk.column];
+			if (fkValue === null || fkValue === void 0) continue;
+			try {
+				const result = db.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [fkValue]);
+				if (result.length > 0 && result[0].count === 0) return {
+					file,
+					tableName,
+					rowIndex,
+					issues: [],
+					type: "foreignKey",
+					foreignKeyError: {
+						column: fk.column,
+						value: fkValue,
+						referencedTable: fk.references.table,
+						referencedColumn: fk.references.column
+					}
+				};
+			} catch (_) {}
+		}
+		return {
+			file,
+			tableName,
+			rowIndex,
+			issues: [{
+				message: errorMessage,
+				path: []
+			}],
+			type: "schema"
+		};
+	}
+	/**
+	* Quote SQL identifier
+	*/
+	quoteIdentifier(name) {
+		return `"${name.replace(/"/g, "\"\"")}"`;
+	}
+	/**
 	* Validate a single JSONL file
 	*/
 	async validateFile(filePath) {
@@ -1154,8 +1259,9 @@ var Validator = class {
 		}
 		if (errors.length === 0) {
 			const dirPath = dirname(filePath);
-			const dbErrors = await this.validateWithDatabase(dirPath, [filePath]);
-			errors.push(...dbErrors);
+			const allJsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
+			const dbErrors = await this.validateWithDatabase(dirPath, allJsonlFiles);
+			errors.push(...dbErrors.filter((e) => e.file === filePath));
 		}
 		return {
 			valid: errors.length === 0,