@toiroakr/lines-db 0.3.0 → 0.4.1

package/dist/index.js

@@ -261,9 +261,9 @@ var DirectoryScanner = class {
 * const schema = defineSchema(
 *   v.object({ id: v.number(), customerId: v.number() }),
 *   {
-*     primaryKey: ['id'],
+*     primaryKey: 'id',
 *     foreignKeys: [
-*       { columns: ['customerId'], references: { table: 'users', columns: ['id'] } }
+*       { column: 'customerId', references: { table: 'users', column: 'id' } }
 *     ]
 *   }
 * );
@@ -337,7 +337,8 @@ var LinesDB = class LinesDB {
 			let foreignKeys;
 			try {
 				const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
-				foreignKeys = (await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`)).foreignKeys;
+				const schemaModule = await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+				foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
 			} catch {}
 			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
 				const referencedTable = fk.references.table;
@@ -364,9 +365,13 @@ var LinesDB = class LinesDB {
 		if (!config.validationSchema) try {
 			const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
 			const schemaModule = await import(`${pathToFileURL$1(config.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
-			if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
-			if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
-			if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
+			const schemaExport = schemaModule.schema || schemaModule.default;
+			if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
+			else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
+			if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
+			else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
+			if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
+			else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
 		} catch (_error) {}
 		this.validationSchemas.set(tableName, validationSchema);
 		const validationErrors = [];
@@ -402,9 +407,12 @@ var LinesDB = class LinesDB {
 		const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
 		const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
 		const indexes = biSchema?.indexes || schemaMetadata.indexes;
-		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) for (const pkColumn of primaryKey) {
-			const col = schema.columns.find((c) => c.name === pkColumn);
+		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const col = schema.columns.find((c) => c.name === primaryKey);
 			if (col) col.primaryKey = true;
+		} else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
 		}
 		if (foreignKeys) schema.foreignKeys = foreignKeys;
 		if (indexes) schema.indexes = indexes;
@@ -428,7 +436,7 @@ var LinesDB = class LinesDB {
 		});
 		const foreignKeyDefs = [];
 		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
-			const fkParts = [`FOREIGN KEY (${fk.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${fk.references.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`];
+			const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
 			if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
 			if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
 			foreignKeyDefs.push(fkParts.join(" "));
@@ -1066,9 +1074,9 @@ var Validator = class {
 			allErrors.push(...result.errors);
 			allWarnings.push(...result.warnings);
 		}
-		if (filesWithSchema.length > 0) {
-			const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-			allErrors.push(...fkErrors);
+		if (filesWithSchema.length > 0 && allErrors.length === 0) {
+			const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+			allErrors.push(...dbErrors);
 		}
 		return {
 			valid: allErrors.length === 0,
@@ -1077,57 +1085,159 @@ var Validator = class {
 		};
 	}
 	/**
-	* Validate foreign key constraints across all tables
+	* Validate by loading data into database one row at a time
+	* This catches constraint violations and extracts detailed error information
 	*/
-	async validateForeignKeys(dirPath, jsonlFiles) {
+	async validateWithDatabase(dirPath, jsonlFiles) {
 		const errors = [];
-		const tableData = /* @__PURE__ */ new Map();
-		const tableSchemas = /* @__PURE__ */ new Map();
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const data = await JsonlReader.read(file);
-			const schema = await SchemaLoader.loadSchema(file);
-			tableData.set(tableName, data);
-			tableSchemas.set(tableName, schema);
-		}
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const schema = tableSchemas.get(tableName);
-			const data = tableData.get(tableName);
-			if (!schema || !data || !schema.foreignKeys) continue;
-			for (const fk of schema.foreignKeys) {
-				const referencedTable = fk.references.table;
-				const referencedData = tableData.get(referencedTable);
-				if (!referencedData) continue;
-				const referencedValues = /* @__PURE__ */ new Set();
-				for (const refRow of referencedData) {
-					const keyValues = fk.references.columns.map((col) => refRow[col]);
-					const compositeKey = JSON.stringify(keyValues);
-					referencedValues.add(compositeKey);
+		try {
+			const db = LinesDB.create({ dataDir: ":memory:" });
+			for (const file of jsonlFiles) {
+				const tableName = basename(file, ".jsonl");
+				const data = await JsonlReader.read(file);
+				let schema;
+				let foreignKeys = [];
+				let indexes = [];
+				let primaryKey;
+				try {
+					schema = await SchemaLoader.loadSchema(file);
+					const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+					const schemaModule = await import(`${pathToFileURL$1(file.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+					const schemaExport = schemaModule.schema || schemaModule.default;
+					if (schemaExport?.foreignKeys) foreignKeys = schemaExport.foreignKeys;
+					if (schemaExport?.indexes) indexes = schemaExport.indexes;
+					if (schemaExport?.primaryKey) primaryKey = schemaExport.primaryKey;
+				} catch (_error) {
+					continue;
 				}
-				for (let i = 0; i < data.length; i++) {
-					const row = data[i];
-					const foreignKeyValues = fk.columns.map((col) => row[col]);
-					const compositeKey = JSON.stringify(foreignKeyValues);
-					if (!referencedValues.has(compositeKey)) errors.push({
-						file,
-						tableName,
-						rowIndex: i,
-						issues: [],
-						type: "foreignKey",
-						foreignKeyError: {
-							column: fk.columns.join(", "),
-							value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-							referencedTable,
-							referencedColumn: fk.references.columns.join(", ")
+				try {
+					const tableSchema = this.createTableSchema(tableName, data, schema, foreignKeys, indexes, primaryKey);
+					this.createTableInDb(db, tableSchema);
+					for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+						const row = data[rowIndex];
+						try {
+							this.insertRowIntoDb(db, tableName, tableSchema, row);
+						} catch (error) {
+							const constraintError = this.analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db);
+							if (constraintError) errors.push(constraintError);
 						}
-					});
+					}
+				} catch (_error) {
+					continue;
 				}
 			}
+			await db.close();
+		} catch (error) {
+			errors.push({
+				file: dirPath,
+				tableName: "database",
+				rowIndex: 0,
+				issues: [{
+					message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+					path: []
+				}],
+				type: "schema"
+			});
 		}
 		return errors;
 	}
 	/**
+	* Create table schema from data and validation schema
+	*/
+	createTableSchema(tableName, data, validationSchema, foreignKeys, indexes, primaryKey) {
+		if (data.length === 0) throw new Error(`No data found in ${tableName}`);
+		const schema = JsonlReader.inferSchema(tableName, data);
+		if (primaryKey) {
+			const pkColumn = schema.columns.find((col) => col.name === primaryKey);
+			if (pkColumn) pkColumn.primaryKey = true;
+		} else if (!schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
+		}
+		if (foreignKeys && foreignKeys.length > 0) schema.foreignKeys = foreignKeys;
+		if (indexes && indexes.length > 0) schema.indexes = indexes;
+		return schema;
+	}
+	/**
+	* Create table in database
+	*/
+	createTableInDb(db, schema) {
+		const columns = schema.columns.map((col) => {
+			let colDef = `${this.quoteIdentifier(col.name)} ${col.type.toUpperCase()}`;
+			if (col.primaryKey) colDef += " PRIMARY KEY";
+			return colDef;
+		});
+		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) columns.push(`FOREIGN KEY (${this.quoteIdentifier(fk.column)}) REFERENCES ${this.quoteIdentifier(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`);
+		const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(schema.name)} (${columns.join(", ")})`;
+		db.execute(sql);
+		if (schema.indexes && schema.indexes.length > 0) for (const index of schema.indexes) {
+			const indexName = index.name || `idx_${schema.name}_${index.columns.join("_")}`;
+			const uniqueKeyword = index.unique ? "UNIQUE" : "";
+			const indexColumns = index.columns.map((col) => this.quoteIdentifier(col)).join(", ");
+			const indexSql = `CREATE ${uniqueKeyword} INDEX IF NOT EXISTS ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(schema.name)} (${indexColumns})`;
+			db.execute(indexSql);
+		}
+	}
+	/**
+	* Insert a row into database
+	*/
+	insertRowIntoDb(db, tableName, schema, row) {
+		const columnNames = schema.columns.map((col) => col.name);
+		const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
+		const placeholders = columnNames.map(() => "?").join(", ");
+		const sql = `INSERT INTO ${this.quoteIdentifier(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
+		const values = columnNames.map((col) => {
+			const value = row[col];
+			if (value === null || value === void 0) return null;
+			if (typeof value === "object") return JSON.stringify(value);
+			if (typeof value === "boolean") return value ? 1 : 0;
+			return value;
+		});
+		db.execute(sql, values);
+	}
+	/**
+	* Analyze constraint error and extract detailed information
+	*/
+	analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db) {
+		const errorMessage = error instanceof Error ? error.message : String(error);
+		if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
+			const fkValue = row[fk.column];
+			if (fkValue === null || fkValue === void 0) continue;
+			try {
+				const result = db.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [fkValue]);
+				if (result.length > 0 && result[0].count === 0) return {
+					file,
+					tableName,
+					rowIndex,
+					issues: [],
+					type: "foreignKey",
+					foreignKeyError: {
+						column: fk.column,
+						value: fkValue,
+						referencedTable: fk.references.table,
+						referencedColumn: fk.references.column
+					}
+				};
+			} catch (_) {}
+		}
+		return {
+			file,
+			tableName,
+			rowIndex,
+			issues: [{
+				message: errorMessage,
+				path: []
+			}],
+			type: "schema"
+		};
+	}
+	/**
+	* Quote SQL identifier
+	*/
+	quoteIdentifier(name) {
+		return `"${name.replace(/"/g, "\"\"")}"`;
+	}
+	/**
 	* Validate a single JSONL file
 	*/
 	async validateFile(filePath) {
@@ -1147,6 +1257,12 @@ var Validator = class {
 				type: "schema"
 			});
 		}
+		if (errors.length === 0) {
+			const dirPath = dirname(filePath);
+			const allJsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
+			const dbErrors = await this.validateWithDatabase(dirPath, allJsonlFiles);
+			errors.push(...dbErrors.filter((e) => e.file === filePath));
+		}
 		return {
 			valid: errors.length === 0,
 			errors,