@toiroakr/lines-db 0.3.0 → 0.4.1

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @toiroakr/lines-db
 
+## 0.4.1
+
+### Patch Changes
+
+- b281dc8: Fix constraint validation in validator to properly detect primary key and unique index violations
+
+  Previously, the validator was not creating indexes from schema metadata and was missing the default primaryKey behavior, causing constraint violations to go undetected. This fix ensures:
+  - Indexes (both unique and non-unique) are now properly created from schema metadata in the validation database
+  - Primary key defaults to 'id' column when not explicitly specified, matching database.ts behavior
+  - Constraint violations are properly detected by inserting rows into an in-memory database and catching SQLite exceptions
+  - Detailed error information is extracted from SQLite error messages for better diagnostics
+
+  Added comprehensive regression tests to prevent this issue from recurring.
+
+## 0.4.0
+
+### Minor Changes
+
+- a662484: - Allow flexible schema export methods (support loading from `schema` or `default` exports)
+  - Enhance constraint validation by loading data into an actual database (catches unique, primary key, and foreign key violations)
+  - Add fallback logic to automatically use `id` column as primary key when it exists and no primary key is explicitly defined
+
 ## 0.3.0
 
 ### Minor Changes

package/bin/cli.js

@@ -248,133 +248,6 @@ var SchemaLoader = class {
 	}
 };
 
-//#endregion
-//#region src/validator.ts
-var Validator = class {
-	path;
-	projectRoot;
-	constructor(options) {
-		this.path = options.path;
-		this.projectRoot = options.projectRoot || process.cwd();
-	}
-	/**
-	* Validate JSONL file(s)
-	*/
-	async validate() {
-		const fullPath = this.path.startsWith("/") ? this.path : join(this.projectRoot, this.path);
-		const stats = await stat(fullPath);
-		if (stats.isDirectory()) return this.validateDirectory(fullPath);
-		else if (stats.isFile() && fullPath.endsWith(".jsonl")) return this.validateFile(fullPath);
-		else throw new Error(`Invalid path: ${this.path}. Must be a directory or .jsonl file.`);
-	}
-	/**
-	* Validate all JSONL files in a directory
-	*/
-	async validateDirectory(dirPath) {
-		const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
-		if (jsonlFiles.length === 0) throw new Error(`No JSONL files found in directory: ${dirPath}`);
-		const allErrors = [];
-		const allWarnings = [];
-		const filesWithSchema = [];
-		for (const file of jsonlFiles) if (await SchemaLoader.hasSchema(file)) filesWithSchema.push(file);
-		else {
-			const tableName = basename(file, ".jsonl");
-			allWarnings.push(`Skipping validation for '${tableName}': schema file not found`);
-		}
-		for (const file of filesWithSchema) {
-			const result = await this.validateFile(file);
-			allErrors.push(...result.errors);
-			allWarnings.push(...result.warnings);
-		}
-		if (filesWithSchema.length > 0) {
-			const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-			allErrors.push(...fkErrors);
-		}
-		return {
-			valid: allErrors.length === 0,
-			errors: allErrors,
-			warnings: allWarnings
-		};
-	}
-	/**
-	* Validate foreign key constraints across all tables
-	*/
-	async validateForeignKeys(dirPath, jsonlFiles) {
-		const errors = [];
-		const tableData = /* @__PURE__ */ new Map();
-		const tableSchemas = /* @__PURE__ */ new Map();
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const data = await JsonlReader.read(file);
-			const schema = await SchemaLoader.loadSchema(file);
-			tableData.set(tableName, data);
-			tableSchemas.set(tableName, schema);
-		}
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const schema = tableSchemas.get(tableName);
-			const data = tableData.get(tableName);
-			if (!schema || !data || !schema.foreignKeys) continue;
-			for (const fk of schema.foreignKeys) {
-				const referencedTable = fk.references.table;
-				const referencedData = tableData.get(referencedTable);
-				if (!referencedData) continue;
-				const referencedValues = /* @__PURE__ */ new Set();
-				for (const refRow of referencedData) {
-					const keyValues = fk.references.columns.map((col) => refRow[col]);
-					const compositeKey = JSON.stringify(keyValues);
-					referencedValues.add(compositeKey);
-				}
-				for (let i = 0; i < data.length; i++) {
-					const row = data[i];
-					const foreignKeyValues = fk.columns.map((col) => row[col]);
-					const compositeKey = JSON.stringify(foreignKeyValues);
-					if (!referencedValues.has(compositeKey)) errors.push({
-						file,
-						tableName,
-						rowIndex: i,
-						issues: [],
-						type: "foreignKey",
-						foreignKeyError: {
-							column: fk.columns.join(", "),
-							value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-							referencedTable,
-							referencedColumn: fk.references.columns.join(", ")
-						}
-					});
-				}
-			}
-		}
-		return errors;
-	}
-	/**
-	* Validate a single JSONL file
-	*/
-	async validateFile(filePath) {
-		const tableName = basename(filePath, ".jsonl");
-		const data = await JsonlReader.read(filePath);
-		const schema = await SchemaLoader.loadSchema(filePath);
-		const errors = [];
-		for (let i = 0; i < data.length; i++) {
-			const row = data[i];
-			const result = schema["~standard"].validate(row);
-			if (result instanceof Promise) throw new Error("Asynchronous validation is not supported.");
-			if (result.issues && result.issues.length > 0) errors.push({
-				file: filePath,
-				tableName,
-				rowIndex: i,
-				issues: result.issues,
-				type: "schema"
-			});
-		}
-		return {
-			valid: errors.length === 0,
-			errors,
-			warnings: []
-		};
-	}
-};
-
 //#endregion
 //#region src/runtime.ts
 function detectRuntime() {
@@ -529,7 +402,8 @@ var LinesDB = class LinesDB {
 			let foreignKeys;
 			try {
 				const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
-				foreignKeys = (await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`)).foreignKeys;
+				const schemaModule = await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+				foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
 			} catch {}
 			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
 				const referencedTable = fk.references.table;
@@ -556,9 +430,13 @@ var LinesDB = class LinesDB {
 		if (!config.validationSchema) try {
 			const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
 			const schemaModule = await import(`${pathToFileURL$1(config.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
-			if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
-			if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
-			if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
+			const schemaExport = schemaModule.schema || schemaModule.default;
+			if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
+			else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
+			if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
+			else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
+			if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
+			else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
 		} catch (_error) {}
 		this.validationSchemas.set(tableName, validationSchema);
 		const validationErrors = [];
@@ -594,9 +472,12 @@ var LinesDB = class LinesDB {
 		const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
 		const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
 		const indexes = biSchema?.indexes || schemaMetadata.indexes;
-		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) for (const pkColumn of primaryKey) {
-			const col = schema.columns.find((c) => c.name === pkColumn);
+		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const col = schema.columns.find((c) => c.name === primaryKey);
 			if (col) col.primaryKey = true;
+		} else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
 		}
 		if (foreignKeys) schema.foreignKeys = foreignKeys;
 		if (indexes) schema.indexes = indexes;
@@ -620,7 +501,7 @@ var LinesDB = class LinesDB {
 		});
 		const foreignKeyDefs = [];
 		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
-			const fkParts = [`FOREIGN KEY (${fk.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${fk.references.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`];
+			const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
 			if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
 			if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
 			foreignKeyDefs.push(fkParts.join(" "));
@@ -1111,6 +992,241 @@ var LinesDB = class LinesDB {
 	}
 };
 
+//#endregion
+//#region src/validator.ts
+var Validator = class {
+	path;
+	projectRoot;
+	constructor(options) {
+		this.path = options.path;
+		this.projectRoot = options.projectRoot || process.cwd();
+	}
+	/**
+	* Validate JSONL file(s)
+	*/
+	async validate() {
+		const fullPath = this.path.startsWith("/") ? this.path : join(this.projectRoot, this.path);
+		const stats = await stat(fullPath);
+		if (stats.isDirectory()) return this.validateDirectory(fullPath);
+		else if (stats.isFile() && fullPath.endsWith(".jsonl")) return this.validateFile(fullPath);
+		else throw new Error(`Invalid path: ${this.path}. Must be a directory or .jsonl file.`);
+	}
+	/**
+	* Validate all JSONL files in a directory
+	*/
+	async validateDirectory(dirPath) {
+		const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
+		if (jsonlFiles.length === 0) throw new Error(`No JSONL files found in directory: ${dirPath}`);
+		const allErrors = [];
+		const allWarnings = [];
+		const filesWithSchema = [];
+		for (const file of jsonlFiles) if (await SchemaLoader.hasSchema(file)) filesWithSchema.push(file);
+		else {
+			const tableName = basename(file, ".jsonl");
+			allWarnings.push(`Skipping validation for '${tableName}': schema file not found`);
+		}
+		for (const file of filesWithSchema) {
+			const result = await this.validateFile(file);
+			allErrors.push(...result.errors);
+			allWarnings.push(...result.warnings);
+		}
+		if (filesWithSchema.length > 0 && allErrors.length === 0) {
+			const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+			allErrors.push(...dbErrors);
+		}
+		return {
+			valid: allErrors.length === 0,
+			errors: allErrors,
+			warnings: allWarnings
+		};
+	}
+	/**
+	* Validate by loading data into database one row at a time
+	* This catches constraint violations and extracts detailed error information
+	*/
+	async validateWithDatabase(dirPath, jsonlFiles) {
+		const errors = [];
+		try {
+			const db = LinesDB.create({ dataDir: ":memory:" });
+			for (const file of jsonlFiles) {
+				const tableName = basename(file, ".jsonl");
+				const data = await JsonlReader.read(file);
+				let schema;
+				let foreignKeys = [];
+				let indexes = [];
+				let primaryKey;
+				try {
+					schema = await SchemaLoader.loadSchema(file);
+					const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+					const schemaModule = await import(`${pathToFileURL$1(file.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+					const schemaExport = schemaModule.schema || schemaModule.default;
+					if (schemaExport?.foreignKeys) foreignKeys = schemaExport.foreignKeys;
+					if (schemaExport?.indexes) indexes = schemaExport.indexes;
+					if (schemaExport?.primaryKey) primaryKey = schemaExport.primaryKey;
+				} catch (_error) {
+					continue;
+				}
+				try {
+					const tableSchema = this.createTableSchema(tableName, data, schema, foreignKeys, indexes, primaryKey);
+					this.createTableInDb(db, tableSchema);
+					for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+						const row = data[rowIndex];
+						try {
+							this.insertRowIntoDb(db, tableName, tableSchema, row);
+						} catch (error) {
+							const constraintError = this.analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db);
+							if (constraintError) errors.push(constraintError);
+						}
+					}
+				} catch (_error) {
+					continue;
+				}
+			}
+			await db.close();
+		} catch (error) {
+			errors.push({
+				file: dirPath,
+				tableName: "database",
+				rowIndex: 0,
+				issues: [{
+					message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+					path: []
+				}],
+				type: "schema"
+			});
+		}
+		return errors;
+	}
+	/**
+	* Create table schema from data and validation schema
+	*/
+	createTableSchema(tableName, data, validationSchema, foreignKeys, indexes, primaryKey) {
+		if (data.length === 0) throw new Error(`No data found in ${tableName}`);
+		const schema = JsonlReader.inferSchema(tableName, data);
+		if (primaryKey) {
+			const pkColumn = schema.columns.find((col) => col.name === primaryKey);
+			if (pkColumn) pkColumn.primaryKey = true;
+		} else if (!schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
+		}
+		if (foreignKeys && foreignKeys.length > 0) schema.foreignKeys = foreignKeys;
+		if (indexes && indexes.length > 0) schema.indexes = indexes;
+		return schema;
+	}
+	/**
+	* Create table in database
+	*/
+	createTableInDb(db, schema) {
+		const columns = schema.columns.map((col) => {
+			let colDef = `${this.quoteIdentifier(col.name)} ${col.type.toUpperCase()}`;
+			if (col.primaryKey) colDef += " PRIMARY KEY";
+			return colDef;
+		});
+		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) columns.push(`FOREIGN KEY (${this.quoteIdentifier(fk.column)}) REFERENCES ${this.quoteIdentifier(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`);
+		const sql = `CREATE TABLE IF NOT EXISTS ${this.quoteIdentifier(schema.name)} (${columns.join(", ")})`;
+		db.execute(sql);
+		if (schema.indexes && schema.indexes.length > 0) for (const index of schema.indexes) {
+			const indexName = index.name || `idx_${schema.name}_${index.columns.join("_")}`;
+			const uniqueKeyword = index.unique ? "UNIQUE" : "";
+			const indexColumns = index.columns.map((col) => this.quoteIdentifier(col)).join(", ");
+			const indexSql = `CREATE ${uniqueKeyword} INDEX IF NOT EXISTS ${this.quoteIdentifier(indexName)} ON ${this.quoteIdentifier(schema.name)} (${indexColumns})`;
+			db.execute(indexSql);
+		}
+	}
+	/**
+	* Insert a row into database
+	*/
+	insertRowIntoDb(db, tableName, schema, row) {
+		const columnNames = schema.columns.map((col) => col.name);
+		const quotedColumns = columnNames.map((name) => this.quoteIdentifier(name));
+		const placeholders = columnNames.map(() => "?").join(", ");
+		const sql = `INSERT INTO ${this.quoteIdentifier(tableName)} (${quotedColumns.join(", ")}) VALUES (${placeholders})`;
+		const values = columnNames.map((col) => {
+			const value = row[col];
+			if (value === null || value === void 0) return null;
+			if (typeof value === "object") return JSON.stringify(value);
+			if (typeof value === "boolean") return value ? 1 : 0;
+			return value;
+		});
+		db.execute(sql, values);
+	}
+	/**
+	* Analyze constraint error and extract detailed information
+	*/
+	analyzeConstraintError(error, file, tableName, rowIndex, row, foreignKeys, db) {
+		const errorMessage = error instanceof Error ? error.message : String(error);
+		if (errorMessage.includes("FOREIGN KEY constraint failed")) for (const fk of foreignKeys) {
+			const fkValue = row[fk.column];
+			if (fkValue === null || fkValue === void 0) continue;
+			try {
+				const result = db.query(`SELECT COUNT(*) as count FROM ${this.quoteIdentifier(fk.references.table)} WHERE ${this.quoteIdentifier(fk.references.column)} = ?`, [fkValue]);
+				if (result.length > 0 && result[0].count === 0) return {
+					file,
+					tableName,
+					rowIndex,
+					issues: [],
+					type: "foreignKey",
+					foreignKeyError: {
+						column: fk.column,
+						value: fkValue,
+						referencedTable: fk.references.table,
+						referencedColumn: fk.references.column
+					}
+				};
+			} catch (_) {}
+		}
+		return {
+			file,
+			tableName,
+			rowIndex,
+			issues: [{
+				message: errorMessage,
+				path: []
+			}],
+			type: "schema"
+		};
+	}
+	/**
+	* Quote SQL identifier
+	*/
+	quoteIdentifier(name) {
+		return `"${name.replace(/"/g, "\"\"")}"`;
+	}
+	/**
+	* Validate a single JSONL file
+	*/
+	async validateFile(filePath) {
+		const tableName = basename(filePath, ".jsonl");
+		const data = await JsonlReader.read(filePath);
+		const schema = await SchemaLoader.loadSchema(filePath);
+		const errors = [];
+		for (let i = 0; i < data.length; i++) {
+			const row = data[i];
+			const result = schema["~standard"].validate(row);
+			if (result instanceof Promise) throw new Error("Asynchronous validation is not supported.");
+			if (result.issues && result.issues.length > 0) errors.push({
+				file: filePath,
+				tableName,
+				rowIndex: i,
+				issues: result.issues,
+				type: "schema"
+			});
+		}
+		if (errors.length === 0) {
+			const dirPath = dirname(filePath);
+			const allJsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
+			const dbErrors = await this.validateWithDatabase(dirPath, allJsonlFiles);
+			errors.push(...dbErrors.filter((e) => e.file === filePath));
+		}
+		return {
+			valid: errors.length === 0,
+			errors,
+			warnings: []
+		};
+	}
+};
+
 //#endregion
 //#region src/error-formatter.ts
 var ErrorFormatter = class {