@toiroakr/lines-db 0.2.1 → 0.4.0

package/dist/index.js

@@ -29,6 +29,7 @@ function createDatabase(path = ":memory:") {
 function createNodeDatabase(path) {
 	const { DatabaseSync } = __require("node:sqlite");
 	const db = new DatabaseSync(path);
+	db.exec("PRAGMA foreign_keys = ON");
 	return {
 		prepare(sql) {
 			const stmt = db.prepare(sql);
@@ -124,7 +125,7 @@ var JsonlReader = class {
 		};
 	}
 	static inferType(value) {
-		if (value === null) return "NULL";
+		if (value === null || value === void 0) return "NULL";
 		if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
 		if (typeof value === "string") return "TEXT";
 		if (typeof value === "boolean") return "INTEGER";
@@ -260,9 +261,9 @@ var DirectoryScanner = class {
 * const schema = defineSchema(
 *   v.object({ id: v.number(), customerId: v.number() }),
 *   {
-*     primaryKey: ['id'],
+*     primaryKey: 'id',
 *     foreignKeys: [
-*       { columns: ['customerId'], references: { table: 'users', columns: ['id'] } }
+*       { column: 'customerId', references: { table: 'users', column: 'id' } }
 *     ]
 *   }
 * );
@@ -308,57 +309,78 @@ var LinesDB = class LinesDB {
 	}
 	/**
 	* Initialize database by loading all JSONL files
+	* Uses dependency resolution to ensure foreign key references are loaded in correct order
 	*/
 	async initialize() {
 		this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
-		for (const [tableName, tableConfig] of this.tables) try {
-			await this.loadTable(tableName, tableConfig);
+		const loadedTables = /* @__PURE__ */ new Set();
+		const loadingTables = /* @__PURE__ */ new Set();
+		for (const [tableName] of this.tables) if (!loadedTables.has(tableName)) try {
+			await this.loadTableWithDependencies(tableName, loadedTables, loadingTables);
 		} catch (error) {
 			console.warn(`Warning: Failed to load table '${tableName}':`, error instanceof Error ? error.message : String(error));
 			this.tables.delete(tableName);
+			this.schemas.delete(tableName);
+			this.validationSchemas.delete(tableName);
+		}
+	}
+	/**
+	* Load a table and its dependencies recursively
+	*/
+	async loadTableWithDependencies(tableName, loadedTables, loadingTables) {
+		if (loadedTables.has(tableName)) return;
+		if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
+		const tableConfig = this.tables.get(tableName);
+		if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
+		loadingTables.add(tableName);
+		try {
+			let foreignKeys;
+			try {
+				const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+				const schemaModule = await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+				foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
+			} catch {}
+			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
+				const referencedTable = fk.references.table;
+				if (!loadedTables.has(referencedTable)) if (this.tables.has(referencedTable)) await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables);
+				else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
+			}
+			if (await this.loadTable(tableName, tableConfig)) loadedTables.add(tableName);
+			else this.tables.delete(tableName);
+		} finally {
+			loadingTables.delete(tableName);
 		}
 	}
 	/**
 	* Load a single table from JSONL file
+	* @returns true if table was loaded, false if skipped
 	*/
 	async loadTable(tableName, config) {
 		const data = await JsonlReader.read(config.jsonlPath);
-		if (data.length === 0) {
-			console.warn(`Warning: Table ${tableName} has no data`);
-			return;
-		}
 		let validationSchema = config.validationSchema;
+		const schemaMetadata = {};
 		if (!validationSchema) try {
 			validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
-		} catch (error) {
-			console.log(`[LinesDB] No validation schema for table '${tableName}':`, error instanceof Error ? error.message : String(error));
-		}
-		console.log(`[LinesDB] Loaded validation schema for table '${tableName}':`, validationSchema ? "FOUND" : "NOT FOUND");
-		if (validationSchema) {
-			console.log(`[LinesDB] Schema type:`, typeof validationSchema);
-			console.log(`[LinesDB] Schema has '~standard':`, "~standard" in validationSchema);
-		}
+		} catch (_error) {}
+		if (!config.validationSchema) try {
+			const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+			const schemaModule = await import(`${pathToFileURL$1(config.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+			const schemaExport = schemaModule.schema || schemaModule.default;
+			if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
+			else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
+			if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
+			else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
+			if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
+			else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
+		} catch (_error) {}
 		this.validationSchemas.set(tableName, validationSchema);
-		let schema;
-		if (config.schema) schema = config.schema;
-		else if (config.autoInferSchema !== false) schema = JsonlReader.inferSchema(tableName, data);
-		else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
-		if (validationSchema) {
-			const biSchema = validationSchema;
-			if (biSchema.primaryKey && !schema.columns.some((col) => col.primaryKey)) for (const pkColumn of biSchema.primaryKey) {
-				const col = schema.columns.find((c) => c.name === pkColumn);
-				if (col) col.primaryKey = true;
-			}
-			if (biSchema.foreignKeys) schema.foreignKeys = biSchema.foreignKeys;
-			if (biSchema.indexes) schema.indexes = biSchema.indexes;
-		}
-		this.schemas.set(tableName, schema);
-		this.createTable(schema);
 		const validationErrors = [];
+		const validatedData = [];
 		for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
 			const row = data[rowIndex];
 			try {
-				this.validateData(tableName, row);
+				const validatedRow = this.validateAndTransform(tableName, row);
+				validatedData.push(validatedRow);
 			} catch (error) {
 				if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
 					rowIndex,
@@ -375,13 +397,34 @@ var LinesDB = class LinesDB {
 			enhancedError.issues = validationErrors[0].error.issues;
 			throw enhancedError;
 		}
-		this.insertData(tableName, schema, data);
+		let schema;
+		if (config.schema) schema = config.schema;
+		else if (config.autoInferSchema !== false) {
+			if (validatedData.length === 0) return false;
+			schema = JsonlReader.inferSchema(tableName, validatedData);
+		} else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
+		const biSchema = validationSchema;
+		const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
+		const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
+		const indexes = biSchema?.indexes || schemaMetadata.indexes;
+		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const col = schema.columns.find((c) => c.name === primaryKey);
+			if (col) col.primaryKey = true;
+		} else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
+		}
+		if (foreignKeys) schema.foreignKeys = foreignKeys;
+		if (indexes) schema.indexes = indexes;
+		this.schemas.set(tableName, schema);
+		this.createTable(schema);
+		this.insertData(tableName, schema, validatedData);
+		return true;
 	}
 	/**
 	* Create table in SQLite with constraints and indexes
 	*/
 	createTable(schema) {
-		this.db.exec("PRAGMA foreign_keys = ON");
 		const quotedTableName = this.quoteTableName(schema.name);
 		const columnDefs = schema.columns.map((col) => {
 			const sqlType = col.type === "JSON" ? "TEXT" : col.type;
@@ -393,7 +436,7 @@ var LinesDB = class LinesDB {
 		});
 		const foreignKeyDefs = [];
 		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
-			const fkParts = [`FOREIGN KEY (${fk.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${fk.references.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`];
+			const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
 			if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
 			if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
 			foreignKeyDefs.push(fkParts.join(" "));
@@ -508,17 +551,12 @@ var LinesDB = class LinesDB {
 		return deserializedRow;
 	}
 	/**
-	* Validate data using StandardSchema
+	* Validate data using StandardSchema and return the transformed value
 	* Note: Only synchronous validation is supported
 	*/
-	validateData(tableName, data) {
+	validateAndTransform(tableName, data) {
 		const schema = this.validationSchemas.get(tableName);
-		console.log(`[LinesDB] validateData called for table '${tableName}', schema exists:`, !!schema);
-		if (!schema) {
-			console.log(`[LinesDB] No validation schema found for table '${tableName}', skipping validation`);
-			return;
-		}
-		console.log(`[LinesDB] Validating data:`, JSON.stringify(data));
+		if (!schema) return data;
 		const result = schema["~standard"].validate(data);
 		if (result instanceof Promise) throw new Error("Asynchronous validation is not supported. Please use synchronous validation schemas.");
 		if (result.issues && result.issues.length > 0) {
@@ -535,6 +573,17 @@ var LinesDB = class LinesDB {
 			error.issues = result.issues;
 			throw error;
 		}
+		const transformedValue = "value" in result ? result.value : data;
+		const normalizedValue = {};
+		for (const [key, value] of Object.entries(transformedValue)) normalizedValue[key] = value === void 0 ? null : value;
+		return normalizedValue;
+	}
+	/**
+	* Validate data using StandardSchema (without returning transformed value)
+	* Note: Only synchronous validation is supported
+	*/
+	validateData(tableName, data) {
+		this.validateAndTransform(tableName, data);
 	}
 	/**
 	* Insert a row into a table with validation
@@ -857,7 +906,7 @@ var LinesDB = class LinesDB {
 			await this.sync();
 			return result;
 		} catch (error) {
-			this.db.exec("ROLLBACK");
+			if (this.inTransaction) this.db.exec("ROLLBACK");
 			this.inTransaction = false;
 			throw error;
 		}
@@ -1025,9 +1074,9 @@ var Validator = class {
 			allErrors.push(...result.errors);
 			allWarnings.push(...result.warnings);
 		}
-		if (filesWithSchema.length > 0) {
-			const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-			allErrors.push(...fkErrors);
+		if (filesWithSchema.length > 0 && allErrors.length === 0) {
+			const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+			allErrors.push(...dbErrors);
 		}
 		return {
 			valid: allErrors.length === 0,
@@ -1036,53 +1085,50 @@ var Validator = class {
 		};
 	}
 	/**
-	* Validate foreign key constraints across all tables
+	* Validate by loading data into an actual database
+	* This catches constraint violations (unique, primary key, foreign key, etc.)
 	*/
-	async validateForeignKeys(dirPath, jsonlFiles) {
+	async validateWithDatabase(dirPath, jsonlFiles) {
 		const errors = [];
-		const tableData = /* @__PURE__ */ new Map();
-		const tableSchemas = /* @__PURE__ */ new Map();
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const data = await JsonlReader.read(file);
-			const schema = await SchemaLoader.loadSchema(file);
-			tableData.set(tableName, data);
-			tableSchemas.set(tableName, schema);
-		}
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const schema = tableSchemas.get(tableName);
-			const data = tableData.get(tableName);
-			if (!schema || !data || !schema.foreignKeys) continue;
-			for (const fk of schema.foreignKeys) {
-				const referencedTable = fk.references.table;
-				const referencedData = tableData.get(referencedTable);
-				if (!referencedData) continue;
-				const referencedValues = /* @__PURE__ */ new Set();
-				for (const refRow of referencedData) {
-					const keyValues = fk.references.columns.map((col) => refRow[col]);
-					const compositeKey = JSON.stringify(keyValues);
-					referencedValues.add(compositeKey);
-				}
-				for (let i = 0; i < data.length; i++) {
-					const row = data[i];
-					const foreignKeyValues = fk.columns.map((col) => row[col]);
-					const compositeKey = JSON.stringify(foreignKeyValues);
-					if (!referencedValues.has(compositeKey)) errors.push({
-						file,
-						tableName,
-						rowIndex: i,
-						issues: [],
-						type: "foreignKey",
-						foreignKeyError: {
-							column: fk.columns.join(", "),
-							value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-							referencedTable,
-							referencedColumn: fk.references.columns.join(", ")
-						}
-					});
-				}
+		const warnMessages = [];
+		const originalWarn = console.warn;
+		console.warn = (...args) => {
+			const message = args.map((arg) => String(arg)).join(" ");
+			warnMessages.push(message);
+			originalWarn(...args);
+		};
+		try {
+			const db = LinesDB.create({ dataDir: dirPath });
+			await db.initialize();
+			await db.close();
+			for (const message of warnMessages) if (message.includes("Failed to load table")) {
+				const tableNameMatch = message.match(/Failed to load table '([^']+)'/);
+				const tableName = tableNameMatch ? tableNameMatch[1] : "unknown";
+				const file = jsonlFiles.find((f) => basename(f, ".jsonl") === tableName);
+				errors.push({
+					file: file || `${dirPath}/${tableName}.jsonl`,
+					tableName,
+					rowIndex: 0,
+					issues: [{
+						message: message.replace(/^Warning:\s*/, ""),
+						path: []
+					}],
+					type: "schema"
+				});
 			}
+		} catch (error) {
+			errors.push({
+				file: dirPath,
+				tableName: "database",
+				rowIndex: 0,
+				issues: [{
+					message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+					path: []
+				}],
+				type: "schema"
+			});
+		} finally {
+			console.warn = originalWarn;
 		}
 		return errors;
 	}
@@ -1106,6 +1152,11 @@ var Validator = class {
 				type: "schema"
 			});
 		}
+		if (errors.length === 0) {
+			const dirPath = dirname(filePath);
+			const dbErrors = await this.validateWithDatabase(dirPath, [filePath]);
+			errors.push(...dbErrors);
+		}
 		return {
 			valid: errors.length === 0,
 			errors,