@toiroakr/lines-db 0.2.1 → 0.4.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @toiroakr/lines-db
 
+## 0.4.0
+
+### Minor Changes
+
+- a662484: - Allow flexible schema export methods (support loading from `schema` or `default` exports)
+  - Enhance constraint validation by loading data into an actual database (catches unique, primary key, and foreign key violations)
+  - Add fallback logic to automatically use `id` column as primary key when it exists and no primary key is explicitly defined
+
+## 0.3.0
+
+### Minor Changes
+
+- 50266c5: - Enhanced database initialization with dependency resolution and error handling
+  - Added support for undefined values in schema inference
+  - Implemented validation that automatically adds columns during data insertion
+
 ## 0.2.1
 
 ### Patch Changes

package/bin/cli.js

@@ -190,7 +190,7 @@ var JsonlReader = class {
 		};
 	}
 	static inferType(value) {
-		if (value === null) return "NULL";
+		if (value === null || value === void 0) return "NULL";
 		if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
 		if (typeof value === "string") return "TEXT";
 		if (typeof value === "boolean") return "INTEGER";
@@ -248,133 +248,6 @@ var SchemaLoader = class {
 	}
 };
 
-//#endregion
-//#region src/validator.ts
-var Validator = class {
-	path;
-	projectRoot;
-	constructor(options) {
-		this.path = options.path;
-		this.projectRoot = options.projectRoot || process.cwd();
-	}
-	/**
-	* Validate JSONL file(s)
-	*/
-	async validate() {
-		const fullPath = this.path.startsWith("/") ? this.path : join(this.projectRoot, this.path);
-		const stats = await stat(fullPath);
-		if (stats.isDirectory()) return this.validateDirectory(fullPath);
-		else if (stats.isFile() && fullPath.endsWith(".jsonl")) return this.validateFile(fullPath);
-		else throw new Error(`Invalid path: ${this.path}. Must be a directory or .jsonl file.`);
-	}
-	/**
-	* Validate all JSONL files in a directory
-	*/
-	async validateDirectory(dirPath) {
-		const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
-		if (jsonlFiles.length === 0) throw new Error(`No JSONL files found in directory: ${dirPath}`);
-		const allErrors = [];
-		const allWarnings = [];
-		const filesWithSchema = [];
-		for (const file of jsonlFiles) if (await SchemaLoader.hasSchema(file)) filesWithSchema.push(file);
-		else {
-			const tableName = basename(file, ".jsonl");
-			allWarnings.push(`Skipping validation for '${tableName}': schema file not found`);
-		}
-		for (const file of filesWithSchema) {
-			const result = await this.validateFile(file);
-			allErrors.push(...result.errors);
-			allWarnings.push(...result.warnings);
-		}
-		if (filesWithSchema.length > 0) {
-			const fkErrors = await this.validateForeignKeys(dirPath, filesWithSchema);
-			allErrors.push(...fkErrors);
-		}
-		return {
-			valid: allErrors.length === 0,
-			errors: allErrors,
-			warnings: allWarnings
-		};
-	}
-	/**
-	* Validate foreign key constraints across all tables
-	*/
-	async validateForeignKeys(dirPath, jsonlFiles) {
-		const errors = [];
-		const tableData = /* @__PURE__ */ new Map();
-		const tableSchemas = /* @__PURE__ */ new Map();
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const data = await JsonlReader.read(file);
-			const schema = await SchemaLoader.loadSchema(file);
-			tableData.set(tableName, data);
-			tableSchemas.set(tableName, schema);
-		}
-		for (const file of jsonlFiles) {
-			const tableName = basename(file, ".jsonl");
-			const schema = tableSchemas.get(tableName);
-			const data = tableData.get(tableName);
-			if (!schema || !data || !schema.foreignKeys) continue;
-			for (const fk of schema.foreignKeys) {
-				const referencedTable = fk.references.table;
-				const referencedData = tableData.get(referencedTable);
-				if (!referencedData) continue;
-				const referencedValues = /* @__PURE__ */ new Set();
-				for (const refRow of referencedData) {
-					const keyValues = fk.references.columns.map((col) => refRow[col]);
-					const compositeKey = JSON.stringify(keyValues);
-					referencedValues.add(compositeKey);
-				}
-				for (let i = 0; i < data.length; i++) {
-					const row = data[i];
-					const foreignKeyValues = fk.columns.map((col) => row[col]);
-					const compositeKey = JSON.stringify(foreignKeyValues);
-					if (!referencedValues.has(compositeKey)) errors.push({
-						file,
-						tableName,
-						rowIndex: i,
-						issues: [],
-						type: "foreignKey",
-						foreignKeyError: {
-							column: fk.columns.join(", "),
-							value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
-							referencedTable,
-							referencedColumn: fk.references.columns.join(", ")
-						}
-					});
-				}
-			}
-		}
-		return errors;
-	}
-	/**
-	* Validate a single JSONL file
-	*/
-	async validateFile(filePath) {
-		const tableName = basename(filePath, ".jsonl");
-		const data = await JsonlReader.read(filePath);
-		const schema = await SchemaLoader.loadSchema(filePath);
-		const errors = [];
-		for (let i = 0; i < data.length; i++) {
-			const row = data[i];
-			const result = schema["~standard"].validate(row);
-			if (result instanceof Promise) throw new Error("Asynchronous validation is not supported.");
-			if (result.issues && result.issues.length > 0) errors.push({
-				file: filePath,
-				tableName,
-				rowIndex: i,
-				issues: result.issues,
-				type: "schema"
-			});
-		}
-		return {
-			valid: errors.length === 0,
-			errors,
-			warnings: []
-		};
-	}
-};
-
 //#endregion
 //#region src/runtime.ts
 function detectRuntime() {
@@ -398,6 +271,7 @@ function createDatabase(path = ":memory:") {
 function createNodeDatabase(path) {
 	const { DatabaseSync } = __require("node:sqlite");
 	const db = new DatabaseSync(path);
+	db.exec("PRAGMA foreign_keys = ON");
 	return {
 		prepare(sql) {
 			const stmt = db.prepare(sql);
@@ -500,57 +374,78 @@ var LinesDB = class LinesDB {
 	}
 	/**
 	* Initialize database by loading all JSONL files
+	* Uses dependency resolution to ensure foreign key references are loaded in correct order
 	*/
 	async initialize() {
 		this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
-		for (const [tableName, tableConfig] of this.tables) try {
-			await this.loadTable(tableName, tableConfig);
+		const loadedTables = /* @__PURE__ */ new Set();
+		const loadingTables = /* @__PURE__ */ new Set();
+		for (const [tableName] of this.tables) if (!loadedTables.has(tableName)) try {
+			await this.loadTableWithDependencies(tableName, loadedTables, loadingTables);
 		} catch (error) {
 			console.warn(`Warning: Failed to load table '${tableName}':`, error instanceof Error ? error.message : String(error));
 			this.tables.delete(tableName);
+			this.schemas.delete(tableName);
+			this.validationSchemas.delete(tableName);
+		}
+	}
+	/**
+	* Load a table and its dependencies recursively
+	*/
+	async loadTableWithDependencies(tableName, loadedTables, loadingTables) {
+		if (loadedTables.has(tableName)) return;
+		if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
+		const tableConfig = this.tables.get(tableName);
+		if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
+		loadingTables.add(tableName);
+		try {
+			let foreignKeys;
+			try {
+				const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+				const schemaModule = await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+				foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
+			} catch {}
+			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
+				const referencedTable = fk.references.table;
+				if (!loadedTables.has(referencedTable)) if (this.tables.has(referencedTable)) await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables);
+				else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
+			}
+			if (await this.loadTable(tableName, tableConfig)) loadedTables.add(tableName);
+			else this.tables.delete(tableName);
+		} finally {
+			loadingTables.delete(tableName);
 		}
 	}
 	/**
 	* Load a single table from JSONL file
+	* @returns true if table was loaded, false if skipped
 	*/
 	async loadTable(tableName, config) {
 		const data = await JsonlReader.read(config.jsonlPath);
-		if (data.length === 0) {
-			console.warn(`Warning: Table ${tableName} has no data`);
-			return;
-		}
 		let validationSchema = config.validationSchema;
+		const schemaMetadata = {};
 		if (!validationSchema) try {
 			validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
-		} catch (error) {
-			console.log(`[LinesDB] No validation schema for table '${tableName}':`, error instanceof Error ? error.message : String(error));
-		}
-		console.log(`[LinesDB] Loaded validation schema for table '${tableName}':`, validationSchema ? "FOUND" : "NOT FOUND");
-		if (validationSchema) {
-			console.log(`[LinesDB] Schema type:`, typeof validationSchema);
-			console.log(`[LinesDB] Schema has '~standard':`, "~standard" in validationSchema);
-		}
+		} catch (_error) {}
+		if (!config.validationSchema) try {
+			const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+			const schemaModule = await import(`${pathToFileURL$1(config.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+			const schemaExport = schemaModule.schema || schemaModule.default;
+			if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
+			else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
+			if (schemaExport?.foreignKeys) schemaMetadata.foreignKeys = schemaExport.foreignKeys;
+			else if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
+			if (schemaExport?.indexes) schemaMetadata.indexes = schemaExport.indexes;
+			else if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
+		} catch (_error) {}
 		this.validationSchemas.set(tableName, validationSchema);
-		let schema;
-		if (config.schema) schema = config.schema;
-		else if (config.autoInferSchema !== false) schema = JsonlReader.inferSchema(tableName, data);
-		else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
-		if (validationSchema) {
-			const biSchema = validationSchema;
-			if (biSchema.primaryKey && !schema.columns.some((col) => col.primaryKey)) for (const pkColumn of biSchema.primaryKey) {
-				const col = schema.columns.find((c) => c.name === pkColumn);
-				if (col) col.primaryKey = true;
-			}
-			if (biSchema.foreignKeys) schema.foreignKeys = biSchema.foreignKeys;
-			if (biSchema.indexes) schema.indexes = biSchema.indexes;
-		}
-		this.schemas.set(tableName, schema);
-		this.createTable(schema);
 		const validationErrors = [];
+		const validatedData = [];
 		for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
 			const row = data[rowIndex];
 			try {
-				this.validateData(tableName, row);
+				const validatedRow = this.validateAndTransform(tableName, row);
+				validatedData.push(validatedRow);
 			} catch (error) {
 				if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
 					rowIndex,
@@ -567,13 +462,34 @@ var LinesDB = class LinesDB {
 			enhancedError.issues = validationErrors[0].error.issues;
 			throw enhancedError;
 		}
-		this.insertData(tableName, schema, data);
+		let schema;
+		if (config.schema) schema = config.schema;
+		else if (config.autoInferSchema !== false) {
+			if (validatedData.length === 0) return false;
+			schema = JsonlReader.inferSchema(tableName, validatedData);
+		} else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
+		const biSchema = validationSchema;
+		const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
+		const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
+		const indexes = biSchema?.indexes || schemaMetadata.indexes;
+		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const col = schema.columns.find((c) => c.name === primaryKey);
+			if (col) col.primaryKey = true;
+		} else if (!primaryKey && !schema.columns.some((col) => col.primaryKey)) {
+			const idColumn = schema.columns.find((c) => c.name === "id");
+			if (idColumn) idColumn.primaryKey = true;
+		}
+		if (foreignKeys) schema.foreignKeys = foreignKeys;
+		if (indexes) schema.indexes = indexes;
+		this.schemas.set(tableName, schema);
+		this.createTable(schema);
+		this.insertData(tableName, schema, validatedData);
+		return true;
 	}
 	/**
 	* Create table in SQLite with constraints and indexes
 	*/
 	createTable(schema) {
-		this.db.exec("PRAGMA foreign_keys = ON");
 		const quotedTableName = this.quoteTableName(schema.name);
 		const columnDefs = schema.columns.map((col) => {
 			const sqlType = col.type === "JSON" ? "TEXT" : col.type;
@@ -585,7 +501,7 @@ var LinesDB = class LinesDB {
 		});
 		const foreignKeyDefs = [];
 		if (schema.foreignKeys && schema.foreignKeys.length > 0) for (const fk of schema.foreignKeys) {
-			const fkParts = [`FOREIGN KEY (${fk.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${fk.references.columns.map((col) => this.quoteIdentifier(col)).join(", ")})`];
+			const fkParts = [`FOREIGN KEY (${this.quoteIdentifier(fk.column)})`, `REFERENCES ${this.quoteTableName(fk.references.table)}(${this.quoteIdentifier(fk.references.column)})`];
 			if (fk.onDelete) fkParts.push(`ON DELETE ${fk.onDelete}`);
 			if (fk.onUpdate) fkParts.push(`ON UPDATE ${fk.onUpdate}`);
 			foreignKeyDefs.push(fkParts.join(" "));
@@ -700,17 +616,12 @@ var LinesDB = class LinesDB {
 		return deserializedRow;
 	}
 	/**
-	* Validate data using StandardSchema
+	* Validate data using StandardSchema and return the transformed value
 	* Note: Only synchronous validation is supported
 	*/
-	validateData(tableName, data) {
+	validateAndTransform(tableName, data) {
 		const schema = this.validationSchemas.get(tableName);
-		console.log(`[LinesDB] validateData called for table '${tableName}', schema exists:`, !!schema);
-		if (!schema) {
-			console.log(`[LinesDB] No validation schema found for table '${tableName}', skipping validation`);
-			return;
-		}
-		console.log(`[LinesDB] Validating data:`, JSON.stringify(data));
+		if (!schema) return data;
 		const result = schema["~standard"].validate(data);
 		if (result instanceof Promise) throw new Error("Asynchronous validation is not supported. Please use synchronous validation schemas.");
 		if (result.issues && result.issues.length > 0) {
@@ -727,6 +638,17 @@ var LinesDB = class LinesDB {
 			error.issues = result.issues;
 			throw error;
 		}
+		const transformedValue = "value" in result ? result.value : data;
+		const normalizedValue = {};
+		for (const [key, value] of Object.entries(transformedValue)) normalizedValue[key] = value === void 0 ? null : value;
+		return normalizedValue;
+	}
+	/**
+	* Validate data using StandardSchema (without returning transformed value)
+	* Note: Only synchronous validation is supported
+	*/
+	validateData(tableName, data) {
+		this.validateAndTransform(tableName, data);
 	}
 	/**
 	* Insert a row into a table with validation
@@ -1049,7 +971,7 @@ var LinesDB = class LinesDB {
 			await this.sync();
 			return result;
 		} catch (error) {
-			this.db.exec("ROLLBACK");
+			if (this.inTransaction) this.db.exec("ROLLBACK");
 			this.inTransaction = false;
 			throw error;
 		}
@@ -1070,6 +992,135 @@ var LinesDB = class LinesDB {
 	}
 };
 
+//#endregion
+//#region src/validator.ts
+var Validator = class {
+	path;
+	projectRoot;
+	constructor(options) {
+		this.path = options.path;
+		this.projectRoot = options.projectRoot || process.cwd();
+	}
+	/**
+	* Validate JSONL file(s)
+	*/
+	async validate() {
+		const fullPath = this.path.startsWith("/") ? this.path : join(this.projectRoot, this.path);
+		const stats = await stat(fullPath);
+		if (stats.isDirectory()) return this.validateDirectory(fullPath);
+		else if (stats.isFile() && fullPath.endsWith(".jsonl")) return this.validateFile(fullPath);
+		else throw new Error(`Invalid path: ${this.path}. Must be a directory or .jsonl file.`);
+	}
+	/**
+	* Validate all JSONL files in a directory
+	*/
+	async validateDirectory(dirPath) {
+		const jsonlFiles = (await readdir(dirPath, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => join(dirPath, entry.name));
+		if (jsonlFiles.length === 0) throw new Error(`No JSONL files found in directory: ${dirPath}`);
+		const allErrors = [];
+		const allWarnings = [];
+		const filesWithSchema = [];
+		for (const file of jsonlFiles) if (await SchemaLoader.hasSchema(file)) filesWithSchema.push(file);
+		else {
+			const tableName = basename(file, ".jsonl");
+			allWarnings.push(`Skipping validation for '${tableName}': schema file not found`);
+		}
+		for (const file of filesWithSchema) {
+			const result = await this.validateFile(file);
+			allErrors.push(...result.errors);
+			allWarnings.push(...result.warnings);
+		}
+		if (filesWithSchema.length > 0 && allErrors.length === 0) {
+			const dbErrors = await this.validateWithDatabase(dirPath, filesWithSchema);
+			allErrors.push(...dbErrors);
+		}
+		return {
+			valid: allErrors.length === 0,
+			errors: allErrors,
+			warnings: allWarnings
+		};
+	}
+	/**
+	* Validate by loading data into an actual database
+	* This catches constraint violations (unique, primary key, foreign key, etc.)
+	*/
+	async validateWithDatabase(dirPath, jsonlFiles) {
+		const errors = [];
+		const warnMessages = [];
+		const originalWarn = console.warn;
+		console.warn = (...args) => {
+			const message = args.map((arg) => String(arg)).join(" ");
+			warnMessages.push(message);
+			originalWarn(...args);
+		};
+		try {
+			const db = LinesDB.create({ dataDir: dirPath });
+			await db.initialize();
+			await db.close();
+			for (const message of warnMessages) if (message.includes("Failed to load table")) {
+				const tableNameMatch = message.match(/Failed to load table '([^']+)'/);
+				const tableName = tableNameMatch ? tableNameMatch[1] : "unknown";
+				const file = jsonlFiles.find((f) => basename(f, ".jsonl") === tableName);
+				errors.push({
+					file: file || `${dirPath}/${tableName}.jsonl`,
+					tableName,
+					rowIndex: 0,
+					issues: [{
+						message: message.replace(/^Warning:\s*/, ""),
+						path: []
+					}],
+					type: "schema"
+				});
+			}
+		} catch (error) {
+			errors.push({
+				file: dirPath,
+				tableName: "database",
+				rowIndex: 0,
+				issues: [{
+					message: `Database initialization failed: ${error instanceof Error ? error.message : String(error)}`,
+					path: []
+				}],
+				type: "schema"
+			});
+		} finally {
+			console.warn = originalWarn;
+		}
+		return errors;
+	}
+	/**
+	* Validate a single JSONL file
+	*/
+	async validateFile(filePath) {
+		const tableName = basename(filePath, ".jsonl");
+		const data = await JsonlReader.read(filePath);
+		const schema = await SchemaLoader.loadSchema(filePath);
+		const errors = [];
+		for (let i = 0; i < data.length; i++) {
+			const row = data[i];
+			const result = schema["~standard"].validate(row);
+			if (result instanceof Promise) throw new Error("Asynchronous validation is not supported.");
+			if (result.issues && result.issues.length > 0) errors.push({
+				file: filePath,
+				tableName,
+				rowIndex: i,
+				issues: result.issues,
+				type: "schema"
+			});
+		}
+		if (errors.length === 0) {
+			const dirPath = dirname(filePath);
+			const dbErrors = await this.validateWithDatabase(dirPath, [filePath]);
+			errors.push(...dbErrors);
+		}
+		return {
+			valid: errors.length === 0,
+			errors,
+			warnings: []
+		};
+	}
+};
+
 //#endregion
 //#region src/error-formatter.ts
 var ErrorFormatter = class {
@@ -1306,7 +1357,6 @@ program.command("migrate").description("Migrate data with transformation functio
 			process.exit(0);
 		}
 		const transformedRows = rowsToMigrate.map((row) => transform(row));
-		console.log("\nApplying migration in transaction...\n");
 		try {
 			await db.transaction(async () => {
 				db.batchUpdate(tableName, transformedRows, { validate: true });
@@ -1351,6 +1401,7 @@ program.command("migrate").description("Migrate data with transformation functio
 				}
 			} else if (error instanceof Error) {
 				console.error(`\n  ${error.message}`);
+				if (options.verbose && error.stack) console.error(`\nStack trace:\n${error.stack}`);
 				if (error.message.includes("UNIQUE constraint failed") || error.message.includes("FOREIGN KEY constraint failed") || error.message.includes("NOT NULL constraint failed") || error.message.includes("CHECK constraint failed")) {
 					console.error("\n  This is a SQLite constraint violation.");
 					console.error("  Please check your data and schema requirements.");