@toiroakr/lines-db 0.2.1 → 0.3.0

package/dist/index.js

@@ -29,6 +29,7 @@ function createDatabase(path = ":memory:") {
 function createNodeDatabase(path) {
 	const { DatabaseSync } = __require("node:sqlite");
 	const db = new DatabaseSync(path);
+	db.exec("PRAGMA foreign_keys = ON");
 	return {
 		prepare(sql) {
 			const stmt = db.prepare(sql);
@@ -124,7 +125,7 @@ var JsonlReader = class {
 		};
 	}
 	static inferType(value) {
-		if (value === null) return "NULL";
+		if (value === null || value === void 0) return "NULL";
 		if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
 		if (typeof value === "string") return "TEXT";
 		if (typeof value === "boolean") return "INTEGER";
@@ -308,57 +309,73 @@ var LinesDB = class LinesDB {
 	}
 	/**
 	* Initialize database by loading all JSONL files
+	* Uses dependency resolution to ensure foreign key references are loaded in correct order
 	*/
 	async initialize() {
 		this.tables = await DirectoryScanner.scanDirectory(this.config.dataDir);
-		for (const [tableName, tableConfig] of this.tables) try {
-			await this.loadTable(tableName, tableConfig);
+		const loadedTables = /* @__PURE__ */ new Set();
+		const loadingTables = /* @__PURE__ */ new Set();
+		for (const [tableName] of this.tables) if (!loadedTables.has(tableName)) try {
+			await this.loadTableWithDependencies(tableName, loadedTables, loadingTables);
 		} catch (error) {
 			console.warn(`Warning: Failed to load table '${tableName}':`, error instanceof Error ? error.message : String(error));
 			this.tables.delete(tableName);
+			this.schemas.delete(tableName);
+			this.validationSchemas.delete(tableName);
+		}
+	}
+	/**
+	* Load a table and its dependencies recursively
+	*/
+	async loadTableWithDependencies(tableName, loadedTables, loadingTables) {
+		if (loadedTables.has(tableName)) return;
+		if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
+		const tableConfig = this.tables.get(tableName);
+		if (!tableConfig) throw new Error(`Table configuration not found for '${tableName}'`);
+		loadingTables.add(tableName);
+		try {
+			let foreignKeys;
+			try {
+				const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+				foreignKeys = (await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`)).foreignKeys;
+			} catch {}
+			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
+				const referencedTable = fk.references.table;
+				if (!loadedTables.has(referencedTable)) if (this.tables.has(referencedTable)) await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables);
+				else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
+			}
+			if (await this.loadTable(tableName, tableConfig)) loadedTables.add(tableName);
+			else this.tables.delete(tableName);
+		} finally {
+			loadingTables.delete(tableName);
 		}
 	}
 	/**
 	* Load a single table from JSONL file
+	* @returns true if table was loaded, false if skipped
 	*/
 	async loadTable(tableName, config) {
 		const data = await JsonlReader.read(config.jsonlPath);
-		if (data.length === 0) {
-			console.warn(`Warning: Table ${tableName} has no data`);
-			return;
-		}
 		let validationSchema = config.validationSchema;
+		const schemaMetadata = {};
 		if (!validationSchema) try {
 			validationSchema = await SchemaLoader.loadSchema(config.jsonlPath);
-		} catch (error) {
-			console.log(`[LinesDB] No validation schema for table '${tableName}':`, error instanceof Error ? error.message : String(error));
-		}
-		console.log(`[LinesDB] Loaded validation schema for table '${tableName}':`, validationSchema ? "FOUND" : "NOT FOUND");
-		if (validationSchema) {
-			console.log(`[LinesDB] Schema type:`, typeof validationSchema);
-			console.log(`[LinesDB] Schema has '~standard':`, "~standard" in validationSchema);
-		}
+		} catch (_error) {}
+		if (!config.validationSchema) try {
+			const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
+			const schemaModule = await import(`${pathToFileURL$1(config.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+			if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
+			if (schemaModule.foreignKeys) schemaMetadata.foreignKeys = schemaModule.foreignKeys;
+			if (schemaModule.indexes) schemaMetadata.indexes = schemaModule.indexes;
+		} catch (_error) {}
 		this.validationSchemas.set(tableName, validationSchema);
-		let schema;
-		if (config.schema) schema = config.schema;
-		else if (config.autoInferSchema !== false) schema = JsonlReader.inferSchema(tableName, data);
-		else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
-		if (validationSchema) {
-			const biSchema = validationSchema;
-			if (biSchema.primaryKey && !schema.columns.some((col) => col.primaryKey)) for (const pkColumn of biSchema.primaryKey) {
-				const col = schema.columns.find((c) => c.name === pkColumn);
-				if (col) col.primaryKey = true;
-			}
-			if (biSchema.foreignKeys) schema.foreignKeys = biSchema.foreignKeys;
-			if (biSchema.indexes) schema.indexes = biSchema.indexes;
-		}
-		this.schemas.set(tableName, schema);
-		this.createTable(schema);
 		const validationErrors = [];
+		const validatedData = [];
 		for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
 			const row = data[rowIndex];
 			try {
-				this.validateData(tableName, row);
+				const validatedRow = this.validateAndTransform(tableName, row);
+				validatedData.push(validatedRow);
 			} catch (error) {
 				if (error instanceof Error && error.name === "ValidationError") validationErrors.push({
 					rowIndex,
@@ -375,13 +392,31 @@ var LinesDB = class LinesDB {
 			enhancedError.issues = validationErrors[0].error.issues;
 			throw enhancedError;
 		}
-		this.insertData(tableName, schema, data);
+		let schema;
+		if (config.schema) schema = config.schema;
+		else if (config.autoInferSchema !== false) {
+			if (validatedData.length === 0) return false;
+			schema = JsonlReader.inferSchema(tableName, validatedData);
+		} else throw new Error(`No schema provided for table ${tableName} and autoInferSchema is disabled`);
+		const biSchema = validationSchema;
+		const primaryKey = biSchema?.primaryKey || schemaMetadata.primaryKey;
+		const foreignKeys = biSchema?.foreignKeys || schemaMetadata.foreignKeys;
+		const indexes = biSchema?.indexes || schemaMetadata.indexes;
+		if (primaryKey && !schema.columns.some((col) => col.primaryKey)) for (const pkColumn of primaryKey) {
+			const col = schema.columns.find((c) => c.name === pkColumn);
+			if (col) col.primaryKey = true;
+		}
+		if (foreignKeys) schema.foreignKeys = foreignKeys;
+		if (indexes) schema.indexes = indexes;
+		this.schemas.set(tableName, schema);
+		this.createTable(schema);
+		this.insertData(tableName, schema, validatedData);
+		return true;
 	}
 	/**
 	* Create table in SQLite with constraints and indexes
 	*/
 	createTable(schema) {
-		this.db.exec("PRAGMA foreign_keys = ON");
 		const quotedTableName = this.quoteTableName(schema.name);
 		const columnDefs = schema.columns.map((col) => {
 			const sqlType = col.type === "JSON" ? "TEXT" : col.type;
@@ -508,17 +543,12 @@ var LinesDB = class LinesDB {
 		return deserializedRow;
 	}
 	/**
-	* Validate data using StandardSchema
+	* Validate data using StandardSchema and return the transformed value
 	* Note: Only synchronous validation is supported
 	*/
-	validateData(tableName, data) {
+	validateAndTransform(tableName, data) {
 		const schema = this.validationSchemas.get(tableName);
-		console.log(`[LinesDB] validateData called for table '${tableName}', schema exists:`, !!schema);
-		if (!schema) {
-			console.log(`[LinesDB] No validation schema found for table '${tableName}', skipping validation`);
-			return;
-		}
-		console.log(`[LinesDB] Validating data:`, JSON.stringify(data));
+		if (!schema) return data;
 		const result = schema["~standard"].validate(data);
 		if (result instanceof Promise) throw new Error("Asynchronous validation is not supported. Please use synchronous validation schemas.");
 		if (result.issues && result.issues.length > 0) {
@@ -535,6 +565,17 @@ var LinesDB = class LinesDB {
 			error.issues = result.issues;
 			throw error;
 		}
+		const transformedValue = "value" in result ? result.value : data;
+		const normalizedValue = {};
+		for (const [key, value] of Object.entries(transformedValue)) normalizedValue[key] = value === void 0 ? null : value;
+		return normalizedValue;
+	}
+	/**
+	* Validate data using StandardSchema (without returning transformed value)
+	* Note: Only synchronous validation is supported
+	*/
+	validateData(tableName, data) {
+		this.validateAndTransform(tableName, data);
 	}
 	/**
 	* Insert a row into a table with validation
@@ -857,7 +898,7 @@ var LinesDB = class LinesDB {
 			await this.sync();
 			return result;
 		} catch (error) {
-			this.db.exec("ROLLBACK");
+			if (this.inTransaction) this.db.exec("ROLLBACK");
 			this.inTransaction = false;
 			throw error;
 		}