@toiroakr/lines-db 0.6.1 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @toiroakr/lines-db
 
+## 0.8.0
+
+### Minor Changes
+
+- c408b92: feat: display per-table validation results for directory validation
+
+  The `validate` command now shows individual results per table when validating a directory, including record counts for successful tables (e.g., `✓ users (3 records)`).
+  - Added `TableValidationResult` type and `tableResults` field to `ValidationResult`
+  - Each table result includes `tableName`, `valid`, `rowCount`, `errors`, and `warnings`
+
+### Patch Changes
+
+- e61a4ee: fix: gracefully handle foreign key validation when referenced table has errors
+
+  When validating a directory, if a table had validation errors, any table referencing it via foreign key would crash with a misleading `no such table` SQLite error. Now, foreign key constraints to failed tables are skipped with a clear warning (e.g., `⚠ Skipping foreign key validation for table 'child': referenced table 'parent' has validation errors`), and the child table's own schema validation still runs normally.
+
+## 0.7.0
+
+### Minor Changes
+
+- 4597383: feat: support .mts and .cts schema file extensions
+
+  Schema files are now auto-detected with the following priority: `.schema.ts` > `.schema.mts` > `.schema.cts`. Mixed extensions within a single project are supported.
+  - Added `--output` option to `generate` command for specifying the output file path (e.g., `--output ./data/db.mts`)
+  - Import paths are correctly rewritten: `.ts`→`.js`, `.mts`→`.mjs`, `.cts`→`.cjs`
+  - New exported utilities: `findSchemaFile`, `isSchemaFile`, `extractTableNameFromSchemaFile`, `rewriteExtensionForImport`, `SCHEMA_EXTENSIONS`
+
 ## 0.6.1
 
 ### Patch Changes

package/bin/cli.js

@@ -10,6 +10,57 @@ import { runInNewContext } from "node:vm";
 //#region rolldown:runtime
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
+//#endregion
+//#region src/schema-extensions.ts
+/**
+* Supported schema file extensions, in priority order.
+* The first match wins when discovering schema files.
+*/
+const SCHEMA_EXTENSIONS = [
+	".schema.ts",
+	".schema.mts",
+	".schema.cts"
+];
+/**
+* Map from schema extensions to their JavaScript import counterparts.
+*/
+const SCHEMA_TO_JS_IMPORT_MAP = {
+	".schema.ts": ".schema.js",
+	".schema.mts": ".schema.mjs",
+	".schema.cts": ".schema.cjs"
+};
+/**
+* Try each supported schema extension and return the full path of the first
+* one that exists on disk. Returns undefined if none is found.
+*/
+async function findSchemaFile(dir, tableName) {
+	for (const ext of SCHEMA_EXTENSIONS) {
+		const candidate = join(dir, `${tableName}${ext}`);
+		try {
+			await access(candidate);
+			return candidate;
+		} catch {}
+	}
+}
+/**
+* Synchronously find a schema file among directory entries.
+* Returns the full path of the first match, or undefined.
+*/
+function findSchemaFileInEntries(dataDirPath, tableName, entries) {
+	for (const ext of SCHEMA_EXTENSIONS) {
+		const candidateName = `${tableName}${ext}`;
+		if (entries.some((e) => e.isFile() && e.name === candidateName)) return join(dataDirPath, candidateName);
+	}
+}
+/**
+* Rewrite a TypeScript path to its JavaScript counterpart for ESM imports.
+* ".schema.ts" -> ".schema.js", ".schema.mts" -> ".schema.mjs", ".schema.cts" -> ".schema.cjs"
+*/
+function rewriteExtensionForImport(filePath) {
+	for (const [tsExt, jsExt] of Object.entries(SCHEMA_TO_JS_IMPORT_MAP)) if (filePath.endsWith(tsExt)) return filePath.slice(0, -tsExt.length) + jsExt;
+	return filePath;
+}
+
 //#endregion
 //#region src/type-generator.ts
 var TypeGenerator = class {
@@ -22,7 +73,7 @@ var TypeGenerator = class {
 		this.projectRoot = envProjectRoot !== void 0 ? envProjectRoot : options.projectRoot || process.cwd();
 		this.dataDir = options.dataDir;
 		this.dataDirPath = isAbsolute(this.dataDir) ? this.dataDir : join(this.projectRoot, this.dataDir);
-		this.outputFile = join(this.dataDirPath, "db.ts");
+		this.outputFile = options.output ? isAbsolute(options.output) ? options.output : join(this.projectRoot, options.output) : join(this.dataDirPath, "db.ts");
 	}
 	/**
 	* Generate types file from JSONL files and their optional schema files.
@@ -45,12 +96,10 @@ var TypeGenerator = class {
 			const tables = [];
 			for (const entry of entries) if (entry.isFile() && entry.name.endsWith(".jsonl")) {
 				const tableName = basename(entry.name, ".jsonl");
-				const schemaFileName = `${tableName}.schema.ts`;
-				const schemaFilePath = join(this.dataDirPath, schemaFileName);
-				const hasSchema = entries.some((e) => e.isFile() && e.name === schemaFileName);
+				const schemaFilePath = findSchemaFileInEntries(this.dataDirPath, tableName, entries);
 				tables.push({
 					tableName,
-					schemaFile: hasSchema ? schemaFilePath : void 0
+					schemaFile: schemaFilePath
 				});
 			}
 			return tables;
@@ -71,7 +120,7 @@ var TypeGenerator = class {
 			if (table.schemaFile) {
 				const schemaIdentifier = this.createSchemaIdentifier(table.tableName, usedAliases);
 				usedAliases.add(schemaIdentifier);
-				let relativePath = relative(join(this.outputFile, ".."), table.schemaFile).replace(/\\/g, "/").replace(".ts", ".js");
+				let relativePath = rewriteExtensionForImport(relative(join(this.outputFile, ".."), table.schemaFile).replace(/\\/g, "/"));
 				if (!relativePath.startsWith(".")) relativePath = "./" + relativePath;
 				imports.push(`import { schema as ${schemaIdentifier} } from '${relativePath}';`);
 				tableEntries.push(`  ${tableKey}: InferOutput<typeof ${schemaIdentifier}>;`);
@@ -279,27 +328,17 @@ var SchemaLoader = class {
 	* Check if a schema file exists for a table
 	*/
 	static async hasSchema(jsonlPath) {
-		const schemaPath = join(dirname(jsonlPath), `${basename(jsonlPath, ".jsonl")}.schema.ts`);
-		try {
-			await access(schemaPath);
-			return true;
-		} catch {
-			return false;
-		}
+		return await findSchemaFile(dirname(jsonlPath), basename(jsonlPath, ".jsonl")) !== void 0;
 	}
 	/**
 	* Load a validation schema file for a table
-	* Requires ${tableName}.schema.ts to exist alongside the JSONL file
+	* Requires ${tableName}.schema.{ts,mts,cts} to exist alongside the JSONL file
 	*/
 	static async loadSchema(jsonlPath) {
 		const dir = dirname(jsonlPath);
 		const tableName = basename(jsonlPath, ".jsonl");
-		const schemaPath = join(dir, `${tableName}.schema.ts`);
-		try {
-			await access(schemaPath);
-		} catch (error) {
-			throw new Error(`Schema file not found for table '${tableName}'. Expected: ${schemaPath}`, { cause: error instanceof Error ? error : void 0 });
-		}
+		const schemaPath = await findSchemaFile(dir, tableName);
+		if (!schemaPath) throw new Error(`Schema file not found for table '${tableName}'. Expected one of: ${SCHEMA_EXTENSIONS.map((ext) => `${tableName}${ext}`).join(", ")}`);
 		try {
 			const module = await import(`${pathToFileURL(schemaPath).href}?t=${Date.now()}`);
 			const schema = module.default || module.schema;
@@ -384,6 +423,7 @@ var LinesDB = class LinesDB {
 	async initialize(options) {
 		const allErrors = [];
 		const allWarnings = [];
+		const allRowCounts = /* @__PURE__ */ new Map();
 		const tableName = options?.tableName;
 		const detailedValidate = options?.detailedValidate ?? false;
 		const transform = options?.transform;
@@ -395,14 +435,27 @@ var LinesDB = class LinesDB {
 		const attemptedTables = /* @__PURE__ */ new Set();
 		for (const tableNameToLoad of tablesToLoad) if (!attemptedTables.has(tableNameToLoad)) {
 			const tableTransform = tableNameToLoad === tableName ? transform : void 0;
-			const { errors, warnings } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
+			const { errors, warnings, rowCounts: tableRowCounts } = await this.loadTableWithDependencies(tableNameToLoad, loadedTables, loadingTables, attemptedTables, detailedValidate, tableTransform);
 			allErrors.push(...errors);
 			allWarnings.push(...warnings);
+			for (const [k, v] of tableRowCounts) allRowCounts.set(k, v);
 		}
+		const tableResults = tablesToLoad.map((name) => {
+			const tableErrors = allErrors.filter((e) => e.tableName === name);
+			const tableWarnings = allWarnings.filter((w) => w.includes(`'${name}'`));
+			return {
+				tableName: name,
+				valid: tableErrors.length === 0,
+				rowCount: allRowCounts.get(name) ?? 0,
+				errors: tableErrors,
+				warnings: tableWarnings
+			};
+		});
 		return {
 			valid: allErrors.length === 0,
 			errors: allErrors,
-			warnings: allWarnings
+			warnings: allWarnings,
+			tableResults
 		};
 	}
 	/**
@@ -411,9 +464,11 @@ var LinesDB = class LinesDB {
 	async loadTableWithDependencies(tableName, loadedTables, loadingTables, attemptedTables, detailedValidate, transform) {
 		const errors = [];
 		const warnings = [];
+		const rowCounts = /* @__PURE__ */ new Map();
 		if (attemptedTables.has(tableName)) return {
 			errors,
-			warnings
+			warnings,
+			rowCounts
 		};
 		attemptedTables.add(tableName);
 		if (loadingTables.has(tableName)) throw new Error(`Circular dependency detected for table '${tableName}'`);
@@ -424,8 +479,11 @@ var LinesDB = class LinesDB {
 			let foreignKeys;
 			try {
 				const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
-				const schemaModule = await import(`${pathToFileURL$1(tableConfig.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
-				foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
+				const schemaPath = await findSchemaFile(dirname(tableConfig.jsonlPath), basename(tableConfig.jsonlPath, ".jsonl"));
+				if (schemaPath) {
+					const schemaModule = await import(`${pathToFileURL$1(schemaPath).href}?t=${Date.now()}`);
+					foreignKeys = (schemaModule.schema || schemaModule.default)?.foreignKeys || schemaModule.foreignKeys;
+				}
 			} catch {}
 			if (foreignKeys && foreignKeys.length > 0) for (const fk of foreignKeys) {
 				const referencedTable = fk.references.table;
@@ -434,10 +492,21 @@ var LinesDB = class LinesDB {
 					const depResult = await this.loadTableWithDependencies(referencedTable, loadedTables, loadingTables, attemptedTables, detailedValidate, void 0);
 					errors.push(...depResult.errors);
 					warnings.push(...depResult.warnings);
+					for (const [k, v] of depResult.rowCounts) rowCounts.set(k, v);
 				} else throw new Error(`Foreign key reference to non-existent table '${referencedTable}' in table '${tableName}'`);
 			}
-			const { loaded, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform);
+			const failedDependencies = /* @__PURE__ */ new Set();
+			if (foreignKeys && foreignKeys.length > 0) {
+				for (const fk of foreignKeys) {
+					const referencedTable = fk.references.table;
+					if (referencedTable === tableName) continue;
+					if (attemptedTables.has(referencedTable) && !loadedTables.has(referencedTable)) failedDependencies.add(referencedTable);
+				}
+				if (failedDependencies.size > 0) for (const dep of failedDependencies) warnings.push(`Skipping foreign key validation for table '${tableName}': referenced table '${dep}' has validation errors`);
+			}
+			const { loaded, rowCount, errors: loadErrors } = await this.loadTable(tableName, tableConfig, detailedValidate, transform, failedDependencies);
 			errors.push(...loadErrors);
+			rowCounts.set(tableName, rowCount);
 			if (loaded) loadedTables.add(tableName);
 			else {
 				warnings.push(`Table '${tableName}' was not loaded (no data or skipped)`);
@@ -448,14 +517,15 @@ var LinesDB = class LinesDB {
 		}
 		return {
 			errors,
-			warnings
+			warnings,
+			rowCounts
 		};
 	}
 	/**
 	* Load a single table from JSONL file
 	* @returns Object with loaded status and validation errors
 	*/
-	async loadTable(tableName, config, detailedValidate, transform) {
+	async loadTable(tableName, config, detailedValidate, transform, failedDependencies) {
 		let data = await JsonlReader.read(config.jsonlPath);
 		if (transform) data = data.map((row) => transform(row));
 		let validationSchema = config.validationSchema;
@@ -465,7 +535,9 @@ var LinesDB = class LinesDB {
 		} catch (_error) {}
 		if (!config.validationSchema) try {
 			const { pathToFileURL: pathToFileURL$1 } = await import("node:url");
-			const schemaModule = await import(`${pathToFileURL$1(config.jsonlPath.replace(".jsonl", ".schema.ts")).href}?t=${Date.now()}`);
+			const schemaPath = await findSchemaFile(dirname(config.jsonlPath), basename(config.jsonlPath, ".jsonl"));
+			if (!schemaPath) throw new Error("Schema file not found");
+			const schemaModule = await import(`${pathToFileURL$1(schemaPath).href}?t=${Date.now()}`);
 			const schemaExport = schemaModule.schema || schemaModule.default;
 			if (schemaExport?.primaryKey) schemaMetadata.primaryKey = schemaExport.primaryKey;
 			else if (schemaModule.primaryKey) schemaMetadata.primaryKey = schemaModule.primaryKey;
@@ -508,6 +580,7 @@ var LinesDB = class LinesDB {
 		}));
 		if (validationErrors.length > 0) return {
 			loaded: false,
+			rowCount: data.length,
 			errors: validationErrorDetails
 		};
 		let schema;
@@ -522,6 +595,7 @@ var LinesDB = class LinesDB {
 		} else if (config.autoInferSchema !== false) {
 			if (validatedData.length === 0) return {
 				loaded: false,
+				rowCount: 0,
 				errors: []
 			};
 			schema = inferredSchema;
@@ -537,7 +611,7 @@ var LinesDB = class LinesDB {
 			const idColumn = schema.columns.find((c) => c.name === "id");
 			if (idColumn) idColumn.primaryKey = true;
 		}
-		if (foreignKeys) schema.foreignKeys = foreignKeys;
+		if (foreignKeys) schema.foreignKeys = failedDependencies && failedDependencies.size > 0 ? foreignKeys.filter((fk) => !failedDependencies.has(fk.references.table)) : foreignKeys;
 		if (indexes) {
 			schema.indexes = indexes;
 			for (const index of indexes) if (index.unique && index.columns.length === 1) {
@@ -551,11 +625,13 @@ var LinesDB = class LinesDB {
 			const insertErrors = this.insertDataWithDetailedValidation(tableName, schema, validatedData, config.jsonlPath);
 			if (insertErrors.length > 0) return {
 				loaded: false,
+				rowCount: data.length,
 				errors: insertErrors
 			};
 		} else this.insertData(tableName, schema, validatedData);
 		return {
 			loaded: true,
+			rowCount: data.length,
 			errors: []
 		};
 	}
@@ -1277,9 +1353,12 @@ function runInSandbox(expression, context = {}) {
 }
 const program = new Command();
 program.name("@toiroakr/lines-db").description("Database utilities for JSONL files").version("1.0.0");
-program.command("generate").description("Generate TypeScript type definitions from schema files").argument("<dataDir>", "Directory containing JSONL and schema files").action(async (dataDir) => {
+program.command("generate").description("Generate TypeScript type definitions from schema files").argument("<dataDir>", "Directory containing JSONL and schema files").option("-o, --output <path>", "Output file path (default: db.ts in dataDir)").action(async (dataDir, options) => {
 	try {
-		await new TypeGenerator({ dataDir }).generate();
+		await new TypeGenerator({
+			dataDir,
+			output: options.output
+		}).generate();
 		console.log("Type generation completed successfully!");
 	} catch (error) {
 		console.error("Error:", error instanceof Error ? error.message : String(error));
@@ -1306,48 +1385,73 @@ program.command("validate").description("Validate JSONL file(s) against schema")
 		} finally {
 			await db.close();
 		}
-		if (result.warnings.length > 0) {
-			for (const warning of result.warnings) console.warn(styleText("yellow", `⚠ ${warning}`));
-			console.log("");
-		}
-		if (result.valid) {
-			console.log("✓ All records are valid");
-			process.exit(0);
-		} else {
+		if (!tableName) {
 			const formatter = new ErrorFormatter({ verbose: options.verbose });
-			const errorsByFile = /* @__PURE__ */ new Map();
-			for (const error of result.errors) {
-				const fileErrors = errorsByFile.get(error.file) || [];
-				fileErrors.push(error);
-				errorsByFile.set(error.file, fileErrors);
-			}
-			for (const [file, fileErrors] of errorsByFile) {
-				console.error(formatter.formatErrorHeader(fileErrors.length, file));
+			for (const tableResult of result.tableResults) if (tableResult.valid && tableResult.warnings.length === 0) console.log(styleText("green", `✓ ${tableResult.tableName} (${tableResult.rowCount} records)`));
+			else if (tableResult.valid && tableResult.warnings.length > 0) for (const warning of tableResult.warnings) console.warn(styleText("yellow", `⚠ ${warning}`));
+			else {
+				const fileErrors = tableResult.errors;
+				console.error(formatter.formatErrorHeader(fileErrors.length, fileErrors[0]?.file));
 				console.error("");
 				const validationErrors = fileErrors.filter((e) => e.type !== "foreignKey" || !e.foreignKeyError);
 				const foreignKeyErrors = fileErrors.filter((e) => e.type === "foreignKey" && e.foreignKeyError);
-				if (validationErrors.length > 0) {
-					const formattedValidation = formatter.formatValidationErrors(validationErrors.map((e) => ({
+				if (validationErrors.length > 0) console.error(formatter.formatValidationErrors(validationErrors.map((e) => ({
+					file: e.file,
+					rowIndex: e.rowIndex,
+					issues: e.issues
+				}))));
+				for (const error of foreignKeyErrors) if (error.foreignKeyError) console.error(formatter.formatForeignKeyError({
+					file: error.file,
+					rowIndex: error.rowIndex,
+					column: error.foreignKeyError.column,
+					value: error.foreignKeyError.value,
+					referencedTable: error.foreignKeyError.referencedTable,
+					referencedColumn: error.foreignKeyError.referencedColumn
+				}));
+				console.error("");
+			}
+			if (result.valid) {
+				console.log("");
+				console.log(styleText("green", "✓ All records are valid"));
+				process.exit(0);
+			} else process.exit(1);
+		} else {
+			if (result.warnings.length > 0) {
+				for (const warning of result.warnings) console.warn(styleText("yellow", `⚠ ${warning}`));
+				console.log("");
+			}
+			if (result.valid) {
+				console.log(styleText("green", "✓ All records are valid"));
+				process.exit(0);
+			} else {
+				const formatter = new ErrorFormatter({ verbose: options.verbose });
+				for (const [, fileErrors] of result.errors.reduce((map, error) => {
+					const errors = map.get(error.file) || [];
+					errors.push(error);
+					map.set(error.file, errors);
+					return map;
+				}, /* @__PURE__ */ new Map())) {
+					console.error(formatter.formatErrorHeader(fileErrors.length, fileErrors[0]?.file));
+					console.error("");
+					const validationErrors = fileErrors.filter((e) => e.type !== "foreignKey" || !e.foreignKeyError);
+					const foreignKeyErrors = fileErrors.filter((e) => e.type === "foreignKey" && e.foreignKeyError);
+					if (validationErrors.length > 0) console.error(formatter.formatValidationErrors(validationErrors.map((e) => ({
 						file: e.file,
 						rowIndex: e.rowIndex,
 						issues: e.issues
-					})));
-					console.error(formattedValidation);
-				}
-				for (const error of foreignKeyErrors) if (error.foreignKeyError) {
-					const formattedFk = formatter.formatForeignKeyError({
+					}))));
+					for (const error of foreignKeyErrors) if (error.foreignKeyError) console.error(formatter.formatForeignKeyError({
 						file: error.file,
 						rowIndex: error.rowIndex,
 						column: error.foreignKeyError.column,
 						value: error.foreignKeyError.value,
 						referencedTable: error.foreignKeyError.referencedTable,
 						referencedColumn: error.foreignKeyError.referencedColumn
-					});
-					console.error(formattedFk);
+					}));
+					console.error("");
 				}
-				console.error("");
+				process.exit(1);
 			}
-			process.exit(1);
 		}
 	} catch (error) {
 		console.error("Error:", error instanceof Error ? error.message : String(error));