@toiroakr/lines-db 0.1.0

package/src/validator.test.ts

@@ -0,0 +1,337 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { Validator } from './validator.js';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('Validator', () => {
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `validator-test-${Date.now()}`);
+    await mkdir(testDir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  describe('validate file', () => {
+    it('should validate file with valid data', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => {
+              const issues = [];
+              if (!data.id || !data.name) {
+                issues.push({ message: 'Missing required fields' });
+              }
+              return { value: data, issues };
+            }
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(true);
+      expect(result.errors).toHaveLength(0);
+    });
+
+    it('should detect validation errors', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1,"name":"Alice"}\n{"id":2}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => {
+              const issues = [];
+              if (!data.name) {
+                issues.push({ message: 'Missing name field', path: ['name'] });
+              }
+              return { value: data, issues };
+            }
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].rowIndex).toBe(2);
+      expect(result.errors[0].tableName).toBe('users');
+      expect(result.errors[0].issues).toHaveLength(1);
+    });
+
+    it('should throw when no schema exists', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      await writeFile(jsonlPath, '{"id":1}\n{"id":2}\n');
+
+      const validator = new Validator({ path: jsonlPath });
+      await expect(validator.validate()).rejects.toThrow(/Schema file not found/);
+    });
+
+    it('should handle multiple validation errors in single file', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1}\n{"id":2}\n{"id":3,"name":"Charlie"}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => {
+              const issues = [];
+              if (!data.name) {
+                issues.push({ message: 'Missing name' });
+              }
+              return { value: data, issues };
+            }
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(2);
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[1].rowIndex).toBe(2);
+    });
+
+    it('should return 0-based rowIndex (first line = index 0)', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // Line 1: valid, Line 2: invalid (no name), Line 3: invalid (no name)
+      await writeFile(
+        jsonlPath,
+        '{"id":1,"name":"Alice"}\n{"id":2}\n{"id":3}\n',
+      );
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => {
+              const issues = [];
+              if (!data.name) {
+                issues.push({ message: 'Missing name field', path: ['name'] });
+              }
+              return { value: data, issues };
+            }
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(2);
+      // First error is on line 2 of the file, which is index 1 (0-based)
+      expect(result.errors[0].rowIndex).toBe(1);
+      expect(result.errors[0].file).toBe(jsonlPath);
+      // Second error is on line 3 of the file, which is index 2 (0-based)
+      expect(result.errors[1].rowIndex).toBe(2);
+      expect(result.errors[1].file).toBe(jsonlPath);
+    });
+
+    it('should return 0-based rowIndex for first line error', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      // First line has error
+      await writeFile(jsonlPath, '{"id":1}\n{"id":2,"name":"Bob"}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => {
+              const issues = [];
+              if (!data.name) {
+                issues.push({ message: 'Missing name field', path: ['name'] });
+              }
+              return { value: data, issues };
+            }
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(1);
+      // First line (line 1 in file) should be index 0
+      expect(result.errors[0].rowIndex).toBe(0);
+      expect(result.errors[0].file).toBe(jsonlPath);
+    });
+  });
+
+  describe('validate directory', () => {
+    it('should validate all JSONL files in directory', async () => {
+      const usersPath = join(testDir, 'users.jsonl');
+      const productsPath = join(testDir, 'products.jsonl');
+      const usersSchemaPath = join(testDir, 'users.schema.ts');
+      const productsSchemaPath = join(testDir, 'products.schema.ts');
+
+      await writeFile(usersPath, '{"id":1,"name":"Alice"}\n');
+      await writeFile(productsPath, '{"id":1,"name":"Product"}\n');
+      await writeFile(
+        usersSchemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          }
+        };
+      `,
+      );
+      await writeFile(
+        productsSchemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({ value: data, issues: [] })
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: testDir });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(true);
+      expect(result.errors).toHaveLength(0);
+    });
+
+    it('should collect errors from multiple files', async () => {
+      const usersPath = join(testDir, 'users.jsonl');
+      const usersSchemaPath = join(testDir, 'users.schema.ts');
+      const productsPath = join(testDir, 'products.jsonl');
+      const productsSchemaPath = join(testDir, 'products.schema.ts');
+
+      await writeFile(usersPath, '{"id":1}\n');
+      await writeFile(
+        usersSchemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({
+              value: data,
+              issues: data.name ? [] : [{ message: 'Missing name' }]
+            })
+          }
+        };
+      `,
+      );
+
+      await writeFile(productsPath, '{"id":1}\n');
+      await writeFile(
+        productsSchemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: (data) => ({
+              value: data,
+              issues: data.price ? [] : [{ message: 'Missing price' }]
+            })
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: testDir });
+      const result = await validator.validate();
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toHaveLength(2);
+    });
+
+    it('should throw error for directory with no JSONL files', async () => {
+      await writeFile(join(testDir, 'readme.txt'), 'no jsonl files here');
+
+      const validator = new Validator({ path: testDir });
+
+      await expect(validator.validate()).rejects.toThrow('No JSONL files found');
+    });
+  });
+
+  describe('error handling', () => {
+    it('should throw error for non-JSONL file', async () => {
+      const txtPath = join(testDir, 'test.txt');
+      await writeFile(txtPath, 'not a jsonl file');
+
+      const validator = new Validator({ path: txtPath });
+
+      await expect(validator.validate()).rejects.toThrow('Invalid path');
+    });
+
+    it('should reject async validation', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: async (data) => {
+              return { value: data, issues: [] };
+            }
+          }
+        };
+      `,
+      );
+
+      const validator = new Validator({ path: jsonlPath });
+
+      await expect(validator.validate()).rejects.toThrow(
+        'Asynchronous validation is not supported',
+      );
+    });
+  });
+});

package/src/validator.ts

@@ -0,0 +1,207 @@
+import { readdir, stat } from 'node:fs/promises';
+import { join, basename } from 'node:path';
+import { JsonlReader } from './jsonl-reader.js';
+import { SchemaLoader } from './schema-loader.js';
+import type { StandardSchemaIssue, JsonObject } from './types.js';
+import type { BiDirectionalSchema } from './schema.js';
+
+export interface ValidationResult {
+  valid: boolean;
+  errors: ValidationErrorDetail[];
+}
+
+export interface ValidationErrorDetail {
+  file: string;
+  tableName: string;
+  rowIndex: number;
+  issues: ReadonlyArray<StandardSchemaIssue>;
+  type?: 'schema' | 'foreignKey';
+  foreignKeyError?: {
+    column: string;
+    value: unknown;
+    referencedTable: string;
+    referencedColumn: string;
+  };
+}
+
+export interface ValidatorOptions {
+  path: string; // File or directory path
+  projectRoot?: string;
+}
+
+export class Validator {
+  private path: string;
+  private projectRoot: string;
+
+  constructor(options: ValidatorOptions) {
+    this.path = options.path;
+    this.projectRoot = options.projectRoot || process.cwd();
+  }
+
+  /**
+   * Validate JSONL file(s)
+   */
+  async validate(): Promise<ValidationResult> {
+    // Use absolute path if provided, otherwise resolve relative to projectRoot
+    const fullPath = this.path.startsWith('/') ? this.path : join(this.projectRoot, this.path);
+    const stats = await stat(fullPath);
+
+    if (stats.isDirectory()) {
+      return this.validateDirectory(fullPath);
+    } else if (stats.isFile() && fullPath.endsWith('.jsonl')) {
+      return this.validateFile(fullPath);
+    } else {
+      throw new Error(`Invalid path: ${this.path}. Must be a directory or .jsonl file.`);
+    }
+  }
+
+  /**
+   * Validate all JSONL files in a directory
+   */
+  private async validateDirectory(dirPath: string): Promise<ValidationResult> {
+    const entries = await readdir(dirPath, { withFileTypes: true });
+    const jsonlFiles = entries
+      .filter((entry) => entry.isFile() && entry.name.endsWith('.jsonl'))
+      .map((entry) => join(dirPath, entry.name));
+
+    if (jsonlFiles.length === 0) {
+      throw new Error(`No JSONL files found in directory: ${dirPath}`);
+    }
+
+    const allErrors: ValidationErrorDetail[] = [];
+
+    // First, validate schema for each file
+    for (const file of jsonlFiles) {
+      const result = await this.validateFile(file);
+      allErrors.push(...result.errors);
+    }
+
+    // Then, validate foreign keys across all tables
+    const fkErrors = await this.validateForeignKeys(dirPath, jsonlFiles);
+    allErrors.push(...fkErrors);
+
+    return {
+      valid: allErrors.length === 0,
+      errors: allErrors,
+    };
+  }
+
+  /**
+   * Validate foreign key constraints across all tables
+   */
+  private async validateForeignKeys(
+    dirPath: string,
+    jsonlFiles: string[],
+  ): Promise<ValidationErrorDetail[]> {
+    const errors: ValidationErrorDetail[] = [];
+
+    // Load all table data
+    const tableData = new Map<string, JsonObject[]>();
+    const tableSchemas = new Map<string, BiDirectionalSchema>();
+
+    for (const file of jsonlFiles) {
+      const tableName = basename(file, '.jsonl');
+      const data = await JsonlReader.read(file);
+      const schema = await SchemaLoader.loadSchema(file);
+
+      tableData.set(tableName, data);
+      tableSchemas.set(tableName, schema as BiDirectionalSchema);
+    }
+
+    // Check foreign keys for each table
+    for (const file of jsonlFiles) {
+      const tableName = basename(file, '.jsonl');
+      const schema = tableSchemas.get(tableName);
+      const data = tableData.get(tableName);
+
+      if (!schema || !data || !schema.foreignKeys) {
+        continue;
+      }
+
+      // Check each foreign key constraint
+      for (const fk of schema.foreignKeys) {
+        const referencedTable = fk.references.table;
+        const referencedData = tableData.get(referencedTable);
+
+        if (!referencedData) {
+          // Referenced table not found - skip validation
+          continue;
+        }
+
+        // Build index of referenced values for fast lookup
+        const referencedValues = new Set<string>();
+        for (const refRow of referencedData) {
+          // Build composite key from referenced columns
+          const keyValues = fk.references.columns.map((col) => refRow[col]);
+          const compositeKey = JSON.stringify(keyValues);
+          referencedValues.add(compositeKey);
+        }
+
+        // Check each row in current table
+        for (let i = 0; i < data.length; i++) {
+          const row = data[i];
+          const foreignKeyValues = fk.columns.map((col) => row[col]);
+          const compositeKey = JSON.stringify(foreignKeyValues);
+
+          // Check if foreign key value exists in referenced table
+          if (!referencedValues.has(compositeKey)) {
+            errors.push({
+              file,
+              tableName,
+              rowIndex: i, // 0-indexed, will be converted to 1-indexed in formatter
+              issues: [],
+              type: 'foreignKey',
+              foreignKeyError: {
+                column: fk.columns.join(', '),
+                value: foreignKeyValues.length === 1 ? foreignKeyValues[0] : foreignKeyValues,
+                referencedTable: referencedTable,
+                referencedColumn: fk.references.columns.join(', '),
+              },
+            });
+          }
+        }
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Validate a single JSONL file
+   */
+  private async validateFile(filePath: string): Promise<ValidationResult> {
+    const tableName = basename(filePath, '.jsonl');
+    const data = await JsonlReader.read(filePath);
+
+    // Try to load schema
+    const schema = await SchemaLoader.loadSchema(filePath);
+
+    const errors: ValidationErrorDetail[] = [];
+
+    // Validate each row
+    for (let i = 0; i < data.length; i++) {
+      const row = data[i];
+      const result = schema['~standard'].validate(row);
+
+      // Only synchronous validation is supported
+      if (result instanceof Promise) {
+        throw new Error('Asynchronous validation is not supported.');
+      }
+
+      if (result.issues && result.issues.length > 0) {
+        errors.push({
+          file: filePath,
+          tableName,
+          rowIndex: i, // 0-indexed, will be converted to 1-indexed in formatter
+          issues: result.issues,
+          type: 'schema',
+        });
+      }
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors,
+    };
+  }
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsconfig.test.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "types": ["vitest/globals", "node"]
+  },
+  "include": ["src/**/*.test.ts", "src/**/*.ts"]
+}

package/tsdown.config.ts

@@ -0,0 +1,26 @@
+import { defineConfig } from 'tsdown';
+
+export default defineConfig([
+  // CLI build (ESM only)
+  {
+    entry: ['src/cli.ts'],
+    format: ['esm'],
+    platform: 'node',
+    target: 'node18',
+    clean: true,
+    shims: true,
+    outDir: 'bin',
+    dts: false,
+    treeshake: true,
+  },
+  // Library build (ESM + CJS)
+  {
+    entry: ['src/index.ts'],
+    format: ['esm', 'cjs'],
+    platform: 'node',
+    target: 'node18',
+    outDir: 'dist',
+    dts: true,
+    treeshake: true,
+  },
+]);

package/vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['src/**/*.test.ts'],
+  },
+});