@toiroakr/lines-db 0.1.0

package/src/directory-scanner.test.ts

@@ -0,0 +1,91 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { DirectoryScanner } from './directory-scanner.js';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('DirectoryScanner', () => {
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `scanner-test-${Date.now()}`);
+    await mkdir(testDir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  describe('scanDirectory', () => {
+    it('should find JSONL files in directory', async () => {
+      await writeFile(join(testDir, 'users.jsonl'), '{"id":1}\n');
+      await writeFile(join(testDir, 'products.jsonl'), '{"id":1}\n');
+
+      const tables = await DirectoryScanner.scanDirectory(testDir);
+
+      expect(tables.size).toBe(2);
+      expect(tables.has('users')).toBe(true);
+      expect(tables.has('products')).toBe(true);
+    });
+
+    it('should create correct table config', async () => {
+      await writeFile(join(testDir, 'users.jsonl'), '{"id":1}\n');
+
+      const tables = await DirectoryScanner.scanDirectory(testDir);
+      const usersConfig = tables.get('users');
+
+      expect(usersConfig).toBeDefined();
+      expect(usersConfig?.jsonlPath).toBe(join(testDir, 'users.jsonl'));
+      expect(usersConfig?.autoInferSchema).toBe(true);
+    });
+
+    it('should ignore non-JSONL files', async () => {
+      await writeFile(join(testDir, 'users.jsonl'), '{"id":1}\n');
+      await writeFile(join(testDir, 'readme.txt'), 'readme');
+      await writeFile(join(testDir, 'config.json'), '{}');
+
+      const tables = await DirectoryScanner.scanDirectory(testDir);
+
+      expect(tables.size).toBe(1);
+      expect(tables.has('users')).toBe(true);
+    });
+
+    it('should handle empty directory', async () => {
+      const tables = await DirectoryScanner.scanDirectory(testDir);
+
+      expect(tables.size).toBe(0);
+    });
+
+    it('should throw error for non-existent directory', async () => {
+      const nonExistentDir = join(testDir, 'nonexistent');
+
+      await expect(DirectoryScanner.scanDirectory(nonExistentDir)).rejects.toThrow(
+        'Failed to scan directory',
+      );
+    });
+
+    it('should handle multiple JSONL files', async () => {
+      const fileNames = ['users', 'products', 'orders', 'categories', 'reviews'];
+
+      for (const name of fileNames) {
+        await writeFile(join(testDir, `${name}.jsonl`), '{"id":1}\n');
+      }
+
+      const tables = await DirectoryScanner.scanDirectory(testDir);
+
+      expect(tables.size).toBe(5);
+      for (const name of fileNames) {
+        expect(tables.has(name)).toBe(true);
+      }
+    });
+
+    it('should handle files with multiple dots in name', async () => {
+      await writeFile(join(testDir, 'test.backup.jsonl'), '{"id":1}\n');
+
+      const tables = await DirectoryScanner.scanDirectory(testDir);
+
+      expect(tables.size).toBe(1);
+      expect(tables.has('test.backup')).toBe(true);
+    });
+  });
+});

package/src/directory-scanner.ts

@@ -0,0 +1,38 @@
+import { readdir } from 'node:fs/promises';
+import { join, basename, extname } from 'node:path';
+import type { TableConfig } from './types.js';
+
+export class DirectoryScanner {
+  /**
+   * Scan directory for JSONL files and create table configurations
+   */
+  static async scanDirectory(dataDir: string): Promise<Map<string, TableConfig>> {
+    const tables = new Map<string, TableConfig>();
+
+    try {
+      const files = await readdir(dataDir);
+
+      for (const file of files) {
+        if (extname(file) === '.jsonl') {
+          const tableName = basename(file, '.jsonl');
+          const jsonlPath = join(dataDir, file);
+
+          tables.set(tableName, {
+            jsonlPath,
+            autoInferSchema: true,
+          });
+        }
+      }
+
+      if (tables.size === 0) {
+        console.warn(`Warning: No JSONL files found in directory: ${dataDir}`);
+      }
+
+      return tables;
+    } catch (error) {
+      throw new Error(
+        `Failed to scan directory ${dataDir}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}

package/src/error-formatter.ts

@@ -0,0 +1,166 @@
+import type { StandardSchemaIssue } from './types.js';
+import { styleText } from 'node:util';
+
+export interface ValidationErrorInfo {
+  file: string;
+  rowIndex: number;
+  issues: ReadonlyArray<StandardSchemaIssue>;
+  data?: unknown;
+  originalData?: unknown;
+}
+
+export interface ForeignKeyErrorInfo {
+  file: string;
+  rowIndex: number;
+  column: string;
+  value: unknown;
+  referencedTable: string;
+  referencedColumn: string;
+  data?: unknown;
+}
+
+export interface ErrorFormatterOptions {
+  verbose?: boolean;
+}
+
+export class ErrorFormatter {
+  private verbose: boolean;
+
+  constructor(options: ErrorFormatterOptions = {}) {
+    this.verbose = options.verbose ?? false;
+  }
+
+  /**
+   * Format validation errors
+   */
+  formatValidationErrors(errors: ValidationErrorInfo[]): string {
+    if (this.verbose) {
+      return this.formatValidationErrorsVerbose(errors);
+    }
+    return this.formatValidationErrorsCompact(errors);
+  }
+
+  /**
+   * Format foreign key error
+   */
+  formatForeignKeyError(error: ForeignKeyErrorInfo): string {
+    if (this.verbose) {
+      return this.formatForeignKeyErrorVerbose(error);
+    }
+    return this.formatForeignKeyErrorCompact(error);
+  }
+
+  /**
+   * Format compact (default) validation errors
+   */
+  private formatValidationErrorsCompact(errors: ValidationErrorInfo[]): string {
+    const lines: string[] = [];
+
+    for (const error of errors) {
+      for (const issue of error.issues) {
+        const fieldPath = this.getFieldPath(issue);
+        const line = `${error.file}:${error.rowIndex + 1} • ${fieldPath}: ${issue.message}`;
+        lines.push(line);
+      }
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format verbose validation errors
+   */
+  private formatValidationErrorsVerbose(errors: ValidationErrorInfo[]): string {
+    const blocks: string[] = [];
+
+    for (let i = 0; i < errors.length; i++) {
+      const error = errors[i];
+      const isLast = i === errors.length - 1;
+      const prefix = isLast ? '└─' : '├─';
+      const linePrefix = isLast ? '   ' : '│  ';
+
+      const lines: string[] = [`${prefix} ${error.file}:${error.rowIndex + 1}`];
+
+      for (const issue of error.issues) {
+        const fieldPath = this.getFieldPath(issue);
+        lines.push(`${linePrefix}Field: ${fieldPath}`);
+        lines.push(`${linePrefix}Error: ${issue.message}`);
+      }
+
+      // Show original data if available (migrate case)
+      if (error.originalData !== undefined) {
+        lines.push(`${linePrefix}Original data: ${JSON.stringify(error.originalData)}`);
+      }
+
+      // Show data
+      if (error.data !== undefined) {
+        const label = error.originalData !== undefined ? 'Transformed data' : 'Data';
+        lines.push(`${linePrefix}${label}: ${JSON.stringify(error.data)}`);
+      }
+
+      blocks.push(lines.join('\n'));
+    }
+
+    return blocks.join('\n│\n');
+  }
+
+  /**
+   * Format compact foreign key error
+   */
+  private formatForeignKeyErrorCompact(error: ForeignKeyErrorInfo): string {
+    return `${error.file}:${error.rowIndex + 1} • ${error.column}: Foreign key constraint failed - Referenced value ${JSON.stringify(error.value)} does not exist in ${error.referencedTable}(${error.referencedColumn})`;
+  }
+
+  /**
+   * Format verbose foreign key error
+   */
+  private formatForeignKeyErrorVerbose(error: ForeignKeyErrorInfo): string {
+    const lines: string[] = [
+      `└─ ${error.file}:${error.rowIndex + 1}`,
+      `   Type: Foreign Key Violation`,
+      `   Field: ${error.column}`,
+      `   Value: ${JSON.stringify(error.value)}`,
+      `   References: ${error.referencedTable}(${error.referencedColumn})`,
+      `   Error: Referenced value does not exist in target table`,
+    ];
+
+    if (error.data !== undefined) {
+      lines.push(`   Data: ${JSON.stringify(error.data)}`);
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Get field path from issue
+   */
+  private getFieldPath(issue: StandardSchemaIssue): string {
+    if (!issue.path || issue.path.length === 0) {
+      return 'root';
+    }
+
+    return issue.path
+      .map((segment) => {
+        if (typeof segment === 'object' && segment !== null && 'key' in segment) {
+          return String(segment.key);
+        }
+        return String(segment);
+      })
+      .join('.');
+  }
+
+  /**
+   * Format error header with count
+   */
+  formatErrorHeader(count: number, file?: string): string {
+    const fileInfo = file ? ` in ${file}` : '';
+    return styleText('red', `✗ Found ${count} error(s)${fileInfo}`);
+  }
+
+  /**
+   * Format migration failure header
+   */
+  formatMigrationFailureHeader(): string {
+    return styleText('red', '\n✗ Migration failed and was rolled back');
+  }
+}

package/src/index.ts

@@ -0,0 +1,35 @@
+export { LinesDB } from './database.js';
+export { JsonlReader } from './jsonl-reader.js';
+export { JsonlWriter } from './jsonl-writer.js';
+export { SchemaLoader } from './schema-loader.js';
+export { DirectoryScanner } from './directory-scanner.js';
+export { defineSchema, hasBackward } from './schema.js';
+export { TypeGenerator } from './type-generator.js';
+export { Validator } from './validator.js';
+export { ensureTableRowsValid } from './jsonl-migration.js';
+export type { TableValidationOptions } from './jsonl-migration.js';
+export { detectRuntime, RUNTIME } from './runtime.js';
+export type { RuntimeEnvironment } from './runtime.js';
+export type { SQLiteDatabase, SQLiteStatement } from './sqlite-adapter.js';
+export type { TypeGeneratorOptions } from './type-generator.js';
+export type { ValidatorOptions, ValidationResult, ValidationErrorDetail } from './validator.js';
+export type {
+  TableSchema,
+  ColumnDefinition,
+  DatabaseConfig,
+  TableConfig,
+  JsonValue,
+  JsonObject,
+  JsonArray,
+  StandardSchema,
+  StandardSchemaResult,
+  StandardSchemaIssue,
+  ValidationError,
+  InferInput,
+  InferOutput,
+  Table,
+  TableDefs,
+  ForeignKeyDefinition,
+  IndexDefinition,
+} from './types.js';
+export type { BiDirectionalSchema, SchemaOptions } from './schema.js';

package/src/jsonl-migration.ts

@@ -0,0 +1,76 @@
+import { join } from 'node:path';
+import type { JsonObject } from './types.js';
+import { JsonlReader } from './jsonl-reader.js';
+import { LinesDB } from './database.js';
+
+export interface TableValidationOptions {
+  dataDir: string;
+  tableName: string;
+  rows: JsonObject[];
+}
+
+/**
+ * Validate a table by temporarily supplying in-memory rows while reusing the existing LinesDB validation pipeline.
+ * If validation fails, the underlying LinesDB error is rethrown so callers can inspect validation details.
+ */
+export async function ensureTableRowsValid(options: TableValidationOptions): Promise<void> {
+  console.log('[ensureTableRowsValid] START');
+  console.log('[ensureTableRowsValid] dataDir:', options.dataDir);
+  console.log('[ensureTableRowsValid] tableName:', options.tableName);
+  console.log('[ensureTableRowsValid] rows count:', options.rows.length);
+
+  const tablePath = join(options.dataDir, `${options.tableName}.jsonl`);
+  const overrides = new Map<string, JsonObject[]>([[tablePath, options.rows]]);
+  console.log('[ensureTableRowsValid] tablePath:', tablePath);
+
+  let capturedError: Error | null = null;
+
+  // Intercept console.warn to capture validation errors
+  const originalWarn = console.warn;
+  const warnMessages: string[] = [];
+  console.warn = (...args: any[]) => {
+    const message = args.join(' ');
+    console.log('[ensureTableRowsValid] Captured warn:', message);
+    warnMessages.push(message);
+    // Check if this is a validation error for our table
+    if (
+      message.includes(`Failed to load table '${options.tableName}'`) &&
+      message.includes('Validation failed')
+    ) {
+      // Extract the original error from the warn message
+      capturedError = new Error(message);
+      console.log('[ensureTableRowsValid] Captured validation error!');
+    }
+  };
+
+  try {
+    console.log('[ensureTableRowsValid] Calling JsonlReader.withOverrides');
+    await JsonlReader.withOverrides(overrides, async () => {
+      console.log('[ensureTableRowsValid] Inside withOverrides callback');
+      const db = LinesDB.create({ dataDir: options.dataDir });
+      console.log('[ensureTableRowsValid] LinesDB created');
+      try {
+        console.log('[ensureTableRowsValid] Calling db.initialize()');
+        await db.initialize();
+        console.log('[ensureTableRowsValid] db.initialize() completed');
+      } finally {
+        console.log('[ensureTableRowsValid] Calling db.close()');
+        await db.close();
+      }
+    });
+    console.log('[ensureTableRowsValid] withOverrides completed');
+  } finally {
+    // Restore original console.warn
+    console.warn = originalWarn;
+  }
+
+  console.log('[ensureTableRowsValid] Warnings captured:', warnMessages.length);
+  console.log('[ensureTableRowsValid] capturedError:', capturedError ? 'YES' : 'NO');
+
+  if (capturedError) {
+    console.log('[ensureTableRowsValid] Throwing captured error');
+    throw capturedError;
+  }
+
+  console.log('[ensureTableRowsValid] END (success)');
+}

package/src/jsonl-reader.test.ts

@@ -0,0 +1,168 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { JsonlReader } from './jsonl-reader.js';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('JsonlReader', () => {
+  let testDir: string;
+  let testFilePath: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `jsonl-test-${Date.now()}`);
+    await mkdir(testDir, { recursive: true });
+    testFilePath = join(testDir, 'test.jsonl');
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  describe('read', () => {
+    it('should read and parse JSONL file', async () => {
+      const content = '{"id": 1, "name": "Alice"}\n{"id": 2, "name": "Bob"}\n';
+      await writeFile(testFilePath, content);
+
+      const result = await JsonlReader.read(testFilePath);
+
+      expect(result).toHaveLength(2);
+      expect(result[0]).toEqual({ id: 1, name: 'Alice' });
+      expect(result[1]).toEqual({ id: 2, name: 'Bob' });
+    });
+
+    it('should handle empty lines', async () => {
+      const content = '{"id": 1}\n\n{"id": 2}\n';
+      await writeFile(testFilePath, content);
+
+      const result = await JsonlReader.read(testFilePath);
+
+      expect(result).toHaveLength(2);
+      expect(result[0]).toEqual({ id: 1 });
+      expect(result[1]).toEqual({ id: 2 });
+    });
+
+    it('should handle trailing newlines', async () => {
+      const content = '{"id": 1}\n{"id": 2}\n\n\n';
+      await writeFile(testFilePath, content);
+
+      const result = await JsonlReader.read(testFilePath);
+
+      expect(result).toHaveLength(2);
+    });
+
+    it('should throw error for invalid JSON', async () => {
+      const content = '{"id": 1}\n{invalid json}\n';
+      await writeFile(testFilePath, content);
+
+      await expect(JsonlReader.read(testFilePath)).rejects.toThrow('Failed to parse JSON line');
+    });
+  });
+
+  describe('inferSchema', () => {
+    it('should infer schema with basic types', () => {
+      const data = [
+        { id: 1, name: 'Alice', age: 30, active: true },
+        { id: 2, name: 'Bob', age: 25, active: false },
+      ];
+
+      const schema = JsonlReader.inferSchema('users', data);
+
+      expect(schema.name).toBe('users');
+      expect(schema.columns).toHaveLength(4);
+
+      const idColumn = schema.columns.find((col) => col.name === 'id');
+      expect(idColumn).toEqual({
+        name: 'id',
+        type: 'INTEGER',
+        notNull: true,
+        primaryKey: true,
+      });
+
+      const nameColumn = schema.columns.find((col) => col.name === 'name');
+      expect(nameColumn?.type).toBe('TEXT');
+
+      const ageColumn = schema.columns.find((col) => col.name === 'age');
+      expect(ageColumn?.type).toBe('INTEGER');
+
+      const activeColumn = schema.columns.find((col) => col.name === 'active');
+      expect(activeColumn?.type).toBe('INTEGER'); // Boolean stored as INTEGER
+    });
+
+    it('should infer JSON type for objects', () => {
+      const data1 = { id: 1, metadata: { key: 'value' } };
+      const data2 = { id: 2, metadata: { foo: 'bar' } };
+
+      const schema = JsonlReader.inferSchema('records', [data1, data2]);
+
+      const metadataColumn = schema.columns.find((col) => col.name === 'metadata');
+      expect(metadataColumn?.type).toBe('JSON');
+    });
+
+    it('should infer JSON type for arrays', () => {
+      const data = [
+        { id: 1, tags: ['a', 'b', 'c'] },
+        { id: 2, tags: ['x', 'y'] },
+      ];
+
+      const schema = JsonlReader.inferSchema('records', data);
+
+      const tagsColumn = schema.columns.find((col) => col.name === 'tags');
+      expect(tagsColumn?.type).toBe('JSON');
+    });
+
+    it('should handle nullable columns', () => {
+      const data = [
+        { id: 1, name: 'Alice', email: null },
+        { id: 2, name: 'Bob', email: 'bob@example.com' },
+      ];
+
+      const schema = JsonlReader.inferSchema('users', data);
+
+      const emailColumn = schema.columns.find((col) => col.name === 'email');
+      expect(emailColumn?.type).toBe('TEXT');
+      expect(emailColumn?.notNull).toBe(false);
+    });
+
+    it('should handle mixed numeric types as REAL', () => {
+      const data = [
+        { id: 1, value: 10 },
+        { id: 2, value: 10.5 },
+      ];
+
+      const schema = JsonlReader.inferSchema('records', data);
+
+      const valueColumn = schema.columns.find((col) => col.name === 'value');
+      expect(valueColumn?.type).toBe('REAL');
+    });
+
+    it('should use TEXT as fallback for mixed types', () => {
+      const data = [
+        { id: 1, mixed: 'text' },
+        { id: 2, mixed: 123 },
+      ];
+
+      const schema = JsonlReader.inferSchema('records', data);
+
+      const mixedColumn = schema.columns.find((col) => col.name === 'mixed');
+      expect(mixedColumn?.type).toBe('TEXT');
+    });
+
+    it('should throw error for empty data', () => {
+      expect(() => JsonlReader.inferSchema('empty', [])).toThrow(
+        'Cannot infer schema from empty data',
+      );
+    });
+
+    it('should handle REAL numbers', () => {
+      const data = [
+        { id: 1, price: 99.99 },
+        { id: 2, price: 149.5 },
+      ];
+
+      const schema = JsonlReader.inferSchema('products', data);
+
+      const priceColumn = schema.columns.find((col) => col.name === 'price');
+      expect(priceColumn?.type).toBe('REAL');
+    });
+  });
+});