@toiroakr/lines-db 0.1.0

package/src/jsonl-reader.ts

@@ -0,0 +1,135 @@
+import { readFile } from 'node:fs/promises';
+import { normalize } from 'node:path';
+import type { JsonObject, ColumnDefinition, TableSchema } from './types.js';
+
+export class JsonlReader {
+  private static overrides: Map<string, JsonObject[]> | null = null;
+
+  /**
+   * Temporarily override the data returned for specific JSONL files.
+   * Useful for scenarios like migration validation where in-memory data should be used.
+   */
+  static async withOverrides<T>(
+    overrides: Map<string, JsonObject[]>,
+    fn: () => Promise<T>,
+  ): Promise<T> {
+    const normalized = new Map<string, JsonObject[]>();
+    for (const [filePath, rows] of overrides) {
+      normalized.set(normalize(filePath), rows);
+    }
+
+    const previousOverrides = this.overrides;
+    this.overrides = normalized;
+
+    try {
+      return await fn();
+    } finally {
+      this.overrides = previousOverrides;
+    }
+  }
+
+  /**
+   * Read JSONL file and parse each line as JSON
+   */
+  static async read(filePath: string): Promise<JsonObject[]> {
+    const overrideRows = this.overrides?.get(normalize(filePath));
+    if (overrideRows) {
+      // Return clones to avoid accidental mutations from consumers
+      return overrideRows.map((row) => JSON.parse(JSON.stringify(row)) as JsonObject);
+    }
+
+    const content = await readFile(filePath, 'utf-8');
+    const lines = content.trim().split('\n');
+
+    return lines
+      .filter((line) => line.trim().length > 0)
+      .map((line) => {
+        try {
+          return JSON.parse(line) as JsonObject;
+        } catch (error) {
+          throw new Error(`Failed to parse JSON line: ${line}`, { cause: error });
+        }
+      });
+  }
+
+  /**
+   * Infer schema from JSONL data
+   */
+  static inferSchema(tableName: string, data: JsonObject[]): TableSchema {
+    if (data.length === 0) {
+      throw new Error('Cannot infer schema from empty data');
+    }
+
+    const columnTypes = new Map<string, Set<string>>();
+    const booleanColumns = new Set<string>();
+    const nonBooleanColumns = new Set<string>();
+
+    // Collect all column names and their types
+    for (const row of data) {
+      for (const [key, value] of Object.entries(row)) {
+        if (!columnTypes.has(key)) {
+          columnTypes.set(key, new Set());
+        }
+        columnTypes.get(key)!.add(this.inferType(value));
+
+        if (typeof value === 'boolean') {
+          booleanColumns.add(key);
+        } else if (value !== null) {
+          nonBooleanColumns.add(key);
+        }
+      }
+    }
+
+    // Convert to column definitions
+    const columns: ColumnDefinition[] = [];
+    for (const [columnName, types] of columnTypes.entries()) {
+      const typeArray = Array.from(types);
+
+      // If multiple types exist, prefer TEXT as a safe fallback
+      let sqlType: ColumnDefinition['type'] = 'TEXT';
+
+      if (typeArray.length === 1) {
+        sqlType = typeArray[0] as ColumnDefinition['type'];
+      } else if (typeArray.every((t) => t === 'INTEGER' || t === 'REAL')) {
+        sqlType = 'REAL'; // Use REAL if we have mixed numeric types
+      } else if (!typeArray.includes('NULL')) {
+        // If there are multiple non-null types, use TEXT
+        sqlType = 'TEXT';
+      } else if (typeArray.length === 2 && typeArray.includes('NULL')) {
+        // If one type + NULL, use the non-null type
+        sqlType = typeArray.find((t) => t !== 'NULL') as ColumnDefinition['type'];
+      }
+
+      const isBooleanColumn = booleanColumns.has(columnName) && !nonBooleanColumns.has(columnName);
+
+      columns.push({
+        name: columnName,
+        type: sqlType,
+        notNull: !typeArray.includes('NULL'),
+        valueType: isBooleanColumn ? 'boolean' : undefined,
+      });
+    }
+
+    // If there's an 'id' column, make it primary key
+    const idColumn = columns.find((col) => col.name === 'id');
+    if (idColumn) {
+      idColumn.primaryKey = true;
+    }
+
+    return {
+      name: tableName,
+      columns,
+    };
+  }
+
+  private static inferType(value: unknown): string {
+    if (value === null) return 'NULL';
+    if (typeof value === 'number') {
+      return Number.isInteger(value) ? 'INTEGER' : 'REAL';
+    }
+    if (typeof value === 'string') return 'TEXT';
+    if (typeof value === 'boolean') return 'INTEGER'; // SQLite stores booleans as integers
+    if (typeof value === 'object') return 'JSON'; // Store objects/arrays as JSON
+    return 'TEXT';
+  }
+}

package/src/jsonl-writer.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { JsonlWriter } from './jsonl-writer.js';
+import { readFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('JsonlWriter', () => {
+  let testDir: string;
+  let testFilePath: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `jsonl-writer-test-${Date.now()}`);
+    await mkdir(testDir, { recursive: true });
+    testFilePath = join(testDir, 'test.jsonl');
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  describe('write', () => {
+    it('should write data to JSONL file', async () => {
+      const data = [
+        { id: 1, name: 'Alice' },
+        { id: 2, name: 'Bob' },
+      ];
+
+      await JsonlWriter.write(testFilePath, data);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      expect(content).toBe('{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n');
+    });
+
+    it('should write empty array', async () => {
+      await JsonlWriter.write(testFilePath, []);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      expect(content).toBe('\n');
+    });
+
+    it('should write complex objects', async () => {
+      const data1 = { id: 1, metadata: { key: 'value' }, tags: ['a', 'b'] };
+      const data2 = { id: 2, active: true, score: 99.5 };
+
+      await JsonlWriter.write(testFilePath, [data1, data2]);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      const lines = content.trim().split('\n');
+
+      expect(lines).toHaveLength(2);
+      expect(JSON.parse(lines[0])).toEqual(data1);
+      expect(JSON.parse(lines[1])).toEqual(data2);
+    });
+
+    it('should overwrite existing file', async () => {
+      await JsonlWriter.write(testFilePath, [{ id: 1 }]);
+      await JsonlWriter.write(testFilePath, [{ id: 2 }]);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      expect(content).toBe('{"id":2}\n');
+    });
+  });
+
+  describe('append', () => {
+    it('should append data to existing file', async () => {
+      await JsonlWriter.write(testFilePath, [{ id: 1, name: 'Alice' }]);
+      await JsonlWriter.append(testFilePath, [{ id: 2, name: 'Bob' }]);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      expect(content).toBe('{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n');
+    });
+
+    it('should create file if it does not exist', async () => {
+      await JsonlWriter.append(testFilePath, [{ id: 1, name: 'Alice' }]);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      expect(content).toBe('{"id":1,"name":"Alice"}\n');
+    });
+
+    it('should append multiple records', async () => {
+      await JsonlWriter.write(testFilePath, [{ id: 1 }]);
+      await JsonlWriter.append(testFilePath, [{ id: 2 }, { id: 3 }, { id: 4 }]);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      const lines = content.trim().split('\n');
+
+      expect(lines).toHaveLength(4);
+      expect(JSON.parse(lines[0])).toEqual({ id: 1 });
+      expect(JSON.parse(lines[3])).toEqual({ id: 4 });
+    });
+
+    it('should handle empty array append', async () => {
+      await JsonlWriter.write(testFilePath, [{ id: 1 }]);
+      await JsonlWriter.append(testFilePath, []);
+
+      const content = await readFile(testFilePath, 'utf-8');
+      // Appending empty array adds just a newline
+      expect(content).toBe('{"id":1}\n\n');
+    });
+  });
+});

package/src/jsonl-writer.ts

@@ -0,0 +1,33 @@
+import { writeFile } from 'node:fs/promises';
+import type { JsonObject } from './types.js';
+
+export class JsonlWriter {
+  /**
+   * Write data to JSONL file
+   */
+  static async write(filePath: string, data: JsonObject[]): Promise<void> {
+    const lines = data.map((obj) => JSON.stringify(obj)).join('\n');
+    await writeFile(filePath, lines + '\n', 'utf-8');
+  }
+
+  /**
+   * Append data to JSONL file
+   */
+  static async append(filePath: string, data: JsonObject[]): Promise<void> {
+    const { readFile, writeFile } = await import('node:fs/promises');
+
+    try {
+      const existing = await readFile(filePath, 'utf-8');
+      const lines = data.map((obj) => JSON.stringify(obj)).join('\n');
+      const newContent = existing.trim() + '\n' + lines + '\n';
+      await writeFile(filePath, newContent, 'utf-8');
+    } catch (error) {
+      // If file doesn't exist, just write the data
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+        await this.write(filePath, data);
+      } else {
+        throw error;
+      }
+    }
+  }
+}

package/src/runtime.ts

@@ -0,0 +1,34 @@
+/**
+ * Runtime detection utilities
+ */
+
+export type RuntimeEnvironment = 'node' | 'bun' | 'deno' | 'unknown';
+
+export function detectRuntime(): RuntimeEnvironment {
+  // Check for Bun
+  if (
+    typeof globalThis !== 'undefined' &&
+    'Bun' in globalThis &&
+    typeof (globalThis as { Bun?: unknown }).Bun !== 'undefined'
+  ) {
+    return 'bun';
+  }
+
+  // Check for Deno
+  if (
+    typeof globalThis !== 'undefined' &&
+    'Deno' in globalThis &&
+    typeof (globalThis as { Deno?: unknown }).Deno !== 'undefined'
+  ) {
+    return 'deno';
+  }
+
+  // Check for Node.js
+  if (typeof process !== 'undefined' && process.versions && process.versions.node) {
+    return 'node';
+  }
+
+  return 'unknown';
+}
+
+export const RUNTIME = detectRuntime();

package/src/schema-loader.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { SchemaLoader } from './schema-loader.js';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('SchemaLoader', () => {
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `schema-loader-test-${Date.now()}`);
+    await mkdir(testDir, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  describe('loadSchema', () => {
+    it('should load schema from .ts file', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test',
+            validate: () => ({ value: {} })
+          }
+        };
+      `,
+      );
+
+      const schema = await SchemaLoader.loadSchema(jsonlPath);
+
+      expect(schema).toBeDefined();
+      expect(schema['~standard'].version).toBe(1);
+      expect(schema['~standard'].vendor).toBe('test');
+    });
+
+    it('should load default export', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export default {
+          '~standard': {
+            version: 1,
+            vendor: 'default-export',
+            validate: () => ({ value: {} })
+          }
+        };
+      `,
+      );
+
+      const schema = await SchemaLoader.loadSchema(jsonlPath);
+
+      expect(schema['~standard'].vendor).toBe('default-export');
+    });
+
+    it('should throw when no schema file exists', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      await writeFile(jsonlPath, '{"id":1}\n');
+
+      await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(/Schema file not found/);
+    });
+
+    it('should throw for invalid schema export', async () => {
+      const jsonlPath = join(testDir, 'users.jsonl');
+      const schemaPath = join(testDir, 'users.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1}\n');
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          notAStandardSchema: true
+        };
+      `,
+      );
+
+      await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
+        /does not export a valid StandardSchema/,
+      );
+    });
+
+    it('should validate StandardSchema structure', async () => {
+      const jsonlPath = join(testDir, 'test.jsonl');
+      const schemaPath = join(testDir, 'test.schema.ts');
+
+      await writeFile(jsonlPath, '{"id":1}\n');
+
+      // Missing validate function
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 1,
+            vendor: 'test'
+          }
+        };
+      `,
+      );
+
+      await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
+        /does not export a valid StandardSchema/,
+      );
+
+      // Wrong version
+      await writeFile(
+        schemaPath,
+        `
+        export const schema = {
+          '~standard': {
+            version: 2,
+            vendor: 'test',
+            validate: () => ({ value: {} })
+          }
+        };
+      `,
+      );
+
+      await expect(SchemaLoader.loadSchema(jsonlPath)).rejects.toThrow(
+        /does not export a valid StandardSchema/,
+      );
+    });
+  });
+});

package/src/schema-loader.ts

@@ -0,0 +1,64 @@
+import { pathToFileURL } from 'node:url';
+import { access } from 'node:fs/promises';
+import { dirname, join, basename } from 'node:path';
+import type { StandardSchema } from './types.js';
+
+export class SchemaLoader {
+  /**
+   * Load a validation schema file for a table
+   * Requires ${tableName}.schema.ts to exist alongside the JSONL file
+   */
+  static async loadSchema(jsonlPath: string): Promise<StandardSchema> {
+    const dir = dirname(jsonlPath);
+    const tableName = basename(jsonlPath, '.jsonl');
+    const schemaPath = join(dir, `${tableName}.schema.ts`);
+
+    try {
+      await access(schemaPath);
+    } catch (error) {
+      throw new Error(`Schema file not found for table '${tableName}'. Expected: ${schemaPath}`, {
+        cause: error instanceof Error ? error : undefined,
+      });
+    }
+
+    try {
+      const schemaUrl = pathToFileURL(schemaPath).href;
+      // Add cache busting query parameter to force reload on each import
+      // This ensures schema changes are picked up immediately
+      const cacheBustedUrl = `${schemaUrl}?t=${Date.now()}`;
+      const module = await import(cacheBustedUrl);
+      const schema = module.default || module.schema;
+
+      if (schema && this.isStandardSchema(schema)) {
+        return schema;
+      }
+
+      throw new Error(`Schema file ${schemaPath} does not export a valid StandardSchema`);
+    } catch (error) {
+      throw new Error(
+        `Failed to load schema for table '${tableName}' from ${schemaPath}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error instanceof Error ? error : undefined },
+      );
+    }
+  }
+
+  /**
+   * Check if an object implements the StandardSchema interface
+   */
+  private static isStandardSchema(obj: unknown): obj is StandardSchema {
+    if (!obj || typeof obj !== 'object') return false;
+
+    const schema = obj as Record<string, unknown>;
+    const standard = schema['~standard'];
+
+    if (!standard || typeof standard !== 'object') return false;
+
+    const standardObj = standard as Record<string, unknown>;
+
+    return (
+      standardObj.version === 1 &&
+      typeof standardObj.vendor === 'string' &&
+      typeof standardObj.validate === 'function'
+    );
+  }
+}

package/src/schema.ts

@@ -0,0 +1,135 @@
+import type { StandardSchema, Table, ForeignKeyDefinition, IndexDefinition } from './types.js';
+
+/**
+ * Schema options for defining constraints and indexes
+ */
+export interface SchemaOptions {
+  /**
+   * Primary key columns
+   */
+  primaryKey?: string[];
+
+  /**
+   * Foreign key constraints
+   */
+  foreignKeys?: ForeignKeyDefinition[];
+
+  /**
+   * Indexes to create
+   */
+  indexes?: IndexDefinition[];
+
+  /**
+   * Backward transformation from Output to Input
+   * Required when Input and Output types differ (e.g., with transformations)
+   */
+  backward?: (output: Table) => Table;
+}
+
+/**
+ * BiDirectional Schema interface
+ * Extends StandardSchema with optional backward transformation and schema metadata
+ */
+export interface BiDirectionalSchema<Input extends Table = Table, Output extends Table = Input>
+  extends StandardSchema<Input, Output> {
+  /**
+   * Backward transformation from Output to Input
+   * Required when Input and Output types differ (e.g., with transformations)
+   */
+  backward?: (output: Output) => Input;
+
+  /**
+   * Primary key columns
+   */
+  primaryKey?: string[];
+
+  /**
+   * Foreign key constraints
+   */
+  foreignKeys?: ForeignKeyDefinition[];
+
+  /**
+   * Indexes to create
+   */
+  indexes?: IndexDefinition[];
+}
+
+/**
+ * Define a bidirectional schema with optional backward transformation
+ *
+ * @param schema - Standard Schema for validation
+ * @param optionsOrBackward - Optional SchemaOptions object or backward transformation function (Output → Input)
+ *                            Required when schema performs transformations
+ *
+ * @example
+ * // No transformation - backward not needed
+ * const schema = defineSchema(
+ *   v.object({ id: v.number(), name: v.string() })
+ * );
+ *
+ * @example
+ * // With transformation - backward recommended (legacy)
+ * const schema = defineSchema(
+ *   v.pipe(v.string(), v.transform(Number)),
+ *   (num) => String(num)  // backward: number → string
+ * );
+ *
+ * @example
+ * // With primary key and foreign key
+ * const schema = defineSchema(
+ *   v.object({ id: v.number(), customerId: v.number() }),
+ *   {
+ *     primaryKey: ['id'],
+ *     foreignKeys: [
+ *       { columns: ['customerId'], references: { table: 'users', columns: ['id'] } }
+ *     ]
+ *   }
+ * );
+ */
+export function defineSchema<Input extends Table, Output extends Table>(
+  schema: StandardSchema<Input, Output>,
+  optionsOrBackward?: SchemaOptions | ((output: Output) => Input),
+): BiDirectionalSchema<Input, Output> {
+  // Create a new object that extends the schema
+  const bidirectionalSchema = Object.create(schema) as BiDirectionalSchema<Input, Output>;
+
+  // Handle options or backward function
+  if (optionsOrBackward) {
+    if (typeof optionsOrBackward === 'function') {
+      // Legacy: backward function only
+      bidirectionalSchema.backward = optionsOrBackward;
+    } else {
+      // New: options object
+      if (optionsOrBackward.backward) {
+        bidirectionalSchema.backward = optionsOrBackward.backward as (output: Output) => Input;
+      }
+      if (optionsOrBackward.primaryKey) {
+        bidirectionalSchema.primaryKey = optionsOrBackward.primaryKey;
+      }
+      if (optionsOrBackward.foreignKeys) {
+        bidirectionalSchema.foreignKeys = optionsOrBackward.foreignKeys;
+      }
+      if (optionsOrBackward.indexes) {
+        bidirectionalSchema.indexes = optionsOrBackward.indexes;
+      }
+    }
+  }
+
+  // Copy '~standard' property
+  Object.defineProperty(bidirectionalSchema, '~standard', {
+    value: schema['~standard'],
+    enumerable: true,
+    configurable: true,
+  });
+
+  return bidirectionalSchema;
+}
+
+/**
+ * Check if a schema has backward transformation
+ */
+export function hasBackward<Input extends Table, Output extends Table>(
+  schema: StandardSchema<Input, Output>,
+): schema is BiDirectionalSchema<Input, Output> {
+  return 'backward' in schema && typeof (schema as { backward?: unknown }).backward === 'function';
+}