@mauryasumit/driftdb 2.0.0

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@mauryasumit/driftdb",
+  "version": "2.0.0",
+  "description": "Local-first SQLite database with automatic S3 sync — offline-first, no infrastructure required",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "typecheck": "tsc --noEmit",
+    "test": "jest",
+    "test:coverage": "jest --coverage",
+    "lint": "eslint src --ext .ts",
+    "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
+    "prebuild": "npm run clean",
+    "prepublishOnly": "npm run typecheck && npm run test && npm run build",
+    "publish:patch": "npm version patch && npm publish --access public",
+    "publish:minor": "npm version minor && npm publish --access public",
+    "publish:major": "npm version major && npm publish --access public",
+    "publish:dry": "npm publish --dry-run --access public"
+  },
+  "keywords": [
+    "sqlite",
+    "s3",
+    "local-first",
+    "offline",
+    "sync",
+    "database",
+    "orm",
+    "local-first",
+    "durable"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.600.0",
+    "@aws-sdk/lib-storage": "^3.600.0",
+    "better-sqlite3": "^12.8.0",
+    "uuid": "^9.0.1"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.10",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.14.0",
+    "@types/uuid": "^9.0.8",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.4",
+    "typescript": "^5.4.5"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/src/db.ts

@@ -0,0 +1,154 @@
+import BetterSqlite3 from 'better-sqlite3';
+import type Database from 'better-sqlite3';
+import { mkdirSync, existsSync } from 'fs';
+import { dirname } from 'path';
+import type { DBConfig, ModelSchema, SyncMetrics } from './types.js';
+import { Repository } from './orm/repository.js';
+import { SyncEngine } from './sync/engine.js';
+import { generateNodeId } from './utils/id.js';
+import type { Model, ModelStatic } from './orm/model.js';
+import type { BaseRecord } from './types.js';
+
+const META_SCHEMA = `
+  CREATE TABLE IF NOT EXISTS _driftdb_meta (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+  );
+`;
+
+export class DB {
+  private readonly sqliteDb: Database.Database;
+  private readonly config: DBConfig;
+  private readonly nodeId: string;
+  private readonly syncEngine: SyncEngine;
+  private readonly repos = new Map<string, Repository<BaseRecord>>();
+
+  constructor(config: DBConfig) {
+    this.config = config;
+
+    if (config.sqlitePath !== ':memory:') {
+      const dir = dirname(config.sqlitePath);
+      if (dir && dir !== '.') {
+        mkdirSync(dir, { recursive: true });
+      }
+    }
+
+    this.sqliteDb = new BetterSqlite3(config.sqlitePath);
+    this.sqliteDb.pragma('journal_mode = WAL');
+    this.sqliteDb.pragma('synchronous = NORMAL');
+    this.sqliteDb.pragma('foreign_keys = ON');
+    this.sqliteDb.pragma('cache_size = -64000');
+    this.sqliteDb.pragma('temp_store = MEMORY');
+
+    this.sqliteDb.exec(META_SCHEMA);
+
+    this.nodeId = this.getOrCreateNodeId(config.nodeId);
+    this.syncEngine = new SyncEngine(this.sqliteDb, this.nodeId, config);
+
+    if (config.autoSync !== false && config.s3Config) {
+      this.syncEngine.start();
+    }
+  }
+
+  private getOrCreateNodeId(preferred?: string): string {
+    if (preferred) {
+      this.sqliteDb
+        .prepare(`INSERT OR REPLACE INTO _driftdb_meta (key, value) VALUES ('nodeId', ?)`)
+        .run(preferred);
+      return preferred;
+    }
+
+    const row = this.sqliteDb
+      .prepare(`SELECT value FROM _driftdb_meta WHERE key = 'nodeId'`)
+      .get() as { value: string } | undefined;
+
+    if (row) return row.value;
+
+    const id = generateNodeId();
+    this.sqliteDb
+      .prepare(`INSERT INTO _driftdb_meta (key, value) VALUES ('nodeId', ?)`)
+      .run(id);
+    return id;
+  }
+
+  define<S extends ModelSchema>(
+    tableName: string,
+    schema: S
+  ): Repository<BaseRecord & { [K in keyof S]: unknown }> {
+    const repo = new Repository<BaseRecord & { [K in keyof S]: unknown }>(
+      this.sqliteDb,
+      tableName,
+      schema,
+      this.syncEngine.getChangeLog()
+    );
+    this.repos.set(tableName, repo as unknown as Repository<BaseRecord>);
+    return repo;
+  }
+
+  registerModel<T extends Model>(ModelClass: ModelStatic<T>): void {
+    if (!ModelClass.tableName) {
+      throw new Error(`Model ${ModelClass.name} must define a static 'tableName'`);
+    }
+    if (!ModelClass.schema) {
+      throw new Error(`Model ${ModelClass.name} must define a static 'schema'`);
+    }
+
+    const repo = new Repository<T>(
+      this.sqliteDb,
+      ModelClass.tableName,
+      ModelClass.schema,
+      this.syncEngine.getChangeLog()
+    );
+
+    (ModelClass as unknown as { _repo: Repository<T> })._repo = repo;
+    this.repos.set(ModelClass.tableName, repo as unknown as Repository<BaseRecord>);
+  }
+
+  getNodeId(): string {
+    return this.nodeId;
+  }
+
+  getMetrics(): Readonly<SyncMetrics> {
+    return this.syncEngine.getMetrics();
+  }
+
+  async flush(): Promise<void> {
+    return this.syncEngine.flush();
+  }
+
+  async snapshot(): Promise<void> {
+    return this.syncEngine.triggerSnapshot();
+  }
+
+  startSync(): void {
+    this.syncEngine.start();
+  }
+
+  stopSync(): void {
+    this.syncEngine.stop();
+  }
+
+  raw(): Database.Database {
+    return this.sqliteDb;
+  }
+
+  close(): void {
+    this.syncEngine.stop();
+    this.sqliteDb.close();
+  }
+
+  transaction<T>(fn: () => T): T {
+    return this.sqliteDb.transaction(fn)();
+  }
+
+  vacuum(): void {
+    this.sqliteDb.exec('VACUUM');
+  }
+
+  integrityCheck(): boolean {
+    const result = this.sqliteDb
+      .prepare('PRAGMA integrity_check')
+      .get() as { integrity_check: string };
+    return result.integrity_check === 'ok';
+  }
+}

package/src/index.ts

@@ -0,0 +1,24 @@
+export { DB } from './db.js';
+export { Model } from './orm/model.js';
+export type { ModelStatic } from './orm/model.js';
+export { Repository } from './orm/repository.js';
+export { Column } from './orm/schema.js';
+export type { RecordOf } from './orm/repository.js';
+
+export type {
+  DBConfig,
+  S3Config,
+  ModelSchema,
+  ColumnDef,
+  ColumnType,
+  BaseRecord,
+  WhereClause,
+  FindOptions,
+  SyncMetrics,
+  RetryConfig,
+  EncryptionConfig,
+  LogBatch,
+  ChangeLogEntry,
+  SyncJob,
+  SyncManifest,
+} from './types.js';

package/src/orm/model.ts

@@ -0,0 +1,95 @@
+import type { BaseRecord, FindOptions, ModelSchema, WhereClause } from '../types.js';
+import type { Repository } from './repository.js';
+
+export abstract class Model implements BaseRecord {
+  id!: string;
+  createdAt!: number;
+  updatedAt!: number;
+
+  static tableName: string;
+  static schema: ModelSchema;
+
+  static _repo: Repository<BaseRecord>;
+
+  static async create<T extends Model>(
+    this: ModelStatic<T>,
+    data: Partial<Omit<T, 'id' | 'createdAt' | 'updatedAt'>>
+  ): Promise<T> {
+    return (this._repo as Repository<T>).create(data) as Promise<T>;
+  }
+
+  static async findById<T extends Model>(
+    this: ModelStatic<T>,
+    id: string
+  ): Promise<T | null> {
+    return (this._repo as Repository<T>).findById(id) as Promise<T | null>;
+  }
+
+  static async findOne<T extends Model>(
+    this: ModelStatic<T>,
+    where: WhereClause<T>
+  ): Promise<T | null> {
+    return (this._repo as Repository<T>).findOne(where) as Promise<T | null>;
+  }
+
+  static async find<T extends Model>(
+    this: ModelStatic<T>,
+    options: FindOptions<T> = {}
+  ): Promise<T[]> {
+    return (this._repo as Repository<T>).find(options) as Promise<T[]>;
+  }
+
+  static async filter<T extends Model>(
+    this: ModelStatic<T>,
+    where: WhereClause<T> = {},
+    options: Omit<FindOptions<T>, 'where'> = {}
+  ): Promise<T[]> {
+    return (this._repo as Repository<T>).filter(where, options) as Promise<T[]>;
+  }
+
+  static async update<T extends Model>(
+    this: ModelStatic<T>,
+    where: WhereClause<T>,
+    data: Partial<Omit<T, 'id' | 'createdAt'>>
+  ): Promise<number> {
+    return (this._repo as Repository<T>).update(where, data);
+  }
+
+  static async delete<T extends Model>(
+    this: ModelStatic<T>,
+    where: WhereClause<T>
+  ): Promise<number> {
+    return (this._repo as Repository<T>).delete(where);
+  }
+
+  static async count<T extends Model>(
+    this: ModelStatic<T>,
+    where: WhereClause<T> = {}
+  ): Promise<number> {
+    return (this._repo as Repository<T>).count(where);
+  }
+
+  static async upsert<T extends Model>(
+    this: ModelStatic<T>,
+    where: WhereClause<T>,
+    data: Partial<Omit<T, 'id' | 'createdAt'>>
+  ): Promise<T> {
+    return (this._repo as Repository<T>).upsert(where, data) as Promise<T>;
+  }
+}
+
+export interface ModelStatic<T extends Model = Model> {
+  new(): T;
+  tableName: string;
+  schema: ModelSchema;
+  _repo: Repository<BaseRecord>;
+  create(data: Partial<Omit<T, 'id' | 'createdAt' | 'updatedAt'>>): Promise<T>;
+  findById(id: string): Promise<T | null>;
+  findOne(where: WhereClause<T>): Promise<T | null>;
+  find(options?: FindOptions<T>): Promise<T[]>;
+  filter(where?: WhereClause<T>, options?: Omit<FindOptions<T>, 'where'>): Promise<T[]>;
+  update(where: WhereClause<T>, data: Partial<Omit<T, 'id' | 'createdAt'>>): Promise<number>;
+  delete(where: WhereClause<T>): Promise<number>;
+  count(where?: WhereClause<T>): Promise<number>;
+  upsert(where: WhereClause<T>, data: Partial<Omit<T, 'id' | 'createdAt'>>): Promise<T>;
+}

package/src/orm/query-builder.ts

@@ -0,0 +1,100 @@
+import type { WhereClause, FindOptions } from '../types.js';
+
+export interface QueryParts {
+  sql: string;
+  params: unknown[];
+}
+
+export function buildWhereClause<T>(where: WhereClause<T>): QueryParts {
+  const conditions: string[] = [];
+  const params: unknown[] = [];
+
+  for (const [key, value] of Object.entries(where)) {
+    if (value === undefined) continue;
+
+    if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+      const ops = value as Record<string, unknown>;
+
+      if ('$gt' in ops) {
+        conditions.push(`"${key}" > ?`);
+        params.push(ops['$gt']);
+      }
+      if ('$gte' in ops) {
+        conditions.push(`"${key}" >= ?`);
+        params.push(ops['$gte']);
+      }
+      if ('$lt' in ops) {
+        conditions.push(`"${key}" < ?`);
+        params.push(ops['$lt']);
+      }
+      if ('$lte' in ops) {
+        conditions.push(`"${key}" <= ?`);
+        params.push(ops['$lte']);
+      }
+      if ('$ne' in ops) {
+        conditions.push(`"${key}" != ?`);
+        params.push(ops['$ne']);
+      }
+      if ('$like' in ops) {
+        conditions.push(`"${key}" LIKE ?`);
+        params.push(ops['$like']);
+      }
+      if ('$in' in ops && Array.isArray(ops['$in'])) {
+        const arr = ops['$in'] as unknown[];
+        if (arr.length === 0) {
+          conditions.push('0 = 1');
+        } else {
+          conditions.push(`"${key}" IN (${arr.map(() => '?').join(',')})`);
+          params.push(...arr);
+        }
+      }
+    } else {
+      if (value === null) {
+        conditions.push(`"${key}" IS NULL`);
+      } else {
+        conditions.push(`"${key}" = ?`);
+        params.push(value);
+      }
+    }
+  }
+
+  return {
+    sql: conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '',
+    params,
+  };
+}
+
+export function buildSelectSQL<T>(
+  tableName: string,
+  options: FindOptions<T> = {}
+): QueryParts {
+  const parts: string[] = [`SELECT * FROM "${tableName}"`];
+  const params: unknown[] = [];
+
+  if (options.where && Object.keys(options.where).length > 0) {
+    const { sql, params: wParams } = buildWhereClause(options.where);
+    if (sql) {
+      parts.push(sql);
+      params.push(...wParams);
+    }
+  }
+
+  if (options.orderBy) {
+    const orderClauses = Object.entries(options.orderBy)
+      .map(([col, dir]) => `"${col}" ${dir ?? 'ASC'}`)
+      .join(', ');
+    if (orderClauses) parts.push(`ORDER BY ${orderClauses}`);
+  }
+
+  if (options.limit !== undefined) {
+    parts.push(`LIMIT ?`);
+    params.push(options.limit);
+  }
+
+  if (options.offset !== undefined) {
+    parts.push(`OFFSET ?`);
+    params.push(options.offset);
+  }
+
+  return { sql: parts.join(' '), params };
+}

package/src/orm/repository.ts

@@ -0,0 +1,156 @@
+import type Database from 'better-sqlite3';
+import type { BaseRecord, FindOptions, ModelSchema, WhereClause } from '../types.js';
+import { buildCreateTableSQL, normalizeSchema } from './schema.js';
+import { buildSelectSQL, buildWhereClause } from './query-builder.js';
+import { generateId } from '../utils/id.js';
+import type { ChangeLog } from '../sync/change-log.js';
+
+export type RecordOf<S extends ModelSchema> = BaseRecord & {
+  [K in keyof S]: S[K] extends { type: 'INTEGER' }
+    ? number
+    : S[K] extends { type: 'REAL' }
+    ? number
+    : S[K] extends { type: 'BOOLEAN' }
+    ? boolean
+    : S[K] extends { type: 'BLOB' }
+    ? Buffer
+    : string;
+};
+
+export class Repository<T extends BaseRecord> {
+  private readonly db: Database.Database;
+  private readonly tableName: string;
+  private readonly schema: ModelSchema;
+  private readonly changeLog: ChangeLog | null;
+
+  constructor(
+    db: Database.Database,
+    tableName: string,
+    schema: ModelSchema,
+    changeLog: ChangeLog | null
+  ) {
+    this.db = db;
+    this.tableName = tableName;
+    this.schema = normalizeSchema(schema);
+    this.changeLog = changeLog;
+    this.initTable();
+  }
+
+  private initTable(): void {
+    const sql = buildCreateTableSQL(this.tableName, this.schema);
+    this.db.exec(sql);
+  }
+
+  async create(data: Partial<Omit<T, 'id' | 'createdAt' | 'updatedAt'>>): Promise<T> {
+    const now = Date.now();
+    const id = generateId();
+    const record = { id, createdAt: now, updatedAt: now, ...data } as T;
+
+    const keys = Object.keys(record);
+    const placeholders = keys.map(() => '?').join(', ');
+    const cols = keys.map((k) => `"${k}"`).join(', ');
+    const values = keys.map((k) => (record as Record<string, unknown>)[k]);
+
+    this.db
+      .prepare(`INSERT INTO "${this.tableName}" (${cols}) VALUES (${placeholders})`)
+      .run(...values);
+
+    this.changeLog?.append(this.tableName, 'insert', record as Record<string, unknown>);
+
+    return record;
+  }
+
+  async findById(id: string): Promise<T | null> {
+    const row = this.db
+      .prepare(`SELECT * FROM "${this.tableName}" WHERE id = ?`)
+      .get(id) as T | undefined;
+    return row ?? null;
+  }
+
+  async findOne(where: WhereClause<T>): Promise<T | null> {
+    const { sql, params } = buildSelectSQL<T>(this.tableName, { where, limit: 1 });
+    const row = this.db.prepare(sql).get(...params) as T | undefined;
+    return row ?? null;
+  }
+
+  async find(options: FindOptions<T> = {}): Promise<T[]> {
+    const { sql, params } = buildSelectSQL<T>(this.tableName, options);
+    return this.db.prepare(sql).all(...params) as T[];
+  }
+
+  async filter(where: WhereClause<T> = {}, options: Omit<FindOptions<T>, 'where'> = {}): Promise<T[]> {
+    return this.find({ where, ...options });
+  }
+
+  async update(where: WhereClause<T>, data: Partial<Omit<T, 'id' | 'createdAt'>>): Promise<number> {
+    const now = Date.now();
+    const updateData = { ...data, updatedAt: now };
+
+    const setCols = Object.keys(updateData)
+      .map((k) => `"${k}" = ?`)
+      .join(', ');
+    const setValues = Object.values(updateData);
+
+    const { sql: whereSQL, params: whereParams } = buildWhereClause(where);
+    const sql = `UPDATE "${this.tableName}" SET ${setCols} ${whereSQL}`;
+
+    const result = this.db.prepare(sql).run(...setValues, ...whereParams);
+
+    if (result.changes > 0) {
+      this.changeLog?.append(this.tableName, 'update', {
+        where: where as Record<string, unknown>,
+        data: updateData as Record<string, unknown>,
+      });
+    }
+
+    return result.changes;
+  }
+
+  async delete(where: WhereClause<T>): Promise<number> {
+    const { sql: whereSQL, params: whereParams } = buildWhereClause(where);
+    const sql = `DELETE FROM "${this.tableName}" ${whereSQL}`;
+
+    const result = this.db.prepare(sql).run(...whereParams);
+
+    if (result.changes > 0) {
+      this.changeLog?.append(this.tableName, 'delete', {
+        where: where as Record<string, unknown>,
+      });
+    }
+
+    return result.changes;
+  }
+
+  async deleteById(id: string): Promise<boolean> {
+    const changes = await this.delete({ id } as WhereClause<T>);
+    return changes > 0;
+  }
+
+  async count(where: WhereClause<T> = {}): Promise<number> {
+    const { sql: whereSQL, params } = buildWhereClause(where);
+    const sql = `SELECT COUNT(*) as cnt FROM "${this.tableName}" ${whereSQL}`;
+    const row = this.db.prepare(sql).get(...params) as { cnt: number };
+    return row.cnt;
+  }
+
+  async upsert(
+    where: WhereClause<T>,
+    data: Partial<Omit<T, 'id' | 'createdAt'>>
+  ): Promise<T> {
+    const existing = await this.findOne(where);
+    if (existing) {
+      await this.update(where, data);
+      return (await this.findById(existing.id))!;
+    }
+    return this.create(data as Partial<Omit<T, 'id' | 'createdAt' | 'updatedAt'>>);
+  }
+
+  raw<R = unknown>(sql: string, params: unknown[] = []): R[] {
+    return this.db.prepare(sql).all(...params) as R[];
+  }
+
+  runRaw(sql: string, params: unknown[] = []): { changes: number; lastInsertRowid: number | bigint } {
+    const result = this.db.prepare(sql).run(...params);
+    return { changes: result.changes, lastInsertRowid: result.lastInsertRowid };
+  }
+}

package/src/orm/schema.ts

@@ -0,0 +1,92 @@
+import type { ColumnDef, ColumnType, ModelSchema } from '../types.js';
+
+export class ColumnBuilder {
+  private readonly def: ColumnDef;
+
+  constructor(type: ColumnType) {
+    this.def = { type };
+  }
+
+  required(): this {
+    this.def.notNull = true;
+    return this;
+  }
+
+  unique(): this {
+    this.def.unique = true;
+    return this;
+  }
+
+  default(value: string | number | boolean | null): this {
+    this.def.default = value;
+    return this;
+  }
+
+  index(): this {
+    this.def.index = true;
+    return this;
+  }
+
+  build(): ColumnDef {
+    return { ...this.def };
+  }
+}
+
+export const Column = {
+  text(): ColumnBuilder {
+    return new ColumnBuilder('TEXT');
+  },
+  integer(): ColumnBuilder {
+    return new ColumnBuilder('INTEGER');
+  },
+  real(): ColumnBuilder {
+    return new ColumnBuilder('REAL');
+  },
+  blob(): ColumnBuilder {
+    return new ColumnBuilder('BLOB');
+  },
+  boolean(): ColumnBuilder {
+    return new ColumnBuilder('BOOLEAN');
+  },
+};
+
+export function buildCreateTableSQL(tableName: string, schema: ModelSchema): string {
+  const cols: string[] = [
+    'id TEXT PRIMARY KEY',
+    'createdAt INTEGER NOT NULL',
+    'updatedAt INTEGER NOT NULL',
+  ];
+
+  const indices: string[] = [];
+
+  for (const [name, rawDef] of Object.entries(schema)) {
+    const def: ColumnDef =
+      rawDef instanceof ColumnBuilder ? rawDef.build() : rawDef;
+
+    let col = `"${name}" ${def.type}`;
+    if (def.notNull) col += ' NOT NULL';
+    if (def.unique) col += ' UNIQUE';
+    if (def.default !== undefined) {
+      const v = def.default;
+      col += ` DEFAULT ${typeof v === 'string' ? `'${v}'` : String(v)}`;
+    }
+    cols.push(col);
+
+    if (def.index) {
+      indices.push(
+        `CREATE INDEX IF NOT EXISTS "idx_${tableName}_${name}" ON "${tableName}" ("${name}");`
+      );
+    }
+  }
+
+  const create = `CREATE TABLE IF NOT EXISTS "${tableName}" (\n  ${cols.join(',\n  ')}\n);`;
+  return [create, ...indices].join('\n');
+}
+
+export function normalizeSchema(schema: ModelSchema): ModelSchema {
+  const normalized: ModelSchema = {};
+  for (const [key, val] of Object.entries(schema)) {
+    normalized[key] = val instanceof ColumnBuilder ? val.build() : val;
+  }
+  return normalized;
+}