@mauryasumit/driftdb 2.0.0

package/src/queue/queue.ts

@@ -0,0 +1,138 @@
+import type Database from 'better-sqlite3';
+import type { SyncJob } from '../types.js';
+import { generateId } from '../utils/id.js';
+import { nextRetryAt } from '../utils/retry.js';
+import type { RetryConfig } from '../types.js';
+
+const SCHEMA = `
+  CREATE TABLE IF NOT EXISTS _driftdb_queue (
+    id TEXT PRIMARY KEY,
+    type TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'pending',
+    attempts INTEGER NOT NULL DEFAULT 0,
+    nextRetryAt INTEGER NOT NULL DEFAULT 0,
+    createdAt INTEGER NOT NULL,
+    error TEXT
+  );
+  CREATE INDEX IF NOT EXISTS idx_driftdb_queue_status ON _driftdb_queue (status, nextRetryAt);
+`;
+
+export class SyncQueue {
+  private readonly db: Database.Database;
+  private readonly retryConfig: RetryConfig;
+
+  constructor(db: Database.Database, retryConfig: RetryConfig) {
+    this.db = db;
+    this.retryConfig = retryConfig;
+    db.exec(SCHEMA);
+  }
+
+  enqueue(type: SyncJob['type'], payload: object): SyncJob {
+    const job: SyncJob = {
+      id: generateId(),
+      type,
+      payload: JSON.stringify(payload),
+      status: 'pending',
+      attempts: 0,
+      nextRetryAt: 0,
+      createdAt: Date.now(),
+      error: null,
+    };
+
+    this.db
+      .prepare(
+        `INSERT INTO _driftdb_queue (id, type, payload, status, attempts, nextRetryAt, createdAt, error)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+      )
+      .run(job.id, job.type, job.payload, job.status, job.attempts, job.nextRetryAt, job.createdAt, job.error);
+
+    return job;
+  }
+
+  dequeue(limit = 5): SyncJob[] {
+    const now = Date.now();
+    const rows = this.db
+      .prepare(
+        `SELECT * FROM _driftdb_queue
+         WHERE status IN ('pending', 'failed')
+           AND nextRetryAt <= ?
+         ORDER BY createdAt ASC
+         LIMIT ?`
+      )
+      .all(now, limit) as SyncJob[];
+
+    if (rows.length === 0) return [];
+
+    const ids = rows.map((r) => r.id);
+    const placeholders = ids.map(() => '?').join(',');
+    this.db
+      .prepare(`UPDATE _driftdb_queue SET status = 'processing' WHERE id IN (${placeholders})`)
+      .run(...ids);
+
+    return rows.map((r) => ({ ...r, status: 'processing' as const }));
+  }
+
+  markDone(id: string): void {
+    this.db
+      .prepare(`UPDATE _driftdb_queue SET status = 'done', error = NULL WHERE id = ?`)
+      .run(id);
+  }
+
+  markFailed(id: string, error: string): void {
+    const job = this.db
+      .prepare(`SELECT attempts FROM _driftdb_queue WHERE id = ?`)
+      .get(id) as Pick<SyncJob, 'attempts'> | undefined;
+
+    if (!job) return;
+
+    const attempts = job.attempts + 1;
+    const willRetry = attempts <= this.retryConfig.maxRetries;
+    const status = willRetry ? 'failed' : 'failed';
+    const retryAt = willRetry ? nextRetryAt(attempts, this.retryConfig) : 0;
+
+    this.db
+      .prepare(
+        `UPDATE _driftdb_queue
+         SET status = ?, attempts = ?, nextRetryAt = ?, error = ?
+         WHERE id = ?`
+      )
+      .run(status, attempts, retryAt, error.slice(0, 1000), id);
+  }
+
+  resetStuck(): void {
+    const staleThreshold = Date.now() - 5 * 60 * 1000;
+    this.db
+      .prepare(
+        `UPDATE _driftdb_queue
+         SET status = 'pending', nextRetryAt = 0
+         WHERE status = 'processing' AND createdAt < ?`
+      )
+      .run(staleThreshold);
+  }
+
+  pendingCount(): number {
+    const row = this.db
+      .prepare(
+        `SELECT COUNT(*) as cnt FROM _driftdb_queue WHERE status IN ('pending', 'processing', 'failed')`
+      )
+      .get() as { cnt: number };
+    return row.cnt;
+  }
+
+  purgeCompleted(olderThanMs = 24 * 60 * 60 * 1000): void {
+    const cutoff = Date.now() - olderThanMs;
+    this.db
+      .prepare(`DELETE FROM _driftdb_queue WHERE status = 'done' AND createdAt < ?`)
+      .run(cutoff);
+  }
+
+  hasPendingOfType(type: SyncJob['type']): boolean {
+    const row = this.db
+      .prepare(
+        `SELECT 1 FROM _driftdb_queue WHERE type = ? AND status IN ('pending', 'processing', 'failed') LIMIT 1`
+      )
+      .get(type) as { 1: number } | undefined;
+    return row !== undefined;
+  }
+}

package/src/storage/s3-adapter.ts

@@ -0,0 +1,181 @@
+import {
+  S3Client,
+  GetObjectCommand,
+  PutObjectCommand,
+  ListObjectsV2Command,
+  HeadObjectCommand,
+  type S3ClientConfig,
+} from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import { Readable } from 'stream';
+import type { S3Config, SyncManifest } from '../types.js';
+import { compress, decompress } from '../utils/compress.js';
+import { encrypt, decrypt } from '../utils/crypto.js';
+
+export interface S3UploadOptions {
+  compress?: boolean;
+  encryptionKey?: string;
+}
+
+export class S3Adapter {
+  private readonly client: S3Client;
+  private readonly bucket: string;
+  private readonly prefix: string;
+
+  constructor(config: S3Config) {
+    const clientConfig: S3ClientConfig = {
+      region: config.region,
+    };
+
+    if (config.accessKeyId && config.secretAccessKey) {
+      clientConfig.credentials = {
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey,
+      };
+    }
+
+    if (config.endpoint) {
+      clientConfig.endpoint = config.endpoint;
+      clientConfig.forcePathStyle = config.forcePathStyle ?? true;
+    }
+
+    this.client = new S3Client(clientConfig);
+    this.bucket = config.bucket;
+    this.prefix = config.prefix ? config.prefix.replace(/\/$/, '') : 'driftdb';
+  }
+
+  private key(path: string): string {
+    return `${this.prefix}/${path}`;
+  }
+
+  async upload(
+    path: string,
+    data: Buffer,
+    options: S3UploadOptions = {}
+  ): Promise<void> {
+    let payload = data;
+
+    if (options.compress) {
+      payload = await compress(payload);
+    }
+
+    if (options.encryptionKey) {
+      payload = encrypt(payload, options.encryptionKey);
+    }
+
+    const contentEncoding = options.compress ? 'gzip' : undefined;
+    const metadata: Record<string, string> = {};
+    if (options.compress) metadata['x-driftdb-compressed'] = '1';
+    if (options.encryptionKey) metadata['x-driftdb-encrypted'] = '1';
+
+    if (payload.length > 5 * 1024 * 1024) {
+      const upload = new Upload({
+        client: this.client,
+        params: {
+          Bucket: this.bucket,
+          Key: this.key(path),
+          Body: Readable.from(payload),
+          ContentEncoding: contentEncoding,
+          Metadata: metadata,
+        },
+      });
+      await upload.done();
+    } else {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: this.key(path),
+          Body: payload,
+          ContentEncoding: contentEncoding,
+          Metadata: metadata,
+        })
+      );
+    }
+  }
+
+  async download(
+    path: string,
+    options: S3UploadOptions = {}
+  ): Promise<Buffer> {
+    const response = await this.client.send(
+      new GetObjectCommand({ Bucket: this.bucket, Key: this.key(path) })
+    );
+
+    if (!response.Body) {
+      throw new Error(`Empty response body for key: ${path}`);
+    }
+
+    const chunks: Buffer[] = [];
+    for await (const chunk of response.Body as AsyncIterable<Uint8Array>) {
+      chunks.push(Buffer.from(chunk));
+    }
+    let data = Buffer.concat(chunks);
+
+    if (options.encryptionKey) {
+      data = Buffer.from(decrypt(data, options.encryptionKey));
+    }
+
+    if (options.compress) {
+      data = Buffer.from(await decompress(data));
+    }
+
+    return data;
+  }
+
+  async exists(path: string): Promise<boolean> {
+    try {
+      await this.client.send(
+        new HeadObjectCommand({ Bucket: this.bucket, Key: this.key(path) })
+      );
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async listKeys(prefix: string): Promise<string[]> {
+    const fullPrefix = this.key(prefix);
+    const keys: string[] = [];
+    let continuationToken: string | undefined;
+
+    do {
+      const response = await this.client.send(
+        new ListObjectsV2Command({
+          Bucket: this.bucket,
+          Prefix: fullPrefix,
+          ContinuationToken: continuationToken,
+        })
+      );
+
+      for (const obj of response.Contents ?? []) {
+        if (obj.Key) {
+          keys.push(obj.Key.slice(this.prefix.length + 1));
+        }
+      }
+
+      continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+
+    return keys;
+  }
+
+  async putManifest(nodeId: string, manifest: SyncManifest): Promise<void> {
+    const data = Buffer.from(JSON.stringify(manifest), 'utf8');
+    await this.upload(`nodes/${nodeId}/manifest.json`, data);
+  }
+
+  async getManifest(nodeId: string): Promise<SyncManifest | null> {
+    const path = `nodes/${nodeId}/manifest.json`;
+    if (!(await this.exists(path))) return null;
+    const data = await this.download(path);
+    return JSON.parse(data.toString('utf8')) as SyncManifest;
+  }
+
+  logKey(nodeId: string, fromSeq: number, toSeq: number): string {
+    return `nodes/${nodeId}/logs/${String(fromSeq).padStart(12, '0')}-${String(toSeq).padStart(12, '0')}.json`;
+  }
+
+  snapshotKey(nodeId: string, timestamp: number): string {
+    return `nodes/${nodeId}/snapshots/${timestamp}.sqlite`;
+  }
+}

package/src/sync/change-log.ts

@@ -0,0 +1,101 @@
+import type Database from 'better-sqlite3';
+import type { ChangeLogEntry, LogBatch } from '../types.js';
+
+const SCHEMA = `
+  CREATE TABLE IF NOT EXISTS _driftdb_log (
+    sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+    timestamp INTEGER NOT NULL,
+    nodeId TEXT NOT NULL,
+    \`table\` TEXT NOT NULL,
+    operation TEXT NOT NULL,
+    data TEXT,
+    synced INTEGER NOT NULL DEFAULT 0
+  );
+  CREATE INDEX IF NOT EXISTS idx_driftdb_log_synced ON _driftdb_log (synced, sequence);
+`;
+
+export class ChangeLog {
+  private readonly db: Database.Database;
+  private readonly nodeId: string;
+
+  constructor(db: Database.Database, nodeId: string) {
+    this.db = db;
+    this.nodeId = nodeId;
+    db.exec(SCHEMA);
+  }
+
+  append(
+    table: string,
+    operation: ChangeLogEntry['operation'],
+    data: Record<string, unknown> | null
+  ): number {
+    const result = this.db
+      .prepare(
+        `INSERT INTO _driftdb_log (timestamp, nodeId, \`table\`, operation, data, synced)
+         VALUES (?, ?, ?, ?, ?, 0)`
+      )
+      .run(Date.now(), this.nodeId, table, operation, data ? JSON.stringify(data) : null);
+
+    return Number(result.lastInsertRowid);
+  }
+
+  pendingEntries(limit: number): ChangeLogEntry[] {
+    return this.db
+      .prepare(
+        `SELECT * FROM _driftdb_log WHERE synced = 0 ORDER BY sequence ASC LIMIT ?`
+      )
+      .all(limit) as ChangeLogEntry[];
+  }
+
+  markSynced(fromSequence: number, toSequence: number): void {
+    this.db
+      .prepare(
+        `UPDATE _driftdb_log SET synced = 1 WHERE sequence >= ? AND sequence <= ?`
+      )
+      .run(fromSequence, toSequence);
+  }
+
+  pendingCount(): number {
+    const row = this.db
+      .prepare(`SELECT COUNT(*) as cnt FROM _driftdb_log WHERE synced = 0`)
+      .get() as { cnt: number };
+    return row.cnt;
+  }
+
+  latestSyncedSequence(): number {
+    const row = this.db
+      .prepare(`SELECT MAX(sequence) as seq FROM _driftdb_log WHERE synced = 1`)
+      .get() as { seq: number | null };
+    return row.seq ?? 0;
+  }
+
+  buildBatch(entries: ChangeLogEntry[]): LogBatch {
+    return {
+      version: 1,
+      nodeId: this.nodeId,
+      fromSequence: entries[0]?.sequence ?? 0,
+      toSequence: entries[entries.length - 1]?.sequence ?? 0,
+      entries: entries.map((e) => ({
+        sequence: e.sequence,
+        timestamp: e.timestamp,
+        table: e.table,
+        operation: e.operation,
+        data: e.data ? (JSON.parse(e.data) as Record<string, unknown>) : null,
+      })),
+    };
+  }
+
+  purgeOldSynced(keepLatest = 5000): void {
+    const row = this.db
+      .prepare(
+        `SELECT sequence FROM _driftdb_log WHERE synced = 1 ORDER BY sequence DESC LIMIT 1 OFFSET ?`
+      )
+      .get(keepLatest) as { sequence: number } | undefined;
+
+    if (row) {
+      this.db
+        .prepare(`DELETE FROM _driftdb_log WHERE synced = 1 AND sequence <= ?`)
+        .run(row.sequence);
+    }
+  }
+}

package/src/sync/engine.ts

@@ -0,0 +1,249 @@
+import type Database from 'better-sqlite3';
+import type { DBConfig, SyncJob, SyncMetrics, UploadLogPayload, UploadSnapshotPayload } from '../types.js';
+import { SyncQueue } from '../queue/queue.js';
+import { ChangeLog } from './change-log.js';
+import { SnapshotManager } from './snapshot-manager.js';
+import { S3Adapter } from '../storage/s3-adapter.js';
+import { withRetry } from '../utils/retry.js';
+import { statSync, existsSync } from 'fs';
+
+const DEFAULT_SYNC_INTERVAL_MS = 5_000;
+const DEFAULT_SNAPSHOT_EVERY_N_LOGS = 1_000;
+const DEFAULT_MAX_BATCH_SIZE = 100;
+const DEFAULT_RETRY_CONFIG = { maxRetries: 5, baseDelayMs: 500, maxDelayMs: 30_000 };
+
+export class SyncEngine {
+  private readonly db: Database.Database;
+  private readonly config: DBConfig;
+  private readonly nodeId: string;
+  private readonly queue: SyncQueue;
+  private readonly changeLog: ChangeLog;
+  private readonly snapshotManager: SnapshotManager | null;
+  private readonly s3: S3Adapter | null;
+
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private isProcessing = false;
+
+  private metrics: SyncMetrics = {
+    lastSyncAt: null,
+    lastSnapshotAt: null,
+    pendingChanges: 0,
+    dbSizeBytes: 0,
+    totalSynced: 0,
+    syncErrors: 0,
+    isRunning: false,
+  };
+
+  constructor(db: Database.Database, nodeId: string, config: DBConfig) {
+    this.db = db;
+    this.config = config;
+    this.nodeId = nodeId;
+
+    const retryConfig = { ...DEFAULT_RETRY_CONFIG, ...config.retryConfig };
+    this.queue = new SyncQueue(db, retryConfig);
+    this.changeLog = new ChangeLog(db, nodeId);
+
+    if (config.s3Config) {
+      this.s3 = new S3Adapter(config.s3Config);
+      const uploadOptions = {
+        compress: config.compression !== false,
+        encryptionKey: config.encryption?.key,
+      };
+      this.snapshotManager = new SnapshotManager(
+        db,
+        this.s3,
+        nodeId,
+        config.sqlitePath,
+        uploadOptions
+      );
+    } else {
+      this.s3 = null;
+      this.snapshotManager = null;
+    }
+  }
+
+  getChangeLog(): ChangeLog {
+    return this.changeLog;
+  }
+
+  getQueue(): SyncQueue {
+    return this.queue;
+  }
+
+  start(): void {
+    if (this.timer) return;
+    this.metrics.isRunning = true;
+    this.queue.resetStuck();
+
+    const intervalMs = this.config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS;
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, intervalMs);
+
+    if (this.timer.unref) {
+      this.timer.unref();
+    }
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    this.metrics.isRunning = false;
+  }
+
+  async flush(): Promise<void> {
+    await this.tick();
+  }
+
+  getMetrics(): Readonly<SyncMetrics> {
+    return {
+      ...this.metrics,
+      pendingChanges: this.changeLog.pendingCount(),
+      dbSizeBytes: this.getDbSizeBytes(),
+    };
+  }
+
+  private getDbSizeBytes(): number {
+    const path = this.config.sqlitePath;
+    if (path === ':memory:' || !existsSync(path)) return 0;
+    try {
+      return statSync(path).size;
+    } catch {
+      return 0;
+    }
+  }
+
+  private async tick(): Promise<void> {
+    if (this.isProcessing || !this.s3) return;
+    this.isProcessing = true;
+
+    try {
+      await this.enqueuePendingLogs();
+      await this.processQueue();
+    } catch (err) {
+      this.metrics.syncErrors++;
+    } finally {
+      this.isProcessing = false;
+    }
+  }
+
+  private async enqueuePendingLogs(): Promise<void> {
+    const maxBatch = this.config.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE;
+    const pending = this.changeLog.pendingEntries(maxBatch);
+    if (pending.length === 0) return;
+
+    const batch = this.changeLog.buildBatch(pending);
+    const s3Key = this.s3!.logKey(this.nodeId, batch.fromSequence, batch.toSequence);
+
+    const alreadyQueued = this.queue.hasPendingOfType('upload_log');
+    if (!alreadyQueued) {
+      const payload: UploadLogPayload = {
+        fromSequence: batch.fromSequence,
+        toSequence: batch.toSequence,
+        s3Key,
+      };
+      this.queue.enqueue('upload_log', { ...payload, batch });
+    }
+  }
+
+  private async processQueue(): Promise<void> {
+    const jobs = this.queue.dequeue(3);
+
+    await Promise.allSettled(
+      jobs.map((job) => this.processJob(job))
+    );
+
+    this.queue.purgeCompleted();
+  }
+
+  private async processJob(job: SyncJob): Promise<void> {
+    const retryConfig = { ...DEFAULT_RETRY_CONFIG, ...this.config.retryConfig };
+    const uploadOptions = {
+      compress: this.config.compression !== false,
+      encryptionKey: this.config.encryption?.key,
+    };
+
+    try {
+      await withRetry(
+        async () => {
+          if (job.type === 'upload_log') {
+            const p = JSON.parse(job.payload) as UploadLogPayload & { batch: unknown };
+            const batchBuffer = Buffer.from(JSON.stringify(p.batch), 'utf8');
+            await this.s3!.upload(p.s3Key, batchBuffer, uploadOptions);
+            this.changeLog.markSynced(p.fromSequence, p.toSequence);
+
+            const manifest = await this.s3!.getManifest(this.nodeId);
+            const latestSeq = Math.max(
+              manifest?.latestLogSequence ?? 0,
+              p.toSequence
+            );
+            await this.s3!.putManifest(this.nodeId, {
+              nodeId: this.nodeId,
+              latestSnapshotKey: manifest?.latestSnapshotKey ?? null,
+              latestSnapshotTimestamp: manifest?.latestSnapshotTimestamp ?? null,
+              latestLogSequence: latestSeq,
+              updatedAt: Date.now(),
+            });
+
+            this.metrics.totalSynced += (p.toSequence - p.fromSequence + 1);
+            this.metrics.lastSyncAt = Date.now();
+
+            await this.maybeSnapshot(latestSeq);
+          } else if (job.type === 'upload_snapshot') {
+            const p = JSON.parse(job.payload) as UploadSnapshotPayload;
+            if (this.snapshotManager) {
+              const { key, timestamp } = await this.snapshotManager.takeAndUpload();
+              const manifest = await this.s3!.getManifest(this.nodeId);
+              await this.s3!.putManifest(this.nodeId, {
+                nodeId: this.nodeId,
+                latestSnapshotKey: key,
+                latestSnapshotTimestamp: timestamp,
+                latestLogSequence: manifest?.latestLogSequence ?? 0,
+                updatedAt: Date.now(),
+              });
+              this.metrics.lastSnapshotAt = Date.now();
+            }
+            void p;
+          }
+        },
+        retryConfig
+      );
+
+      this.queue.markDone(job.id);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      this.queue.markFailed(job.id, msg);
+      this.metrics.syncErrors++;
+    }
+  }
+
+  private async maybeSnapshot(latestSequence: number): Promise<void> {
+    const threshold = this.config.snapshotEveryNLogs ?? DEFAULT_SNAPSHOT_EVERY_N_LOGS;
+    if (latestSequence > 0 && latestSequence % threshold === 0) {
+      if (!this.queue.hasPendingOfType('upload_snapshot')) {
+        const payload: UploadSnapshotPayload = {
+          timestamp: Date.now(),
+          s3Key: this.s3!.snapshotKey(this.nodeId, Date.now()),
+          dbPath: this.config.sqlitePath,
+        };
+        this.queue.enqueue('upload_snapshot', payload);
+      }
+    }
+  }
+
+  async triggerSnapshot(): Promise<void> {
+    if (!this.snapshotManager || !this.s3) return;
+    const { key, timestamp } = await this.snapshotManager.takeAndUpload();
+    const manifest = await this.s3.getManifest(this.nodeId);
+    await this.s3.putManifest(this.nodeId, {
+      nodeId: this.nodeId,
+      latestSnapshotKey: key,
+      latestSnapshotTimestamp: timestamp,
+      latestLogSequence: manifest?.latestLogSequence ?? 0,
+      updatedAt: Date.now(),
+    });
+    this.metrics.lastSnapshotAt = Date.now();
+  }
+}