@mauryasumit/driftdb 2.0.0

package/src/sync/snapshot-manager.ts

@@ -0,0 +1,80 @@
+import { readFileSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { tmpdir } from 'os';
+import { mkdirSync, copyFileSync, unlinkSync } from 'fs';
+import type Database from 'better-sqlite3';
+import type { S3Adapter } from '../storage/s3-adapter.js';
+import type { S3UploadOptions } from '../storage/s3-adapter.js';
+
+export class SnapshotManager {
+  private readonly db: Database.Database;
+  private readonly s3: S3Adapter;
+  private readonly nodeId: string;
+  private readonly sqlitePath: string;
+  private readonly uploadOptions: S3UploadOptions;
+
+  constructor(
+    db: Database.Database,
+    s3: S3Adapter,
+    nodeId: string,
+    sqlitePath: string,
+    uploadOptions: S3UploadOptions
+  ) {
+    this.db = db;
+    this.s3 = s3;
+    this.nodeId = nodeId;
+    this.sqlitePath = sqlitePath;
+    this.uploadOptions = uploadOptions;
+  }
+
+  async takeAndUpload(): Promise<{ key: string; timestamp: number }> {
+    const timestamp = Date.now();
+    const tempPath = join(tmpdir(), `driftdb-snap-${this.nodeId}-${timestamp}.sqlite`);
+
+    try {
+      this.db.exec('PRAGMA wal_checkpoint(FULL)');
+
+      if (this.sqlitePath === ':memory:') {
+        const backup = this.db.serialize();
+        await this.s3.upload(
+          this.s3.snapshotKey(this.nodeId, timestamp),
+          Buffer.from(backup),
+          this.uploadOptions
+        );
+      } else {
+        copyFileSync(this.sqlitePath, tempPath);
+        const data = readFileSync(tempPath);
+        await this.s3.upload(
+          this.s3.snapshotKey(this.nodeId, timestamp),
+          data,
+          this.uploadOptions
+        );
+      }
+
+      const key = this.s3.snapshotKey(this.nodeId, timestamp);
+      return { key, timestamp };
+    } finally {
+      if (existsSync(tempPath)) {
+        try { unlinkSync(tempPath); } catch { /* ignore */ }
+      }
+    }
+  }
+
+  async restoreLatest(): Promise<boolean> {
+    const manifest = await this.s3.getManifest(this.nodeId);
+    if (!manifest?.latestSnapshotKey) return false;
+
+    const data = await this.s3.download(manifest.latestSnapshotKey, this.uploadOptions);
+
+    const restorePath = this.sqlitePath !== ':memory:' ? this.sqlitePath : null;
+    if (!restorePath) return false;
+
+    const dir = dirname(restorePath);
+    mkdirSync(dir, { recursive: true });
+
+    const { writeFileSync } = await import('fs');
+    writeFileSync(restorePath, data);
+
+    return true;
+  }
+}

package/src/types.ts

@@ -0,0 +1,130 @@
+export type ColumnType = 'TEXT' | 'INTEGER' | 'REAL' | 'BLOB' | 'BOOLEAN';
+
+export interface ColumnDef {
+  type: ColumnType;
+  notNull?: boolean;
+  unique?: boolean;
+  default?: string | number | boolean | null;
+  index?: boolean;
+}
+
+export type ModelSchema = Record<string, ColumnDef>;
+
+export interface BaseRecord {
+  id: string;
+  createdAt: number;
+  updatedAt: number;
+}
+
+export type WhereValue<V> =
+  | V
+  | { $gt?: V; $gte?: V; $lt?: V; $lte?: V; $in?: V[]; $like?: string; $ne?: V };
+
+export type WhereClause<T> = {
+  [K in keyof T]?: WhereValue<T[K]>;
+};
+
+export interface FindOptions<T> {
+  where?: WhereClause<T>;
+  orderBy?: Partial<Record<keyof T, 'ASC' | 'DESC'>>;
+  limit?: number;
+  offset?: number;
+}
+
+export interface S3Config {
+  bucket: string;
+  region: string;
+  prefix?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  endpoint?: string;
+  forcePathStyle?: boolean;
+}
+
+export interface RetryConfig {
+  maxRetries: number;
+  baseDelayMs: number;
+  maxDelayMs: number;
+}
+
+export interface EncryptionConfig {
+  key: string;
+}
+
+export interface DBConfig {
+  sqlitePath: string;
+  s3Config?: S3Config;
+  nodeId?: string;
+  syncIntervalMs?: number;
+  snapshotEveryNLogs?: number;
+  maxBatchSize?: number;
+  compression?: boolean;
+  encryption?: EncryptionConfig;
+  retryConfig?: RetryConfig;
+  autoSync?: boolean;
+}
+
+export interface ChangeLogEntry {
+  sequence: number;
+  timestamp: number;
+  nodeId: string;
+  table: string;
+  operation: 'insert' | 'update' | 'delete';
+  data: string | null;
+  synced: 0 | 1;
+}
+
+export interface LogBatch {
+  version: 1;
+  nodeId: string;
+  fromSequence: number;
+  toSequence: number;
+  entries: Array<{
+    sequence: number;
+    timestamp: number;
+    table: string;
+    operation: 'insert' | 'update' | 'delete';
+    data: Record<string, unknown> | null;
+  }>;
+}
+
+export interface SyncJob {
+  id: string;
+  type: 'upload_log' | 'upload_snapshot';
+  payload: string;
+  status: 'pending' | 'processing' | 'done' | 'failed';
+  attempts: number;
+  nextRetryAt: number;
+  createdAt: number;
+  error: string | null;
+}
+
+export interface SyncManifest {
+  nodeId: string;
+  latestSnapshotKey: string | null;
+  latestSnapshotTimestamp: number | null;
+  latestLogSequence: number;
+  updatedAt: number;
+}
+
+export interface SyncMetrics {
+  lastSyncAt: number | null;
+  lastSnapshotAt: number | null;
+  pendingChanges: number;
+  dbSizeBytes: number;
+  totalSynced: number;
+  syncErrors: number;
+  isRunning: boolean;
+}
+
+export interface UploadLogPayload {
+  fromSequence: number;
+  toSequence: number;
+  s3Key: string;
+}
+
+export interface UploadSnapshotPayload {
+  timestamp: number;
+  s3Key: string;
+  dbPath: string;
+}

package/src/utils/compress.ts

@@ -0,0 +1,14 @@
+import { gzip, gunzip, constants } from 'zlib';
+import { promisify } from 'util';
+
+const gzipAsync = promisify(gzip);
+const gunzipAsync = promisify(gunzip);
+
+export async function compress(data: Buffer | string): Promise<Buffer> {
+  const buf = typeof data === 'string' ? Buffer.from(data, 'utf8') : data;
+  return gzipAsync(buf, { level: constants.Z_BEST_SPEED });
+}
+
+export async function decompress(data: Buffer): Promise<Buffer> {
+  return gunzipAsync(data);
+}

package/src/utils/crypto.ts

@@ -0,0 +1,33 @@
+import { createCipheriv, createDecipheriv, randomBytes } from 'crypto';
+
+const ALGORITHM = 'aes-256-gcm';
+const IV_LENGTH = 12;
+const TAG_LENGTH = 16;
+const KEY_LENGTH = 32;
+
+export function normalizeKey(hexKey: string): Buffer {
+  const buf = Buffer.from(hexKey, 'hex');
+  if (buf.length !== KEY_LENGTH) {
+    throw new Error(`Encryption key must be ${KEY_LENGTH * 2} hex characters (${KEY_LENGTH} bytes)`);
+  }
+  return buf;
+}
+
+export function encrypt(data: Buffer, hexKey: string): Buffer {
+  const key = normalizeKey(hexKey);
+  const iv = randomBytes(IV_LENGTH);
+  const cipher = createCipheriv(ALGORITHM, key, iv);
+  const encrypted = Buffer.concat([cipher.update(data), cipher.final()]);
+  const tag = cipher.getAuthTag();
+  return Buffer.concat([iv, tag, encrypted]);
+}
+
+export function decrypt(data: Buffer, hexKey: string): Buffer {
+  const key = normalizeKey(hexKey);
+  const iv = data.subarray(0, IV_LENGTH);
+  const tag = data.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
+  const encrypted = data.subarray(IV_LENGTH + TAG_LENGTH);
+  const decipher = createDecipheriv(ALGORITHM, key, iv);
+  decipher.setAuthTag(tag);
+  return Buffer.concat([decipher.update(encrypted), decipher.final()]);
+}

package/src/utils/id.ts

@@ -0,0 +1,10 @@
+import { v4 as uuidv4 } from 'uuid';
+import { randomBytes } from 'crypto';
+
+export function generateId(): string {
+  return uuidv4();
+}
+
+export function generateNodeId(): string {
+  return randomBytes(6).toString('hex');
+}

package/src/utils/retry.ts

@@ -0,0 +1,38 @@
+import type { RetryConfig } from '../types.js';
+
+const DEFAULT_RETRY_CONFIG: RetryConfig = {
+  maxRetries: 5,
+  baseDelayMs: 500,
+  maxDelayMs: 30_000,
+};
+
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  config: Partial<RetryConfig> = {}
+): Promise<T> {
+  const cfg = { ...DEFAULT_RETRY_CONFIG, ...config };
+  let lastError: Error | undefined;
+
+  for (let attempt = 0; attempt <= cfg.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err instanceof Error ? err : new Error(String(err));
+      if (attempt === cfg.maxRetries) break;
+      const delay = Math.min(cfg.baseDelayMs * Math.pow(2, attempt), cfg.maxDelayMs);
+      const jitter = Math.random() * delay * 0.2;
+      await sleep(delay + jitter);
+    }
+  }
+
+  throw lastError ?? new Error('Retry failed');
+}
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function nextRetryAt(attempt: number, config: RetryConfig = DEFAULT_RETRY_CONFIG): number {
+  const delay = Math.min(config.baseDelayMs * Math.pow(2, attempt), config.maxDelayMs);
+  return Date.now() + delay;
+}