@nsky/sync 0.1.0

package/README.md

@@ -0,0 +1,280 @@
+# @nsky/sync
+
+Headless local-first sync primitives for applications that keep UI reads and writes local, then synchronize modeled changes in the background.
+
+The package follows the `SQLite + Operation Log + HLC + Incremental Sync` design in this workspace. It does not sync database files. It syncs application-level `Change` records through storage and transport adapters.
+
+## What It Provides
+
+- `SyncEngine`: runs push, pull, cursor persistence, and remote apply cycles.
+- `TypedSyncEventBus`: reports sync lifecycle events without binding to any UI framework.
+- `LWWResolver`: resolves conflicts with HLC-based last-write-wins semantics and tombstone priority.
+- `HLCClock`: creates monotonic hybrid logical timestamps for change versions.
+- `MemoryDatabaseAdapter`: in-memory database adapter for tests and local prototypes.
+- `DexieDatabaseAdapter`: IndexedDB-backed adapter for browser and extension contexts.
+- `FetchSyncTransport`: HTTP transport for `/push` and `/pull` endpoints.
+- `WebSocketSyncTransport`: request/response transport with real-time change notifications.
+- `DefaultSyncScheduler`: interval, foreground, network-recovery, retry, and tombstone cleanup scheduling.
+- `ExponentialBackoffPolicy`: reusable retry policy for transports and scheduling.
+
+## Import Paths
+
+```ts
+import { SyncEngine } from "@nsky/sync/core";
+import { FetchSyncTransport, WebSocketSyncTransport } from "@nsky/sync/http";
+import { DexieDatabaseAdapter, MemoryDatabaseAdapter } from "@nsky/sync/sql";
+import { HLCClock, createChangeId, createDeviceId } from "@nsky/sync/utils";
+```
+
+The root export also re-exports all modules:
+
+```ts
+import { SyncEngine, FetchSyncTransport, HLCClock } from "@nsky/sync";
+```
+
+## Quick Start
+
+```ts
+import { SyncEngine } from "@nsky/sync/core";
+import { FetchSyncTransport } from "@nsky/sync/http";
+import { MemoryDatabaseAdapter } from "@nsky/sync/sql";
+import { HLCClock, createChangeId, createDeviceId } from "@nsky/sync/utils";
+
+const deviceId = createDeviceId();
+const clock = new HLCClock(deviceId);
+
+const database = new MemoryDatabaseAdapter({
+  pendingChanges: [
+    {
+      id: createChangeId(),
+      table: "bookmarks",
+      entityId: "bookmark-1",
+      action: "update",
+      payload: { title: "Local-first sync" },
+      version: clock.now(),
+      deviceId,
+      createdAt: Date.now(),
+    },
+  ],
+});
+
+const transport = new FetchSyncTransport({
+  baseUrl: "https://api.example.com/sync",
+  headers: {
+    Authorization: "Bearer <token>",
+  },
+});
+
+const engine = new SyncEngine({
+  database,
+  transport,
+  deviceId,
+  batchSize: 100,
+});
+
+engine.events.on("statusChange", (status) => {
+  console.log("sync status:", status);
+});
+
+engine.events.on("error", (error) => {
+  console.error("sync failed:", error.code, error.message);
+});
+
+const result = await engine.sync();
+console.log(result);
+```
+
+## Browser IndexedDB Adapter
+
+Use `DexieDatabaseAdapter` when the app runs in a browser, extension, or other IndexedDB-capable runtime:
+
+```ts
+import { DexieDatabaseAdapter } from "@nsky/sync/sql";
+
+const database = new DexieDatabaseAdapter({
+  databaseName: "my-app-sync",
+});
+
+await database.recordLocalChange({
+  id: createChangeId(),
+  table: "bookmarks",
+  entityId: "bookmark-1",
+  action: "update",
+  payload: { title: "Updated locally" },
+  version: clock.now(),
+  deviceId,
+  createdAt: Date.now(),
+});
+```
+
+`recordLocalChange()` is the hook your Repository layer should call in the same user action that updates local business data. The adapter stores pending changes, remote row snapshots, cursors, and tombstones; it intentionally does not know your app-specific tables.
+
+## WebSocket Transport
+
+`WebSocketSyncTransport` implements the same `SyncTransport` contract as HTTP and can also wake the engine when the server announces new changes:
+
+```ts
+import { WebSocketSyncTransport } from "@nsky/sync/http";
+
+const transport = new WebSocketSyncTransport({
+  url: "wss://api.example.com/sync",
+});
+```
+
+Client-to-server messages use generated `id` values:
+
+- `{ id, type: "push", payload }`
+- `{ id, type: "pull", cursor, limit }`
+
+Server responses should echo the matching `id`:
+
+- `{ id, type: "pushResult" }`
+- `{ id, type: "pullResult", result }`
+- `{ type: "changes", cursor }` for real-time notifications.
+
+## Scheduler
+
+Wrap an engine with `DefaultSyncScheduler` when you want background sync behavior:
+
+```ts
+import { DefaultSyncScheduler, SyncEngine } from "@nsky/sync/core";
+import { ExponentialBackoffPolicy } from "@nsky/sync/utils";
+
+const engine = new SyncEngine({ database, transport, deviceId });
+const scheduler = new DefaultSyncScheduler({
+  engine,
+  database,
+  retryPolicy: new ExponentialBackoffPolicy({
+    maxAttempts: 5,
+    baseDelayMs: 1_000,
+    maxDelayMs: 60_000,
+  }),
+});
+
+scheduler.start({
+  intervalMs: 30_000,
+  syncOnNetworkRecover: true,
+  syncOnForeground: true,
+  tombstoneCleanupIntervalMs: 7 * 24 * 60 * 60 * 1_000,
+  tombstoneMaxAgeMs: 30 * 24 * 60 * 60 * 1_000,
+});
+
+await scheduler.triggerSync();
+scheduler.stop();
+```
+
+The scheduler prevents overlapping `sync()` calls. Failed sync results are passed through the configured `RetryPolicy`; successful sync resets the retry state.
+
+## Sync Flow
+
+`SyncEngine.sync()` performs one complete cycle:
+
+1. Read pending local changes from `DatabaseAdapter.getPendingChanges()`.
+2. Push them with `SyncTransport.push()`.
+3. Mark pushed changes as synced.
+4. Load the local cursor.
+5. Pull remote pages with `SyncTransport.pull(cursor, limit)`.
+6. Apply each page and save its cursor inside `runInTransaction()`.
+7. Emit lifecycle events and return a `SyncResult`.
+
+Concurrent `sync()` calls share the same in-flight promise, so one engine instance does not run overlapping sync cycles.
+
+## Change Model
+
+```ts
+type ChangeAction = "insert" | "update" | "delete";
+
+interface Change<TPayload = unknown> {
+  readonly id: string;
+  readonly table: string;
+  readonly entityId: string;
+  readonly action: ChangeAction;
+  readonly payload: TPayload;
+  readonly version: string;
+  readonly deviceId: string;
+  readonly createdAt: number;
+}
+```
+
+Use HLC timestamps for `version` so devices can compare writes consistently even when wall clocks drift.
+
+## Adapter Contracts
+
+Production apps should provide a durable `DatabaseAdapter`, usually backed by SQLite or IndexedDB. The adapter must make `runInTransaction()` atomic.
+
+```ts
+interface DatabaseAdapter {
+  runInTransaction<T>(workload: () => Promise<T>): Promise<T>;
+  getLocalCursor(): Promise<SyncCursor | null>;
+  saveLocalCursor(cursor: SyncCursor): Promise<void>;
+  getPendingChanges(limit?: number): Promise<Change[]>;
+  markAsSynced(changeIds: string[]): Promise<void>;
+  applyRemoteChanges(changes: Change[]): Promise<void>;
+  clearTombstones(beforeTimestamp: number): Promise<number>;
+}
+```
+
+Transport implementations push local batches and pull ordered remote pages:
+
+```ts
+interface SyncTransport {
+  readonly type: "http" | "websocket" | "webrtc" | "grpc";
+  push(payload: PushPayload): Promise<void>;
+  pull(cursor: string | null, limit?: number): Promise<PullResult>;
+  subscribe?: (onChanges: (cursor: string) => void) => () => void;
+}
+```
+
+`FetchSyncTransport` and `WebSocketSyncTransport` both accept a `retryPolicy`. By default, `ExponentialBackoffPolicy` retries recoverable network, transport, server, and unknown errors; it does not retry non-recoverable errors such as auth failures.
+
+## HTTP Endpoint Shape
+
+`FetchSyncTransport` expects these endpoints relative to `baseUrl`:
+
+- `POST /push` with body `{ changes, deviceId }`; any 2xx response is accepted.
+- `GET /pull?cursor=<cursor>&limit=<limit>` returning:
+
+```ts
+{
+  cursor: { value: "opaque-server-cursor", lastSyncedAt: 1710000000000 },
+  changes: [],
+  hasMore: false,
+}
+```
+
+HTTP failures are converted to sync errors:
+
+- `401` or `403` -> `AUTH_FAILED`
+- `410` -> `CURSOR_EXPIRED`
+- `5xx` -> `SERVER_ERROR`
+- other non-2xx statuses -> `TRANSPORT_ERROR`
+
+## Conflict Behavior
+
+The default resolver is `LWWResolver`:
+
+- delete changes win over non-delete changes for the same entity.
+- otherwise, the higher HLC `version` wins.
+- equal versions pick the remote change for deterministic convergence.
+
+Pass a custom `ConflictResolver` to `SyncEngine` when a domain needs field-level merge, CRDT behavior, or manual conflict queues.
+
+## Testing Helpers
+
+`MemoryDatabaseAdapter` is intended for tests and prototypes:
+
+```ts
+import { MemoryDatabaseAdapter } from "@nsky/sync/sql";
+
+const database = new MemoryDatabaseAdapter({
+  pendingChanges: [],
+  rows: [],
+  cursor: null,
+});
+```
+
+It implements the same adapter contract as a durable database adapter, including rollback behavior for failed `runInTransaction()` workloads.
+
+## Current Scope
+
+This package provides the headless engine, core contracts, in-memory and IndexedDB storage adapters, HTTP and WebSocket transports, scheduler, HLC utilities, and retry policy primitives. Platform-specific durable adapters such as SQLite are expected to implement the exported `DatabaseAdapter` interface.

package/dist/core/index.cjs

@@ -0,0 +1,277 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_utils_index = require("../utils/index.cjs");
+//#region src/core/conflict.ts
+var LWWResolver = class {
+	resolve(context) {
+		const { local, remote } = context;
+		if (local.action === "delete" && remote.action !== "delete") return {
+			winner: local,
+			strategy: "Tombstone wins over non-delete change"
+		};
+		if (remote.action === "delete" && local.action !== "delete") return {
+			winner: remote,
+			strategy: "Tombstone wins over non-delete change"
+		};
+		const order = require_utils_index.compareHLC(remote.version, local.version);
+		if (order > 0) return {
+			winner: remote,
+			strategy: "LWW: remote version newer"
+		};
+		if (order < 0) return {
+			winner: local,
+			strategy: "LWW: local version newer"
+		};
+		return {
+			winner: remote,
+			strategy: "LWW: equal version, remote wins"
+		};
+	}
+};
+//#endregion
+//#region src/core/errors.ts
+function createSyncError(code, message, cause) {
+	return {
+		code,
+		message,
+		cause,
+		occurredAt: Date.now()
+	};
+}
+function isSyncError(error) {
+	return typeof error === "object" && error !== null && "code" in error && "message" in error && "occurredAt" in error;
+}
+function normalizeSyncError(error) {
+	if (isSyncError(error)) return error;
+	if (error instanceof Error) return createSyncError("UNKNOWN", error.message, error);
+	return createSyncError("UNKNOWN", "Unknown sync error.", error);
+}
+//#endregion
+//#region src/core/event-bus.ts
+var TypedSyncEventBus = class {
+	#listeners = /* @__PURE__ */ new Map();
+	on(event, listener) {
+		const listeners = this.#listeners.get(event) ?? /* @__PURE__ */ new Set();
+		listeners.add(listener);
+		this.#listeners.set(event, listeners);
+		return () => this.off(event, listener);
+	}
+	once(event, listener) {
+		const wrapped = ((payload) => {
+			this.off(event, wrapped);
+			if (payload === void 0) listener();
+			else listener(payload);
+		});
+		this.on(event, wrapped);
+	}
+	off(event, listener) {
+		this.#listeners.get(event)?.delete(listener);
+	}
+	emit(event, ...args) {
+		const listeners = [...this.#listeners.get(event) ?? []];
+		for (const listener of listeners) if (args.length === 0) listener();
+		else listener(args[0]);
+	}
+};
+//#endregion
+//#region src/core/engine.ts
+var SyncEngine = class {
+	#database;
+	#transport;
+	#deviceId;
+	#batchSize;
+	#unsubscribe;
+	#status = "idle";
+	#inFlight = null;
+	resolver;
+	events = new TypedSyncEventBus();
+	constructor(options) {
+		this.#database = options.database;
+		this.#transport = options.transport;
+		this.#deviceId = options.deviceId;
+		this.#batchSize = options.batchSize ?? 100;
+		this.resolver = options.resolver ?? new LWWResolver();
+		this.#unsubscribe = this.#transport.subscribe?.(() => {
+			this.sync();
+		});
+	}
+	get status() {
+		return this.#status;
+	}
+	sync() {
+		if (this.#inFlight !== null) return this.#inFlight;
+		this.#inFlight = this.#syncOnce().finally(() => {
+			this.#inFlight = null;
+		});
+		return this.#inFlight;
+	}
+	async dispose() {
+		this.#unsubscribe?.();
+	}
+	async #syncOnce() {
+		const startedAt = Date.now();
+		let uploadedCount = 0;
+		let downloadedCount = 0;
+		this.events.emit("syncStart");
+		try {
+			this.#setStatus("uploading");
+			const pendingChanges = await this.#database.getPendingChanges(this.#batchSize);
+			if (pendingChanges.length > 0) {
+				await this.#transport.push({
+					changes: pendingChanges,
+					deviceId: this.#deviceId
+				});
+				await this.#database.markAsSynced(pendingChanges.map((change) => change.id));
+				uploadedCount = pendingChanges.length;
+			}
+			this.events.emit("uploadComplete", { uploadedCount });
+			let cursor = await this.#database.getLocalCursor();
+			let hasMore = true;
+			while (hasMore) {
+				this.#setStatus("downloading");
+				const pulled = await this.#transport.pull(cursor?.value ?? null, this.#batchSize);
+				this.events.emit("downloadComplete", { downloadedCount: pulled.changes.length });
+				this.#setStatus("applying");
+				await this.#database.runInTransaction(async () => {
+					await this.#database.applyRemoteChanges([...pulled.changes]);
+					await this.#database.saveLocalCursor(pulled.cursor);
+				});
+				downloadedCount += pulled.changes.length;
+				cursor = pulled.cursor;
+				hasMore = pulled.hasMore;
+			}
+			this.#setStatus("idle");
+			const result = {
+				success: true,
+				uploadedCount,
+				downloadedCount,
+				durationMs: Date.now() - startedAt
+			};
+			this.events.emit("syncFinish", result);
+			return result;
+		} catch (cause) {
+			const error = normalizeSyncError(cause);
+			const result = {
+				success: false,
+				uploadedCount,
+				downloadedCount,
+				durationMs: Date.now() - startedAt,
+				error
+			};
+			this.events.emit("error", error);
+			this.#setStatus("idle");
+			this.events.emit("syncFinish", result);
+			return result;
+		}
+	}
+	#setStatus(status) {
+		this.#status = status;
+		this.events.emit("statusChange", status);
+	}
+};
+//#endregion
+//#region src/core/scheduler.ts
+const DEFAULT_INTERVAL_MS = 3e4;
+const DEFAULT_TOMBSTONE_CLEANUP_INTERVAL_MS = 10080 * 60 * 1e3;
+const DEFAULT_TOMBSTONE_MAX_AGE_MS = 720 * 60 * 60 * 1e3;
+var DefaultSyncScheduler = class {
+	#engine;
+	#database;
+	#defaultRetryPolicy;
+	#window;
+	#document;
+	#syncTimer = null;
+	#retryTimer = null;
+	#tombstoneTimer = null;
+	#inFlight = null;
+	#retryAttempt = 0;
+	#firstFailedAt = 0;
+	#retryPolicy;
+	#handleOnline = () => {
+		this.triggerSync();
+	};
+	#handleVisibilityChange = () => {
+		if (this.#document?.visibilityState === "visible") this.triggerSync();
+	};
+	constructor(options) {
+		this.#engine = options.engine;
+		this.#database = options.database;
+		this.#defaultRetryPolicy = options.retryPolicy ?? new require_utils_index.ExponentialBackoffPolicy();
+		this.#retryPolicy = this.#defaultRetryPolicy;
+		this.#window = options.window ?? globalThis.window;
+		this.#document = options.document ?? globalThis.document;
+	}
+	start(options = {}) {
+		this.stop();
+		this.#retryPolicy = options.retryPolicy ?? this.#defaultRetryPolicy;
+		const intervalMs = options.intervalMs ?? DEFAULT_INTERVAL_MS;
+		if (intervalMs > 0) this.#syncTimer = setInterval(() => {
+			this.triggerSync();
+		}, intervalMs);
+		if (options.syncOnNetworkRecover ?? true) this.#window?.addEventListener("online", this.#handleOnline);
+		if (options.syncOnForeground ?? true) this.#document?.addEventListener("visibilitychange", this.#handleVisibilityChange);
+		const tombstoneCleanupIntervalMs = options.tombstoneCleanupIntervalMs ?? DEFAULT_TOMBSTONE_CLEANUP_INTERVAL_MS;
+		if (this.#database !== void 0 && tombstoneCleanupIntervalMs > 0) {
+			const tombstoneMaxAgeMs = options.tombstoneMaxAgeMs ?? DEFAULT_TOMBSTONE_MAX_AGE_MS;
+			this.#tombstoneTimer = setInterval(() => {
+				this.#database?.clearTombstones(Date.now() - tombstoneMaxAgeMs);
+			}, tombstoneCleanupIntervalMs);
+		}
+	}
+	stop() {
+		if (this.#syncTimer !== null) {
+			clearInterval(this.#syncTimer);
+			this.#syncTimer = null;
+		}
+		if (this.#retryTimer !== null) {
+			clearTimeout(this.#retryTimer);
+			this.#retryTimer = null;
+		}
+		if (this.#tombstoneTimer !== null) {
+			clearInterval(this.#tombstoneTimer);
+			this.#tombstoneTimer = null;
+		}
+		this.#window?.removeEventListener("online", this.#handleOnline);
+		this.#document?.removeEventListener("visibilitychange", this.#handleVisibilityChange);
+	}
+	async triggerSync() {
+		if (this.#inFlight !== null) return await this.#inFlight;
+		this.#inFlight = this.#engine.sync().then((result) => {
+			this.#handleResult(result);
+			return result;
+		});
+		try {
+			return await this.#inFlight;
+		} finally {
+			this.#inFlight = null;
+		}
+	}
+	#handleResult(result) {
+		if (result.success) {
+			this.#retryAttempt = 0;
+			this.#firstFailedAt = 0;
+			return;
+		}
+		if (result.error === void 0) return;
+		this.#firstFailedAt = this.#firstFailedAt === 0 ? result.error.occurredAt : this.#firstFailedAt;
+		const decision = this.#retryPolicy.decide({
+			attempt: this.#retryAttempt,
+			firstFailedAt: this.#firstFailedAt,
+			lastError: result.error
+		});
+		this.#retryAttempt += 1;
+		if (decision.action === "abort") return;
+		if (this.#retryTimer !== null) clearTimeout(this.#retryTimer);
+		this.#retryTimer = setTimeout(() => {
+			this.#retryTimer = null;
+			this.triggerSync();
+		}, decision.delayMs);
+	}
+};
+//#endregion
+exports.DefaultSyncScheduler = DefaultSyncScheduler;
+exports.LWWResolver = LWWResolver;
+exports.SyncEngine = SyncEngine;
+exports.TypedSyncEventBus = TypedSyncEventBus;
+exports.createSyncError = createSyncError;
+exports.isSyncError = isSyncError;
+exports.normalizeSyncError = normalizeSyncError;

package/dist/core/index.d.cts

@@ -0,0 +1,2 @@
+import { A as SyncStatus, C as DeviceId, D as SyncError, E as SyncCursor, O as SyncErrorCode, S as DatabaseAdapter, T as PushPayload, _ as ConflictResolution, a as SyncSchedulerDatabase, b as Change, c as isSyncError, d as SyncEngineOptions, f as SyncEventBus, g as ConflictContext, h as TypedSyncEventBus, i as SyncScheduler, j as SyncTransport, k as SyncResult, l as normalizeSyncError, m as SyncEventMap, n as DefaultSyncSchedulerOptions, o as SyncSchedulerEngine, p as SyncEventListener, r as SchedulerOptions, s as createSyncError, t as DefaultSyncScheduler, u as SyncEngine, v as ConflictResolver, w as PullResult, x as ChangeAction, y as LWWResolver } from "../index2.cjs";
+export { Change, ChangeAction, ConflictContext, ConflictResolution, ConflictResolver, DatabaseAdapter, DefaultSyncScheduler, DefaultSyncSchedulerOptions, DeviceId, LWWResolver, PullResult, PushPayload, SchedulerOptions, SyncCursor, SyncEngine, SyncEngineOptions, SyncError, SyncErrorCode, SyncEventBus, SyncEventListener, SyncEventMap, SyncResult, SyncScheduler, SyncSchedulerDatabase, SyncSchedulerEngine, SyncStatus, SyncTransport, TypedSyncEventBus, createSyncError, isSyncError, normalizeSyncError };

package/dist/core/index.d.mts

@@ -0,0 +1,2 @@
+import { A as SyncStatus, C as DeviceId, D as SyncError, E as SyncCursor, O as SyncErrorCode, S as DatabaseAdapter, T as PushPayload, _ as ConflictResolution, a as SyncSchedulerDatabase, b as Change, c as isSyncError, d as SyncEngineOptions, f as SyncEventBus, g as ConflictContext, h as TypedSyncEventBus, i as SyncScheduler, j as SyncTransport, k as SyncResult, l as normalizeSyncError, m as SyncEventMap, n as DefaultSyncSchedulerOptions, o as SyncSchedulerEngine, p as SyncEventListener, r as SchedulerOptions, s as createSyncError, t as DefaultSyncScheduler, u as SyncEngine, v as ConflictResolver, w as PullResult, x as ChangeAction, y as LWWResolver } from "../index2.mjs";
+export { Change, ChangeAction, ConflictContext, ConflictResolution, ConflictResolver, DatabaseAdapter, DefaultSyncScheduler, DefaultSyncSchedulerOptions, DeviceId, LWWResolver, PullResult, PushPayload, SchedulerOptions, SyncCursor, SyncEngine, SyncEngineOptions, SyncError, SyncErrorCode, SyncEventBus, SyncEventListener, SyncEventMap, SyncResult, SyncScheduler, SyncSchedulerDatabase, SyncSchedulerEngine, SyncStatus, SyncTransport, TypedSyncEventBus, createSyncError, isSyncError, normalizeSyncError };