@anfenn/dync 1.0.0

package/src/addVisibilityChangeListener.native.ts

@@ -0,0 +1,33 @@
+import type { VisibilitySubscription } from './types';
+
+export function addVisibilityChangeListener(
+    add: boolean,
+    currentSubscription: VisibilitySubscription | undefined,
+    onVisibilityChange: (isVisible: boolean) => void,
+): VisibilitySubscription | undefined {
+    if (add && !currentSubscription) {
+        let nativeSubscription: { remove: () => void } | undefined;
+
+        void (async () => {
+            try {
+                const ReactNative = await import('react-native');
+                const handler = (state: Parameters<typeof ReactNative.AppState.addEventListener>[1] extends (state: infer S) => any ? S : never) => {
+                    onVisibilityChange(state === 'active');
+                };
+                nativeSubscription = ReactNative.AppState.addEventListener('change', handler);
+            } catch {
+                // AppState unavailable
+            }
+        })();
+
+        return {
+            remove: () => {
+                nativeSubscription?.remove();
+            },
+        };
+    } else if (!add && currentSubscription) {
+        currentSubscription.remove();
+        return undefined;
+    }
+    return currentSubscription;
+}

package/src/addVisibilityChangeListener.ts

@@ -0,0 +1,24 @@
+import type { VisibilitySubscription } from './types';
+
+export function addVisibilityChangeListener(
+    add: boolean,
+    currentSubscription: VisibilitySubscription | undefined,
+    onVisibilityChange: (isVisible: boolean) => void,
+): VisibilitySubscription | undefined {
+    if (add && !currentSubscription) {
+        const handler = () => {
+            onVisibilityChange(document.visibilityState === 'visible');
+        };
+        document.addEventListener('visibilitychange', handler);
+
+        return {
+            remove: () => {
+                document.removeEventListener('visibilitychange', handler);
+            },
+        };
+    } else if (!add && currentSubscription) {
+        currentSubscription.remove();
+        return undefined;
+    }
+    return currentSubscription;
+}

package/src/capacitor.ts

@@ -0,0 +1,4 @@
+// Capacitor SQLite Drivers
+// Import this entry point only in Capacitor/native builds
+export { CapacitorSQLiteDriver } from './storage/sqlite/drivers/CapacitorSQLiteDriver';
+export { CapacitorFastSqlDriver, type FastSqlDriverOptions } from './storage/sqlite/drivers/CapacitorFastSqlDriver';

package/src/core/StateManager.ts

@@ -0,0 +1,272 @@
+import { deleteKeyIfEmptyObject, omitFields } from '../helpers';
+import { LOCAL_PK, UPDATED_AT, SyncAction, type PendingChange, type PersistedSyncState, type SyncState, type SyncStatus } from '../types';
+import type { StorageAdapter } from '../storage/types';
+
+const LOCAL_ONLY_SYNC_FIELDS = [LOCAL_PK, UPDATED_AT];
+
+// Internal table name used for storing sync state
+export const DYNC_STATE_TABLE = '_dync_state';
+const SYNC_STATE_KEY = 'sync_state';
+
+interface StateRow {
+    [LOCAL_PK]: string;
+    value: string;
+}
+
+const DEFAULT_STATE: PersistedSyncState = {
+    firstLoadDone: false,
+    pendingChanges: [],
+    lastPulled: {},
+};
+
+interface StateContext {
+    initialStatus?: SyncStatus;
+    storageAdapter?: StorageAdapter;
+}
+
+export interface StateHelpers {
+    hydrate(): Promise<void>;
+    getState(): PersistedSyncState;
+    setState(setterOrState: PersistedSyncState | ((state: PersistedSyncState) => Partial<PersistedSyncState>)): Promise<void>;
+    setErrorInMemory(error: Error): void;
+    addPendingChange(change: Omit<PendingChange, 'version'>): Promise<void>;
+    samePendingVersion(stateKey: string, localId: string, version: number): boolean;
+    removePendingChange(localId: string, stateKey: string): Promise<void>;
+    updatePendingChange(stateKey: string, localId: string, action: SyncAction, id?: any): Promise<void>;
+    setPendingChangeBefore(stateKey: string, localId: string, before: any): Promise<void>;
+    hasConflicts(localId: string): boolean;
+    getSyncStatus(): SyncStatus;
+    setSyncStatus(status: SyncStatus): void;
+    getSyncState(): SyncState;
+    subscribe(listener: (state: SyncState) => void): () => void;
+}
+
+export class StateManager implements StateHelpers {
+    private persistedState: PersistedSyncState;
+    private syncStatus: SyncStatus;
+    private readonly listeners = new Set<(state: SyncState) => void>();
+    private readonly storageAdapter?: StorageAdapter;
+    private hydrated = false;
+
+    constructor(ctx: StateContext) {
+        this.storageAdapter = ctx.storageAdapter;
+        // Start with default state, hydrate() will load from database
+        this.persistedState = DEFAULT_STATE;
+        this.syncStatus = ctx.initialStatus ?? 'disabled';
+    }
+
+    /**
+     * Load state from the database. Called after stores() defines the schema.
+     */
+    async hydrate(): Promise<void> {
+        if (this.hydrated) return;
+
+        if (!this.storageAdapter) {
+            throw new Error('Cannot hydrate state without a storage adapter');
+        }
+
+        const table = this.storageAdapter.table<StateRow>(DYNC_STATE_TABLE);
+        const row = await table.get(SYNC_STATE_KEY);
+        if (row?.value) {
+            this.persistedState = parseStoredState(row.value);
+        }
+
+        this.hydrated = true;
+
+        // Notify subscribers if state was loaded from database
+        this.emit();
+    }
+
+    private emit(): void {
+        this.listeners.forEach((fn) => fn(this.getSyncState()));
+    }
+
+    private async persist(): Promise<void> {
+        // Only persist after hydration (i.e., after database is opened)
+        if (!this.hydrated || !this.storageAdapter) return;
+
+        this.emit();
+
+        const table = this.storageAdapter.table<StateRow>(DYNC_STATE_TABLE);
+        await table.put({ [LOCAL_PK]: SYNC_STATE_KEY, value: JSON.stringify(this.persistedState) } as StateRow);
+    }
+
+    getState(): PersistedSyncState {
+        return clonePersistedState(this.persistedState);
+    }
+
+    setState(setterOrState: PersistedSyncState | ((state: PersistedSyncState) => Partial<PersistedSyncState>)): Promise<void> {
+        this.persistedState = resolveNextState(this.persistedState, setterOrState);
+        return this.persist();
+    }
+
+    /**
+     * Set error in memory only without persisting to database.
+     * Used when the database itself failed to open.
+     */
+    setErrorInMemory(error: Error): void {
+        this.persistedState = { ...this.persistedState, error };
+        this.emit();
+    }
+
+    addPendingChange(change: Omit<PendingChange, 'version'>): Promise<void> {
+        const next = clonePersistedState(this.persistedState);
+        const queueItem = next.pendingChanges.find((p) => p.localId === change.localId && p.stateKey === change.stateKey);
+
+        const omittedChanges = omitFields(change.changes, LOCAL_ONLY_SYNC_FIELDS);
+        const omittedBefore = omitFields(change.before, LOCAL_ONLY_SYNC_FIELDS);
+        const omittedAfter = omitFields(change.after, LOCAL_ONLY_SYNC_FIELDS);
+        const hasChanges = Object.keys(omittedChanges || {}).length > 0;
+        const action = change.action;
+
+        if (queueItem) {
+            if (queueItem.action === SyncAction.Remove) {
+                return Promise.resolve();
+            }
+
+            queueItem.version += 1;
+
+            if (action === SyncAction.Remove) {
+                queueItem.action = SyncAction.Remove;
+            } else if (hasChanges) {
+                queueItem.changes = { ...queueItem.changes, ...omittedChanges };
+                queueItem.after = { ...queueItem.after, ...omittedAfter };
+            }
+        } else if (action === SyncAction.Remove || hasChanges) {
+            next.pendingChanges = [...next.pendingChanges];
+            next.pendingChanges.push({
+                action,
+                stateKey: change.stateKey,
+                localId: change.localId,
+                id: change.id,
+                version: 1,
+                changes: omittedChanges,
+                before: omittedBefore,
+                after: omittedAfter,
+            });
+        }
+
+        this.persistedState = next;
+        return this.persist();
+    }
+
+    samePendingVersion(stateKey: string, localId: string, version: number): boolean {
+        return this.persistedState.pendingChanges.find((p) => p.localId === localId && p.stateKey === stateKey)?.version === version;
+    }
+
+    removePendingChange(localId: string, stateKey: string): Promise<void> {
+        const next = clonePersistedState(this.persistedState);
+        next.pendingChanges = next.pendingChanges.filter((p) => !(p.localId === localId && p.stateKey === stateKey));
+        this.persistedState = next;
+        return this.persist();
+    }
+
+    updatePendingChange(stateKey: string, localId: string, action: SyncAction, id?: any): Promise<void> {
+        const next = clonePersistedState(this.persistedState);
+        const changeItem = next.pendingChanges.find((p) => p.stateKey === stateKey && p.localId === localId);
+        if (changeItem) {
+            changeItem.action = action;
+            if (id) changeItem.id = id;
+            this.persistedState = next;
+            return this.persist();
+        }
+        return Promise.resolve();
+    }
+
+    setPendingChangeBefore(stateKey: string, localId: string, before: any): Promise<void> {
+        const next = clonePersistedState(this.persistedState);
+        const changeItem = next.pendingChanges.find((p) => p.stateKey === stateKey && p.localId === localId);
+        if (changeItem) {
+            changeItem.before = { ...(changeItem.before ?? {}), ...before };
+            this.persistedState = next;
+            return this.persist();
+        }
+        return Promise.resolve();
+    }
+
+    hasConflicts(localId: string): boolean {
+        return Boolean(this.persistedState.conflicts?.[localId]);
+    }
+
+    getSyncStatus(): SyncStatus {
+        return this.syncStatus;
+    }
+
+    setSyncStatus(status: SyncStatus): void {
+        if (this.syncStatus === status) return;
+        this.syncStatus = status;
+        this.emit();
+    }
+
+    getSyncState(): SyncState {
+        return buildSyncState(this.persistedState, this.syncStatus, this.hydrated);
+    }
+
+    subscribe(listener: (state: SyncState) => void): () => void {
+        this.listeners.add(listener);
+        return () => this.listeners.delete(listener);
+    }
+}
+
+function parseStoredState(stored: string): PersistedSyncState {
+    const parsed = JSON.parse(stored);
+    if (parsed.pendingChanges) {
+        parsed.pendingChanges = parsed.pendingChanges.map((change: any) => ({
+            ...change,
+            timestamp: change.timestamp ? new Date(change.timestamp) : change.timestamp,
+        }));
+    }
+    return parsed;
+}
+
+function resolveNextState(
+    current: PersistedSyncState,
+    setterOrState: PersistedSyncState | ((state: PersistedSyncState) => Partial<PersistedSyncState>),
+): PersistedSyncState {
+    if (typeof setterOrState === 'function') {
+        return { ...current, ...setterOrState(clonePersistedState(current)) };
+    }
+    return { ...current, ...setterOrState };
+}
+
+function buildSyncState(state: PersistedSyncState, status: SyncStatus, hydrated: boolean): SyncState {
+    const persisted = clonePersistedState(state);
+    const syncState: SyncState = {
+        ...persisted,
+        status,
+        hydrated,
+    };
+    deleteKeyIfEmptyObject(syncState, 'conflicts');
+    return syncState;
+}
+
+function clonePersistedState(state: PersistedSyncState): PersistedSyncState {
+    return {
+        ...state,
+        pendingChanges: state.pendingChanges.map((change) => ({
+            ...change,
+            changes: cloneRecord(change.changes),
+            before: cloneRecord(change.before),
+            after: cloneRecord(change.after),
+        })),
+        lastPulled: { ...state.lastPulled },
+        conflicts: cloneConflicts(state.conflicts),
+    };
+}
+
+function cloneConflicts(conflicts: SyncState['conflicts'] | undefined): SyncState['conflicts'] | undefined {
+    if (!conflicts) return undefined;
+    const next: NonNullable<SyncState['conflicts']> = {};
+    for (const [key, value] of Object.entries(conflicts)) {
+        next[key] = {
+            stateKey: value.stateKey,
+            fields: value.fields.map((field) => ({ ...field })),
+        };
+    }
+    return next;
+}
+
+function cloneRecord<T extends Record<string, any> | null | undefined>(record: T): T {
+    if (!record) return record;
+    return { ...record } as T;
+}

package/src/core/firstLoad.ts

@@ -0,0 +1,332 @@
+import { createLocalId, sleep } from '../helpers';
+import type { Logger } from '../logger';
+import type { ApiFunctions, BatchFirstLoadResult, BatchSync, FirstLoadProgressCallback } from '../types';
+import type { StorageTable } from '../storage/types';
+import { DYNC_STATE_TABLE, type StateHelpers } from './StateManager';
+import type { WithTransaction } from './types';
+
+export interface FirstLoadBaseContext {
+    logger: Logger;
+    state: StateHelpers;
+    table: <T>(name: string) => StorageTable<T>;
+    withTransaction: WithTransaction;
+    onProgress?: FirstLoadProgressCallback;
+}
+
+export interface FirstLoadContext extends FirstLoadBaseContext {
+    syncApis: Record<string, ApiFunctions>;
+}
+
+export interface FirstLoadBatchContext extends FirstLoadBaseContext {
+    batchSync: BatchSync;
+}
+
+interface RemoteRecord {
+    id?: unknown;
+    updated_at?: string;
+    deleted?: boolean;
+    _localId?: string;
+    [key: string]: unknown;
+}
+
+// Yield to event loop to prevent UI freezing on constrained devices
+const yieldToEventLoop = (): Promise<void> => sleep(0);
+
+const WRITE_BATCH_SIZE = 200;
+
+export async function startFirstLoad(ctx: FirstLoadContext): Promise<void> {
+    ctx.logger.debug('[dync] Starting first load...');
+
+    if (ctx.state.getState().firstLoadDone) {
+        ctx.logger.debug('[dync] First load already completed');
+        return;
+    }
+
+    let error: Error | undefined;
+
+    for (const [stateKey, api] of Object.entries(ctx.syncApis)) {
+        if (!api.firstLoad) {
+            ctx.logger.error(`[dync] firstLoad:no-api-function stateKey=${stateKey}`);
+            continue;
+        }
+
+        try {
+            ctx.logger.info(`[dync] firstLoad:start stateKey=${stateKey}`);
+
+            let lastId: unknown;
+            let isEmptyTable = true;
+            let batchCount = 0;
+            let totalInserted = 0;
+            let totalUpdated = 0;
+
+            while (true) {
+                const batch = await api.firstLoad(lastId);
+                if (!batch?.length) break;
+
+                batchCount++;
+
+                // Process batch in smaller chunks to manage memory and allow UI updates
+                const { inserted, updated } = await processBatchInChunks(ctx, stateKey, batch, isEmptyTable, lastId === undefined);
+                totalInserted += inserted;
+                totalUpdated += updated;
+
+                // Report progress if callback provided
+                if (ctx.onProgress) {
+                    ctx.onProgress({
+                        table: stateKey,
+                        inserted: totalInserted,
+                        updated: totalUpdated,
+                        total: totalInserted + totalUpdated,
+                    });
+                }
+
+                // After first batch, we know if table was empty
+                if (lastId === undefined) {
+                    isEmptyTable = (await ctx.table(stateKey).count()) === batch.length;
+                }
+
+                if (lastId !== undefined && lastId === batch[batch.length - 1].id) {
+                    throw new Error(`Duplicate records downloaded, stopping to prevent infinite loop`);
+                }
+
+                lastId = batch[batch.length - 1].id;
+
+                // Yield between API batches to allow UI updates
+                if (batchCount % 5 === 0) {
+                    await yieldToEventLoop();
+                }
+            }
+
+            ctx.logger.info(`[dync] firstLoad:done stateKey=${stateKey} inserted=${totalInserted} updated=${totalUpdated}`);
+        } catch (err) {
+            error = error ?? (err as Error);
+            ctx.logger.error(`[dync] firstLoad:error stateKey=${stateKey}`, err);
+        }
+    }
+
+    await ctx.state.setState((syncState) => ({
+        ...syncState,
+        firstLoadDone: true,
+        error,
+    }));
+
+    ctx.logger.debug('[dync] First load completed');
+}
+
+interface BatchResult {
+    inserted: number;
+    updated: number;
+}
+
+async function processBatchInChunks(
+    ctx: FirstLoadBaseContext,
+    stateKey: string,
+    batch: RemoteRecord[],
+    isEmptyTable: boolean,
+    isFirstBatch: boolean,
+): Promise<BatchResult> {
+    let newest = new Date(ctx.state.getState().lastPulled[stateKey] || 0);
+
+    return ctx.withTransaction('rw', [stateKey, DYNC_STATE_TABLE], async (tables) => {
+        const txTable = tables[stateKey]!;
+
+        // Check if table is empty on first batch
+        let tableIsEmpty = isEmptyTable;
+        if (isFirstBatch) {
+            const count = await txTable.count();
+            tableIsEmpty = count === 0;
+        }
+
+        // Pre-filter and prepare records, mutating in place to reduce allocations
+        const activeRecords: RemoteRecord[] = [];
+        for (const remote of batch) {
+            const remoteUpdated = new Date(remote.updated_at || 0);
+            if (remoteUpdated > newest) newest = remoteUpdated;
+
+            if (remote.deleted) continue;
+
+            // Mutate in place instead of spreading
+            delete remote.deleted;
+            remote._localId = createLocalId();
+            activeRecords.push(remote);
+        }
+
+        let inserted = 0;
+        let updated = 0;
+
+        if (tableIsEmpty) {
+            // Fast path: no existing records, bulk add in chunks
+            for (let i = 0; i < activeRecords.length; i += WRITE_BATCH_SIZE) {
+                const chunk = activeRecords.slice(i, i + WRITE_BATCH_SIZE);
+                await txTable.raw.bulkAdd(chunk as any);
+                inserted += chunk.length;
+            }
+        } else {
+            // Slower path: need to check for existing records
+            // Process in chunks to limit memory usage of lookup map
+            for (let i = 0; i < activeRecords.length; i += WRITE_BATCH_SIZE) {
+                const chunk = activeRecords.slice(i, i + WRITE_BATCH_SIZE);
+                const chunkResult = await processChunkWithLookup(txTable, chunk);
+                inserted += chunkResult.inserted;
+                updated += chunkResult.updated;
+            }
+        }
+
+        await ctx.state.setState((syncState) => ({
+            ...syncState,
+            lastPulled: {
+                ...syncState.lastPulled,
+                [stateKey]: newest.toISOString(),
+            },
+        }));
+
+        return { inserted, updated };
+    });
+}
+
+async function processChunkWithLookup(txTable: StorageTable<any>, chunk: RemoteRecord[]): Promise<BatchResult> {
+    // Collect server IDs for this chunk
+    const serverIds = chunk.filter((r) => r.id != null).map((r) => r.id);
+
+    // Bulk lookup existing records
+    const existingByServerId = new Map<unknown, { _localId: string }>();
+    if (serverIds.length > 0) {
+        const existingRecords = await txTable.where('id').anyOf(serverIds).toArray();
+        for (const existing of existingRecords) {
+            existingByServerId.set((existing as any).id, existing as { _localId: string });
+        }
+    }
+
+    // Separate into adds and updates
+    const toAdd: RemoteRecord[] = [];
+    let updated = 0;
+
+    for (const remote of chunk) {
+        const existing = remote.id != null ? existingByServerId.get(remote.id) : undefined;
+        if (existing) {
+            // Update: merge and update in place
+            const merged = Object.assign({}, existing, remote, { _localId: existing._localId });
+            await txTable.raw.update(existing._localId, merged as any);
+            updated++;
+        } else {
+            toAdd.push(remote);
+        }
+    }
+
+    // Bulk add new records
+    if (toAdd.length > 0) {
+        await txTable.raw.bulkAdd(toAdd as any);
+    }
+
+    // Clear the map to help GC
+    existingByServerId.clear();
+
+    return { inserted: toAdd.length, updated };
+}
+
+// ============================================================================
+// Batch Mode First Load Operations
+// ============================================================================
+
+export async function startFirstLoadBatch(ctx: FirstLoadBatchContext): Promise<void> {
+    ctx.logger.debug('[dync] Starting batch first load...');
+
+    if (ctx.state.getState().firstLoadDone) {
+        ctx.logger.debug('[dync] First load already completed');
+        return;
+    }
+
+    if (!ctx.batchSync.firstLoad) {
+        ctx.logger.warn('[dync] firstLoad:batch:no-firstLoad-function');
+        await ctx.state.setState((syncState) => ({
+            ...syncState,
+            firstLoadDone: true,
+        }));
+        return;
+    }
+
+    let error: Error | undefined;
+
+    try {
+        ctx.logger.info(`[dync] firstLoad:batch:start tables=${[...ctx.batchSync.syncTables].join(',')}`);
+
+        // Track progress per table
+        const progress: Record<string, { inserted: number; updated: number }> = {};
+        for (const tableName of ctx.batchSync.syncTables) {
+            progress[tableName] = { inserted: 0, updated: 0 };
+        }
+
+        // Initialize cursors for all tables
+        let cursors: Record<string, any> = {};
+        for (const tableName of ctx.batchSync.syncTables) {
+            cursors[tableName] = undefined;
+        }
+
+        let batchCount = 0;
+
+        while (true) {
+            const result: BatchFirstLoadResult = await ctx.batchSync.firstLoad(cursors);
+
+            if (!result.hasMore && Object.values(result.data).every((d) => !d?.length)) {
+                break;
+            }
+
+            batchCount++;
+
+            // Process each table's data
+            for (const [stateKey, batch] of Object.entries(result.data)) {
+                if (!ctx.batchSync.syncTables.includes(stateKey)) {
+                    ctx.logger.warn(`[dync] firstLoad:batch:unknown-table stateKey=${stateKey}`);
+                    continue;
+                }
+
+                if (!batch?.length) continue;
+
+                const isFirstBatch = progress[stateKey]!.inserted === 0 && progress[stateKey]!.updated === 0;
+                const isEmptyTable = isFirstBatch && (await ctx.table(stateKey).count()) === 0;
+
+                const { inserted, updated } = await processBatchInChunks(ctx, stateKey, batch, isEmptyTable, isFirstBatch);
+                progress[stateKey]!.inserted += inserted;
+                progress[stateKey]!.updated += updated;
+
+                // Report progress if callback provided
+                if (ctx.onProgress) {
+                    ctx.onProgress({
+                        table: stateKey,
+                        inserted: progress[stateKey]!.inserted,
+                        updated: progress[stateKey]!.updated,
+                        total: progress[stateKey]!.inserted + progress[stateKey]!.updated,
+                    });
+                }
+            }
+
+            // Update cursors for next batch
+            cursors = result.cursors;
+
+            // Yield between API batches to allow UI updates
+            if (batchCount % 5 === 0) {
+                await yieldToEventLoop();
+            }
+
+            if (!result.hasMore) {
+                break;
+            }
+        }
+
+        // Log completion for each table
+        for (const [stateKey, p] of Object.entries(progress)) {
+            ctx.logger.info(`[dync] firstLoad:batch:done stateKey=${stateKey} inserted=${p.inserted} updated=${p.updated}`);
+        }
+    } catch (err) {
+        error = err as Error;
+        ctx.logger.error('[dync] firstLoad:batch:error', err);
+    }
+
+    await ctx.state.setState((syncState) => ({
+        ...syncState,
+        firstLoadDone: true,
+        error,
+    }));
+
+    ctx.logger.debug('[dync] Batch first load completed');
+}