@anfenn/zync 0.1.22 → 0.2.0

package/README.md

@@ -2,16 +2,19 @@
 
 [![npm version](https://img.shields.io/npm/v/@anfenn/zync.svg)](https://www.npmjs.com/package/@anfenn/zync)
 
-Unopinionated, bullet-proof, offline-first sync middleware for Zustand.
+Simple, bullet-proof, offline-first sync middleware for Zustand.
 
 ## Benefits
 
-- Simple to sync any state with a backend
+- Easy to sync non-nested array state with a backend (i.e. mirror remote database tables locally)
+- **"It just works"** philosophy
+- Batteries optionally included:
+    - IndexedDB helper (based on [idb](https://www.npmjs.com/package/idb))
 - Uses the official persist middleware as the local storage (localStorage, IndexedDB, etc.)
 - Zync's persistWithSync() is a drop-in replacement for Zustand's persist()
 - Allows for idiomatic use of Zustand
 - Leaves the api requests up to you (RESTful, GraphQL, etc.), just provide add(), update(), remove() and list()
-- **_Coming soon_**: Customisable conflict resolution. Currently last-write-wins.
+- **_Coming soon_**: Customisable conflict resolution. Currently local-wins.
 
 ## Requirements
 
@@ -30,7 +33,7 @@ npm install @anfenn/zync
 ### Zustand store creation (store.ts):
 
 ```ts
-import { SyncAction, type UseStoreWithSync, persistWithSync } from '@anfenn/zync';
+import { type UseStoreWithSync, persistWithSync } from '@anfenn/zync';
 import { create } from 'zustand';
 import { createJSONStorage } from 'zustand/middleware';
 import { useShallow } from 'zustand/react/shallow';
@@ -45,32 +48,25 @@ type Store = {
 
 export const useStore = create<any>()(
     persistWithSync<Store>(
-        (set, get, queueToSync) => ({
-            // Standard Zustand state and mutation functions with new queueToSync()
+        (set, get, setAndSync) => ({
+            // Standard Zustand state and mutation functions with new setAndSync()
 
             facts: [],
             addFact: (item: Fact) => {
                 const updated_at = new Date().toISOString();
                 const newItem = { ...item, created_at: updated_at, updated_at };
 
-                set((state: Store) => ({
+                setAndSync((state: Store) => ({
                     facts: [...state.facts, newItem],
                 }));
-
-                // Never call queueToSync() inside Zustand set() due to itself calling set(), so may cause lost state changes
-                queueToSync(SyncAction.CreateOrUpdate, 'facts', item._localId);
             },
             updateFact: (localId: string, changes: Partial<Fact>) => {
-                set((state: Store) => ({
+                setAndSync((state: Store) => ({
                     facts: state.facts.map((item) => (item._localId === localId ? { ...item, ...changes } : item)),
                 }));
-
-                queueToSync(SyncAction.CreateOrUpdate, 'facts', localId);
             },
             removeFact: (localId: string) => {
-                queueToSync(SyncAction.Remove, 'facts', localId);
-
-                set((state: Store) => ({
+                setAndSync((state: Store) => ({
                     facts: state.facts.filter((item) => item._localId !== localId),
                 }));
             },
@@ -107,6 +103,9 @@ export const useFacts = () =>
     );
 ```
 
+**_NOTE_**: Zync uses an internal timer (setInterval) to sync, so it's advised to just have one store. You could have multiple, with different store names (see Zustand persist options above), but if both stores use Zync, although it would work fine, it wouldn't offer much advantage. If one store becomes large with many state keys and functions, then you could separate them into multiple files and import than with object spreading
+`e.g. {...storeState1, ...storeState2}`
+
 ### In your component:
 
 ```ts
@@ -242,23 +241,19 @@ async function firstLoad(lastId: any) {
 
 ## Optional IndexedDB storage
 
-When using IndexedDB Zustand saves the whole store under one key, which means indexes cannot be used to accelerate querying. However, if this becomes a performance issue due to the size of the store, then libraries like dexie.js instead of Zustand would be a better solution and provide the syntax for high performance queries.
+Using async IndexedDB over sync localStorage gives the advantage of a responsive UI when reading/writing a very large store, as IndexedDB is running in it's own thread.
 
 If you want to use the bundled `createIndexedDBStorage()` helper, install `idb` in your project. It's intentionally optional so projects that don't use IndexedDB won't pull the dependency into their bundles.
 
-Install for runtime usage:
+[idb](https://www.npmjs.com/package/idb) is an extremely popular and lightweight wrapper to simplify IndexedDB's verbose events based api into a simple Promise based one. It also handles the inconsistencies found when running in different browsers.
 
 ```bash
 npm install idb
 ```
 
-Or add it as an optional dependency of your package:
-
-```bash
-npm install --save-optional idb
-```
+When using IndexedDB Zustand saves the whole store under one key, which means indexes cannot be used to accelerate querying. However, if this becomes a performance issue due to the size of the store, then libraries like dexie.js instead of Zustand would be a better solution and provide the syntax for high performance queries.
 
-The library will throw a helpful runtime error if `idb` isn't installed when `createIndexedDBStorage()` is invoked.
+From testing I've found Zustand and Zync are lightening fast even with 100,000 average sized state objects.
 
 ## Community
 

package/dist/index.cjs

@@ -349,9 +349,42 @@ function findApi(stateKey, syncApi) {
   }
   return api;
 }
+function findChanges(current, updated) {
+  const currentMap = /* @__PURE__ */ new Map();
+  for (const item of current) {
+    if (item && item._localId) {
+      currentMap.set(item._localId, item);
+    }
+  }
+  const changesMap = /* @__PURE__ */ new Map();
+  for (const item of updated) {
+    if (item && item._localId) {
+      const curr = currentMap.get(item._localId);
+      if (curr) {
+        const diff = {};
+        for (const key in curr) {
+          if (key !== "_localId" && curr[key] !== item[key]) {
+            diff[key] = item[key];
+          }
+        }
+        if (Object.keys(diff).length > 0) {
+          changesMap.set(item._localId, { currentItem: curr, updatedItem: item, changes: diff });
+        }
+      } else {
+        changesMap.set(item._localId, { currentItem: null, updatedItem: item, changes: item });
+      }
+    }
+  }
+  for (const [localId, curr] of currentMap) {
+    if (!updated.some((u) => u && u._localId === localId)) {
+      changesMap.set(localId, { currentItem: curr, updatedItem: null, changes: null });
+    }
+  }
+  return changesMap;
+}
 
 // src/pull.ts
-async function pull(set, get, stateKey, api, logger) {
+async function pull(set, get, stateKey, api, logger, conflictResolutionStrategy) {
   const lastPulled = get().syncState.lastPulled || {};
   const lastPulledAt = new Date(lastPulled[stateKey] || /* @__PURE__ */ new Date(0));
   logger.debug(`[zync] pull:start stateKey=${stateKey} since=${lastPulledAt.toISOString()}`);
@@ -381,15 +414,45 @@ async function pull(set, get, stateKey, api, logger) {
       }
       delete remote.deleted;
       const pending = localItem && pendingChanges.some((p) => p.stateKey === stateKey && p.localId === localItem._localId);
-      if (localItem && !pending) {
-        const merged = {
-          ...localItem,
-          ...remote,
-          _localId: localItem._localId
-        };
-        nextItems = nextItems.map((i) => i._localId === localItem._localId ? merged : i);
-        logger.debug(`[zync] pull:merge stateKey=${stateKey} id=${remote.id}`);
-      } else if (!localItem) {
+      if (localItem) {
+        if (pending) {
+          switch (conflictResolutionStrategy) {
+            case "local-wins":
+              logger.debug(`[zync] pull:conflict-strategy:${conflictResolutionStrategy} stateKey=${stateKey} id=${remote.id}`);
+              break;
+            case "remote-wins": {
+              const merged = {
+                ...remote,
+                _localId: localItem._localId
+              };
+              nextItems = nextItems.map((i) => i._localId === localItem._localId ? merged : i);
+              logger.debug(`[zync] pull:conflict-strategy:${conflictResolutionStrategy} stateKey=${stateKey} id=${remote.id}`);
+              break;
+            }
+            // case 'try-shallow-merge':
+            //     // Try and merge all fields, fail if not possible due to conflicts
+            //     // throw new ConflictError('Details...');
+            //     break;
+            // case 'custom':
+            //     // Hook to allow custom userland logic
+            //     // const error = onConflict(localItem, remote, stateKey, pending);
+            //     // logger.debug(`[zync] pull:conflict-strategy:${conflictResolutionStrategy} stateKey=${stateKey} id=${remote.id} error=${error}`);
+            //     // if (error) throw new ConflictError(error);
+            //     break;
+            default:
+              logger.error(`[zync] pull:conflict-strategy:unknown stateKey=${stateKey} id=${remote.id} strategy=${conflictResolutionStrategy}`);
+              break;
+          }
+        } else {
+          const merged = {
+            ...localItem,
+            ...remote,
+            _localId: localItem._localId
+          };
+          nextItems = nextItems.map((i) => i._localId === localItem._localId ? merged : i);
+          logger.debug(`[zync] pull:merge stateKey=${stateKey} id=${remote.id}`);
+        }
+      } else {
         nextItems = [...nextItems, { ...remote, _localId: nextLocalId() }];
         logger.debug(`[zync] pull:add stateKey=${stateKey} id=${remote.id}`);
       }
@@ -409,7 +472,7 @@ async function pull(set, get, stateKey, api, logger) {
 
 // src/push.ts
 var SYNC_FIELDS = ["id", "_localId", "updated_at", "deleted"];
-async function pushOne(set, get, change, api, logger, queueToSync, missingStrategy, onMissingRemoteRecordDuringUpdate, onAfterRemoteAdd) {
+async function pushOne(set, get, change, api, logger, setAndQueueToSync, missingStrategy, onMissingRemoteRecordDuringUpdate, onAfterRemoteAdd) {
   logger.debug(`[zync] push:attempt action=${change.action} stateKey=${change.stateKey} localId=${change.localId}`);
   const { action, stateKey, localId, id, version } = change;
   switch (action) {
@@ -449,26 +512,27 @@ async function pushOne(set, get, change, api, logger, queueToSync, missingStrate
           }
           return;
         } else {
-          logger.warn("[zync] push:update:missing-remote", {
-            stateKey,
-            localId,
-            id: item.id
-          });
           switch (missingStrategy) {
             case "delete-local-record":
               set((s) => ({
                 [stateKey]: (s[stateKey] || []).filter((i) => i._localId !== localId)
               }));
+              logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
               break;
-            case "insert-remote-record": {
+            case "insert-remote-record":
               omittedItem._localId = crypto.randomUUID();
               omittedItem.updated_at = (/* @__PURE__ */ new Date()).toISOString();
-              set((s) => ({
+              setAndQueueToSync((s) => ({
                 [stateKey]: (s[stateKey] || []).map((i) => i._localId === localId ? omittedItem : i)
               }));
-              queueToSync("create-or-update" /* CreateOrUpdate */, stateKey, omittedItem._localId);
+              logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
+              break;
+            case "ignore":
+              logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
+              break;
+            default:
+              logger.error(`[zync] push:missing-remote:unknown stateKey=${stateKey} id=${item.id} strategy=${missingStrategy}`);
               break;
-            }
           }
           removeFromPendingChanges(set, localId, stateKey);
           onMissingRemoteRecordDuringUpdate?.(missingStrategy, item, omittedItem._localId);
@@ -488,7 +552,7 @@ async function pushOne(set, get, change, api, logger, queueToSync, missingStrate
         if (samePendingVersion(get, stateKey, localId, version)) {
           removeFromPendingChanges(set, localId, stateKey);
         }
-        onAfterRemoteAdd?.(set, get, queueToSync, stateKey, {
+        onAfterRemoteAdd?.(set, get, setAndQueueToSync, stateKey, {
           ...item,
           ...result
         });
@@ -588,6 +652,7 @@ var DEFAULT_SYNC_INTERVAL_MILLIS = 5e3;
 var DEFAULT_LOGGER = console;
 var DEFAULT_MIN_LOG_LEVEL = "debug";
 var DEFAULT_MISSING_REMOTE_RECORD_STRATEGY = "ignore";
+var DEFAULT_CONFLICT_RESOLUTION_STRATEGY = "local-wins";
 function createWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}) {
   const store = (0, import_zustand.create)(persistWithSync(stateCreator, persistOptions, syncApi, syncOptions));
   return new Promise((resolve) => {
@@ -599,6 +664,7 @@ function createWithSync(stateCreator, persistOptions, syncApi, syncOptions = {})
 function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}) {
   const syncInterval = syncOptions.syncInterval ?? DEFAULT_SYNC_INTERVAL_MILLIS;
   const missingStrategy = syncOptions.missingRemoteRecordDuringUpdateStrategy ?? DEFAULT_MISSING_REMOTE_RECORD_STRATEGY;
+  const conflictResolutionStrategy = syncOptions.conflictResolutionStrategy ?? DEFAULT_CONFLICT_RESOLUTION_STRATEGY;
   const logger = newLogger(syncOptions.logger ?? DEFAULT_LOGGER, syncOptions.minLogLevel ?? DEFAULT_MIN_LOG_LEVEL);
   const baseOnRehydrate = persistOptions?.onRehydrateStorage;
   const basePartialize = persistOptions?.partialize;
@@ -654,15 +720,15 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
       for (const stateKey of Object.keys(syncApi)) {
         try {
           const api = findApi(stateKey, syncApi);
-          await pull(set, get, stateKey, api, logger);
+          await pull(set, get, stateKey, api, logger, conflictResolutionStrategy);
         } catch (err) {
           syncError = syncError ?? err;
           logger.error(`[zync] pull:error stateKey=${stateKey}`, err);
         }
       }
-      const snapshot = [...get().syncState.pendingChanges || []];
-      snapshot.sort((a, b) => orderFor(a.action) - orderFor(b.action));
-      for (const change of snapshot) {
+      const changesSnapshot = [...get().syncState.pendingChanges || []];
+      changesSnapshot.sort((a, b) => orderFor(a.action) - orderFor(b.action));
+      for (const change of changesSnapshot) {
         try {
           const api = findApi(change.stateKey, syncApi);
           await pushOne(
@@ -671,7 +737,7 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
             change,
             api,
             logger,
-            queueToSync,
+            setAndQueueToSync,
             missingStrategy,
             syncOptions.onMissingRemoteRecordDuringUpdate,
             syncOptions.onAfterRemoteAdd
@@ -755,40 +821,7 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
         }
       }));
     }
-    function queueToSync(action, stateKey, ...localIds) {
-      set((state) => {
-        const pendingChanges = state.syncState.pendingChanges || [];
-        for (const localId of localIds) {
-          const item = state[stateKey].find((i) => i._localId === localId);
-          if (!item) {
-            logger.error(`[zync] queueToSync:no-local-item localId=${localId}`);
-            continue;
-          }
-          const queueItem = pendingChanges.find((p) => p.localId === localId && p.stateKey === stateKey);
-          if (queueItem) {
-            queueItem.version += 1;
-            if (queueItem.action === "create-or-update" /* CreateOrUpdate */ && action === "remove" /* Remove */ && item.id) {
-              queueItem.action = "remove" /* Remove */;
-              queueItem.id = item.id;
-              logger.debug(`[zync] queueToSync:changed-to-remove action=${action} localId=${localId} v=${queueItem.version}`);
-            } else {
-              logger.debug(`[zync] queueToSync:re-queued action=${action} localId=${localId} v=${queueItem.version}`);
-            }
-          } else {
-            pendingChanges.push({ action, stateKey, localId, id: item.id, version: 1 });
-            logger.debug(`[zync] queueToSync:added action=${action} localId=${localId}`);
-          }
-        }
-        return {
-          syncState: {
-            ...state.syncState || {},
-            pendingChanges
-          }
-        };
-      });
-      syncOnce();
-    }
-    function setAndSync(partial) {
+    function setAndSyncOnce(partial) {
       if (typeof partial === "function") {
         set((state) => ({ ...partial(state) }));
       } else {
@@ -796,6 +829,49 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
       }
       syncOnce();
     }
+    function setAndQueueToSync(partial) {
+      if (typeof partial === "function") {
+        set((state) => newSyncState(state, partial(state)));
+      } else {
+        set((state) => newSyncState(state, partial));
+      }
+      syncOnce();
+    }
+    function newSyncState(state, partial) {
+      const pendingChanges = state.syncState.pendingChanges || [];
+      Object.keys(partial).map((stateKey) => {
+        const current = state[stateKey];
+        const updated = partial[stateKey];
+        const changes = findChanges(current, updated);
+        addToPendingChanges(pendingChanges, stateKey, changes);
+      });
+      return {
+        ...partial,
+        syncState: {
+          ...state.syncState || {},
+          pendingChanges
+        }
+      };
+    }
+    function addToPendingChanges(pendingChanges, stateKey, changes) {
+      for (const [localId, change] of changes) {
+        const action = change.updatedItem === null ? "remove" /* Remove */ : "create-or-update" /* CreateOrUpdate */;
+        const queueItem = pendingChanges.find((p) => p.localId === localId && p.stateKey === stateKey);
+        if (queueItem) {
+          queueItem.version += 1;
+          if (queueItem.action === "create-or-update" /* CreateOrUpdate */ && action === "remove" /* Remove */ && change.currentItem.id) {
+            queueItem.action = "remove" /* Remove */;
+            queueItem.id = change.currentItem.id;
+            logger.debug(`[zync] addToPendingChanges:changed-to-remove action=${action} localId=${localId} v=${queueItem.version}`);
+          } else {
+            logger.debug(`[zync] addToPendingChanges:re-queued action=${action} localId=${localId} v=${queueItem.version}`);
+          }
+        } else {
+          pendingChanges.push({ action, stateKey, localId, id: change.currentItem?.id, version: 1 });
+          logger.debug(`[zync] addToPendingChanges:added action=${action} localId=${localId}`);
+        }
+      }
+    }
     function enable(enabled) {
       set((state) => ({
         syncState: {
@@ -834,7 +910,7 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
       enable,
       startFirstLoad
     };
-    const userState = stateCreator(setAndSync, get, queueToSync);
+    const userState = stateCreator(setAndSyncOnce, get, setAndQueueToSync);
     return {
       ...userState,
       syncState: {