@anfenn/zync 0.2.0 → 0.3.0

package/README.md

@@ -4,27 +4,32 @@
 
 Simple, bullet-proof, offline-first sync middleware for Zustand.
 
+**_STATUS_**: Actively developed in alpha stage while requirements are being understood. Api may change, requests are welcome.
+
 ## Benefits
 
 - Easy to sync non-nested array state with a backend (i.e. mirror remote database tables locally)
 - **"It just works"** philosophy
+- Optimistic UI updates
 - Batteries optionally included:
     - IndexedDB helper (based on [idb](https://www.npmjs.com/package/idb))
+    - UUID helper
 - Uses the official persist middleware as the local storage (localStorage, IndexedDB, etc.)
 - Zync's persistWithSync() is a drop-in replacement for Zustand's persist()
 - Allows for idiomatic use of Zustand
 - Leaves the api requests up to you (RESTful, GraphQL, etc.), just provide add(), update(), remove() and list()
+- Client or server assigned primary key, of any datatype
 - **_Coming soon_**: Customisable conflict resolution. Currently local-wins.
 
 ## Requirements
 
 - Client records will have a `_localId` field which is stable and never sent to the server. It is ideal for use as a key in JSX. The provided helper function `nextLocalId()` returns a UUID, but you could use any unique value
 - Server records must have:
-    - `id`: Server assigned unique identifier (any datatype)
+    - `id`: Any datatype, can be a server OR client assigned value
     - `updated_at`: Server assigned **_millisecond_** timestamp (db trigger or api layer). The client will never send this as the client clock is unlikely to be in sync with the server, so is never used for change detection. If it has a higher precision than millisecond, like PostgreSQL's microsecond timestampz, updates could be ignored.
     - `deleted`: Boolean, used for soft deletes, to allow other clients to download deleted records to keep their local records in sync
 
-## Quickstart
+## Quickstart - Server assigned id example
 
 ```bash
 npm install @anfenn/zync
@@ -53,7 +58,7 @@ export const useStore = create<any>()(
 
             facts: [],
             addFact: (item: Fact) => {
-                const updated_at = new Date().toISOString();
+                const updated_at = new Date().toISOString(); // Used as an optimistic UI update only, never sent to server
                 const newItem = { ...item, created_at: updated_at, updated_at };
 
                 setAndSync((state: Store) => ({
@@ -103,7 +108,7 @@ export const useFacts = () =>
     );
 ```
 
-**_NOTE_**: Zync uses an internal timer (setInterval) to sync, so it's advised to just have one store. You could have multiple, with different store names (see Zustand persist options above), but if both stores use Zync, although it would work fine, it wouldn't offer much advantage. If one store becomes large with many state keys and functions, then you could separate them into multiple files and import than with object spreading
+**_NOTE_**: Zync uses an internal timer (setInterval) to sync, so it's advised to just have one store. You could have multiple, with different store names (see Zustand persist options above), but if both stores use Zync, although it would work fine, it wouldn't offer much advantage. If one store becomes large with many state keys and functions, then you could separate them into multiple files and import them with object spreading
 `e.g. {...storeState1, ...storeState2}`
 
 ### In your component:
@@ -239,6 +244,10 @@ async function firstLoad(lastId: any) {
 }
 ```
 
+## Client side assigned id
+
+As simple as just setting the id field when creating a new record (and amend model types if using Typescript)
+
 ## Optional IndexedDB storage
 
 Using async IndexedDB over sync localStorage gives the advantage of a responsive UI when reading/writing a very large store, as IndexedDB is running in it's own thread.

package/dist/index.cjs

@@ -310,20 +310,23 @@ function newLogger(base, min) {
 }
 
 // src/helpers.ts
+var SYNC_FIELDS = ["_localId", "updated_at", "deleted"];
 function nextLocalId() {
   return crypto.randomUUID();
 }
 function orderFor(a) {
   switch (a) {
-    case "create-or-update" /* CreateOrUpdate */:
+    case "create" /* Create */:
       return 1;
-    case "remove" /* Remove */:
+    case "update" /* Update */:
       return 2;
+    case "remove" /* Remove */:
+      return 3;
   }
 }
-function omitSyncFields(item, fields) {
+function omitSyncFields(item) {
   const result = { ...item };
-  for (const k of fields) delete result[k];
+  for (const k of SYNC_FIELDS) delete result[k];
   return result;
 }
 function samePendingVersion(get, stateKey, localId, version) {
@@ -342,6 +345,38 @@ function removeFromPendingChanges(set, localId, stateKey) {
     };
   });
 }
+function tryAddToPendingChanges(pendingChanges, stateKey, changes) {
+  for (const [localId, change] of changes) {
+    let omittedItem = omitSyncFields(change.changes);
+    const queueItem = pendingChanges.find((p) => p.localId === localId && p.stateKey === stateKey);
+    const hasChanges = Object.keys(omittedItem).length > 0;
+    const action = change.updatedItem === null ? "remove" /* Remove */ : change.currentItem === null ? "create" /* Create */ : "update" /* Update */;
+    if (action === "update" /* Update */ && change.updatedItem && change.currentItem && change.currentItem._localId !== change.updatedItem._localId) {
+      omittedItem = omitSyncFields(change.updatedItem);
+    }
+    if (queueItem) {
+      if (queueItem.action === "remove" /* Remove */) {
+        continue;
+      }
+      queueItem.version += 1;
+      if (action === "remove" /* Remove */) {
+        queueItem.action = "remove" /* Remove */;
+      } else if (hasChanges) {
+        queueItem.changes = { ...queueItem.changes, ...omittedItem };
+      }
+    } else if (action === "remove" /* Remove */ || hasChanges) {
+      pendingChanges.push({ action, stateKey, localId, id: change.id, version: 1, changes: omittedItem });
+    }
+  }
+}
+function setPendingChangeToUpdate(get, stateKey, localId, id) {
+  const pendingChanges = get().syncState.pendingChanges || [];
+  const change = pendingChanges.find((p) => p.stateKey === stateKey && p.localId === localId);
+  if (change) {
+    change.action = "update" /* Update */;
+    if (id) change.id = id;
+  }
+}
 function findApi(stateKey, syncApi) {
   const api = syncApi[stateKey];
   if (!api || !api.add || !api.update || !api.remove || !api.list || !api.firstLoad) {
@@ -353,11 +388,12 @@ function findChanges(current, updated) {
   const currentMap = /* @__PURE__ */ new Map();
   for (const item of current) {
     if (item && item._localId) {
-      currentMap.set(item._localId, item);
+      currentMap.set(item._localId, { ...item });
     }
   }
   const changesMap = /* @__PURE__ */ new Map();
-  for (const item of updated) {
+  for (const update of updated) {
+    const item = { ...update };
     if (item && item._localId) {
       const curr = currentMap.get(item._localId);
       if (curr) {
@@ -368,16 +404,16 @@ function findChanges(current, updated) {
           }
         }
         if (Object.keys(diff).length > 0) {
-          changesMap.set(item._localId, { currentItem: curr, updatedItem: item, changes: diff });
+          changesMap.set(item._localId, { currentItem: curr, updatedItem: item, changes: diff, id: curr.id ?? item.id });
         }
       } else {
-        changesMap.set(item._localId, { currentItem: null, updatedItem: item, changes: item });
+        changesMap.set(item._localId, { currentItem: null, updatedItem: item, changes: item, id: item.id });
       }
     }
   }
   for (const [localId, curr] of currentMap) {
     if (!updated.some((u) => u && u._localId === localId)) {
-      changesMap.set(localId, { currentItem: curr, updatedItem: null, changes: null });
+      changesMap.set(localId, { currentItem: curr, updatedItem: null, changes: null, id: curr.id });
     }
   }
   return changesMap;
@@ -413,9 +449,9 @@ async function pull(set, get, stateKey, api, logger, conflictResolutionStrategy)
         continue;
       }
       delete remote.deleted;
-      const pending = localItem && pendingChanges.some((p) => p.stateKey === stateKey && p.localId === localItem._localId);
       if (localItem) {
-        if (pending) {
+        const pendingChange = pendingChanges.find((p) => p.stateKey === stateKey && p.localId === localItem._localId);
+        if (pendingChange) {
           switch (conflictResolutionStrategy) {
             case "local-wins":
               logger.debug(`[zync] pull:conflict-strategy:${conflictResolutionStrategy} stateKey=${stateKey} id=${remote.id}`);
@@ -446,11 +482,10 @@ async function pull(set, get, stateKey, api, logger, conflictResolutionStrategy)
         } else {
           const merged = {
             ...localItem,
-            ...remote,
-            _localId: localItem._localId
+            ...remote
           };
           nextItems = nextItems.map((i) => i._localId === localItem._localId ? merged : i);
-          logger.debug(`[zync] pull:merge stateKey=${stateKey} id=${remote.id}`);
+          logger.debug(`[zync] pull:merge-remote stateKey=${stateKey} id=${remote.id}`);
         }
       } else {
         nextItems = [...nextItems, { ...remote, _localId: nextLocalId() }];
@@ -471,96 +506,85 @@ async function pull(set, get, stateKey, api, logger, conflictResolutionStrategy)
 }
 
 // src/push.ts
-var SYNC_FIELDS = ["id", "_localId", "updated_at", "deleted"];
 async function pushOne(set, get, change, api, logger, setAndQueueToSync, missingStrategy, onMissingRemoteRecordDuringUpdate, onAfterRemoteAdd) {
   logger.debug(`[zync] push:attempt action=${change.action} stateKey=${change.stateKey} localId=${change.localId}`);
-  const { action, stateKey, localId, id, version } = change;
+  const { action, stateKey, localId, id, version, changes } = change;
+  const changesClone = { ...changes };
   switch (action) {
     case "remove" /* Remove */:
       if (!id) {
-        logger.warn(`[zync] push:remove:no-id ${stateKey} ${localId}`);
+        logger.warn(`[zync] push:remove:no-id stateKey=${stateKey} localId=${localId}`);
         removeFromPendingChanges(set, localId, stateKey);
         return;
       }
       await api.remove(id);
-      logger.debug(`[zync] push:remove:success ${stateKey} ${localId} ${id}`);
+      logger.debug(`[zync] push:remove:success stateKey=${stateKey} localId=${localId} id=${id}`);
       removeFromPendingChanges(set, localId, stateKey);
       break;
-    case "create-or-update" /* CreateOrUpdate */: {
-      const state = get();
-      const items = state[stateKey] || [];
-      const item = items.find((i) => i._localId === localId);
-      if (!item) {
-        logger.warn(`[zync] push:create-or-update:no-local-item`, {
-          stateKey,
-          localId
-        });
-        removeFromPendingChanges(set, localId, stateKey);
+    case "update" /* Update */: {
+      const changed = await api.update(id, changesClone);
+      if (changed) {
+        logger.debug(`[zync] push:update:success stateKey=${stateKey} localId=${localId} id=${id}`);
+        if (samePendingVersion(get, stateKey, localId, version)) {
+          removeFromPendingChanges(set, localId, stateKey);
+        }
         return;
-      }
-      const omittedItem = omitSyncFields(item, SYNC_FIELDS);
-      if (item.id) {
-        const changed = await api.update(item.id, omittedItem);
-        if (changed) {
-          logger.debug("[zync] push:update:success", {
-            stateKey,
-            localId,
-            id: item.id
-          });
-          if (samePendingVersion(get, stateKey, localId, version)) {
-            removeFromPendingChanges(set, localId, stateKey);
-          }
+      } else {
+        const state = get();
+        const items = state[stateKey] || [];
+        const item = items.find((i) => i._localId === localId);
+        if (!item) {
+          logger.warn(`[zync] push:missing-remote:no-local-item stateKey=${stateKey} localId=${localId}`);
+          removeFromPendingChanges(set, localId, stateKey);
           return;
-        } else {
-          switch (missingStrategy) {
-            case "delete-local-record":
-              set((s) => ({
-                [stateKey]: (s[stateKey] || []).filter((i) => i._localId !== localId)
-              }));
-              logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
-              break;
-            case "insert-remote-record":
-              omittedItem._localId = crypto.randomUUID();
-              omittedItem.updated_at = (/* @__PURE__ */ new Date()).toISOString();
-              setAndQueueToSync((s) => ({
-                [stateKey]: (s[stateKey] || []).map((i) => i._localId === localId ? omittedItem : i)
-              }));
-              logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
-              break;
-            case "ignore":
-              logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
-              break;
-            default:
-              logger.error(`[zync] push:missing-remote:unknown stateKey=${stateKey} id=${item.id} strategy=${missingStrategy}`);
-              break;
+        }
+        switch (missingStrategy) {
+          case "delete-local-record":
+            set((s) => ({
+              [stateKey]: (s[stateKey] || []).filter((i) => i._localId !== localId)
+            }));
+            logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
+            break;
+          case "insert-remote-record": {
+            const newItem = {
+              ...item,
+              _localId: nextLocalId(),
+              updated_at: (/* @__PURE__ */ new Date()).toISOString()
+            };
+            setAndQueueToSync((s) => ({
+              [stateKey]: (s[stateKey] || []).map((i) => i._localId === localId ? newItem : i)
+            }));
+            logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${newItem.id}`);
+            break;
           }
-          removeFromPendingChanges(set, localId, stateKey);
-          onMissingRemoteRecordDuringUpdate?.(missingStrategy, item, omittedItem._localId);
+          case "ignore":
+            logger.debug(`[zync] push:missing-remote:${missingStrategy} stateKey=${stateKey} id=${item.id}`);
+            break;
+          default:
+            logger.error(`[zync] push:missing-remote:unknown-strategy stateKey=${stateKey} id=${item.id} strategy=${missingStrategy}`);
+            break;
         }
-        return;
+        removeFromPendingChanges(set, localId, stateKey);
+        onMissingRemoteRecordDuringUpdate?.(missingStrategy, item);
       }
-      const result = await api.add(omittedItem);
+      break;
+    }
+    case "create" /* Create */: {
+      const result = await api.add(changesClone);
       if (result) {
-        logger.debug("[zync] push:create:success", {
-          stateKey,
-          localId,
-          id: result.id
-        });
+        logger.debug(`[zync] push:create:success stateKey=${stateKey} localId=${localId} id=${id}`);
         set((s) => ({
           [stateKey]: (s[stateKey] || []).map((i) => i._localId === localId ? { ...i, ...result } : i)
         }));
         if (samePendingVersion(get, stateKey, localId, version)) {
           removeFromPendingChanges(set, localId, stateKey);
+        } else {
+          setPendingChangeToUpdate(get, stateKey, localId, result.id);
         }
-        onAfterRemoteAdd?.(set, get, setAndQueueToSync, stateKey, {
-          ...item,
-          ...result
-        });
+        const finalItem = { ...changes, ...result, _localId: localId };
+        onAfterRemoteAdd?.(set, get, setAndQueueToSync, stateKey, finalItem);
       } else {
-        logger.warn("[zync] push:create:no-result", {
-          stateKey,
-          localId
-        });
+        logger.warn(`[zync] push:create:no-result stateKey=${stateKey} localId=${localId} id=${id}`);
         if (samePendingVersion(get, stateKey, localId, version)) {
           removeFromPendingChanges(set, localId, stateKey);
         }
@@ -570,6 +594,74 @@ async function pushOne(set, get, change, api, logger, setAndQueueToSync, missing
   }
 }
 
+// src/firstLoad.ts
+async function startFirstLoad(set, syncApi, logger) {
+  let syncError;
+  for (const stateKey of Object.keys(syncApi)) {
+    try {
+      logger.info(`[zync] firstLoad:start stateKey=${stateKey}`);
+      const api = findApi(stateKey, syncApi);
+      let lastId;
+      while (true) {
+        const batch = await api.firstLoad(lastId);
+        if (!batch?.length) break;
+        set((state) => {
+          const local = state[stateKey] || [];
+          const localById = new Map(local.filter((l) => l.id).map((l) => [l.id, l]));
+          let newest = new Date(state.syncState.lastPulled[stateKey] || 0);
+          const next = [...local];
+          for (const remote of batch) {
+            const remoteUpdated = new Date(remote.updated_at || 0);
+            if (remoteUpdated > newest) newest = remoteUpdated;
+            if (remote.deleted) continue;
+            delete remote.deleted;
+            const localItem = remote.id ? localById.get(remote.id) : void 0;
+            if (localItem) {
+              const merged = {
+                ...localItem,
+                ...remote,
+                _localId: localItem._localId
+              };
+              const idx = next.findIndex((i) => i._localId === localItem._localId);
+              if (idx >= 0) next[idx] = merged;
+            } else {
+              next.push({
+                ...remote,
+                _localId: nextLocalId()
+              });
+            }
+          }
+          return {
+            [stateKey]: next,
+            syncState: {
+              ...state.syncState || {},
+              lastPulled: {
+                ...state.syncState.lastPulled || {},
+                [stateKey]: newest.toISOString()
+              }
+            }
+          };
+        });
+        if (lastId !== void 0 && lastId === batch[batch.length - 1].id) {
+          throw new Error(`Duplicate records downloaded, stopping to prevent infinite loop`);
+        }
+        lastId = batch[batch.length - 1].id;
+      }
+      logger.info(`[zync] firstLoad:done stateKey=${stateKey}`);
+    } catch (err) {
+      syncError = syncError ?? err;
+      logger.error(`[zync] firstLoad:error stateKey=${stateKey}`, err);
+    }
+  }
+  set((state) => ({
+    syncState: {
+      ...state.syncState || {},
+      firstLoadDone: true,
+      error: syncError
+    }
+  }));
+}
+
 // src/indexedDBStorage.ts
 function createIndexedDBStorage(options) {
   const dbName = options.dbName;
@@ -644,7 +736,8 @@ function createIndexedDBStorage(options) {
 
 // src/index.ts
 var SyncAction = /* @__PURE__ */ ((SyncAction2) => {
-  SyncAction2["CreateOrUpdate"] = "create-or-update";
+  SyncAction2["Create"] = "create";
+  SyncAction2["Update"] = "update";
   SyncAction2["Remove"] = "remove";
   return SyncAction2;
 })(SyncAction || {});
@@ -716,13 +809,13 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
           status: "syncing"
         }
       }));
-      let syncError;
+      let firstSyncError;
       for (const stateKey of Object.keys(syncApi)) {
         try {
           const api = findApi(stateKey, syncApi);
           await pull(set, get, stateKey, api, logger, conflictResolutionStrategy);
         } catch (err) {
-          syncError = syncError ?? err;
+          firstSyncError = firstSyncError ?? err;
           logger.error(`[zync] pull:error stateKey=${stateKey}`, err);
         }
       }
@@ -743,7 +836,7 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
             syncOptions.onAfterRemoteAdd
           );
         } catch (err) {
-          syncError = syncError ?? err;
+          firstSyncError = firstSyncError ?? err;
           logger.error(`[zync] push:error change=${change}`, err);
         }
       }
@@ -751,76 +844,13 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
         syncState: {
           ...state2.syncState || {},
           status: "idle",
-          error: syncError
+          error: firstSyncError
         }
       }));
-      if (get().syncState.pendingChanges.length > 0 && !syncError) {
+      if (get().syncState.pendingChanges.length > 0 && !firstSyncError) {
         await syncOnce();
       }
     }
-    async function startFirstLoad() {
-      let syncError;
-      for (const stateKey of Object.keys(syncApi)) {
-        try {
-          logger.info(`[zync] firstLoad:start stateKey=${stateKey}`);
-          const api = findApi(stateKey, syncApi);
-          let lastId;
-          while (true) {
-            const batch = await api.firstLoad(lastId);
-            if (!batch?.length) break;
-            set((state) => {
-              const local = state[stateKey] || [];
-              const localById = new Map(local.filter((l) => l.id).map((l) => [l.id, l]));
-              let newest = new Date(state.syncState.lastPulled[stateKey] || 0);
-              const next = [...local];
-              for (const remote of batch) {
-                const remoteUpdated = new Date(remote.updated_at || 0);
-                if (remoteUpdated > newest) newest = remoteUpdated;
-                if (remote.deleted) continue;
-                delete remote.deleted;
-                const localItem = remote.id ? localById.get(remote.id) : void 0;
-                if (localItem) {
-                  const merged = {
-                    ...localItem,
-                    ...remote,
-                    _localId: localItem._localId
-                  };
-                  const idx = next.findIndex((i) => i._localId === localItem._localId);
-                  if (idx >= 0) next[idx] = merged;
-                } else {
-                  next.push({
-                    ...remote,
-                    _localId: nextLocalId()
-                  });
-                }
-              }
-              return {
-                [stateKey]: next,
-                syncState: {
-                  ...state.syncState || {},
-                  lastPulled: {
-                    ...state.syncState.lastPulled || {},
-                    [stateKey]: newest.toISOString()
-                  }
-                }
-              };
-            });
-            lastId = batch[batch.length - 1].id;
-          }
-          logger.info(`[zync] firstLoad:done stateKey=${stateKey}`);
-        } catch (err) {
-          syncError = syncError ?? err;
-          logger.error(`[zync] firstLoad:error stateKey=${stateKey}`, err);
-        }
-      }
-      set((state) => ({
-        syncState: {
-          ...state.syncState || {},
-          firstLoadDone: true,
-          error: syncError
-        }
-      }));
-    }
     function setAndSyncOnce(partial) {
       if (typeof partial === "function") {
         set((state) => ({ ...partial(state) }));
@@ -843,7 +873,7 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
         const current = state[stateKey];
         const updated = partial[stateKey];
         const changes = findChanges(current, updated);
-        addToPendingChanges(pendingChanges, stateKey, changes);
+        tryAddToPendingChanges(pendingChanges, stateKey, changes);
       });
       return {
         ...partial,
@@ -853,25 +883,6 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
         }
       };
     }
-    function addToPendingChanges(pendingChanges, stateKey, changes) {
-      for (const [localId, change] of changes) {
-        const action = change.updatedItem === null ? "remove" /* Remove */ : "create-or-update" /* CreateOrUpdate */;
-        const queueItem = pendingChanges.find((p) => p.localId === localId && p.stateKey === stateKey);
-        if (queueItem) {
-          queueItem.version += 1;
-          if (queueItem.action === "create-or-update" /* CreateOrUpdate */ && action === "remove" /* Remove */ && change.currentItem.id) {
-            queueItem.action = "remove" /* Remove */;
-            queueItem.id = change.currentItem.id;
-            logger.debug(`[zync] addToPendingChanges:changed-to-remove action=${action} localId=${localId} v=${queueItem.version}`);
-          } else {
-            logger.debug(`[zync] addToPendingChanges:re-queued action=${action} localId=${localId} v=${queueItem.version}`);
-          }
-        } else {
-          pendingChanges.push({ action, stateKey, localId, id: change.currentItem?.id, version: 1 });
-          logger.debug(`[zync] addToPendingChanges:added action=${action} localId=${localId}`);
-        }
-      }
-    }
     function enable(enabled) {
       set((state) => ({
         syncState: {
@@ -908,7 +919,7 @@ function persistWithSync(stateCreator, persistOptions, syncApi, syncOptions = {}
     }
     storeApi.sync = {
       enable,
-      startFirstLoad
+      startFirstLoad: () => startFirstLoad(set, syncApi, logger)
     };
     const userState = stateCreator(setAndSyncOnce, get, setAndQueueToSync);
     return {