@jcbuisson/express-x 3.1.6 → 3.1.12

package/CLAUDE.md

@@ -61,7 +61,7 @@ No build step — the source is run directly as ES modules.
 
 The sync system reconciles a client Dexie cache with a server database. The protocol:
 
-1. Client calls `sync.go(modelName, where, cutoffDate, clientMetadataDict)` with its local metadata snapshot.
+1. Client calls `sync.go(modelName, where, clientMetadataDict)` with its local metadata snapshot.
 2. Server computes diff using `computeSyncResult` (exported from `server.mjs`, pure function, no I/O).
 3. Server returns `{ addClient, updateClient, deleteClient, addDatabase, updateDatabase }`.
 4. Client applies each batch: puts addClient records into Dexie, deletes deleteClient, updates updateClient, then calls `createWithMeta`/`updateWithMeta` for addDatabase/updateDatabase.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jcbuisson/express-x",
-  "version": "3.1.6",
+  "version": "3.1.12",
   "description": "",
   "type": "module",
   "main": "src/server.mjs",
@@ -29,8 +29,8 @@
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.4.5",
-    "@jcbuisson/express-x-client": "^3.1.6",
-    "@jcbuisson/express-x-drizzle": "^3.1.6",
+    "@jcbuisson/express-x-client": "^3.1.11",
+    "@jcbuisson/express-x-drizzle": "^3.1.11",
     "@vueuse/core": "^14.3.0",
     "dexie": "^4.4.2",
     "drizzle-orm": "^0.45.2",

package/src/server.mjs

@@ -46,6 +46,17 @@ export class Mutex {
 
 //////////////////////////       SYNC ALGORITHM (common to all offline plugins)       //////////////////////////
 
+// This sync algorithm uses the metadata table as its only source of truth for existence and recency. Manual SQL changes
+// create a split reality — the data table and the metadata table diverge — and the algorithm's set-intersection logic
+// misidentifies the divergence as intentional client-side changes that must be propagated.
+// If you need to seed or manipulate data outside the framework, you must also write the corresponding rows into the
+// metadata table with valid created_at/updated_at/deleted_at values.
+
+// The metadata table is a tombstone log that grows monotonically. Every record ever created leaves a permanent entry. For
+// long-lived applications with high churn this is worth planning for: you can only safely purge a tombstone once you're certain no
+// client still holds a copy of that record (i.e., every client has synced at least once after the deletion). There's currently no
+// pruning mechanism in the framework.
+
 export function computeSyncResult(databaseValuesDict, clientMetadataDict, databaseMetadataDict) {
    const onlyDatabaseIds = new Set()
    const onlyClientIds = new Set()
@@ -79,11 +90,18 @@ export function computeSyncResult(databaseValuesDict, clientMetadataDict, databa
 
    for (const uid of databaseAndClientIds) {
       const clientMetaData = clientMetadataDict[uid]
+      const databaseMetaData = databaseMetadataDict[uid] || { uid, created_at: null }
       if (clientMetaData.deleted_at) {
-         deleteDatabase.push(uid)
-         deleteClient.push([uid, clientMetaData.deleted_at])
+         const clientDeletedAt = new Date(clientMetaData.deleted_at)
+         const databaseUpdatedAt = new Date(databaseMetaData.updated_at || databaseMetaData.created_at)
+         const diff = clientDeletedAt - databaseUpdatedAt
+         if (diff >= 0) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, clientMetaData.deleted_at])
+         } else {
+            updateClient.push([databaseValuesDict[uid], databaseMetaData])
+         }
       } else {
-         const databaseMetaData = databaseMetadataDict[uid] || { uid, created_at: null }
          const clientUpdatedAt = new Date(clientMetaData.updated_at || clientMetaData.created_at)
          const databaseUpdatedAt = new Date(databaseMetaData.updated_at || databaseMetaData.created_at)
          const diff = clientUpdatedAt - databaseUpdatedAt

package/test/offline.test.mjs

@@ -8,6 +8,7 @@ process.on('unhandledRejection', (reason, promise) => {
 import { test, describe, after } from 'node:test'
 import assert from 'node:assert/strict'
 import { io as ioc } from 'socket.io-client'
+import { firstValueFrom } from 'rxjs'
 import { PGlite } from '@electric-sql/pglite'
 import { drizzle } from 'drizzle-orm/pglite'
 import { pgTable, text, timestamp, integer } from 'drizzle-orm/pg-core'
@@ -308,6 +309,41 @@ describe('Full offline-first client ↔ server protocol', () => {
       }
    })
 
+   test('record in both, DB created_at newer → client metadata is fully replaced', async () => {
+      const modelName = `model${++dbCounter}`
+      const { pglite, db, metaTable, modelTable } = await createTestDb(modelName)
+
+      // Server is newer by created_at only; updated_at is null.
+      await db.insert(modelTable).values({ uid: 's1', label: 'server-created-later' })
+      await db.insert(metaTable).values({ uid: 's1', created_at: T2 })
+
+      const { clientApp, cleanup } = await createTestContext(
+         serverApp => serverApp.configure(drizzleOfflinePlugin, db, metaTable, [modelTable]),
+         { useOfflinePlugin: true },
+      )
+
+      try {
+         const model = clientApp.createOfflineModel(modelName, ['label'])
+         await model.db.values.add({ uid: 's1', label: 'client-created-earlier' })
+         await model.db.metadata.add({ uid: 's1', created_at: T1 })
+         await model.addSynchroWhere({})
+
+         await model.synchronizeAll()
+
+         const s1 = await model.db.values.get('s1')
+         const meta = await model.db.metadata.get('s1')
+         assert.equal(s1.label, 'server-created-later')
+         assert.equal(new Date(meta.created_at).getTime(), T2.getTime())
+
+         await model.synchronizeAll()
+         const s1Again = await model.db.values.get('s1')
+         assert.equal(s1Again.label, 'server-created-later')
+      } finally {
+         await cleanup()
+         pglite.close()
+      }
+   })
+
    test('one failed updateWithMeta does not abort remaining updateDatabase entries', async () => {
       const modelName = `model${++dbCounter}`
       const serverUpdated = {}
@@ -315,7 +351,7 @@ describe('Full offline-first client ↔ server protocol', () => {
       const { clientApp, cleanup } = await createTestContext(serverApp => {
          serverApp.createService('sync', {
             // Tell the client both u1 and u2 need to be pushed (client is newer for both)
-            go: async (mn, where, cutoff, clientMetadataDict) => ({
+            go: async (mn, where, clientMetadataDict) => ({
                addClient: [], updateClient: [], deleteClient: [], addDatabase: [],
                updateDatabase: [ clientMetadataDict['u1'], clientMetadataDict['u2'] ],
             }),
@@ -489,6 +525,102 @@ describe('Full offline-first client ↔ server protocol', () => {
       pglite.close()
    })
 
+   test('reconnect keeps disconnectedDate until automatic sync completes', async () => {
+      const modelName = `model${++dbCounter}`
+      const serverRows = []
+      let resolveSyncStarted
+      let releaseSync
+      let resolveCreate
+      const syncStarted = new Promise(resolve => { resolveSyncStarted = resolve })
+      const syncRelease = new Promise(resolve => { releaseSync = resolve })
+      const createDone = new Promise(resolve => { resolveCreate = resolve })
+
+      function registerServices(app, { slowSync = false } = {}) {
+         app.createService(modelName, {
+            createWithMeta: async (uid, data, created_at) => {
+               serverRows.push({ uid, ...data })
+               resolveCreate()
+               return [{ uid, ...data }, { uid, created_at, updated_at: null, deleted_at: null }]
+            },
+            updateWithMeta: async () => {},
+            deleteWithMeta: async () => {},
+            findMany: async () => [],
+         })
+         app.createService('sync', {
+            go: async (_modelName, _where, clientMetadataDict) => {
+               if (slowSync) {
+                  resolveSyncStarted()
+                  await syncRelease
+               }
+               return {
+                  addClient: [],
+                  updateClient: [],
+                  deleteClient: [],
+                  addDatabase: Object.values(clientMetadataDict),
+                  updateDatabase: [],
+               }
+            },
+         })
+      }
+
+      const serverApp1 = expressX({})
+      registerServices(serverApp1)
+      await new Promise(resolve => serverApp1.httpServer.listen(0, resolve))
+      const port = serverApp1.httpServer.address().port
+
+      const socket = ioc(`http://localhost:${port}`, {
+         transports: ['websocket'],
+         autoConnect: false,
+      })
+      const clientApp = createClient(socket, { debug: false })
+      offlinePlugin(clientApp)
+      socket.connect()
+      await new Promise((resolve, reject) => {
+         socket.on('connect', resolve)
+         socket.on('connect_error', reject)
+      })
+
+      const model = clientApp.createOfflineModel(modelName, ['label'])
+      await model.addSynchroWhere({})
+      await model.synchronizeAll()
+
+      const disconnected = new Promise(resolve => socket.once('disconnect', resolve))
+      serverApp1.io.disconnectSockets(true)
+      await new Promise(resolve => serverApp1.io.close(resolve))
+      await disconnected
+
+      await model.db.values.add({ uid: 'reconnect-1', label: 'from reconnect' })
+      await model.db.metadata.add({ uid: 'reconnect-1', created_at: T1 })
+
+      const serverApp2 = expressX({})
+      registerServices(serverApp2, { slowSync: true })
+      await new Promise(resolve => serverApp2.httpServer.listen(port, resolve))
+
+      const reconnected = new Promise((resolve, reject) => {
+         const timer = setTimeout(() => reject(new Error('reconnect timeout')), 5000)
+         socket.once('connect', () => { clearTimeout(timer); resolve() })
+      })
+      socket.connect()
+      await reconnected
+      await syncStarted
+
+      assert.ok(clientApp.disconnectedDate, 'disconnect marker should remain while reconnect sync is running')
+      assert.equal(serverRows.length, 0, 'offline row should not be pushed before sync is released')
+
+      releaseSync()
+      await createDone
+
+      for (let i = 0; i < 50 && clientApp.disconnectedDate; i++) {
+         await new Promise(resolve => setTimeout(resolve, 10))
+      }
+
+      assert.equal(clientApp.disconnectedDate, null, 'disconnect marker should clear after reconnect sync completes')
+      assert.deepEqual(serverRows, [{ uid: 'reconnect-1', label: 'from reconnect' }])
+
+      socket.disconnect()
+      await new Promise(resolve => serverApp2.io.close(resolve))
+   })
+
    test('updateWithMeta pub/sub handler tolerates undefined value (concurrent delete race)', async () => {
       // updateWithMeta does `const [value] = UPDATE ... RETURNING`.  If the model row
       // was deleted by a concurrent operation between the sync's findMany snapshot and
@@ -1115,6 +1247,132 @@ describe('Full offline-first client ↔ server protocol', () => {
       }
    })
 
+   test('sync.go serializes overlapping where scopes on the server', async () => {
+      const modelName = `model${++dbCounter}`
+      const { pglite, db, metaTable, modelTable } = await createTestDb(modelName)
+
+      await db.insert(modelTable).values({ uid: 'r1', label: 'x' })
+      await db.insert(metaTable).values({ uid: 'r1', created_at: T0 })
+
+      const serverApp = expressX({})
+      serverApp.configure(drizzleOfflinePlugin, db, metaTable, [modelTable])
+
+      const modelService = serverApp.service(modelName)
+      const originalFindMany = modelService.findMany
+      let releaseBroadFindMany
+      let broadFindManyEntered
+      let narrowFindManyEntered = false
+      const broadFindManyStarted = new Promise(resolve => { broadFindManyEntered = resolve })
+      const broadFindManyRelease = new Promise(resolve => { releaseBroadFindMany = resolve })
+
+      modelService.findMany = async (where) => {
+         if (Object.keys(where).length === 0) {
+            broadFindManyEntered()
+            await broadFindManyRelease
+         } else {
+            narrowFindManyEntered = true
+         }
+         return originalFindMany(where)
+      }
+
+      try {
+         const broadSync = serverApp.service('sync').go(modelName, {}, {})
+         await broadFindManyStarted
+
+         const narrowSync = serverApp.service('sync').go(modelName, { label: 'x' }, {})
+         await new Promise(resolve => setTimeout(resolve, 20))
+         assert.equal(narrowFindManyEntered, false, 'narrow overlapping sync must wait for broad sync')
+
+         releaseBroadFindMany()
+         await Promise.all([broadSync, narrowSync])
+         assert.equal(narrowFindManyEntered, true, 'narrow sync should run after broad sync releases')
+      } finally {
+         pglite.close()
+      }
+   })
+
+   test('getObservable syncs persisted where scopes on a new client session', async () => {
+      // If whereList was left in IndexedDB by a previous run, addSynchroDBWhere()
+      // returns false. getObservable() must still sync once for this app session;
+      // otherwise the first emission can be stale forever until a manual sync/reconnect.
+      const modelName = `model${++dbCounter}`
+      const { pglite, db, metaTable, modelTable } = await createTestDb(modelName)
+
+      await db.insert(modelTable).values({ uid: 'r1', label: 'from-server' })
+      await db.insert(metaTable).values({ uid: 'r1', created_at: T0 })
+
+      const { clientApp, cleanup } = await createTestContext(
+         serverApp => serverApp.configure(drizzleOfflinePlugin, db, metaTable, [modelTable]),
+         { useOfflinePlugin: true },
+      )
+
+      try {
+         const model = clientApp.createOfflineModel(modelName, ['label'])
+         await model.db.whereList.add({ sortedjson: '{}' })
+
+         const rows = await firstValueFrom(model.getObservable({}))
+
+         assert.deepEqual(rows.map(row => row.uid), ['r1'])
+      } finally {
+         await cleanup()
+         pglite.close()
+      }
+   })
+
+   test('initial connect syncs scopes registered while the client started offline', async () => {
+      const modelName = `model${++dbCounter}`
+      const { pglite, db, metaTable, modelTable } = await createTestDb(modelName)
+
+      await db.insert(modelTable).values({ uid: 'r1', label: 'from-server' })
+      await db.insert(metaTable).values({ uid: 'r1', created_at: T0 })
+
+      const serverApp = expressX({})
+      serverApp.configure(drizzleOfflinePlugin, db, metaTable, [modelTable])
+      await new Promise(resolve => serverApp.httpServer.listen(0, resolve))
+      const port = serverApp.httpServer.address().port
+
+      const socket = ioc(`http://localhost:${port}`, {
+         transports: ['websocket'],
+         autoConnect: false,
+      })
+      const clientApp = createClient(socket, { debug: false })
+      offlinePlugin(clientApp)
+      const model = clientApp.createOfflineModel(modelName, ['label'])
+      let subscription
+
+      try {
+         await model.db.whereList.add({ sortedjson: '{}' })
+
+         const observedServerRow = new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => reject(new Error('timed out waiting for synced row')), 1000)
+            subscription = model.getObservable({}).subscribe({
+               next: rows => {
+                  if (rows.some(row => row.uid === 'r1')) {
+                     clearTimeout(timeout)
+                     subscription.unsubscribe()
+                     resolve(rows)
+                  }
+               },
+               error: reject,
+            })
+         })
+
+         socket.connect()
+         await new Promise((resolve, reject) => {
+            socket.on('connect', resolve)
+            socket.on('connect_error', reject)
+         })
+
+         const rows = await observedServerRow
+         assert.deepEqual(rows.map(row => row.uid), ['r1'])
+      } finally {
+         if (subscription) subscription.unsubscribe()
+         socket.disconnect()
+         await new Promise(resolve => serverApp.io.close(resolve))
+         pglite.close()
+      }
+   })
+
    test('update() rollback clears stale updated_at from metadata', async () => {
       // Optimistic update sets updated_at = now in metadata before the server responds.
       // On rejection the rollback does db.metadata.update(uid, previousMetadata), but

package/test/sync.test.mjs

@@ -99,6 +99,26 @@ describe('computeSyncResult', () => {
       assert.deepEqual(result.deleteClient, [['g', T1]])
    })
 
+   test('record in both, client delete newer than DB update → delete on DB and notify client', () => {
+      const value      = { uid: 'h', label: 'server-old' }
+      const dbMeta     = { uid: 'h', created_at: T0, updated_at: T1 }
+      const clientMeta = { uid: 'h', created_at: T0, deleted_at: T2 }
+      const result = computeSyncResult({ h: value }, { h: clientMeta }, { h: dbMeta })
+      assert.deepEqual(result.deleteDatabase, ['h'])
+      assert.deepEqual(result.deleteClient, [['h', T2]])
+      assert.deepEqual(result.updateClient, [])
+   })
+
+   test('record in both, DB update newer than client delete → restore client from DB', () => {
+      const value      = { uid: 'i', label: 'server-new' }
+      const dbMeta     = { uid: 'i', created_at: T0, updated_at: T2 }
+      const clientMeta = { uid: 'i', created_at: T0, deleted_at: T1 }
+      const result = computeSyncResult({ i: value }, { i: clientMeta }, { i: dbMeta })
+      assert.deepEqual(result.updateClient, [[value, dbMeta]])
+      assert.deepEqual(result.deleteDatabase, [])
+      assert.deepEqual(result.deleteClient, [])
+   })
+
    test('mixed scenario: one of each case', () => {
       const dbOnly      = { uid: 'db', label: 'db-only' }
       const dbOnlyMeta  = { uid: 'db', created_at: T0 }