@jcbuisson/express-x-plugins 5.0.2 → 5.0.4

package/README.md

@@ -13,7 +13,7 @@ ALTER SYSTEM SET max_replication_slots = 10;
 ALTER SYSTEM SET max_wal_senders = 10;
 ALTER ROLE chris WITH REPLICATION;
 
-(restart postgres: `sudo systemctl restart postgresql`)
+(restart postgres)
 
 ## Use HTTP2
 nginx: `listen 443 ssl http2;`
@@ -100,6 +100,29 @@ rows. Configure `tombstoneData` for required columns without defaults, such as a
 required title. Keep the cursor table and tombstones to protect delayed retries and
 offline clients. This does not resolve concurrent edits by different clients.
 
+`tombstoneData` can also be a function receiving `{ id }` and returning a plain
+object (or a promise of one). It runs inside the mutation transaction whenever a
+tombstone is written, including an update targeting a missing or deleted row.
+Replayed or stale revisions return the saved result without calling it again.
+For a unique, required email column, derive a dummy value from the row's UUID:
+
+```js
+{
+  name: 'user',
+  primaryKey: 'uid',
+  tombstoneData: ({ id }) => ({
+    email: `deleted-${id}@tombstone.invalid`,
+    name: '',
+    color: '',
+  }),
+}
+```
+
+The returned keys must be valid column identifiers and cannot include the primary
+key, `version`, or `deleted`. Reserve the dummy email pattern for tombstones so
+active users cannot occupy those values. A deterministic value remains stable
+across repeated deletions; a function can also generate a fresh random value.
+
 Import `prepareElectricSyncSchema` alongside `electricServerPlugin`. It adds
 `version` and `deleted` columns and creates the shared version sequence and cursor
 table. The primary key must be UUID in sync mode. All writes to synced tables must
@@ -239,11 +262,9 @@ can pass plugin-level `localDb` to retain automatic ownership management.
 
 ### Explicit offline models
 
-For offline writes, install `@electric-sql/pglite`, pass a shared PGlite database
-as `localDb`, and use the sync
-server mode above. `PGliteWorker` with an IndexedDB data directory is recommended
-for multiple tabs. Only one worker should use `ownsSync: true`; use a shared
-`BroadcastChannel` so other tabs receive change notifications.
+For offline writes, install `@electric-sql/pglite`, pass a shared PGlite database as `localDb`, and use the sync server mode above.
+`PGliteWorker` with an IndexedDB data directory is recommended for multiple tabs.
+Only one worker should use `ownsSync: true`; use a shared `BroadcastChannel` so other tabs receive change notifications.
 
 ```js
 const todo = app.createElectricModel('todos', {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jcbuisson/express-x-plugins",
-  "version": "5.0.2",
+  "version": "5.0.4",
   "description": "Plugins for express-x",
   "type": "module",
   "main": "./src/electric-server-plugin.mjs",

package/src/electric-offline-model.mjs

@@ -158,20 +158,23 @@ export function createOfflineElectricModel({ db, service, modelName, url, primar
 
    async function reconcileSnapshot(rows) {
       await retryLeader(() => db.transaction(async tx => {
-         for (const row of rows) {
-            const id = String(row[primaryKey])
-            if (row.version != null) {
-               await tx.query(`DELETE FROM electric_mutation_queue
-                  WHERE model_name = $1 AND row_id = $2 AND acknowledged_version <= $3::numeric`,
-               [modelName, id, String(row.version)])
-            }
-            if (row.deleted) continue
-            await tx.query(`INSERT INTO electric_local_rows (model_name, row_id, data)
-               SELECT $1, $2, $3::jsonb WHERE NOT EXISTS
-                  (SELECT 1 FROM electric_mutation_queue WHERE model_name = $1 AND row_id = $2)
-               ON CONFLICT (model_name, row_id) DO UPDATE SET data = EXCLUDED.data`,
-            [modelName, id, json(row)])
-         }
+         // Send the snapshot once instead of making two worker round trips per row.
+         const incoming = json(rows)
+         await tx.query(`DELETE FROM electric_mutation_queue AS pending
+            USING jsonb_array_elements($2::jsonb) AS incoming(data)
+            WHERE pending.model_name = $1
+              AND pending.row_id = incoming.data->>$3
+              AND pending.acknowledged_version <= (incoming.data->>'version')::numeric`,
+         [modelName, incoming, primaryKey])
+         await tx.query(`INSERT INTO electric_local_rows (model_name, row_id, data)
+            SELECT $1, incoming.data->>$3, incoming.data
+            FROM jsonb_array_elements($2::jsonb) AS incoming(data)
+            WHERE NOT COALESCE((incoming.data->>'deleted')::boolean, false)
+              AND NOT EXISTS (SELECT 1 FROM electric_mutation_queue AS pending
+                 WHERE pending.model_name = $1 AND pending.row_id = incoming.data->>$3)
+            ON CONFLICT (model_name, row_id) DO UPDATE SET data = EXCLUDED.data
+            WHERE electric_local_rows.data IS DISTINCT FROM EXCLUDED.data`,
+         [modelName, incoming, primaryKey])
          const activeIds = rows.filter(row => !row.deleted).map(row => String(row[primaryKey]))
          await tx.query(`DELETE FROM electric_local_rows AS local
             WHERE local.model_name = $1 AND NOT (local.row_id = ANY($2::text[]))

package/src/electric-server-plugin.mjs

@@ -210,7 +210,11 @@ async function syncMutation(db, model, action, id, data, mutation) {
 }
 
 async function writeTombstone(tx, model, id) {
-   const tombstone = Object.entries(model.tombstoneData)
+   const data = typeof model.tombstoneData === 'function'
+      ? await model.tombstoneData({ id })
+      : model.tombstoneData
+   validateTombstoneData(data, model.primaryKey)
+   const tombstone = Object.entries(data)
    const columns = [model.quotedPrimaryKey, ...tombstone.map(([column]) => quoteIdentifier(column, 'tombstone column')), 'deleted']
    const values = [id, ...tombstone.map(([, value]) => value)]
    const parameters = values.map((_, index) => `$${index + 1}`)
@@ -247,11 +251,7 @@ function normalizeModels(models) {
       const primaryKey = config.primaryKey ?? 'id'
       quoteIdentifier(name, 'model name')
       const tombstoneData = config.tombstoneData ?? {}
-      assertPlainObject(tombstoneData, 'tombstoneData')
-      for (const column of Object.keys(tombstoneData)) {
-         quoteIdentifier(column, 'tombstone column')
-         if ([primaryKey, 'version', 'deleted'].includes(column)) throw new TypeError('tombstoneData contains a sync-managed column')
-      }
+      if (typeof tombstoneData !== 'function') validateTombstoneData(tombstoneData, primaryKey)
       return {
          name,
          table,
@@ -263,6 +263,14 @@ function normalizeModels(models) {
    })
 }
 
+function validateTombstoneData(data, primaryKey) {
+   assertPlainObject(data, 'tombstoneData')
+   for (const column of Object.keys(data)) {
+      quoteIdentifier(column, 'tombstone column')
+      if ([primaryKey, 'version', 'deleted'].includes(column)) throw new TypeError('tombstoneData contains a sync-managed column')
+   }
+}
+
 function assertPlainObject(value, label) {
    if (!value || typeof value !== 'object' || Array.isArray(value) || Object.prototype.toString.call(value) !== '[object Object]') {
       throw new TypeError(`${label} must be a plain object`)

package/test/electric-offline-model.test.mjs

@@ -66,3 +66,43 @@ test('offline writes stay local until the acknowledged Electric version arrives'
       await db.close()
    }
 })
+
+test('large snapshots use three queries and preserve bigint versions and tombstones', async () => {
+   const db = new PGlite()
+   const { createOfflineElectricModel } = await import('../src/electric-offline-model.mjs')
+   let queries = 0
+   const countedDb = {
+      exec: (...args) => db.exec(...args),
+      query: (...args) => db.query(...args),
+      transaction: callback => db.transaction(tx => callback({
+         query: (...args) => { queries++; return tx.query(...args) },
+      })),
+   }
+   const model = createOfflineElectricModel({
+      db: countedDb, service: {}, modelName: 'range', primaryKey: 'uid',
+      ShapeStreamClass: Stream, ShapeClass: Shape, ownsSync: true,
+      network: { onLine: true }, retryMs: 60_000,
+   })
+   try {
+      await model.prepare()
+      model.start()
+      await model.waitForIdle()
+      const rows = Array.from({ length: 100 }, () => ({
+         uid: crypto.randomUUID(), label: 'shared', version: '9007199254740993', deleted: false,
+      }))
+      Shape.current.emit({ rows })
+      await model.waitForIdle()
+      assert.equal(queries, 3)
+      assert.equal((await model.findMany()).length, 100)
+      assert.equal((await model.findMany())[0].version, '9007199254740993')
+      queries = 0
+      Shape.current.emit({ rows: [{ ...rows[0], deleted: true }, { ...rows[1], label: 'changed' }] })
+      await model.waitForIdle()
+      assert.equal(queries, 3)
+      assert.deepEqual(await model.findMany(), [{ ...rows[1], label: 'changed' }])
+   } finally {
+      model.stop()
+      await model.waitForIdle()
+      await db.close()
+   }
+})

package/test/electric-server-plugin.test.mjs

@@ -223,3 +223,81 @@ test('sync delete writes a versioned tombstone and clears configured fields', as
    assert.match(tombstone.sql, /version = nextval/)
    assert.deepEqual(tombstone.values, [id, ''])
 })
+
+
+test('sync tombstone factories generate per-row values and skip replayed revisions', async () => {
+   const queries = []
+   const cursors = new Map()
+   const calls = []
+   const client = {
+      async query(sql, values = []) {
+         queries.push({ sql, values })
+         if (sql.includes('SELECT revision, result FROM electric_mutation_cursor')) {
+            return { rows: [cursors.get(values[2]) ?? { revision: '0', result: null }] }
+         }
+         if (sql.startsWith('UPDATE "users"')) return { rows: [] }
+         if (sql.includes('INSERT INTO "users"')) {
+            return { rows: [{ uid: values[0], email: values[1], deleted: true, version: '7' }] }
+         }
+         if (sql.includes('UPDATE electric_mutation_cursor')) {
+            cursors.set(values[2], { revision: values[3], result: JSON.parse(values[4]) })
+         }
+         return { rows: [] }
+      },
+      release() {},
+   }
+   const db = { query: client.query.bind(client), async connect() { return client } }
+   const services = new Map()
+   const app = { createService(name, methods) { services.set(name, methods) }, get() {} }
+   electricServerPlugin(app, db, [{
+      name: 'users', primaryKey: 'uid',
+      tombstoneData: async ({ id }) => {
+         calls.push(id)
+         return { email: `deleted-${id}@tombstone.invalid` }
+      },
+   }], { sync: true })
+   const service = services.get('users')
+   const first = '64d76168-775f-481e-8974-18d31d835d9e'
+   const second = '64d76168-775f-481e-8974-18d31d835d9f'
+   const metadata = { clientId: '5e199e1c-23a4-473e-9989-14d4a1fb857e', revision: '2' }
+   const result = await service.delete.call({}, first, metadata)
+   assert.deepEqual(await service.delete.call({}, first, metadata), result)
+   assert.deepEqual(await service.delete.call({}, first, { ...metadata, revision: '1' }), result)
+   await service.update.call({}, second, { name: 'missing' }, metadata)
+   await service.delete.call({}, first, { ...metadata, revision: '3' })
+   assert.deepEqual(calls, [first, second, first])
+   assert.deepEqual(queries.filter(({ sql }) => sql.includes('INSERT INTO "users"')).map(({ values }) => values), [
+      [first, `deleted-${first}@tombstone.invalid`],
+      [second, `deleted-${second}@tombstone.invalid`],
+      [first, `deleted-${first}@tombstone.invalid`],
+   ])
+})
+
+test('invalid tombstone factory results roll back before writing a tombstone', async () => {
+   for (const [data, message] of [
+      [null, /plain object/],
+      [[], /plain object/],
+      [{ 'email; DROP TABLE users': '' }, /simple SQL identifier/],
+      [{ uid: 'override' }, /sync-managed column/],
+      [{ version: '0' }, /sync-managed column/],
+      [{ deleted: false }, /sync-managed column/],
+   ]) {
+      const queries = []
+      const client = {
+         async query(sql) {
+            queries.push(sql)
+            return { rows: sql.includes('SELECT revision, result') ? [{ revision: '0', result: null }] : [] }
+         },
+         release() {},
+      }
+      const services = new Map()
+      electricServerPlugin({ createService(name, methods) { services.set(name, methods) }, get() {} },
+         { query: client.query.bind(client), async connect() { return client } },
+         [{ name: 'users', primaryKey: 'uid', tombstoneData: () => data }], { sync: true })
+      await assert.rejects(services.get('users').delete.call({}, '64d76168-775f-481e-8974-18d31d835d9e', {
+         clientId: '5e199e1c-23a4-473e-9989-14d4a1fb857e', revision: '1',
+      }), message)
+      assert.ok(queries.includes('ROLLBACK'))
+      assert.ok(!queries.some(sql => sql.includes('INSERT INTO "users"')))
+   }
+})