@jcbuisson/express-x-plugins 5.0.6 → 5.0.8

package/src/electric-offline-model.mjs

@@ -196,8 +196,9 @@ export function createOfflineElectricModel({ db, service, modelName, url, primar
             const mutation = rows[0]
             if (!mutation) break
             try {
-               await sendMutation(mutation)
-               lastError = null
+               const rejection = await sendMutation(mutation)
+               if (rejection) report(rejection)
+               else lastError = null
             } catch (error) {
                report(error)
                if (!isPermanent(error)) break
@@ -222,6 +223,7 @@ export function createOfflineElectricModel({ db, service, modelName, url, primar
             ? await service.update(id, withoutKey(data), metadata)
             : await service.delete(id, metadata)
       if (result?.version == null) throw new Error('sync mutation did not return a version')
+      let discarded = false
       await retryLeader(() => db.transaction(async tx => {
          const current = await tx.query('SELECT action, revision, data FROM electric_mutation_queue WHERE model_name = $1 AND row_id = $2', [modelName, id])
          const queued = current.rows[0]
@@ -229,6 +231,10 @@ export function createOfflineElectricModel({ db, service, modelName, url, primar
          if (String(queued.revision) === String(mutation.revision) && queued.action === action) {
             await tx.query(`UPDATE electric_mutation_queue SET acknowledged_version = $3::numeric
                WHERE model_name = $1 AND row_id = $2`, [modelName, id, String(result.version)])
+            if (result.deleted && action !== 'delete') {
+               await tx.query('DELETE FROM electric_local_rows WHERE model_name = $1 AND row_id = $2', [modelName, id])
+               discarded = true
+            }
          } else if (action === 'create' && queued.action === 'create') {
             // The create was sent, then edited locally. Its next request is an update.
             await tx.query(`UPDATE electric_mutation_queue SET action = 'update',
@@ -237,6 +243,12 @@ export function createOfflineElectricModel({ db, service, modelName, url, primar
       }))
       await reconcile()
       notify()
+      if (discarded) {
+         const error = new Error(result.syncRejection?.message ?? `Change to '${modelName}' discarded because the row was deleted`)
+         error.code = result.syncRejection?.code ?? 'sync-row-deleted'
+         error.rowId = id
+         return error
+      }
    }
 
    function withoutKey(data) {

package/src/electric-server-plugin.mjs

@@ -1,3 +1,5 @@
+import { discoverSyncRelations } from './electric-sync-relations.mjs'
+
 /**
  * Register Express-X mutation services and an Electric Shape proxy.
  *
@@ -14,6 +16,12 @@ export function electricServerPlugin(app, db, models, options = {}) {
    const configuredModels = normalizeModels(models)
    const sync = options.sync === true
    if (sync && typeof db.connect !== 'function') throw new TypeError('sync requires a pg Pool with connect()')
+   let relationsPromise
+   const getRelations = () => relationsPromise ??= discoverSyncRelations(db, configuredModels).catch(error => {
+      relationsPromise = undefined
+      throw error
+   })
+   const mutate = async (...args) => syncMutation(db, ...args, await getRelations())
    const electricUrl = new URL(options.electricUrl ?? process.env.ELECTRIC_URL ?? 'http://localhost:3000/v1/shape')
    const shapePath = options.shapePath ?? '/electric/v1/shape/:model'
    const fetchImpl = options.fetch ?? globalThis.fetch
@@ -70,7 +78,7 @@ export function electricServerPlugin(app, db, models, options = {}) {
                : { ...mutationData }
             if (sync) {
                if (!hasClientId) throw new TypeError('sync mutations require a client-generated id')
-               return syncMutation(db, model, 'create', idOrData, safeData, mutation)
+               return mutate(model, 'create', idOrData, safeData, mutation)
             }
             const entries = Object.entries(safeData).filter(([, value]) => value !== undefined)
             const columns = entries.map(([column]) => quoteIdentifier(column, 'data column'))
@@ -101,7 +109,7 @@ export function electricServerPlugin(app, db, models, options = {}) {
 
          update: async function(id, data, mutation) {
             await authorize(this, model.name, 'update', sync ? [id, data, mutation] : [id, data])
-            if (sync) return syncMutation(db, model, 'update', id, data, mutation)
+            if (sync) return mutate(model, 'update', id, data, mutation)
             const set = buildSet(data, model.primaryKey)
             return withTransaction(db, async client => {
                const result = await client.query(
@@ -114,7 +122,7 @@ export function electricServerPlugin(app, db, models, options = {}) {
 
          delete: async function(id, mutation) {
             await authorize(this, model.name, 'delete', sync ? [id, mutation] : [id])
-            if (sync) return syncMutation(db, model, 'delete', id, null, mutation)
+            if (sync) return mutate(model, 'delete', id, null, mutation)
             return withTransaction(db, async client => {
                const result = await client.query(
                   `DELETE FROM ${model.quotedTable} WHERE ${model.quotedPrimaryKey} = $1 RETURNING *`, [id],
@@ -166,9 +174,11 @@ export async function prepareElectricSyncSchema(db, models) {
       await db.query(`ALTER TABLE ${model.quotedTable} ADD COLUMN IF NOT EXISTS version BIGINT NOT NULL DEFAULT nextval('electric_sync_version_seq')`)
       await db.query(`ALTER TABLE ${model.quotedTable} ADD COLUMN IF NOT EXISTS deleted BOOLEAN NOT NULL DEFAULT false`)
    }
+   const relations = await discoverSyncRelations(db, configuredModels)
+   await withTransaction(db, tx => relations.repair(tx, writeTombstone))
 }
 
-async function syncMutation(db, model, action, id, data, mutation) {
+async function syncMutation(db, model, action, id, data, mutation, relations) {
    if (!mutation || typeof mutation !== 'object') throw new TypeError('sync mutation metadata is required')
    const { clientId, revision } = mutation
    if (typeof clientId !== 'string' || !UUID.test(clientId)) throw new TypeError('clientId must be a UUID')
@@ -179,6 +189,7 @@ async function syncMutation(db, model, action, id, data, mutation) {
    if (action !== 'delete') assertPlainObject(data, 'mutation data')
    if (data && ('version' in data || 'deleted' in data)) throw new TypeError('version and deleted are managed by sync')
    return withTransaction(db, async tx => {
+      await relations.lock(tx, model)
       const key = [model.table, clientId, String(id)]
       await tx.query(`INSERT INTO electric_mutation_cursor (table_name, client_id, row_id)
          VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`, key)
@@ -191,7 +202,10 @@ async function syncMutation(db, model, action, id, data, mutation) {
       const editedCreate = action === 'create' && BigInt(cursor.revision) > 0n
          && Object.entries(data).some(([column, value]) => column !== model.primaryKey && value !== undefined)
       let result
-      if (action === 'create' && !editedCreate) {
+      const rejection = await relations.deletedParent(tx, model, action, id, data, editedCreate)
+      if (rejection) {
+         result = { ...await writeTombstone(tx, model, id), syncRejection: rejection }
+      } else if (action === 'create' && !editedCreate) {
          const entries = Object.entries({ ...data, [model.primaryKey]: id }).filter(([, value]) => value !== undefined)
          const columns = entries.map(([column]) => quoteIdentifier(column, 'data column'))
          const values = entries.map(([, value]) => value)
@@ -208,6 +222,7 @@ async function syncMutation(db, model, action, id, data, mutation) {
       } else {
          result = await writeTombstone(tx, model, id)
       }
+      if (result?.deleted) await relations.cascade(tx, model, id, writeTombstone)
       await tx.query(`UPDATE electric_mutation_cursor SET revision = $4, result = $5::jsonb
          WHERE table_name = $1 AND client_id = $2 AND row_id = $3`, [...key, String(revision), JSON.stringify(result ?? null)])
       return result
@@ -253,7 +268,7 @@ function normalizeModels(models) {
       }
       const name = config.name
       const table = config.table ?? name
-      const primaryKey = config.primaryKey ?? 'id'
+      const primaryKey = config.primaryKey ?? 'uid'
       quoteIdentifier(name, 'model name')
       const tombstoneData = config.tombstoneData ?? {}
       if (typeof tombstoneData !== 'function') validateTombstoneData(tombstoneData, primaryKey)
@@ -321,6 +336,7 @@ function buildSet(data, primaryKey, startIndex = 1) {
 }
 
 async function withTransaction(db, operation) {
+   if (typeof db?.connect !== 'function' && typeof db?.transaction === 'function') return db.transaction(operation)
    if (typeof db?.connect !== 'function') return operation(db)
    const client = await db.connect()
    try {

package/src/electric-sync-relations.mjs

@@ -0,0 +1,99 @@
+// PostgreSQL foreign keys remain satisfied by soft-deleted rows. Mirror CASCADE
+// relationships at the sync boundary, where deletions must produce tombstones.
+export async function discoverSyncRelations(db, models) {
+   const { rows } = await db.query(`SELECT child.table_name AS child_table, parent.table_name AS parent_table,
+      ARRAY(SELECT attname::text FROM unnest(fk.conkey) WITH ORDINALITY AS key(attnum, position)
+         JOIN pg_attribute ON attrelid = fk.conrelid AND pg_attribute.attnum = key.attnum ORDER BY position) AS child_columns,
+      ARRAY(SELECT attname::text FROM unnest(fk.confkey) WITH ORDINALITY AS key(attnum, position)
+         JOIN pg_attribute ON attrelid = fk.confrelid AND pg_attribute.attnum = key.attnum ORDER BY position) AS parent_columns
+      FROM pg_constraint AS fk
+      JOIN unnest($1::text[]) AS child(table_name) ON fk.conrelid = to_regclass(child.table_name)
+      JOIN unnest($1::text[]) AS parent(table_name) ON fk.confrelid = to_regclass(parent.table_name)
+      WHERE fk.contype = 'f' AND fk.confdeltype = 'c'`, [models.map(model => model.quotedTable)])
+   const byTable = new Map(models.map(model => [model.quotedTable, model]))
+   const relations = rows.map(row => {
+      const child = byTable.get(row.child_table)
+      const parent = byTable.get(row.parent_table)
+      if (row.child_columns.length !== 1 || row.parent_columns.length !== 1 || row.parent_columns[0] !== parent.primaryKey) {
+         throw new TypeError('sync CASCADE foreign keys must reference a single configured primary key')
+      }
+      // Names come from PostgreSQL, not request data; still quote embedded quotes.
+      const column = row.child_columns[0]
+      return { child, parent, column, quotedColumn: `"${column.replaceAll('"', '""')}"` }
+   })
+   const members = new Set(relations.flatMap(({ child, parent }) => [child, parent]))
+
+   async function lock(tx, model) {
+      if (!members.size || model && !members.has(model)) return
+      // One transaction-scoped lock, acquired before any cursor/row locks. This
+      // serializes relational mutations across processes and handles cyclic
+      // graphs without lock-order inversions. Unrelated models remain parallel.
+      await tx.query('SELECT pg_advisory_xact_lock(178693771, 1)')
+   }
+
+   async function deletedParent(tx, model, action, id, data, editedCreate) {
+      const parents = relations.filter(relation => relation.child === model)
+      if (action === 'delete' || !parents.length) return null
+      const { rows } = await tx.query(`SELECT * FROM ${model.quotedTable} WHERE ${model.quotedPrimaryKey} = $1 FOR UPDATE`, [id])
+      const current = rows[0]
+      // A colliding create cannot overwrite or delete another client's row.
+      if (action === 'create' && !editedCreate && current) return null
+      if (action === 'update' || editedCreate) {
+         if (!current || current.deleted) return null
+      }
+      const changes = Object.fromEntries(Object.entries(data).filter(([column, value]) => column !== model.primaryKey && value !== undefined))
+      const proposed = { ...current, ...changes, [model.primaryKey]: id }
+      for (const relation of parents) {
+         const parentId = proposed[relation.column]
+         if (parentId == null) continue
+         const { parent } = relation
+         // PostgreSQL permits a newly inserted row to reference itself.
+         if (parent.table === model.table && String(parentId) === String(id)) continue
+         const result = await tx.query(`SELECT deleted FROM ${parent.quotedTable} WHERE ${parent.quotedPrimaryKey} = $1 FOR UPDATE`, [parentId])
+         if (!result.rows[0]) {
+            // The parent may still be queued on another model. Retry rather than
+            // discarding a legitimate offline parent+child creation.
+            const error = new Error(`Waiting for '${parent.name}' parent '${parentId}' to synchronize`)
+            error.code = 'sync-parent-missing'
+            throw error
+         }
+         if (result.rows[0].deleted) return {
+            code: 'sync-parent-deleted',
+            message: `Change to '${model.name}' discarded because '${parent.name}' parent '${parentId}' was deleted`,
+         }
+      }
+      return null
+   }
+
+   async function cascade(tx, model, id, writeTombstone, visited = new Set()) {
+      const key = JSON.stringify([model.table, id])
+      if (visited.has(key)) return
+      visited.add(key)
+      for (const relation of relations.filter(relation => relation.parent === model)) {
+         const { child } = relation
+         const { rows } = await tx.query(`SELECT ${child.quotedPrimaryKey} FROM ${child.quotedTable}
+            WHERE ${relation.quotedColumn} = $1 AND NOT deleted ORDER BY ${child.quotedPrimaryKey} FOR UPDATE`, [id])
+         for (const row of rows) {
+            const childId = row[child.primaryKey]
+            await writeTombstone(tx, child, childId)
+            await cascade(tx, child, childId, writeTombstone, visited)
+         }
+      }
+   }
+
+   async function repair(tx, writeTombstone) {
+      await lock(tx)
+      for (const { child, parent, quotedColumn } of relations) {
+         const { rows } = await tx.query(`SELECT child.${child.quotedPrimaryKey} FROM ${child.quotedTable} AS child
+            JOIN ${parent.quotedTable} AS parent ON child.${quotedColumn} = parent.${parent.quotedPrimaryKey}
+            WHERE NOT child.deleted AND parent.deleted ORDER BY child.${child.quotedPrimaryKey} FOR UPDATE OF child`)
+         for (const row of rows) {
+            const id = row[child.primaryKey]
+            await writeTombstone(tx, child, id)
+            await cascade(tx, child, id, writeTombstone)
+         }
+      }
+   }
+
+   return { lock, deletedParent, cascade, repair }
+}

package/src/sync-algorithm.mjs

@@ -0,0 +1,146 @@
+// Restored from express-x commit edb1db3 (last complete custom sync algorithm).
+
+// This sync algorithm uses the metadata table as its only source of truth for existence and recency. Manual SQL changes
+// create a split reality — the data table and the metadata table diverge — and the algorithm's set-intersection logic
+// misidentifies the divergence as intentional client-side changes that must be propagated.
+// If you need to seed or manipulate data outside the framework, you must also write the corresponding rows into the
+// metadata table with valid created_at/updated_at/deleted_at values.
+
+// The metadata table is a tombstone log that grows monotonically. Every record ever created leaves a permanent entry. For
+// long-lived applications with high churn this is worth planning for: you can only safely purge a tombstone once you're certain no
+// client still holds a copy of that record (i.e., every client has synced at least once after the deletion). There's currently no
+// pruning mechanism in the framework.
+
+export function computeSyncResult(databaseValuesDict, clientMetadataDict, databaseMetadataDict) {
+   assertValidMetadataTimestamps(clientMetadataDict, 'client')
+   assertValidMetadataTimestamps(databaseMetadataDict, 'database')
+   const onlyDatabaseIds = new Set()
+   const onlyClientIds = new Set()
+   const databaseAndClientIds = new Set()
+
+   for (const uid of Object.keys(databaseValuesDict)) {
+      if (Object.hasOwn(clientMetadataDict, uid)) databaseAndClientIds.add(uid)
+      else onlyDatabaseIds.add(uid)
+   }
+   for (const uid of Object.keys(clientMetadataDict)) {
+      if (Object.hasOwn(databaseValuesDict, uid)) databaseAndClientIds.add(uid)
+      else onlyClientIds.add(uid)
+   }
+
+   const addDatabase = [], updateDatabase = [], deleteDatabase = []
+   const addClient = [], updateClient = [], deleteClient = []
+
+   for (const uid of onlyDatabaseIds) {
+      const databaseMetaData = databaseMetadataDict[uid] || { uid, created_at: null }
+      if (databaseMetaData.deleted_at) {
+         deleteDatabase.push(uid)
+      } else {
+         addClient.push([databaseValuesDict[uid], databaseMetaData])
+      }
+   }
+
+   for (const uid of onlyClientIds) {
+      const clientMetaData = clientMetadataDict[uid]
+      const databaseMetaData = databaseMetadataDict[uid]
+      if (databaseMetaData?.deleted_at) {
+         const clientUpdatedAt = new Date(clientMetaData.deleted_at || clientMetaData.updated_at || clientMetaData.created_at)
+         const databaseDeletedAt = new Date(databaseMetaData.deleted_at)
+         if (databaseDeletedAt >= clientUpdatedAt) {
+            deleteClient.push([uid, databaseMetaData.deleted_at])
+         } else if (clientMetaData.deleted_at) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, clientMetaData.deleted_at])
+         } else {
+            addDatabase.push(clientMetaData)
+         }
+      } else if (databaseMetaData) {
+         const clientUpdatedAt = new Date(clientMetaData.deleted_at || clientMetaData.updated_at || clientMetaData.created_at)
+         const databaseUpdatedAt = new Date(databaseMetaData.updated_at || databaseMetaData.created_at)
+         if (databaseUpdatedAt >= clientUpdatedAt) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, databaseUpdatedAt])
+         } else if (clientMetaData.deleted_at) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, clientMetaData.deleted_at])
+         } else {
+            addDatabase.push(clientMetaData)
+         }
+      } else if (clientMetaData.deleted_at) {
+         deleteClient.push([uid, clientMetaData.deleted_at])
+      } else {
+         addDatabase.push(clientMetaData)
+      }
+   }
+
+   for (const uid of databaseAndClientIds) {
+      const clientMetaData = clientMetadataDict[uid]
+      const databaseMetaData = databaseMetadataDict[uid] || { uid, created_at: null }
+      if (databaseMetaData.deleted_at) {
+         const clientUpdatedAt = new Date(clientMetaData.deleted_at || clientMetaData.updated_at || clientMetaData.created_at)
+         const databaseDeletedAt = new Date(databaseMetaData.deleted_at)
+         if (databaseDeletedAt >= clientUpdatedAt) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, databaseMetaData.deleted_at])
+         } else if (clientMetaData.deleted_at) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, clientMetaData.deleted_at])
+         } else {
+            updateDatabase.push(clientMetaData)
+         }
+      } else if (clientMetaData.deleted_at) {
+         const clientDeletedAt = new Date(clientMetaData.deleted_at)
+         const databaseUpdatedAt = new Date(databaseMetaData.updated_at || databaseMetaData.created_at)
+         const diff = clientDeletedAt - databaseUpdatedAt
+         if (diff >= 0) {
+            deleteDatabase.push(uid)
+            deleteClient.push([uid, clientMetaData.deleted_at])
+         } else {
+            updateClient.push([databaseValuesDict[uid], databaseMetaData])
+         }
+      } else {
+         const clientUpdatedAt = new Date(clientMetaData.updated_at || clientMetaData.created_at)
+         const databaseUpdatedAt = new Date(databaseMetaData.updated_at || databaseMetaData.created_at)
+         const diff = clientUpdatedAt - databaseUpdatedAt
+         if (diff > 0) updateDatabase.push(clientMetaData)
+         else if (diff < 0) updateClient.push([databaseValuesDict[uid], databaseMetaData])
+      }
+   }
+
+   return {
+      addClient,
+      updateClient,
+      deleteClient,
+      addDatabase,
+      updateDatabase,
+      deleteDatabase,
+   }
+}
+
+function assertValidMetadataTimestamps(metadataDict, source) {
+   if (!metadataDict || typeof metadataDict !== 'object' || Array.isArray(metadataDict)) {
+      throw new TypeError(`${source} metadata must be an object`)
+   }
+   for (const [uid, metadata] of Object.entries(metadataDict)) {
+      if (!metadata || typeof metadata !== 'object') {
+         throw new TypeError(`${source} metadata for '${uid}' must be an object`)
+      }
+      if (metadata.uid !== uid) {
+         throw new TypeError(`${source} metadata key '${uid}' does not match uid '${metadata.uid}'`)
+      }
+      const timestampFields = ['created_at', 'updated_at', 'deleted_at']
+      // Timestamp-free database metadata can exist transiently while the
+      // Drizzle plugin locks/repairs a row. Client metadata must always carry
+      // a usable version.
+      if (source !== 'database' && !timestampFields.some(field => metadata[field] != null)) {
+         throw new TypeError(`${source} metadata '${uid}' must contain a timestamp`)
+      }
+      for (const field of timestampFields) {
+         const value = metadata[field]
+         if (value == null) continue
+         if (Number.isNaN(new Date(value).getTime())) {
+            throw new TypeError(`${source} metadata '${uid}.${field}' must be a valid timestamp`)
+         }
+      }
+   }
+}
+