@jcbuisson/express-x-plugins 5.0.7 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +180 -1
- package/docs/electric-fsm.md +244 -0
- package/docs/sync-fsm.md +117 -0
- package/package.json +34 -2
- package/src/electric-offline-model.mjs +14 -2
- package/src/electric-server-plugin.mjs +21 -5
- package/src/electric-sql-client-plugin.mjs +52 -0
- package/src/electric-sql-server-plugin.mjs +69 -0
- package/src/electric-sync-relations.mjs +99 -0
- package/src/sync-algorithm.mjs +146 -0
- package/src/sync-client-plugin.mjs +1023 -0
- package/src/sync-server-plugin.mjs +541 -0
- package/test/electric-relations-postgres.test.mjs +113 -0
- package/test/electric-relations.test.mjs +220 -0
- package/test/electric-server-plugin.test.mjs +3 -2
- package/test/electric-sql-plugin.test.mjs +74 -0
- package/test/sync-algorithm.test.mjs +229 -0
- package/test/sync-offline.test.mjs +4643 -0
|
@@ -0,0 +1,541 @@
|
|
|
1
|
+
// Restored from express-x-drizzle commit ddf6804.
|
|
2
|
+
import { and, eq, gt, gte, lt, lte, inArray, isNull, getTableName } from "drizzle-orm";
|
|
3
|
+
|
|
4
|
+
import { computeSyncResult } from './sync-algorithm.mjs'
|
|
5
|
+
|
|
6
|
+
export function truncateString(str, maxLength = 300, ellipsis = '...') {
|
|
7
|
+
// Check if the string already fits
|
|
8
|
+
if (str.length <= maxLength) return str;
|
|
9
|
+
// Calculate the cut-off point, accounting for the ellipsis length
|
|
10
|
+
const cutLength = maxLength - ellipsis.length;
|
|
11
|
+
// Ensure the string is long enough to be cut
|
|
12
|
+
if (cutLength < 0) return str.substring(0, maxLength); // Just cut it off if ellipsis doesn't fit
|
|
13
|
+
// Truncate the string and add the ellipsis
|
|
14
|
+
return str.substring(0, cutLength) + ellipsis;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
////////////////////////// UTILITIES //////////////////////////
|
|
22
|
+
|
|
23
|
+
function whereToDrizzleFilters(table, where) {
|
|
24
|
+
validateWhere(where)
|
|
25
|
+
const conditions = Object.entries(where)
|
|
26
|
+
.filter(([_, value]) => value !== undefined)
|
|
27
|
+
.map(([key, value]) => {
|
|
28
|
+
const column = table[key]
|
|
29
|
+
if (value === null) return isNull(column)
|
|
30
|
+
if (typeof value === 'object') {
|
|
31
|
+
// Collect ALL range bounds — a compound { gte:1, lte:10 } needs both
|
|
32
|
+
const bounds = []
|
|
33
|
+
if ('gte' in value) bounds.push(gte(column, coerceWhereValue(column, value.gte)))
|
|
34
|
+
if ('gt' in value) bounds.push(gt(column, coerceWhereValue(column, value.gt)))
|
|
35
|
+
if ('lte' in value) bounds.push(lte(column, coerceWhereValue(column, value.lte)))
|
|
36
|
+
if ('lt' in value) bounds.push(lt(column, coerceWhereValue(column, value.lt)))
|
|
37
|
+
if (bounds.length > 0) return and(...bounds)
|
|
38
|
+
}
|
|
39
|
+
return eq(column, coerceWhereValue(column, value))
|
|
40
|
+
})
|
|
41
|
+
return conditions.length ? and(...conditions) : undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function validateWhere(where) {
|
|
45
|
+
if (!where || typeof where !== 'object' || Array.isArray(where)
|
|
46
|
+
|| Object.prototype.toString.call(where) !== '[object Object]') {
|
|
47
|
+
throw new TypeError('where must be a plain object')
|
|
48
|
+
}
|
|
49
|
+
validateJsonFilterValue(where, 'where')
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function validateJsonFilterValue(value, path) {
|
|
53
|
+
if (value === undefined || typeof value === 'function'
|
|
54
|
+
|| typeof value === 'symbol' || typeof value === 'bigint') {
|
|
55
|
+
throw new TypeError(`${path} contains a non-serializable value`)
|
|
56
|
+
}
|
|
57
|
+
if (typeof value === 'number' && !Number.isFinite(value)) {
|
|
58
|
+
throw new TypeError(`${path} contains a non-finite number`)
|
|
59
|
+
}
|
|
60
|
+
if (value instanceof Date) {
|
|
61
|
+
if (Number.isNaN(value.getTime())) throw new TypeError(`${path} contains an invalid Date`)
|
|
62
|
+
return
|
|
63
|
+
}
|
|
64
|
+
if (Array.isArray(value)) {
|
|
65
|
+
value.forEach((entry, index) => validateJsonFilterValue(entry, `${path}[${index}]`))
|
|
66
|
+
return
|
|
67
|
+
}
|
|
68
|
+
if (value && typeof value === 'object') {
|
|
69
|
+
if (Object.prototype.toString.call(value) !== '[object Object]') {
|
|
70
|
+
throw new TypeError(`${path} contains a non-plain object`)
|
|
71
|
+
}
|
|
72
|
+
for (const [key, entry] of Object.entries(value)) validateJsonFilterValue(entry, `${path}.${key}`)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function coerceWhereValue(column, value) {
|
|
77
|
+
if (column?.columnType === 'PgTimestamp' && typeof value === 'string') return new Date(value)
|
|
78
|
+
return value
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function normalizeMutationTimestamp(value, field) {
|
|
82
|
+
const timestamp = new Date(value)
|
|
83
|
+
if (Number.isNaN(timestamp.getTime())) throw new TypeError(`${field} must be a valid timestamp`)
|
|
84
|
+
const now = Date.now()
|
|
85
|
+
const maxFutureSkew = 5 * 60 * 1000
|
|
86
|
+
return timestamp.getTime() > now + maxFutureSkew ? new Date(now) : timestamp
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function metadataStorageUid(modelName, uid) {
|
|
90
|
+
return JSON.stringify([modelName, uid])
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function externalizeMetadata(row, uid) {
|
|
94
|
+
return row ? { ...row, uid } : null
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function metadataTimestamp(meta, fields = ['deleted_at', 'updated_at', 'created_at']) {
|
|
98
|
+
if (!meta) return null
|
|
99
|
+
for (const field of fields) {
|
|
100
|
+
if (meta[field] != null) return new Date(meta[field])
|
|
101
|
+
}
|
|
102
|
+
return null
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function sanitizeMutationData(data) {
|
|
106
|
+
if (!data || typeof data !== 'object' || Array.isArray(data)) {
|
|
107
|
+
throw new TypeError('mutation data must be an object')
|
|
108
|
+
}
|
|
109
|
+
const { uid: _uid, __deleted__: _deleted, ...safeData } = data
|
|
110
|
+
return safeData
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function isPlainObject(value) {
|
|
114
|
+
return value && typeof value === 'object' && !Array.isArray(value) && Object.prototype.toString.call(value) === '[object Object]'
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function hasRangeOperator(value) {
|
|
118
|
+
return isPlainObject(value) && ['gte', 'gt', 'lte', 'lt'].some(key => key in value)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function valueWithinRange(value, range) {
|
|
122
|
+
if (value === null || value === undefined) return false
|
|
123
|
+
if ('gte' in range && compareWhereValues(value, range.gte) < 0) return false
|
|
124
|
+
if ('gt' in range && compareWhereValues(value, range.gt) <= 0) return false
|
|
125
|
+
if ('lte' in range && compareWhereValues(value, range.lte) > 0) return false
|
|
126
|
+
if ('lt' in range && compareWhereValues(value, range.lt) >= 0) return false
|
|
127
|
+
return true
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function rangesOverlap(a, b) {
|
|
131
|
+
const lower = [
|
|
132
|
+
'gte' in a && { value: a.gte, inclusive: true },
|
|
133
|
+
'gt' in a && { value: a.gt, inclusive: false },
|
|
134
|
+
'gte' in b && { value: b.gte, inclusive: true },
|
|
135
|
+
'gt' in b && { value: b.gt, inclusive: false },
|
|
136
|
+
].filter(Boolean).sort((x, y) => compareRangeBounds(y, x) || Number(x.inclusive) - Number(y.inclusive))[0]
|
|
137
|
+
|
|
138
|
+
const upper = [
|
|
139
|
+
'lte' in a && { value: a.lte, inclusive: true },
|
|
140
|
+
'lt' in a && { value: a.lt, inclusive: false },
|
|
141
|
+
'lte' in b && { value: b.lte, inclusive: true },
|
|
142
|
+
'lt' in b && { value: b.lt, inclusive: false },
|
|
143
|
+
].filter(Boolean).sort((x, y) => compareRangeBounds(x, y) || Number(x.inclusive) - Number(y.inclusive))[0]
|
|
144
|
+
|
|
145
|
+
if (!lower || !upper) return true
|
|
146
|
+
const diff = compareWhereValues(lower.value, upper.value)
|
|
147
|
+
if (diff < 0) return true
|
|
148
|
+
if (diff > 0) return false
|
|
149
|
+
return lower.inclusive && upper.inclusive
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function compareRangeBounds(a, b) {
|
|
153
|
+
return compareWhereValues(a.value, b.value)
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function compareWhereValues(a, b) {
|
|
157
|
+
if (a instanceof Date || b instanceof Date) {
|
|
158
|
+
const aTime = new Date(a).getTime()
|
|
159
|
+
const bTime = new Date(b).getTime()
|
|
160
|
+
if (!Number.isNaN(aTime) && !Number.isNaN(bTime)) return aTime - bTime
|
|
161
|
+
}
|
|
162
|
+
if (a === b) return 0
|
|
163
|
+
return a > b ? 1 : -1
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function constraintsOverlap(a, b) {
|
|
167
|
+
if (a === undefined || b === undefined) return true
|
|
168
|
+
if (a === null || b === null) return a === null && b === null
|
|
169
|
+
|
|
170
|
+
const aRange = hasRangeOperator(a)
|
|
171
|
+
const bRange = hasRangeOperator(b)
|
|
172
|
+
|
|
173
|
+
if (aRange && bRange) return rangesOverlap(a, b)
|
|
174
|
+
if (aRange) return valueWithinRange(b, a)
|
|
175
|
+
if (bRange) return valueWithinRange(a, b)
|
|
176
|
+
if (isPlainObject(a) || isPlainObject(b)) return true
|
|
177
|
+
return a === b
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function whereScopesOverlap(a = {}, b = {}) {
|
|
181
|
+
const sharedKeys = Object.keys(a).filter(key => Object.hasOwn(b, key))
|
|
182
|
+
for (const key of sharedKeys) {
|
|
183
|
+
if (!constraintsOverlap(a[key], b[key])) return false
|
|
184
|
+
}
|
|
185
|
+
return true
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
class OverlapLock {
|
|
189
|
+
constructor() {
|
|
190
|
+
this.active = []
|
|
191
|
+
this.queue = []
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
acquire(where) {
|
|
195
|
+
return new Promise(resolve => {
|
|
196
|
+
const entry = { where, resolve }
|
|
197
|
+
this.queue.push(entry)
|
|
198
|
+
this.pump()
|
|
199
|
+
})
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
pump() {
|
|
203
|
+
for (let i = 0; i < this.queue.length; i++) {
|
|
204
|
+
const entry = this.queue[i]
|
|
205
|
+
if (this.active.some(active => whereScopesOverlap(active.where, entry.where))) continue
|
|
206
|
+
// Preserve fairness: a later scope may bypass earlier disjoint scopes,
|
|
207
|
+
// but never an earlier queued scope that it overlaps.
|
|
208
|
+
if (this.queue.slice(0, i).some(queued => whereScopesOverlap(queued.where, entry.where))) continue
|
|
209
|
+
this.queue.splice(i, 1)
|
|
210
|
+
i -= 1
|
|
211
|
+
this.active.push(entry)
|
|
212
|
+
entry.resolve(() => {
|
|
213
|
+
this.active = this.active.filter(active => active !== entry)
|
|
214
|
+
this.pump()
|
|
215
|
+
})
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
get idle() {
|
|
220
|
+
return this.active.length === 0 && this.queue.length === 0
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
////////////////////////// DRIZZLE OFFLINE PLUGIN //////////////////////////
|
|
226
|
+
|
|
227
|
+
export function drizzleOfflinePlugin(app, db, metadata, models, options = {}) {
|
|
228
|
+
if (typeof options.authorize !== 'function') {
|
|
229
|
+
throw new TypeError('drizzleOfflinePlugin requires an authorize(context, operation) policy')
|
|
230
|
+
}
|
|
231
|
+
const computeSynchronization = options.computeSyncResult ?? computeSyncResult
|
|
232
|
+
const configuredModels = new Map(models.map(model => [getTableName(model), model]))
|
|
233
|
+
const storageUidFor = (modelName, uid) =>
|
|
234
|
+
models.length === 1 ? uid : metadataStorageUid(modelName, uid)
|
|
235
|
+
const maxSyncRecords = options.maxSyncRecords ?? 10_000
|
|
236
|
+
if (!Number.isInteger(maxSyncRecords) || maxSyncRecords < 1) {
|
|
237
|
+
throw new TypeError('maxSyncRecords must be a positive integer')
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function lockMetadataRow(tx, modelName, uid) {
|
|
241
|
+
const storageUid = storageUidFor(modelName, uid)
|
|
242
|
+
await tx.insert(metadata).values({ uid: storageUid }).onConflictDoNothing({ target: metadata.uid })
|
|
243
|
+
let row = (await tx.select().from(metadata).where(eq(metadata.uid, storageUid)).for('update'))[0]
|
|
244
|
+
|
|
245
|
+
return { storageUid, meta: externalizeMetadata(row, uid) }
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function lockUidOwnership(tx, uid) {
|
|
249
|
+
if (models.length === 1) return
|
|
250
|
+
const lockUid = JSON.stringify(['__uid_owner__', uid])
|
|
251
|
+
await tx.insert(metadata).values({ uid: lockUid }).onConflictDoNothing({ target: metadata.uid })
|
|
252
|
+
await tx.select({ uid: metadata.uid }).from(metadata).where(eq(metadata.uid, lockUid)).for('update')
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async function authorize(context, modelName, action, args) {
|
|
256
|
+
const allowed = await options.authorize(context, { modelName, action, args })
|
|
257
|
+
if (!allowed) {
|
|
258
|
+
const error = new Error(`not authorized to ${action} '${modelName}'`)
|
|
259
|
+
error.code = 'forbidden'
|
|
260
|
+
throw error
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function assertUidUniqueToModel(tx, currentModel, uid) {
|
|
265
|
+
for (const candidate of models) {
|
|
266
|
+
if (candidate === currentModel) continue
|
|
267
|
+
const row = (await tx.select({ uid: candidate.uid }).from(candidate).where(eq(candidate.uid, uid)))[0]
|
|
268
|
+
if (row) throw new Error(`uid '${uid}' already belongs to model '${getTableName(candidate)}'`)
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// add a database service for each model
|
|
273
|
+
for (const model of models) {
|
|
274
|
+
const modelName = getTableName(model)
|
|
275
|
+
|
|
276
|
+
app.createService(modelName, {
|
|
277
|
+
|
|
278
|
+
findUnique: async function(where) {
|
|
279
|
+
await authorize(this, modelName, 'findUnique', [where])
|
|
280
|
+
const rows = await db.select().from(model).where(whereToDrizzleFilters(model, where));
|
|
281
|
+
return rows[0] ?? null;
|
|
282
|
+
},
|
|
283
|
+
|
|
284
|
+
findMany: async function(where, queryOptions = {}) {
|
|
285
|
+
await authorize(this, modelName, 'findMany', [where, queryOptions])
|
|
286
|
+
let query = db.select().from(model).where(whereToDrizzleFilters(model, where))
|
|
287
|
+
if (queryOptions.limit != null) query = query.limit(queryOptions.limit)
|
|
288
|
+
return await query;
|
|
289
|
+
},
|
|
290
|
+
|
|
291
|
+
createWithMeta: async function(uid, data, created_at) {
|
|
292
|
+
await authorize(this, modelName, 'createWithMeta', [uid, data, created_at])
|
|
293
|
+
const safeData = sanitizeMutationData(data)
|
|
294
|
+
const ts = normalizeMutationTimestamp(created_at, 'created_at')
|
|
295
|
+
return await db.transaction(async (tx) => {
|
|
296
|
+
await lockUidOwnership(tx, uid)
|
|
297
|
+
const { storageUid, meta: existingMeta } = await lockMetadataRow(tx, modelName, uid)
|
|
298
|
+
await assertUidUniqueToModel(tx, model, uid)
|
|
299
|
+
const existingTime = metadataTimestamp(existingMeta)
|
|
300
|
+
if (existingTime && existingTime >= ts) {
|
|
301
|
+
const value = (await tx.select().from(model).where(eq(model.uid, uid)))[0] ?? undefined
|
|
302
|
+
if (existingMeta.deleted_at) {
|
|
303
|
+
if (value) await tx.delete(model).where(eq(model.uid, uid))
|
|
304
|
+
return [undefined, existingMeta]
|
|
305
|
+
}
|
|
306
|
+
if (!value && !existingMeta.deleted_at) {
|
|
307
|
+
const [meta] = await tx.insert(metadata)
|
|
308
|
+
.values({ uid: storageUid, deleted_at: existingTime })
|
|
309
|
+
.onConflictDoUpdate({ target: metadata.uid, set: { deleted_at: existingTime } })
|
|
310
|
+
.returning();
|
|
311
|
+
return [undefined, externalizeMetadata(meta, uid)]
|
|
312
|
+
}
|
|
313
|
+
return [value, existingMeta]
|
|
314
|
+
}
|
|
315
|
+
// Upsert: if the model row already exists (e.g. a concurrent createWithMeta
|
|
316
|
+
// from the direct create() path landed before the sync's addDatabase step),
|
|
317
|
+
// update it instead of throwing a PK conflict that would rollback the
|
|
318
|
+
// client's Dexie record.
|
|
319
|
+
const [value] = await tx.insert(model)
|
|
320
|
+
.values({ ...safeData, uid })
|
|
321
|
+
.onConflictDoUpdate({ target: model.uid, set: safeData })
|
|
322
|
+
.returning();
|
|
323
|
+
// Upsert metadata: handles re-creation after a prior deleteWithMeta.
|
|
324
|
+
const [meta] = await tx.insert(metadata)
|
|
325
|
+
.values({ uid: storageUid, created_at: ts })
|
|
326
|
+
.onConflictDoUpdate({
|
|
327
|
+
target: metadata.uid,
|
|
328
|
+
set: { created_at: ts, deleted_at: null, updated_at: null },
|
|
329
|
+
})
|
|
330
|
+
.returning();
|
|
331
|
+
return [value, externalizeMetadata(meta, uid)]
|
|
332
|
+
})
|
|
333
|
+
},
|
|
334
|
+
|
|
335
|
+
updateWithMeta: async function(uid, data, updated_at) {
|
|
336
|
+
await authorize(this, modelName, 'updateWithMeta', [uid, data, updated_at])
|
|
337
|
+
const safeData = sanitizeMutationData(data)
|
|
338
|
+
const ts = updated_at ? normalizeMutationTimestamp(updated_at, 'updated_at') : null
|
|
339
|
+
return await db.transaction(async (tx) => {
|
|
340
|
+
await lockUidOwnership(tx, uid)
|
|
341
|
+
const { storageUid, meta: existingMeta } = await lockMetadataRow(tx, modelName, uid)
|
|
342
|
+
await assertUidUniqueToModel(tx, model, uid)
|
|
343
|
+
if (existingMeta?.deleted_at) {
|
|
344
|
+
const deletedAt = new Date(existingMeta.deleted_at)
|
|
345
|
+
if (ts && ts > deletedAt) {
|
|
346
|
+
const [value] = await tx.insert(model)
|
|
347
|
+
.values({ ...safeData, uid })
|
|
348
|
+
.onConflictDoUpdate({ target: model.uid, set: safeData })
|
|
349
|
+
.returning();
|
|
350
|
+
const [meta] = await tx.insert(metadata)
|
|
351
|
+
.values({ uid: storageUid, updated_at: ts })
|
|
352
|
+
.onConflictDoUpdate({ target: metadata.uid, set: { updated_at: ts, deleted_at: null } })
|
|
353
|
+
.returning();
|
|
354
|
+
return [value, externalizeMetadata(meta, uid)]
|
|
355
|
+
}
|
|
356
|
+
await tx.delete(model).where(eq(model.uid, uid))
|
|
357
|
+
return [undefined, existingMeta]
|
|
358
|
+
}
|
|
359
|
+
const existingTime = metadataTimestamp(existingMeta, ['updated_at', 'created_at'])
|
|
360
|
+
if (existingTime && (!ts || existingTime >= ts)) {
|
|
361
|
+
const value = (await tx.select().from(model).where(eq(model.uid, uid)))[0] ?? undefined
|
|
362
|
+
if (!value) {
|
|
363
|
+
const [meta] = await tx.insert(metadata)
|
|
364
|
+
.values({ uid: storageUid, deleted_at: existingTime })
|
|
365
|
+
.onConflictDoUpdate({ target: metadata.uid, set: { deleted_at: existingTime } })
|
|
366
|
+
.returning();
|
|
367
|
+
return [undefined, externalizeMetadata(meta, uid)]
|
|
368
|
+
}
|
|
369
|
+
return [value, existingMeta]
|
|
370
|
+
}
|
|
371
|
+
const [value] = await tx.insert(model)
|
|
372
|
+
.values({ ...safeData, uid })
|
|
373
|
+
.onConflictDoUpdate({ target: model.uid, set: safeData })
|
|
374
|
+
.returning();
|
|
375
|
+
// Upsert metadata: if metadata is missing (data-integrity gap where the
|
|
376
|
+
// model row exists but no metadata row), create it so the loop stops.
|
|
377
|
+
const [meta] = await tx.insert(metadata)
|
|
378
|
+
.values({ uid: storageUid, updated_at: ts })
|
|
379
|
+
.onConflictDoUpdate({ target: metadata.uid, set: { updated_at: ts, deleted_at: null } })
|
|
380
|
+
.returning();
|
|
381
|
+
return [value, externalizeMetadata(meta, uid)]
|
|
382
|
+
})
|
|
383
|
+
},
|
|
384
|
+
|
|
385
|
+
deleteWithMeta: async function(uid, deleted_at) {
|
|
386
|
+
await authorize(this, modelName, 'deleteWithMeta', [uid, deleted_at])
|
|
387
|
+
return await db.transaction(async (tx) => {
|
|
388
|
+
await lockUidOwnership(tx, uid)
|
|
389
|
+
const { storageUid, meta: existingMeta } = await lockMetadataRow(tx, modelName, uid)
|
|
390
|
+
await assertUidUniqueToModel(tx, model, uid)
|
|
391
|
+
const ts = normalizeMutationTimestamp(deleted_at, 'deleted_at')
|
|
392
|
+
const existingTime = metadataTimestamp(existingMeta)
|
|
393
|
+
if (existingTime && existingTime > ts) {
|
|
394
|
+
const value = (await tx.select().from(model).where(eq(model.uid, uid)))[0] ?? undefined
|
|
395
|
+
if (existingMeta.deleted_at) {
|
|
396
|
+
if (value) await tx.delete(model).where(eq(model.uid, uid))
|
|
397
|
+
return [undefined, existingMeta]
|
|
398
|
+
}
|
|
399
|
+
if (!value && !existingMeta.deleted_at) {
|
|
400
|
+
const [meta] = await tx.insert(metadata)
|
|
401
|
+
.values({ uid: storageUid, deleted_at: existingTime })
|
|
402
|
+
.onConflictDoUpdate({ target: metadata.uid, set: { deleted_at: existingTime } })
|
|
403
|
+
.returning();
|
|
404
|
+
return [undefined, externalizeMetadata(meta, uid)]
|
|
405
|
+
}
|
|
406
|
+
return [value, existingMeta]
|
|
407
|
+
}
|
|
408
|
+
const [value] = await tx.delete(model).where(eq(model.uid, uid)).returning();
|
|
409
|
+
const [meta] = await tx.insert(metadata)
|
|
410
|
+
.values({ uid: storageUid, deleted_at: ts })
|
|
411
|
+
.onConflictDoUpdate({ target: metadata.uid, set: { deleted_at: ts } })
|
|
412
|
+
.returning();
|
|
413
|
+
return [value, externalizeMetadata(meta, uid)]
|
|
414
|
+
})
|
|
415
|
+
},
|
|
416
|
+
})
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const syncLocks = new Map()
|
|
420
|
+
|
|
421
|
+
// add a synchronization service
|
|
422
|
+
app.createService('sync', {
|
|
423
|
+
|
|
424
|
+
go: async function(modelName, where, clientMetadataDict) {
|
|
425
|
+
if (!configuredModels.has(modelName)) throw new Error(`model '${modelName}' is not configured for synchronization`)
|
|
426
|
+
await authorize(this, modelName, 'sync', [where, clientMetadataDict])
|
|
427
|
+
if (!clientMetadataDict || typeof clientMetadataDict !== 'object' || Array.isArray(clientMetadataDict)) {
|
|
428
|
+
throw new TypeError('client metadata must be an object')
|
|
429
|
+
}
|
|
430
|
+
const clientUids = Object.keys(clientMetadataDict)
|
|
431
|
+
if (clientUids.length > maxSyncRecords) {
|
|
432
|
+
throw new Error(`sync metadata exceeds maxSyncRecords (${maxSyncRecords})`)
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// overlap-aware lock so independent scopes can still run in parallel, but overlapping where predicates do not
|
|
436
|
+
if (!syncLocks.has(modelName)) syncLocks.set(modelName, new OverlapLock())
|
|
437
|
+
const syncLock = syncLocks.get(modelName)
|
|
438
|
+
const releaseSyncLock = await syncLock.acquire(where)
|
|
439
|
+
|
|
440
|
+
try {
|
|
441
|
+
console.log('>>>>> SYNC', modelName, where)
|
|
442
|
+
const databaseService = app.service(modelName)
|
|
443
|
+
|
|
444
|
+
// STEP1: get existing database `where` values and build a dictionary
|
|
445
|
+
const databaseValues = await databaseService.callWithContext(
|
|
446
|
+
this, 'findMany', where, { limit: maxSyncRecords + 1 },
|
|
447
|
+
)
|
|
448
|
+
if (databaseValues.length > maxSyncRecords) {
|
|
449
|
+
throw new Error(`sync result exceeds maxSyncRecords (${maxSyncRecords})`)
|
|
450
|
+
}
|
|
451
|
+
const databaseValuesDict = databaseValues.reduce((accu, value) => {
|
|
452
|
+
accu[value.uid] = value
|
|
453
|
+
return accu
|
|
454
|
+
}, Object.create(null))
|
|
455
|
+
|
|
456
|
+
// STEP 2: fetch metadata for each database record and each client-sent uid.
|
|
457
|
+
// Client uids matter when the server row was deleted: only the tombstone remains.
|
|
458
|
+
const effectiveClientMetadata = Object.assign(Object.create(null), clientMetadataDict)
|
|
459
|
+
|
|
460
|
+
// A client-supplied uid whose real row exists outside `where` must not
|
|
461
|
+
// participate in this scoped sync. Treating it as a missing row can
|
|
462
|
+
// otherwise schedule an out-of-scope deletion.
|
|
463
|
+
const candidateOutsideUids = clientUids.filter(uid => !Object.hasOwn(databaseValuesDict, uid))
|
|
464
|
+
if (candidateOutsideUids.length > 0) {
|
|
465
|
+
const outsideRows = await db.select({ uid: configuredModels.get(modelName).uid })
|
|
466
|
+
.from(configuredModels.get(modelName))
|
|
467
|
+
.where(inArray(configuredModels.get(modelName).uid, candidateOutsideUids))
|
|
468
|
+
for (const row of outsideRows) delete effectiveClientMetadata[row.uid]
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
const databaseMetadataDict = Object.create(null)
|
|
472
|
+
const metadataUids = new Set([
|
|
473
|
+
...Object.keys(databaseValuesDict),
|
|
474
|
+
...Object.keys(effectiveClientMetadata),
|
|
475
|
+
])
|
|
476
|
+
const metadataUidList = [...metadataUids]
|
|
477
|
+
if (metadataUidList.length > 0) {
|
|
478
|
+
const storageToUid = new Map(metadataUidList.map(uid => [storageUidFor(modelName, uid), uid]))
|
|
479
|
+
const rows = await db.select().from(metadata)
|
|
480
|
+
.where(inArray(metadata.uid, [...storageToUid.keys()]))
|
|
481
|
+
for (const row of rows) {
|
|
482
|
+
const uid = storageToUid.get(row.uid)
|
|
483
|
+
databaseMetadataDict[uid] = externalizeMetadata(row, uid)
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
}
|
|
487
|
+
const metadataRepairTimestamp = new Date()
|
|
488
|
+
for (const uid of Object.keys(databaseValuesDict)) {
|
|
489
|
+
if (databaseMetadataDict[uid]) continue
|
|
490
|
+
if (Object.hasOwn(effectiveClientMetadata, uid)
|
|
491
|
+
&& !effectiveClientMetadata[uid]?.__repaired__) continue
|
|
492
|
+
|
|
493
|
+
const [meta] = await db.insert(metadata)
|
|
494
|
+
.values({ uid: storageUidFor(modelName, uid), created_at: metadataRepairTimestamp })
|
|
495
|
+
.onConflictDoUpdate({
|
|
496
|
+
target: metadata.uid,
|
|
497
|
+
set: { created_at: metadataRepairTimestamp, updated_at: null, deleted_at: null },
|
|
498
|
+
})
|
|
499
|
+
.returning()
|
|
500
|
+
databaseMetadataDict[uid] = externalizeMetadata(meta, uid)
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// STEP 3: compute sync result
|
|
504
|
+
const result = computeSynchronization(databaseValuesDict, effectiveClientMetadata, databaseMetadataDict)
|
|
505
|
+
|
|
506
|
+
// STEP 4: execute server-side deletions
|
|
507
|
+
for (const uid of result.deleteDatabase) {
|
|
508
|
+
const deletedAt = effectiveClientMetadata[uid]?.deleted_at
|
|
509
|
+
?? databaseMetadataDict[uid]?.deleted_at
|
|
510
|
+
?? databaseMetadataDict[uid]?.updated_at
|
|
511
|
+
?? databaseMetadataDict[uid]?.created_at
|
|
512
|
+
await databaseService.callWithContext(this, 'deleteWithMeta', uid, deletedAt)
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
console.log('addDatabase', truncateString(JSON.stringify(result.addDatabase)))
|
|
516
|
+
console.log('updateDatabase', truncateString(JSON.stringify(result.updateDatabase)))
|
|
517
|
+
console.log('addClient', truncateString(JSON.stringify(result.addClient)))
|
|
518
|
+
console.log('deleteClient', truncateString(JSON.stringify(result.deleteClient)))
|
|
519
|
+
console.log('updateClient', truncateString(JSON.stringify(result.updateClient)))
|
|
520
|
+
|
|
521
|
+
return {
|
|
522
|
+
addClient: result.addClient,
|
|
523
|
+
updateClient: result.updateClient,
|
|
524
|
+
deleteClient: result.deleteClient,
|
|
525
|
+
addDatabase: result.addDatabase,
|
|
526
|
+
updateDatabase: result.updateDatabase,
|
|
527
|
+
}
|
|
528
|
+
} catch(err) {
|
|
529
|
+
console.log('*** err sync', err)
|
|
530
|
+
throw err
|
|
531
|
+
} finally {
|
|
532
|
+
releaseSyncLock()
|
|
533
|
+
if (syncLock.idle) syncLocks.delete(modelName)
|
|
534
|
+
}
|
|
535
|
+
},
|
|
536
|
+
})
|
|
537
|
+
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
export { computeSyncResult, drizzleOfflinePlugin as syncServerPlugin }
|
|
541
|
+
export default drizzleOfflinePlugin
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import test from 'node:test'
|
|
3
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
4
|
+
import { electricServerPlugin, prepareElectricSyncSchema } from '../src/electric-server-plugin.mjs'
|
|
5
|
+
|
|
6
|
+
const connectionString = process.env.ELECTRIC_TEST_DATABASE_URL
|
|
7
|
+
|
|
8
|
+
function register(db, models) {
|
|
9
|
+
const services = new Map()
|
|
10
|
+
electricServerPlugin({ createService(name, methods) { services.set(name, methods) }, get() {} }, db, models, { sync: true })
|
|
11
|
+
return services
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function commitGate(pool) {
|
|
15
|
+
let reached
|
|
16
|
+
let release
|
|
17
|
+
const waiting = new Promise(resolve => { reached = resolve })
|
|
18
|
+
const released = new Promise(resolve => { release = resolve })
|
|
19
|
+
return {
|
|
20
|
+
waiting, release,
|
|
21
|
+
db: {
|
|
22
|
+
query: (...args) => pool.query(...args),
|
|
23
|
+
async connect() {
|
|
24
|
+
const client = await pool.connect()
|
|
25
|
+
return {
|
|
26
|
+
async query(sql, values) {
|
|
27
|
+
if (sql === 'COMMIT') { reached(); await released }
|
|
28
|
+
return client.query(sql, values)
|
|
29
|
+
},
|
|
30
|
+
release: () => client.release(),
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
test('PostgreSQL serializes concurrent membership writes and soft deletes across plugin instances', {
|
|
38
|
+
skip: !connectionString,
|
|
39
|
+
timeout: 30_000,
|
|
40
|
+
}, async t => {
|
|
41
|
+
const { Pool } = await import('pg')
|
|
42
|
+
const schema = `electric_sync_test_${crypto.randomUUID().replaceAll('-', '')}`
|
|
43
|
+
const admin = new Pool({ connectionString })
|
|
44
|
+
let pool
|
|
45
|
+
t.after(async () => {
|
|
46
|
+
try {
|
|
47
|
+
await pool?.end()
|
|
48
|
+
await admin.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`)
|
|
49
|
+
} finally { await admin.end() }
|
|
50
|
+
})
|
|
51
|
+
await admin.query(`CREATE SCHEMA "${schema}"`)
|
|
52
|
+
pool = new Pool({ connectionString, options: `-c search_path=${schema}`, statement_timeout: 10_000 })
|
|
53
|
+
await pool.query(`
|
|
54
|
+
CREATE TABLE people (uid UUID PRIMARY KEY);
|
|
55
|
+
CREATE TABLE groups (uid UUID PRIMARY KEY);
|
|
56
|
+
CREATE TABLE membership (
|
|
57
|
+
uid UUID PRIMARY KEY,
|
|
58
|
+
user_uid UUID REFERENCES people(uid) ON DELETE CASCADE,
|
|
59
|
+
group_uid UUID REFERENCES groups(uid) ON DELETE CASCADE
|
|
60
|
+
);
|
|
61
|
+
`)
|
|
62
|
+
const models = [
|
|
63
|
+
{ name: 'user', table: 'people' },
|
|
64
|
+
{ name: 'group', table: 'groups' },
|
|
65
|
+
{ name: 'membership', tombstoneData: { user_uid: null, group_uid: null } },
|
|
66
|
+
]
|
|
67
|
+
await prepareElectricSyncSchema(pool, models)
|
|
68
|
+
const services = register(pool, models)
|
|
69
|
+
const metadata = () => ({ clientId: crypto.randomUUID(), revision: '1' })
|
|
70
|
+
|
|
71
|
+
for (const first of ['delete', 'create']) {
|
|
72
|
+
await t.test(`${first} reaches the server first`, async () => {
|
|
73
|
+
const user = crypto.randomUUID()
|
|
74
|
+
const group = crypto.randomUUID()
|
|
75
|
+
const membership = crypto.randomUUID()
|
|
76
|
+
await services.get('user').create(user, {}, metadata())
|
|
77
|
+
await services.get('group').create(group, {}, metadata())
|
|
78
|
+
const gate = commitGate(pool)
|
|
79
|
+
const gated = register(gate.db, models)
|
|
80
|
+
const create = service => service.get('membership').create(membership, { user_uid: user, group_uid: group }, metadata())
|
|
81
|
+
const remove = service => service.get('group').delete(group, metadata())
|
|
82
|
+
const firstWrite = first === 'delete' ? remove(gated) : create(gated)
|
|
83
|
+
let secondWrite
|
|
84
|
+
try {
|
|
85
|
+
// Keep the first transaction open after its mutation. The second
|
|
86
|
+
// must wait for the same advisory lock on another connection.
|
|
87
|
+
await Promise.race([gate.waiting, firstWrite.then(() => { throw new Error('commit gate was not reached') })])
|
|
88
|
+
secondWrite = first === 'delete' ? create(services) : remove(services)
|
|
89
|
+
let blocked = false
|
|
90
|
+
for (let attempt = 0; attempt < 100; attempt++) {
|
|
91
|
+
const { rows } = await pool.query(`SELECT EXISTS (
|
|
92
|
+
SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND NOT granted
|
|
93
|
+
AND classid = 178693771 AND objid = 1
|
|
94
|
+
AND database = (SELECT oid FROM pg_database WHERE datname = current_database())
|
|
95
|
+
) AS blocked`)
|
|
96
|
+
if (rows[0].blocked) { blocked = true; break }
|
|
97
|
+
await delay(10)
|
|
98
|
+
}
|
|
99
|
+
assert.equal(blocked, true, 'the second mutation must block until the first commits')
|
|
100
|
+
} finally {
|
|
101
|
+
gate.release()
|
|
102
|
+
await Promise.allSettled([firstWrite, secondWrite])
|
|
103
|
+
}
|
|
104
|
+
const results = await Promise.all([firstWrite, secondWrite])
|
|
105
|
+
if (first === 'delete') assert.equal(results[1].syncRejection.code, 'sync-parent-deleted')
|
|
106
|
+
const row = (await pool.query('SELECT * FROM membership WHERE uid = $1', [membership])).rows[0]
|
|
107
|
+
assert.equal(row.deleted, true)
|
|
108
|
+
assert.equal(row.user_uid, null)
|
|
109
|
+
assert.equal(row.group_uid, null)
|
|
110
|
+
assert.equal((await pool.query('SELECT deleted FROM groups WHERE uid = $1', [group])).rows[0].deleted, true)
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
})
|