@tanstack/electric-db-collection 0.1.30 → 0.1.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/electric.cjs +169 -31
- package/dist/cjs/electric.cjs.map +1 -1
- package/dist/cjs/electric.d.cts +116 -7
- package/dist/cjs/errors.cjs +8 -24
- package/dist/cjs/errors.cjs.map +1 -1
- package/dist/cjs/errors.d.cts +2 -5
- package/dist/cjs/index.cjs +2 -3
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/esm/electric.d.ts +116 -7
- package/dist/esm/electric.js +166 -33
- package/dist/esm/electric.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -5
- package/dist/esm/errors.js +8 -24
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/index.js +3 -4
- package/package.json +2 -2
- package/src/electric.ts +369 -65
- package/src/errors.ts +6 -22
package/dist/esm/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sources":["../../src/errors.ts"],"sourcesContent":["import { TanStackDBError } from \"@tanstack/db\"\n\n// Electric DB Collection Errors\nexport class ElectricDBCollectionError extends TanStackDBError {\n constructor(message: string, collectionId?: string) {\n super(`${collectionId ? `[${collectionId}] ` : ``}${message}`)\n this.name = `ElectricDBCollectionError`\n }\n}\n\nexport class ExpectedNumberInAwaitTxIdError extends ElectricDBCollectionError {\n constructor(txIdType: string, collectionId?: string) {\n super(`Expected number in awaitTxId, received ${txIdType}`, collectionId)\n this.name = `ExpectedNumberInAwaitTxIdError`\n }\n}\n\nexport class TimeoutWaitingForTxIdError extends ElectricDBCollectionError {\n constructor(txId: number, collectionId?: string) {\n super(`Timeout waiting for txId: ${txId}`, collectionId)\n this.name = `TimeoutWaitingForTxIdError`\n }\n}\n\nexport class
|
|
1
|
+
{"version":3,"file":"errors.js","sources":["../../src/errors.ts"],"sourcesContent":["import { TanStackDBError } from \"@tanstack/db\"\n\n// Electric DB Collection Errors\nexport class ElectricDBCollectionError extends TanStackDBError {\n constructor(message: string, collectionId?: string) {\n super(`${collectionId ? `[${collectionId}] ` : ``}${message}`)\n this.name = `ElectricDBCollectionError`\n }\n}\n\nexport class ExpectedNumberInAwaitTxIdError extends ElectricDBCollectionError {\n constructor(txIdType: string, collectionId?: string) {\n super(`Expected number in awaitTxId, received ${txIdType}`, collectionId)\n this.name = `ExpectedNumberInAwaitTxIdError`\n }\n}\n\nexport class TimeoutWaitingForTxIdError extends ElectricDBCollectionError {\n constructor(txId: number, collectionId?: string) {\n super(`Timeout waiting for txId: ${txId}`, collectionId)\n this.name = `TimeoutWaitingForTxIdError`\n }\n}\n\nexport class TimeoutWaitingForMatchError extends ElectricDBCollectionError {\n constructor(collectionId?: string) {\n super(`Timeout waiting for custom match function`, collectionId)\n this.name = `TimeoutWaitingForMatchError`\n }\n}\n\nexport class StreamAbortedError extends ElectricDBCollectionError {\n constructor(collectionId?: string) {\n super(`Stream aborted`, collectionId)\n this.name = `StreamAbortedError`\n }\n}\n"],"names":[],"mappings":";AAGO,MAAM,kCAAkC,gBAAgB;AAAA,EAC7D,YAAY,SAAiB,cAAuB;AAClD,UAAM,GAAG,eAAe,IAAI,YAAY,OAAO,EAAE,GAAG,OAAO,EAAE;AAC7D,SAAK,OAAO;AAAA,EACd;AACF;AAEO,MAAM,uCAAuC,0BAA0B;AAAA,EAC5E,YAAY,UAAkB,cAAuB;AACnD,UAAM,0CAA0C,QAAQ,IAAI,YAAY;AACxE,SAAK,OAAO;AAAA,EACd;AACF;AAEO,MAAM,mCAAmC,0BAA0B;AAAA,EACxE,YAAY,MAAc,cAAuB;AAC/C,UAAM,6BAA6B,IAAI,IAAI,YAAY;AACvD,SAAK,OAAO;AAAA,EACd;AACF;AAEO,MAAM,oCAAoC,0BAA0B;AAAA,EACzE,YAAY,cAAuB;AACjC,UAAM,6CAA6C,YAAY;AAC/D,SAAK,OAAO;AAAA,EACd;AACF;AAEO,MAAM,2BAA2B,0BAA0B;AAAA,EAChE,YAAY,cAAuB;AACjC,UAAM,kBAAkB,YAAY;AACpC,SAAK,OAAO;AAAA,EACd;AACF;"}
|
package/dist/esm/index.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { electricCollectionOptions } from "./electric.js";
|
|
2
|
-
import { ElectricDBCollectionError,
|
|
2
|
+
import { ElectricDBCollectionError, ExpectedNumberInAwaitTxIdError, StreamAbortedError, TimeoutWaitingForMatchError, TimeoutWaitingForTxIdError } from "./errors.js";
|
|
3
3
|
export {
|
|
4
4
|
ElectricDBCollectionError,
|
|
5
|
-
ElectricDeleteHandlerMustReturnTxIdError,
|
|
6
|
-
ElectricInsertHandlerMustReturnTxIdError,
|
|
7
|
-
ElectricUpdateHandlerMustReturnTxIdError,
|
|
8
5
|
ExpectedNumberInAwaitTxIdError,
|
|
6
|
+
StreamAbortedError,
|
|
7
|
+
TimeoutWaitingForMatchError,
|
|
9
8
|
TimeoutWaitingForTxIdError,
|
|
10
9
|
electricCollectionOptions
|
|
11
10
|
};
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tanstack/electric-db-collection",
|
|
3
3
|
"description": "ElectricSQL collection for TanStack DB",
|
|
4
|
-
"version": "0.1.
|
|
4
|
+
"version": "0.1.31",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@electric-sql/client": "^1.0.14",
|
|
7
7
|
"@standard-schema/spec": "^1.0.0",
|
|
8
8
|
"@tanstack/store": "^0.7.7",
|
|
9
9
|
"debug": "^4.4.3",
|
|
10
|
-
"@tanstack/db": "0.4.
|
|
10
|
+
"@tanstack/db": "0.4.7"
|
|
11
11
|
},
|
|
12
12
|
"devDependencies": {
|
|
13
13
|
"@types/debug": "^4.1.12",
|
package/src/electric.ts
CHANGED
|
@@ -7,17 +7,15 @@ import {
|
|
|
7
7
|
import { Store } from "@tanstack/store"
|
|
8
8
|
import DebugModule from "debug"
|
|
9
9
|
import {
|
|
10
|
-
ElectricDeleteHandlerMustReturnTxIdError,
|
|
11
|
-
ElectricInsertHandlerMustReturnTxIdError,
|
|
12
|
-
ElectricUpdateHandlerMustReturnTxIdError,
|
|
13
10
|
ExpectedNumberInAwaitTxIdError,
|
|
11
|
+
StreamAbortedError,
|
|
12
|
+
TimeoutWaitingForMatchError,
|
|
14
13
|
TimeoutWaitingForTxIdError,
|
|
15
14
|
} from "./errors"
|
|
16
15
|
import type {
|
|
17
16
|
BaseCollectionConfig,
|
|
18
17
|
CollectionConfig,
|
|
19
18
|
DeleteMutationFnParams,
|
|
20
|
-
Fn,
|
|
21
19
|
InsertMutationFnParams,
|
|
22
20
|
SyncConfig,
|
|
23
21
|
UpdateMutationFnParams,
|
|
@@ -33,6 +31,9 @@ import type {
|
|
|
33
31
|
ShapeStreamOptions,
|
|
34
32
|
} from "@electric-sql/client"
|
|
35
33
|
|
|
34
|
+
// Re-export for user convenience in custom match functions
|
|
35
|
+
export { isChangeMessage, isControlMessage } from "@electric-sql/client"
|
|
36
|
+
|
|
36
37
|
const debug = DebugModule.debug(`ts/db:electric`)
|
|
37
38
|
|
|
38
39
|
/**
|
|
@@ -41,14 +42,20 @@ const debug = DebugModule.debug(`ts/db:electric`)
|
|
|
41
42
|
export type Txid = number
|
|
42
43
|
|
|
43
44
|
/**
|
|
44
|
-
*
|
|
45
|
+
* Custom match function type - receives stream messages and returns boolean
|
|
46
|
+
* indicating if the mutation has been synchronized
|
|
45
47
|
*/
|
|
46
|
-
type
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
export type MatchFunction<T extends Row<unknown>> = (
|
|
49
|
+
message: Message<T>
|
|
50
|
+
) => boolean
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Matching strategies for Electric synchronization
|
|
54
|
+
* Handlers can return:
|
|
55
|
+
* - Txid strategy: { txid: number | number[] } (recommended)
|
|
56
|
+
* - Void (no return value) - mutation completes without waiting
|
|
57
|
+
*/
|
|
58
|
+
export type MatchingStrategy = { txid: Txid | Array<Txid> } | void
|
|
52
59
|
|
|
53
60
|
/**
|
|
54
61
|
* Type representing a snapshot end message
|
|
@@ -56,7 +63,6 @@ type MaybeTxId =
|
|
|
56
63
|
type SnapshotEndMessage = ControlMessage & {
|
|
57
64
|
headers: { control: `snapshot-end` }
|
|
58
65
|
}
|
|
59
|
-
|
|
60
66
|
// The `InferSchemaOutput` and `ResolveType` are copied from the `@tanstack/db` package
|
|
61
67
|
// but we modified `InferSchemaOutput` slightly to restrict the schema output to `Row<unknown>`
|
|
62
68
|
// This is needed in order for `GetExtensions` to be able to infer the parser extensions type from the schema
|
|
@@ -74,17 +80,109 @@ type InferSchemaOutput<T> = T extends StandardSchemaV1
|
|
|
74
80
|
export interface ElectricCollectionConfig<
|
|
75
81
|
T extends Row<unknown> = Row<unknown>,
|
|
76
82
|
TSchema extends StandardSchemaV1 = never,
|
|
77
|
-
> extends
|
|
78
|
-
T,
|
|
79
|
-
|
|
80
|
-
TSchema,
|
|
81
|
-
Record<string, Fn>,
|
|
82
|
-
{ txid: Txid | Array<Txid> }
|
|
83
|
+
> extends Omit<
|
|
84
|
+
BaseCollectionConfig<T, string | number, TSchema, UtilsRecord, any>,
|
|
85
|
+
`onInsert` | `onUpdate` | `onDelete`
|
|
83
86
|
> {
|
|
84
87
|
/**
|
|
85
88
|
* Configuration options for the ElectricSQL ShapeStream
|
|
86
89
|
*/
|
|
87
90
|
shapeOptions: ShapeStreamOptions<GetExtensions<T>>
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Optional asynchronous handler function called before an insert operation
|
|
94
|
+
* @param params Object containing transaction and collection information
|
|
95
|
+
* @returns Promise resolving to { txid } or void
|
|
96
|
+
* @example
|
|
97
|
+
* // Basic Electric insert handler with txid (recommended)
|
|
98
|
+
* onInsert: async ({ transaction }) => {
|
|
99
|
+
* const newItem = transaction.mutations[0].modified
|
|
100
|
+
* const result = await api.todos.create({
|
|
101
|
+
* data: newItem
|
|
102
|
+
* })
|
|
103
|
+
* return { txid: result.txid }
|
|
104
|
+
* }
|
|
105
|
+
*
|
|
106
|
+
* @example
|
|
107
|
+
* // Insert handler with multiple items - return array of txids
|
|
108
|
+
* onInsert: async ({ transaction }) => {
|
|
109
|
+
* const items = transaction.mutations.map(m => m.modified)
|
|
110
|
+
* const results = await Promise.all(
|
|
111
|
+
* items.map(item => api.todos.create({ data: item }))
|
|
112
|
+
* )
|
|
113
|
+
* return { txid: results.map(r => r.txid) }
|
|
114
|
+
* }
|
|
115
|
+
*
|
|
116
|
+
* @example
|
|
117
|
+
* // Use awaitMatch utility for custom matching
|
|
118
|
+
* onInsert: async ({ transaction, collection }) => {
|
|
119
|
+
* const newItem = transaction.mutations[0].modified
|
|
120
|
+
* await api.todos.create({ data: newItem })
|
|
121
|
+
* await collection.utils.awaitMatch(
|
|
122
|
+
* (message) => isChangeMessage(message) &&
|
|
123
|
+
* message.headers.operation === 'insert' &&
|
|
124
|
+
* message.value.name === newItem.name
|
|
125
|
+
* )
|
|
126
|
+
* }
|
|
127
|
+
*/
|
|
128
|
+
onInsert?: (params: InsertMutationFnParams<T>) => Promise<MatchingStrategy>
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Optional asynchronous handler function called before an update operation
|
|
132
|
+
* @param params Object containing transaction and collection information
|
|
133
|
+
* @returns Promise resolving to { txid } or void
|
|
134
|
+
* @example
|
|
135
|
+
* // Basic Electric update handler with txid (recommended)
|
|
136
|
+
* onUpdate: async ({ transaction }) => {
|
|
137
|
+
* const { original, changes } = transaction.mutations[0]
|
|
138
|
+
* const result = await api.todos.update({
|
|
139
|
+
* where: { id: original.id },
|
|
140
|
+
* data: changes
|
|
141
|
+
* })
|
|
142
|
+
* return { txid: result.txid }
|
|
143
|
+
* }
|
|
144
|
+
*
|
|
145
|
+
* @example
|
|
146
|
+
* // Use awaitMatch utility for custom matching
|
|
147
|
+
* onUpdate: async ({ transaction, collection }) => {
|
|
148
|
+
* const { original, changes } = transaction.mutations[0]
|
|
149
|
+
* await api.todos.update({ where: { id: original.id }, data: changes })
|
|
150
|
+
* await collection.utils.awaitMatch(
|
|
151
|
+
* (message) => isChangeMessage(message) &&
|
|
152
|
+
* message.headers.operation === 'update' &&
|
|
153
|
+
* message.value.id === original.id
|
|
154
|
+
* )
|
|
155
|
+
* }
|
|
156
|
+
*/
|
|
157
|
+
onUpdate?: (params: UpdateMutationFnParams<T>) => Promise<MatchingStrategy>
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Optional asynchronous handler function called before a delete operation
|
|
161
|
+
* @param params Object containing transaction and collection information
|
|
162
|
+
* @returns Promise resolving to { txid } or void
|
|
163
|
+
* @example
|
|
164
|
+
* // Basic Electric delete handler with txid (recommended)
|
|
165
|
+
* onDelete: async ({ transaction }) => {
|
|
166
|
+
* const mutation = transaction.mutations[0]
|
|
167
|
+
* const result = await api.todos.delete({
|
|
168
|
+
* id: mutation.original.id
|
|
169
|
+
* })
|
|
170
|
+
* return { txid: result.txid }
|
|
171
|
+
* }
|
|
172
|
+
*
|
|
173
|
+
* @example
|
|
174
|
+
* // Use awaitMatch utility for custom matching
|
|
175
|
+
* onDelete: async ({ transaction, collection }) => {
|
|
176
|
+
* const mutation = transaction.mutations[0]
|
|
177
|
+
* await api.todos.delete({ id: mutation.original.id })
|
|
178
|
+
* await collection.utils.awaitMatch(
|
|
179
|
+
* (message) => isChangeMessage(message) &&
|
|
180
|
+
* message.headers.operation === 'delete' &&
|
|
181
|
+
* message.value.id === mutation.original.id
|
|
182
|
+
* )
|
|
183
|
+
* }
|
|
184
|
+
*/
|
|
185
|
+
onDelete?: (params: DeleteMutationFnParams<T>) => Promise<MatchingStrategy>
|
|
88
186
|
}
|
|
89
187
|
|
|
90
188
|
function isUpToDateMessage<T extends Row<unknown>>(
|
|
@@ -125,11 +223,21 @@ function hasTxids<T extends Row<unknown>>(
|
|
|
125
223
|
*/
|
|
126
224
|
export type AwaitTxIdFn = (txId: Txid, timeout?: number) => Promise<boolean>
|
|
127
225
|
|
|
226
|
+
/**
|
|
227
|
+
* Type for the awaitMatch utility function
|
|
228
|
+
*/
|
|
229
|
+
export type AwaitMatchFn<T extends Row<unknown>> = (
|
|
230
|
+
matchFn: MatchFunction<T>,
|
|
231
|
+
timeout?: number
|
|
232
|
+
) => Promise<boolean>
|
|
233
|
+
|
|
128
234
|
/**
|
|
129
235
|
* Electric collection utilities type
|
|
130
236
|
*/
|
|
131
|
-
export interface ElectricCollectionUtils extends
|
|
237
|
+
export interface ElectricCollectionUtils<T extends Row<unknown> = Row<unknown>>
|
|
238
|
+
extends UtilsRecord {
|
|
132
239
|
awaitTxId: AwaitTxIdFn
|
|
240
|
+
awaitMatch: AwaitMatchFn<T>
|
|
133
241
|
}
|
|
134
242
|
|
|
135
243
|
/**
|
|
@@ -173,21 +281,72 @@ export function electricCollectionOptions(
|
|
|
173
281
|
} {
|
|
174
282
|
const seenTxids = new Store<Set<Txid>>(new Set([]))
|
|
175
283
|
const seenSnapshots = new Store<Array<PostgresSnapshot>>([])
|
|
284
|
+
const pendingMatches = new Store<
|
|
285
|
+
Map<
|
|
286
|
+
string,
|
|
287
|
+
{
|
|
288
|
+
matchFn: (message: Message<any>) => boolean
|
|
289
|
+
resolve: (value: boolean) => void
|
|
290
|
+
reject: (error: Error) => void
|
|
291
|
+
timeoutId: ReturnType<typeof setTimeout>
|
|
292
|
+
matched: boolean
|
|
293
|
+
}
|
|
294
|
+
>
|
|
295
|
+
>(new Map())
|
|
296
|
+
|
|
297
|
+
// Buffer messages since last up-to-date to handle race conditions
|
|
298
|
+
const currentBatchMessages = new Store<Array<Message<any>>>([])
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Helper function to remove multiple matches from the pendingMatches store
|
|
302
|
+
*/
|
|
303
|
+
const removePendingMatches = (matchIds: Array<string>) => {
|
|
304
|
+
if (matchIds.length > 0) {
|
|
305
|
+
pendingMatches.setState((current) => {
|
|
306
|
+
const newMatches = new Map(current)
|
|
307
|
+
matchIds.forEach((id) => newMatches.delete(id))
|
|
308
|
+
return newMatches
|
|
309
|
+
})
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Helper function to resolve and cleanup matched pending matches
|
|
315
|
+
*/
|
|
316
|
+
const resolveMatchedPendingMatches = () => {
|
|
317
|
+
const matchesToResolve: Array<string> = []
|
|
318
|
+
pendingMatches.state.forEach((match, matchId) => {
|
|
319
|
+
if (match.matched) {
|
|
320
|
+
clearTimeout(match.timeoutId)
|
|
321
|
+
match.resolve(true)
|
|
322
|
+
matchesToResolve.push(matchId)
|
|
323
|
+
debug(
|
|
324
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch resolved on up-to-date for match %s`,
|
|
325
|
+
matchId
|
|
326
|
+
)
|
|
327
|
+
}
|
|
328
|
+
})
|
|
329
|
+
removePendingMatches(matchesToResolve)
|
|
330
|
+
}
|
|
176
331
|
const sync = createElectricSync<any>(config.shapeOptions, {
|
|
177
332
|
seenTxids,
|
|
178
333
|
seenSnapshots,
|
|
334
|
+
pendingMatches,
|
|
335
|
+
currentBatchMessages,
|
|
336
|
+
removePendingMatches,
|
|
337
|
+
resolveMatchedPendingMatches,
|
|
179
338
|
collectionId: config.id,
|
|
180
339
|
})
|
|
181
340
|
|
|
182
341
|
/**
|
|
183
342
|
* Wait for a specific transaction ID to be synced
|
|
184
343
|
* @param txId The transaction ID to wait for as a number
|
|
185
|
-
* @param timeout Optional timeout in milliseconds (defaults to
|
|
344
|
+
* @param timeout Optional timeout in milliseconds (defaults to 5000ms)
|
|
186
345
|
* @returns Promise that resolves when the txId is synced
|
|
187
346
|
*/
|
|
188
347
|
const awaitTxId: AwaitTxIdFn = async (
|
|
189
348
|
txId: Txid,
|
|
190
|
-
timeout: number =
|
|
349
|
+
timeout: number = 5000
|
|
191
350
|
): Promise<boolean> => {
|
|
192
351
|
debug(
|
|
193
352
|
`${config.id ? `[${config.id}] ` : ``}awaitTxId called with txid %d`,
|
|
@@ -246,49 +405,128 @@ export function electricCollectionOptions(
|
|
|
246
405
|
})
|
|
247
406
|
}
|
|
248
407
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
408
|
+
/**
|
|
409
|
+
* Wait for a custom match function to find a matching message
|
|
410
|
+
* @param matchFn Function that returns true when a message matches
|
|
411
|
+
* @param timeout Optional timeout in milliseconds (defaults to 5000ms)
|
|
412
|
+
* @returns Promise that resolves when a matching message is found
|
|
413
|
+
*/
|
|
414
|
+
const awaitMatch: AwaitMatchFn<any> = async (
|
|
415
|
+
matchFn: MatchFunction<any>,
|
|
416
|
+
timeout: number = 3000
|
|
417
|
+
): Promise<boolean> => {
|
|
418
|
+
debug(
|
|
419
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch called with custom function`
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
return new Promise((resolve, reject) => {
|
|
423
|
+
const matchId = Math.random().toString(36)
|
|
424
|
+
|
|
425
|
+
const cleanupMatch = () => {
|
|
426
|
+
pendingMatches.setState((current) => {
|
|
427
|
+
const newMatches = new Map(current)
|
|
428
|
+
newMatches.delete(matchId)
|
|
429
|
+
return newMatches
|
|
430
|
+
})
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
const onTimeout = () => {
|
|
434
|
+
cleanupMatch()
|
|
435
|
+
reject(new TimeoutWaitingForMatchError(config.id))
|
|
436
|
+
}
|
|
253
437
|
|
|
254
|
-
|
|
255
|
-
((await config.onInsert!(params)) as MaybeTxId) ?? {}
|
|
256
|
-
const txid = handlerResult.txid
|
|
438
|
+
const timeoutId = setTimeout(onTimeout, timeout)
|
|
257
439
|
|
|
258
|
-
|
|
259
|
-
|
|
440
|
+
// We need access to the stream messages to check against the match function
|
|
441
|
+
// This will be handled by the sync configuration
|
|
442
|
+
const checkMatch = (message: Message<any>) => {
|
|
443
|
+
if (matchFn(message)) {
|
|
444
|
+
debug(
|
|
445
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch found matching message, waiting for up-to-date`
|
|
446
|
+
)
|
|
447
|
+
// Mark as matched but don't resolve yet - wait for up-to-date
|
|
448
|
+
pendingMatches.setState((current) => {
|
|
449
|
+
const newMatches = new Map(current)
|
|
450
|
+
const existing = newMatches.get(matchId)
|
|
451
|
+
if (existing) {
|
|
452
|
+
newMatches.set(matchId, { ...existing, matched: true })
|
|
453
|
+
}
|
|
454
|
+
return newMatches
|
|
455
|
+
})
|
|
456
|
+
return true
|
|
260
457
|
}
|
|
458
|
+
return false
|
|
459
|
+
}
|
|
261
460
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
461
|
+
// Check against current batch messages first to handle race conditions
|
|
462
|
+
for (const message of currentBatchMessages.state) {
|
|
463
|
+
if (matchFn(message)) {
|
|
464
|
+
debug(
|
|
465
|
+
`${config.id ? `[${config.id}] ` : ``}awaitMatch found immediate match in current batch, waiting for up-to-date`
|
|
466
|
+
)
|
|
467
|
+
// Register match as already matched
|
|
468
|
+
pendingMatches.setState((current) => {
|
|
469
|
+
const newMatches = new Map(current)
|
|
470
|
+
newMatches.set(matchId, {
|
|
471
|
+
matchFn: checkMatch,
|
|
472
|
+
resolve,
|
|
473
|
+
reject,
|
|
474
|
+
timeoutId,
|
|
475
|
+
matched: true, // Already matched
|
|
476
|
+
})
|
|
477
|
+
return newMatches
|
|
478
|
+
})
|
|
479
|
+
return
|
|
267
480
|
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// Store the match function for the sync process to use
|
|
484
|
+
// We'll add this to a pending matches store
|
|
485
|
+
pendingMatches.setState((current) => {
|
|
486
|
+
const newMatches = new Map(current)
|
|
487
|
+
newMatches.set(matchId, {
|
|
488
|
+
matchFn: checkMatch,
|
|
489
|
+
resolve,
|
|
490
|
+
reject,
|
|
491
|
+
timeoutId,
|
|
492
|
+
matched: false,
|
|
493
|
+
})
|
|
494
|
+
return newMatches
|
|
495
|
+
})
|
|
496
|
+
})
|
|
497
|
+
}
|
|
268
498
|
|
|
499
|
+
/**
|
|
500
|
+
* Process matching strategy and wait for synchronization
|
|
501
|
+
*/
|
|
502
|
+
const processMatchingStrategy = async (
|
|
503
|
+
result: MatchingStrategy
|
|
504
|
+
): Promise<void> => {
|
|
505
|
+
// Only wait if result contains txid
|
|
506
|
+
if (result && `txid` in result) {
|
|
507
|
+
// Handle both single txid and array of txids
|
|
508
|
+
if (Array.isArray(result.txid)) {
|
|
509
|
+
await Promise.all(result.txid.map(awaitTxId))
|
|
510
|
+
} else {
|
|
511
|
+
await awaitTxId(result.txid)
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
// If result is void/undefined, don't wait - mutation completes immediately
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// Create wrapper handlers for direct persistence operations that handle different matching strategies
|
|
518
|
+
const wrappedOnInsert = config.onInsert
|
|
519
|
+
? async (params: InsertMutationFnParams<any>) => {
|
|
520
|
+
const handlerResult = await config.onInsert!(params)
|
|
521
|
+
await processMatchingStrategy(handlerResult)
|
|
269
522
|
return handlerResult
|
|
270
523
|
}
|
|
271
524
|
: undefined
|
|
272
525
|
|
|
273
526
|
const wrappedOnUpdate = config.onUpdate
|
|
274
527
|
? async (params: UpdateMutationFnParams<any>) => {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
const handlerResult =
|
|
278
|
-
((await config.onUpdate!(params)) as MaybeTxId) ?? {}
|
|
279
|
-
const txid = handlerResult.txid
|
|
280
|
-
|
|
281
|
-
if (!txid) {
|
|
282
|
-
throw new ElectricUpdateHandlerMustReturnTxIdError(config.id)
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
// Handle both single txid and array of txids
|
|
286
|
-
if (Array.isArray(txid)) {
|
|
287
|
-
await Promise.all(txid.map((id) => awaitTxId(id)))
|
|
288
|
-
} else {
|
|
289
|
-
await awaitTxId(txid)
|
|
290
|
-
}
|
|
291
|
-
|
|
528
|
+
const handlerResult = await config.onUpdate!(params)
|
|
529
|
+
await processMatchingStrategy(handlerResult)
|
|
292
530
|
return handlerResult
|
|
293
531
|
}
|
|
294
532
|
: undefined
|
|
@@ -296,17 +534,7 @@ export function electricCollectionOptions(
|
|
|
296
534
|
const wrappedOnDelete = config.onDelete
|
|
297
535
|
? async (params: DeleteMutationFnParams<any>) => {
|
|
298
536
|
const handlerResult = await config.onDelete!(params)
|
|
299
|
-
|
|
300
|
-
throw new ElectricDeleteHandlerMustReturnTxIdError(config.id)
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// Handle both single txid and array of txids
|
|
304
|
-
if (Array.isArray(handlerResult.txid)) {
|
|
305
|
-
await Promise.all(handlerResult.txid.map((id) => awaitTxId(id)))
|
|
306
|
-
} else {
|
|
307
|
-
await awaitTxId(handlerResult.txid)
|
|
308
|
-
}
|
|
309
|
-
|
|
537
|
+
await processMatchingStrategy(handlerResult)
|
|
310
538
|
return handlerResult
|
|
311
539
|
}
|
|
312
540
|
: undefined
|
|
@@ -328,7 +556,8 @@ export function electricCollectionOptions(
|
|
|
328
556
|
onDelete: wrappedOnDelete,
|
|
329
557
|
utils: {
|
|
330
558
|
awaitTxId,
|
|
331
|
-
|
|
559
|
+
awaitMatch,
|
|
560
|
+
} as ElectricCollectionUtils<any>,
|
|
332
561
|
}
|
|
333
562
|
}
|
|
334
563
|
|
|
@@ -340,10 +569,34 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
340
569
|
options: {
|
|
341
570
|
seenTxids: Store<Set<Txid>>
|
|
342
571
|
seenSnapshots: Store<Array<PostgresSnapshot>>
|
|
572
|
+
pendingMatches: Store<
|
|
573
|
+
Map<
|
|
574
|
+
string,
|
|
575
|
+
{
|
|
576
|
+
matchFn: (message: Message<T>) => boolean
|
|
577
|
+
resolve: (value: boolean) => void
|
|
578
|
+
reject: (error: Error) => void
|
|
579
|
+
timeoutId: ReturnType<typeof setTimeout>
|
|
580
|
+
matched: boolean
|
|
581
|
+
}
|
|
582
|
+
>
|
|
583
|
+
>
|
|
584
|
+
currentBatchMessages: Store<Array<Message<T>>>
|
|
585
|
+
removePendingMatches: (matchIds: Array<string>) => void
|
|
586
|
+
resolveMatchedPendingMatches: () => void
|
|
343
587
|
collectionId?: string
|
|
344
588
|
}
|
|
345
589
|
): SyncConfig<T> {
|
|
346
|
-
const {
|
|
590
|
+
const {
|
|
591
|
+
seenTxids,
|
|
592
|
+
seenSnapshots,
|
|
593
|
+
pendingMatches,
|
|
594
|
+
currentBatchMessages,
|
|
595
|
+
removePendingMatches,
|
|
596
|
+
resolveMatchedPendingMatches,
|
|
597
|
+
collectionId,
|
|
598
|
+
} = options
|
|
599
|
+
const MAX_BATCH_MESSAGES = 1000 // Safety limit for message buffer
|
|
347
600
|
|
|
348
601
|
// Store for the relation schema information
|
|
349
602
|
const relationSchema = new Store<string | undefined>(undefined)
|
|
@@ -387,6 +640,17 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
387
640
|
}
|
|
388
641
|
}
|
|
389
642
|
|
|
643
|
+
// Cleanup pending matches on abort
|
|
644
|
+
abortController.signal.addEventListener(`abort`, () => {
|
|
645
|
+
pendingMatches.setState((current) => {
|
|
646
|
+
current.forEach((match) => {
|
|
647
|
+
clearTimeout(match.timeoutId)
|
|
648
|
+
match.reject(new StreamAbortedError())
|
|
649
|
+
})
|
|
650
|
+
return new Map() // Clear all pending matches
|
|
651
|
+
})
|
|
652
|
+
})
|
|
653
|
+
|
|
390
654
|
const stream = new ShapeStream({
|
|
391
655
|
...shapeOptions,
|
|
392
656
|
signal: abortController.signal,
|
|
@@ -420,11 +684,45 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
420
684
|
let hasUpToDate = false
|
|
421
685
|
|
|
422
686
|
for (const message of messages) {
|
|
687
|
+
// Add message to current batch buffer (for race condition handling)
|
|
688
|
+
if (isChangeMessage(message)) {
|
|
689
|
+
currentBatchMessages.setState((currentBuffer) => {
|
|
690
|
+
const newBuffer = [...currentBuffer, message]
|
|
691
|
+
// Limit buffer size for safety
|
|
692
|
+
if (newBuffer.length > MAX_BATCH_MESSAGES) {
|
|
693
|
+
newBuffer.splice(0, newBuffer.length - MAX_BATCH_MESSAGES)
|
|
694
|
+
}
|
|
695
|
+
return newBuffer
|
|
696
|
+
})
|
|
697
|
+
}
|
|
698
|
+
|
|
423
699
|
// Check for txids in the message and add them to our store
|
|
424
700
|
if (hasTxids(message)) {
|
|
425
701
|
message.headers.txids?.forEach((txid) => newTxids.add(txid))
|
|
426
702
|
}
|
|
427
703
|
|
|
704
|
+
// Check pending matches against this message
|
|
705
|
+
// Note: matchFn will mark matches internally, we don't resolve here
|
|
706
|
+
const matchesToRemove: Array<string> = []
|
|
707
|
+
pendingMatches.state.forEach((match, matchId) => {
|
|
708
|
+
if (!match.matched) {
|
|
709
|
+
try {
|
|
710
|
+
match.matchFn(message)
|
|
711
|
+
} catch (err) {
|
|
712
|
+
// If matchFn throws, clean up and reject the promise
|
|
713
|
+
clearTimeout(match.timeoutId)
|
|
714
|
+
match.reject(
|
|
715
|
+
err instanceof Error ? err : new Error(String(err))
|
|
716
|
+
)
|
|
717
|
+
matchesToRemove.push(matchId)
|
|
718
|
+
debug(`matchFn error: %o`, err)
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
})
|
|
722
|
+
|
|
723
|
+
// Remove matches that errored
|
|
724
|
+
removePendingMatches(matchesToRemove)
|
|
725
|
+
|
|
428
726
|
if (isChangeMessage(message)) {
|
|
429
727
|
// Check if the message contains schema information
|
|
430
728
|
const schema = message.headers.schema
|
|
@@ -469,6 +767,9 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
469
767
|
}
|
|
470
768
|
|
|
471
769
|
if (hasUpToDate) {
|
|
770
|
+
// Clear the current batch buffer since we're now up-to-date
|
|
771
|
+
currentBatchMessages.setState(() => [])
|
|
772
|
+
|
|
472
773
|
// Commit transaction if one was started
|
|
473
774
|
if (transactionStarted) {
|
|
474
775
|
commit()
|
|
@@ -504,6 +805,9 @@ function createElectricSync<T extends Row<unknown>>(
|
|
|
504
805
|
newSnapshots.length = 0
|
|
505
806
|
return seen
|
|
506
807
|
})
|
|
808
|
+
|
|
809
|
+
// Resolve all matched pending matches on up-to-date
|
|
810
|
+
resolveMatchedPendingMatches()
|
|
507
811
|
}
|
|
508
812
|
})
|
|
509
813
|
|
package/src/errors.ts
CHANGED
|
@@ -22,32 +22,16 @@ export class TimeoutWaitingForTxIdError extends ElectricDBCollectionError {
|
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
export class
|
|
25
|
+
export class TimeoutWaitingForMatchError extends ElectricDBCollectionError {
|
|
26
26
|
constructor(collectionId?: string) {
|
|
27
|
-
super(
|
|
28
|
-
|
|
29
|
-
collectionId
|
|
30
|
-
)
|
|
31
|
-
this.name = `ElectricInsertHandlerMustReturnTxIdError`
|
|
27
|
+
super(`Timeout waiting for custom match function`, collectionId)
|
|
28
|
+
this.name = `TimeoutWaitingForMatchError`
|
|
32
29
|
}
|
|
33
30
|
}
|
|
34
31
|
|
|
35
|
-
export class
|
|
32
|
+
export class StreamAbortedError extends ElectricDBCollectionError {
|
|
36
33
|
constructor(collectionId?: string) {
|
|
37
|
-
super(
|
|
38
|
-
|
|
39
|
-
collectionId
|
|
40
|
-
)
|
|
41
|
-
this.name = `ElectricUpdateHandlerMustReturnTxIdError`
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export class ElectricDeleteHandlerMustReturnTxIdError extends ElectricDBCollectionError {
|
|
46
|
-
constructor(collectionId?: string) {
|
|
47
|
-
super(
|
|
48
|
-
`Electric collection onDelete handler must return a txid or array of txids`,
|
|
49
|
-
collectionId
|
|
50
|
-
)
|
|
51
|
-
this.name = `ElectricDeleteHandlerMustReturnTxIdError`
|
|
34
|
+
super(`Stream aborted`, collectionId)
|
|
35
|
+
this.name = `StreamAbortedError`
|
|
52
36
|
}
|
|
53
37
|
}
|