@tanstack/db 0.5.10 → 0.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/SortedMap.cjs +40 -26
- package/dist/cjs/SortedMap.cjs.map +1 -1
- package/dist/cjs/SortedMap.d.cts +10 -15
- package/dist/cjs/collection/change-events.cjs +1 -1
- package/dist/cjs/collection/change-events.cjs.map +1 -1
- package/dist/cjs/collection/changes.cjs.map +1 -1
- package/dist/cjs/collection/events.cjs.map +1 -1
- package/dist/cjs/collection/events.d.cts +12 -4
- package/dist/cjs/collection/index.cjs +2 -1
- package/dist/cjs/collection/index.cjs.map +1 -1
- package/dist/cjs/collection/indexes.cjs.map +1 -1
- package/dist/cjs/collection/lifecycle.cjs.map +1 -1
- package/dist/cjs/collection/mutations.cjs +5 -2
- package/dist/cjs/collection/mutations.cjs.map +1 -1
- package/dist/cjs/collection/state.cjs +6 -5
- package/dist/cjs/collection/state.cjs.map +1 -1
- package/dist/cjs/collection/state.d.cts +4 -1
- package/dist/cjs/collection/subscription.cjs +60 -53
- package/dist/cjs/collection/subscription.cjs.map +1 -1
- package/dist/cjs/collection/subscription.d.cts +18 -4
- package/dist/cjs/collection/sync.cjs.map +1 -1
- package/dist/cjs/errors.cjs +9 -0
- package/dist/cjs/errors.cjs.map +1 -1
- package/dist/cjs/errors.d.cts +3 -0
- package/dist/cjs/event-emitter.cjs.map +1 -1
- package/dist/cjs/index.cjs +4 -0
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +2 -1
- package/dist/cjs/indexes/auto-index.cjs.map +1 -1
- package/dist/cjs/indexes/base-index.cjs.map +1 -1
- package/dist/cjs/indexes/btree-index.cjs +8 -6
- package/dist/cjs/indexes/btree-index.cjs.map +1 -1
- package/dist/cjs/indexes/lazy-index.cjs.map +1 -1
- package/dist/cjs/indexes/reverse-index.cjs.map +1 -1
- package/dist/cjs/local-only.cjs.map +1 -1
- package/dist/cjs/local-storage.cjs.map +1 -1
- package/dist/cjs/optimistic-action.cjs.map +1 -1
- package/dist/cjs/paced-mutations.cjs.map +1 -1
- package/dist/cjs/proxy.cjs.map +1 -1
- package/dist/cjs/query/builder/functions.cjs.map +1 -1
- package/dist/cjs/query/builder/index.cjs.map +1 -1
- package/dist/cjs/query/builder/ref-proxy.cjs.map +1 -1
- package/dist/cjs/query/compiler/evaluators.cjs.map +1 -1
- package/dist/cjs/query/compiler/expressions.cjs.map +1 -1
- package/dist/cjs/query/compiler/group-by.cjs.map +1 -1
- package/dist/cjs/query/compiler/index.cjs.map +1 -1
- package/dist/cjs/query/compiler/joins.cjs.map +1 -1
- package/dist/cjs/query/compiler/order-by.cjs +91 -38
- package/dist/cjs/query/compiler/order-by.cjs.map +1 -1
- package/dist/cjs/query/compiler/order-by.d.cts +6 -2
- package/dist/cjs/query/compiler/select.cjs.map +1 -1
- package/dist/cjs/query/expression-helpers.cjs.map +1 -1
- package/dist/cjs/query/index.d.cts +1 -1
- package/dist/cjs/query/ir.cjs.map +1 -1
- package/dist/cjs/query/live/collection-config-builder.cjs.map +1 -1
- package/dist/cjs/query/live/collection-registry.cjs.map +1 -1
- package/dist/cjs/query/live/collection-subscriber.cjs +30 -15
- package/dist/cjs/query/live/collection-subscriber.cjs.map +1 -1
- package/dist/cjs/query/live/internal.cjs.map +1 -1
- package/dist/cjs/query/live-query-collection.cjs.map +1 -1
- package/dist/cjs/query/optimizer.cjs.map +1 -1
- package/dist/cjs/query/predicate-utils.cjs +19 -2
- package/dist/cjs/query/predicate-utils.cjs.map +1 -1
- package/dist/cjs/query/predicate-utils.d.cts +32 -1
- package/dist/cjs/query/subset-dedupe.cjs.map +1 -1
- package/dist/cjs/scheduler.cjs.map +1 -1
- package/dist/cjs/strategies/debounceStrategy.cjs.map +1 -1
- package/dist/cjs/strategies/queueStrategy.cjs.map +1 -1
- package/dist/cjs/strategies/throttleStrategy.cjs.map +1 -1
- package/dist/cjs/transactions.cjs.map +1 -1
- package/dist/cjs/types.d.cts +43 -5
- package/dist/cjs/utils/browser-polyfills.cjs.map +1 -1
- package/dist/cjs/utils/btree.cjs.map +1 -1
- package/dist/cjs/utils/comparison.cjs.map +1 -1
- package/dist/cjs/utils/cursor.cjs +39 -0
- package/dist/cjs/utils/cursor.cjs.map +1 -0
- package/dist/cjs/utils/cursor.d.cts +18 -0
- package/dist/cjs/utils/index-optimization.cjs.map +1 -1
- package/dist/cjs/utils.cjs.map +1 -1
- package/dist/esm/SortedMap.d.ts +10 -15
- package/dist/esm/SortedMap.js +40 -26
- package/dist/esm/SortedMap.js.map +1 -1
- package/dist/esm/collection/change-events.js +1 -1
- package/dist/esm/collection/change-events.js.map +1 -1
- package/dist/esm/collection/changes.js.map +1 -1
- package/dist/esm/collection/events.d.ts +12 -4
- package/dist/esm/collection/events.js.map +1 -1
- package/dist/esm/collection/index.js +2 -1
- package/dist/esm/collection/index.js.map +1 -1
- package/dist/esm/collection/indexes.js.map +1 -1
- package/dist/esm/collection/lifecycle.js.map +1 -1
- package/dist/esm/collection/mutations.js +6 -3
- package/dist/esm/collection/mutations.js.map +1 -1
- package/dist/esm/collection/state.d.ts +4 -1
- package/dist/esm/collection/state.js +6 -5
- package/dist/esm/collection/state.js.map +1 -1
- package/dist/esm/collection/subscription.d.ts +18 -4
- package/dist/esm/collection/subscription.js +61 -54
- package/dist/esm/collection/subscription.js.map +1 -1
- package/dist/esm/collection/sync.js.map +1 -1
- package/dist/esm/errors.d.ts +3 -0
- package/dist/esm/errors.js +9 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/event-emitter.js.map +1 -1
- package/dist/esm/index.d.ts +2 -1
- package/dist/esm/index.js +6 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/indexes/auto-index.js.map +1 -1
- package/dist/esm/indexes/base-index.js.map +1 -1
- package/dist/esm/indexes/btree-index.js +8 -6
- package/dist/esm/indexes/btree-index.js.map +1 -1
- package/dist/esm/indexes/lazy-index.js.map +1 -1
- package/dist/esm/indexes/reverse-index.js.map +1 -1
- package/dist/esm/local-only.js.map +1 -1
- package/dist/esm/local-storage.js.map +1 -1
- package/dist/esm/optimistic-action.js.map +1 -1
- package/dist/esm/paced-mutations.js.map +1 -1
- package/dist/esm/proxy.js.map +1 -1
- package/dist/esm/query/builder/functions.js.map +1 -1
- package/dist/esm/query/builder/index.js.map +1 -1
- package/dist/esm/query/builder/ref-proxy.js.map +1 -1
- package/dist/esm/query/compiler/evaluators.js.map +1 -1
- package/dist/esm/query/compiler/expressions.js.map +1 -1
- package/dist/esm/query/compiler/group-by.js.map +1 -1
- package/dist/esm/query/compiler/index.js.map +1 -1
- package/dist/esm/query/compiler/joins.js.map +1 -1
- package/dist/esm/query/compiler/order-by.d.ts +6 -2
- package/dist/esm/query/compiler/order-by.js +91 -38
- package/dist/esm/query/compiler/order-by.js.map +1 -1
- package/dist/esm/query/compiler/select.js.map +1 -1
- package/dist/esm/query/expression-helpers.js.map +1 -1
- package/dist/esm/query/index.d.ts +1 -1
- package/dist/esm/query/ir.js.map +1 -1
- package/dist/esm/query/live/collection-config-builder.js.map +1 -1
- package/dist/esm/query/live/collection-registry.js.map +1 -1
- package/dist/esm/query/live/collection-subscriber.js +30 -15
- package/dist/esm/query/live/collection-subscriber.js.map +1 -1
- package/dist/esm/query/live/internal.js.map +1 -1
- package/dist/esm/query/live-query-collection.js.map +1 -1
- package/dist/esm/query/optimizer.js.map +1 -1
- package/dist/esm/query/predicate-utils.d.ts +32 -1
- package/dist/esm/query/predicate-utils.js +19 -2
- package/dist/esm/query/predicate-utils.js.map +1 -1
- package/dist/esm/query/subset-dedupe.js.map +1 -1
- package/dist/esm/scheduler.js.map +1 -1
- package/dist/esm/strategies/debounceStrategy.js.map +1 -1
- package/dist/esm/strategies/queueStrategy.js.map +1 -1
- package/dist/esm/strategies/throttleStrategy.js.map +1 -1
- package/dist/esm/transactions.js.map +1 -1
- package/dist/esm/types.d.ts +43 -5
- package/dist/esm/utils/browser-polyfills.js.map +1 -1
- package/dist/esm/utils/btree.js.map +1 -1
- package/dist/esm/utils/comparison.js.map +1 -1
- package/dist/esm/utils/cursor.d.ts +18 -0
- package/dist/esm/utils/cursor.js +39 -0
- package/dist/esm/utils/cursor.js.map +1 -0
- package/dist/esm/utils/index-optimization.js.map +1 -1
- package/dist/esm/utils.js.map +1 -1
- package/package.json +30 -28
- package/src/SortedMap.ts +50 -31
- package/src/collection/change-events.ts +23 -21
- package/src/collection/changes.ts +12 -12
- package/src/collection/events.ts +20 -10
- package/src/collection/index.ts +47 -46
- package/src/collection/indexes.ts +14 -14
- package/src/collection/lifecycle.ts +16 -16
- package/src/collection/mutations.ts +25 -20
- package/src/collection/state.ts +43 -36
- package/src/collection/subscription.ts +114 -83
- package/src/collection/sync.ts +13 -13
- package/src/duplicate-instance-check.ts +1 -1
- package/src/errors.ts +49 -40
- package/src/event-emitter.ts +5 -5
- package/src/index.ts +21 -20
- package/src/indexes/auto-index.ts +11 -11
- package/src/indexes/base-index.ts +13 -13
- package/src/indexes/btree-index.ts +21 -17
- package/src/indexes/index-options.ts +3 -3
- package/src/indexes/lazy-index.ts +8 -8
- package/src/indexes/reverse-index.ts +5 -5
- package/src/local-only.ts +12 -12
- package/src/local-storage.ts +17 -17
- package/src/optimistic-action.ts +5 -5
- package/src/paced-mutations.ts +6 -6
- package/src/proxy.ts +43 -43
- package/src/query/builder/functions.ts +28 -28
- package/src/query/builder/index.ts +22 -22
- package/src/query/builder/ref-proxy.ts +4 -4
- package/src/query/builder/types.ts +8 -8
- package/src/query/compiler/evaluators.ts +9 -9
- package/src/query/compiler/expressions.ts +6 -6
- package/src/query/compiler/group-by.ts +24 -24
- package/src/query/compiler/index.ts +44 -44
- package/src/query/compiler/joins.ts +37 -37
- package/src/query/compiler/order-by.ts +170 -77
- package/src/query/compiler/select.ts +13 -13
- package/src/query/compiler/types.ts +2 -2
- package/src/query/expression-helpers.ts +16 -16
- package/src/query/index.ts +10 -9
- package/src/query/ir.ts +13 -13
- package/src/query/live/collection-config-builder.ts +53 -53
- package/src/query/live/collection-registry.ts +6 -6
- package/src/query/live/collection-subscriber.ts +87 -48
- package/src/query/live/internal.ts +1 -1
- package/src/query/live/types.ts +4 -4
- package/src/query/live-query-collection.ts +15 -15
- package/src/query/optimizer.ts +29 -29
- package/src/query/predicate-utils.ts +105 -50
- package/src/query/subset-dedupe.ts +6 -6
- package/src/scheduler.ts +3 -3
- package/src/strategies/debounceStrategy.ts +6 -6
- package/src/strategies/index.ts +4 -4
- package/src/strategies/queueStrategy.ts +5 -5
- package/src/strategies/throttleStrategy.ts +6 -6
- package/src/strategies/types.ts +2 -2
- package/src/transactions.ts +9 -9
- package/src/types.ts +51 -12
- package/src/utils/array-utils.ts +1 -1
- package/src/utils/browser-polyfills.ts +2 -2
- package/src/utils/btree.ts +22 -22
- package/src/utils/comparison.ts +3 -3
- package/src/utils/cursor.ts +78 -0
- package/src/utils/index-optimization.ts +14 -14
- package/src/utils.ts +4 -4
package/src/collection/state.ts
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
import { deepEquals } from
|
|
2
|
-
import { SortedMap } from
|
|
3
|
-
import type { Transaction } from
|
|
4
|
-
import type { StandardSchemaV1 } from
|
|
1
|
+
import { deepEquals } from '../utils'
|
|
2
|
+
import { SortedMap } from '../SortedMap'
|
|
3
|
+
import type { Transaction } from '../transactions'
|
|
4
|
+
import type { StandardSchemaV1 } from '@standard-schema/spec'
|
|
5
5
|
import type {
|
|
6
6
|
ChangeMessage,
|
|
7
7
|
CollectionConfig,
|
|
8
8
|
OptimisticChangeMessage,
|
|
9
|
-
} from
|
|
10
|
-
import type { CollectionImpl } from
|
|
11
|
-
import type { CollectionLifecycleManager } from
|
|
12
|
-
import type { CollectionChangesManager } from
|
|
13
|
-
import type { CollectionIndexesManager } from
|
|
9
|
+
} from '../types'
|
|
10
|
+
import type { CollectionImpl } from './index.js'
|
|
11
|
+
import type { CollectionLifecycleManager } from './lifecycle'
|
|
12
|
+
import type { CollectionChangesManager } from './changes'
|
|
13
|
+
import type { CollectionIndexesManager } from './indexes'
|
|
14
|
+
import type { CollectionEventsManager } from './events'
|
|
14
15
|
|
|
15
16
|
interface PendingSyncedTransaction<
|
|
16
17
|
T extends object = Record<string, unknown>,
|
|
@@ -37,13 +38,14 @@ export class CollectionStateManager<
|
|
|
37
38
|
public lifecycle!: CollectionLifecycleManager<TOutput, TKey, TSchema, TInput>
|
|
38
39
|
public changes!: CollectionChangesManager<TOutput, TKey, TSchema, TInput>
|
|
39
40
|
public indexes!: CollectionIndexesManager<TOutput, TKey, TSchema, TInput>
|
|
41
|
+
private _events!: CollectionEventsManager
|
|
40
42
|
|
|
41
43
|
// Core state - make public for testing
|
|
42
44
|
public transactions: SortedMap<string, Transaction<any>>
|
|
43
45
|
public pendingSyncedTransactions: Array<
|
|
44
46
|
PendingSyncedTransaction<TOutput, TKey>
|
|
45
47
|
> = []
|
|
46
|
-
public syncedData:
|
|
48
|
+
public syncedData: SortedMap<TKey, TOutput>
|
|
47
49
|
public syncedMetadata = new Map<TKey, unknown>()
|
|
48
50
|
|
|
49
51
|
// Optimistic state tracking - make public for testing
|
|
@@ -66,15 +68,12 @@ export class CollectionStateManager<
|
|
|
66
68
|
constructor(config: CollectionConfig<TOutput, TKey, TSchema>) {
|
|
67
69
|
this.config = config
|
|
68
70
|
this.transactions = new SortedMap<string, Transaction<any>>((a, b) =>
|
|
69
|
-
a.compareCreatedAt(b)
|
|
71
|
+
a.compareCreatedAt(b),
|
|
70
72
|
)
|
|
71
73
|
|
|
72
|
-
// Set up data storage
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
} else {
|
|
76
|
-
this.syncedData = new Map<TKey, TOutput>()
|
|
77
|
-
}
|
|
74
|
+
// Set up data storage - always use SortedMap for deterministic iteration.
|
|
75
|
+
// If a custom compare function is provided, use it; otherwise entries are sorted by key only.
|
|
76
|
+
this.syncedData = new SortedMap<TKey, TOutput>(config.compare)
|
|
78
77
|
}
|
|
79
78
|
|
|
80
79
|
setDeps(deps: {
|
|
@@ -82,11 +81,13 @@ export class CollectionStateManager<
|
|
|
82
81
|
lifecycle: CollectionLifecycleManager<TOutput, TKey, TSchema, TInput>
|
|
83
82
|
changes: CollectionChangesManager<TOutput, TKey, TSchema, TInput>
|
|
84
83
|
indexes: CollectionIndexesManager<TOutput, TKey, TSchema, TInput>
|
|
84
|
+
events: CollectionEventsManager
|
|
85
85
|
}) {
|
|
86
86
|
this.collection = deps.collection
|
|
87
87
|
this.lifecycle = deps.lifecycle
|
|
88
88
|
this.changes = deps.changes
|
|
89
89
|
this.indexes = deps.indexes
|
|
90
|
+
this._events = deps.events
|
|
90
91
|
}
|
|
91
92
|
|
|
92
93
|
/**
|
|
@@ -185,7 +186,7 @@ export class CollectionStateManager<
|
|
|
185
186
|
* Execute a callback for each entry in the collection
|
|
186
187
|
*/
|
|
187
188
|
public forEach(
|
|
188
|
-
callbackfn: (value: TOutput, key: TKey, index: number) => void
|
|
189
|
+
callbackfn: (value: TOutput, key: TKey, index: number) => void,
|
|
189
190
|
): void {
|
|
190
191
|
let index = 0
|
|
191
192
|
for (const [key, value] of this.entries()) {
|
|
@@ -197,7 +198,7 @@ export class CollectionStateManager<
|
|
|
197
198
|
* Create a new array with the results of calling a function for each entry in the collection
|
|
198
199
|
*/
|
|
199
200
|
public map<U>(
|
|
200
|
-
callbackfn: (value: TOutput, key: TKey, index: number) => U
|
|
201
|
+
callbackfn: (value: TOutput, key: TKey, index: number) => U,
|
|
201
202
|
): Array<U> {
|
|
202
203
|
const result: Array<U> = []
|
|
203
204
|
let index = 0
|
|
@@ -213,7 +214,7 @@ export class CollectionStateManager<
|
|
|
213
214
|
* @returns True if the given collection is this collection, false otherwise
|
|
214
215
|
*/
|
|
215
216
|
private isThisCollection(
|
|
216
|
-
collection: CollectionImpl<any, any, any, any, any
|
|
217
|
+
collection: CollectionImpl<any, any, any, any, any>,
|
|
217
218
|
): boolean {
|
|
218
219
|
return collection === this.collection
|
|
219
220
|
}
|
|
@@ -222,7 +223,7 @@ export class CollectionStateManager<
|
|
|
222
223
|
* Recompute optimistic state from active transactions
|
|
223
224
|
*/
|
|
224
225
|
public recomputeOptimisticState(
|
|
225
|
-
triggeredByUserAction: boolean = false
|
|
226
|
+
triggeredByUserAction: boolean = false,
|
|
226
227
|
): void {
|
|
227
228
|
// Skip redundant recalculations when we're in the middle of committing sync transactions
|
|
228
229
|
// While the sync pipeline is replaying a large batch we still want to honour
|
|
@@ -257,7 +258,7 @@ export class CollectionStateManager<
|
|
|
257
258
|
case `update`:
|
|
258
259
|
this.optimisticUpserts.set(
|
|
259
260
|
mutation.key,
|
|
260
|
-
mutation.modified as TOutput
|
|
261
|
+
mutation.modified as TOutput,
|
|
261
262
|
)
|
|
262
263
|
this.optimisticDeletes.delete(mutation.key)
|
|
263
264
|
break
|
|
@@ -316,8 +317,8 @@ export class CollectionStateManager<
|
|
|
316
317
|
// We can infer this by checking if we have no remaining optimistic mutations for this key
|
|
317
318
|
const hasActiveOptimisticMutation = activeTransactions.some((tx) =>
|
|
318
319
|
tx.mutations.some(
|
|
319
|
-
(m) => this.isThisCollection(m.collection) && m.key === event.key
|
|
320
|
-
)
|
|
320
|
+
(m) => this.isThisCollection(m.collection) && m.key === event.key,
|
|
321
|
+
),
|
|
321
322
|
)
|
|
322
323
|
|
|
323
324
|
if (!hasActiveOptimisticMutation) {
|
|
@@ -348,10 +349,10 @@ export class CollectionStateManager<
|
|
|
348
349
|
private calculateSize(): number {
|
|
349
350
|
const syncedSize = this.syncedData.size
|
|
350
351
|
const deletesFromSynced = Array.from(this.optimisticDeletes).filter(
|
|
351
|
-
(key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key)
|
|
352
|
+
(key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key),
|
|
352
353
|
).length
|
|
353
354
|
const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter(
|
|
354
|
-
(key) => !this.syncedData.has(key)
|
|
355
|
+
(key) => !this.syncedData.has(key),
|
|
355
356
|
).length
|
|
356
357
|
|
|
357
358
|
return syncedSize - deletesFromSynced + upsertsNotInSynced
|
|
@@ -363,7 +364,7 @@ export class CollectionStateManager<
|
|
|
363
364
|
private collectOptimisticChanges(
|
|
364
365
|
previousUpserts: Map<TKey, TOutput>,
|
|
365
366
|
previousDeletes: Set<TKey>,
|
|
366
|
-
events: Array<ChangeMessage<TOutput, TKey
|
|
367
|
+
events: Array<ChangeMessage<TOutput, TKey>>,
|
|
367
368
|
): void {
|
|
368
369
|
const allKeys = new Set([
|
|
369
370
|
...previousUpserts.keys(),
|
|
@@ -377,7 +378,7 @@ export class CollectionStateManager<
|
|
|
377
378
|
const previousValue = this.getPreviousValue(
|
|
378
379
|
key,
|
|
379
380
|
previousUpserts,
|
|
380
|
-
previousDeletes
|
|
381
|
+
previousDeletes,
|
|
381
382
|
)
|
|
382
383
|
|
|
383
384
|
if (previousValue !== undefined && currentValue === undefined) {
|
|
@@ -405,7 +406,7 @@ export class CollectionStateManager<
|
|
|
405
406
|
private getPreviousValue(
|
|
406
407
|
key: TKey,
|
|
407
408
|
previousUpserts: Map<TKey, TOutput>,
|
|
408
|
-
previousDeletes: Set<TKey
|
|
409
|
+
previousDeletes: Set<TKey>,
|
|
409
410
|
): TOutput | undefined {
|
|
410
411
|
if (previousDeletes.has(key)) {
|
|
411
412
|
return undefined
|
|
@@ -456,7 +457,7 @@ export class CollectionStateManager<
|
|
|
456
457
|
PendingSyncedTransaction<TOutput, TKey>
|
|
457
458
|
>,
|
|
458
459
|
hasTruncateSync: false,
|
|
459
|
-
}
|
|
460
|
+
},
|
|
460
461
|
)
|
|
461
462
|
|
|
462
463
|
if (!hasPersistingTransaction || hasTruncateSync) {
|
|
@@ -528,6 +529,12 @@ export class CollectionStateManager<
|
|
|
528
529
|
for (const key of changedKeys) {
|
|
529
530
|
currentVisibleState.delete(key)
|
|
530
531
|
}
|
|
532
|
+
|
|
533
|
+
// 4) Emit truncate event so subscriptions can reset their cursor tracking state
|
|
534
|
+
this._events.emit(`truncate`, {
|
|
535
|
+
type: `truncate`,
|
|
536
|
+
collection: this.collection,
|
|
537
|
+
})
|
|
531
538
|
}
|
|
532
539
|
|
|
533
540
|
for (const operation of transaction.operations) {
|
|
@@ -545,8 +552,8 @@ export class CollectionStateManager<
|
|
|
545
552
|
Object.assign(
|
|
546
553
|
{},
|
|
547
554
|
this.syncedMetadata.get(key),
|
|
548
|
-
operation.metadata
|
|
549
|
-
)
|
|
555
|
+
operation.metadata,
|
|
556
|
+
),
|
|
550
557
|
)
|
|
551
558
|
break
|
|
552
559
|
case `delete`:
|
|
@@ -564,7 +571,7 @@ export class CollectionStateManager<
|
|
|
564
571
|
const updatedValue = Object.assign(
|
|
565
572
|
{},
|
|
566
573
|
this.syncedData.get(key),
|
|
567
|
-
operation.value
|
|
574
|
+
operation.value,
|
|
568
575
|
)
|
|
569
576
|
this.syncedData.set(key, updatedValue)
|
|
570
577
|
} else {
|
|
@@ -597,10 +604,10 @@ export class CollectionStateManager<
|
|
|
597
604
|
// Build re-apply sets from the snapshot taken at the start of this function.
|
|
598
605
|
// This prevents losing optimistic state if transactions complete during truncate processing.
|
|
599
606
|
const reapplyUpserts = new Map<TKey, TOutput>(
|
|
600
|
-
truncateOptimisticSnapshot!.upserts
|
|
607
|
+
truncateOptimisticSnapshot!.upserts,
|
|
601
608
|
)
|
|
602
609
|
const reapplyDeletes = new Set<TKey>(
|
|
603
|
-
truncateOptimisticSnapshot!.deletes
|
|
610
|
+
truncateOptimisticSnapshot!.deletes,
|
|
604
611
|
)
|
|
605
612
|
|
|
606
613
|
// Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete.
|
|
@@ -679,7 +686,7 @@ export class CollectionStateManager<
|
|
|
679
686
|
case `update`:
|
|
680
687
|
this.optimisticUpserts.set(
|
|
681
688
|
mutation.key,
|
|
682
|
-
mutation.modified as TOutput
|
|
689
|
+
mutation.modified as TOutput,
|
|
683
690
|
)
|
|
684
691
|
this.optimisticDeletes.delete(mutation.key)
|
|
685
692
|
break
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { ensureIndexForExpression } from
|
|
2
|
-
import { and, eq,
|
|
3
|
-
import { Value } from
|
|
4
|
-
import { EventEmitter } from
|
|
1
|
+
import { ensureIndexForExpression } from '../indexes/auto-index.js'
|
|
2
|
+
import { and, eq, gte, lt } from '../query/builder/functions.js'
|
|
3
|
+
import { Value } from '../query/ir.js'
|
|
4
|
+
import { EventEmitter } from '../event-emitter.js'
|
|
5
|
+
import { buildCursor } from '../utils/cursor.js'
|
|
5
6
|
import {
|
|
6
7
|
createFilterFunctionFromExpression,
|
|
7
8
|
createFilteredCallback,
|
|
8
|
-
} from
|
|
9
|
-
import type { BasicExpression, OrderBy } from
|
|
10
|
-
import type { IndexInterface } from
|
|
9
|
+
} from './change-events.js'
|
|
10
|
+
import type { BasicExpression, OrderBy } from '../query/ir.js'
|
|
11
|
+
import type { IndexInterface } from '../indexes/base-index.js'
|
|
11
12
|
import type {
|
|
12
13
|
ChangeMessage,
|
|
13
14
|
LoadSubsetOptions,
|
|
@@ -15,19 +16,26 @@ import type {
|
|
|
15
16
|
SubscriptionEvents,
|
|
16
17
|
SubscriptionStatus,
|
|
17
18
|
SubscriptionUnsubscribedEvent,
|
|
18
|
-
} from
|
|
19
|
-
import type { CollectionImpl } from
|
|
19
|
+
} from '../types.js'
|
|
20
|
+
import type { CollectionImpl } from './index.js'
|
|
20
21
|
|
|
21
22
|
type RequestSnapshotOptions = {
|
|
22
23
|
where?: BasicExpression<boolean>
|
|
23
24
|
optimizedOnly?: boolean
|
|
24
25
|
trackLoadSubsetPromise?: boolean
|
|
26
|
+
/** Optional orderBy to pass to loadSubset for backend optimization */
|
|
27
|
+
orderBy?: OrderBy
|
|
28
|
+
/** Optional limit to pass to loadSubset for backend optimization */
|
|
29
|
+
limit?: number
|
|
25
30
|
}
|
|
26
31
|
|
|
27
32
|
type RequestLimitedSnapshotOptions = {
|
|
28
33
|
orderBy: OrderBy
|
|
29
34
|
limit: number
|
|
30
|
-
|
|
35
|
+
/** All column values for cursor (first value used for local index, all values for sync layer) */
|
|
36
|
+
minValues?: Array<unknown>
|
|
37
|
+
/** Row offset for offset-based pagination (passed to sync layer) */
|
|
38
|
+
offset?: number
|
|
31
39
|
}
|
|
32
40
|
|
|
33
41
|
type CollectionSubscriptionOptions = {
|
|
@@ -57,6 +65,12 @@ export class CollectionSubscription
|
|
|
57
65
|
// Keep track of the keys we've sent (needed for join and orderBy optimizations)
|
|
58
66
|
private sentKeys = new Set<string | number>()
|
|
59
67
|
|
|
68
|
+
// Track the count of rows sent via requestLimitedSnapshot for offset-based pagination
|
|
69
|
+
private limitedSnapshotRowCount = 0
|
|
70
|
+
|
|
71
|
+
// Track the last key sent via requestLimitedSnapshot for cursor-based pagination
|
|
72
|
+
private lastSentKey: string | number | undefined
|
|
73
|
+
|
|
60
74
|
private filteredCallback: (changes: Array<ChangeMessage<any, any>>) => void
|
|
61
75
|
|
|
62
76
|
private orderByIndex: IndexInterface<string | number> | undefined
|
|
@@ -72,7 +86,7 @@ export class CollectionSubscription
|
|
|
72
86
|
constructor(
|
|
73
87
|
private collection: CollectionImpl<any, any, any, any, any>,
|
|
74
88
|
private callback: (changes: Array<ChangeMessage<any, any>>) => void,
|
|
75
|
-
private options: CollectionSubscriptionOptions
|
|
89
|
+
private options: CollectionSubscriptionOptions,
|
|
76
90
|
) {
|
|
77
91
|
super()
|
|
78
92
|
if (options.onUnsubscribe) {
|
|
@@ -85,7 +99,7 @@ export class CollectionSubscription
|
|
|
85
99
|
}
|
|
86
100
|
|
|
87
101
|
const callbackWithSentKeysTracking = (
|
|
88
|
-
changes: Array<ChangeMessage<any, any
|
|
102
|
+
changes: Array<ChangeMessage<any, any>>,
|
|
89
103
|
) => {
|
|
90
104
|
callback(changes)
|
|
91
105
|
this.trackSentKeys(changes)
|
|
@@ -203,6 +217,9 @@ export class CollectionSubscription
|
|
|
203
217
|
const loadOptions: LoadSubsetOptions = {
|
|
204
218
|
where: stateOpts.where,
|
|
205
219
|
subscription: this,
|
|
220
|
+
// Include orderBy and limit if provided so sync layer can optimize the query
|
|
221
|
+
orderBy: opts?.orderBy,
|
|
222
|
+
limit: opts?.limit,
|
|
206
223
|
}
|
|
207
224
|
const syncResult = this.collection._sync.loadSubset(loadOptions)
|
|
208
225
|
|
|
@@ -224,7 +241,7 @@ export class CollectionSubscription
|
|
|
224
241
|
|
|
225
242
|
// Only send changes that have not been sent yet
|
|
226
243
|
const filteredSnapshot = snapshot.filter(
|
|
227
|
-
(change) => !this.sentKeys.has(change.key)
|
|
244
|
+
(change) => !this.sentKeys.has(change.key),
|
|
228
245
|
)
|
|
229
246
|
|
|
230
247
|
this.snapshotSent = true
|
|
@@ -233,26 +250,37 @@ export class CollectionSubscription
|
|
|
233
250
|
}
|
|
234
251
|
|
|
235
252
|
/**
|
|
236
|
-
* Sends a snapshot that fulfills the `where` clause and all rows are bigger or equal to
|
|
253
|
+
* Sends a snapshot that fulfills the `where` clause and all rows are bigger or equal to the cursor.
|
|
237
254
|
* Requires a range index to be set with `setOrderByIndex` prior to calling this method.
|
|
238
255
|
* It uses that range index to load the items in the order of the index.
|
|
239
|
-
*
|
|
256
|
+
*
|
|
257
|
+
* For multi-column orderBy:
|
|
258
|
+
* - Uses first value from `minValues` for LOCAL index operations (wide bounds, ensures no missed rows)
|
|
259
|
+
* - Uses all `minValues` to build a precise composite cursor for SYNC layer loadSubset
|
|
260
|
+
*
|
|
261
|
+
* Note 1: it may load more rows than the provided LIMIT because it loads all values equal to the first cursor value + limit values greater.
|
|
240
262
|
* This is needed to ensure that it does not accidentally skip duplicate values when the limit falls in the middle of some duplicated values.
|
|
241
263
|
* Note 2: it does not send keys that have already been sent before.
|
|
242
264
|
*/
|
|
243
265
|
requestLimitedSnapshot({
|
|
244
266
|
orderBy,
|
|
245
267
|
limit,
|
|
246
|
-
|
|
268
|
+
minValues,
|
|
269
|
+
offset,
|
|
247
270
|
}: RequestLimitedSnapshotOptions) {
|
|
248
271
|
if (!limit) throw new Error(`limit is required`)
|
|
249
272
|
|
|
250
273
|
if (!this.orderByIndex) {
|
|
251
274
|
throw new Error(
|
|
252
|
-
`Ordered snapshot was requested but no index was found. You have to call setOrderByIndex before requesting an ordered snapshot
|
|
275
|
+
`Ordered snapshot was requested but no index was found. You have to call setOrderByIndex before requesting an ordered snapshot.`,
|
|
253
276
|
)
|
|
254
277
|
}
|
|
255
278
|
|
|
279
|
+
// Derive first column value from minValues (used for local index operations)
|
|
280
|
+
const minValue = minValues?.[0]
|
|
281
|
+
// Cast for index operations (index expects string | number)
|
|
282
|
+
const minValueForIndex = minValue as string | number | undefined
|
|
283
|
+
|
|
256
284
|
const index = this.orderByIndex
|
|
257
285
|
const where = this.options.whereExpression
|
|
258
286
|
const whereFilterFn = where
|
|
@@ -272,7 +300,7 @@ export class CollectionSubscription
|
|
|
272
300
|
return whereFilterFn?.(value) ?? true
|
|
273
301
|
}
|
|
274
302
|
|
|
275
|
-
let biggestObservedValue =
|
|
303
|
+
let biggestObservedValue = minValueForIndex
|
|
276
304
|
const changes: Array<ChangeMessage<any, string | number>> = []
|
|
277
305
|
|
|
278
306
|
// If we have a minValue we need to handle the case
|
|
@@ -281,12 +309,16 @@ export class CollectionSubscription
|
|
|
281
309
|
// so if minValue is 3 then the previous snapshot may not have included all 3s
|
|
282
310
|
// e.g. if it was offset 0 and limit 3 it would only have loaded the first 3
|
|
283
311
|
// so we load all rows equal to minValue first, to be sure we don't skip any duplicate values
|
|
312
|
+
//
|
|
313
|
+
// For multi-column orderBy, we use the first column value for index operations (wide bounds)
|
|
314
|
+
// This may load some duplicates but ensures we never miss any rows.
|
|
284
315
|
let keys: Array<string | number> = []
|
|
285
|
-
if (
|
|
286
|
-
// First, get all items with the same value as minValue
|
|
316
|
+
if (minValueForIndex !== undefined) {
|
|
317
|
+
// First, get all items with the same FIRST COLUMN value as minValue
|
|
318
|
+
// This provides wide bounds for the local index
|
|
287
319
|
const { expression } = orderBy[0]!
|
|
288
320
|
const allRowsWithMinValue = this.collection.currentStateAsChanges({
|
|
289
|
-
where: eq(expression, new Value(
|
|
321
|
+
where: eq(expression, new Value(minValueForIndex)),
|
|
290
322
|
})
|
|
291
323
|
|
|
292
324
|
if (allRowsWithMinValue) {
|
|
@@ -300,15 +332,15 @@ export class CollectionSubscription
|
|
|
300
332
|
// Then get items greater than minValue
|
|
301
333
|
const keysGreaterThanMin = index.take(
|
|
302
334
|
limit - keys.length,
|
|
303
|
-
|
|
304
|
-
filterFn
|
|
335
|
+
minValueForIndex,
|
|
336
|
+
filterFn,
|
|
305
337
|
)
|
|
306
338
|
keys.push(...keysGreaterThanMin)
|
|
307
339
|
} else {
|
|
308
|
-
keys = index.take(limit,
|
|
340
|
+
keys = index.take(limit, minValueForIndex, filterFn)
|
|
309
341
|
}
|
|
310
342
|
} else {
|
|
311
|
-
keys = index.take(limit,
|
|
343
|
+
keys = index.take(limit, minValueForIndex, filterFn)
|
|
312
344
|
}
|
|
313
345
|
|
|
314
346
|
const valuesNeeded = () => Math.max(limit - changes.length, 0)
|
|
@@ -331,76 +363,75 @@ export class CollectionSubscription
|
|
|
331
363
|
keys = index.take(valuesNeeded(), biggestObservedValue, filterFn)
|
|
332
364
|
}
|
|
333
365
|
|
|
366
|
+
// Track row count for offset-based pagination (before sending to callback)
|
|
367
|
+
// Use the current count as the offset for this load
|
|
368
|
+
const currentOffset = this.limitedSnapshotRowCount
|
|
369
|
+
|
|
334
370
|
this.callback(changes)
|
|
335
371
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
372
|
+
// Update the row count and last key after sending (for next call's offset/cursor)
|
|
373
|
+
this.limitedSnapshotRowCount += changes.length
|
|
374
|
+
if (changes.length > 0) {
|
|
375
|
+
this.lastSentKey = changes[changes.length - 1]!.key
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// Build cursor expressions for sync layer loadSubset
|
|
379
|
+
// The cursor expressions are separate from the main where clause
|
|
380
|
+
// so the sync layer can choose cursor-based or offset-based pagination
|
|
381
|
+
let cursorExpressions:
|
|
382
|
+
| {
|
|
383
|
+
whereFrom: BasicExpression<boolean>
|
|
384
|
+
whereCurrent: BasicExpression<boolean>
|
|
385
|
+
lastKey?: string | number
|
|
386
|
+
}
|
|
387
|
+
| undefined
|
|
388
|
+
|
|
389
|
+
if (minValues !== undefined && minValues.length > 0) {
|
|
390
|
+
const whereFromCursor = buildCursor(orderBy, minValues)
|
|
391
|
+
|
|
392
|
+
if (whereFromCursor) {
|
|
393
|
+
const { expression } = orderBy[0]!
|
|
394
|
+
const minValue = minValues[0]
|
|
395
|
+
|
|
396
|
+
// Build the whereCurrent expression for the first orderBy column
|
|
397
|
+
// For Date values, we need to handle precision differences between JS (ms) and backends (μs)
|
|
398
|
+
// A JS Date represents a 1ms range, so we query for all values within that range
|
|
399
|
+
let whereCurrentCursor: BasicExpression<boolean>
|
|
400
|
+
if (minValue instanceof Date) {
|
|
401
|
+
const minValuePlus1ms = new Date(minValue.getTime() + 1)
|
|
402
|
+
whereCurrentCursor = and(
|
|
403
|
+
gte(expression, new Value(minValue)),
|
|
404
|
+
lt(expression, new Value(minValuePlus1ms)),
|
|
405
|
+
)
|
|
406
|
+
} else {
|
|
407
|
+
whereCurrentCursor = eq(expression, new Value(minValue))
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
cursorExpressions = {
|
|
411
|
+
whereFrom: whereFromCursor,
|
|
412
|
+
whereCurrent: whereCurrentCursor,
|
|
413
|
+
lastKey: this.lastSentKey,
|
|
414
|
+
}
|
|
415
|
+
}
|
|
343
416
|
}
|
|
344
417
|
|
|
345
418
|
// Request the sync layer to load more data
|
|
346
419
|
// don't await it, we will load the data into the collection when it comes in
|
|
347
|
-
|
|
348
|
-
|
|
420
|
+
// Note: `where` does NOT include cursor expressions - they are passed separately
|
|
421
|
+
// The sync layer can choose to use cursor-based or offset-based pagination
|
|
422
|
+
const loadOptions: LoadSubsetOptions = {
|
|
423
|
+
where, // Main filter only, no cursor
|
|
349
424
|
limit,
|
|
350
425
|
orderBy,
|
|
426
|
+
cursor: cursorExpressions, // Cursor expressions passed separately
|
|
427
|
+
offset: offset ?? currentOffset, // Use provided offset, or auto-tracked offset
|
|
351
428
|
subscription: this,
|
|
352
429
|
}
|
|
353
|
-
const syncResult = this.collection._sync.loadSubset(
|
|
430
|
+
const syncResult = this.collection._sync.loadSubset(loadOptions)
|
|
354
431
|
|
|
355
432
|
// Track this loadSubset call
|
|
356
|
-
this.loadedSubsets.push(
|
|
357
|
-
|
|
358
|
-
// Make parallel loadSubset calls for values equal to minValue and values greater than minValue
|
|
359
|
-
const promises: Array<Promise<void>> = []
|
|
360
|
-
|
|
361
|
-
// First promise: load all values equal to minValue
|
|
362
|
-
if (typeof minValue !== `undefined`) {
|
|
363
|
-
const { expression } = orderBy[0]!
|
|
364
|
-
|
|
365
|
-
// For Date values, we need to handle precision differences between JS (ms) and backends (μs)
|
|
366
|
-
// A JS Date represents a 1ms range, so we query for all values within that range
|
|
367
|
-
let exactValueFilter
|
|
368
|
-
if (minValue instanceof Date) {
|
|
369
|
-
const minValuePlus1ms = new Date(minValue.getTime() + 1)
|
|
370
|
-
exactValueFilter = and(
|
|
371
|
-
gte(expression, new Value(minValue)),
|
|
372
|
-
lt(expression, new Value(minValuePlus1ms))
|
|
373
|
-
)
|
|
374
|
-
} else {
|
|
375
|
-
exactValueFilter = eq(expression, new Value(minValue))
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
const loadOptions2: LoadSubsetOptions = {
|
|
379
|
-
where: exactValueFilter,
|
|
380
|
-
subscription: this,
|
|
381
|
-
}
|
|
382
|
-
const equalValueResult = this.collection._sync.loadSubset(loadOptions2)
|
|
383
|
-
|
|
384
|
-
// Track this loadSubset call
|
|
385
|
-
this.loadedSubsets.push(loadOptions2)
|
|
386
|
-
|
|
387
|
-
if (equalValueResult instanceof Promise) {
|
|
388
|
-
promises.push(equalValueResult)
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
// Second promise: load values greater than minValue
|
|
393
|
-
if (syncResult instanceof Promise) {
|
|
394
|
-
promises.push(syncResult)
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// Track the combined promise
|
|
398
|
-
if (promises.length > 0) {
|
|
399
|
-
const combinedPromise = Promise.all(promises).then(() => {})
|
|
400
|
-
this.trackLoadSubsetPromise(combinedPromise)
|
|
401
|
-
} else {
|
|
402
|
-
this.trackLoadSubsetPromise(syncResult)
|
|
403
|
-
}
|
|
433
|
+
this.loadedSubsets.push(loadOptions)
|
|
434
|
+
this.trackLoadSubsetPromise(syncResult)
|
|
404
435
|
}
|
|
405
436
|
|
|
406
437
|
// TODO: also add similar test but that checks that it can also load it from the collection's loadSubset function
|
package/src/collection/sync.ts
CHANGED
|
@@ -7,22 +7,22 @@ import {
|
|
|
7
7
|
SyncCleanupError,
|
|
8
8
|
SyncTransactionAlreadyCommittedError,
|
|
9
9
|
SyncTransactionAlreadyCommittedWriteError,
|
|
10
|
-
} from
|
|
11
|
-
import { deepEquals } from
|
|
12
|
-
import { LIVE_QUERY_INTERNAL } from
|
|
13
|
-
import type { StandardSchemaV1 } from
|
|
10
|
+
} from '../errors'
|
|
11
|
+
import { deepEquals } from '../utils'
|
|
12
|
+
import { LIVE_QUERY_INTERNAL } from '../query/live/internal.js'
|
|
13
|
+
import type { StandardSchemaV1 } from '@standard-schema/spec'
|
|
14
14
|
import type {
|
|
15
15
|
ChangeMessage,
|
|
16
16
|
CleanupFn,
|
|
17
17
|
CollectionConfig,
|
|
18
18
|
LoadSubsetOptions,
|
|
19
19
|
SyncConfigRes,
|
|
20
|
-
} from
|
|
21
|
-
import type { CollectionImpl } from
|
|
22
|
-
import type { CollectionStateManager } from
|
|
23
|
-
import type { CollectionLifecycleManager } from
|
|
24
|
-
import type { CollectionEventsManager } from
|
|
25
|
-
import type { LiveQueryCollectionUtils } from
|
|
20
|
+
} from '../types'
|
|
21
|
+
import type { CollectionImpl } from './index.js'
|
|
22
|
+
import type { CollectionStateManager } from './state'
|
|
23
|
+
import type { CollectionLifecycleManager } from './lifecycle'
|
|
24
|
+
import type { CollectionEventsManager } from './events.js'
|
|
25
|
+
import type { LiveQueryCollectionUtils } from '../query/live/collection-config-builder.js'
|
|
26
26
|
|
|
27
27
|
export class CollectionSyncManager<
|
|
28
28
|
TOutput extends object = Record<string, unknown>,
|
|
@@ -202,7 +202,7 @@ export class CollectionSyncManager<
|
|
|
202
202
|
deletes: new Set(this.state.optimisticDeletes),
|
|
203
203
|
}
|
|
204
204
|
},
|
|
205
|
-
})
|
|
205
|
+
}),
|
|
206
206
|
)
|
|
207
207
|
|
|
208
208
|
// Store cleanup function if provided
|
|
@@ -218,7 +218,7 @@ export class CollectionSyncManager<
|
|
|
218
218
|
if (this.syncMode === `on-demand` && !this.syncLoadSubsetFn) {
|
|
219
219
|
throw new CollectionConfigurationError(
|
|
220
220
|
`Collection "${this.id}" is configured with syncMode "on-demand" but the sync function did not return a loadSubset handler. ` +
|
|
221
|
-
`Either provide a loadSubset handler or use syncMode "eager"
|
|
221
|
+
`Either provide a loadSubset handler or use syncMode "eager".`,
|
|
222
222
|
)
|
|
223
223
|
}
|
|
224
224
|
} catch (error) {
|
|
@@ -242,7 +242,7 @@ export class CollectionSyncManager<
|
|
|
242
242
|
`${this.id ? `[${this.id}] ` : ``}Calling .preload() on a collection with syncMode "on-demand" is a no-op. ` +
|
|
243
243
|
`In on-demand mode, data is only loaded when queries request it. ` +
|
|
244
244
|
`Instead, create a live query and call .preload() on that to load the specific data you need. ` +
|
|
245
|
-
`See https://tanstack.com/blog/tanstack-db-0.5-query-driven-sync for more details
|
|
245
|
+
`See https://tanstack.com/blog/tanstack-db-0.5-query-driven-sync for more details.`,
|
|
246
246
|
)
|
|
247
247
|
}
|
|
248
248
|
|