@tanstack/db 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/collection/events.cjs +9 -51
- package/dist/cjs/collection/events.cjs.map +1 -1
- package/dist/cjs/collection/events.d.cts +18 -7
- package/dist/cjs/collection/index.cjs +9 -12
- package/dist/cjs/collection/index.cjs.map +1 -1
- package/dist/cjs/collection/index.d.cts +13 -14
- package/dist/cjs/collection/subscription.cjs +62 -6
- package/dist/cjs/collection/subscription.cjs.map +1 -1
- package/dist/cjs/collection/subscription.d.cts +16 -3
- package/dist/cjs/collection/sync.cjs +58 -6
- package/dist/cjs/collection/sync.cjs.map +1 -1
- package/dist/cjs/collection/sync.d.cts +18 -4
- package/dist/cjs/errors.cjs +59 -17
- package/dist/cjs/errors.cjs.map +1 -1
- package/dist/cjs/errors.d.cts +44 -8
- package/dist/cjs/event-emitter.cjs +94 -0
- package/dist/cjs/event-emitter.cjs.map +1 -0
- package/dist/cjs/event-emitter.d.cts +45 -0
- package/dist/cjs/index.cjs +9 -4
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/local-only.cjs.map +1 -1
- package/dist/cjs/local-only.d.cts +2 -5
- package/dist/cjs/query/builder/types.d.cts +1 -1
- package/dist/cjs/query/compiler/index.cjs +46 -19
- package/dist/cjs/query/compiler/index.cjs.map +1 -1
- package/dist/cjs/query/compiler/index.d.cts +35 -9
- package/dist/cjs/query/compiler/joins.cjs +91 -66
- package/dist/cjs/query/compiler/joins.cjs.map +1 -1
- package/dist/cjs/query/compiler/joins.d.cts +6 -3
- package/dist/cjs/query/compiler/order-by.cjs +20 -4
- package/dist/cjs/query/compiler/order-by.cjs.map +1 -1
- package/dist/cjs/query/compiler/order-by.d.cts +3 -1
- package/dist/cjs/query/compiler/select.cjs.map +1 -1
- package/dist/cjs/query/compiler/types.d.cts +4 -0
- package/dist/cjs/query/index.d.cts +1 -0
- package/dist/cjs/query/live/collection-config-builder.cjs +306 -46
- package/dist/cjs/query/live/collection-config-builder.cjs.map +1 -1
- package/dist/cjs/query/live/collection-config-builder.d.cts +97 -9
- package/dist/cjs/query/live/collection-registry.cjs +16 -0
- package/dist/cjs/query/live/collection-registry.cjs.map +1 -0
- package/dist/cjs/query/live/collection-registry.d.cts +26 -0
- package/dist/cjs/query/live/collection-subscriber.cjs +86 -58
- package/dist/cjs/query/live/collection-subscriber.cjs.map +1 -1
- package/dist/cjs/query/live/collection-subscriber.d.cts +5 -7
- package/dist/cjs/query/live-query-collection.cjs +11 -5
- package/dist/cjs/query/live-query-collection.cjs.map +1 -1
- package/dist/cjs/query/live-query-collection.d.cts +12 -5
- package/dist/cjs/query/optimizer.cjs +44 -7
- package/dist/cjs/query/optimizer.cjs.map +1 -1
- package/dist/cjs/query/optimizer.d.cts +4 -4
- package/dist/cjs/scheduler.cjs +137 -0
- package/dist/cjs/scheduler.cjs.map +1 -0
- package/dist/cjs/scheduler.d.cts +56 -0
- package/dist/cjs/transactions.cjs +7 -1
- package/dist/cjs/transactions.cjs.map +1 -1
- package/dist/cjs/types.d.cts +82 -11
- package/dist/esm/collection/events.d.ts +18 -7
- package/dist/esm/collection/events.js +9 -51
- package/dist/esm/collection/events.js.map +1 -1
- package/dist/esm/collection/index.d.ts +13 -14
- package/dist/esm/collection/index.js +9 -12
- package/dist/esm/collection/index.js.map +1 -1
- package/dist/esm/collection/subscription.d.ts +16 -3
- package/dist/esm/collection/subscription.js +62 -6
- package/dist/esm/collection/subscription.js.map +1 -1
- package/dist/esm/collection/sync.d.ts +18 -4
- package/dist/esm/collection/sync.js +59 -7
- package/dist/esm/collection/sync.js.map +1 -1
- package/dist/esm/errors.d.ts +44 -8
- package/dist/esm/errors.js +60 -18
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/event-emitter.d.ts +45 -0
- package/dist/esm/event-emitter.js +94 -0
- package/dist/esm/event-emitter.js.map +1 -0
- package/dist/esm/index.js +10 -5
- package/dist/esm/local-only.d.ts +2 -5
- package/dist/esm/local-only.js.map +1 -1
- package/dist/esm/query/builder/types.d.ts +1 -1
- package/dist/esm/query/compiler/index.d.ts +35 -9
- package/dist/esm/query/compiler/index.js +46 -19
- package/dist/esm/query/compiler/index.js.map +1 -1
- package/dist/esm/query/compiler/joins.d.ts +6 -3
- package/dist/esm/query/compiler/joins.js +93 -68
- package/dist/esm/query/compiler/joins.js.map +1 -1
- package/dist/esm/query/compiler/order-by.d.ts +3 -1
- package/dist/esm/query/compiler/order-by.js +20 -4
- package/dist/esm/query/compiler/order-by.js.map +1 -1
- package/dist/esm/query/compiler/select.js.map +1 -1
- package/dist/esm/query/compiler/types.d.ts +4 -0
- package/dist/esm/query/index.d.ts +1 -0
- package/dist/esm/query/live/collection-config-builder.d.ts +97 -9
- package/dist/esm/query/live/collection-config-builder.js +306 -46
- package/dist/esm/query/live/collection-config-builder.js.map +1 -1
- package/dist/esm/query/live/collection-registry.d.ts +26 -0
- package/dist/esm/query/live/collection-registry.js +16 -0
- package/dist/esm/query/live/collection-registry.js.map +1 -0
- package/dist/esm/query/live/collection-subscriber.d.ts +5 -7
- package/dist/esm/query/live/collection-subscriber.js +86 -58
- package/dist/esm/query/live/collection-subscriber.js.map +1 -1
- package/dist/esm/query/live-query-collection.d.ts +12 -5
- package/dist/esm/query/live-query-collection.js +11 -5
- package/dist/esm/query/live-query-collection.js.map +1 -1
- package/dist/esm/query/optimizer.d.ts +4 -4
- package/dist/esm/query/optimizer.js +44 -7
- package/dist/esm/query/optimizer.js.map +1 -1
- package/dist/esm/scheduler.d.ts +56 -0
- package/dist/esm/scheduler.js +137 -0
- package/dist/esm/scheduler.js.map +1 -0
- package/dist/esm/transactions.js +7 -1
- package/dist/esm/transactions.js.map +1 -1
- package/dist/esm/types.d.ts +82 -11
- package/package.json +2 -2
- package/src/collection/events.ts +25 -74
- package/src/collection/index.ts +15 -19
- package/src/collection/subscription.ts +88 -6
- package/src/collection/sync.ts +81 -9
- package/src/errors.ts +91 -13
- package/src/event-emitter.ts +118 -0
- package/src/local-only.ts +5 -12
- package/src/query/builder/types.ts +1 -1
- package/src/query/compiler/index.ts +124 -33
- package/src/query/compiler/joins.ts +187 -128
- package/src/query/compiler/order-by.ts +30 -2
- package/src/query/compiler/select.ts +2 -3
- package/src/query/compiler/types.ts +5 -0
- package/src/query/index.ts +1 -0
- package/src/query/live/collection-config-builder.ts +501 -60
- package/src/query/live/collection-registry.ts +47 -0
- package/src/query/live/collection-subscriber.ts +137 -105
- package/src/query/live-query-collection.ts +47 -18
- package/src/query/optimizer.ts +85 -15
- package/src/scheduler.ts +198 -0
- package/src/transactions.ts +12 -1
- package/src/types.ts +93 -11
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { D2, output } from "@tanstack/db-ivm"
|
|
2
2
|
import { compileQuery } from "../compiler/index.js"
|
|
3
3
|
import { buildQuery, getQueryIR } from "../builder/index.js"
|
|
4
|
+
import {
|
|
5
|
+
MissingAliasInputsError,
|
|
6
|
+
SetWindowRequiresOrderByError,
|
|
7
|
+
} from "../../errors.js"
|
|
8
|
+
import { transactionScopedScheduler } from "../../scheduler.js"
|
|
9
|
+
import { getActiveTransaction } from "../../transactions.js"
|
|
4
10
|
import { CollectionSubscriber } from "./collection-subscriber.js"
|
|
11
|
+
import { getCollectionBuilder } from "./collection-registry.js"
|
|
12
|
+
import type { WindowOptions } from "../compiler/index.js"
|
|
13
|
+
import type { SchedulerContextId } from "../../scheduler.js"
|
|
5
14
|
import type { CollectionSubscription } from "../../collection/subscription.js"
|
|
6
15
|
import type { RootStreamBuilder } from "@tanstack/db-ivm"
|
|
7
16
|
import type { OrderByOptimizationInfo } from "../compiler/order-by.js"
|
|
@@ -11,6 +20,7 @@ import type {
|
|
|
11
20
|
KeyedStream,
|
|
12
21
|
ResultStream,
|
|
13
22
|
SyncConfig,
|
|
23
|
+
UtilsRecord,
|
|
14
24
|
} from "../../types.js"
|
|
15
25
|
import type { Context, GetResult } from "../builder/types.js"
|
|
16
26
|
import type { BasicExpression, QueryIR } from "../ir.js"
|
|
@@ -23,6 +33,22 @@ import type {
|
|
|
23
33
|
} from "./types.js"
|
|
24
34
|
import type { AllCollectionEvents } from "../../collection/events.js"
|
|
25
35
|
|
|
36
|
+
export type LiveQueryCollectionUtils = UtilsRecord & {
|
|
37
|
+
getRunCount: () => number
|
|
38
|
+
getBuilder: () => CollectionConfigBuilder<any, any>
|
|
39
|
+
/**
|
|
40
|
+
* Sets the offset and limit of an ordered query.
|
|
41
|
+
* Is a no-op if the query is not ordered.
|
|
42
|
+
*
|
|
43
|
+
* @returns `true` if no subset loading was triggered, or `Promise<void>` that resolves when the subset has been loaded
|
|
44
|
+
*/
|
|
45
|
+
setWindow: (options: WindowOptions) => true | Promise<void>
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
type PendingGraphRun = {
|
|
49
|
+
loadCallbacks: Set<() => boolean>
|
|
50
|
+
}
|
|
51
|
+
|
|
26
52
|
// Global counter for auto-generated collection IDs
|
|
27
53
|
let liveQueryCollectionCounter = 0
|
|
28
54
|
|
|
@@ -37,6 +63,9 @@ export class CollectionConfigBuilder<
|
|
|
37
63
|
private readonly id: string
|
|
38
64
|
readonly query: QueryIR
|
|
39
65
|
private readonly collections: Record<string, Collection<any, any, any>>
|
|
66
|
+
private readonly collectionByAlias: Record<string, Collection<any, any, any>>
|
|
67
|
+
// Populated during compilation with all aliases (including subquery inner aliases)
|
|
68
|
+
private compiledAliasToCollectionId: Record<string, string> = {}
|
|
40
69
|
|
|
41
70
|
// WeakMap to store the keys of the results
|
|
42
71
|
// so that we can retrieve them in the getKey function
|
|
@@ -48,26 +77,60 @@ export class CollectionConfigBuilder<
|
|
|
48
77
|
private readonly compare?: (val1: TResult, val2: TResult) => number
|
|
49
78
|
|
|
50
79
|
private isGraphRunning = false
|
|
80
|
+
private runCount = 0
|
|
81
|
+
|
|
82
|
+
// Current sync session state (set when sync starts, cleared when it stops)
|
|
83
|
+
// Public for testing purposes (CollectionConfigBuilder is internal, not public API)
|
|
84
|
+
public currentSyncConfig:
|
|
85
|
+
| Parameters<SyncConfig<TResult>[`sync`]>[0]
|
|
86
|
+
| undefined
|
|
87
|
+
public currentSyncState: FullSyncState | undefined
|
|
51
88
|
|
|
52
89
|
// Error state tracking
|
|
53
90
|
private isInErrorState = false
|
|
54
91
|
|
|
55
92
|
// Reference to the live query collection for error state transitions
|
|
56
|
-
|
|
93
|
+
public liveQueryCollection?: Collection<TResult, any, any>
|
|
94
|
+
|
|
95
|
+
private windowFn: ((options: WindowOptions) => void) | undefined
|
|
96
|
+
|
|
97
|
+
private maybeRunGraphFn: (() => void) | undefined
|
|
98
|
+
|
|
99
|
+
private readonly aliasDependencies: Record<
|
|
100
|
+
string,
|
|
101
|
+
Array<CollectionConfigBuilder<any, any>>
|
|
102
|
+
> = {}
|
|
103
|
+
|
|
104
|
+
private readonly builderDependencies = new Set<
|
|
105
|
+
CollectionConfigBuilder<any, any>
|
|
106
|
+
>()
|
|
107
|
+
|
|
108
|
+
// Pending graph runs per scheduler context (e.g., per transaction)
|
|
109
|
+
// The builder manages its own state; the scheduler just orchestrates execution order
|
|
110
|
+
// Only stores callbacks - if sync ends, pending jobs gracefully no-op
|
|
111
|
+
private readonly pendingGraphRuns = new Map<
|
|
112
|
+
SchedulerContextId,
|
|
113
|
+
PendingGraphRun
|
|
114
|
+
>()
|
|
115
|
+
|
|
116
|
+
// Unsubscribe function for scheduler's onClear listener
|
|
117
|
+
// Registered when sync starts, unregistered when sync stops
|
|
118
|
+
// Prevents memory leaks by releasing the scheduler's reference to this builder
|
|
119
|
+
private unsubscribeFromSchedulerClears?: () => void
|
|
57
120
|
|
|
58
121
|
private graphCache: D2 | undefined
|
|
59
122
|
private inputsCache: Record<string, RootStreamBuilder<unknown>> | undefined
|
|
60
123
|
private pipelineCache: ResultStream | undefined
|
|
61
|
-
public
|
|
124
|
+
public sourceWhereClausesCache:
|
|
62
125
|
| Map<string, BasicExpression<boolean>>
|
|
63
126
|
| undefined
|
|
64
127
|
|
|
65
|
-
// Map of
|
|
128
|
+
// Map of source alias to subscription
|
|
66
129
|
readonly subscriptions: Record<string, CollectionSubscription> = {}
|
|
67
|
-
// Map of
|
|
68
|
-
|
|
69
|
-
// Set of
|
|
70
|
-
readonly
|
|
130
|
+
// Map of source aliases to functions that load keys for that lazy source
|
|
131
|
+
lazySourcesCallbacks: Record<string, LazyCollectionCallbacks> = {}
|
|
132
|
+
// Set of source aliases that are lazy (don't load initial state)
|
|
133
|
+
readonly lazySources = new Set<string>()
|
|
71
134
|
// Set of collection IDs that include an optimizable ORDER BY clause
|
|
72
135
|
optimizableOrderByCollections: Record<string, OrderByOptimizationInfo> = {}
|
|
73
136
|
|
|
@@ -79,6 +142,19 @@ export class CollectionConfigBuilder<
|
|
|
79
142
|
|
|
80
143
|
this.query = buildQueryFromConfig(config)
|
|
81
144
|
this.collections = extractCollectionsFromQuery(this.query)
|
|
145
|
+
const collectionAliasesById = extractCollectionAliases(this.query)
|
|
146
|
+
|
|
147
|
+
// Build a reverse lookup map from alias to collection instance.
|
|
148
|
+
// This enables self-join support where the same collection can be referenced
|
|
149
|
+
// multiple times with different aliases (e.g., { employee: col, manager: col })
|
|
150
|
+
this.collectionByAlias = {}
|
|
151
|
+
for (const [collectionId, aliases] of collectionAliasesById.entries()) {
|
|
152
|
+
const collection = this.collections[collectionId]
|
|
153
|
+
if (!collection) continue
|
|
154
|
+
for (const alias of aliases) {
|
|
155
|
+
this.collectionByAlias[alias] = collection
|
|
156
|
+
}
|
|
157
|
+
}
|
|
82
158
|
|
|
83
159
|
// Create compare function for ordering if the query has orderBy
|
|
84
160
|
if (this.query.orderBy && this.query.orderBy.length > 0) {
|
|
@@ -90,7 +166,9 @@ export class CollectionConfigBuilder<
|
|
|
90
166
|
this.compileBasePipeline()
|
|
91
167
|
}
|
|
92
168
|
|
|
93
|
-
getConfig(): CollectionConfigSingleRowOption<TResult> {
|
|
169
|
+
getConfig(): CollectionConfigSingleRowOption<TResult> & {
|
|
170
|
+
utils: LiveQueryCollectionUtils
|
|
171
|
+
} {
|
|
94
172
|
return {
|
|
95
173
|
id: this.id,
|
|
96
174
|
getKey:
|
|
@@ -105,9 +183,69 @@ export class CollectionConfigBuilder<
|
|
|
105
183
|
onDelete: this.config.onDelete,
|
|
106
184
|
startSync: this.config.startSync,
|
|
107
185
|
singleResult: this.query.singleResult,
|
|
186
|
+
utils: {
|
|
187
|
+
getRunCount: this.getRunCount.bind(this),
|
|
188
|
+
getBuilder: () => this,
|
|
189
|
+
setWindow: this.setWindow.bind(this),
|
|
190
|
+
},
|
|
108
191
|
}
|
|
109
192
|
}
|
|
110
193
|
|
|
194
|
+
setWindow(options: WindowOptions): true | Promise<void> {
|
|
195
|
+
if (!this.windowFn) {
|
|
196
|
+
throw new SetWindowRequiresOrderByError()
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
this.windowFn(options)
|
|
200
|
+
this.maybeRunGraphFn?.()
|
|
201
|
+
|
|
202
|
+
// Check if loading a subset was triggered
|
|
203
|
+
if (this.liveQueryCollection?.isLoadingSubset) {
|
|
204
|
+
// Loading was triggered, return a promise that resolves when it completes
|
|
205
|
+
return new Promise<void>((resolve) => {
|
|
206
|
+
const unsubscribe = this.liveQueryCollection!.on(
|
|
207
|
+
`loadingSubset:change`,
|
|
208
|
+
(event) => {
|
|
209
|
+
if (!event.isLoadingSubset) {
|
|
210
|
+
unsubscribe()
|
|
211
|
+
resolve()
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
)
|
|
215
|
+
})
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// No loading was triggered
|
|
219
|
+
return true
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Resolves a collection alias to its collection ID.
|
|
224
|
+
*
|
|
225
|
+
* Uses a two-tier lookup strategy:
|
|
226
|
+
* 1. First checks compiled aliases (includes subquery inner aliases)
|
|
227
|
+
* 2. Falls back to declared aliases from the query's from/join clauses
|
|
228
|
+
*
|
|
229
|
+
* @param alias - The alias to resolve (e.g., "employee", "manager")
|
|
230
|
+
* @returns The collection ID that the alias references
|
|
231
|
+
* @throws {Error} If the alias is not found in either lookup
|
|
232
|
+
*/
|
|
233
|
+
getCollectionIdForAlias(alias: string): string {
|
|
234
|
+
const compiled = this.compiledAliasToCollectionId[alias]
|
|
235
|
+
if (compiled) {
|
|
236
|
+
return compiled
|
|
237
|
+
}
|
|
238
|
+
const collection = this.collectionByAlias[alias]
|
|
239
|
+
if (collection) {
|
|
240
|
+
return collection.id
|
|
241
|
+
}
|
|
242
|
+
throw new Error(`Unknown source alias "${alias}"`)
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
isLazyAlias(alias: string): boolean {
|
|
246
|
+
return this.lazySources.has(alias)
|
|
247
|
+
}
|
|
248
|
+
|
|
111
249
|
// The callback function is called after the graph has run.
|
|
112
250
|
// This gives the callback a chance to load more data if needed,
|
|
113
251
|
// that's used to optimize orderBy operators that set a limit,
|
|
@@ -115,12 +253,8 @@ export class CollectionConfigBuilder<
|
|
|
115
253
|
// That can happen because even though we load N rows, the pipeline might filter some of these rows out
|
|
116
254
|
// causing the orderBy operator to receive less than N rows or even no rows at all.
|
|
117
255
|
// So this callback would notice that it doesn't have enough rows and load some more.
|
|
118
|
-
// The callback returns a boolean, when it's true it's done loading data.
|
|
119
|
-
maybeRunGraph(
|
|
120
|
-
config: SyncMethods<TResult>,
|
|
121
|
-
syncState: FullSyncState,
|
|
122
|
-
callback?: () => boolean
|
|
123
|
-
) {
|
|
256
|
+
// The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready.
|
|
257
|
+
maybeRunGraph(callback?: () => boolean) {
|
|
124
258
|
if (this.isGraphRunning) {
|
|
125
259
|
// no nested runs of the graph
|
|
126
260
|
// which is possible if the `callback`
|
|
@@ -128,10 +262,18 @@ export class CollectionConfigBuilder<
|
|
|
128
262
|
return
|
|
129
263
|
}
|
|
130
264
|
|
|
265
|
+
// Should only be called when sync is active
|
|
266
|
+
if (!this.currentSyncConfig || !this.currentSyncState) {
|
|
267
|
+
throw new Error(
|
|
268
|
+
`maybeRunGraph called without active sync session. This should not happen.`
|
|
269
|
+
)
|
|
270
|
+
}
|
|
271
|
+
|
|
131
272
|
this.isGraphRunning = true
|
|
132
273
|
|
|
133
274
|
try {
|
|
134
|
-
const { begin, commit } =
|
|
275
|
+
const { begin, commit } = this.currentSyncConfig
|
|
276
|
+
const syncState = this.currentSyncState
|
|
135
277
|
|
|
136
278
|
// Don't run if the live query is in an error state
|
|
137
279
|
if (this.isInErrorState) {
|
|
@@ -152,7 +294,7 @@ export class CollectionConfigBuilder<
|
|
|
152
294
|
commit()
|
|
153
295
|
// After initial commit, check if we should mark ready
|
|
154
296
|
// (in case all sources were already ready before we subscribed)
|
|
155
|
-
this.updateLiveQueryStatus(
|
|
297
|
+
this.updateLiveQueryStatus(this.currentSyncConfig)
|
|
156
298
|
}
|
|
157
299
|
}
|
|
158
300
|
} finally {
|
|
@@ -160,6 +302,158 @@ export class CollectionConfigBuilder<
|
|
|
160
302
|
}
|
|
161
303
|
}
|
|
162
304
|
|
|
305
|
+
/**
|
|
306
|
+
* Schedules a graph run with the transaction-scoped scheduler.
|
|
307
|
+
* Ensures each builder runs at most once per transaction, with automatic dependency tracking
|
|
308
|
+
* to run parent queries before child queries. Outside a transaction, runs immediately.
|
|
309
|
+
*
|
|
310
|
+
* Multiple calls during a transaction are coalesced into a single execution.
|
|
311
|
+
* Dependencies are auto-discovered from subscribed live queries, or can be overridden.
|
|
312
|
+
* Load callbacks are combined when entries merge.
|
|
313
|
+
*
|
|
314
|
+
* Uses the current sync session's config and syncState from instance properties.
|
|
315
|
+
*
|
|
316
|
+
* @param callback - Optional callback to load more data if needed (returns true when done)
|
|
317
|
+
* @param options - Optional scheduling configuration
|
|
318
|
+
* @param options.contextId - Transaction ID to group work; defaults to active transaction
|
|
319
|
+
* @param options.jobId - Unique identifier for this job; defaults to this builder instance
|
|
320
|
+
* @param options.alias - Source alias that triggered this schedule; adds alias-specific dependencies
|
|
321
|
+
* @param options.dependencies - Explicit dependency list; overrides auto-discovered dependencies
|
|
322
|
+
*/
|
|
323
|
+
scheduleGraphRun(
|
|
324
|
+
callback?: () => boolean,
|
|
325
|
+
options?: {
|
|
326
|
+
contextId?: SchedulerContextId
|
|
327
|
+
jobId?: unknown
|
|
328
|
+
alias?: string
|
|
329
|
+
dependencies?: Array<CollectionConfigBuilder<any, any>>
|
|
330
|
+
}
|
|
331
|
+
) {
|
|
332
|
+
const contextId = options?.contextId ?? getActiveTransaction()?.id
|
|
333
|
+
// Use the builder instance as the job ID for deduplication. This is memory-safe
|
|
334
|
+
// because the scheduler's context Map is deleted after flushing (no long-term retention).
|
|
335
|
+
const jobId = options?.jobId ?? this
|
|
336
|
+
const dependentBuilders = (() => {
|
|
337
|
+
if (options?.dependencies) {
|
|
338
|
+
return options.dependencies
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const deps = new Set(this.builderDependencies)
|
|
342
|
+
if (options?.alias) {
|
|
343
|
+
const aliasDeps = this.aliasDependencies[options.alias]
|
|
344
|
+
if (aliasDeps) {
|
|
345
|
+
for (const dep of aliasDeps) {
|
|
346
|
+
deps.add(dep)
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
deps.delete(this)
|
|
352
|
+
|
|
353
|
+
return Array.from(deps)
|
|
354
|
+
})()
|
|
355
|
+
|
|
356
|
+
// We intentionally scope deduplication to the builder instance. Each instance
|
|
357
|
+
// owns caches and compiled pipelines, so sharing work across instances that
|
|
358
|
+
// merely reuse the same string id would execute the wrong builder's graph.
|
|
359
|
+
|
|
360
|
+
if (!this.currentSyncConfig || !this.currentSyncState) {
|
|
361
|
+
throw new Error(
|
|
362
|
+
`scheduleGraphRun called without active sync session. This should not happen.`
|
|
363
|
+
)
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// Manage our own state - get or create pending callbacks for this context
|
|
367
|
+
let pending = contextId ? this.pendingGraphRuns.get(contextId) : undefined
|
|
368
|
+
if (!pending) {
|
|
369
|
+
pending = {
|
|
370
|
+
loadCallbacks: new Set(),
|
|
371
|
+
}
|
|
372
|
+
if (contextId) {
|
|
373
|
+
this.pendingGraphRuns.set(contextId, pending)
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// Add callback if provided (this is what accumulates between schedules)
|
|
378
|
+
if (callback) {
|
|
379
|
+
pending.loadCallbacks.add(callback)
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Schedule execution (scheduler just orchestrates order, we manage state)
|
|
383
|
+
// For immediate execution (no contextId), pass pending directly since it won't be in the map
|
|
384
|
+
const pendingToPass = contextId ? undefined : pending
|
|
385
|
+
transactionScopedScheduler.schedule({
|
|
386
|
+
contextId,
|
|
387
|
+
jobId,
|
|
388
|
+
dependencies: dependentBuilders,
|
|
389
|
+
run: () => this.executeGraphRun(contextId, pendingToPass),
|
|
390
|
+
})
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* Clears pending graph run state for a specific context.
|
|
395
|
+
* Called when the scheduler clears a context (e.g., transaction rollback/abort).
|
|
396
|
+
*/
|
|
397
|
+
clearPendingGraphRun(contextId: SchedulerContextId): void {
|
|
398
|
+
this.pendingGraphRuns.delete(contextId)
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Executes a pending graph run. Called by the scheduler when dependencies are satisfied.
|
|
403
|
+
* Clears the pending state BEFORE execution so that any re-schedules during the run
|
|
404
|
+
* create fresh state and don't interfere with the current execution.
|
|
405
|
+
* Uses instance sync state - if sync has ended, gracefully returns without executing.
|
|
406
|
+
*
|
|
407
|
+
* @param contextId - Optional context ID to look up pending state
|
|
408
|
+
* @param pendingParam - For immediate execution (no context), pending state is passed directly
|
|
409
|
+
*/
|
|
410
|
+
private executeGraphRun(
|
|
411
|
+
contextId?: SchedulerContextId,
|
|
412
|
+
pendingParam?: PendingGraphRun
|
|
413
|
+
): void {
|
|
414
|
+
// Get pending state: either from parameter (no context) or from map (with context)
|
|
415
|
+
// Remove from map BEFORE checking sync state to prevent leaking entries when sync ends
|
|
416
|
+
// before the transaction flushes (e.g., unsubscribe during in-flight transaction)
|
|
417
|
+
const pending =
|
|
418
|
+
pendingParam ??
|
|
419
|
+
(contextId ? this.pendingGraphRuns.get(contextId) : undefined)
|
|
420
|
+
if (contextId) {
|
|
421
|
+
this.pendingGraphRuns.delete(contextId)
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// If no pending state, nothing to execute (context was cleared)
|
|
425
|
+
if (!pending) {
|
|
426
|
+
return
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// If sync session has ended, don't execute (graph is finalized, subscriptions cleared)
|
|
430
|
+
if (!this.currentSyncConfig || !this.currentSyncState) {
|
|
431
|
+
return
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
this.incrementRunCount()
|
|
435
|
+
|
|
436
|
+
const combinedLoader = () => {
|
|
437
|
+
let allDone = true
|
|
438
|
+
let firstError: unknown
|
|
439
|
+
pending.loadCallbacks.forEach((loader) => {
|
|
440
|
+
try {
|
|
441
|
+
allDone = loader() && allDone
|
|
442
|
+
} catch (error) {
|
|
443
|
+
allDone = false
|
|
444
|
+
firstError ??= error
|
|
445
|
+
}
|
|
446
|
+
})
|
|
447
|
+
if (firstError) {
|
|
448
|
+
throw firstError
|
|
449
|
+
}
|
|
450
|
+
// Returning false signals that callers should schedule another pass.
|
|
451
|
+
return allDone
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
this.maybeRunGraph(combinedLoader)
|
|
455
|
+
}
|
|
456
|
+
|
|
163
457
|
private getSyncConfig(): SyncConfig<TResult> {
|
|
164
458
|
return {
|
|
165
459
|
rowUpdateMode: `full`,
|
|
@@ -167,9 +461,19 @@ export class CollectionConfigBuilder<
|
|
|
167
461
|
}
|
|
168
462
|
}
|
|
169
463
|
|
|
464
|
+
incrementRunCount() {
|
|
465
|
+
this.runCount++
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
getRunCount() {
|
|
469
|
+
return this.runCount
|
|
470
|
+
}
|
|
471
|
+
|
|
170
472
|
private syncFn(config: SyncMethods<TResult>) {
|
|
171
473
|
// Store reference to the live query collection for error state transitions
|
|
172
474
|
this.liveQueryCollection = config.collection
|
|
475
|
+
// Store config and syncState as instance properties for the duration of this sync session
|
|
476
|
+
this.currentSyncConfig = config
|
|
173
477
|
|
|
174
478
|
const syncState: SyncState = {
|
|
175
479
|
messagesCount: 0,
|
|
@@ -182,58 +486,102 @@ export class CollectionConfigBuilder<
|
|
|
182
486
|
config,
|
|
183
487
|
syncState
|
|
184
488
|
)
|
|
489
|
+
this.currentSyncState = fullSyncState
|
|
185
490
|
|
|
186
|
-
|
|
491
|
+
// Listen for scheduler context clears to clean up our pending state
|
|
492
|
+
// Re-register on each sync start so the listener is active for the sync session's lifetime
|
|
493
|
+
this.unsubscribeFromSchedulerClears = transactionScopedScheduler.onClear(
|
|
494
|
+
(contextId) => {
|
|
495
|
+
this.clearPendingGraphRun(contextId)
|
|
496
|
+
}
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
const loadSubsetDataCallbacks = this.subscribeToAllCollections(
|
|
187
500
|
config,
|
|
188
501
|
fullSyncState
|
|
189
502
|
)
|
|
190
503
|
|
|
504
|
+
this.maybeRunGraphFn = () => this.scheduleGraphRun(loadSubsetDataCallbacks)
|
|
505
|
+
|
|
191
506
|
// Initial run with callback to load more data if needed
|
|
192
|
-
this.
|
|
507
|
+
this.scheduleGraphRun(loadSubsetDataCallbacks)
|
|
193
508
|
|
|
194
509
|
// Return the unsubscribe function
|
|
195
510
|
return () => {
|
|
196
511
|
syncState.unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe())
|
|
197
512
|
|
|
513
|
+
// Clear current sync session state
|
|
514
|
+
this.currentSyncConfig = undefined
|
|
515
|
+
this.currentSyncState = undefined
|
|
516
|
+
|
|
517
|
+
// Clear all pending graph runs to prevent memory leaks from in-flight transactions
|
|
518
|
+
// that may flush after the sync session ends
|
|
519
|
+
this.pendingGraphRuns.clear()
|
|
520
|
+
|
|
198
521
|
// Reset caches so a fresh graph/pipeline is compiled on next start
|
|
199
522
|
// This avoids reusing a finalized D2 graph across GC restarts
|
|
200
523
|
this.graphCache = undefined
|
|
201
524
|
this.inputsCache = undefined
|
|
202
525
|
this.pipelineCache = undefined
|
|
203
|
-
this.
|
|
526
|
+
this.sourceWhereClausesCache = undefined
|
|
204
527
|
|
|
205
|
-
// Reset lazy
|
|
206
|
-
this.
|
|
528
|
+
// Reset lazy source alias state
|
|
529
|
+
this.lazySources.clear()
|
|
207
530
|
this.optimizableOrderByCollections = {}
|
|
208
|
-
this.
|
|
531
|
+
this.lazySourcesCallbacks = {}
|
|
532
|
+
|
|
533
|
+
// Clear subscription references to prevent memory leaks
|
|
534
|
+
// Note: Individual subscriptions are already unsubscribed via unsubscribeCallbacks
|
|
535
|
+
Object.keys(this.subscriptions).forEach(
|
|
536
|
+
(key) => delete this.subscriptions[key]
|
|
537
|
+
)
|
|
538
|
+
this.compiledAliasToCollectionId = {}
|
|
539
|
+
|
|
540
|
+
// Unregister from scheduler's onClear listener to prevent memory leaks
|
|
541
|
+
// The scheduler's listener Set would otherwise keep a strong reference to this builder
|
|
542
|
+
this.unsubscribeFromSchedulerClears?.()
|
|
543
|
+
this.unsubscribeFromSchedulerClears = undefined
|
|
209
544
|
}
|
|
210
545
|
}
|
|
211
546
|
|
|
547
|
+
/**
|
|
548
|
+
* Compiles the query pipeline with all declared aliases.
|
|
549
|
+
*/
|
|
212
550
|
private compileBasePipeline() {
|
|
213
551
|
this.graphCache = new D2()
|
|
214
552
|
this.inputsCache = Object.fromEntries(
|
|
215
|
-
Object.
|
|
216
|
-
|
|
553
|
+
Object.keys(this.collectionByAlias).map((alias) => [
|
|
554
|
+
alias,
|
|
217
555
|
this.graphCache!.newInput<any>(),
|
|
218
556
|
])
|
|
219
557
|
)
|
|
220
558
|
|
|
221
|
-
|
|
222
|
-
const {
|
|
223
|
-
pipeline: pipelineCache,
|
|
224
|
-
collectionWhereClauses: collectionWhereClausesCache,
|
|
225
|
-
} = compileQuery(
|
|
559
|
+
const compilation = compileQuery(
|
|
226
560
|
this.query,
|
|
227
561
|
this.inputsCache as Record<string, KeyedStream>,
|
|
228
562
|
this.collections,
|
|
229
563
|
this.subscriptions,
|
|
230
|
-
this.
|
|
231
|
-
this.
|
|
232
|
-
this.optimizableOrderByCollections
|
|
564
|
+
this.lazySourcesCallbacks,
|
|
565
|
+
this.lazySources,
|
|
566
|
+
this.optimizableOrderByCollections,
|
|
567
|
+
(windowFn: (options: WindowOptions) => void) => {
|
|
568
|
+
this.windowFn = windowFn
|
|
569
|
+
}
|
|
233
570
|
)
|
|
234
571
|
|
|
235
|
-
this.pipelineCache =
|
|
236
|
-
this.
|
|
572
|
+
this.pipelineCache = compilation.pipeline
|
|
573
|
+
this.sourceWhereClausesCache = compilation.sourceWhereClauses
|
|
574
|
+
this.compiledAliasToCollectionId = compilation.aliasToCollectionId
|
|
575
|
+
|
|
576
|
+
// Defensive check: verify all compiled aliases have corresponding inputs
|
|
577
|
+
// This should never happen since all aliases come from user declarations,
|
|
578
|
+
// but catch it early if the assumption is violated in the future.
|
|
579
|
+
const missingAliases = Object.keys(this.compiledAliasToCollectionId).filter(
|
|
580
|
+
(alias) => !Object.hasOwn(this.inputsCache!, alias)
|
|
581
|
+
)
|
|
582
|
+
if (missingAliases.length > 0) {
|
|
583
|
+
throw new MissingAliasInputsError(missingAliases)
|
|
584
|
+
}
|
|
237
585
|
}
|
|
238
586
|
|
|
239
587
|
private maybeCompileBasePipeline() {
|
|
@@ -400,50 +748,84 @@ export class CollectionConfigBuilder<
|
|
|
400
748
|
)
|
|
401
749
|
}
|
|
402
750
|
|
|
751
|
+
/**
|
|
752
|
+
* Creates per-alias subscriptions enabling self-join support.
|
|
753
|
+
* Each alias gets its own subscription with independent filters, even for the same collection.
|
|
754
|
+
* Example: `{ employee: col, manager: col }` creates two separate subscriptions.
|
|
755
|
+
*/
|
|
403
756
|
private subscribeToAllCollections(
|
|
404
757
|
config: SyncMethods<TResult>,
|
|
405
758
|
syncState: FullSyncState
|
|
406
759
|
) {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
)
|
|
760
|
+
// Use compiled aliases as the source of truth - these include all aliases from the query
|
|
761
|
+
// including those from subqueries, which may not be in collectionByAlias
|
|
762
|
+
const compiledAliases = Object.entries(this.compiledAliasToCollectionId)
|
|
763
|
+
if (compiledAliases.length === 0) {
|
|
764
|
+
throw new Error(
|
|
765
|
+
`Compiler returned no alias metadata for query '${this.id}'. This should not happen; please report.`
|
|
766
|
+
)
|
|
767
|
+
}
|
|
416
768
|
|
|
417
|
-
|
|
418
|
-
|
|
769
|
+
// Create a separate subscription for each alias, enabling self-joins where the same
|
|
770
|
+
// collection can be used multiple times with different filters and subscriptions
|
|
771
|
+
const loaders = compiledAliases.map(([alias, collectionId]) => {
|
|
772
|
+
// Try collectionByAlias first (for declared aliases), fall back to collections (for subquery aliases)
|
|
773
|
+
const collection =
|
|
774
|
+
this.collectionByAlias[alias] ?? this.collections[collectionId]!
|
|
775
|
+
|
|
776
|
+
const dependencyBuilder = getCollectionBuilder(collection)
|
|
777
|
+
if (dependencyBuilder && dependencyBuilder !== this) {
|
|
778
|
+
this.aliasDependencies[alias] = [dependencyBuilder]
|
|
779
|
+
this.builderDependencies.add(dependencyBuilder)
|
|
780
|
+
} else {
|
|
781
|
+
this.aliasDependencies[alias] = []
|
|
782
|
+
}
|
|
419
783
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
784
|
+
// CollectionSubscriber handles the actual subscription to the source collection
|
|
785
|
+
// and feeds data into the D2 graph inputs for this specific alias
|
|
786
|
+
const collectionSubscriber = new CollectionSubscriber(
|
|
787
|
+
alias,
|
|
788
|
+
collectionId,
|
|
789
|
+
collection,
|
|
790
|
+
this
|
|
791
|
+
)
|
|
425
792
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
793
|
+
// Subscribe to status changes for status flow
|
|
794
|
+
const statusUnsubscribe = collection.on(`status:change`, (event) => {
|
|
795
|
+
this.handleSourceStatusChange(config, collectionId, event)
|
|
796
|
+
})
|
|
797
|
+
syncState.unsubscribeCallbacks.add(statusUnsubscribe)
|
|
430
798
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
799
|
+
const subscription = collectionSubscriber.subscribe()
|
|
800
|
+
// Store subscription by alias (not collection ID) to support lazy loading
|
|
801
|
+
// which needs to look up subscriptions by their query alias
|
|
802
|
+
this.subscriptions[alias] = subscription
|
|
803
|
+
|
|
804
|
+
// Create a callback for loading more data if needed (used by OrderBy optimization)
|
|
805
|
+
const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(
|
|
806
|
+
collectionSubscriber,
|
|
807
|
+
subscription
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
return loadMore
|
|
811
|
+
})
|
|
434
812
|
|
|
435
|
-
|
|
813
|
+
// Combine all loaders into a single callback that initiates loading more data
|
|
814
|
+
// from any source that needs it. Returns true once all loaders have been called,
|
|
815
|
+
// but the actual async loading may still be in progress.
|
|
816
|
+
const loadSubsetDataCallbacks = () => {
|
|
436
817
|
loaders.map((loader) => loader())
|
|
437
818
|
return true
|
|
438
819
|
}
|
|
439
820
|
|
|
440
|
-
// Mark
|
|
821
|
+
// Mark as subscribed so the graph can start running
|
|
822
|
+
// (graph only runs when all collections are subscribed)
|
|
441
823
|
syncState.subscribedToAllCollections = true
|
|
442
824
|
|
|
443
825
|
// Initial status check after all subscriptions are set up
|
|
444
826
|
this.updateLiveQueryStatus(config)
|
|
445
827
|
|
|
446
|
-
return
|
|
828
|
+
return loadSubsetDataCallbacks
|
|
447
829
|
}
|
|
448
830
|
}
|
|
449
831
|
|
|
@@ -524,6 +906,65 @@ function extractCollectionsFromQuery(
|
|
|
524
906
|
return collections
|
|
525
907
|
}
|
|
526
908
|
|
|
909
|
+
/**
|
|
910
|
+
* Extracts all aliases used for each collection across the entire query tree.
|
|
911
|
+
*
|
|
912
|
+
* Traverses the QueryIR recursively to build a map from collection ID to all aliases
|
|
913
|
+
* that reference that collection. This is essential for self-join support, where the
|
|
914
|
+
* same collection may be referenced multiple times with different aliases.
|
|
915
|
+
*
|
|
916
|
+
* For example, given a query like:
|
|
917
|
+
* ```ts
|
|
918
|
+
* q.from({ employee: employeesCollection })
|
|
919
|
+
* .join({ manager: employeesCollection }, ({ employee, manager }) =>
|
|
920
|
+
* eq(employee.managerId, manager.id)
|
|
921
|
+
* )
|
|
922
|
+
* ```
|
|
923
|
+
*
|
|
924
|
+
* This function would return:
|
|
925
|
+
* ```
|
|
926
|
+
* Map { "employees" => Set { "employee", "manager" } }
|
|
927
|
+
* ```
|
|
928
|
+
*
|
|
929
|
+
* @param query - The query IR to extract aliases from
|
|
930
|
+
* @returns A map from collection ID to the set of all aliases referencing that collection
|
|
931
|
+
*/
|
|
932
|
+
function extractCollectionAliases(query: QueryIR): Map<string, Set<string>> {
|
|
933
|
+
const aliasesById = new Map<string, Set<string>>()
|
|
934
|
+
|
|
935
|
+
function recordAlias(source: any) {
|
|
936
|
+
if (!source) return
|
|
937
|
+
|
|
938
|
+
if (source.type === `collectionRef`) {
|
|
939
|
+
const { id } = source.collection
|
|
940
|
+
const existing = aliasesById.get(id)
|
|
941
|
+
if (existing) {
|
|
942
|
+
existing.add(source.alias)
|
|
943
|
+
} else {
|
|
944
|
+
aliasesById.set(id, new Set([source.alias]))
|
|
945
|
+
}
|
|
946
|
+
} else if (source.type === `queryRef`) {
|
|
947
|
+
traverse(source.query)
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
function traverse(q?: QueryIR) {
|
|
952
|
+
if (!q) return
|
|
953
|
+
|
|
954
|
+
recordAlias(q.from)
|
|
955
|
+
|
|
956
|
+
if (q.join) {
|
|
957
|
+
for (const joinClause of q.join) {
|
|
958
|
+
recordAlias(joinClause.from)
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
traverse(query)
|
|
964
|
+
|
|
965
|
+
return aliasesById
|
|
966
|
+
}
|
|
967
|
+
|
|
527
968
|
function accumulateChanges<T>(
|
|
528
969
|
acc: Map<unknown, Changes<T>>,
|
|
529
970
|
[[key, tupleData], multiplicity]: [
|