@tanstack/db 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/dist/cjs/collection/events.cjs +9 -51
  2. package/dist/cjs/collection/events.cjs.map +1 -1
  3. package/dist/cjs/collection/events.d.cts +18 -7
  4. package/dist/cjs/collection/index.cjs +9 -12
  5. package/dist/cjs/collection/index.cjs.map +1 -1
  6. package/dist/cjs/collection/index.d.cts +13 -14
  7. package/dist/cjs/collection/subscription.cjs +62 -6
  8. package/dist/cjs/collection/subscription.cjs.map +1 -1
  9. package/dist/cjs/collection/subscription.d.cts +16 -3
  10. package/dist/cjs/collection/sync.cjs +58 -6
  11. package/dist/cjs/collection/sync.cjs.map +1 -1
  12. package/dist/cjs/collection/sync.d.cts +18 -4
  13. package/dist/cjs/errors.cjs +59 -17
  14. package/dist/cjs/errors.cjs.map +1 -1
  15. package/dist/cjs/errors.d.cts +44 -8
  16. package/dist/cjs/event-emitter.cjs +94 -0
  17. package/dist/cjs/event-emitter.cjs.map +1 -0
  18. package/dist/cjs/event-emitter.d.cts +45 -0
  19. package/dist/cjs/index.cjs +9 -4
  20. package/dist/cjs/index.cjs.map +1 -1
  21. package/dist/cjs/local-only.cjs.map +1 -1
  22. package/dist/cjs/local-only.d.cts +2 -5
  23. package/dist/cjs/query/builder/types.d.cts +1 -1
  24. package/dist/cjs/query/compiler/index.cjs +46 -19
  25. package/dist/cjs/query/compiler/index.cjs.map +1 -1
  26. package/dist/cjs/query/compiler/index.d.cts +35 -9
  27. package/dist/cjs/query/compiler/joins.cjs +91 -66
  28. package/dist/cjs/query/compiler/joins.cjs.map +1 -1
  29. package/dist/cjs/query/compiler/joins.d.cts +6 -3
  30. package/dist/cjs/query/compiler/order-by.cjs +20 -4
  31. package/dist/cjs/query/compiler/order-by.cjs.map +1 -1
  32. package/dist/cjs/query/compiler/order-by.d.cts +3 -1
  33. package/dist/cjs/query/compiler/select.cjs.map +1 -1
  34. package/dist/cjs/query/compiler/types.d.cts +4 -0
  35. package/dist/cjs/query/index.d.cts +1 -0
  36. package/dist/cjs/query/live/collection-config-builder.cjs +306 -46
  37. package/dist/cjs/query/live/collection-config-builder.cjs.map +1 -1
  38. package/dist/cjs/query/live/collection-config-builder.d.cts +97 -9
  39. package/dist/cjs/query/live/collection-registry.cjs +16 -0
  40. package/dist/cjs/query/live/collection-registry.cjs.map +1 -0
  41. package/dist/cjs/query/live/collection-registry.d.cts +26 -0
  42. package/dist/cjs/query/live/collection-subscriber.cjs +86 -58
  43. package/dist/cjs/query/live/collection-subscriber.cjs.map +1 -1
  44. package/dist/cjs/query/live/collection-subscriber.d.cts +5 -7
  45. package/dist/cjs/query/live-query-collection.cjs +11 -5
  46. package/dist/cjs/query/live-query-collection.cjs.map +1 -1
  47. package/dist/cjs/query/live-query-collection.d.cts +12 -5
  48. package/dist/cjs/query/optimizer.cjs +44 -7
  49. package/dist/cjs/query/optimizer.cjs.map +1 -1
  50. package/dist/cjs/query/optimizer.d.cts +4 -4
  51. package/dist/cjs/scheduler.cjs +137 -0
  52. package/dist/cjs/scheduler.cjs.map +1 -0
  53. package/dist/cjs/scheduler.d.cts +56 -0
  54. package/dist/cjs/transactions.cjs +7 -1
  55. package/dist/cjs/transactions.cjs.map +1 -1
  56. package/dist/cjs/types.d.cts +82 -11
  57. package/dist/esm/collection/events.d.ts +18 -7
  58. package/dist/esm/collection/events.js +9 -51
  59. package/dist/esm/collection/events.js.map +1 -1
  60. package/dist/esm/collection/index.d.ts +13 -14
  61. package/dist/esm/collection/index.js +9 -12
  62. package/dist/esm/collection/index.js.map +1 -1
  63. package/dist/esm/collection/subscription.d.ts +16 -3
  64. package/dist/esm/collection/subscription.js +62 -6
  65. package/dist/esm/collection/subscription.js.map +1 -1
  66. package/dist/esm/collection/sync.d.ts +18 -4
  67. package/dist/esm/collection/sync.js +59 -7
  68. package/dist/esm/collection/sync.js.map +1 -1
  69. package/dist/esm/errors.d.ts +44 -8
  70. package/dist/esm/errors.js +60 -18
  71. package/dist/esm/errors.js.map +1 -1
  72. package/dist/esm/event-emitter.d.ts +45 -0
  73. package/dist/esm/event-emitter.js +94 -0
  74. package/dist/esm/event-emitter.js.map +1 -0
  75. package/dist/esm/index.js +10 -5
  76. package/dist/esm/local-only.d.ts +2 -5
  77. package/dist/esm/local-only.js.map +1 -1
  78. package/dist/esm/query/builder/types.d.ts +1 -1
  79. package/dist/esm/query/compiler/index.d.ts +35 -9
  80. package/dist/esm/query/compiler/index.js +46 -19
  81. package/dist/esm/query/compiler/index.js.map +1 -1
  82. package/dist/esm/query/compiler/joins.d.ts +6 -3
  83. package/dist/esm/query/compiler/joins.js +93 -68
  84. package/dist/esm/query/compiler/joins.js.map +1 -1
  85. package/dist/esm/query/compiler/order-by.d.ts +3 -1
  86. package/dist/esm/query/compiler/order-by.js +20 -4
  87. package/dist/esm/query/compiler/order-by.js.map +1 -1
  88. package/dist/esm/query/compiler/select.js.map +1 -1
  89. package/dist/esm/query/compiler/types.d.ts +4 -0
  90. package/dist/esm/query/index.d.ts +1 -0
  91. package/dist/esm/query/live/collection-config-builder.d.ts +97 -9
  92. package/dist/esm/query/live/collection-config-builder.js +306 -46
  93. package/dist/esm/query/live/collection-config-builder.js.map +1 -1
  94. package/dist/esm/query/live/collection-registry.d.ts +26 -0
  95. package/dist/esm/query/live/collection-registry.js +16 -0
  96. package/dist/esm/query/live/collection-registry.js.map +1 -0
  97. package/dist/esm/query/live/collection-subscriber.d.ts +5 -7
  98. package/dist/esm/query/live/collection-subscriber.js +86 -58
  99. package/dist/esm/query/live/collection-subscriber.js.map +1 -1
  100. package/dist/esm/query/live-query-collection.d.ts +12 -5
  101. package/dist/esm/query/live-query-collection.js +11 -5
  102. package/dist/esm/query/live-query-collection.js.map +1 -1
  103. package/dist/esm/query/optimizer.d.ts +4 -4
  104. package/dist/esm/query/optimizer.js +44 -7
  105. package/dist/esm/query/optimizer.js.map +1 -1
  106. package/dist/esm/scheduler.d.ts +56 -0
  107. package/dist/esm/scheduler.js +137 -0
  108. package/dist/esm/scheduler.js.map +1 -0
  109. package/dist/esm/transactions.js +7 -1
  110. package/dist/esm/transactions.js.map +1 -1
  111. package/dist/esm/types.d.ts +82 -11
  112. package/package.json +2 -2
  113. package/src/collection/events.ts +25 -74
  114. package/src/collection/index.ts +15 -19
  115. package/src/collection/subscription.ts +88 -6
  116. package/src/collection/sync.ts +81 -9
  117. package/src/errors.ts +91 -13
  118. package/src/event-emitter.ts +118 -0
  119. package/src/local-only.ts +5 -12
  120. package/src/query/builder/types.ts +1 -1
  121. package/src/query/compiler/index.ts +124 -33
  122. package/src/query/compiler/joins.ts +187 -128
  123. package/src/query/compiler/order-by.ts +30 -2
  124. package/src/query/compiler/select.ts +2 -3
  125. package/src/query/compiler/types.ts +5 -0
  126. package/src/query/index.ts +1 -0
  127. package/src/query/live/collection-config-builder.ts +501 -60
  128. package/src/query/live/collection-registry.ts +47 -0
  129. package/src/query/live/collection-subscriber.ts +137 -105
  130. package/src/query/live-query-collection.ts +47 -18
  131. package/src/query/optimizer.ts +85 -15
  132. package/src/scheduler.ts +198 -0
  133. package/src/transactions.ts +12 -1
  134. package/src/types.ts +93 -11
@@ -1,7 +1,16 @@
1
1
  import { D2, output } from "@tanstack/db-ivm"
2
2
  import { compileQuery } from "../compiler/index.js"
3
3
  import { buildQuery, getQueryIR } from "../builder/index.js"
4
+ import {
5
+ MissingAliasInputsError,
6
+ SetWindowRequiresOrderByError,
7
+ } from "../../errors.js"
8
+ import { transactionScopedScheduler } from "../../scheduler.js"
9
+ import { getActiveTransaction } from "../../transactions.js"
4
10
  import { CollectionSubscriber } from "./collection-subscriber.js"
11
+ import { getCollectionBuilder } from "./collection-registry.js"
12
+ import type { WindowOptions } from "../compiler/index.js"
13
+ import type { SchedulerContextId } from "../../scheduler.js"
5
14
  import type { CollectionSubscription } from "../../collection/subscription.js"
6
15
  import type { RootStreamBuilder } from "@tanstack/db-ivm"
7
16
  import type { OrderByOptimizationInfo } from "../compiler/order-by.js"
@@ -11,6 +20,7 @@ import type {
11
20
  KeyedStream,
12
21
  ResultStream,
13
22
  SyncConfig,
23
+ UtilsRecord,
14
24
  } from "../../types.js"
15
25
  import type { Context, GetResult } from "../builder/types.js"
16
26
  import type { BasicExpression, QueryIR } from "../ir.js"
@@ -23,6 +33,22 @@ import type {
23
33
  } from "./types.js"
24
34
  import type { AllCollectionEvents } from "../../collection/events.js"
25
35
 
36
+ export type LiveQueryCollectionUtils = UtilsRecord & {
37
+ getRunCount: () => number
38
+ getBuilder: () => CollectionConfigBuilder<any, any>
39
+ /**
40
+ * Sets the offset and limit of an ordered query.
41
+ * Is a no-op if the query is not ordered.
42
+ *
43
+ * @returns `true` if no subset loading was triggered, or `Promise<void>` that resolves when the subset has been loaded
44
+ */
45
+ setWindow: (options: WindowOptions) => true | Promise<void>
46
+ }
47
+
48
+ type PendingGraphRun = {
49
+ loadCallbacks: Set<() => boolean>
50
+ }
51
+
26
52
  // Global counter for auto-generated collection IDs
27
53
  let liveQueryCollectionCounter = 0
28
54
 
@@ -37,6 +63,9 @@ export class CollectionConfigBuilder<
37
63
  private readonly id: string
38
64
  readonly query: QueryIR
39
65
  private readonly collections: Record<string, Collection<any, any, any>>
66
+ private readonly collectionByAlias: Record<string, Collection<any, any, any>>
67
+ // Populated during compilation with all aliases (including subquery inner aliases)
68
+ private compiledAliasToCollectionId: Record<string, string> = {}
40
69
 
41
70
  // WeakMap to store the keys of the results
42
71
  // so that we can retrieve them in the getKey function
@@ -48,26 +77,60 @@ export class CollectionConfigBuilder<
48
77
  private readonly compare?: (val1: TResult, val2: TResult) => number
49
78
 
50
79
  private isGraphRunning = false
80
+ private runCount = 0
81
+
82
+ // Current sync session state (set when sync starts, cleared when it stops)
83
+ // Public for testing purposes (CollectionConfigBuilder is internal, not public API)
84
+ public currentSyncConfig:
85
+ | Parameters<SyncConfig<TResult>[`sync`]>[0]
86
+ | undefined
87
+ public currentSyncState: FullSyncState | undefined
51
88
 
52
89
  // Error state tracking
53
90
  private isInErrorState = false
54
91
 
55
92
  // Reference to the live query collection for error state transitions
56
- private liveQueryCollection?: Collection<TResult, any, any>
93
+ public liveQueryCollection?: Collection<TResult, any, any>
94
+
95
+ private windowFn: ((options: WindowOptions) => void) | undefined
96
+
97
+ private maybeRunGraphFn: (() => void) | undefined
98
+
99
+ private readonly aliasDependencies: Record<
100
+ string,
101
+ Array<CollectionConfigBuilder<any, any>>
102
+ > = {}
103
+
104
+ private readonly builderDependencies = new Set<
105
+ CollectionConfigBuilder<any, any>
106
+ >()
107
+
108
+ // Pending graph runs per scheduler context (e.g., per transaction)
109
+ // The builder manages its own state; the scheduler just orchestrates execution order
110
+ // Only stores callbacks - if sync ends, pending jobs gracefully no-op
111
+ private readonly pendingGraphRuns = new Map<
112
+ SchedulerContextId,
113
+ PendingGraphRun
114
+ >()
115
+
116
+ // Unsubscribe function for scheduler's onClear listener
117
+ // Registered when sync starts, unregistered when sync stops
118
+ // Prevents memory leaks by releasing the scheduler's reference to this builder
119
+ private unsubscribeFromSchedulerClears?: () => void
57
120
 
58
121
  private graphCache: D2 | undefined
59
122
  private inputsCache: Record<string, RootStreamBuilder<unknown>> | undefined
60
123
  private pipelineCache: ResultStream | undefined
61
- public collectionWhereClausesCache:
124
+ public sourceWhereClausesCache:
62
125
  | Map<string, BasicExpression<boolean>>
63
126
  | undefined
64
127
 
65
- // Map of collection ID to subscription
128
+ // Map of source alias to subscription
66
129
  readonly subscriptions: Record<string, CollectionSubscription> = {}
67
- // Map of collection IDs to functions that load keys for that lazy collection
68
- lazyCollectionsCallbacks: Record<string, LazyCollectionCallbacks> = {}
69
- // Set of collection IDs that are lazy collections
70
- readonly lazyCollections = new Set<string>()
130
+ // Map of source aliases to functions that load keys for that lazy source
131
+ lazySourcesCallbacks: Record<string, LazyCollectionCallbacks> = {}
132
+ // Set of source aliases that are lazy (don't load initial state)
133
+ readonly lazySources = new Set<string>()
71
134
  // Set of collection IDs that include an optimizable ORDER BY clause
72
135
  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo> = {}
73
136
 
@@ -79,6 +142,19 @@ export class CollectionConfigBuilder<
79
142
 
80
143
  this.query = buildQueryFromConfig(config)
81
144
  this.collections = extractCollectionsFromQuery(this.query)
145
+ const collectionAliasesById = extractCollectionAliases(this.query)
146
+
147
+ // Build a reverse lookup map from alias to collection instance.
148
+ // This enables self-join support where the same collection can be referenced
149
+ // multiple times with different aliases (e.g., { employee: col, manager: col })
150
+ this.collectionByAlias = {}
151
+ for (const [collectionId, aliases] of collectionAliasesById.entries()) {
152
+ const collection = this.collections[collectionId]
153
+ if (!collection) continue
154
+ for (const alias of aliases) {
155
+ this.collectionByAlias[alias] = collection
156
+ }
157
+ }
82
158
 
83
159
  // Create compare function for ordering if the query has orderBy
84
160
  if (this.query.orderBy && this.query.orderBy.length > 0) {
@@ -90,7 +166,9 @@ export class CollectionConfigBuilder<
90
166
  this.compileBasePipeline()
91
167
  }
92
168
 
93
- getConfig(): CollectionConfigSingleRowOption<TResult> {
169
+ getConfig(): CollectionConfigSingleRowOption<TResult> & {
170
+ utils: LiveQueryCollectionUtils
171
+ } {
94
172
  return {
95
173
  id: this.id,
96
174
  getKey:
@@ -105,9 +183,69 @@ export class CollectionConfigBuilder<
105
183
  onDelete: this.config.onDelete,
106
184
  startSync: this.config.startSync,
107
185
  singleResult: this.query.singleResult,
186
+ utils: {
187
+ getRunCount: this.getRunCount.bind(this),
188
+ getBuilder: () => this,
189
+ setWindow: this.setWindow.bind(this),
190
+ },
108
191
  }
109
192
  }
110
193
 
194
+ setWindow(options: WindowOptions): true | Promise<void> {
195
+ if (!this.windowFn) {
196
+ throw new SetWindowRequiresOrderByError()
197
+ }
198
+
199
+ this.windowFn(options)
200
+ this.maybeRunGraphFn?.()
201
+
202
+ // Check if loading a subset was triggered
203
+ if (this.liveQueryCollection?.isLoadingSubset) {
204
+ // Loading was triggered, return a promise that resolves when it completes
205
+ return new Promise<void>((resolve) => {
206
+ const unsubscribe = this.liveQueryCollection!.on(
207
+ `loadingSubset:change`,
208
+ (event) => {
209
+ if (!event.isLoadingSubset) {
210
+ unsubscribe()
211
+ resolve()
212
+ }
213
+ }
214
+ )
215
+ })
216
+ }
217
+
218
+ // No loading was triggered
219
+ return true
220
+ }
221
+
222
+ /**
223
+ * Resolves a collection alias to its collection ID.
224
+ *
225
+ * Uses a two-tier lookup strategy:
226
+ * 1. First checks compiled aliases (includes subquery inner aliases)
227
+ * 2. Falls back to declared aliases from the query's from/join clauses
228
+ *
229
+ * @param alias - The alias to resolve (e.g., "employee", "manager")
230
+ * @returns The collection ID that the alias references
231
+ * @throws {Error} If the alias is not found in either lookup
232
+ */
233
+ getCollectionIdForAlias(alias: string): string {
234
+ const compiled = this.compiledAliasToCollectionId[alias]
235
+ if (compiled) {
236
+ return compiled
237
+ }
238
+ const collection = this.collectionByAlias[alias]
239
+ if (collection) {
240
+ return collection.id
241
+ }
242
+ throw new Error(`Unknown source alias "${alias}"`)
243
+ }
244
+
245
+ isLazyAlias(alias: string): boolean {
246
+ return this.lazySources.has(alias)
247
+ }
248
+
111
249
  // The callback function is called after the graph has run.
112
250
  // This gives the callback a chance to load more data if needed,
113
251
  // that's used to optimize orderBy operators that set a limit,
@@ -115,12 +253,8 @@ export class CollectionConfigBuilder<
115
253
  // That can happen because even though we load N rows, the pipeline might filter some of these rows out
116
254
  // causing the orderBy operator to receive less than N rows or even no rows at all.
117
255
  // So this callback would notice that it doesn't have enough rows and load some more.
118
- // The callback returns a boolean, when it's true it's done loading data.
119
- maybeRunGraph(
120
- config: SyncMethods<TResult>,
121
- syncState: FullSyncState,
122
- callback?: () => boolean
123
- ) {
256
+ // The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready.
257
+ maybeRunGraph(callback?: () => boolean) {
124
258
  if (this.isGraphRunning) {
125
259
  // no nested runs of the graph
126
260
  // which is possible if the `callback`
@@ -128,10 +262,18 @@ export class CollectionConfigBuilder<
128
262
  return
129
263
  }
130
264
 
265
+ // Should only be called when sync is active
266
+ if (!this.currentSyncConfig || !this.currentSyncState) {
267
+ throw new Error(
268
+ `maybeRunGraph called without active sync session. This should not happen.`
269
+ )
270
+ }
271
+
131
272
  this.isGraphRunning = true
132
273
 
133
274
  try {
134
- const { begin, commit } = config
275
+ const { begin, commit } = this.currentSyncConfig
276
+ const syncState = this.currentSyncState
135
277
 
136
278
  // Don't run if the live query is in an error state
137
279
  if (this.isInErrorState) {
@@ -152,7 +294,7 @@ export class CollectionConfigBuilder<
152
294
  commit()
153
295
  // After initial commit, check if we should mark ready
154
296
  // (in case all sources were already ready before we subscribed)
155
- this.updateLiveQueryStatus(config)
297
+ this.updateLiveQueryStatus(this.currentSyncConfig)
156
298
  }
157
299
  }
158
300
  } finally {
@@ -160,6 +302,158 @@ export class CollectionConfigBuilder<
160
302
  }
161
303
  }
162
304
 
305
+ /**
306
+ * Schedules a graph run with the transaction-scoped scheduler.
307
+ * Ensures each builder runs at most once per transaction, with automatic dependency tracking
308
+ * to run parent queries before child queries. Outside a transaction, runs immediately.
309
+ *
310
+ * Multiple calls during a transaction are coalesced into a single execution.
311
+ * Dependencies are auto-discovered from subscribed live queries, or can be overridden.
312
+ * Load callbacks are combined when entries merge.
313
+ *
314
+ * Uses the current sync session's config and syncState from instance properties.
315
+ *
316
+ * @param callback - Optional callback to load more data if needed (returns true when done)
317
+ * @param options - Optional scheduling configuration
318
+ * @param options.contextId - Transaction ID to group work; defaults to active transaction
319
+ * @param options.jobId - Unique identifier for this job; defaults to this builder instance
320
+ * @param options.alias - Source alias that triggered this schedule; adds alias-specific dependencies
321
+ * @param options.dependencies - Explicit dependency list; overrides auto-discovered dependencies
322
+ */
323
+ scheduleGraphRun(
324
+ callback?: () => boolean,
325
+ options?: {
326
+ contextId?: SchedulerContextId
327
+ jobId?: unknown
328
+ alias?: string
329
+ dependencies?: Array<CollectionConfigBuilder<any, any>>
330
+ }
331
+ ) {
332
+ const contextId = options?.contextId ?? getActiveTransaction()?.id
333
+ // Use the builder instance as the job ID for deduplication. This is memory-safe
334
+ // because the scheduler's context Map is deleted after flushing (no long-term retention).
335
+ const jobId = options?.jobId ?? this
336
+ const dependentBuilders = (() => {
337
+ if (options?.dependencies) {
338
+ return options.dependencies
339
+ }
340
+
341
+ const deps = new Set(this.builderDependencies)
342
+ if (options?.alias) {
343
+ const aliasDeps = this.aliasDependencies[options.alias]
344
+ if (aliasDeps) {
345
+ for (const dep of aliasDeps) {
346
+ deps.add(dep)
347
+ }
348
+ }
349
+ }
350
+
351
+ deps.delete(this)
352
+
353
+ return Array.from(deps)
354
+ })()
355
+
356
+ // We intentionally scope deduplication to the builder instance. Each instance
357
+ // owns caches and compiled pipelines, so sharing work across instances that
358
+ // merely reuse the same string id would execute the wrong builder's graph.
359
+
360
+ if (!this.currentSyncConfig || !this.currentSyncState) {
361
+ throw new Error(
362
+ `scheduleGraphRun called without active sync session. This should not happen.`
363
+ )
364
+ }
365
+
366
+ // Manage our own state - get or create pending callbacks for this context
367
+ let pending = contextId ? this.pendingGraphRuns.get(contextId) : undefined
368
+ if (!pending) {
369
+ pending = {
370
+ loadCallbacks: new Set(),
371
+ }
372
+ if (contextId) {
373
+ this.pendingGraphRuns.set(contextId, pending)
374
+ }
375
+ }
376
+
377
+ // Add callback if provided (this is what accumulates between schedules)
378
+ if (callback) {
379
+ pending.loadCallbacks.add(callback)
380
+ }
381
+
382
+ // Schedule execution (scheduler just orchestrates order, we manage state)
383
+ // For immediate execution (no contextId), pass pending directly since it won't be in the map
384
+ const pendingToPass = contextId ? undefined : pending
385
+ transactionScopedScheduler.schedule({
386
+ contextId,
387
+ jobId,
388
+ dependencies: dependentBuilders,
389
+ run: () => this.executeGraphRun(contextId, pendingToPass),
390
+ })
391
+ }
392
+
393
+ /**
394
+ * Clears pending graph run state for a specific context.
395
+ * Called when the scheduler clears a context (e.g., transaction rollback/abort).
396
+ */
397
+ clearPendingGraphRun(contextId: SchedulerContextId): void {
398
+ this.pendingGraphRuns.delete(contextId)
399
+ }
400
+
401
+ /**
402
+ * Executes a pending graph run. Called by the scheduler when dependencies are satisfied.
403
+ * Clears the pending state BEFORE execution so that any re-schedules during the run
404
+ * create fresh state and don't interfere with the current execution.
405
+ * Uses instance sync state - if sync has ended, gracefully returns without executing.
406
+ *
407
+ * @param contextId - Optional context ID to look up pending state
408
+ * @param pendingParam - For immediate execution (no context), pending state is passed directly
409
+ */
410
+ private executeGraphRun(
411
+ contextId?: SchedulerContextId,
412
+ pendingParam?: PendingGraphRun
413
+ ): void {
414
+ // Get pending state: either from parameter (no context) or from map (with context)
415
+ // Remove from map BEFORE checking sync state to prevent leaking entries when sync ends
416
+ // before the transaction flushes (e.g., unsubscribe during in-flight transaction)
417
+ const pending =
418
+ pendingParam ??
419
+ (contextId ? this.pendingGraphRuns.get(contextId) : undefined)
420
+ if (contextId) {
421
+ this.pendingGraphRuns.delete(contextId)
422
+ }
423
+
424
+ // If no pending state, nothing to execute (context was cleared)
425
+ if (!pending) {
426
+ return
427
+ }
428
+
429
+ // If sync session has ended, don't execute (graph is finalized, subscriptions cleared)
430
+ if (!this.currentSyncConfig || !this.currentSyncState) {
431
+ return
432
+ }
433
+
434
+ this.incrementRunCount()
435
+
436
+ const combinedLoader = () => {
437
+ let allDone = true
438
+ let firstError: unknown
439
+ pending.loadCallbacks.forEach((loader) => {
440
+ try {
441
+ allDone = loader() && allDone
442
+ } catch (error) {
443
+ allDone = false
444
+ firstError ??= error
445
+ }
446
+ })
447
+ if (firstError) {
448
+ throw firstError
449
+ }
450
+ // Returning false signals that callers should schedule another pass.
451
+ return allDone
452
+ }
453
+
454
+ this.maybeRunGraph(combinedLoader)
455
+ }
456
+
163
457
  private getSyncConfig(): SyncConfig<TResult> {
164
458
  return {
165
459
  rowUpdateMode: `full`,
@@ -167,9 +461,19 @@ export class CollectionConfigBuilder<
167
461
  }
168
462
  }
169
463
 
464
+ incrementRunCount() {
465
+ this.runCount++
466
+ }
467
+
468
+ getRunCount() {
469
+ return this.runCount
470
+ }
471
+
170
472
  private syncFn(config: SyncMethods<TResult>) {
171
473
  // Store reference to the live query collection for error state transitions
172
474
  this.liveQueryCollection = config.collection
475
+ // Store config and syncState as instance properties for the duration of this sync session
476
+ this.currentSyncConfig = config
173
477
 
174
478
  const syncState: SyncState = {
175
479
  messagesCount: 0,
@@ -182,58 +486,102 @@ export class CollectionConfigBuilder<
182
486
  config,
183
487
  syncState
184
488
  )
489
+ this.currentSyncState = fullSyncState
185
490
 
186
- const loadMoreDataCallbacks = this.subscribeToAllCollections(
491
+ // Listen for scheduler context clears to clean up our pending state
492
+ // Re-register on each sync start so the listener is active for the sync session's lifetime
493
+ this.unsubscribeFromSchedulerClears = transactionScopedScheduler.onClear(
494
+ (contextId) => {
495
+ this.clearPendingGraphRun(contextId)
496
+ }
497
+ )
498
+
499
+ const loadSubsetDataCallbacks = this.subscribeToAllCollections(
187
500
  config,
188
501
  fullSyncState
189
502
  )
190
503
 
504
+ this.maybeRunGraphFn = () => this.scheduleGraphRun(loadSubsetDataCallbacks)
505
+
191
506
  // Initial run with callback to load more data if needed
192
- this.maybeRunGraph(config, fullSyncState, loadMoreDataCallbacks)
507
+ this.scheduleGraphRun(loadSubsetDataCallbacks)
193
508
 
194
509
  // Return the unsubscribe function
195
510
  return () => {
196
511
  syncState.unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe())
197
512
 
513
+ // Clear current sync session state
514
+ this.currentSyncConfig = undefined
515
+ this.currentSyncState = undefined
516
+
517
+ // Clear all pending graph runs to prevent memory leaks from in-flight transactions
518
+ // that may flush after the sync session ends
519
+ this.pendingGraphRuns.clear()
520
+
198
521
  // Reset caches so a fresh graph/pipeline is compiled on next start
199
522
  // This avoids reusing a finalized D2 graph across GC restarts
200
523
  this.graphCache = undefined
201
524
  this.inputsCache = undefined
202
525
  this.pipelineCache = undefined
203
- this.collectionWhereClausesCache = undefined
526
+ this.sourceWhereClausesCache = undefined
204
527
 
205
- // Reset lazy collection state
206
- this.lazyCollections.clear()
528
+ // Reset lazy source alias state
529
+ this.lazySources.clear()
207
530
  this.optimizableOrderByCollections = {}
208
- this.lazyCollectionsCallbacks = {}
531
+ this.lazySourcesCallbacks = {}
532
+
533
+ // Clear subscription references to prevent memory leaks
534
+ // Note: Individual subscriptions are already unsubscribed via unsubscribeCallbacks
535
+ Object.keys(this.subscriptions).forEach(
536
+ (key) => delete this.subscriptions[key]
537
+ )
538
+ this.compiledAliasToCollectionId = {}
539
+
540
+ // Unregister from scheduler's onClear listener to prevent memory leaks
541
+ // The scheduler's listener Set would otherwise keep a strong reference to this builder
542
+ this.unsubscribeFromSchedulerClears?.()
543
+ this.unsubscribeFromSchedulerClears = undefined
209
544
  }
210
545
  }
211
546
 
547
+ /**
548
+ * Compiles the query pipeline with all declared aliases.
549
+ */
212
550
  private compileBasePipeline() {
213
551
  this.graphCache = new D2()
214
552
  this.inputsCache = Object.fromEntries(
215
- Object.entries(this.collections).map(([key]) => [
216
- key,
553
+ Object.keys(this.collectionByAlias).map((alias) => [
554
+ alias,
217
555
  this.graphCache!.newInput<any>(),
218
556
  ])
219
557
  )
220
558
 
221
- // Compile the query and get both pipeline and collection WHERE clauses
222
- const {
223
- pipeline: pipelineCache,
224
- collectionWhereClauses: collectionWhereClausesCache,
225
- } = compileQuery(
559
+ const compilation = compileQuery(
226
560
  this.query,
227
561
  this.inputsCache as Record<string, KeyedStream>,
228
562
  this.collections,
229
563
  this.subscriptions,
230
- this.lazyCollectionsCallbacks,
231
- this.lazyCollections,
232
- this.optimizableOrderByCollections
564
+ this.lazySourcesCallbacks,
565
+ this.lazySources,
566
+ this.optimizableOrderByCollections,
567
+ (windowFn: (options: WindowOptions) => void) => {
568
+ this.windowFn = windowFn
569
+ }
233
570
  )
234
571
 
235
- this.pipelineCache = pipelineCache
236
- this.collectionWhereClausesCache = collectionWhereClausesCache
572
+ this.pipelineCache = compilation.pipeline
573
+ this.sourceWhereClausesCache = compilation.sourceWhereClauses
574
+ this.compiledAliasToCollectionId = compilation.aliasToCollectionId
575
+
576
+ // Defensive check: verify all compiled aliases have corresponding inputs
577
+ // This should never happen since all aliases come from user declarations,
578
+ // but catch it early if the assumption is violated in the future.
579
+ const missingAliases = Object.keys(this.compiledAliasToCollectionId).filter(
580
+ (alias) => !Object.hasOwn(this.inputsCache!, alias)
581
+ )
582
+ if (missingAliases.length > 0) {
583
+ throw new MissingAliasInputsError(missingAliases)
584
+ }
237
585
  }
238
586
 
239
587
  private maybeCompileBasePipeline() {
@@ -400,50 +748,84 @@ export class CollectionConfigBuilder<
400
748
  )
401
749
  }
402
750
 
751
+ /**
752
+ * Creates per-alias subscriptions enabling self-join support.
753
+ * Each alias gets its own subscription with independent filters, even for the same collection.
754
+ * Example: `{ employee: col, manager: col }` creates two separate subscriptions.
755
+ */
403
756
  private subscribeToAllCollections(
404
757
  config: SyncMethods<TResult>,
405
758
  syncState: FullSyncState
406
759
  ) {
407
- const loaders = Object.entries(this.collections).map(
408
- ([collectionId, collection]) => {
409
- const collectionSubscriber = new CollectionSubscriber(
410
- collectionId,
411
- collection,
412
- config,
413
- syncState,
414
- this
415
- )
760
+ // Use compiled aliases as the source of truth - these include all aliases from the query
761
+ // including those from subqueries, which may not be in collectionByAlias
762
+ const compiledAliases = Object.entries(this.compiledAliasToCollectionId)
763
+ if (compiledAliases.length === 0) {
764
+ throw new Error(
765
+ `Compiler returned no alias metadata for query '${this.id}'. This should not happen; please report.`
766
+ )
767
+ }
416
768
 
417
- const subscription = collectionSubscriber.subscribe()
418
- this.subscriptions[collectionId] = subscription
769
+ // Create a separate subscription for each alias, enabling self-joins where the same
770
+ // collection can be used multiple times with different filters and subscriptions
771
+ const loaders = compiledAliases.map(([alias, collectionId]) => {
772
+ // Try collectionByAlias first (for declared aliases), fall back to collections (for subquery aliases)
773
+ const collection =
774
+ this.collectionByAlias[alias] ?? this.collections[collectionId]!
775
+
776
+ const dependencyBuilder = getCollectionBuilder(collection)
777
+ if (dependencyBuilder && dependencyBuilder !== this) {
778
+ this.aliasDependencies[alias] = [dependencyBuilder]
779
+ this.builderDependencies.add(dependencyBuilder)
780
+ } else {
781
+ this.aliasDependencies[alias] = []
782
+ }
419
783
 
420
- // Subscribe to status changes for status flow
421
- const statusUnsubscribe = collection.on(`status:change`, (event) => {
422
- this.handleSourceStatusChange(config, collectionId, event)
423
- })
424
- syncState.unsubscribeCallbacks.add(statusUnsubscribe)
784
+ // CollectionSubscriber handles the actual subscription to the source collection
785
+ // and feeds data into the D2 graph inputs for this specific alias
786
+ const collectionSubscriber = new CollectionSubscriber(
787
+ alias,
788
+ collectionId,
789
+ collection,
790
+ this
791
+ )
425
792
 
426
- const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(
427
- collectionSubscriber,
428
- subscription
429
- )
793
+ // Subscribe to status changes for status flow
794
+ const statusUnsubscribe = collection.on(`status:change`, (event) => {
795
+ this.handleSourceStatusChange(config, collectionId, event)
796
+ })
797
+ syncState.unsubscribeCallbacks.add(statusUnsubscribe)
430
798
 
431
- return loadMore
432
- }
433
- )
799
+ const subscription = collectionSubscriber.subscribe()
800
+ // Store subscription by alias (not collection ID) to support lazy loading
801
+ // which needs to look up subscriptions by their query alias
802
+ this.subscriptions[alias] = subscription
803
+
804
+ // Create a callback for loading more data if needed (used by OrderBy optimization)
805
+ const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(
806
+ collectionSubscriber,
807
+ subscription
808
+ )
809
+
810
+ return loadMore
811
+ })
434
812
 
435
- const loadMoreDataCallback = () => {
813
+ // Combine all loaders into a single callback that initiates loading more data
814
+ // from any source that needs it. Returns true once all loaders have been called,
815
+ // but the actual async loading may still be in progress.
816
+ const loadSubsetDataCallbacks = () => {
436
817
  loaders.map((loader) => loader())
437
818
  return true
438
819
  }
439
820
 
440
- // Mark the collections as subscribed in the sync state
821
+ // Mark as subscribed so the graph can start running
822
+ // (graph only runs when all collections are subscribed)
441
823
  syncState.subscribedToAllCollections = true
442
824
 
443
825
  // Initial status check after all subscriptions are set up
444
826
  this.updateLiveQueryStatus(config)
445
827
 
446
- return loadMoreDataCallback
828
+ return loadSubsetDataCallbacks
447
829
  }
448
830
  }
449
831
 
@@ -524,6 +906,65 @@ function extractCollectionsFromQuery(
524
906
  return collections
525
907
  }
526
908
 
909
+ /**
910
+ * Extracts all aliases used for each collection across the entire query tree.
911
+ *
912
+ * Traverses the QueryIR recursively to build a map from collection ID to all aliases
913
+ * that reference that collection. This is essential for self-join support, where the
914
+ * same collection may be referenced multiple times with different aliases.
915
+ *
916
+ * For example, given a query like:
917
+ * ```ts
918
+ * q.from({ employee: employeesCollection })
919
+ * .join({ manager: employeesCollection }, ({ employee, manager }) =>
920
+ * eq(employee.managerId, manager.id)
921
+ * )
922
+ * ```
923
+ *
924
+ * This function would return:
925
+ * ```
926
+ * Map { "employees" => Set { "employee", "manager" } }
927
+ * ```
928
+ *
929
+ * @param query - The query IR to extract aliases from
930
+ * @returns A map from collection ID to the set of all aliases referencing that collection
931
+ */
932
+ function extractCollectionAliases(query: QueryIR): Map<string, Set<string>> {
933
+ const aliasesById = new Map<string, Set<string>>()
934
+
935
+ function recordAlias(source: any) {
936
+ if (!source) return
937
+
938
+ if (source.type === `collectionRef`) {
939
+ const { id } = source.collection
940
+ const existing = aliasesById.get(id)
941
+ if (existing) {
942
+ existing.add(source.alias)
943
+ } else {
944
+ aliasesById.set(id, new Set([source.alias]))
945
+ }
946
+ } else if (source.type === `queryRef`) {
947
+ traverse(source.query)
948
+ }
949
+ }
950
+
951
+ function traverse(q?: QueryIR) {
952
+ if (!q) return
953
+
954
+ recordAlias(q.from)
955
+
956
+ if (q.join) {
957
+ for (const joinClause of q.join) {
958
+ recordAlias(joinClause.from)
959
+ }
960
+ }
961
+ }
962
+
963
+ traverse(query)
964
+
965
+ return aliasesById
966
+ }
967
+
527
968
  function accumulateChanges<T>(
528
969
  acc: Map<unknown, Changes<T>>,
529
970
  [[key, tupleData], multiplicity]: [