@tech4good/angular-schematics 20.3.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/.agents/skills/publish-npm/SKILL.md +1 -1
  2. package/.agents/skills/publish-npm/eval.yaml +15 -0
  3. package/.agents/skills/publish-npm/fixtures/package-v21.json.template +8 -0
  4. package/.husky/commit-msg +1 -0
  5. package/.husky/pre-commit +1 -0
  6. package/.t4g/agent-memory.db +0 -0
  7. package/CHANGELOG.md +100 -26
  8. package/commitlint.config.js +15 -0
  9. package/contributing-docs/commit-message.md +3 -6
  10. package/package.json +10 -6
  11. package/src/collection.json +8 -0
  12. package/src/component/files/__name@dasherize__/__name@dasherize__.component.ts.template +1 -3
  13. package/src/component/index_spec.js +57 -4
  14. package/src/component/index_spec.js.map +1 -1
  15. package/src/entity/index.js +7 -1
  16. package/src/entity/index.js.map +1 -1
  17. package/src/entity/index_spec.js +108 -4
  18. package/src/entity/index_spec.js.map +1 -1
  19. package/src/guard/files/__name@dasherize__.guard.spec.ts.template +45 -0
  20. package/src/guard/files/__name@dasherize__.guard.ts.template +32 -0
  21. package/src/guard/index.d.ts +2 -0
  22. package/src/guard/index.js +38 -0
  23. package/src/guard/index.js.map +1 -0
  24. package/src/guard/index_spec.d.ts +1 -0
  25. package/src/guard/index_spec.js +64 -0
  26. package/src/guard/index_spec.js.map +1 -0
  27. package/src/guard/schema.json +46 -0
  28. package/src/ng-add/files/.firebaserc.template +3 -2
  29. package/src/ng-add/files/.github/workflows/angular-github-ci.yml.template +105 -39
  30. package/src/ng-add/files/.gitignore.template +1 -2
  31. package/src/ng-add/files/angular.json.template +1 -5
  32. package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +390 -0
  33. package/src/ng-add/files/docs-infra/load-system-architecture.md.template +166 -0
  34. package/src/ng-add/files/docs-infra/querying-strategy.md.template +157 -0
  35. package/src/ng-add/files/docs-infra/stream-system-architecture.md.template +283 -0
  36. package/src/ng-add/files/eslint.config.js.template +4 -1
  37. package/src/ng-add/files/firebase.json.template +37 -0
  38. package/src/ng-add/files/firestore.rules.spec.ts.template +221 -0
  39. package/src/ng-add/files/firestore.rules.template +67 -0
  40. package/src/ng-add/files/jest.config.ts.template +6 -1
  41. package/src/ng-add/files/package.json.template +41 -31
  42. package/src/ng-add/files/setup-jest.ts.template +13 -5
  43. package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +76 -12
  44. package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.ts.template +1 -3
  45. package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.spec.ts.template +90 -0
  46. package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.ts.template +1 -1
  47. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-counts/admin-count-pill/admin-count-pill.component.spec.ts.template +12 -17
  48. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-counts/admin-count-pill/admin-count-pill.component.ts.template +2 -4
  49. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-counts/admin-user-counts.component.spec.ts.template +40 -19
  50. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-counts/admin-user-counts.component.ts.template +3 -5
  51. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.html.template +2 -2
  52. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +108 -0
  53. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.ts.template +11 -11
  54. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.html.template +6 -2
  55. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +136 -19
  56. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.ts.template +8 -5
  57. package/src/ng-add/files/src/app/app.component.html.template +1 -1
  58. package/src/ng-add/files/src/app/app.component.scss.template +10 -0
  59. package/src/ng-add/files/src/app/app.component.ts.template +1 -1
  60. package/src/ng-add/files/src/app/app.config.ts.template +11 -11
  61. package/src/ng-add/files/src/app/auth-routing.spec.ts.template +101 -0
  62. package/src/ng-add/files/src/app/core/analytics/log.service.ts.template +5 -8
  63. package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +140 -0
  64. package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +25 -7
  65. package/src/ng-add/files/src/app/core/auth/guards/redirect-if-authenticated.guard.ts.template +1 -1
  66. package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.ts.template +1 -1
  67. package/src/ng-add/files/src/app/core/auth/guards/require-auth-and-consent.guard.ts.template +1 -1
  68. package/src/ng-add/files/src/app/core/auth/guards/require-auth.guard.ts.template +1 -1
  69. package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +229 -14
  70. package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +1 -1
  71. package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +34 -9
  72. package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +4 -4
  73. package/src/ng-add/files/src/app/core/firebase/query-logger.ts.template +11 -0
  74. package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +104 -0
  75. package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +127 -0
  76. package/src/ng-add/files/src/app/core/firebase/simulation.mock.ts.template +38 -0
  77. package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +443 -0
  78. package/src/ng-add/files/src/app/core/store/app.store.spec.ts.template +920 -0
  79. package/src/ng-add/files/src/app/core/store/app.store.ts.template +303 -137
  80. package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +65 -23
  81. package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +68 -67
  82. package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +46 -0
  83. package/src/ng-add/files/src/app/core/store/user-context/user-context.mock.ts.template +17 -0
  84. package/src/ng-add/files/src/app/core/testing/test-utils.ts.template +26 -0
  85. package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +3 -3
  86. package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +89 -13
  87. package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +46 -17
  88. package/src/ng-add/files/src/app/first-time/consent/consent.guard.ts.template +1 -1
  89. package/src/ng-add/files/src/app/first-time/early-access/early-access.component.html.template +5 -63
  90. package/src/ng-add/files/src/app/first-time/early-access/early-access.component.scss.template +1 -94
  91. package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +150 -14
  92. package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +15 -42
  93. package/src/ng-add/files/src/app/first-time/early-access/early-access.guard.ts.template +2 -2
  94. package/src/ng-add/files/src/app/first-time/early-access/early-access.model.ts.template +5 -0
  95. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.html.template +42 -0
  96. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.scss.template +96 -0
  97. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.spec.ts.template +37 -0
  98. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.ts.template +63 -0
  99. package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.html.template +16 -0
  100. package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.scss.template +7 -0
  101. package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.spec.ts.template +41 -0
  102. package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.ts.template +26 -0
  103. package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +123 -0
  104. package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.html.template +11 -4
  105. package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.scss.template +17 -4
  106. package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +5 -5
  107. package/src/ng-add/files/src/app/first-time/landing/landing.component.html.template +1 -1
  108. package/src/ng-add/files/src/app/first-time/landing/landing.component.scss.template +0 -1
  109. package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +25 -17
  110. package/src/ng-add/files/src/app/first-time/landing/landing.component.ts.template +4 -12
  111. package/src/ng-add/files/src/app/first-time/landing/rotating-quotes/rotating-quotes.component.ts.template +1 -1
  112. package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +36 -16
  113. package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +1 -1
  114. package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.spec.ts.template +11 -15
  115. package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +1 -3
  116. package/src/ng-add/files/src/app/main/home/home.component.ts.template +1 -3
  117. package/src/ng-add/files/src/app/shared/dev-status-bar/dev-status-bar.component.spec.ts.template +98 -0
  118. package/src/ng-add/files/src/app/shared/dev-status-bar/dev-status-bar.component.ts.template +9 -3
  119. package/src/ng-add/files/src/app/shared/hint-visibility/hint-visibility.directive.spec.ts.template +164 -0
  120. package/src/ng-add/files/src/app/shared/hint-visibility/hint-visibility.directive.ts.template +20 -18
  121. package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +38 -17
  122. package/src/ng-add/files/src/app/shared/navbar/navbar.component.ts.template +1 -1
  123. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.html.template +1 -1
  124. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.spec.ts.template +49 -10
  125. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.ts.template +5 -5
  126. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +86 -0
  127. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +11 -9
  128. package/src/ng-add/files/src/environments/environment.prod.ts.template +1 -0
  129. package/src/ng-add/files/src/environments/environment.test.ts.template +1 -0
  130. package/src/ng-add/files/src/environments/environment.ts.template +2 -8
  131. package/src/ng-add/files/src/styles.scss.template +29 -1
  132. package/src/ng-add/files/tsconfig.app.json.template +6 -6
  133. package/src/ng-add/files/tsconfig.json.template +13 -8
  134. package/src/ng-add/files/tsconfig.spec.json.template +6 -5
  135. package/src/ng-add/index.js +19 -1
  136. package/src/ng-add/index.js.map +1 -1
  137. package/src/component/files/__name@dasherize__/__name@dasherize__.animations.ts.template +0 -4
  138. package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.animations.ts.template +0 -4
  139. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-counts/admin-count-pill/admin-count-pill.animations.ts.template +0 -4
  140. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-counts/admin-user-counts.animations.ts.template +0 -4
  141. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.animations.ts.template +0 -4
  142. package/src/ng-add/files/src/app/first-time/consent/consent.animations.ts.template +0 -4
  143. package/src/ng-add/files/src/app/first-time/early-access/early-access.animations.ts.template +0 -10
  144. package/src/ng-add/files/src/app/first-time/landing/hero/hero.animations.ts.template +0 -15
  145. package/src/ng-add/files/src/app/first-time/landing/landing.animations.ts.template +0 -4
  146. package/src/ng-add/files/src/app/first-time/onboarding/onboarding.animations.ts.template +0 -4
  147. package/src/ng-add/files/src/app/main/home/home.animations.ts.template +0 -4
@@ -0,0 +1,157 @@
1
+ # @tech4good/angular-schematics Querying Strategy
2
+
3
+ This document provides a high-level overview of the @tech4good/angular-schematics framework's querying strategy. It is designed to help developers and AI agents think through the best approach for fetching, streaming, and managing database state for highly collaborative systems.
4
+
5
+ ## 1. Designing Collaborative Systems (The Big Picture)
6
+
7
+ ### Architectural Overview
8
+ The query system is built on a clean, layered architecture:
9
+
10
+ ```mermaid
11
+ graph TD
12
+ Component[Angular Component] -->|Calls load/stream| Store[NgRx Signal Store Feature]
13
+ Store -->|Delegates streaming| CachedListeners[CachedListenersService]
14
+ Store -->|Fetches one-shot data| Database[DatabaseService / Firestore]
15
+ CachedListeners -->|Creates connections| Database
16
+ CachedListeners -->|Patches state| Store
17
+ ```
18
+
19
+ When building an app where multiple users interact at the same time (like a chat app, a shared document, or a live auction), you constantly face a trade-off between **Data Freshness** (how quickly users see updates) and **System Load** (how much stress you put on the network and database).
20
+
21
+ - **Data Freshness (Streaming)** means keeping a permanent, open connection to the database (a websocket) so updates are pushed to the user instantly. It feels magical, but it is expensive and drains resources.
22
+ - **System Load (Loading)** means asking the database for data only once. It is cheap and efficient, but the user won't see new updates until they ask again.
23
+
24
+ ### The Core Philosophy: Stream the Active State, Load the Context
25
+ The golden rule is to only use a real-time stream for data that is actively changing and critical to the user's immediate actions (e.g., incoming chat messages, a live auction bid). For everything else—like historical posts, user profiles, or configuration settings—you should load the data once and rely on alternative strategies to keep it reasonably fresh.
26
+
27
+ ### Strategies for "Fresh" Loading
28
+ If you don't stream, how do users get updates? Here are common, highly efficient patterns:
29
+ 1. **Polling**: Automatically run a `load` query every 5 to 10 seconds. *(Note: This is only viable if you have a highly optimized delta-fetching loading strategy—which our framework implements and describes further below!)*
30
+ 2. **Event-Based Refetches**: Trigger a `load` when the user clicks "Load More", pulls-to-refresh on mobile, or when they switch tabs back to your app.
31
+ 3. **Lightweight "New Items" Streams**: Instead of streaming an entire list of 500 documents, stream just *one* piece of data.
32
+ - *Metadata Document*: Stream a single document that just holds a `lastUpdatedAt` timestamp. When the timestamp changes, show a button saying "New updates available!"
33
+ - *Limit(1) Stream*: Stream the single newest item in a list (e.g., `limit(1)` sorted by date). When it changes, trigger a notification. *(Note: A limit(1) stream only tells you that a new item exists, not how many there are. To show "5 new posts", you would typically trigger an efficient `COUNT()` aggregate query on the database, or use a Metadata Document that explicitly tracks the `unreadCount`.)*
34
+
35
+ ### Combining Strategies in Practice
36
+ In a real application, you rarely use just one of these strategies. You combine them to create an experience that *feels* completely real-time while remaining highly efficient on the backend:
37
+
38
+ **Example 1: A Social Media Feed (e.g., Twitter/X)**
39
+ - **Event-Based Refetching**: The user opens the app, loading the first 20 posts. When they scroll to the bottom, an event triggers loading the next 20.
40
+ - **Lightweight Stream**: A `limit(1)` stream listens for new posts by people they follow. When it triggers, instead of disrupting the user's reading by forcibly injecting the post, the app shows a pill at the top: *"↑ New Posts Available"*. Clicking the pill triggers an **Event-Based Refetch** to load the missing posts.
41
+ - **Polling (Background Refresh)**: The app might quietly poll every 10 seconds for updates purely on the posts *currently visible* on the screen. Because of delta-fetching, this is extremely cheap and allows "Like" and "Comment" counts to tick up organically while the user is reading.
42
+
43
+ **Example 2: A Team Messaging App (e.g., Slack)**
44
+ - **Stream the Active State**: The currently open chat channel is fully streamed. When someone types, you see the message and typing indicators instantly.
45
+ - **Lightweight Stream**: The app doesn't stream the message history for the 50 *other* channels the user is in. Instead, it streams a single `UserMetadata` document containing a `lastMentionedAt` timestamp or an `unreadCount`.
46
+ - **Polling / Event-Based Refetches**: When that metadata document changes, or when the user switches tabs back to the app, it triggers a `load` query to fetch the updated channel list and populate the unread badge counts in the sidebar.
47
+
48
+ *(Note: For ultra-fast, millisecond-level collaboration like live typing indicators or 60fps live mouse cursors, even databases like Firebase are too slow. For those, use dedicated CRDT/WebSocket backends like Yjs).*
49
+
50
+ ### The Relational Problem: Joins in NoSQL
51
+ In NoSQL databases (like Firebase), there are no "Joins" like in SQL. If you load a list of 30 Posts, and you need to show the Author's Name next to each post, a naive implementation would make 30 additional network queries to fetch those 30 authors individually. This is called the **N+1 Problem** and causes massive performance bottlenecks.
52
+
53
+ In our framework, there are two primary ways to handle this:
54
+
55
+ #### 1. Relying on the Nested Query Optimizer (The Default)
56
+ Our framework's custom query syntax natively supports "Nested Queries" (described in the next section). When you declare a nested query, our framework automatically intercepts all the N+1 requests, deduplicates them, and batches them together into highly efficient `in` queries (up to 30 items at a time).
57
+ - **When to use it**: This should be your default approach. It keeps your database normalized (no duplicate data to keep in sync) while avoiding the network performance penalty of the N+1 problem.
58
+ - **Limitations**:
59
+ - *Query Constraints*: Because the optimizer works by extracting IDs and injecting them into a batched `in` clause, it only works for simple one-to-one or one-to-many direct ID lookups (e.g. `['authorId', '==', id]` becomes `['authorId', 'in', [id1, id2, id3...]]`). It cannot handle complex nested queries involving multiple conditions, ranges, or inequalities.
60
+ - *Waterfall Latency*: The parent query must completely finish downloading before the optimizer can extract the IDs and initiate the nested query. This "waterfall" effect slightly increases the initial load time compared to a denormalized document that returns all the data in a single round-trip.
61
+ - *Batch Limits*: Firestore limits `in` queries to 30 items. While our framework automatically chunks larger arrays into multiple batches, querying 300 nested items will still result in 10 separate network requests.
62
+
63
+ #### 2. Denormalization (For Filtering and Scale)
64
+ While our optimizer fixes the network bottleneck, it doesn't solve database limitations. For example, if you want to run a database query to *"fetch all Posts where the Author's Name starts with 'D'"*, you can't do that with nested queries because Firestore cannot filter by a field that isn't on the Post document itself.
65
+ - **When to use it**: If you need to **filter or sort** a list based on a joined field, or if the joined data is permanently fixed, you must use **Denormalization**. This means copying the `authorName` directly onto the `Post` document when it is created. It allows for complex filtering, but requires you to write a backend trigger to keep the copied data synced if the original author changes their name.
66
+ ---
67
+
68
+ ## 2. The Custom Query Syntax
69
+
70
+ To help manage these complexities, our framework uses a custom query syntax. This custom syntax acts as an abstraction layer between our app and the database. By using our syntax instead of raw database commands, our framework can automatically apply powerful optimizations behind the scenes.
71
+
72
+ Crucially, this abstraction also gives us options in the future for **swapping out the backend** while still making it stream well. As our needs grow, we can migrate to backends that natively excel at streaming "joined" relational data without rewriting our UI code. For example:
73
+ - **Supabase**: Leverages PostgreSQL with Realtime extensions to stream joined data natively.
74
+ - **Firebase SQL Connect / Data Connect**: Allows writing GraphQL/SQL-like relational queries backed by Cloud SQL.
75
+ - **Firestore Enterprise**: Offers higher limits and advanced streaming capabilities for enterprise-scale collaboration.
76
+
77
+ A query in our framework consists of three parts:
78
+
79
+ 1. **`queryParams`**: How you filter the data. This is an array of conditions.
80
+ - Example: `[['__userId', '==', 'user123'], ['status', '==', 'active']]`
81
+ 2. **`queryOptions`**: How you organize the results (sorting, limiting).
82
+ - Example: `{ limit: 20, orderBy: ['createdAt', 'desc'] }`
83
+ 3. **Nested Queries**: This is a special feature of our framework. It allows you to declare follow-up queries that depend on the data you just retrieved.
84
+ - Example: "Get me the top 20 Posts. Then, for each Post, use its `authorId` to get the Author's profile."
85
+ - **Code Example**:
86
+ ```typescript
87
+ load(
88
+ [['status', '==', 'published']], // queryParams
89
+ { limit: 20 }, // queryOptions
90
+ (post) => [ // nested queries
91
+ new EntityLoadQuery(usersStore, [['__id', '==', post.authorId]])
92
+ ]
93
+ )
94
+ ```
95
+
96
+ ---
97
+
98
+ ## 3. How We Use `load()` (One-Shot Context)
99
+
100
+ You should use `load()` by default for almost everything. Our framework heavily optimizes `load()` so that every container component can simply ask for the data it needs without worrying about destroying performance.
101
+
102
+ Our optimizations focus on two things:
103
+
104
+ **1. Solving the N+1 Problem (Grouping via `optimize`)**
105
+ When you use our Nested Query syntax (e.g., loading 30 Posts, and asking for their 30 Authors), making 30 separate network requests would be slow.
106
+ Our framework intercepts these nested queries, looks for patterns, and groups them together. Instead of asking for 30 individual authors, it combines them into a single, massive query: "Give me the authors whose ID is in this list of 30 IDs."
107
+
108
+ **2. Stopping Duplicate Fetching (`lastLoadTime`)**
109
+ If Component A loads a User Profile, and later Component B asks for the exact same User Profile, we shouldn't waste network data downloading it again.
110
+ Our framework tracks a `lastLoadTime` for queries. When a component asks to `load()` data, the framework modifies the query behind the scenes to say: *"Only give me documents that have been updated since this exact `lastLoadTime`."* If nothing changed, the network payload is basically zero.
111
+
112
+ Crucially, **for simple `==` and `in` queries, we optimize `lastLoadTime` at the individual entity level**. This means if you load a batch of 30 authors, the framework remembers the exact timestamp *for each specific author*. If you later load a new feed containing 10 authors you've already seen and 5 new ones, the framework is smart enough to only fetch the updates for the 10 known authors and full data for the 5 new ones, merging them perfectly. *(Note: To prevent the system from breaking the 15 authors into 15 separate network requests if they all have slightly different load times, the framework uses a time-bucketing strategy. It rounds timestamps into 5-minute intervals, effectively balancing the tradeoff between over-fetching a few minutes of data vs firing too many queries).*
113
+
114
+ **For complex queries** (e.g., those using inequality operators like `<`, `>`, or those with `queryOptions` like `limit` and `orderBy`), it is impossible to optimize at the individual entity level. For these, we track the `lastLoadTime` for the *exact query string*. This means if you run a complex query for the first time, it will fetch all matching entities—even if some of those specific entities were already previously loaded into the cache by a different query.
115
+
116
+ > **Deep Dive**: See [Load System Architecture](./load-system-architecture.md) for code-level details on how we chunk and track time.
117
+
118
+ ---
119
+
120
+ ## 4. How We Use `stream()` (Real-Time Collaboration)
121
+
122
+ `stream()` sets up a persistent websocket to the database. Whenever the data changes on the server, the new data is instantly pushed to the client.
123
+
124
+ Because maintaining websockets is expensive, our framework manages a **Listener Pool**. If Component A and Component B both ask to stream the same chat room, the framework only opens *one* connection to the database and shares the data between them.
125
+
126
+ **Warning: The Dangers of Nested Streaming**
127
+ Our framework allows you to use the Nested Query syntax with `stream()`. For example, you can stream a list of Projects, and automatically stream the real-time presence (online/offline status) of every project owner.
128
+
129
+ However, Firebase has strict limits. If you have more than ~100 active listeners, your app's performance will severely degrade and connections will freeze. If you stream 30 projects and open 30 nested streams for their owners, you will hit this limit almost immediately.
130
+
131
+ To prevent hitting this limit, the framework attempts to group nested streams into chunks of 30 using an `in` query, just like it does for `load()`. But because streaming connections are permanent, grouping them creates two problems that our framework attempts to address:
132
+
133
+ **Problem A: Deduplication Failures**
134
+ Imagine two components on the screen.
135
+ - Component A needs to stream User 1 and User 2. It groups them: `['__id', 'in', ['user1', 'user2']]`.
136
+ - Component B needs to stream User 2 and User 3. It groups them: `['__id', 'in', ['user2', 'user3']]`.
137
+
138
+ Because the grouped arrays are different, the framework doesn't realize they share `user2`. It opens two separate database listeners, forcing the client to download and stream `user2` twice. This destroys efficiency.
139
+
140
+ **Problem B: Listener Teardown Churn**
141
+ Imagine you are streaming a feed of 30 posts, and you grouped their authors into a single listener: `['__id', 'in', ['user1', 'user2', ... 'user30']]`.
142
+ Suddenly, `user2` deletes their post. It disappears from the screen. We no longer need to stream `user2`'s profile.
143
+
144
+ Because they are grouped into a single websocket, we cannot simply "unsubscribe" from `user2`. To stop streaming `user2`, we must destroy the entire websocket for all 30 users, and instantly build a brand new websocket for the remaining 29 users (`['__id', 'in', ['user1', 'user3', ... 'user30']]`). During the fraction of a second where we tear down and rebuild the socket, we might miss real-time updates for the other 29 users, causing data gaps or UI flickering.
145
+
146
+ ### The Global Multiplexer Solution
147
+ *(Note: This multiplexing strategy only works for simple `==` and `in` nested queries. Because of the extreme complexity of managing listener chunks, nested streams are strictly prohibited from using complex conditions, inequality operators, or `queryOptions` like limits.)*
148
+
149
+ To solve these problems, the framework uses a complex **Global Multiplexer with Smart Lazy Batching**. It untangles the nested streams so that components ask for individual IDs, but behind the scenes, the multiplexer tracks all active IDs and smartly manages the listeners:
150
+ - **Smart Additions**: When a new entity is added, it doesn't destroy and rebuild a grouped listener. Instead, it creates a lightweight single-entity listener.
151
+ - **Lazy Deletions**: When an entity is removed, it doesn't tear down the batch (to avoid Problem B). It orphans the ID and allows its incoming data to continue flowing into the central data store. Because each UI component is responsible for selecting only the specific data it needs from the store, this orphaned data is safely ignored by the UI—it just means we temporarily over-fetch a small amount of data in the background until the batch is repacked.
152
+ - **Empty Listener Cleanup**: If every single ID inside a grouped listener is eventually orphaned (meaning the listener has 0 active IDs left), the framework immediately tears down and removes the websocket to free up connections.
153
+ - **Smart Repacking**: Once single listeners pile up or a specific batch accumulates too many orphans, it triggers a mathematically optimized repack. It calculates exactly which batches have empty slots and carefully merges single listeners into them, strictly minimizing the number of websockets that need to be torn down and rebuilt.
154
+
155
+ If you have a complex scenario that doesn't fit these rules, you must either denormalize the data into the parent document, or use `load()` instead.
156
+
157
+ > **Deep Dive**: See [Stream System Architecture](./stream-system-architecture.md) for code-level details on the Global Multiplexer and listener teardown logic.
@@ -0,0 +1,283 @@
1
+ # Stream System Architecture
2
+
3
+ This document details the code-level architecture of the `stream()` method. While `load()` is a one-shot fetch, `stream()` maintains an active websocket to Firestore. For a high-level overview of our querying strategy, refer to [querying-strategy.md](./querying-strategy.md).
4
+
5
+ ## 1. Data Structures & Connection Caching
6
+
7
+ Because websockets are expensive and Firestore limits active listeners (100 concurrent `in` queries per client), we aggressively cache and multiplex connections.
8
+
9
+ The `CachedListenersService` maintains the cache for complex legacy queries:
10
+
11
+ ```typescript
12
+ // From cached-listeners.service.ts
13
+ // Key: "collection.queryParams.queryOptions"
14
+ private connectionByQueryKey: Record<string, CachedConnection> = {};
15
+
16
+ export interface CachedConnection {
17
+ queryKey: string;
18
+ registeredSet: Set<DestroyRef>; // Tracks active components
19
+ }
20
+ ```
21
+
22
+ Separately, the `CachedListenersService` maintains a map of generic `StreamMultiplexer` instances, which are delegated to handle simple queries:
23
+
24
+ ```typescript
25
+ // From cached-listeners.service.ts
26
+ // Key: "collection.field"
27
+ private streamMultiplexers = new Map<string, StreamMultiplexer>();
28
+ ```
29
+
30
+ Inside the `StreamMultiplexer` class itself, the state is split to track component lifecycles and batch pools:
31
+
32
+ ```typescript
33
+ // From stream-multiplexer.ts
34
+ export class StreamMultiplexer {
35
+ // Bi-directional tracking between a unique ID and the components (DestroyRefs) that requested it
36
+ private destroyRefsById = new Map<string, Set<DestroyRef>>();
37
+ private idsByDestroyRef = new Map<DestroyRef, Set<string>>();
38
+
39
+ // Two pools of active Firestore websocket connections
40
+ private batchedListeners: Array<{ ids: string[], unsubscribe: () => void }> = []; // Full batches of 30
41
+ private singleListeners: Array<{ ids: string[], unsubscribe: () => void }> = []; // Sparse streams
42
+ }
43
+ ```
44
+
45
+ ## 2. The `stream()` Flow Walkthrough
46
+
47
+ ```mermaid
48
+ sequenceDiagram
49
+ participant Component
50
+ participant Store
51
+ participant CachedListeners as CachedListenersService
52
+ participant DB as DatabaseService
53
+
54
+ Component->>Store: stream(queryParams, queryOptions, injector, streamQueries)
55
+ Store->>CachedListeners: processStreamQuery(collection, store, queryParams, queryOptions, injector, streamQueries)
56
+
57
+ alt Connection Exists in Cache
58
+ CachedListeners->>CachedListeners: Add component's DestroyRef to connection.registeredSet
59
+ else Connection New
60
+ CachedListeners->>CachedListeners: Create connection.registeredSet with DestroyRef
61
+ CachedListeners->>CachedListeners: Initialize connectionByQueryKey[queryKey]
62
+ CachedListeners->>DB: streamEntitiesChanges(collection, queryParams, queryOptions)
63
+ DB-->>CachedListeners: Observable of DocumentChanges (added/modified/removed)
64
+ CachedListeners->>Store: handleStreamChanges(added, modified, removed)
65
+ end
66
+
67
+ rect rgb(255, 240, 245)
68
+ Note over CachedListeners: Nested Streaming Setup
69
+ CachedListeners->>Store: computed selectEntities(queryParams, queryOptions)
70
+ CachedListeners->>CachedListeners: toObservable(streamEntities)
71
+ CachedListeners->>CachedListeners: pipe(pairwise()) to get added/modified parent docs
72
+ loop For each change batch
73
+ CachedListeners->>CachedListeners: Map to streamQueries(doc)
74
+ CachedListeners->>CachedListeners: Transpose and optimize via EntityStreamQuery.optimize()
75
+ CachedListeners->>CachedListeners: Recursively call processStreamQuery() for nested queries
76
+ end
77
+ end
78
+ ```
79
+
80
+ ### Step 1: Execution Path Split (Legacy vs Optimized)
81
+
82
+ When a component calls `stream()`, the execution path dynamically splits based on the query complexity.
83
+
84
+ #### Path A: The Legacy/Complex Path (`processStreamQuery`)
85
+ For complex queries (inequalities, limits, multi-field), we fall back to a standard connection cache using `CachedConnection`.
86
+ - **Deduplication**: If the exact `queryKey` already exists, we simply add the component's `DestroyRef` to the `registeredSet` and skip database work.
87
+ - **Fetch & Soft Deletes**: If new, we open a `streamEntitiesChanges` websocket to Firestore. The stream automatically filters incoming payloads, intercepting documents where `_deleted == true` and routing them to the store as deletions rather than modifications.
88
+
89
+ #### Path B: The Optimized Equality Path (`processSimpleStreamQuery`)
90
+ For simple equality queries (`==`, `in`), binding the websocket lifecycle strictly to the exact query string causes massive teardown churn. If a component requests `in [u1, u2, u3]` and then drops `u2`, the legacy framework would destroy the whole listener and rebuild `in [u1, u3]`.
91
+
92
+ Instead, we route these to the **Global Stream Multiplexer**. The component passes its requested IDs to `StreamMultiplexer.registerIds()`, which evaluates whether to spawn a new sparse stream or merge it into an existing batch.
93
+
94
+ ### Step 2: Reactive Nested Streaming
95
+
96
+ Once the stream is established (from either Path A or Path B), it begins patching new data into the NgRx store. This triggers the nested streaming pipeline.
97
+
98
+ Handling nested streams is notoriously complex. We cannot simply listen to the `added`/`modified` payloads coming directly from the Firestore websocket, because we deduplicate parent streams globally. A parent stream might be serving three different components, but component C has a unique nested query it needs to trigger for those parents.
99
+
100
+ **The Solution:** We create a secondary "store stream" that monitors the *local NgRx store* for the parent entities.
101
+
102
+ ```typescript
103
+ // 1. Create a reactive signal tracking the parent entities in the store
104
+ const streamEntities = computed(() => {
105
+ return store.selectEntities(queryParams, queryOptions);
106
+ });
107
+
108
+ // 2. Convert to an RxJS pipeline to diff changes (pairwise)
109
+ toObservable(streamEntities, { injector }).pipe(
110
+ map(entities => /* ... convert to ID map ... */),
111
+ startWith({}),
112
+ pairwise(), // Compare current vs previous
113
+ map(([prevEntities, currEntities]) => {
114
+ // 3. Diffing logic: determine what was truly added/modified in the store
115
+ const changedEntities = [];
116
+ for (const key in currEntities) {
117
+ if (!(key in prevEntities)) {
118
+ changedEntities.push({ type: 'added', doc: currEntities[key] });
119
+ } else if (currEntities[key]['_updatedAt'] !== prevEntities[key]['_updatedAt']) {
120
+ changedEntities.push({ type: 'modified', doc: currEntities[key] });
121
+ }
122
+ }
123
+ return changedEntities;
124
+ }),
125
+ ).subscribe(async (changes) => {
126
+ // 4. For every modified/added parent entity, execute its nested streams
127
+ const queriesByEntity = changes.map((change) => streamQueries(change.doc));
128
+
129
+ // 5. Transpose the matrix: [QueryType][Entities]
130
+ const queriesByQueryType = queriesByEntity[0].map((_, colIndex) =>
131
+ queriesByEntity.map((row) => row[colIndex])
132
+ );
133
+
134
+ // 6. Optimize and recursively stream each query type
135
+ queriesByQueryType.forEach((queriesForOneType) => {
136
+ const optimizedQueries = EntityStreamQuery.optimize(queriesForOneType);
137
+ // Concurrently run processStreamQuery() recursively for the child queries
138
+ });
139
+ });
140
+ ```
141
+
142
+ As the store iterates over the changed parent entities, it collects an array of nested queries for *each* parent. This forms a 2D array: `[Parent Index][Query Index]`. To optimize these queries and prevent an N+1 cascade of websocket connections, we group them by their *query type* across all parents by transposing the matrix:
143
+
144
+ ```mermaid
145
+ graph TD
146
+ subgraph "Original Layout (By Parent Entity)"
147
+ E1["Entity 1<br/>[QueryA, QueryB]"]
148
+ E2["Entity 2<br/>[QueryA, QueryB]"]
149
+ E3["Entity 3<br/>[QueryA, QueryB]"]
150
+ end
151
+
152
+ subgraph "Transposed Layout (By Query Type)"
153
+ QA["Query A List<br/>[Entity 1's Query A, Entity 2's Query A, Entity 3's Query A]"]
154
+ QB["Query B List<br/>[Entity 1's Query B, Entity 2's Query B, Entity 3's Query B]"]
155
+ end
156
+
157
+ E1 -.-> QA
158
+ E1 -.-> QB
159
+ E2 -.-> QA
160
+ E2 -.-> QB
161
+ E3 -.-> QA
162
+ E3 -.-> QB
163
+ ```
164
+
165
+ `EntityStreamQuery.optimize()` detects multiple `==` queries for the same field and collapses them into a single massive `in` query containing all requested IDs. This ensures nested queries execute reliably regardless of how the parent entity arrived in the store. The resulting massive `in` query is recursively passed back into `stream()`. Because nested streams are strictly prohibited from using complex conditions, they **always** hit Path B (the Stream Multiplexer), allowing them to be efficiently structurally batched across the entire application.
166
+
167
+ ### Step 3: Multiplexer Background Operations (Smart Repacking)
168
+
169
+ As users navigate the app and new data arrives, the optimized Path B multiplexer faces two background challenges over time:
170
+
171
+ 1. **New ID Sprawl**: When a new parent entity is added to the store (e.g., via pagination), it triggers a new nested query. We *cannot* immediately add this newly requested ID into an existing batched websocket, because modifying a Firestore listener requires tearing it down and rebuilding it, causing UI flicker. Instead, the multiplexer spins up a temporary "sparse" stream specifically for the new IDs, adding it to the `singleListeners` pool.
172
+ 2. **Orphaned IDs**: As components unmount (detailed in Step 4), IDs become "orphaned" (reference count drops to 0). If they are in an active batch, **we do not tear down the listener**. Their data simply continues flowing into the store where the UI safely ignores it.
173
+
174
+ To prevent both orphaned IDs and sparse single listeners from piling up and eating our 100-listener limit, the multiplexer constantly monitors the pools. If either threshold is crossed (too many orphans, or too many single listeners), it executes two garbage collection passes:
175
+
176
+ **1. Dead Stream Purging (`cleanupEmptyListeners`)**:
177
+ Before attempting a complex repack, the system identifies any batch or single listener where *every single ID* has 0 active holds. These completely abandoned streams are immediately unsubscribed and purged from the system, freeing up websocket connections with zero overhead.
178
+
179
+ **2. Mathematical Repacking (`executeSmartRepack`)**:
180
+ For the remaining streams that contain a mix of active and orphaned IDs, the multiplexer executes a greedy algorithm to mathematically consolidate them:
181
+
182
+ ```typescript
183
+ // From stream-multiplexer.ts: executeSmartRepack()
184
+
185
+ // 1. Gather all "sparse" single listeners and candidate batches (<= 25 active IDs)
186
+ const candidateBatches = this.batchedListeners
187
+ .filter(b => b.activeCount <= 25)
188
+ .sort((a, b) => a.activeCount - b.activeCount);
189
+
190
+ // 2. Greedily calculate if absorbing a sparse batch reduces the total stream count
191
+ for (let i = 0; i < candidateBatches.length; i++) {
192
+ currentT.push(candidateBatches[i].batch);
193
+ currentA += candidateBatches[i].activeCount;
194
+
195
+ // If tearing down these batches and packing them with the single listeners
196
+ // requires fewer total chunks of 30 than we currently have running...
197
+ const newBatchesCount = Math.ceil((S + currentA) / 30);
198
+ const reduction = S + currentT.length - newBatchesCount;
199
+
200
+ if (reduction > bestReduction) {
201
+ bestReduction = reduction;
202
+ bestT = [...currentT]; // Save these batches for teardown
203
+ }
204
+ }
205
+ ```
206
+
207
+ To ensure absolutely zero UI flickering during this repack, the multiplexer implements an **overlapping teardown** mechanism.
208
+
209
+ ```typescript
210
+ // From stream-multiplexer.ts: startNewBatches()
211
+
212
+ // We pass the old sockets as an argument, but we do NOT unsubscribe them yet.
213
+ const unsubscribeBatch = this.db.streamEntitiesChanges(this.collection, queryParams, {}).subscribe(changes => {
214
+ this.patchToStore(changes, queryParams);
215
+
216
+ if (isFirstEmission) {
217
+ isFirstEmission = false;
218
+ // Only after the new socket has successfully connected and emitted its first
219
+ // payload do we tear down the old sockets it replaces!
220
+ this.teardownOldListeners(oldBatchesToTeardown);
221
+ }
222
+ });
223
+ ```
224
+
225
+ ### Step 4: Component Lifecycle & Graceful Teardown
226
+
227
+ When a component is finally destroyed, it triggers the teardown sequence. Because we have two different caching strategies, teardown is handled differently for Path A vs Path B.
228
+
229
+ #### Path A Teardown: Complex Queries
230
+ For standard cached connections, the `CachedListenersService` waits 5 seconds (the "Grace Period") before unsubscribing from Firestore. If the user clicks a link and navigates away but immediately comes back, we don't want to tear down and rebuild the websocket, which causes UI flicker and burns DB reads.
231
+
232
+ ```typescript
233
+ // From cached-listeners.service.ts
234
+ destroyRef.onDestroy(() => {
235
+ setTimeout(() => {
236
+ connection.registeredSet.delete(destroyRef);
237
+ if (connection.registeredSet.size === 0) {
238
+ // Last component listening to this exact query string was destroyed
239
+ this.closeConnection$.next(connection.queryKey);
240
+ delete this.connectionByQueryKey[connection.queryKey];
241
+ }
242
+ }, 5000); // 5-second Grace Period
243
+ });
244
+ ```
245
+
246
+ #### Path B Teardown: Stream Multiplexer
247
+ For simple multiplexed queries, `StreamMultiplexer` also employs a 5-second grace period. However, because a component might have requested a dozen specific IDs, the multiplexer must selectively remove that component's "claim" on each individual ID.
248
+
249
+ ```typescript
250
+ // From stream-multiplexer.ts
251
+ destroyRef.onDestroy(() => {
252
+ setTimeout(() => {
253
+ let triggeredRemove = false;
254
+
255
+ // Retrieve all specific IDs requested by this component
256
+ const idsToClean = this.idsByDestroyRef.get(destroyRef);
257
+ if (idsToClean) {
258
+ idsToClean.forEach(id => {
259
+ const registeredSet = this.destroyRefsById.get(id);
260
+ if (registeredSet && registeredSet.has(destroyRef)) {
261
+ registeredSet.delete(destroyRef); // Remove claim
262
+
263
+ if (registeredSet.size === 0) {
264
+ this.destroyRefsById.delete(id); // ID is now entirely orphaned
265
+ triggeredRemove = true;
266
+ }
267
+ }
268
+ });
269
+ this.idsByDestroyRef.delete(destroyRef);
270
+ }
271
+
272
+ // If IDs were orphaned, evaluate if we should garbage collect
273
+ if (triggeredRemove) {
274
+ this.cleanupEmptyListeners(); // Immediately purge fully abandoned sockets
275
+ if (this.hasBatchWithTooManyOrphans()) {
276
+ this.executeSmartRepack(); // Optional background repack
277
+ }
278
+ }
279
+ }, 5000);
280
+ });
281
+ ```
282
+
283
+ As seen in the code, tearing down Path B rarely results in an immediate websocket disconnection unless a socket is 100% abandoned. Most often, the IDs simply become "orphaned" and are cleaned up asynchronously during the next Smart Repack.
@@ -30,7 +30,10 @@ export default defineConfig([
30
30
  ecmaVersion: 2020,
31
31
  sourceType: "module",
32
32
  parserOptions: {
33
- project: ["tsconfig.json"],
33
+ // We include app and spec tsconfigs because the root tsconfig.json
34
+ // explicitly excludes files ("files": []) and only serves to reference the others.
35
+ // Without these explicit references, ESLint cannot discover the application files.
36
+ project: ["tsconfig.json", "tsconfig.app.json", "tsconfig.spec.json"],
34
37
  createDefaultProgram: true,
35
38
  },
36
39
  },
@@ -0,0 +1,37 @@
1
+ {
2
+ "firestore": {
3
+ "database": "(default)",
4
+ "location": "nam5",
5
+ "rules": "firestore.rules",
6
+ "indexes": "firestore.indexes.json"
7
+ },
8
+ "hosting": {
9
+ "public": "dist/<%= dasherize(project) %>/browser",
10
+ "ignore": [
11
+ "firebase.json",
12
+ "**/.*",
13
+ "**/node_modules/**"
14
+ ],
15
+ "rewrites": [
16
+ {
17
+ "source": "**",
18
+ "destination": "/index.html"
19
+ }
20
+ ]
21
+ },
22
+ "emulators": {
23
+ "auth": {
24
+ "port": 9099
25
+ },
26
+ "firestore": {
27
+ "port": 8080
28
+ },
29
+ "hosting": {
30
+ "port": 5000
31
+ },
32
+ "ui": {
33
+ "enabled": true
34
+ },
35
+ "singleProjectMode": true
36
+ }
37
+ }