@tech4good/angular-schematics 21.3.0 → 21.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.t4g/agent-memory.db +0 -0
- package/CHANGELOG.md +24 -0
- package/package.json +1 -1
- package/src/entity/index.js +42 -9
- package/src/entity/index.js.map +1 -1
- package/src/entity/index_spec.js +42 -8
- package/src/entity/index_spec.js.map +1 -1
- package/src/entity/schema.json +0 -5
- package/src/ng-add/files/.gitignore.template +1 -0
- package/src/ng-add/files/angular.json.template +3 -0
- package/src/ng-add/files/firestore.rules.spec.ts.template +88 -1
- package/src/ng-add/files/firestore.rules.template +9 -0
- package/src/ng-add/files/package.json.template +4 -2
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +3 -3
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +0 -1
- package/src/ng-add/files/src/app/app.component.html.template +1 -1
- package/src/ng-add/files/src/app/app.component.scss.template +9 -1
- package/src/ng-add/files/src/app/app.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/auth-routing.spec.ts.template +1 -1
- package/src/ng-add/files/src/app/core/analytics/log.service.ts.template +1 -1
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +53 -51
- package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +6 -0
- package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +14 -14
- package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +50 -48
- package/src/ng-add/files/src/app/core/logging/logging.service.spec.ts.template +118 -0
- package/src/ng-add/files/src/app/core/logging/logging.service.ts.template +168 -0
- package/src/ng-add/files/src/app/core/store/app.model.ts.template +3 -1
- package/src/ng-add/files/src/app/core/store/app.store.ts.template +11 -9
- package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +5 -4
- package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +13 -4
- package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +0 -4
- package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +0 -1
- package/src/ng-add/files/src/app/core/store/user-private/user-private.mock.ts.template +15 -0
- package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +9 -0
- package/src/ng-add/files/src/app/core/store/user-private/user-private.store.ts.template +41 -0
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +0 -1
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +1 -1
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +3 -3
- package/src/ng-add/files/src/app/first-time/landing/landing.component.scss.template +1 -1
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +3 -3
- package/src/ng-add/files/src/app/main/home/home.component.spec.ts.template +1 -1
- package/src/ng-add/files/src/app/main/home/home.component.ts.template +4 -4
- package/src/ng-add/files/src/app/shared/dev-status-bar/dev-status-bar.component.scss.template +1 -0
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +41 -5
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +25 -2
- package/src/ng-add/files/src/environments/environment.test.ts.template +1 -1
- package/src/shared/statements-adder.d.ts +1 -0
- package/src/shared/statements-adder.js +8 -0
- package/src/shared/statements-adder.js.map +1 -1
- package/src/ng-add/files/src/app/core/firebase/query-logger.ts.template +0 -11
- package/tech4good-angular-schematics-21.0.0.tgz +0 -0
|
@@ -8,7 +8,9 @@ import { QueryParams, QueryOptions, AnyEntity } from '../store/app.model';
|
|
|
8
8
|
import { DocumentChange } from '@angular/fire/firestore';
|
|
9
9
|
import { DATABASE_SERVICE, DatabaseService } from './database.service';
|
|
10
10
|
import { StreamMultiplexer } from './stream-multiplexer';
|
|
11
|
-
import {
|
|
11
|
+
import { getLogger } from '../logging/logging.service';
|
|
12
|
+
|
|
13
|
+
const queryLogger = getLogger('query');
|
|
12
14
|
|
|
13
15
|
const selectId: SelectEntityId<AnyEntity> = (entity) => entity.__id;
|
|
14
16
|
|
|
@@ -19,22 +21,22 @@ export interface CachedConnection {
|
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
/**
|
|
22
|
-
* CachedListenersService manages database connections and their lifecycles to
|
|
24
|
+
* CachedListenersService manages database connections and their lifecycles to
|
|
23
25
|
* ensure optimal performance, prevent memory leaks, and solve the N+1 query problem.
|
|
24
26
|
*
|
|
25
27
|
* Responsibilities:
|
|
26
|
-
* 1. **Listener Pooling**: Deduplicates active Firestore snapshot listeners. If multiple
|
|
27
|
-
* components request a stream for the exact same query, only one actual database
|
|
28
|
+
* 1. **Listener Pooling**: Deduplicates active Firestore snapshot listeners. If multiple
|
|
29
|
+
* components request a stream for the exact same query, only one actual database
|
|
28
30
|
* connection is opened.
|
|
29
|
-
* 2. **Reference Counting & Lifecycle Management**: Tracks which components are using
|
|
30
|
-
* which stream via their Angular `DestroyRef`. When a component is destroyed, its
|
|
31
|
+
* 2. **Reference Counting & Lifecycle Management**: Tracks which components are using
|
|
32
|
+
* which stream via their Angular `DestroyRef`. When a component is destroyed, its
|
|
31
33
|
* reference is removed.
|
|
32
|
-
* 3. **Graceful Teardown**: Implements a 5-second grace period before fully closing a
|
|
33
|
-
* database connection after its last reference drops. This prevents UI flicker and
|
|
34
|
+
* 3. **Graceful Teardown**: Implements a 5-second grace period before fully closing a
|
|
35
|
+
* database connection after its last reference drops. This prevents UI flicker and
|
|
34
36
|
* unnecessary DB reads if the user navigates away and immediately back.
|
|
35
|
-
* 4. **Load Time Tracking (One-shot Loads)**: Tracks `lastLoadTime` for general queries
|
|
36
|
-
* and precise `entityLoadTime` for optimized batch equality queries. This enables
|
|
37
|
-
* incremental (delta-only) fetching, drastically reducing bandwidth by only fetching
|
|
37
|
+
* 4. **Load Time Tracking (One-shot Loads)**: Tracks `lastLoadTime` for general queries
|
|
38
|
+
* and precise `entityLoadTime` for optimized batch equality queries. This enables
|
|
39
|
+
* incremental (delta-only) fetching, drastically reducing bandwidth by only fetching
|
|
38
40
|
* entities modified since the last load.
|
|
39
41
|
*/
|
|
40
42
|
@Injectable({
|
|
@@ -106,7 +108,7 @@ export class CachedListenersService {
|
|
|
106
108
|
}
|
|
107
109
|
}
|
|
108
110
|
this.connectionByQueryKey = {};
|
|
109
|
-
|
|
111
|
+
|
|
110
112
|
for (const multiplexer of this.streamMultiplexers.values()) {
|
|
111
113
|
multiplexer.teardownAll();
|
|
112
114
|
}
|
|
@@ -166,7 +168,7 @@ export class CachedListenersService {
|
|
|
166
168
|
loading?: WritableSignal<boolean>,
|
|
167
169
|
},
|
|
168
170
|
): Promise<CachedConnection> {
|
|
169
|
-
|
|
171
|
+
queryLogger.log(`[CachedListenersService] processStreamQuery called for collection: '${collection}', query:`, JSON.stringify(queryParams));
|
|
170
172
|
// console.log("PROCESS QUERY: " + this.getQueryKey(collection, queryParams, queryOptions));
|
|
171
173
|
|
|
172
174
|
// Get destroyRef for the component that called processStreamQuery
|
|
@@ -288,7 +290,7 @@ export class CachedListenersService {
|
|
|
288
290
|
// Process queries concurrently using Promise.all
|
|
289
291
|
await Promise.all(optimizedQueries.filter((q) => !!q).map(async (query) => {
|
|
290
292
|
const isSimpleChild = (!query.queryOptions || Object.keys(query.queryOptions).length === 0) &&
|
|
291
|
-
query.queryParams.length === 1 &&
|
|
293
|
+
query.queryParams.length === 1 &&
|
|
292
294
|
(query.queryParams[0][1] === '==' || query.queryParams[0][1] === 'in');
|
|
293
295
|
|
|
294
296
|
if (isSimpleChild) {
|
|
@@ -296,10 +298,10 @@ export class CachedListenersService {
|
|
|
296
298
|
const childOp = query.queryParams[0][1];
|
|
297
299
|
const childVal = query.queryParams[0][2];
|
|
298
300
|
const childIds = childOp === 'in' ? childVal : [childVal];
|
|
299
|
-
|
|
301
|
+
queryLogger.log(`[CachedListenersService] Nested query optimized to simple stream query: ${childField} in [${childIds}]`);
|
|
300
302
|
await this.processSimpleStreamQuery(query.collection, childField, childIds, query.store, injector, query.streamQueries, { loading: options?.loading });
|
|
301
303
|
} else {
|
|
302
|
-
|
|
304
|
+
queryLogger.log(`[CachedListenersService] Nested query falling back to standard stream query:`, JSON.stringify(query.queryParams));
|
|
303
305
|
await this.processStreamQuery(query.collection, query.store, query.queryParams, query.queryOptions, injector, query.streamQueries, { loading: options?.loading });
|
|
304
306
|
}
|
|
305
307
|
}));
|
|
@@ -317,9 +319,9 @@ export class CachedListenersService {
|
|
|
317
319
|
|
|
318
320
|
/**
|
|
319
321
|
* Optimized fast-path for simple streaming queries (e.g., `id in [1, 2, 3]`).
|
|
320
|
-
* Instead of managing distinct complex RxJS subscription chains for every component, this delegates
|
|
321
|
-
* ID registration to a centralized `StreamMultiplexer` (per collection/field).
|
|
322
|
-
*
|
|
322
|
+
* Instead of managing distinct complex RxJS subscription chains for every component, this delegates
|
|
323
|
+
* ID registration to a centralized `StreamMultiplexer` (per collection/field).
|
|
324
|
+
*
|
|
323
325
|
* It also handles nested `streamQueries` (e.g. streaming a post's author) by listening to changes
|
|
324
326
|
* to the parent entities, diffing them (`pairwise`), and setting up streams for the new children.
|
|
325
327
|
*/
|
|
@@ -332,8 +334,8 @@ export class CachedListenersService {
|
|
|
332
334
|
streamQueries?: (entity: T) => EntityStreamQuery<AnyEntity>[],
|
|
333
335
|
options?: { loading?: WritableSignal<boolean> },
|
|
334
336
|
): Promise<void> {
|
|
335
|
-
|
|
336
|
-
|
|
337
|
+
queryLogger.log(`[CachedListenersService] processSimpleStreamQuery called for collection: '${collection}', field: '${field}', ids: [${ids}]`);
|
|
338
|
+
|
|
337
339
|
if (!ids || ids.length === 0) {
|
|
338
340
|
options?.loading?.set(false);
|
|
339
341
|
return;
|
|
@@ -341,7 +343,7 @@ export class CachedListenersService {
|
|
|
341
343
|
|
|
342
344
|
const destroyRef = injector.get(DestroyRef);
|
|
343
345
|
const multiplexerKey = `${collection}.${field}`;
|
|
344
|
-
|
|
346
|
+
|
|
345
347
|
let multiplexer = this.streamMultiplexers.get(multiplexerKey);
|
|
346
348
|
if (!multiplexer) {
|
|
347
349
|
multiplexer = new StreamMultiplexer(collection, field, this.db, store);
|
|
@@ -350,7 +352,7 @@ export class CachedListenersService {
|
|
|
350
352
|
multiplexer.registerIds(ids, destroyRef);
|
|
351
353
|
|
|
352
354
|
if (streamQueries) {
|
|
353
|
-
// 1. We select the entities from the store.
|
|
355
|
+
// 1. We select the entities from the store.
|
|
354
356
|
// Because `app.store.ts` caches identical array references, `filteredSignal` will NOT trigger
|
|
355
357
|
// spurious emissions if the underlying data has not actually changed.
|
|
356
358
|
const filteredSignal = computed(() => {
|
|
@@ -371,7 +373,7 @@ export class CachedListenersService {
|
|
|
371
373
|
startWith({}),
|
|
372
374
|
pairwise(),
|
|
373
375
|
map(([prevEntities, currEntities]) => {
|
|
374
|
-
// 3. Diff the current entities against the previous entities to determine exactly
|
|
376
|
+
// 3. Diff the current entities against the previous entities to determine exactly
|
|
375
377
|
// what was added, removed, or modified. We rely on the `_deleted` flag and `_updatedAt`.
|
|
376
378
|
const changedEntities = [];
|
|
377
379
|
for (const key in currEntities) {
|
|
@@ -388,38 +390,38 @@ export class CachedListenersService {
|
|
|
388
390
|
map((changes) => changes.filter((c) => c.type !== 'removed')),
|
|
389
391
|
filter((changes) => changes.length > 0),
|
|
390
392
|
).subscribe(async (changes) => {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
query.queryParams.length === 1 &&
|
|
393
|
+
// 4. For every changed entity, evaluate its nested stream queries.
|
|
394
|
+
const queriesByEntity = changes.map((change) => streamQueries(change.doc));
|
|
395
|
+
if (queriesByEntity.length === 0 || queriesByEntity[0].length === 0) return;
|
|
396
|
+
|
|
397
|
+
// 5. Transpose the matrix so each row corresponds to all queries of the same query type.
|
|
398
|
+
// This allows us to group and optimize queries (e.g., combining 5 single equality queries into 1 'in' query).
|
|
399
|
+
const queriesByQueryType = queriesByEntity[0].map((_, colIndex) => queriesByEntity.map((row) => row[colIndex]));
|
|
400
|
+
|
|
401
|
+
await Promise.all(queriesByQueryType.map(async (queriesForOneType) => {
|
|
402
|
+
const optimizedQueries = EntityStreamQuery.optimize(queriesForOneType);
|
|
403
|
+
await Promise.all(optimizedQueries.filter((q) => !!q).map(async (query) => {
|
|
404
|
+
const isSimpleChild = (!query.queryOptions || Object.keys(query.queryOptions).length === 0) &&
|
|
405
|
+
query.queryParams.length === 1 &&
|
|
404
406
|
(query.queryParams[0][1] === '==' || query.queryParams[0][1] === 'in');
|
|
405
407
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
408
|
+
if (isSimpleChild) {
|
|
409
|
+
const childField = query.queryParams[0][0];
|
|
410
|
+
const childOp = query.queryParams[0][1];
|
|
411
|
+
const childVal = query.queryParams[0][2];
|
|
412
|
+
const childIds = childOp === 'in' ? childVal : [childVal];
|
|
413
|
+
queryLogger.log(`[CachedListenersService] Nested query optimized to simple stream query: ${childField} in [${childIds}]`);
|
|
414
|
+
await this.processSimpleStreamQuery(query.collection, childField, childIds, query.store, injector, query.streamQueries, { loading: options?.loading });
|
|
415
|
+
} else {
|
|
416
|
+
queryLogger.log(`[CachedListenersService] Nested query falling back to standard stream query:`, JSON.stringify(query.queryParams));
|
|
417
|
+
await this.processStreamQuery(query.collection, query.store, query.queryParams, query.queryOptions, injector, query.streamQueries, { loading: options?.loading });
|
|
418
|
+
}
|
|
419
|
+
}));
|
|
420
|
+
}));
|
|
419
421
|
});
|
|
420
422
|
}
|
|
421
423
|
|
|
422
|
-
// We'll turn off the loading indicator right away since we don't have a good way
|
|
424
|
+
// We'll turn off the loading indicator right away since we don't have a good way
|
|
423
425
|
// to know when all individual streams are initialized yet.
|
|
424
426
|
options?.loading?.set(false);
|
|
425
427
|
}
|
|
@@ -12,6 +12,7 @@ import { AuthMockDB } from '../store/auth/auth.mock';
|
|
|
12
12
|
// Entity Model DB Data
|
|
13
13
|
import { UserMockDB, USER_DB } from '../store/user/user.mock';
|
|
14
14
|
import { UserContextMockDB, USERCONTEXT_DB } from '../store/user-context/user-context.mock';
|
|
15
|
+
import { UserPrivateMockDB, USERPRIVATE_DB } from '../store/user-private/user-private.mock';
|
|
15
16
|
|
|
16
17
|
const selectId: SelectEntityId<AnyEntity> = (entity) => entity.__id;
|
|
17
18
|
|
|
@@ -23,6 +24,7 @@ export class FirebaseMockService implements DatabaseService {
|
|
|
23
24
|
readonly DB = {
|
|
24
25
|
'users': inject(UserMockDB),
|
|
25
26
|
'userContexts': inject(UserContextMockDB),
|
|
27
|
+
'userPrivates': inject(UserPrivateMockDB),
|
|
26
28
|
};
|
|
27
29
|
private user$ = new BehaviorSubject(this.authDB.afUser());
|
|
28
30
|
|
|
@@ -50,6 +52,7 @@ export class FirebaseMockService implements DatabaseService {
|
|
|
50
52
|
// Init Entity Models
|
|
51
53
|
this.DB['users'].init(USER_DB);
|
|
52
54
|
this.DB['userContexts'].init(USERCONTEXT_DB);
|
|
55
|
+
this.DB['userPrivates'].init(USERPRIVATE_DB);
|
|
53
56
|
}
|
|
54
57
|
|
|
55
58
|
/** Creates a unique id */
|
|
@@ -326,12 +329,14 @@ export class FirebaseMockService implements DatabaseService {
|
|
|
326
329
|
export function provideFirebaseMock({
|
|
327
330
|
users,
|
|
328
331
|
userContexts,
|
|
332
|
+
userPrivates,
|
|
329
333
|
disableLatency,
|
|
330
334
|
failCollections,
|
|
331
335
|
stallCollections,
|
|
332
336
|
}: {
|
|
333
337
|
users?: any[];
|
|
334
338
|
userContexts?: any[];
|
|
339
|
+
userPrivates?: any[];
|
|
335
340
|
disableLatency?: boolean;
|
|
336
341
|
/** Collections that throw/reject on any read. Mutable mid-test via `db.failCollections`. */
|
|
337
342
|
failCollections?: string[];
|
|
@@ -351,6 +356,7 @@ export function provideFirebaseMock({
|
|
|
351
356
|
if (stallCollections) service.stallCollections = stallCollections;
|
|
352
357
|
if (users) service.DB['users'].init(users);
|
|
353
358
|
if (userContexts) service.DB['userContexts'].init(userContexts);
|
|
359
|
+
if (userPrivates) service.DB['userPrivates'].init(userPrivates);
|
|
354
360
|
return service;
|
|
355
361
|
},
|
|
356
362
|
deps: [Injector],
|
|
@@ -4,7 +4,7 @@ import { withEntitiesAndSelectMethods, withEntitiesAndDBMethods } from '../store
|
|
|
4
4
|
import { DATABASE_SERVICE } from './database.service';
|
|
5
5
|
import { provideFirebaseMock } from './firebase.mock.service';
|
|
6
6
|
import { MOCK_SIMULATION_USERS } from './simulation.mock';
|
|
7
|
-
import
|
|
7
|
+
import { LoggingService } from '../logging/logging.service';
|
|
8
8
|
import { Timestamp } from '@angular/fire/firestore';
|
|
9
9
|
|
|
10
10
|
export const UserStore = signalStore(
|
|
@@ -39,7 +39,7 @@ describe('Simulation for Load Bucketing Strategy', () => {
|
|
|
39
39
|
const userStore = TestBed.inject(UserStore) as any;
|
|
40
40
|
const db = TestBed.inject(DATABASE_SERVICE) as any;
|
|
41
41
|
|
|
42
|
-
const logSpy = jest.spyOn(
|
|
42
|
+
const logSpy = jest.spyOn(LoggingService.prototype, 'logForNamespace').mockImplementation(() => {});
|
|
43
43
|
const getEntitiesSpy = jest.spyOn(db, 'getEntities');
|
|
44
44
|
const getEntitySpy = jest.spyOn(db, 'getEntity');
|
|
45
45
|
|
|
@@ -8,7 +8,7 @@ import { withEntitiesAndSelectMethods, withEntitiesAndDBMethods } from '../store
|
|
|
8
8
|
|
|
9
9
|
import { provideFirebaseMock } from './firebase.mock.service';
|
|
10
10
|
import { MOCK_SIMULATION_USERS, MOCK_SIMULATION_USER_CONTEXTS } from './simulation.mock';
|
|
11
|
-
import
|
|
11
|
+
import { LoggingService } from '../logging/logging.service';
|
|
12
12
|
|
|
13
13
|
// Define realistic Stores
|
|
14
14
|
export const UserStore = signalStore(
|
|
@@ -40,13 +40,13 @@ describe('Simulation with Actual Store Syntax', () => {
|
|
|
40
40
|
const userContextStore = TestBed.inject(UserContextStore) as any;
|
|
41
41
|
const injector = TestBed.inject(EnvironmentInjector);
|
|
42
42
|
|
|
43
|
-
const logSpy = jest.spyOn(
|
|
43
|
+
const logSpy = jest.spyOn(LoggingService.prototype, 'logForNamespace').mockImplementation(() => {});
|
|
44
44
|
console.log('=== STARTING REALISTIC STORE SIMULATION ===\n');
|
|
45
|
-
|
|
45
|
+
|
|
46
46
|
// Create mock injectors and destroy refs for separate components
|
|
47
47
|
const componentA = { onDestroyFns: [] as any[] };
|
|
48
48
|
const mockInjectorA = { get: (token: any, notFound?: any, flags?: any) => token === DestroyRef ? { onDestroy: (cb: any) => componentA.onDestroyFns.push(cb) } : injector.get(token, notFound, flags) };
|
|
49
|
-
|
|
49
|
+
|
|
50
50
|
const componentB = { onDestroyFns: [] as any[] };
|
|
51
51
|
const mockInjectorB = { get: (token: any, notFound?: any, flags?: any) => token === DestroyRef ? { onDestroy: (cb: any) => componentB.onDestroyFns.push(cb) } : injector.get(token, notFound, flags) };
|
|
52
52
|
|
|
@@ -64,8 +64,8 @@ describe('Simulation with Actual Store Syntax', () => {
|
|
|
64
64
|
// 3. Component C asks for 3 IDs (2 overlap with A, 1 new) and has a NESTED stream query
|
|
65
65
|
// Streaming the user's UserContext
|
|
66
66
|
userStore.stream(
|
|
67
|
-
[['__id', 'in', ['id1', 'id2', 'id9']]],
|
|
68
|
-
{},
|
|
67
|
+
[['__id', 'in', ['id1', 'id2', 'id9']]],
|
|
68
|
+
{},
|
|
69
69
|
mockInjectorC,
|
|
70
70
|
(user: any) => {
|
|
71
71
|
const contextId = `${user.__id}_context`;
|
|
@@ -99,28 +99,28 @@ describe('Simulation with Actual Store Syntax', () => {
|
|
|
99
99
|
// 8. Test quick recreation (Component E requests id100, dies, Component F requests id100 immediately)
|
|
100
100
|
const componentE = { onDestroyFns: [] as any[] };
|
|
101
101
|
const mockInjectorE = { get: (token: any, notFound?: any, flags?: any) => token === DestroyRef ? { onDestroy: (cb: any) => componentE.onDestroyFns.push(cb) } : injector.get(token, notFound, flags) };
|
|
102
|
-
|
|
102
|
+
|
|
103
103
|
console.log('\n---> Action: Component E mounts, requests id100');
|
|
104
104
|
userStore.stream([['__id', 'in', ['id100']]], {}, mockInjectorE);
|
|
105
105
|
await new Promise((r) => setTimeout(r, 50));
|
|
106
|
-
|
|
106
|
+
|
|
107
107
|
console.log('\n---> Action: Component E destroys itself');
|
|
108
108
|
componentE.onDestroyFns.forEach((fn) => fn());
|
|
109
109
|
await new Promise((r) => setTimeout(r, 1000)); // wait 1s (less than 5s)
|
|
110
110
|
|
|
111
111
|
const componentF = { onDestroyFns: [] as any[] };
|
|
112
112
|
const mockInjectorF = { get: (token: any, notFound?: any, flags?: any) => token === DestroyRef ? { onDestroy: (cb: any) => componentF.onDestroyFns.push(cb) } : injector.get(token, notFound, flags) };
|
|
113
|
-
|
|
113
|
+
|
|
114
114
|
console.log('\n---> Action: Component F mounts and requests id100 BEFORE the 5s timeout finishes');
|
|
115
115
|
userStore.stream([['__id', 'in', ['id100']]], {}, mockInjectorF);
|
|
116
|
-
|
|
116
|
+
|
|
117
117
|
await new Promise((r) => setTimeout(r, 4500)); // Wait out the rest of Component E's 5s timeout
|
|
118
118
|
|
|
119
119
|
// Verify key behaviors through logs
|
|
120
|
-
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[StreamMultiplexer] Opening 1 new websocket stream(s) for chunks:'));
|
|
121
|
-
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[StreamMultiplexer] Tearing down 9 old websocket stream(s).'));
|
|
122
|
-
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[StreamMultiplexer] Smart Repack Triggered!'));
|
|
123
|
-
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[StreamMultiplexer] Component destroyed 5 seconds ago! Releasing its holds on IDs: [id100]'));
|
|
120
|
+
expect(logSpy).toHaveBeenCalledWith('query', expect.stringContaining('[StreamMultiplexer] Opening 1 new websocket stream(s) for chunks:'));
|
|
121
|
+
expect(logSpy).toHaveBeenCalledWith('query', expect.stringContaining('[StreamMultiplexer] Tearing down 9 old websocket stream(s).'));
|
|
122
|
+
expect(logSpy).toHaveBeenCalledWith('query', expect.stringContaining('[StreamMultiplexer] Smart Repack Triggered!'));
|
|
123
|
+
expect(logSpy).toHaveBeenCalledWith('query', expect.stringContaining('[StreamMultiplexer] Component destroyed 5 seconds ago! Releasing its holds on IDs: [id100]'));
|
|
124
124
|
|
|
125
125
|
console.log('\n=== FULL STREAMING SIMULATION COMPLETE ===');
|
|
126
126
|
}, 15000);
|
|
@@ -1,41 +1,43 @@
|
|
|
1
1
|
import { DestroyRef } from '@angular/core';
|
|
2
2
|
import { DatabaseService } from './database.service';
|
|
3
3
|
import { entityInQueryResults } from '../store/app.store';
|
|
4
|
-
import {
|
|
4
|
+
import { getLogger } from '../logging/logging.service';
|
|
5
5
|
import { QueryParams, AnyEntity } from '../store/app.model';
|
|
6
6
|
|
|
7
|
+
const queryLogger = getLogger('query');
|
|
8
|
+
|
|
7
9
|
/**
|
|
8
|
-
* StreamMultiplexer optimizes Firestore realtime subscriptions by multiplexing multiple
|
|
10
|
+
* StreamMultiplexer optimizes Firestore realtime subscriptions by multiplexing multiple
|
|
9
11
|
* independent component requests into a minimal number of batched 'in' queries.
|
|
10
|
-
*
|
|
12
|
+
*
|
|
11
13
|
* It manages two pools of websockets:
|
|
12
14
|
* 1. `batchedListeners`: fully packed 'in' queries (chunks of 30 IDs).
|
|
13
15
|
* 2. `singleListeners`: sparse 'in' queries handling newly requested IDs.
|
|
14
|
-
*
|
|
16
|
+
*
|
|
15
17
|
* It balances stream connection limits with UI rendering performance by maintaining
|
|
16
18
|
* sparse streams until a threshold is reached, at which point it automatically triggers
|
|
17
19
|
* an `executeSmartRepack` to consolidate streams and free up websocket connections.
|
|
18
|
-
*
|
|
19
|
-
* It tracks component lifecycles using `DestroyRef` to gracefully tear down streams
|
|
20
|
+
*
|
|
21
|
+
* It tracks component lifecycles using `DestroyRef` to gracefully tear down streams
|
|
20
22
|
* when components are destroyed and their requested IDs become "orphaned".
|
|
21
23
|
*/
|
|
22
24
|
export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
23
25
|
// Maps a unique query value ID (e.g., a specific user ID) to the set of component DestroyRefs that requested it.
|
|
24
26
|
private destroyRefsById = new Map<string, Set<DestroyRef>>();
|
|
25
|
-
|
|
26
|
-
// Maps a component's DestroyRef to the set of IDs it requested. This allows us to safely clean up all of a component's
|
|
27
|
+
|
|
28
|
+
// Maps a component's DestroyRef to the set of IDs it requested. This allows us to safely clean up all of a component's
|
|
27
29
|
// requested IDs when it is destroyed, even if it called registerIds multiple times.
|
|
28
30
|
private idsByDestroyRef = new Map<DestroyRef, Set<string>>();
|
|
29
|
-
|
|
31
|
+
|
|
30
32
|
// Tracks optimized batches of up to 30 IDs
|
|
31
33
|
private batchedListeners: Array<{ ids: string[], unsubscribe: (() => void) | null }> = [];
|
|
32
|
-
|
|
34
|
+
|
|
33
35
|
// Tracks newly added small groups of IDs that haven't been repacked yet
|
|
34
36
|
private singleListeners: Array<{ ids: string[], unsubscribe: (() => void) | null }> = [];
|
|
35
37
|
|
|
36
38
|
// Threshold for determining when a batch is "sparse" enough to warrant evaluating a repack operation.
|
|
37
39
|
private readonly MAX_ORPHANED_IDS_PER_BATCH = 10;
|
|
38
|
-
|
|
40
|
+
|
|
39
41
|
// Threshold for determining when we have accumulated enough single listeners to warrant repacking them into a batch.
|
|
40
42
|
private readonly MAX_SINGLE_LISTENERS_BEFORE_REPACK = 10;
|
|
41
43
|
|
|
@@ -59,13 +61,13 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
59
61
|
* Automatically handles teardown when the provided destroyRef fires.
|
|
60
62
|
*/
|
|
61
63
|
registerIds(ids: string[], destroyRef: DestroyRef) {
|
|
62
|
-
|
|
64
|
+
queryLogger.log(`[StreamMultiplexer] registerIds called with: [${Array.from(ids).join(', ')}]`);
|
|
63
65
|
const newIds: string[] = [];
|
|
64
66
|
|
|
65
67
|
// We maintain a reverse lookup from DestroyRef -> IDs to easily clean up when the component is destroyed.
|
|
66
68
|
let idsForDestroyRef = this.idsByDestroyRef.get(destroyRef);
|
|
67
69
|
const isFirstRegistrationForRef = !idsForDestroyRef;
|
|
68
|
-
|
|
70
|
+
|
|
69
71
|
// Initialize the set if this is the very first time this DestroyRef is calling registerIds()
|
|
70
72
|
if (!idsForDestroyRef) {
|
|
71
73
|
idsForDestroyRef = new Set();
|
|
@@ -75,15 +77,15 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
75
77
|
ids.forEach((id) => {
|
|
76
78
|
// Add the requested ID to the component's tracking set so it can be safely cleaned up later
|
|
77
79
|
idsForDestroyRef.add(id);
|
|
78
|
-
|
|
80
|
+
|
|
79
81
|
// Look up if this exact ID is already being actively streamed by anyone else in the application
|
|
80
82
|
let registeredSet = this.destroyRefsById.get(id);
|
|
81
83
|
if (!registeredSet) {
|
|
82
|
-
// First time this ID has been requested anywhere!
|
|
84
|
+
// First time this ID has been requested anywhere!
|
|
83
85
|
// We create a new Set to track all the specific DestroyRefs that have requested it.
|
|
84
86
|
registeredSet = new Set();
|
|
85
87
|
this.destroyRefsById.set(id, registeredSet);
|
|
86
|
-
|
|
88
|
+
|
|
87
89
|
// Track this as a new ID that needs to be streamed
|
|
88
90
|
newIds.push(id);
|
|
89
91
|
}
|
|
@@ -95,20 +97,20 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
95
97
|
destroyRef.onDestroy(() => {
|
|
96
98
|
setTimeout(() => {
|
|
97
99
|
let triggeredRemove = false;
|
|
98
|
-
|
|
100
|
+
|
|
99
101
|
// Retrieve the dynamically accumulated set of IDs that were requested by this specific component's DestroyRef.
|
|
100
102
|
// By pulling from this Map instead of a closed-over lexical variable, we ensure we clean up ALL IDs
|
|
101
103
|
// requested across multiple calls to registerIds() by the same component.
|
|
102
104
|
const idsToClean = this.idsByDestroyRef.get(destroyRef);
|
|
103
105
|
if (idsToClean) {
|
|
104
|
-
|
|
106
|
+
queryLogger.log(`[StreamMultiplexer] Component destroyed 5 seconds ago! Releasing its holds on IDs: [${Array.from(idsToClean).join(', ')}]`);
|
|
105
107
|
idsToClean.forEach((id) => {
|
|
106
108
|
const registeredSet = this.destroyRefsById.get(id);
|
|
107
109
|
if (registeredSet && registeredSet.has(destroyRef)) {
|
|
108
110
|
// Remove this component's claim to the ID
|
|
109
111
|
registeredSet.delete(destroyRef);
|
|
110
|
-
|
|
111
|
-
// If no other components are currently listening to this ID,
|
|
112
|
+
|
|
113
|
+
// If no other components are currently listening to this ID,
|
|
112
114
|
// completely remove it from the globally active set.
|
|
113
115
|
if (registeredSet.size === 0) {
|
|
114
116
|
this.destroyRefsById.delete(id);
|
|
@@ -116,11 +118,11 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
116
118
|
}
|
|
117
119
|
}
|
|
118
120
|
});
|
|
119
|
-
|
|
121
|
+
|
|
120
122
|
// Cleanup the tracking Map itself to prevent memory leaks
|
|
121
123
|
this.idsByDestroyRef.delete(destroyRef);
|
|
122
124
|
}
|
|
123
|
-
|
|
125
|
+
|
|
124
126
|
if (triggeredRemove) {
|
|
125
127
|
this.cleanupEmptyListeners();
|
|
126
128
|
if (this.hasBatchWithTooManyOrphans()) {
|
|
@@ -139,10 +141,10 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
139
141
|
* A repack is executed if either:
|
|
140
142
|
* 1. A new ID request pushes the projected single listener count over the MAX_SINGLE_LISTENERS_BEFORE_REPACK threshold.
|
|
141
143
|
* 2. An orphaned batch cleanup check triggered this evaluation.
|
|
142
|
-
*
|
|
144
|
+
*
|
|
143
145
|
* If new IDs are requested but the threshold is not met, this method simply spins up
|
|
144
146
|
* a single sparse stream for those IDs to prioritize immediate UI rendering speed.
|
|
145
|
-
*
|
|
147
|
+
*
|
|
146
148
|
* @param newIds - Array of newly requested IDs that are not yet streamed.
|
|
147
149
|
*/
|
|
148
150
|
private checkAndExecuteRepack(newIds: string[]) {
|
|
@@ -158,7 +160,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
158
160
|
// Otherwise, spin up a new listener for the newly requested IDs
|
|
159
161
|
const newChunks = this.chunkArray(newIds, 30);
|
|
160
162
|
const newBatches = this.startNewBatches(newChunks, []);
|
|
161
|
-
|
|
163
|
+
|
|
162
164
|
// We separate full batches from sparse single listeners
|
|
163
165
|
newBatches.forEach((b) => {
|
|
164
166
|
if (b.ids.length === 30) {
|
|
@@ -180,7 +182,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
180
182
|
if (b.unsubscribe) b.unsubscribe();
|
|
181
183
|
});
|
|
182
184
|
this.batchedListeners = [];
|
|
183
|
-
|
|
185
|
+
|
|
184
186
|
this.singleListeners.forEach((s) => {
|
|
185
187
|
if (s.unsubscribe) s.unsubscribe();
|
|
186
188
|
});
|
|
@@ -200,7 +202,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
200
202
|
|
|
201
203
|
const emptyBatches = this.batchedListeners.filter((b) => b.ids.every((id) => !activeIds.has(id)));
|
|
202
204
|
if (emptyBatches.length > 0) {
|
|
203
|
-
|
|
205
|
+
queryLogger.log(`[StreamMultiplexer] Purging ${emptyBatches.length} completely empty batched stream(s).`);
|
|
204
206
|
emptyBatches.forEach((b) => {
|
|
205
207
|
if (b.unsubscribe) {
|
|
206
208
|
b.unsubscribe();
|
|
@@ -212,7 +214,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
212
214
|
|
|
213
215
|
const emptySingles = this.singleListeners.filter((s) => s.ids.every((id) => !activeIds.has(id)));
|
|
214
216
|
if (emptySingles.length > 0) {
|
|
215
|
-
|
|
217
|
+
queryLogger.log(`[StreamMultiplexer] Purging ${emptySingles.length} completely empty single stream(s).`);
|
|
216
218
|
emptySingles.forEach((s) => {
|
|
217
219
|
if (s.unsubscribe) {
|
|
218
220
|
s.unsubscribe();
|
|
@@ -239,22 +241,22 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
239
241
|
|
|
240
242
|
/**
|
|
241
243
|
* Consolidates sparse streams and tears down bloated batches.
|
|
242
|
-
*
|
|
244
|
+
*
|
|
243
245
|
* It uses a greedy mathematical algorithm to evaluate "candidate batches" (those with <= 25 active IDs)
|
|
244
|
-
* and determine if tearing them down and bundling their active IDs with the single listeners will
|
|
246
|
+
* and determine if tearing them down and bundling their active IDs with the single listeners will
|
|
245
247
|
* result in a net reduction in the total number of websocket streams.
|
|
246
|
-
*
|
|
248
|
+
*
|
|
247
249
|
* Note: This method does not check threshold conditions before executing. Callers
|
|
248
|
-
* (e.g. checkAndExecuteRepack) must verify that a repack is strictly necessary
|
|
250
|
+
* (e.g. checkAndExecuteRepack) must verify that a repack is strictly necessary
|
|
249
251
|
* before invoking this method.
|
|
250
|
-
*
|
|
252
|
+
*
|
|
251
253
|
* @param newIds - Array of newly requested IDs that need to be streamed.
|
|
252
254
|
*/
|
|
253
255
|
private executeSmartRepack(newIds: string[] = []) {
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
queryLogger.log(`[StreamMultiplexer] Smart Repack Triggered!`);
|
|
257
|
+
|
|
256
258
|
const activeIds = new Set(this.destroyRefsById.keys());
|
|
257
|
-
|
|
259
|
+
|
|
258
260
|
// 1. Gather all single listeners and new IDs into one pool
|
|
259
261
|
const singleIds: string[] = [];
|
|
260
262
|
this.singleListeners.forEach((s) => singleIds.push(...s.ids.filter((id) => activeIds.has(id))));
|
|
@@ -274,9 +276,9 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
274
276
|
candidateBatches.sort((a, b) => a.activeCount - b.activeCount);
|
|
275
277
|
|
|
276
278
|
// Baseline reduction: bundling all single listeners into tight chunks
|
|
277
|
-
let bestReduction = S > 0 ? S - Math.ceil(S / 30) : 0;
|
|
279
|
+
let bestReduction = S > 0 ? S - Math.ceil(S / 30) : 0;
|
|
278
280
|
let bestT: Array<{ ids: string[], unsubscribe: (() => void) | null }> = [];
|
|
279
|
-
|
|
281
|
+
|
|
280
282
|
let currentA = 0;
|
|
281
283
|
const currentT: Array<{ ids: string[], unsubscribe: (() => void) | null }> = [];
|
|
282
284
|
|
|
@@ -294,7 +296,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
294
296
|
}
|
|
295
297
|
}
|
|
296
298
|
|
|
297
|
-
// 5. Ensure we also forcibly tear down any bloated batches (>= 10 orphans)
|
|
299
|
+
// 5. Ensure we also forcibly tear down any bloated batches (>= 10 orphans)
|
|
298
300
|
// even if they didn't mathematically reduce the total stream count.
|
|
299
301
|
const bloatedBatches = this.batchedListeners.filter((batch) => {
|
|
300
302
|
const orphans = batch.ids.filter((id) => !activeIds.has(id)).length;
|
|
@@ -309,7 +311,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
309
311
|
|
|
310
312
|
const batchesToTeardown = bestT;
|
|
311
313
|
const singleListenersToTeardown = [...this.singleListeners];
|
|
312
|
-
|
|
314
|
+
|
|
313
315
|
// 6. Collect all the actively requested IDs from the streams we are about to tear down
|
|
314
316
|
const idsToChunk: string[] = [...singleIds];
|
|
315
317
|
batchesToTeardown.forEach((b) => {
|
|
@@ -324,17 +326,17 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
324
326
|
|
|
325
327
|
// 8. Break the collected IDs into optimal chunks of 30 for Firestore 'in' queries
|
|
326
328
|
const newChunks = this.chunkArray(idsToChunk, 30);
|
|
327
|
-
|
|
329
|
+
|
|
328
330
|
// 9. Spin up the new websocket streams, passing the old ones to be torn down only AFTER the new ones emit data
|
|
329
331
|
const newBatches = this.startNewBatches(newChunks, [...batchesToTeardown, ...singleListenersToTeardown]);
|
|
330
|
-
|
|
332
|
+
|
|
331
333
|
// 10. Update our internal tracking arrays to reflect the new state
|
|
332
334
|
this.finalizeRepackState(batchesToTeardown, singleListenersToTeardown, newBatches);
|
|
333
335
|
}
|
|
334
336
|
|
|
335
337
|
/**
|
|
336
338
|
* Opens new Firestore snapshot streams for the given chunks of IDs.
|
|
337
|
-
*
|
|
339
|
+
*
|
|
338
340
|
* @param newChunks - The fully chunked arrays of IDs to stream (max 30 per chunk).
|
|
339
341
|
* @param oldBatchesToTeardown - The old websocket listeners that these new chunks are replacing.
|
|
340
342
|
* They are only torn down *after* the new stream emits its first
|
|
@@ -342,8 +344,8 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
342
344
|
*/
|
|
343
345
|
private startNewBatches(newChunks: string[][], oldBatchesToTeardown: Array<{ ids: string[], unsubscribe: (() => void) | null }>) {
|
|
344
346
|
if (newChunks.length > 0) {
|
|
345
|
-
|
|
346
|
-
newChunks.forEach((c) =>
|
|
347
|
+
queryLogger.log(`[StreamMultiplexer] Opening ${newChunks.length} new websocket stream(s) for chunks:`);
|
|
348
|
+
newChunks.forEach((c) => queryLogger.log(` - [${c.join(', ')}]`));
|
|
347
349
|
}
|
|
348
350
|
const newBatches: Array<{ ids: string[], unsubscribe: (() => void) | null }> = [];
|
|
349
351
|
|
|
@@ -370,9 +372,9 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
370
372
|
*/
|
|
371
373
|
private teardownOldListeners(oldBatches: Array<{ ids: string[], unsubscribe: (() => void) | null }>) {
|
|
372
374
|
if (oldBatches.length > 0) {
|
|
373
|
-
|
|
375
|
+
queryLogger.log(`[StreamMultiplexer] Tearing down ${oldBatches.length} old websocket stream(s).`);
|
|
374
376
|
oldBatches.forEach((b) => {
|
|
375
|
-
|
|
377
|
+
queryLogger.log(` - [${b.ids.join(', ')}]`);
|
|
376
378
|
if (b.unsubscribe) {
|
|
377
379
|
b.unsubscribe();
|
|
378
380
|
b.unsubscribe = null;
|
|
@@ -392,7 +394,7 @@ export class StreamMultiplexer<T extends AnyEntity = AnyEntity> {
|
|
|
392
394
|
) {
|
|
393
395
|
this.batchedListeners = this.batchedListeners.filter((b) => !oldBatchesToTeardown.includes(b));
|
|
394
396
|
this.singleListeners = this.singleListeners.filter((s) => !oldSingleListenersToTeardown.includes(s));
|
|
395
|
-
|
|
397
|
+
|
|
396
398
|
newBatches.forEach((b) => {
|
|
397
399
|
if (b.ids.length === 30) {
|
|
398
400
|
this.batchedListeners.push(b);
|