@futdevpro/nts-dynamo 1.15.78 → 1.15.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/.dynamo/logs/cicd-pipeline/output.log +1670 -1668
  2. package/.dynamo/logs/cicd-pipeline/status.json +32 -32
  3. package/build/_collections/global-settings.const.d.ts.map +1 -1
  4. package/build/_collections/global-settings.const.js +11 -0
  5. package/build/_collections/global-settings.const.js.map +1 -1
  6. package/build/_models/interfaces/global-settings.interface.d.ts +18 -0
  7. package/build/_models/interfaces/global-settings.interface.d.ts.map +1 -1
  8. package/build/_services/base/data.service.d.ts +8 -0
  9. package/build/_services/base/data.service.d.ts.map +1 -1
  10. package/build/_services/base/data.service.js +17 -0
  11. package/build/_services/base/data.service.js.map +1 -1
  12. package/build/_services/core/collection-growth-monitor.service.d.ts +77 -0
  13. package/build/_services/core/collection-growth-monitor.service.d.ts.map +1 -0
  14. package/build/_services/core/collection-growth-monitor.service.js +147 -0
  15. package/build/_services/core/collection-growth-monitor.service.js.map +1 -0
  16. package/build/_services/server/app.server.d.ts +21 -7
  17. package/build/_services/server/app.server.d.ts.map +1 -1
  18. package/build/_services/server/app.server.js +45 -17
  19. package/build/_services/server/app.server.js.map +1 -1
  20. package/build/index.d.ts +1 -0
  21. package/build/index.d.ts.map +1 -1
  22. package/build/index.js +1 -0
  23. package/build/index.js.map +1 -1
  24. package/package.json +1 -1
  25. package/src/_collections/global-settings.const.ts +12 -0
  26. package/src/_models/interfaces/global-settings.interface.ts +19 -0
  27. package/src/_services/base/data.service.spec.ts +56 -1
  28. package/src/_services/base/data.service.ts +23 -2
  29. package/src/_services/core/collection-growth-monitor.service.spec.ts +67 -0
  30. package/src/_services/core/collection-growth-monitor.service.ts +211 -0
  31. package/src/_services/server/app.server-retention.spec.ts +10 -10
  32. package/src/_services/server/app.server.ts +55 -19
  33. package/src/index.ts +1 -0
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA2BA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,+CAA+C,CAAC;AAC9D,cAAc,qDAAqD,CAAC;AACpE,cAAc,6CAA6C,CAAC;AAC5D,cAAc,0CAA0C,CAAC;AACzD,cAAc,8CAA8C,CAAC;AAC7D,cAAc,sCAAsC,CAAC;AAIrD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,qCAAqC,CAAC;AACpD,cAAc,qCAAqC,CAAC;AACpD,cAAc,8BAA8B,CAAC;AAK7C,mEAAmE;AACnE,cAAc,uDAAuD,CAAC;AACtE,cAAc,qDAAqD,CAAC;AACpE,cAAc,wDAAwD,CAAC;AACvE,cAAc,gDAAgD,CAAC;AAC/D,cAAc,wDAAwD,CAAC;AACvE,cAAc,uDAAuD,CAAC;AACtE,cAAc,8CAA8C,CAAC;AAG7D,cAAc,wDAAwD,CAAC;AACvE,cAAc,gEAAgE,CAAA;AAC9E,cAAc,mDAAmD,CAAC;AAClE,cAAc,4DAA4D,CAAC;AAC3E,cAAc,wDAAwD,CAAC;AACvE,cAAc,sDAAsD,CAAC;AACrE,cAAc,uDAAuD,CAAC;AAGtE,cAAc,gCAAgC,CAAC;AAK/C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,+BAA+B,CAAC;AAE9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AAEtD,cAAc,6CAA6C,CAAC;AAG5D,cAAc,mCAAmC,CAAC;AAClD,cAAc,uCAAuC,CAAC;AACtD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yCAAyC,CAAC;AACxD,cAAc,oCAAoC,CAAC;AAGnD,cAAc,+BAA+B,CAAC;AAG9C,cAAc,sCAAsC,CAAC;AACrD,cAAc,0CAA0C,CAAC;AAGzD,cAAc,mCAAmC,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA2BA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,+CAA+C,CAAC;AAC9D,cAAc,qDAAqD,CAAC;AACpE,cAAc,6CAA6C,CAAC;AAC5D,cAAc,0CAA0C,CAAC;AACzD,cAAc,8CAA8C,CAAC;AAC7D,cAAc,sCAAsC,CAAC;AAIrD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,qCAAqC,CAAC;AACpD,cAAc,qCAAqC,CAAC;AACpD,cAAc,8BAA8B,CAAC;AAK7C,mEAAmE;AACnE,cAAc,uDAAuD,CAAC;AACtE,cAAc,qDAAqD,CAAC;AACpE,cAAc,wDAAwD,CAAC;AACvE,cAAc,gDAAgD,CAAC;AAC/D,cAAc,wDAAwD,CAAC;AACvE,cAAc,uDAAuD,CAAC;AACtE,cAAc,8CAA8C,CAAC;AAG7D,cAAc,wDAAwD,CAAC;AACvE,cAAc,gEAAgE,CAAA;AAC9E,cAAc,mDAAmD,CAAC;AAClE,cAAc,4DAA4D,CAAC;AAC3E,cAAc,wDAAwD,CAAC;AACvE,cAAc,sDAAsD,CAAC;AACrE,cAAc,uDAAuD,CAAC;AAGtE,cAAc,gCAAgC,CAAC;AAK/C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,+BAA+B,CAAC;AAE9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oDAAoD,CAAC;AACnE,cAAc,iCAAiC,CAAC;AAChD,cAAc,uCAAuC,CAAC;AAEtD,cAAc,6CAA6C,CAAC;AAG5D,cAAc,mCAAmC,CAAC;AAClD,cAAc,uCAAuC,CAAC;AACtD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yCAAyC,CAAC;AACxD,cAAc,oCAAoC,CAAC;AAGnD,cAAc,+BAA+B,CAAC;AAG9C,cAAc,sCAAsC,CAAC;AACrD,cAAc,0CAA0C,CAAC;AAGzD,cAAc,mCAAmC,CAAC"}
package/build/index.js CHANGED
@@ -61,6 +61,7 @@ tslib_1.__exportStar(require("./_models/types/db-update.type"), exports);
61
61
  tslib_1.__exportStar(require("./_services/core/api.service"), exports);
62
62
  tslib_1.__exportStar(require("./_services/core/auth.service"), exports);
63
63
  tslib_1.__exportStar(require("./_services/core/email.service"), exports);
64
+ tslib_1.__exportStar(require("./_services/core/collection-growth-monitor.service"), exports);
64
65
  tslib_1.__exportStar(require("./_services/core/global.service"), exports);
65
66
  tslib_1.__exportStar(require("./_services/core/memory-guard.service"), exports);
66
67
  tslib_1.__exportStar(require("./_services/core/service-collection.service"), exports);
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;;;;;;;IAsBI;;;AAEJ,cAAc;AACd,sEAA4C;AAC5C,wFAA8D;AAC9D,8FAAoE;AACpE,sFAA4D;AAC5D,mFAAyD;AACzD,uFAA6D;AAC7D,+EAAqD;AAGrD,QAAQ;AACR,wEAA8C;AAC9C,8EAAoD;AACpD,8EAAoD;AACpD,uEAA6C;AAG7C,6CAA6C;AAE7C,mEAAmE;AACnE,gGAAsE;AACtE,8FAAoE;AACpE,iGAAuE;AACvE,yFAA+D;AAC/D,iGAAuE;AACvE,gGAAsE;AACtE,uFAA6D;AAE7D,wBAAwB;AACxB,iGAAuE;AACvE,yGAA8E;AAC9E,4FAAkE;AAClE,qGAA2E;AAC3E,iGAAuE;AACvE,+FAAqE;AACrE,gGAAsE;AAEtE,eAAe;AACf,yEAA+C;AAG/C,WAAW;AACX,gBAAgB;AAChB,uEAA6C;AAC7C,wEAA8C;AAE9C,yEAA+C;AAC/C,0EAAgD;AAChD,gFAAsD;AAEtD,sFAA4D;AAE5D,gBAAgB;AAChB,4EAAkD;AAClD,gFAAsD;AACtD,wEAA8C;AAC9C,sEAA4C;AAC5C,kFAAwD;AACxD,6EAAmD;AAEnD,kBAAkB;AAClB,wEAA8C;AAE9C,iBAAiB;AACjB,+EAAqD;AACrD,mFAAyD;AAEzD,kBAAkB;AAClB,4EAAkD"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;;;;;;;IAsBI;;;AAEJ,cAAc;AACd,sEAA4C;AAC5C,wFAA8D;AAC9D,8FAAoE;AACpE,sFAA4D;AAC5D,mFAAyD;AACzD,uFAA6D;AAC7D,+EAAqD;AAGrD,QAAQ;AACR,wEAA8C;AAC9C,8EAAoD;AACpD,8EAAoD;AACpD,uEAA6C;AAG7C,6CAA6C;AAE7C,mEAAmE;AACnE,gGAAsE;AACtE,8FAAoE;AACpE,iGAAuE;AACvE,yFAA+D;AAC/D,iGAAuE;AACvE,gGAAsE;AACtE,uFAA6D;AAE7D,wBAAwB;AACxB,iGAAuE;AACvE,yGAA8E;AAC9E,4FAAkE;AAClE,qGAA2E;AAC3E,iGAAuE;AACvE,+FAAqE;AACrE,gGAAsE;AAEtE,eAAe;AACf,yEAA+C;AAG/C,WAAW;AACX,gBAAgB;AAChB,uEAA6C;AAC7C,wEAA8C;AAE9C,yEAA+C;AAC/C,6FAAmE;AACnE,0EAAgD;AAChD,gFAAsD;AAEtD,sFAA4D;AAE5D,gBAAgB;AAChB,4EAAkD;AAClD,gFAAsD;AACtD,wEAA8C;AAC9C,sEAA4C;AAC5C,kFAAwD;AACxD,6EAAmD;AAEnD,kBAAkB;AAClB,wEAA8C;AAE9C,iBAAiB;AACjB,+EAAqD;AACrD,mFAAyD;AAEzD,kBAAkB;AAClB,4EAAkD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@futdevpro/nts-dynamo",
3
- "version": "01.15.78",
3
+ "version": "01.15.81",
4
4
  "description": "Dynamic NodeTS (NodeJS-Typescript), MongoDB Backend System Framework by Future Development Program Ltd.",
5
5
  "DyBu_settings": {
6
6
  "packageType": "server-package",
@@ -93,5 +93,17 @@ export const DyNTS_global_settings: DyNTS_Global_Settings = {
93
93
  exitCode: 137,
94
94
  bootGraceMs: 60000,
95
95
  },
96
+
97
+ // FR-258 / SR-4 — proactive collection-growth monitor (companion to memoryGuard). Periodically
98
+ // samples each collection's document count (scan-free estimatedDocumentCount) and warns into the
99
+ // Errors-sink BEFORE an unbounded collection can be loaded into heap and OOM. Observation only —
100
+ // never deletes (retention SR-1/SR-2 does the deleting). Slow cadence (5 min) — growth is slow.
101
+ collectionGrowthMonitor: {
102
+ enabled: true,
103
+ pollIntervalMs: 300000,
104
+ sizeThresholdDocCount: 250000,
105
+ growthRateThresholdPct: 50,
106
+ reAlertCooldownMs: 3600000,
107
+ },
96
108
  };
97
109
 
@@ -213,4 +213,23 @@ export interface DyNTS_Global_Settings {
213
213
  */
214
214
  bootGraceMs?: number;
215
215
  };
216
+
217
+ /**
218
+ * FR-258 / SR-4 — proactive collection-growth monitor (companion to {@link memoryGuard}). Samples
219
+ * each registered collection's document count (scan-free `estimatedDocumentCount`) on a slow
220
+ * cadence and raises a warning into the Errors-sink BEFORE an unbounded collection can be loaded
221
+ * into heap and OOM. Observation only — never deletes (retention SR-1/SR-2 does the deleting).
222
+ */
223
+ collectionGrowthMonitor?: {
224
+ /** Bekapcsolja a monitort (a base App startup feltelepíti). Default: true. */
225
+ enabled: boolean;
226
+ /** Poll-intervallum ms-ben. Default: 300000 (5 min). */
227
+ pollIntervalMs?: number;
228
+ /** Warning, ha egy collection doc-száma ezt meghaladja. Default: 250000. */
229
+ sizeThresholdDocCount?: number;
230
+ /** Warning, ha egy collection az előző poll óta ennél többel nőtt (%). Default: 50. */
231
+ growthRateThresholdPct?: number;
232
+ /** Re-alert cooldown collection-önként + alert-típusonként, ms (anti-flood). Default: 3600000 (1 h). */
233
+ reAlertCooldownMs?: number;
234
+ };
216
235
  }
@@ -3,7 +3,7 @@ import { DyNTS_DataService } from './data.service';
3
3
  import { DyNTS_DBService } from './db.service';
4
4
  import { DyNTS_GlobalService } from '../core/global.service';
5
5
  import { DyNTS_ArchiveDataService } from './archive-data.service';
6
- import { DyFM_Metadata, DyFM_DataModel_Params, DyFM_Error, DyFM_DBFilter, DyFM_SearchQuery, DyFM_BasicProperty_Type, DyFM_EnvironmentFlag } from '@futdevpro/fsm-dynamo';
6
+ import { DyFM_Metadata, DyFM_DataModel_Params, DyFM_Error, DyFM_DBFilter, DyFM_SearchQuery, DyFM_BasicProperty_Type, DyFM_EnvironmentFlag, DyFM_Log } from '@futdevpro/fsm-dynamo';
7
7
  import { DyNTS_global_settings } from '../../_collections/global-settings.const';
8
8
 
9
9
  // Initialize global settings before any test runs
@@ -273,6 +273,61 @@ describe('| DyNTS_DataService', () => {
273
273
  expect(result).toEqual(mockDataList);
274
274
  expect(service.dataList).toEqual(mockDataList);
275
275
  });
276
+
277
+ // FR-258 / SR-3 — over-fetch guard (unbounded find → heap → OOM precursor). Warns, never caps.
278
+ describe('| over-fetch warning (FR-258 / SR-3)', () => {
279
+ let originalThreshold: number;
280
+
281
+ beforeEach(() => {
282
+ originalThreshold = DyNTS_DataService.findDataListWarnThreshold;
283
+ });
284
+
285
+ afterEach(() => {
286
+ DyNTS_DataService.findDataListWarnThreshold = originalThreshold;
287
+ });
288
+
289
+ function listOf(count: number): TestMetadata[] {
290
+ return Array.from({ length: count }, (_v, i: number): TestMetadata =>
291
+ new TestMetadata({ _id: `id-${i}`, name: `n-${i}` } as TestMetadata));
292
+ }
293
+
294
+ it('| warns when the result EXCEEDS the threshold', async () => {
295
+ DyNTS_DataService.findDataListWarnThreshold = 3;
296
+ const warnSpy = spyOn(DyFM_Log, 'warn');
297
+ mockDBService.find.and.returnValue(Promise.resolve(listOf(5)));
298
+
299
+ const result = await service.findDataList({ name: 'Test' });
300
+
301
+ expect(result.length).toBe(5);
302
+ const overFetchCall = warnSpy.calls.allArgs().find((args: any[]): boolean =>
303
+ typeof args[0] === 'string' && args[0].includes('[FR-258 SR-3]'));
304
+ expect(overFetchCall).toBeDefined();
305
+ });
306
+
307
+ it('| does NOT warn at exactly the threshold (strictly greater)', async () => {
308
+ DyNTS_DataService.findDataListWarnThreshold = 5;
309
+ const warnSpy = spyOn(DyFM_Log, 'warn');
310
+ mockDBService.find.and.returnValue(Promise.resolve(listOf(5)));
311
+
312
+ await service.findDataList({ name: 'Test' });
313
+
314
+ const overFetchCall = warnSpy.calls.allArgs().find((args: any[]): boolean =>
315
+ typeof args[0] === 'string' && args[0].includes('[FR-258 SR-3]'));
316
+ expect(overFetchCall).toBeUndefined();
317
+ });
318
+
319
+ it('| disabled (threshold <= 0) → never warns regardless of size', async () => {
320
+ DyNTS_DataService.findDataListWarnThreshold = 0;
321
+ const warnSpy = spyOn(DyFM_Log, 'warn');
322
+ mockDBService.find.and.returnValue(Promise.resolve(listOf(50000)));
323
+
324
+ await service.findDataList({ name: 'Test' });
325
+
326
+ const overFetchCall = warnSpy.calls.allArgs().find((args: any[]): boolean =>
327
+ typeof args[0] === 'string' && args[0].includes('[FR-258 SR-3]'));
328
+ expect(overFetchCall).toBeUndefined();
329
+ });
330
+ });
276
331
  });
277
332
 
278
333
  describe('| updateData', () => {
@@ -81,6 +81,15 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
81
81
  /** error code base */
82
82
  ecBase: string = `${DyNTS_global_settings.systemShortCodeName}|`;
83
83
 
84
+ /**
85
+ * FR-258 / SR-3 — over-fetch küszöb. Ennél több dokumentumot visszaadó {@link findDataList} hívás
86
+ * az unbounded-growth → GC-thrash/OOM vektor (2026-06-23 incidens): a teljes (akár több-100k-s)
87
+ * collection a heap-be töltődik. NEM capeljük a query-t (az törné a hívókat) — csak WARNINGot
88
+ * logolunk, hogy az over-fetch LÁTHATÓ legyen és paginálható/szűkíthető. Runtime felülírható
89
+ * (`DyNTS_DataService.findDataListWarnThreshold = N`). Default: 10000.
90
+ */
91
+ static findDataListWarnThreshold: number = 10000;
92
+
84
93
  dataDBService: DyNTS_DBService<T>;
85
94
  haveArchiveDataService: boolean;
86
95
 
@@ -676,11 +685,23 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
676
685
  this.dataList = dataListExists;
677
686
  }
678
687
 
688
+ // FR-258 / SR-3 — over-fetch guard. Egy szűretlen find, ami egy nagy collection EGÉSZÉT a heap-be
689
+ // tölti, a GC-thrash/OOM vektor (2026-06-23). NEM capeljük a query-t (az törné a hívókat), csak
690
+ // WARNINGot logolunk, hogy az over-fetch LÁTHATÓ legyen → paginate/szűkebb filter a megoldás.
691
+ const overFetchThreshold: number = DyNTS_DataService.findDataListWarnThreshold;
692
+ if (overFetchThreshold > 0 && dataListExists.length > overFetchThreshold) {
693
+ DyFM_Log.warn(
694
+ `[FR-258 SR-3] findDataList "${this.dataParams.dataName}" returned ${dataListExists.length} documents ` +
695
+ `(> ${overFetchThreshold}) — the whole result is held in heap. Consider a tighter filter or ` +
696
+ `pagination (skip/limit) to avoid unbounded memory growth.`,
697
+ );
698
+ }
699
+
679
700
  return dataListExists;
680
701
  } catch (error) {
681
- throw new DyFM_Error({
702
+ throw new DyFM_Error({
682
703
  ...this._getDefaultErrorSettings('findDataList', error),
683
-
704
+
684
705
  errorCode: `${this.ecBase}DyNTS-DS0-FDS0`,
685
706
  });
686
707
  }
@@ -0,0 +1,67 @@
1
+ import { DyNTS_CollectionGrowthAlert, DyNTS_CollectionGrowthMonitor } from './collection-growth-monitor.service';
2
+
3
+ /**
4
+ * FR-258 / SR-4 — unit tests for the pure growth-evaluation core
5
+ * (`DyNTS_CollectionGrowthMonitor.evaluateGrowth`). No I/O, no timers — covers size-threshold,
6
+ * growth-rate, both/neither, and the first-poll (no prior snapshot) edge cases.
7
+ */
8
+ describe('| DyNTS_CollectionGrowthMonitor.evaluateGrowth (FR-258 / SR-4)', () => {
9
+
10
+ const SIZE_THRESHOLD: number = 250000;
11
+ const GROWTH_PCT: number = 50;
12
+
13
+ function kinds(alerts: DyNTS_CollectionGrowthAlert[]): string[] {
14
+ return alerts.map((a: DyNTS_CollectionGrowthAlert): string => a.kind).sort();
15
+ }
16
+
17
+ it('| under threshold + no prior snapshot → no alerts', () => {
18
+ const alerts: DyNTS_CollectionGrowthAlert[] =
19
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('logs', 100, undefined, SIZE_THRESHOLD, GROWTH_PCT);
20
+ expect(alerts).toEqual([]);
21
+ });
22
+
23
+ it('| over size threshold → a "size" alert with the right detail', () => {
24
+ const alerts: DyNTS_CollectionGrowthAlert[] =
25
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('cdp_report_details_archiveds', 300000, undefined, SIZE_THRESHOLD, GROWTH_PCT);
26
+ expect(kinds(alerts)).toEqual(['size']);
27
+ expect(alerts[0].dataName).toBe('cdp_report_details_archiveds');
28
+ expect(alerts[0].detail).toEqual({ collection: 'cdp_report_details_archiveds', docCount: 300000, threshold: SIZE_THRESHOLD });
29
+ });
30
+
31
+ it('| exactly AT the threshold → no size alert (strictly greater)', () => {
32
+ const alerts: DyNTS_CollectionGrowthAlert[] =
33
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('logs', SIZE_THRESHOLD, undefined, SIZE_THRESHOLD, GROWTH_PCT);
34
+ expect(alerts).toEqual([]);
35
+ });
36
+
37
+ it('| grew more than the growth-rate % since last poll → a "growth" alert', () => {
38
+ const alerts: DyNTS_CollectionGrowthAlert[] =
39
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('pipeline_jobs', 200, { docCount: 100, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
40
+ expect(kinds(alerts)).toEqual(['growth']);
41
+ expect(alerts[0].detail).toEqual({ collection: 'pipeline_jobs', oldCount: 100, newCount: 200, growthPct: 100 });
42
+ });
43
+
44
+ it('| grew LESS than the growth-rate % → no growth alert', () => {
45
+ const alerts: DyNTS_CollectionGrowthAlert[] =
46
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('logs', 110, { docCount: 100, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
47
+ expect(alerts).toEqual([]);
48
+ });
49
+
50
+ it('| both over-threshold AND fast-growth → BOTH alerts', () => {
51
+ const alerts: DyNTS_CollectionGrowthAlert[] =
52
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('big', 400000, { docCount: 100000, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
53
+ expect(kinds(alerts)).toEqual(['growth', 'size']);
54
+ });
55
+
56
+ it('| prior snapshot with zero docCount → no growth alert (avoid divide-by-zero)', () => {
57
+ const alerts: DyNTS_CollectionGrowthAlert[] =
58
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('fresh', 5000, { docCount: 0, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
59
+ expect(alerts).toEqual([]);
60
+ });
61
+
62
+ it('| shrinking collection (negative growth) → no growth alert', () => {
63
+ const alerts: DyNTS_CollectionGrowthAlert[] =
64
+ DyNTS_CollectionGrowthMonitor.evaluateGrowth('pruned', 50, { docCount: 100, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
65
+ expect(alerts).toEqual([]);
66
+ });
67
+ });
@@ -0,0 +1,211 @@
1
+ import { DyFM_Error, DyFM_ErrorLevel, DyFM_Log } from '@futdevpro/fsm-dynamo';
2
+
3
+ import { DyNTS_global_settings } from '../../_collections/global-settings.const';
4
+ import { DyNTS_SingletonServiceBase } from '../base/singleton.service-base';
5
+ import { DyNTS_GlobalService } from './global.service';
6
+
7
+ /**
8
+ * FR-258 / SR-4 — Collection Growth Monitor (proactive companion to DyNTS_MemoryGuard).
9
+ *
10
+ * The MemoryGuard is REACTIVE: it fires when the heap is already near OOM. This monitor is
11
+ * PROACTIVE: it periodically samples each registered collection's document count (via the cheap,
12
+ * scan-free `estimatedDocumentCount()`) and raises a warning into the Errors-sink BEFORE a bloated
13
+ * collection can be loaded into heap and GC-thrash the process (the 2026-06-23 overseer incident:
14
+ * `cdp_report_details_archiveds` reached 3 GB / `pipeline_jobs` 2 GB, unbounded).
15
+ *
16
+ * It pairs with SR-1/SR-2 (declarative retention TTL): TTL keeps collections bounded; this monitor
17
+ * surfaces the ones that AREN'T (no retention declared, or growing faster than retention prunes) so
18
+ * an operator can add `retention` before it becomes an OOM. Pure observation — never deletes.
19
+ *
20
+ * Like the MemoryGuard it is a singleton, idempotent, fully non-fatal (a monitoring layer must never
21
+ * crash a server), and its timer is `unref()`-ed so it can't keep the process alive on its own.
22
+ */
23
+ export interface DyNTS_CollectionGrowthMonitor_Config {
24
+ /** Master switch. Default: `DyNTS_global_settings.collectionGrowthMonitor.enabled` (true). */
25
+ enabled?: boolean;
26
+ /** Poll cadence in ms. Default: 300000 (5 min) — collection growth is slow, no need to poll often. */
27
+ pollIntervalMs?: number;
28
+ /** Warn when a collection's document count exceeds this. Default: 250000. */
29
+ sizeThresholdDocCount?: number;
30
+ /** Warn when a collection grew by more than this percent since the previous poll. Default: 50. */
31
+ growthRateThresholdPct?: number;
32
+ /** Re-alert cooldown per collection per alert-kind, in ms (anti-flood). Default: 3600000 (1 h). */
33
+ reAlertCooldownMs?: number;
34
+ }
35
+
36
+ /** One growth finding to surface to the Errors-sink. */
37
+ export interface DyNTS_CollectionGrowthAlert {
38
+ kind: 'size' | 'growth';
39
+ dataName: string;
40
+ message: string;
41
+ detail: Record<string, unknown>;
42
+ }
43
+
44
+ interface CollectionSnapshot {
45
+ docCount: number;
46
+ atMs: number;
47
+ }
48
+
49
+ export class DyNTS_CollectionGrowthMonitor extends DyNTS_SingletonServiceBase {
50
+
51
+ static getInstance(): DyNTS_CollectionGrowthMonitor {
52
+ return DyNTS_CollectionGrowthMonitor.getSingletonInstance() as DyNTS_CollectionGrowthMonitor;
53
+ }
54
+
55
+ private timer: ReturnType<typeof setInterval> | null = null;
56
+ private installed: boolean = false;
57
+
58
+ // Resolved (effective) config — filled at install().
59
+ private pollIntervalMs: number = 300000;
60
+ private sizeThresholdDocCount: number = 250000;
61
+ private growthRateThresholdPct: number = 50;
62
+ private reAlertCooldownMs: number = 3600000;
63
+
64
+ private readonly lastSnapshotByCollection: Map<string, CollectionSnapshot> = new Map();
65
+ /** Last time we alerted, keyed by `${dataName}:${kind}` — for the anti-flood cooldown. */
66
+ private readonly lastAlertAtByKey: Map<string, number> = new Map();
67
+
68
+ protected constructor() {
69
+ super();
70
+ }
71
+
72
+ /**
73
+ * Start the monitor. Idempotent (a second call is a no-op). `config` overrides
74
+ * `DyNTS_global_settings.collectionGrowthMonitor`, which overrides the built-in defaults. NEVER
75
+ * throws — every error is try/catch-ed and descriptively logged.
76
+ */
77
+ install(config?: DyNTS_CollectionGrowthMonitor_Config): void {
78
+ if (this.installed) { return; }
79
+
80
+ try {
81
+ const g: NonNullable<typeof DyNTS_global_settings.collectionGrowthMonitor> =
82
+ DyNTS_global_settings.collectionGrowthMonitor ?? { enabled: true };
83
+
84
+ if (!(config?.enabled ?? g.enabled ?? true)) { return; }
85
+
86
+ this.pollIntervalMs = config?.pollIntervalMs ?? g.pollIntervalMs ?? 300000;
87
+ this.sizeThresholdDocCount = config?.sizeThresholdDocCount ?? g.sizeThresholdDocCount ?? 250000;
88
+ this.growthRateThresholdPct = config?.growthRateThresholdPct ?? g.growthRateThresholdPct ?? 50;
89
+ this.reAlertCooldownMs = config?.reAlertCooldownMs ?? g.reAlertCooldownMs ?? 3600000;
90
+
91
+ this.installed = true;
92
+ this.timer = setInterval((): void => { void this.poll(); }, this.pollIntervalMs);
93
+ if (typeof this.timer.unref === 'function') { this.timer.unref(); }
94
+
95
+ DyFM_Log.info(
96
+ `[DyNTS_CollectionGrowthMonitor] installed — poll ${Math.round(this.pollIntervalMs / 1000)}s, ` +
97
+ `size-threshold ${this.sizeThresholdDocCount} docs, growth-rate ${this.growthRateThresholdPct}%/poll`,
98
+ );
99
+ } catch (err: unknown) {
100
+ DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] install failed (non-fatal):', err);
101
+ }
102
+ }
103
+
104
+ /**
105
+ * Pure decision core (unit-tested): given a fresh count + the previous snapshot, return the alerts
106
+ * to raise. NEVER does I/O. A `size` alert fires when `count > sizeThreshold`; a `growth` alert
107
+ * fires when the collection grew by more than `growthRatePct` since the last snapshot (needs a
108
+ * non-zero previous count).
109
+ */
110
+ static evaluateGrowth(
111
+ dataName: string,
112
+ count: number,
113
+ last: CollectionSnapshot | undefined,
114
+ sizeThreshold: number,
115
+ growthRatePct: number,
116
+ ): DyNTS_CollectionGrowthAlert[] {
117
+ const alerts: DyNTS_CollectionGrowthAlert[] = [];
118
+
119
+ if (count > sizeThreshold) {
120
+ alerts.push({
121
+ kind: 'size',
122
+ dataName: dataName,
123
+ message:
124
+ `Collection "${dataName}" has ${count} documents (> ${sizeThreshold} threshold) — ` +
125
+ `unbounded-growth risk. Consider declaring \`retention\` on this model (FR-258 / SR-1).`,
126
+ detail: { collection: dataName, docCount: count, threshold: sizeThreshold },
127
+ });
128
+ }
129
+
130
+ if (last && last.docCount > 0) {
131
+ const growthPct: number = ((count - last.docCount) / last.docCount) * 100;
132
+ if (growthPct > growthRatePct) {
133
+ alerts.push({
134
+ kind: 'growth',
135
+ dataName: dataName,
136
+ message:
137
+ `Collection "${dataName}" grew ${Math.round(growthPct)}% since the last poll ` +
138
+ `(${last.docCount} → ${count} docs) — fast unbounded growth.`,
139
+ detail: { collection: dataName, oldCount: last.docCount, newCount: count, growthPct: Math.round(growthPct) },
140
+ });
141
+ }
142
+ }
143
+
144
+ return alerts;
145
+ }
146
+
147
+ /**
148
+ * One poll cycle: sample every registered collection's `estimatedDocumentCount()` (scan-free) and
149
+ * raise cooldown-gated warnings. Fully non-fatal — a failure on one collection never aborts the rest.
150
+ */
151
+ private async poll(): Promise<void> {
152
+ try {
153
+ const services: ReturnType<typeof DyNTS_GlobalService.getAllDBServices> = DyNTS_GlobalService.getAllDBServices();
154
+ const nowMs: number = Date.now();
155
+
156
+ for (const service of services) {
157
+ const dataName: string | undefined = service?.dataParams?.dataName;
158
+ const model: { estimatedDocumentCount?: () => Promise<number> } | undefined =
159
+ service?.dataModel as unknown as { estimatedDocumentCount?: () => Promise<number> };
160
+ if (!dataName || !model || typeof model.estimatedDocumentCount !== 'function') { continue; }
161
+
162
+ try {
163
+ const count: number = await model.estimatedDocumentCount();
164
+ const last: CollectionSnapshot | undefined = this.lastSnapshotByCollection.get(dataName);
165
+
166
+ const alerts: DyNTS_CollectionGrowthAlert[] = DyNTS_CollectionGrowthMonitor.evaluateGrowth(
167
+ dataName, count, last, this.sizeThresholdDocCount, this.growthRateThresholdPct,
168
+ );
169
+ for (const alert of alerts) {
170
+ this.emitAlert(alert, nowMs);
171
+ }
172
+
173
+ this.lastSnapshotByCollection.set(dataName, { docCount: count, atMs: nowMs });
174
+ } catch (collErr: unknown) {
175
+ DyFM_Log.warn(
176
+ `[DyNTS_CollectionGrowthMonitor] poll for "${dataName}" failed (non-fatal): ` +
177
+ `${collErr instanceof Error ? collErr.message : String(collErr)}`,
178
+ );
179
+ }
180
+ }
181
+ } catch (err: unknown) {
182
+ DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] poll failed (non-fatal):', err);
183
+ }
184
+ }
185
+
186
+ /** Raise one alert to the Errors-sink, gated by the per-(collection,kind) cooldown to avoid flooding. */
187
+ private emitAlert(alert: DyNTS_CollectionGrowthAlert, nowMs: number): void {
188
+ const key: string = `${alert.dataName}:${alert.kind}`;
189
+ const lastAt: number | undefined = this.lastAlertAtByKey.get(key);
190
+ if (lastAt !== undefined && nowMs - lastAt < this.reAlertCooldownMs) { return; }
191
+ this.lastAlertAtByKey.set(key, nowMs);
192
+
193
+ DyFM_Log.warn(`[DyNTS_CollectionGrowthMonitor] ${alert.message}`);
194
+ DyNTS_GlobalService.globalErrorHandler?.(
195
+ new DyFM_Error({
196
+ errorCode: `${DyNTS_global_settings.systemShortCodeName ?? 'DyNTS'}|DyNTS-CGM-${alert.kind === 'size' ? 'SIZE' : 'GROWTH'}`,
197
+ message: alert.message,
198
+ level: DyFM_ErrorLevel.warning,
199
+ additionalContent: alert.detail,
200
+ }),
201
+ );
202
+ }
203
+
204
+ /** Test-only teardown: stop the timer + clear state so specs don't leak between runs. */
205
+ _teardownForTesting(): void {
206
+ if (this.timer) { clearInterval(this.timer); this.timer = null; }
207
+ this.installed = false;
208
+ this.lastSnapshotByCollection.clear();
209
+ this.lastAlertAtByKey.clear();
210
+ }
211
+ }
@@ -3,7 +3,7 @@ import { DyNTS_App, DyNTS_TtlIndexAction, DyNTS_TtlIndexCollection, DyNTS_TtlInd
3
3
  /**
4
4
  * FR-258 / SR-2 — unit tests for the retention TTL-index reconciliation logic
5
5
  * (`DyNTS_App.ensureRetentionTtlIndex`). Pure with respect to a mockable native-driver collection;
6
- * no live MongoDB required. Covers create / no-op / update(drop+recreate) and edge cases.
6
+ * no live MongoDB required. Covers create / no-op / differs(boot-safe, no rebuild) and edge cases.
7
7
  */
8
8
  describe('| DyNTS_App.ensureRetentionTtlIndex (FR-258 / SR-2)', () => {
9
9
 
@@ -51,25 +51,25 @@ describe('| DyNTS_App.ensureRetentionTtlIndex (FR-258 / SR-2)', () => {
51
51
  expect(m.dropped.length).toBe(0);
52
52
  });
53
53
 
54
- it('| existing index with a DIFFERENT ttl → drop + recreate (action "updated")', async () => {
54
+ it('| existing index with a DIFFERENT ttl → BOOT-SAFE: left untouched (action "differs", NO drop/recreate)', async () => {
55
55
  const m: MockColl = makeMockCollection([
56
56
  { name: '__created_1', key: { __created: 1 }, expireAfterSeconds: 999 },
57
57
  ]);
58
58
  const action: DyNTS_TtlIndexAction = await DyNTS_App.ensureRetentionTtlIndex(m.coll, TTL_14D);
59
- expect(action).toBe('updated');
60
- expect(m.dropped).toEqual(['__created_1']);
61
- expect(m.created.length).toBe(1);
62
- expect(m.created[0].options).toEqual({ expireAfterSeconds: TTL_14D });
59
+ expect(action).toBe('differs');
60
+ // Critical: never rebuild a live index at boot (heavy op on large collections → health-probe starvation).
61
+ expect(m.dropped.length).toBe(0);
62
+ expect(m.created.length).toBe(0);
63
63
  });
64
64
 
65
- it('| existing NON-TTL __created index (no expireAfterSeconds) → drop + recreate as TTL', async () => {
65
+ it('| existing NON-TTL __created index (no expireAfterSeconds) → BOOT-SAFE: left untouched (action "differs")', async () => {
66
66
  const m: MockColl = makeMockCollection([
67
67
  { name: '__created_1', key: { __created: 1 } },
68
68
  ]);
69
69
  const action: DyNTS_TtlIndexAction = await DyNTS_App.ensureRetentionTtlIndex(m.coll, 604800);
70
- expect(action).toBe('updated');
71
- expect(m.dropped).toEqual(['__created_1']);
72
- expect(m.created.length).toBe(1);
70
+ expect(action).toBe('differs');
71
+ expect(m.dropped.length).toBe(0);
72
+ expect(m.created.length).toBe(0);
73
73
  });
74
74
 
75
75
  it('| ignores indexes on OTHER fields — only reconciles { __created: 1 }', async () => {
@@ -55,6 +55,7 @@ import {
55
55
  import { DyNTS_DBService } from '../base/db.service';
56
56
  import { DyNTS_SingletonService } from '../base/singleton.service';
57
57
  import { DyNTS_GlobalService } from '../core/global.service';
58
+ import { DyNTS_CollectionGrowthMonitor } from '../core/collection-growth-monitor.service';
58
59
  import { DyNTS_MemoryGuard } from '../core/memory-guard.service';
59
60
  import { DyNTS_RoutingModule } from '../route/routing-module.service';
60
61
  import { DyNTS_getStarRoute } from '../../_collections/star.controller';
@@ -253,8 +254,17 @@ export interface DyNTS_TtlIndexCollection {
253
254
  dropIndex?: (name: string) => Promise<unknown>;
254
255
  }
255
256
 
256
- /** FR-258 / SR-2 — outcome of {@link DyNTS_App.ensureRetentionTtlIndex}. */
257
- export type DyNTS_TtlIndexAction = 'created' | 'updated' | 'noop';
257
+ /**
258
+ * FR-258 / SR-2 outcome of {@link DyNTS_App.ensureRetentionTtlIndex}.
259
+ * - `created`: no `__created` TTL index existed → one was created.
260
+ * - `noop`: an index with the SAME TTL already existed → nothing done.
261
+ * - `differs`: an index exists with a DIFFERENT TTL (or a non-TTL `__created` index) → left UNTOUCHED,
262
+ * only a warning is emitted. We deliberately DO NOT drop+recreate at boot: rebuilding a TTL index on a
263
+ * multi-GB production collection is a heavy server-side op that can starve concurrent boot queries (e.g.
264
+ * the health-probe) and trip a fail-safe deploy revert. A deliberate retention CHANGE must be applied via
265
+ * maintenance, not silently at every startup.
266
+ */
267
+ export type DyNTS_TtlIndexAction = 'created' | 'differs' | 'noop';
258
268
 
259
269
  export abstract class DyNTS_App extends DyNTS_SingletonService {
260
270
 
@@ -579,6 +589,17 @@ export abstract class DyNTS_App extends DyNTS_SingletonService {
579
589
  DyFM_Log.warn('[DyNTS_MemoryGuard] auto-install skipped (non-fatal):', memoryGuardError);
580
590
  }
581
591
 
592
+ // FR-258 / SR-4 — proactive collection-growth monitor (companion to the MemoryGuard). Warns
593
+ // into the Errors-sink when a collection grows unbounded, BEFORE it can be loaded into heap and
594
+ // OOM. Default-on, observation-only, never throws.
595
+ try {
596
+ if (DyNTS_global_settings.collectionGrowthMonitor?.enabled) {
597
+ DyNTS_CollectionGrowthMonitor.getInstance().install();
598
+ }
599
+ } catch (cgmError: unknown) {
600
+ DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] auto-install skipped (non-fatal):', cgmError);
601
+ }
602
+
582
603
  if (!extended) {
583
604
  await this.ready();
584
605
 
@@ -941,10 +962,16 @@ export abstract class DyNTS_App extends DyNTS_SingletonService {
941
962
 
942
963
  resolve();
943
964
 
944
- // FR-258 / SR-2 — install declared-retention TTL indexes. Fire-and-forget +
945
- // fully non-fatal: an index build on a large collection must NOT block startup,
946
- // and any failure here is logged but never crashes the app.
947
- void this.installRetentionTtlIndexes();
965
+ // FR-258 / SR-2 — install declared-retention TTL indexes. DEFERRED + fire-and-forget +
966
+ // fully non-fatal: a create-if-missing index op (and the `indexes()` probes) must NOT
967
+ // compete with boot-readiness. We delay it well past the deploy health-probe window so a
968
+ // cold boot is never slowed by it, then run it detached. `.unref()` keeps it from holding
969
+ // the event loop open. Any failure is logged but never crashes the app. (`'differs'` never
970
+ // rebuilds a live index — see ensureRetentionTtlIndex.)
971
+ const ttlInstallTimer: ReturnType<typeof setTimeout> = setTimeout((): void => {
972
+ void this.installRetentionTtlIndexes();
973
+ }, 30000);
974
+ if (typeof ttlInstallTimer.unref === 'function') { ttlInstallTimer.unref(); }
948
975
  })
949
976
  .on('error', (error): void => {
950
977
  if (!this.systemControls.mongoose.started) {
@@ -1073,14 +1100,21 @@ export abstract class DyNTS_App extends DyNTS_SingletonService {
1073
1100
 
1074
1101
  try {
1075
1102
  const action: DyNTS_TtlIndexAction = await DyNTS_App.ensureRetentionTtlIndex(collection, ttlSeconds);
1076
- if (action === 'created' || action === 'updated') {
1103
+ if (action === 'created') {
1077
1104
  ensured++;
1078
1105
  if (this.logSetup) {
1079
1106
  DyFM_Log.success(
1080
- `[FR-258 retention] TTL index ${action} on "${dataName}" — ` +
1107
+ `[FR-258 retention] TTL index created on "${dataName}" — ` +
1081
1108
  `${Math.round(ttlSeconds / 86400)}d (${ttlSeconds}s)`,
1082
1109
  );
1083
1110
  }
1111
+ } else if (action === 'differs') {
1112
+ // Live index has a different TTL — left untouched at boot (see ensureRetentionTtlIndex doc).
1113
+ DyFM_Log.warn(
1114
+ `[FR-258 retention] "${dataName}" already has a {__created} index with a DIFFERENT TTL than the ` +
1115
+ `declared ${Math.round(ttlSeconds / 86400)}d (${ttlSeconds}s) — left as-is (no boot-time rebuild). ` +
1116
+ `Apply the change via maintenance if intended.`,
1117
+ );
1084
1118
  }
1085
1119
  } catch (idxErr: unknown) {
1086
1120
  DyFM_Log.warn(
@@ -1103,11 +1137,16 @@ export abstract class DyNTS_App extends DyNTS_SingletonService {
1103
1137
 
1104
1138
  /**
1105
1139
  * FR-258 / SR-2 — ensure a single `{ __created: 1 }` TTL index with the given `expireAfterSeconds`.
1106
- * Extracted as a pure-ish static so it is unit-testable with a mock collection. Idempotent:
1107
- * - no existing `__created` index → create it → `'created'`
1108
- * - existing with the SAME TTL → no-op → `'noop'`
1109
- * - existing with a DIFFERENT TTL / non-TTL → drop + recreate → `'updated'`
1110
- * Drop+recreate preserves query coverage (the `{__created:1}` index is restored immediately).
1140
+ * Extracted as a pure-ish static so it is unit-testable with a mock collection. Idempotent + BOOT-SAFE:
1141
+ * - no existing `__created` index → create it → `'created'`
1142
+ * - existing with the SAME TTL → no-op → `'noop'`
1143
+ * - existing with a DIFFERENT TTL / non-TTL → leave it as-is → `'differs'` (warn only — NO drop+recreate)
1144
+ *
1145
+ * The `'differs'` case intentionally does NOT mutate the live index. Dropping + recreating a TTL index on a
1146
+ * large (multi-GB) production collection is a heavy server-side operation; doing it automatically inside the
1147
+ * startup path can starve the concurrent boot queries (including the deploy health-probe) and trigger a
1148
+ * fail-safe revert. Create-if-missing is the essential growth-prevention behaviour; a deliberate retention
1149
+ * CHANGE on an already-indexed collection is a maintenance action, not a silent every-boot rebuild.
1111
1150
  */
1112
1151
  static async ensureRetentionTtlIndex(
1113
1152
  collection: DyNTS_TtlIndexCollection,
@@ -1126,12 +1165,9 @@ export abstract class DyNTS_App extends DyNTS_SingletonService {
1126
1165
  if (existing.expireAfterSeconds === ttlSeconds) {
1127
1166
  return 'noop';
1128
1167
  }
1129
- // Retention changed, or an existing non-TTL `__created` index reconcile by drop+recreate.
1130
- if (existing.name && typeof collection.dropIndex === 'function') {
1131
- await collection.dropIndex(existing.name).catch((): void => undefined);
1132
- }
1133
- await collection.createIndex({ __created: 1 }, { expireAfterSeconds: ttlSeconds });
1134
- return 'updated';
1168
+ // Retention differs from the live index (or it is a non-TTL `__created` index). Leave it UNTOUCHED at
1169
+ // boot only report it. Avoids a heavy drop+recreate competing with boot-readiness on a large collection.
1170
+ return 'differs';
1135
1171
  }
1136
1172
 
1137
1173
  /**
package/src/index.ts CHANGED
@@ -71,6 +71,7 @@ export * from './_services/core/api.service';
71
71
  export * from './_services/core/auth.service';
72
72
 
73
73
  export * from './_services/core/email.service';
74
+ export * from './_services/core/collection-growth-monitor.service';
74
75
  export * from './_services/core/global.service';
75
76
  export * from './_services/core/memory-guard.service';
76
77