@futdevpro/nts-dynamo 1.15.78 → 1.15.80

package/src/_services/base/data.service.spec.ts

@@ -3,7 +3,7 @@ import { DyNTS_DataService } from './data.service';
 import { DyNTS_DBService } from './db.service';
 import { DyNTS_GlobalService } from '../core/global.service';
 import { DyNTS_ArchiveDataService } from './archive-data.service';
-import { DyFM_Metadata, DyFM_DataModel_Params, DyFM_Error, DyFM_DBFilter, DyFM_SearchQuery, DyFM_BasicProperty_Type, DyFM_EnvironmentFlag } from '@futdevpro/fsm-dynamo';
+import { DyFM_Metadata, DyFM_DataModel_Params, DyFM_Error, DyFM_DBFilter, DyFM_SearchQuery, DyFM_BasicProperty_Type, DyFM_EnvironmentFlag, DyFM_Log } from '@futdevpro/fsm-dynamo';
 import { DyNTS_global_settings } from '../../_collections/global-settings.const';
 
 // Initialize global settings before any test runs
@@ -273,6 +273,61 @@ describe('| DyNTS_DataService', () => {
       expect(result).toEqual(mockDataList);
       expect(service.dataList).toEqual(mockDataList);
     });
+
+    // FR-258 / SR-3 — over-fetch guard (unbounded find → heap → OOM precursor). Warns, never caps.
+    describe('| over-fetch warning (FR-258 / SR-3)', () => {
+      let originalThreshold: number;
+
+      beforeEach(() => {
+        originalThreshold = DyNTS_DataService.findDataListWarnThreshold;
+      });
+
+      afterEach(() => {
+        DyNTS_DataService.findDataListWarnThreshold = originalThreshold;
+      });
+
+      function listOf(count: number): TestMetadata[] {
+        return Array.from({ length: count }, (_v, i: number): TestMetadata =>
+          new TestMetadata({ _id: `id-${i}`, name: `n-${i}` } as TestMetadata));
+      }
+
+      it('| warns when the result EXCEEDS the threshold', async () => {
+        DyNTS_DataService.findDataListWarnThreshold = 3;
+        const warnSpy = spyOn(DyFM_Log, 'warn');
+        mockDBService.find.and.returnValue(Promise.resolve(listOf(5)));
+
+        const result = await service.findDataList({ name: 'Test' });
+
+        expect(result.length).toBe(5);
+        const overFetchCall = warnSpy.calls.allArgs().find((args: any[]): boolean =>
+          typeof args[0] === 'string' && args[0].includes('[FR-258 SR-3]'));
+        expect(overFetchCall).toBeDefined();
+      });
+
+      it('| does NOT warn at exactly the threshold (strictly greater)', async () => {
+        DyNTS_DataService.findDataListWarnThreshold = 5;
+        const warnSpy = spyOn(DyFM_Log, 'warn');
+        mockDBService.find.and.returnValue(Promise.resolve(listOf(5)));
+
+        await service.findDataList({ name: 'Test' });
+
+        const overFetchCall = warnSpy.calls.allArgs().find((args: any[]): boolean =>
+          typeof args[0] === 'string' && args[0].includes('[FR-258 SR-3]'));
+        expect(overFetchCall).toBeUndefined();
+      });
+
+      it('| disabled (threshold <= 0) → never warns regardless of size', async () => {
+        DyNTS_DataService.findDataListWarnThreshold = 0;
+        const warnSpy = spyOn(DyFM_Log, 'warn');
+        mockDBService.find.and.returnValue(Promise.resolve(listOf(50000)));
+
+        await service.findDataList({ name: 'Test' });
+
+        const overFetchCall = warnSpy.calls.allArgs().find((args: any[]): boolean =>
+          typeof args[0] === 'string' && args[0].includes('[FR-258 SR-3]'));
+        expect(overFetchCall).toBeUndefined();
+      });
+    });
   });
 
   describe('| updateData', () => {

package/src/_services/base/data.service.ts

@@ -81,6 +81,15 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
   /** error code base */
   ecBase: string = `${DyNTS_global_settings.systemShortCodeName}|`;
 
+  /**
+   * FR-258 / SR-3 — over-fetch küszöb. Ennél több dokumentumot visszaadó {@link findDataList} hívás
+   * az unbounded-growth → GC-thrash/OOM vektor (2026-06-23 incidens): a teljes (akár több-100k-s)
+   * collection a heap-be töltődik. NEM capeljük a query-t (az törné a hívókat) — csak WARNINGot
+   * logolunk, hogy az over-fetch LÁTHATÓ legyen és paginálható/szűkíthető. Runtime felülírható
+   * (`DyNTS_DataService.findDataListWarnThreshold = N`). Default: 10000.
+   */
+  static findDataListWarnThreshold: number = 10000;
+
   dataDBService: DyNTS_DBService<T>;
   haveArchiveDataService: boolean;
 
@@ -676,11 +685,23 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
         this.dataList = dataListExists;
       }
 
+      // FR-258 / SR-3 — over-fetch guard. Egy szűretlen find, ami egy nagy collection EGÉSZÉT a heap-be
+      // tölti, a GC-thrash/OOM vektor (2026-06-23). NEM capeljük a query-t (az törné a hívókat), csak
+      // WARNINGot logolunk, hogy az over-fetch LÁTHATÓ legyen → paginate/szűkebb filter a megoldás.
+      const overFetchThreshold: number = DyNTS_DataService.findDataListWarnThreshold;
+      if (overFetchThreshold > 0 && dataListExists.length > overFetchThreshold) {
+        DyFM_Log.warn(
+          `[FR-258 SR-3] findDataList "${this.dataParams.dataName}" returned ${dataListExists.length} documents ` +
+          `(> ${overFetchThreshold}) — the whole result is held in heap. Consider a tighter filter or ` +
+          `pagination (skip/limit) to avoid unbounded memory growth.`,
+        );
+      }
+
       return dataListExists;
     } catch (error) {
-      throw new DyFM_Error({ 
+      throw new DyFM_Error({
         ...this._getDefaultErrorSettings('findDataList', error),
-        
+
         errorCode: `${this.ecBase}DyNTS-DS0-FDS0`,
       });
     }

package/src/_services/core/collection-growth-monitor.service.spec.ts

@@ -0,0 +1,67 @@
+import { DyNTS_CollectionGrowthAlert, DyNTS_CollectionGrowthMonitor } from './collection-growth-monitor.service';
+
+/**
+ * FR-258 / SR-4 — unit tests for the pure growth-evaluation core
+ * (`DyNTS_CollectionGrowthMonitor.evaluateGrowth`). No I/O, no timers — covers size-threshold,
+ * growth-rate, both/neither, and the first-poll (no prior snapshot) edge cases.
+ */
+describe('| DyNTS_CollectionGrowthMonitor.evaluateGrowth (FR-258 / SR-4)', () => {
+
+  const SIZE_THRESHOLD: number = 250000;
+  const GROWTH_PCT: number = 50;
+
+  function kinds(alerts: DyNTS_CollectionGrowthAlert[]): string[] {
+    return alerts.map((a: DyNTS_CollectionGrowthAlert): string => a.kind).sort();
+  }
+
+  it('| under threshold + no prior snapshot → no alerts', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('logs', 100, undefined, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(alerts).toEqual([]);
+  });
+
+  it('| over size threshold → a "size" alert with the right detail', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('cdp_report_details_archiveds', 300000, undefined, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(kinds(alerts)).toEqual(['size']);
+    expect(alerts[0].dataName).toBe('cdp_report_details_archiveds');
+    expect(alerts[0].detail).toEqual({ collection: 'cdp_report_details_archiveds', docCount: 300000, threshold: SIZE_THRESHOLD });
+  });
+
+  it('| exactly AT the threshold → no size alert (strictly greater)', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('logs', SIZE_THRESHOLD, undefined, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(alerts).toEqual([]);
+  });
+
+  it('| grew more than the growth-rate % since last poll → a "growth" alert', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('pipeline_jobs', 200, { docCount: 100, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(kinds(alerts)).toEqual(['growth']);
+    expect(alerts[0].detail).toEqual({ collection: 'pipeline_jobs', oldCount: 100, newCount: 200, growthPct: 100 });
+  });
+
+  it('| grew LESS than the growth-rate % → no growth alert', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('logs', 110, { docCount: 100, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(alerts).toEqual([]);
+  });
+
+  it('| both over-threshold AND fast-growth → BOTH alerts', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('big', 400000, { docCount: 100000, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(kinds(alerts)).toEqual(['growth', 'size']);
+  });
+
+  it('| prior snapshot with zero docCount → no growth alert (avoid divide-by-zero)', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('fresh', 5000, { docCount: 0, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(alerts).toEqual([]);
+  });
+
+  it('| shrinking collection (negative growth) → no growth alert', () => {
+    const alerts: DyNTS_CollectionGrowthAlert[] =
+      DyNTS_CollectionGrowthMonitor.evaluateGrowth('pruned', 50, { docCount: 100, atMs: 0 }, SIZE_THRESHOLD, GROWTH_PCT);
+    expect(alerts).toEqual([]);
+  });
+});

package/src/_services/core/collection-growth-monitor.service.ts

@@ -0,0 +1,211 @@
+import { DyFM_Error, DyFM_ErrorLevel, DyFM_Log } from '@futdevpro/fsm-dynamo';
+
+import { DyNTS_global_settings } from '../../_collections/global-settings.const';
+import { DyNTS_SingletonServiceBase } from '../base/singleton.service-base';
+import { DyNTS_GlobalService } from './global.service';
+
+/**
+ * FR-258 / SR-4 — Collection Growth Monitor (proactive companion to DyNTS_MemoryGuard).
+ *
+ * The MemoryGuard is REACTIVE: it fires when the heap is already near OOM. This monitor is
+ * PROACTIVE: it periodically samples each registered collection's document count (via the cheap,
+ * scan-free `estimatedDocumentCount()`) and raises a warning into the Errors-sink BEFORE a bloated
+ * collection can be loaded into heap and GC-thrash the process (the 2026-06-23 overseer incident:
+ * `cdp_report_details_archiveds` reached 3 GB / `pipeline_jobs` 2 GB, unbounded).
+ *
+ * It pairs with SR-1/SR-2 (declarative retention TTL): TTL keeps collections bounded; this monitor
+ * surfaces the ones that AREN'T (no retention declared, or growing faster than retention prunes) so
+ * an operator can add `retention` before it becomes an OOM. Pure observation — never deletes.
+ *
+ * Like the MemoryGuard it is a singleton, idempotent, fully non-fatal (a monitoring layer must never
+ * crash a server), and its timer is `unref()`-ed so it can't keep the process alive on its own.
+ */
+export interface DyNTS_CollectionGrowthMonitor_Config {
+  /** Master switch. Default: `DyNTS_global_settings.collectionGrowthMonitor.enabled` (true). */
+  enabled?: boolean;
+  /** Poll cadence in ms. Default: 300000 (5 min) — collection growth is slow, no need to poll often. */
+  pollIntervalMs?: number;
+  /** Warn when a collection's document count exceeds this. Default: 250000. */
+  sizeThresholdDocCount?: number;
+  /** Warn when a collection grew by more than this percent since the previous poll. Default: 50. */
+  growthRateThresholdPct?: number;
+  /** Re-alert cooldown per collection per alert-kind, in ms (anti-flood). Default: 3600000 (1 h). */
+  reAlertCooldownMs?: number;
+}
+
+/** One growth finding to surface to the Errors-sink. */
+export interface DyNTS_CollectionGrowthAlert {
+  kind: 'size' | 'growth';
+  dataName: string;
+  message: string;
+  detail: Record<string, unknown>;
+}
+
+interface CollectionSnapshot {
+  docCount: number;
+  atMs: number;
+}
+
+export class DyNTS_CollectionGrowthMonitor extends DyNTS_SingletonServiceBase {
+
+  static getInstance(): DyNTS_CollectionGrowthMonitor {
+    return DyNTS_CollectionGrowthMonitor.getSingletonInstance() as DyNTS_CollectionGrowthMonitor;
+  }
+
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private installed: boolean = false;
+
+  // Resolved (effective) config — filled at install().
+  private pollIntervalMs: number = 300000;
+  private sizeThresholdDocCount: number = 250000;
+  private growthRateThresholdPct: number = 50;
+  private reAlertCooldownMs: number = 3600000;
+
+  private readonly lastSnapshotByCollection: Map<string, CollectionSnapshot> = new Map();
+  /** Last time we alerted, keyed by `${dataName}:${kind}` — for the anti-flood cooldown. */
+  private readonly lastAlertAtByKey: Map<string, number> = new Map();
+
+  protected constructor() {
+    super();
+  }
+
+  /**
+   * Start the monitor. Idempotent (a second call is a no-op). `config` overrides
+   * `DyNTS_global_settings.collectionGrowthMonitor`, which overrides the built-in defaults. NEVER
+   * throws — every error is try/catch-ed and descriptively logged.
+   */
+  install(config?: DyNTS_CollectionGrowthMonitor_Config): void {
+    if (this.installed) { return; }
+
+    try {
+      const g: NonNullable<typeof DyNTS_global_settings.collectionGrowthMonitor> =
+        DyNTS_global_settings.collectionGrowthMonitor ?? { enabled: true };
+
+      if (!(config?.enabled ?? g.enabled ?? true)) { return; }
+
+      this.pollIntervalMs = config?.pollIntervalMs ?? g.pollIntervalMs ?? 300000;
+      this.sizeThresholdDocCount = config?.sizeThresholdDocCount ?? g.sizeThresholdDocCount ?? 250000;
+      this.growthRateThresholdPct = config?.growthRateThresholdPct ?? g.growthRateThresholdPct ?? 50;
+      this.reAlertCooldownMs = config?.reAlertCooldownMs ?? g.reAlertCooldownMs ?? 3600000;
+
+      this.installed = true;
+      this.timer = setInterval((): void => { void this.poll(); }, this.pollIntervalMs);
+      if (typeof this.timer.unref === 'function') { this.timer.unref(); }
+
+      DyFM_Log.info(
+        `[DyNTS_CollectionGrowthMonitor] installed — poll ${Math.round(this.pollIntervalMs / 1000)}s, ` +
+        `size-threshold ${this.sizeThresholdDocCount} docs, growth-rate ${this.growthRateThresholdPct}%/poll`,
+      );
+    } catch (err: unknown) {
+      DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] install failed (non-fatal):', err);
+    }
+  }
+
+  /**
+   * Pure decision core (unit-tested): given a fresh count + the previous snapshot, return the alerts
+   * to raise. NEVER does I/O. A `size` alert fires when `count > sizeThreshold`; a `growth` alert
+   * fires when the collection grew by more than `growthRatePct` since the last snapshot (needs a
+   * non-zero previous count).
+   */
+  static evaluateGrowth(
+    dataName: string,
+    count: number,
+    last: CollectionSnapshot | undefined,
+    sizeThreshold: number,
+    growthRatePct: number,
+  ): DyNTS_CollectionGrowthAlert[] {
+    const alerts: DyNTS_CollectionGrowthAlert[] = [];
+
+    if (count > sizeThreshold) {
+      alerts.push({
+        kind: 'size',
+        dataName: dataName,
+        message:
+          `Collection "${dataName}" has ${count} documents (> ${sizeThreshold} threshold) — ` +
+          `unbounded-growth risk. Consider declaring \`retention\` on this model (FR-258 / SR-1).`,
+        detail: { collection: dataName, docCount: count, threshold: sizeThreshold },
+      });
+    }
+
+    if (last && last.docCount > 0) {
+      const growthPct: number = ((count - last.docCount) / last.docCount) * 100;
+      if (growthPct > growthRatePct) {
+        alerts.push({
+          kind: 'growth',
+          dataName: dataName,
+          message:
+            `Collection "${dataName}" grew ${Math.round(growthPct)}% since the last poll ` +
+            `(${last.docCount} → ${count} docs) — fast unbounded growth.`,
+          detail: { collection: dataName, oldCount: last.docCount, newCount: count, growthPct: Math.round(growthPct) },
+        });
+      }
+    }
+
+    return alerts;
+  }
+
+  /**
+   * One poll cycle: sample every registered collection's `estimatedDocumentCount()` (scan-free) and
+   * raise cooldown-gated warnings. Fully non-fatal — a failure on one collection never aborts the rest.
+   */
+  private async poll(): Promise<void> {
+    try {
+      const services: ReturnType<typeof DyNTS_GlobalService.getAllDBServices> = DyNTS_GlobalService.getAllDBServices();
+      const nowMs: number = Date.now();
+
+      for (const service of services) {
+        const dataName: string | undefined = service?.dataParams?.dataName;
+        const model: { estimatedDocumentCount?: () => Promise<number> } | undefined =
+          service?.dataModel as unknown as { estimatedDocumentCount?: () => Promise<number> };
+        if (!dataName || !model || typeof model.estimatedDocumentCount !== 'function') { continue; }
+
+        try {
+          const count: number = await model.estimatedDocumentCount();
+          const last: CollectionSnapshot | undefined = this.lastSnapshotByCollection.get(dataName);
+
+          const alerts: DyNTS_CollectionGrowthAlert[] = DyNTS_CollectionGrowthMonitor.evaluateGrowth(
+            dataName, count, last, this.sizeThresholdDocCount, this.growthRateThresholdPct,
+          );
+          for (const alert of alerts) {
+            this.emitAlert(alert, nowMs);
+          }
+
+          this.lastSnapshotByCollection.set(dataName, { docCount: count, atMs: nowMs });
+        } catch (collErr: unknown) {
+          DyFM_Log.warn(
+            `[DyNTS_CollectionGrowthMonitor] poll for "${dataName}" failed (non-fatal): ` +
+            `${collErr instanceof Error ? collErr.message : String(collErr)}`,
+          );
+        }
+      }
+    } catch (err: unknown) {
+      DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] poll failed (non-fatal):', err);
+    }
+  }
+
+  /** Raise one alert to the Errors-sink, gated by the per-(collection,kind) cooldown to avoid flooding. */
+  private emitAlert(alert: DyNTS_CollectionGrowthAlert, nowMs: number): void {
+    const key: string = `${alert.dataName}:${alert.kind}`;
+    const lastAt: number | undefined = this.lastAlertAtByKey.get(key);
+    if (lastAt !== undefined && nowMs - lastAt < this.reAlertCooldownMs) { return; }
+    this.lastAlertAtByKey.set(key, nowMs);
+
+    DyFM_Log.warn(`[DyNTS_CollectionGrowthMonitor] ${alert.message}`);
+    DyNTS_GlobalService.globalErrorHandler?.(
+      new DyFM_Error({
+        errorCode: `${DyNTS_global_settings.systemShortCodeName ?? 'DyNTS'}|DyNTS-CGM-${alert.kind === 'size' ? 'SIZE' : 'GROWTH'}`,
+        message: alert.message,
+        level: DyFM_ErrorLevel.warning,
+        additionalContent: alert.detail,
+      }),
+    );
+  }
+
+  /** Test-only teardown: stop the timer + clear state so specs don't leak between runs. */
+  _teardownForTesting(): void {
+    if (this.timer) { clearInterval(this.timer); this.timer = null; }
+    this.installed = false;
+    this.lastSnapshotByCollection.clear();
+    this.lastAlertAtByKey.clear();
+  }
+}

package/src/_services/server/app.server.ts

@@ -55,6 +55,7 @@ import {
 import { DyNTS_DBService } from '../base/db.service';
 import { DyNTS_SingletonService } from '../base/singleton.service';
 import { DyNTS_GlobalService } from '../core/global.service';
+import { DyNTS_CollectionGrowthMonitor } from '../core/collection-growth-monitor.service';
 import { DyNTS_MemoryGuard } from '../core/memory-guard.service';
 import { DyNTS_RoutingModule } from '../route/routing-module.service';
 import { DyNTS_getStarRoute } from '../../_collections/star.controller';
@@ -579,6 +580,17 @@ export abstract class DyNTS_App extends DyNTS_SingletonService {
         DyFM_Log.warn('[DyNTS_MemoryGuard] auto-install skipped (non-fatal):', memoryGuardError);
       }
 
+      // FR-258 / SR-4 — proactive collection-growth monitor (companion to the MemoryGuard). Warns
+      // into the Errors-sink when a collection grows unbounded, BEFORE it can be loaded into heap and
+      // OOM. Default-on, observation-only, never throws.
+      try {
+        if (DyNTS_global_settings.collectionGrowthMonitor?.enabled) {
+          DyNTS_CollectionGrowthMonitor.getInstance().install();
+        }
+      } catch (cgmError: unknown) {
+        DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] auto-install skipped (non-fatal):', cgmError);
+      }
+
       if (!extended) {
         await this.ready();
 

package/src/index.ts

@@ -71,6 +71,7 @@ export * from './_services/core/api.service';
 export * from './_services/core/auth.service';
 
 export * from './_services/core/email.service';
+export * from './_services/core/collection-growth-monitor.service';
 export * from './_services/core/global.service';
 export * from './_services/core/memory-guard.service';