@futdevpro/nts-dynamo 1.15.151 → 1.15.154

package/src/_services/core/collection-growth-monitor.service.ts

@@ -1,234 +1,234 @@
-import {
-  DyFM_Error, DyFM_ErrorLevel, DyFM_Log, DyFM_RetentionSettings, DyFM_resolveRetentionTtlSeconds,
-} from '@futdevpro/fsm-dynamo';
-
-import { DyNTS_global_settings } from '../../_collections/global-settings.const';
-import { DyNTS_SingletonServiceBase } from '../base/singleton.service-base';
-import { DyNTS_GlobalService } from './global.service';
-
-/**
- * FR-258 / SR-4 — Collection Growth Monitor (proactive companion to DyNTS_MemoryGuard).
- *
- * The MemoryGuard is REACTIVE: it fires when the heap is already near OOM. This monitor is
- * PROACTIVE: it periodically samples each registered collection's document count (via the cheap,
- * scan-free `estimatedDocumentCount()`) and raises a warning into the Errors-sink BEFORE a bloated
- * collection can be loaded into heap and GC-thrash the process (the 2026-06-23 overseer incident:
- * `cdp_report_details_archiveds` reached 3 GB / `pipeline_jobs` 2 GB, unbounded).
- *
- * It pairs with SR-1/SR-2 (declarative retention TTL): TTL keeps collections bounded; this monitor
- * surfaces the ones that AREN'T (no retention declared, or growing faster than retention prunes) so
- * an operator can add `retention` before it becomes an OOM. Pure observation — never deletes.
- *
- * Like the MemoryGuard it is a singleton, idempotent, fully non-fatal (a monitoring layer must never
- * crash a server), and its timer is `unref()`-ed so it can't keep the process alive on its own.
- */
-export interface DyNTS_CollectionGrowthMonitor_Config {
-  /** Master switch. Default: `DyNTS_global_settings.collectionGrowthMonitor.enabled` (true). */
-  enabled?: boolean;
-  /** Poll cadence in ms. Default: 300000 (5 min) — collection growth is slow, no need to poll often. */
-  pollIntervalMs?: number;
-  /** Warn when a collection's document count exceeds this. Default: 250000. */
-  sizeThresholdDocCount?: number;
-  /** Warn when a collection grew by more than this percent since the previous poll. Default: 50. */
-  growthRateThresholdPct?: number;
-  /** Re-alert cooldown per collection per alert-kind, in ms (anti-flood). Default: 3600000 (1 h). */
-  reAlertCooldownMs?: number;
-}
-
-/** One growth finding to surface to the Errors-sink. */
-export interface DyNTS_CollectionGrowthAlert {
-  kind: 'size' | 'growth';
-  dataName: string;
-  message: string;
-  detail: Record<string, unknown>;
-}
-
-interface CollectionSnapshot {
-  docCount: number;
-  atMs: number;
-}
-
-export class DyNTS_CollectionGrowthMonitor extends DyNTS_SingletonServiceBase {
-
-  static getInstance(): DyNTS_CollectionGrowthMonitor {
-    return DyNTS_CollectionGrowthMonitor.getSingletonInstance() as DyNTS_CollectionGrowthMonitor;
-  }
-
-  private timer: ReturnType<typeof setInterval> | null = null;
-  private installed: boolean = false;
-
-  // Resolved (effective) config — filled at install().
-  private pollIntervalMs: number = 300000;
-  private sizeThresholdDocCount: number = 250000;
-  private growthRateThresholdPct: number = 50;
-  private reAlertCooldownMs: number = 3600000;
-
-  private readonly lastSnapshotByCollection: Map<string, CollectionSnapshot> = new Map();
-  /** Last time we alerted, keyed by `${dataName}:${kind}` — for the anti-flood cooldown. */
-  private readonly lastAlertAtByKey: Map<string, number> = new Map();
-
-  protected constructor() {
-    super();
-  }
-
-  /**
-   * Start the monitor. Idempotent (a second call is a no-op). `config` overrides
-   * `DyNTS_global_settings.collectionGrowthMonitor`, which overrides the built-in defaults. NEVER
-   * throws — every error is try/catch-ed and descriptively logged.
-   */
-  install(config?: DyNTS_CollectionGrowthMonitor_Config): void {
-    if (this.installed) { return; }
-
-    try {
-      const g: NonNullable<typeof DyNTS_global_settings.collectionGrowthMonitor> =
-        DyNTS_global_settings.collectionGrowthMonitor ?? { enabled: true };
-
-      if (!(config?.enabled ?? g.enabled ?? true)) { return; }
-
-      this.pollIntervalMs = config?.pollIntervalMs ?? g.pollIntervalMs ?? 300000;
-      this.sizeThresholdDocCount = config?.sizeThresholdDocCount ?? g.sizeThresholdDocCount ?? 250000;
-      this.growthRateThresholdPct = config?.growthRateThresholdPct ?? g.growthRateThresholdPct ?? 50;
-      this.reAlertCooldownMs = config?.reAlertCooldownMs ?? g.reAlertCooldownMs ?? 3600000;
-
-      this.installed = true;
-      this.timer = setInterval((): void => { void this.poll(); }, this.pollIntervalMs);
-      if (typeof this.timer.unref === 'function') { this.timer.unref(); }
-
-      DyFM_Log.info(
-        `[DyNTS_CollectionGrowthMonitor] installed — poll ${Math.round(this.pollIntervalMs / 1000)}s, ` +
-        `size-threshold ${this.sizeThresholdDocCount} docs, growth-rate ${this.growthRateThresholdPct}%/poll`,
-      );
-    } catch (err: unknown) {
-      DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] install failed (non-fatal):', err);
-    }
-  }
-
-  /**
-   * Pure decision core (unit-tested): given a fresh count + the previous snapshot, return the alerts
-   * to raise. NEVER does I/O. A `size` alert fires when `count > sizeThreshold`; a `growth` alert
-   * fires when the collection grew by more than `growthRatePct` since the last snapshot (needs a
-   * non-zero previous count).
-   */
-  static evaluateGrowth(
-    dataName: string,
-    count: number,
-    last: CollectionSnapshot | undefined,
-    sizeThreshold: number,
-    growthRatePct: number,
-    retentionTtlSec?: number,
-  ): DyNTS_CollectionGrowthAlert[] {
-    const alerts: DyNTS_CollectionGrowthAlert[] = [];
-
-    if (count > sizeThreshold) {
-      // FR-311 — RETENTION-TUDATOS tanacs. A korabbi uzenet FELTETEL NELKUL azt ajanlotta, hogy
-      // "declare `retention`" — akkor is, ha a modell MAR deklaralt egyet. Elesben (2026-08-19,
-      // overseer `ov_secret_audit`, 253 617 dokumentum) ez oda vezetett, hogy az operator olyat
-      // kapott feladatul, ami mar KESZ volt, orankent ujra, az error-tarba is elmentve. A ket eset
-      // KULONBOZO teendot kiván, ezert kulon is kell mondani oket.
-      const retentionDays: number | null = retentionTtlSec === undefined
-        ? null
-        : Math.round(retentionTtlSec / 86400);
-      alerts.push({
-        kind: 'size',
-        dataName: dataName,
-        message: retentionDays === null
-          ? `Collection "${dataName}" has ${count} documents (> ${sizeThreshold} threshold) — ` +
-            `unbounded-growth risk. Consider declaring \`retention\` on this model (FR-258 / SR-1).`
-          : `Collection "${dataName}" has ${count} documents (> ${sizeThreshold} threshold) DESPITE a ` +
-            `declared retention of ~${retentionDays} day(s) — the window is too long, or growth outpaces ` +
-            `the TTL. Shorten \`retention\` or reduce the write-rate (FR-258 / SR-1).`,
-        detail: {
-          collection: dataName, docCount: count, threshold: sizeThreshold,
-          retentionDays: retentionDays,
-        },
-      });
-    }
-
-    if (last && last.docCount > 0) {
-      const growthPct: number = ((count - last.docCount) / last.docCount) * 100;
-      if (growthPct > growthRatePct) {
-        alerts.push({
-          kind: 'growth',
-          dataName: dataName,
-          message:
-            `Collection "${dataName}" grew ${Math.round(growthPct)}% since the last poll ` +
-            `(${last.docCount} → ${count} docs) — fast unbounded growth.`,
-          detail: { collection: dataName, oldCount: last.docCount, newCount: count, growthPct: Math.round(growthPct) },
-        });
-      }
-    }
-
-    return alerts;
-  }
-
-  /**
-   * One poll cycle: sample every registered collection's `estimatedDocumentCount()` (scan-free) and
-   * raise cooldown-gated warnings. Fully non-fatal — a failure on one collection never aborts the rest.
-   */
-  private async poll(): Promise<void> {
-    try {
-      const services: ReturnType<typeof DyNTS_GlobalService.getAllDBServices> = DyNTS_GlobalService.getAllDBServices();
-      const nowMs: number = Date.now();
-
-      for (const service of services) {
-        const dataName: string | undefined = service?.dataParams?.dataName;
-        const model: { estimatedDocumentCount?: () => Promise<number> } | undefined =
-          service?.dataModel as unknown as { estimatedDocumentCount?: () => Promise<number> };
-        if (!dataName || !model || typeof model.estimatedDocumentCount !== 'function') { continue; }
-
-        try {
-          const count: number = await model.estimatedDocumentCount();
-          const last: CollectionSnapshot | undefined = this.lastSnapshotByCollection.get(dataName);
-
-          // FR-311 — a modell SAJAT retention-deklaraciojat visszük be a dontesbe. A feloldas az
-          // fsm-dynamo SSOT-resolvereval megy (`ttlSeconds` > `retentionDays`, ervenytelen ertek
-          // elutasitva) — NEM ujraimplementalva, kulonben ketfele igazsag lenne rola.
-          const retentionTtlSec: number | undefined = DyFM_resolveRetentionTtlSeconds(
-            service?.dataParams?.retention as DyFM_RetentionSettings | undefined,
-          );
-          const alerts: DyNTS_CollectionGrowthAlert[] = DyNTS_CollectionGrowthMonitor.evaluateGrowth(
-            dataName, count, last, this.sizeThresholdDocCount, this.growthRateThresholdPct, retentionTtlSec,
-          );
-          for (const alert of alerts) {
-            this.emitAlert(alert, nowMs);
-          }
-
-          this.lastSnapshotByCollection.set(dataName, { docCount: count, atMs: nowMs });
-        } catch (collErr: unknown) {
-          DyFM_Log.warn(
-            `[DyNTS_CollectionGrowthMonitor] poll for "${dataName}" failed (non-fatal): ` +
-            `${collErr instanceof Error ? collErr.message : String(collErr)}`,
-          );
-        }
-      }
-    } catch (err: unknown) {
-      DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] poll failed (non-fatal):', err);
-    }
-  }
-
-  /** Raise one alert to the Errors-sink, gated by the per-(collection,kind) cooldown to avoid flooding. */
-  private emitAlert(alert: DyNTS_CollectionGrowthAlert, nowMs: number): void {
-    const key: string = `${alert.dataName}:${alert.kind}`;
-    const lastAt: number | undefined = this.lastAlertAtByKey.get(key);
-    if (lastAt !== undefined && nowMs - lastAt < this.reAlertCooldownMs) { return; }
-    this.lastAlertAtByKey.set(key, nowMs);
-
-    DyFM_Log.warn(`[DyNTS_CollectionGrowthMonitor] ${alert.message}`);
-    DyNTS_GlobalService.globalErrorHandler?.(
-      new DyFM_Error({
-        errorCode: `${DyNTS_global_settings.systemShortCodeName ?? 'DyNTS'}|DyNTS-CGM-${alert.kind === 'size' ? 'SIZE' : 'GROWTH'}`,
-        message: alert.message,
-        level: DyFM_ErrorLevel.warning,
-        additionalContent: alert.detail,
-      }),
-    );
-  }
-
-  /** Test-only teardown: stop the timer + clear state so specs don't leak between runs. */
-  _teardownForTesting(): void {
-    if (this.timer) { clearInterval(this.timer); this.timer = null; }
-    this.installed = false;
-    this.lastSnapshotByCollection.clear();
-    this.lastAlertAtByKey.clear();
-  }
-}
+import {
+  DyFM_Error, DyFM_ErrorLevel, DyFM_Log, DyFM_RetentionSettings, DyFM_resolveRetentionTtlSeconds,
+} from '@futdevpro/fsm-dynamo';
+
+import { DyNTS_global_settings } from '../../_collections/global-settings.const';
+import { DyNTS_SingletonServiceBase } from '../base/singleton.service-base';
+import { DyNTS_GlobalService } from './global.service';
+
+/**
+ * FR-258 / SR-4 — Collection Growth Monitor (proactive companion to DyNTS_MemoryGuard).
+ *
+ * The MemoryGuard is REACTIVE: it fires when the heap is already near OOM. This monitor is
+ * PROACTIVE: it periodically samples each registered collection's document count (via the cheap,
+ * scan-free `estimatedDocumentCount()`) and raises a warning into the Errors-sink BEFORE a bloated
+ * collection can be loaded into heap and GC-thrash the process (the 2026-06-23 overseer incident:
+ * `cdp_report_details_archiveds` reached 3 GB / `pipeline_jobs` 2 GB, unbounded).
+ *
+ * It pairs with SR-1/SR-2 (declarative retention TTL): TTL keeps collections bounded; this monitor
+ * surfaces the ones that AREN'T (no retention declared, or growing faster than retention prunes) so
+ * an operator can add `retention` before it becomes an OOM. Pure observation — never deletes.
+ *
+ * Like the MemoryGuard it is a singleton, idempotent, fully non-fatal (a monitoring layer must never
+ * crash a server), and its timer is `unref()`-ed so it can't keep the process alive on its own.
+ */
+export interface DyNTS_CollectionGrowthMonitor_Config {
+  /** Master switch. Default: `DyNTS_global_settings.collectionGrowthMonitor.enabled` (true). */
+  enabled?: boolean;
+  /** Poll cadence in ms. Default: 300000 (5 min) — collection growth is slow, no need to poll often. */
+  pollIntervalMs?: number;
+  /** Warn when a collection's document count exceeds this. Default: 250000. */
+  sizeThresholdDocCount?: number;
+  /** Warn when a collection grew by more than this percent since the previous poll. Default: 50. */
+  growthRateThresholdPct?: number;
+  /** Re-alert cooldown per collection per alert-kind, in ms (anti-flood). Default: 3600000 (1 h). */
+  reAlertCooldownMs?: number;
+}
+
+/** One growth finding to surface to the Errors-sink. */
+export interface DyNTS_CollectionGrowthAlert {
+  kind: 'size' | 'growth';
+  dataName: string;
+  message: string;
+  detail: Record<string, unknown>;
+}
+
+interface CollectionSnapshot {
+  docCount: number;
+  atMs: number;
+}
+
+export class DyNTS_CollectionGrowthMonitor extends DyNTS_SingletonServiceBase {
+
+  static getInstance(): DyNTS_CollectionGrowthMonitor {
+    return DyNTS_CollectionGrowthMonitor.getSingletonInstance() as DyNTS_CollectionGrowthMonitor;
+  }
+
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private installed: boolean = false;
+
+  // Resolved (effective) config — filled at install().
+  private pollIntervalMs: number = 300000;
+  private sizeThresholdDocCount: number = 250000;
+  private growthRateThresholdPct: number = 50;
+  private reAlertCooldownMs: number = 3600000;
+
+  private readonly lastSnapshotByCollection: Map<string, CollectionSnapshot> = new Map();
+  /** Last time we alerted, keyed by `${dataName}:${kind}` — for the anti-flood cooldown. */
+  private readonly lastAlertAtByKey: Map<string, number> = new Map();
+
+  protected constructor() {
+    super();
+  }
+
+  /**
+   * Start the monitor. Idempotent (a second call is a no-op). `config` overrides
+   * `DyNTS_global_settings.collectionGrowthMonitor`, which overrides the built-in defaults. NEVER
+   * throws — every error is try/catch-ed and descriptively logged.
+   */
+  install(config?: DyNTS_CollectionGrowthMonitor_Config): void {
+    if (this.installed) { return; }
+
+    try {
+      const g: NonNullable<typeof DyNTS_global_settings.collectionGrowthMonitor> =
+        DyNTS_global_settings.collectionGrowthMonitor ?? { enabled: true };
+
+      if (!(config?.enabled ?? g.enabled ?? true)) { return; }
+
+      this.pollIntervalMs = config?.pollIntervalMs ?? g.pollIntervalMs ?? 300000;
+      this.sizeThresholdDocCount = config?.sizeThresholdDocCount ?? g.sizeThresholdDocCount ?? 250000;
+      this.growthRateThresholdPct = config?.growthRateThresholdPct ?? g.growthRateThresholdPct ?? 50;
+      this.reAlertCooldownMs = config?.reAlertCooldownMs ?? g.reAlertCooldownMs ?? 3600000;
+
+      this.installed = true;
+      this.timer = setInterval((): void => { void this.poll(); }, this.pollIntervalMs);
+      if (typeof this.timer.unref === 'function') { this.timer.unref(); }
+
+      DyFM_Log.info(
+        `[DyNTS_CollectionGrowthMonitor] installed — poll ${Math.round(this.pollIntervalMs / 1000)}s, ` +
+        `size-threshold ${this.sizeThresholdDocCount} docs, growth-rate ${this.growthRateThresholdPct}%/poll`,
+      );
+    } catch (err: unknown) {
+      DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] install failed (non-fatal):', err);
+    }
+  }
+
+  /**
+   * Pure decision core (unit-tested): given a fresh count + the previous snapshot, return the alerts
+   * to raise. NEVER does I/O. A `size` alert fires when `count > sizeThreshold`; a `growth` alert
+   * fires when the collection grew by more than `growthRatePct` since the last snapshot (needs a
+   * non-zero previous count).
+   */
+  static evaluateGrowth(
+    dataName: string,
+    count: number,
+    last: CollectionSnapshot | undefined,
+    sizeThreshold: number,
+    growthRatePct: number,
+    retentionTtlSec?: number,
+  ): DyNTS_CollectionGrowthAlert[] {
+    const alerts: DyNTS_CollectionGrowthAlert[] = [];
+
+    if (count > sizeThreshold) {
+      // FR-311 — RETENTION-TUDATOS tanacs. A korabbi uzenet FELTETEL NELKUL azt ajanlotta, hogy
+      // "declare `retention`" — akkor is, ha a modell MAR deklaralt egyet. Elesben (2026-08-19,
+      // overseer `ov_secret_audit`, 253 617 dokumentum) ez oda vezetett, hogy az operator olyat
+      // kapott feladatul, ami mar KESZ volt, orankent ujra, az error-tarba is elmentve. A ket eset
+      // KULONBOZO teendot kiván, ezert kulon is kell mondani oket.
+      const retentionDays: number | null = retentionTtlSec === undefined
+        ? null
+        : Math.round(retentionTtlSec / 86400);
+      alerts.push({
+        kind: 'size',
+        dataName: dataName,
+        message: retentionDays === null
+          ? `Collection "${dataName}" has ${count} documents (> ${sizeThreshold} threshold) — ` +
+            `unbounded-growth risk. Consider declaring \`retention\` on this model (FR-258 / SR-1).`
+          : `Collection "${dataName}" has ${count} documents (> ${sizeThreshold} threshold) DESPITE a ` +
+            `declared retention of ~${retentionDays} day(s) — the window is too long, or growth outpaces ` +
+            `the TTL. Shorten \`retention\` or reduce the write-rate (FR-258 / SR-1).`,
+        detail: {
+          collection: dataName, docCount: count, threshold: sizeThreshold,
+          retentionDays: retentionDays,
+        },
+      });
+    }
+
+    if (last && last.docCount > 0) {
+      const growthPct: number = ((count - last.docCount) / last.docCount) * 100;
+      if (growthPct > growthRatePct) {
+        alerts.push({
+          kind: 'growth',
+          dataName: dataName,
+          message:
+            `Collection "${dataName}" grew ${Math.round(growthPct)}% since the last poll ` +
+            `(${last.docCount} → ${count} docs) — fast unbounded growth.`,
+          detail: { collection: dataName, oldCount: last.docCount, newCount: count, growthPct: Math.round(growthPct) },
+        });
+      }
+    }
+
+    return alerts;
+  }
+
+  /**
+   * One poll cycle: sample every registered collection's `estimatedDocumentCount()` (scan-free) and
+   * raise cooldown-gated warnings. Fully non-fatal — a failure on one collection never aborts the rest.
+   */
+  private async poll(): Promise<void> {
+    try {
+      const services: ReturnType<typeof DyNTS_GlobalService.getAllDBServices> = DyNTS_GlobalService.getAllDBServices();
+      const nowMs: number = Date.now();
+
+      for (const service of services) {
+        const dataName: string | undefined = service?.dataParams?.dataName;
+        const model: { estimatedDocumentCount?: () => Promise<number> } | undefined =
+          service?.dataModel as unknown as { estimatedDocumentCount?: () => Promise<number> };
+        if (!dataName || !model || typeof model.estimatedDocumentCount !== 'function') { continue; }
+
+        try {
+          const count: number = await model.estimatedDocumentCount();
+          const last: CollectionSnapshot | undefined = this.lastSnapshotByCollection.get(dataName);
+
+          // FR-311 — a modell SAJAT retention-deklaraciojat visszük be a dontesbe. A feloldas az
+          // fsm-dynamo SSOT-resolvereval megy (`ttlSeconds` > `retentionDays`, ervenytelen ertek
+          // elutasitva) — NEM ujraimplementalva, kulonben ketfele igazsag lenne rola.
+          const retentionTtlSec: number | undefined = DyFM_resolveRetentionTtlSeconds(
+            service?.dataParams?.retention as DyFM_RetentionSettings | undefined,
+          );
+          const alerts: DyNTS_CollectionGrowthAlert[] = DyNTS_CollectionGrowthMonitor.evaluateGrowth(
+            dataName, count, last, this.sizeThresholdDocCount, this.growthRateThresholdPct, retentionTtlSec,
+          );
+          for (const alert of alerts) {
+            this.emitAlert(alert, nowMs);
+          }
+
+          this.lastSnapshotByCollection.set(dataName, { docCount: count, atMs: nowMs });
+        } catch (collErr: unknown) {
+          DyFM_Log.warn(
+            `[DyNTS_CollectionGrowthMonitor] poll for "${dataName}" failed (non-fatal): ` +
+            `${collErr instanceof Error ? collErr.message : String(collErr)}`,
+          );
+        }
+      }
+    } catch (err: unknown) {
+      DyFM_Log.warn('[DyNTS_CollectionGrowthMonitor] poll failed (non-fatal):', err);
+    }
+  }
+
+  /** Raise one alert to the Errors-sink, gated by the per-(collection,kind) cooldown to avoid flooding. */
+  private emitAlert(alert: DyNTS_CollectionGrowthAlert, nowMs: number): void {
+    const key: string = `${alert.dataName}:${alert.kind}`;
+    const lastAt: number | undefined = this.lastAlertAtByKey.get(key);
+    if (lastAt !== undefined && nowMs - lastAt < this.reAlertCooldownMs) { return; }
+    this.lastAlertAtByKey.set(key, nowMs);
+
+    DyFM_Log.warn(`[DyNTS_CollectionGrowthMonitor] ${alert.message}`);
+    DyNTS_GlobalService.globalErrorHandler?.(
+      new DyFM_Error({
+        errorCode: `${DyNTS_global_settings.systemShortCodeName ?? 'DyNTS'}|DyNTS-CGM-${alert.kind === 'size' ? 'SIZE' : 'GROWTH'}`,
+        message: alert.message,
+        level: DyFM_ErrorLevel.warning,
+        additionalContent: alert.detail,
+      }),
+    );
+  }
+
+  /** Test-only teardown: stop the timer + clear state so specs don't leak between runs. */
+  _teardownForTesting(): void {
+    if (this.timer) { clearInterval(this.timer); this.timer = null; }
+    this.installed = false;
+    this.lastSnapshotByCollection.clear();
+    this.lastAlertAtByKey.clear();
+  }
+}