@wildo-ai/saas-backend-lib 1.0.1

package/src/databases/services/aggregation/time-series-densifier.backend.service.ts

@@ -0,0 +1,389 @@
+/** ──────────────────────────────────────────────────────────────
+ * TimeSeriesDensifierBackendService
+ *
+ * Post-query in-memory densification for time-series data.
+ * DB compilers return sparse results; this service fills missing
+ * time buckets before computed measures and window functions.
+ *
+ * Features:
+ *   - Range resolution: explicit dateRange → context filters → global filters → data min/max
+ *   - Timezone-aware bucket generation
+ *   - Max 2,000 bucket guard
+ *   - `_filled: true` metadata on generated rows
+ *   - Fill values: 0 for numeric measures, null for non-numeric
+ *
+ * Ref: Densification spec (Q3) → L1560-1577
+ * Ref: fillMissingBuckets field → L631
+ * Ref: Timezone in densifier → L489
+ * ────────────────────────────────────────────────────────────── */
+
+import { injectable, inject } from 'inversify';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import type { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import {
+  AggregateFunction,
+  ErrorType,
+  ErrorCustomMessageReference,
+  TimeGranularity,
+} from '@wildo-ai/saas-models';
+import type {
+  AggregationQuery,
+  AggregationQueryResponseRow,
+} from '@wildo-ai/saas-models';
+
+/** Aggregate functions that always produce numeric results and can safely default to 0 */
+const NUMERIC_AGGREGATES = new Set<AggregateFunction>([
+  AggregateFunction.COUNT,
+  AggregateFunction.COUNT_DISTINCT,
+  AggregateFunction.SUM,
+  AggregateFunction.AVG,
+  AggregateFunction.MIN,
+  AggregateFunction.MAX,
+  AggregateFunction.MEDIAN,
+  AggregateFunction.PERCENTILE,
+  AggregateFunction.STDDEV,
+  AggregateFunction.VARIANCE,
+]);
+import type { ResourceFilter } from '~wildo-backend-lib/databases/models/filter.backend.schemas';
+
+/** Maximum number of generated buckets before rejecting */
+const MAX_BUCKETS = 2_000;
+
+/** Input for the densifier */
+export interface DensifierInput {
+  query: AggregationQuery;
+  data: AggregationQueryResponseRow[];
+  resolvedFilter?: ResourceFilter;
+  timezone: string;
+}
+
+@injectable()
+export class TimeSeriesDensifierBackendService {
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.Logger) private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService) private readonly errorBuilder: ErrorBuilderBackendService,
+  ) {}
+
+  /**
+   * Densify time-series data by filling missing time buckets.
+   * Only operates on dimensions with `fillMissingBuckets: true` and a `timeGranularity`.
+   * Returns the original data if no densification is needed.
+   */
+  densify(input: DensifierInput): AggregationQueryResponseRow[] {
+    const { query, data, resolvedFilter, timezone } = input;
+
+    // Find dimensions that need densification
+    const timeDims = query.dimensions.filter(d => d.fillMissingBuckets && d.timeGranularity);
+
+    if (timeDims.length === 0) {
+      return data;
+    }
+
+    // For MVP: densify on the first qualifying time dimension
+    const timeDim = timeDims[0];
+    const granularity = timeDim.timeGranularity!;
+    const dimRef = timeDim.ref;
+
+    // Resolve date range
+    const range = this.resolveRange(timeDim, dimRef, data, resolvedFilter);
+    if (!range.from || !range.to) {
+      this.logger.debug(`TimeSeriesDensifier: could not resolve range for dimension '${dimRef}', skipping densification`);
+      return data;
+    }
+
+    // Generate all buckets
+    const buckets = this.generateBuckets(range.from, range.to, granularity, timezone);
+
+    // Guard: max buckets
+    if (buckets.length > MAX_BUCKETS) {
+      throw this.errorBuilder.buildError(
+        ErrorType.VALIDATION,
+        undefined,
+        {
+          customMessageReference: ErrorCustomMessageReference.AGGREGATION_MAX_BUCKETS_EXCEEDED,
+          context: {
+            message: `Too many time buckets (${buckets.length}). Max allowed: ${MAX_BUCKETS}. Use a coarser granularity or narrow the date range.`,
+            bucketCount: buckets.length,
+            maxBuckets: MAX_BUCKETS,
+            granularity,
+          },
+        },
+      );
+    }
+
+    // Build a lookup from existing data: bucket key → row
+    const existingRows = this.buildRowLookup(data, dimRef, granularity, timezone);
+
+    // Determine measure refs and their fill values (0 for numeric, null for non-numeric)
+    const measureFillValues = new Map<string, 0 | null>();
+    for (const m of query.measures) {
+      measureFillValues.set(m.ref, NUMERIC_AGGREGATES.has(m.aggregate) ? 0 : null);
+    }
+
+    const otherDimRefs = query.dimensions.filter(d => d.ref !== dimRef).map(d => d.ref);
+
+    // If there are other (non-time) dimensions, densify per unique combination
+    if (otherDimRefs.length > 0) {
+      return this.densifyWithOtherDimensions(buckets, existingRows, dimRef, measureFillValues, otherDimRefs, data);
+    }
+
+    // Simple case: only time dimension
+    return this.densifySimple(buckets, existingRows, dimRef, measureFillValues);
+  }
+
+  // ── Range Resolution ───────────────────────────────────────
+
+  private resolveRange(
+    timeDim: AggregationQuery['dimensions'][0],
+    dimRef: string,
+    data: AggregationQueryResponseRow[],
+    resolvedFilter?: ResourceFilter,
+  ): { from: Date | null; to: Date | null } {
+    // Priority 1: Explicit dateRange on dimension
+    if (timeDim.dateRange?.from || timeDim.dateRange?.to) {
+      return {
+        from: timeDim.dateRange.from ? new Date(timeDim.dateRange.from) : null,
+        to: timeDim.dateRange.to ? new Date(timeDim.dateRange.to) : null,
+      };
+    }
+
+    // Priority 2 & 3: Extract bounds from resolved filter (covers both context filters and global filters)
+    if (resolvedFilter) {
+      const fieldFilter = resolvedFilter[timeDim.field] as Record<string, unknown> | undefined;
+      if (fieldFilter && typeof fieldFilter === 'object') {
+        const from = fieldFilter['$gte'] || fieldFilter['$gt'];
+        const to = fieldFilter['$lte'] || fieldFilter['$lt'];
+        if (from || to) {
+          return {
+            from: from ? new Date(from as string | number) : null,
+            to: to ? new Date(to as string | number) : null,
+          };
+        }
+      }
+    }
+
+    // Priority 4: Fallback to data min/max (never invent unbounded ranges)
+    if (data.length === 0) {
+      return { from: null, to: null };
+    }
+
+    let minDate: Date | null = null;
+    let maxDate: Date | null = null;
+
+    for (const row of data) {
+      const val = row[dimRef];
+      if (val == null) continue;
+      const date = val instanceof Date ? val : new Date(val as string | number);
+      if (isNaN(date.getTime())) continue;
+      if (!minDate || date < minDate) minDate = date;
+      if (!maxDate || date > maxDate) maxDate = date;
+    }
+
+    return { from: minDate, to: maxDate };
+  }
+
+  // ── Bucket Generation ──────────────────────────────────────
+
+  private generateBuckets(from: Date, to: Date, granularity: TimeGranularity, _timezone: string): Date[] {
+    const buckets: Date[] = [];
+    let current = this.truncateToGranularity(new Date(from.getTime()), granularity);
+    const end = to;
+
+    // Safety: limit to MAX_BUCKETS + 1 to detect overflow
+    const maxIterations = MAX_BUCKETS + 1;
+    let iterations = 0;
+
+    while (current <= end && iterations < maxIterations) {
+      buckets.push(new Date(current.getTime()));
+      current = this.advanceBucket(current, granularity);
+      iterations++;
+    }
+
+    return buckets;
+  }
+
+  private truncateToGranularity(date: Date, granularity: TimeGranularity): Date {
+    const d = new Date(date.getTime());
+
+    switch (granularity) {
+      case TimeGranularity.MINUTE:
+        d.setUTCSeconds(0, 0);
+        break;
+      case TimeGranularity.HOUR:
+        d.setUTCMinutes(0, 0, 0);
+        break;
+      case TimeGranularity.DAY:
+        d.setUTCHours(0, 0, 0, 0);
+        break;
+      case TimeGranularity.WEEK:
+        // Truncate to Monday
+        d.setUTCHours(0, 0, 0, 0);
+        const dayOfWeek = d.getUTCDay();
+        const diff = dayOfWeek === 0 ? -6 : 1 - dayOfWeek;
+        d.setUTCDate(d.getUTCDate() + diff);
+        break;
+      case TimeGranularity.MONTH:
+        d.setUTCDate(1);
+        d.setUTCHours(0, 0, 0, 0);
+        break;
+      case TimeGranularity.QUARTER:
+        d.setUTCMonth(Math.floor(d.getUTCMonth() / 3) * 3, 1);
+        d.setUTCHours(0, 0, 0, 0);
+        break;
+      case TimeGranularity.YEAR:
+        d.setUTCMonth(0, 1);
+        d.setUTCHours(0, 0, 0, 0);
+        break;
+    }
+
+    return d;
+  }
+
+  private advanceBucket(date: Date, granularity: TimeGranularity): Date {
+    const d = new Date(date.getTime());
+
+    switch (granularity) {
+      case TimeGranularity.MINUTE:
+        d.setUTCMinutes(d.getUTCMinutes() + 1);
+        break;
+      case TimeGranularity.HOUR:
+        d.setUTCHours(d.getUTCHours() + 1);
+        break;
+      case TimeGranularity.DAY:
+        d.setUTCDate(d.getUTCDate() + 1);
+        break;
+      case TimeGranularity.WEEK:
+        d.setUTCDate(d.getUTCDate() + 7);
+        break;
+      case TimeGranularity.MONTH:
+        d.setUTCMonth(d.getUTCMonth() + 1);
+        break;
+      case TimeGranularity.QUARTER:
+        d.setUTCMonth(d.getUTCMonth() + 3);
+        break;
+      case TimeGranularity.YEAR:
+        d.setUTCFullYear(d.getUTCFullYear() + 1);
+        break;
+    }
+
+    return d;
+  }
+
+  // ── Row Lookup ─────────────────────────────────────────────
+
+  private buildRowLookup(
+    data: AggregationQueryResponseRow[],
+    dimRef: string,
+    granularity: TimeGranularity,
+    _timezone: string,
+  ): Map<string, AggregationQueryResponseRow[]> {
+    const lookup = new Map<string, AggregationQueryResponseRow[]>();
+
+    for (const row of data) {
+      const val = row[dimRef];
+      if (val == null) continue;
+      const date = val instanceof Date ? val : new Date(val as string | number);
+      if (isNaN(date.getTime())) continue;
+
+      const truncated = this.truncateToGranularity(date, granularity);
+      const key = truncated.toISOString();
+
+      const existing = lookup.get(key);
+      if (existing) {
+        existing.push(row);
+      } else {
+        lookup.set(key, [row]);
+      }
+    }
+
+    return lookup;
+  }
+
+  // ── Densification (simple — no other dimensions) ───────────
+
+  private densifySimple(
+    buckets: Date[],
+    existingRows: Map<string, AggregationQueryResponseRow[]>,
+    dimRef: string,
+    measureFillValues: Map<string, 0 | null>,
+  ): AggregationQueryResponseRow[] {
+    const result: AggregationQueryResponseRow[] = [];
+
+    for (const bucket of buckets) {
+      const key = bucket.toISOString();
+      const existing = existingRows.get(key);
+
+      if (existing && existing.length > 0) {
+        // Use existing rows (multiple rows possible if split-by dimension)
+        result.push(...existing);
+      } else {
+        // Generate a filled row
+        const filledRow: AggregationQueryResponseRow = {
+          [dimRef]: bucket.toISOString(),
+          _filled: true,
+        };
+        for (const [mRef, fillValue] of measureFillValues) {
+          filledRow[mRef] = fillValue;
+        }
+        result.push(filledRow);
+      }
+    }
+
+    return result;
+  }
+
+  // ── Densification (with other dimensions — split-by aware) ──
+
+  private densifyWithOtherDimensions(
+    buckets: Date[],
+    existingRows: Map<string, AggregationQueryResponseRow[]>,
+    dimRef: string,
+    measureFillValues: Map<string, 0 | null>,
+    otherDimRefs: string[],
+    allData: AggregationQueryResponseRow[],
+  ): AggregationQueryResponseRow[] {
+    // Collect all unique combinations of "other" dimension values
+    // Use JSON serialization for combo keys to avoid delimiter collisions
+    const combos = new Set<string>();
+    for (const row of allData) {
+      const comboKey = JSON.stringify(otherDimRefs.map(d => row[d] ?? ''));
+      combos.add(comboKey);
+    }
+
+    const result: AggregationQueryResponseRow[] = [];
+
+    for (const bucket of buckets) {
+      const key = bucket.toISOString();
+      const existingForBucket = existingRows.get(key) || [];
+
+      // Track which combos already have data for this bucket
+      const existingCombos = new Set<string>();
+      for (const row of existingForBucket) {
+        const comboKey = JSON.stringify(otherDimRefs.map(d => row[d] ?? ''));
+        existingCombos.add(comboKey);
+        result.push(row);
+      }
+
+      // Fill missing combos for this bucket
+      for (const comboStr of combos) {
+        if (existingCombos.has(comboStr)) continue;
+
+        const comboValues = JSON.parse(comboStr) as unknown[];
+        const filledRow: AggregationQueryResponseRow = {
+          [dimRef]: bucket.toISOString(),
+          _filled: true,
+        };
+        for (let i = 0; i < otherDimRefs.length; i++) {
+          filledRow[otherDimRefs[i]] = comboValues[i];
+        }
+        for (const [mRef, fillValue] of measureFillValues) {
+          filledRow[mRef] = fillValue;
+        }
+        result.push(filledRow);
+      }
+    }
+
+    return result;
+  }
+}

package/src/databases/services/aggregation/topn-processor.backend.service.ts

@@ -0,0 +1,221 @@
+/** ──────────────────────────────────────────────────────────────
+ * TopNProcessorBackendService
+ *
+ * Post-processing step that reduces dimension cardinality:
+ *   1. For each dimension with topN, sort rows by the primary measure (descending)
+ *   2. Keep top N values
+ *   3. If groupOther is true, aggregate the remaining rows into an "Other" bucket
+ *
+ * Pipeline position: After DB execution, BEFORE TimeSeriesDensifier.
+ * This ensures densification only fills buckets for surviving dimension values,
+ * and running totals / window functions operate on the truncated set.
+ * ────────────────────────────────────────────────────────────── */
+
+import { injectable, inject } from 'inversify';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { AggregateFunction } from '@wildo-ai/saas-models';
+import type { AggregationQuery } from '@wildo-ai/saas-models';
+
+const OTHER_BUCKET_LABEL = 'Other';
+
+@injectable()
+export class TopNProcessorBackendService {
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.Logger) private readonly logger: LoggerService,
+  ) {}
+
+  /**
+   * Apply top-N processing for dimensions that have topN configured.
+   *
+   * For each dimension with `topN`, this:
+   * 1. Determines the primary measure to rank by (first measure in the query)
+   * 2. Ranks rows by that measure value (descending)
+   * 3. Keeps only the top N dimension values
+   * 4. If `groupOther` is true, aggregates the rest into an "Other" bucket
+   *
+   * If multiple dimensions have topN, they are processed sequentially.
+   */
+  process(
+    data: Record<string, unknown>[],
+    query: AggregationQuery,
+  ): Record<string, unknown>[] {
+    if (data.length === 0) return data;
+
+    const topNDimensions = query.dimensions.filter(d => d.topN != null && d.topN > 0);
+    if (topNDimensions.length === 0) return data;
+
+    // Determine the primary measure ref (first measure in the query)
+    const primaryMeasureRef = query.measures.length > 0 ? query.measures[0]!.ref : undefined;
+    if (!primaryMeasureRef) {
+      this.logger.warn('TopNProcessor: no measures defined, skipping top-N processing');
+      return data;
+    }
+
+    let result = data;
+
+    for (const dim of topNDimensions) {
+      result = this.applyTopN(result, dim.ref, dim.topN!, dim.groupOther ?? false, primaryMeasureRef, query);
+    }
+
+    return result;
+  }
+
+  /**
+   * Apply top-N for a single dimension.
+   */
+  private applyTopN(
+    data: Record<string, unknown>[],
+    dimensionRef: string,
+    topN: number,
+    groupOther: boolean,
+    primaryMeasureRef: string,
+    query: AggregationQuery,
+  ): Record<string, unknown>[] {
+    // 1. Aggregate measure values by this dimension's value
+    //    This is needed because with split-by, there can be multiple rows per dimension value
+    const measureByDimValue = new Map<string, number>();
+
+    for (const row of data) {
+      const dimValue = String(row[dimensionRef] ?? '');
+      const measureValue = Number(row[primaryMeasureRef] ?? 0);
+      measureByDimValue.set(dimValue, (measureByDimValue.get(dimValue) ?? 0) + measureValue);
+    }
+
+    // 2. Sort dimension values by the aggregated measure value (descending)
+    const rankedValues = [...measureByDimValue.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .map(entry => entry[0]);
+
+    // 3. Separate top-N values from the rest
+    const topValues = new Set(rankedValues.slice(0, topN));
+    const otherValues = new Set(rankedValues.slice(topN));
+
+    if (otherValues.size === 0) {
+      // All values fit in top-N — no filtering needed
+      return data;
+    }
+
+    this.logger.debug(
+      `TopNProcessor: dimension '${dimensionRef}' top-${topN}: keeping ${topValues.size}, ` +
+      `${groupOther ? `aggregating ${otherValues.size} into "Other"` : `removing ${otherValues.size}`}`,
+    );
+
+    // 4. Partition rows
+    const topRows = data.filter(row => topValues.has(String(row[dimensionRef] ?? '')));
+    const otherRows = data.filter(row => otherValues.has(String(row[dimensionRef] ?? '')));
+
+    if (!groupOther || otherRows.length === 0) {
+      return topRows;
+    }
+
+    // 5. Aggregate "Other" rows
+    const otherBucketRows = this.aggregateOtherBucket(
+      otherRows,
+      dimensionRef,
+      query,
+    );
+
+    return [...topRows, ...otherBucketRows];
+  }
+
+  /**
+   * Aggregate overflow rows into an "Other" bucket.
+   *
+   * Groups by all OTHER dimensions (excluding the top-N dimension),
+   * then aggregates numeric measure values using the correct operation
+   * per measure (SUM for sum/count, MIN for min, MAX for max, AVG for avg).
+   *
+   * Note: AVG of averages is an approximation since we don't have
+   * per-group row counts. For exact results, use SUM with a computed
+   * measure that divides by count.
+   */
+  private aggregateOtherBucket(
+    otherRows: Record<string, unknown>[],
+    topNDimensionRef: string,
+    query: AggregationQuery,
+  ): Record<string, unknown>[] {
+    // Other dimensions (to preserve as group keys)
+    const otherDimensionRefs = query.dimensions
+      .map(d => d.ref)
+      .filter(ref => ref !== topNDimensionRef);
+
+    // Build a map of measureRef → aggregate function for correct aggregation
+    const measureAggregateMap = new Map<string, AggregateFunction>();
+    for (const m of query.measures) {
+      measureAggregateMap.set(m.ref, m.aggregate as AggregateFunction);
+    }
+
+    // Group "other" rows by the remaining dimensions
+    const groups = new Map<string, Record<string, unknown>[]>();
+
+    for (const row of otherRows) {
+      const groupKey = otherDimensionRefs.length > 0
+        ? JSON.stringify(otherDimensionRefs.map(ref => row[ref]))
+        : '__all__';
+
+      const group = groups.get(groupKey);
+      if (group) {
+        group.push(row);
+      } else {
+        groups.set(groupKey, [row]);
+      }
+    }
+
+    // Build aggregated rows
+    const result: Record<string, unknown>[] = [];
+
+    for (const [, groupRows] of groups) {
+      const aggregated: Record<string, unknown> = {};
+
+      // Set the top-N dimension to "Other"
+      aggregated[topNDimensionRef] = OTHER_BUCKET_LABEL;
+
+      // Copy other dimension values from the first row (they're the same for the group)
+      for (const dimRef of otherDimensionRefs) {
+        aggregated[dimRef] = groupRows[0]![dimRef];
+      }
+
+      // Aggregate measure values using the correct operation per aggregate function
+      for (const measureDef of query.measures) {
+        const values = groupRows
+          .map(r => Number(r[measureDef.ref] ?? 0))
+          .filter(v => !isNaN(v));
+
+        if (values.length === 0) {
+          aggregated[measureDef.ref] = 0;
+          continue;
+        }
+
+        const agg = measureAggregateMap.get(measureDef.ref);
+        switch (agg) {
+          case AggregateFunction.MIN:
+            aggregated[measureDef.ref] = Math.min(...values);
+            break;
+          case AggregateFunction.MAX:
+            aggregated[measureDef.ref] = Math.max(...values);
+            break;
+          case AggregateFunction.AVG:
+          case AggregateFunction.MEDIAN:
+            // Averaging pre-aggregated averages is an approximation
+            aggregated[measureDef.ref] = values.reduce((a, b) => a + b, 0) / values.length;
+            break;
+          default:
+            // SUM, COUNT, COUNT_DISTINCT, and others: sum the values
+            aggregated[measureDef.ref] = values.reduce((a, b) => a + b, 0);
+            break;
+        }
+      }
+
+      // Preserve _filled flag if any row had it
+      const hasFilledRows = groupRows.some(r => r['_filled'] === true);
+      if (hasFilledRows) {
+        aggregated['_filled'] = true;
+      }
+
+      result.push(aggregated);
+    }
+
+    return result;
+  }
+}