@opendata-ai/openchart-engine 6.25.4 → 6.26.0

package/src/transforms/__tests__/filter-relative.test.ts

@@ -0,0 +1,202 @@
+import { describe, expect, it } from 'vitest';
+import { runFilter } from '../filter';
+import { runTransforms } from '../index';
+
+/**
+ * Generate monthly rows: one row per month for `months` months starting from `startDate`.
+ */
+function monthlyData(startDate: string, months: number) {
+  const rows = [];
+  const start = new Date(startDate);
+  for (let i = 0; i < months; i++) {
+    const d = new Date(start);
+    d.setMonth(d.getMonth() + i);
+    rows.push({ date: d.toISOString().slice(0, 10), value: i + 1 });
+  }
+  return rows;
+}
+
+/**
+ * Generate daily rows: one row per day for `days` days starting from `startDate`.
+ */
+function dailyData(startDate: string, days: number) {
+  const rows = [];
+  const start = new Date(startDate);
+  for (let i = 0; i < days; i++) {
+    const d = new Date(start);
+    d.setDate(d.getDate() + i);
+    rows.push({ date: d.toISOString().slice(0, 10), value: i + 1 });
+  }
+  return rows;
+}
+
+describe('relative-time filter', () => {
+  it('last 1 year from max: 60 monthly rows, gte anchor max offset -1 year returns ~12 rows', () => {
+    const data = monthlyData('2020-01-01', 60); // Jan 2020 through Dec 2024
+    const result = runFilter(data, {
+      field: 'date',
+      gte: { anchor: 'max', offset: -1, unit: 'year' },
+    });
+    // Max date is 2024-12-01, minus 1 year = 2023-12-01
+    // Should include Dec 2023 through Dec 2024 = 13 rows
+    expect(result.length).toBeGreaterThanOrEqual(12);
+    expect(result.length).toBeLessThanOrEqual(13);
+    // All returned dates should be >= 2023-12-01
+    for (const row of result) {
+      expect(new Date(row.date as string).getTime()).toBeGreaterThanOrEqual(
+        new Date('2023-12-01').getTime(),
+      );
+    }
+  });
+
+  it('last 3 years from max: 60 monthly rows, offset -3 returns ~36 rows', () => {
+    const data = monthlyData('2020-01-01', 60);
+    const result = runFilter(data, {
+      field: 'date',
+      gte: { anchor: 'max', offset: -3, unit: 'year' },
+    });
+    // Max is 2024-12-01, minus 3 years = 2021-12-01
+    // Dec 2021 through Dec 2024 = 37 rows
+    expect(result.length).toBeGreaterThanOrEqual(36);
+    expect(result.length).toBeLessThanOrEqual(37);
+  });
+
+  it('last 30 days: 90 daily rows, offset -30 day returns 30-31 rows', () => {
+    const data = dailyData('2024-01-01', 90);
+    const result = runFilter(data, {
+      field: 'date',
+      gte: { anchor: 'max', offset: -30, unit: 'day' },
+    });
+    expect(result.length).toBeGreaterThanOrEqual(30);
+    expect(result.length).toBeLessThanOrEqual(31);
+  });
+
+  it('anchor min + offset: lte anchor min offset +2 year returns rows within 2 years of earliest', () => {
+    const data = monthlyData('2020-01-01', 60);
+    const result = runFilter(data, {
+      field: 'date',
+      lte: { anchor: 'min', offset: 2, unit: 'year' },
+    });
+    // Min is 2020-01-01, plus 2 years = 2022-01-01
+    // Jan 2020 through Jan 2022 = 25 rows
+    expect(result.length).toBeGreaterThanOrEqual(24);
+    expect(result.length).toBeLessThanOrEqual(25);
+    for (const row of result) {
+      expect(new Date(row.date as string).getTime()).toBeLessThanOrEqual(
+        new Date('2022-01-01').getTime(),
+      );
+    }
+  });
+
+  it('quarter unit: offset -4 quarter returns last ~4 quarters', () => {
+    // Generate quarterly data (every 3 months)
+    const data = [];
+    const start = new Date('2020-01-01');
+    for (let i = 0; i < 20; i++) {
+      const d = new Date(start);
+      d.setMonth(d.getMonth() + i * 3);
+      data.push({ date: d.toISOString().slice(0, 10), value: i + 1 });
+    }
+    const result = runFilter(data, {
+      field: 'date',
+      gte: { anchor: 'max', offset: -4, unit: 'quarter' },
+    });
+    // 4 quarters back from max = ~4-5 rows of quarterly data
+    expect(result.length).toBeGreaterThanOrEqual(4);
+    expect(result.length).toBeLessThanOrEqual(5);
+  });
+
+  it('non-ISO dates: data with timestamp numbers resolves correctly', () => {
+    // Use numeric timestamps instead of date strings
+    const baseDate = new Date('2024-01-01').getTime();
+    const dayMs = 86400000;
+    const data = Array.from({ length: 90 }, (_, i) => ({
+      date: baseDate + i * dayMs,
+      value: i,
+    }));
+    const result = runFilter(data, {
+      field: 'date',
+      gte: { anchor: 'max', offset: -30, unit: 'day' },
+    });
+    expect(result.length).toBeGreaterThanOrEqual(30);
+    expect(result.length).toBeLessThanOrEqual(31);
+  });
+
+  it('empty dataset returns [] without throwing', () => {
+    const result = runFilter([], {
+      field: 'date',
+      gte: { anchor: 'max', offset: -1, unit: 'year' },
+    });
+    expect(result).toEqual([]);
+  });
+
+  it('single row: returns that row when anchor matches', () => {
+    const data = [{ date: '2024-06-15', value: 42 }];
+    const result = runFilter(data, {
+      field: 'date',
+      gte: { anchor: 'max', offset: -1, unit: 'year' },
+    });
+    // The single row is both min and max; subtracting 1 year puts the threshold before it
+    expect(result).toHaveLength(1);
+    expect(result[0].value).toBe(42);
+  });
+
+  it('chained transforms: calculate first, then relative filter works on post-transform data', () => {
+    const data = monthlyData('2020-01-01', 60);
+    const result = runTransforms(data, [
+      // Copy date field to a new field (identity via multiply by 1 won't work for dates,
+      // so we just verify the filter operates on the same data)
+      {
+        filter: {
+          field: 'date',
+          gte: { anchor: 'max', offset: -1, unit: 'year' },
+        },
+      },
+    ]);
+    expect(result.length).toBeGreaterThanOrEqual(12);
+    expect(result.length).toBeLessThanOrEqual(13);
+  });
+
+  it('range predicate with two relative refs', () => {
+    const data = monthlyData('2020-01-01', 60);
+    const maxDate = new Date(data[data.length - 1].date as string);
+    const loBound = new Date(maxDate);
+    loBound.setFullYear(loBound.getFullYear() - 2);
+    const hiBound = new Date(maxDate);
+    hiBound.setFullYear(hiBound.getFullYear() - 1);
+
+    const result = runFilter(data, {
+      field: 'date',
+      range: [
+        { anchor: 'max', offset: -2, unit: 'year' },
+        { anchor: 'max', offset: -1, unit: 'year' },
+      ],
+    });
+    expect(result.length).toBeGreaterThanOrEqual(12);
+    expect(result.length).toBeLessThanOrEqual(13);
+    for (const row of result) {
+      const ts = new Date(row.date as string).getTime();
+      expect(ts).toBeGreaterThanOrEqual(loBound.getTime());
+      expect(ts).toBeLessThanOrEqual(hiBound.getTime());
+    }
+  });
+
+  it('logical AND with relative ref and static filter: both applied', () => {
+    const data = monthlyData('2020-01-01', 60).map((r, i) => ({
+      ...r,
+      category: i % 2 === 0 ? 'A' : 'B',
+    }));
+    const result = runFilter(data, {
+      and: [
+        { field: 'date', gte: { anchor: 'max', offset: -1, unit: 'year' } },
+        { field: 'category', equal: 'A' },
+      ],
+    });
+    // ~12-13 rows in last year, roughly half are category A
+    expect(result.length).toBeGreaterThanOrEqual(5);
+    expect(result.length).toBeLessThanOrEqual(7);
+    for (const row of result) {
+      expect(row.category).toBe('A');
+    }
+  });
+});

package/src/transforms/__tests__/window.test.ts

@@ -0,0 +1,286 @@
+import { describe, expect, it } from 'vitest';
+import { runTransforms } from '../index';
+
+describe('window transform', () => {
+  it('lag(1) on 5 sorted rows: first row null, rest match previous', () => {
+    const data = [
+      { month: '2024-01', value: 10 },
+      { month: '2024-02', value: 20 },
+      { month: '2024-03', value: 30 },
+      { month: '2024-04', value: 40 },
+      { month: '2024-05', value: 50 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'lag', field: 'value', offset: 1, as: 'prev_value' }],
+        sort: [{ field: 'month' }],
+      },
+    ]);
+    expect(result.map((r) => r.prev_value)).toEqual([null, 10, 20, 30, 40]);
+  });
+
+  it('lag(12) on 24 monthly rows: rows 1-12 null, row 13 matches row 1', () => {
+    const data = Array.from({ length: 24 }, (_, i) => {
+      const d = new Date(2023, i, 1);
+      return { month: d.toISOString().slice(0, 7), value: (i + 1) * 100 };
+    });
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'lag', field: 'value', offset: 12, as: 'prev_year' }],
+        sort: [{ field: 'month' }],
+      },
+    ]);
+    // First 12 rows should be null
+    for (let i = 0; i < 12; i++) {
+      expect(result[i].prev_year).toBe(null);
+    }
+    // Row 13 (index 12) should match row 1's value (100)
+    expect(result[12].prev_year).toBe(100);
+    // Row 24 (index 23) should match row 12's value (1200)
+    expect(result[23].prev_year).toBe(1200);
+  });
+
+  it('pct_change with offset 1 on [100, 110, 121, 133.1]', () => {
+    const data = [
+      { idx: 1, value: 100 },
+      { idx: 2, value: 110 },
+      { idx: 3, value: 121 },
+      { idx: 4, value: 133.1 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'pct_change', field: 'value', offset: 1, as: 'pct' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result[0].pct).toBe(null);
+    expect(result[1].pct).toBeCloseTo(0.1, 5);
+    expect(result[2].pct).toBeCloseTo(0.1, 5);
+    expect(result[3].pct).toBeCloseTo(0.1, 5);
+  });
+
+  it('pct_change with zero denominator returns null, not Infinity', () => {
+    const data = [
+      { idx: 1, value: 0 },
+      { idx: 2, value: 50 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'pct_change', field: 'value', offset: 1, as: 'pct' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result[0].pct).toBe(null);
+    expect(result[1].pct).toBe(null); // 0 denominator -> null
+  });
+
+  it('diff on [10, 15, 12, 20]', () => {
+    const data = [
+      { idx: 1, value: 10 },
+      { idx: 2, value: 15 },
+      { idx: 3, value: 12 },
+      { idx: 4, value: 20 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'diff', field: 'value', offset: 1, as: 'delta' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result.map((r) => r.delta)).toEqual([null, 5, -3, 8]);
+  });
+
+  it('cumsum on [1, 2, 3, 4, 5]', () => {
+    const data = [
+      { idx: 1, value: 1 },
+      { idx: 2, value: 2 },
+      { idx: 3, value: 3 },
+      { idx: 4, value: 4 },
+      { idx: 5, value: 5 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'cumsum', field: 'value', as: 'running' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result.map((r) => r.running)).toEqual([1, 3, 6, 10, 15]);
+  });
+
+  it('cumsum with nulls [1, null, 3, null, 5] treats nulls as 0', () => {
+    const data = [
+      { idx: 1, value: 1 },
+      { idx: 2, value: null },
+      { idx: 3, value: 3 },
+      { idx: 4, value: null },
+      { idx: 5, value: 5 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'cumsum', field: 'value', as: 'running' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result.map((r) => r.running)).toEqual([1, 1, 4, 4, 9]);
+  });
+
+  it('rank on values [30, 10, 40, 20] sorted ascending gives ranks [1, 2, 3, 4]', () => {
+    // Input is unsorted; window sorts by 'value' ascending
+    const data = [
+      { id: 'a', value: 30 },
+      { id: 'b', value: 10 },
+      { id: 'c', value: 40 },
+      { id: 'd', value: 20 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'rank', field: 'value', as: 'rank' }],
+        sort: [{ field: 'value', order: 'ascending' }],
+      },
+    ]);
+    // Results should be in ORIGINAL input order, so:
+    // a(30) -> rank 3, b(10) -> rank 1, c(40) -> rank 4, d(20) -> rank 2
+    expect(result.map((r) => r.rank)).toEqual([3, 1, 4, 2]);
+  });
+
+  it('first_value on [100, 200, 300] all get 100', () => {
+    const data = [
+      { idx: 1, value: 100 },
+      { idx: 2, value: 200 },
+      { idx: 3, value: 300 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'first_value', field: 'value', as: 'first' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result.map((r) => r.first)).toEqual([100, 100, 100]);
+  });
+
+  it('groupby partitioning: 2 groups of 3 rows, lag(1) computed independently', () => {
+    const data = [
+      { group: 'A', idx: 1, value: 10 },
+      { group: 'B', idx: 1, value: 100 },
+      { group: 'A', idx: 2, value: 20 },
+      { group: 'B', idx: 2, value: 200 },
+      { group: 'A', idx: 3, value: 30 },
+      { group: 'B', idx: 3, value: 300 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'lag', field: 'value', offset: 1, as: 'prev' }],
+        sort: [{ field: 'idx' }],
+        groupby: ['group'],
+      },
+    ]);
+    // Results in original input order
+    // A idx 1 -> null, B idx 1 -> null, A idx 2 -> 10, B idx 2 -> 100, A idx 3 -> 20, B idx 3 -> 200
+    expect(result.map((r) => r.prev)).toEqual([null, null, 10, 100, 20, 200]);
+  });
+
+  it('sorts ISO dates correctly before computing lag', () => {
+    // Input deliberately unsorted
+    const data = [
+      { date: '2024-03-15', value: 300 },
+      { date: '2024-01-10', value: 100 },
+      { date: '2024-02-20', value: 200 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'lag', field: 'value', offset: 1, as: 'prev' }],
+        sort: [{ field: 'date' }],
+      },
+    ]);
+    // Sorted order: Jan(100), Feb(200), Mar(300)
+    // Jan -> null, Feb -> 100, Mar -> 200
+    // Original order: Mar, Jan, Feb -> [200, null, 100]
+    expect(result.map((r) => r.prev)).toEqual([200, null, 100]);
+  });
+
+  it('sorts numeric timestamp strings numerically, not lexicographically', () => {
+    // Lexicographic: "9" > "10" but numeric: 9 < 10
+    const data = [
+      { ts: '10', value: 'b' },
+      { ts: '9', value: 'a' },
+      { ts: '100', value: 'c' },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'lag', field: 'value', offset: 1, as: 'prev' }],
+        sort: [{ field: 'ts' }],
+      },
+    ]);
+    // Numeric sort: 9, 10, 100 -> values: a, b, c
+    // ts=10(b) -> lag is a, ts=9(a) -> lag is null, ts=100(c) -> lag is b
+    expect(result.map((r) => r.prev)).toEqual(['a', null, 'b']);
+  });
+
+  it('lead(1) on 4 rows: last row null, rest match next', () => {
+    const data = [
+      { idx: 1, value: 10 },
+      { idx: 2, value: 20 },
+      { idx: 3, value: 30 },
+      { idx: 4, value: 40 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'lead', field: 'value', offset: 1, as: 'next_value' }],
+        sort: [{ field: 'idx' }],
+      },
+    ]);
+    expect(result.map((r) => r.next_value)).toEqual([20, 30, 40, null]);
+  });
+
+  it('rank with tied values uses competition ranking', () => {
+    const data = [
+      { id: 'a', value: 10 },
+      { id: 'b', value: 20 },
+      { id: 'c', value: 10 },
+      { id: 'd', value: 30 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'rank', field: 'value', as: 'rank' }],
+        sort: [{ field: 'value', order: 'ascending' }],
+      },
+    ]);
+    // Sorted: a(10), c(10), b(20), d(30). Tied values get same rank.
+    // Original order: a(10)->1, b(20)->3, c(10)->1, d(30)->4
+    expect(result.map((r) => r.rank)).toEqual([1, 3, 1, 4]);
+  });
+
+  it('returns [] for empty data', () => {
+    const result = runTransforms(
+      [],
+      [
+        {
+          window: [{ op: 'lag', field: 'value', offset: 1, as: 'prev' }],
+          sort: [{ field: 'value' }],
+        },
+      ],
+    );
+    expect(result).toEqual([]);
+  });
+
+  it('window chained with filter: window then filter on computed field', () => {
+    const data = [
+      { idx: 1, value: 100 },
+      { idx: 2, value: 110 },
+      { idx: 3, value: 90 },
+      { idx: 4, value: 120 },
+    ];
+    const result = runTransforms(data, [
+      {
+        window: [{ op: 'diff', field: 'value', offset: 1, as: 'delta' }],
+        sort: [{ field: 'idx' }],
+      },
+      {
+        filter: { field: 'delta', gt: 0 },
+      },
+    ]);
+    // Diffs: [null, 10, -20, 30] -> filter gt 0 -> [10, 30]
+    expect(result).toHaveLength(2);
+    expect(result.map((r) => r.delta)).toEqual([10, 30]);
+  });
+});

package/src/transforms/filter.ts

@@ -1,17 +1,122 @@
 /**
  * Filter transform: removes rows that don't match a predicate.
+ *
+ * Supports RelativeTimeRef values on comparison properties (lt, lte, gt, gte, range).
+ * These are resolved against the data extent before filtering.
+ */
+
+import type {
+  DataRow,
+  FieldPredicate,
+  FilterPredicate,
+  RelativeTimeRef,
+} from '@opendata-ai/openchart-core';
+import { evaluatePredicate, isRelativeTimeRef } from './predicates';
+
+/**
+ * Apply a time offset to a Date, returning the resulting timestamp.
+ */
+function applyOffset(anchor: Date, offset: number, unit: RelativeTimeRef['unit']): number {
+  const d = new Date(anchor.getTime());
+  switch (unit) {
+    case 'year':
+      d.setFullYear(d.getFullYear() + offset);
+      break;
+    case 'quarter':
+      d.setMonth(d.getMonth() + offset * 3);
+      break;
+    case 'month':
+      d.setMonth(d.getMonth() + offset);
+      break;
+    case 'week':
+      d.setDate(d.getDate() + offset * 7);
+      break;
+    case 'day':
+      d.setDate(d.getDate() + offset);
+      break;
+  }
+  return d.getTime();
+}
+
+/**
+ * Resolve a single RelativeTimeRef against a data array.
+ * Scans the field for min/max date values, then applies the offset.
+ */
+function resolveRef(data: DataRow[], field: string, ref: RelativeTimeRef): number {
+  let anchorMs = ref.anchor === 'max' ? -Infinity : Infinity;
+
+  for (const row of data) {
+    const val = row[field];
+    if (val == null) continue;
+    const ms = new Date(val as string | number).getTime();
+    if (Number.isNaN(ms)) continue;
+    if (ref.anchor === 'max' && ms > anchorMs) anchorMs = ms;
+    if (ref.anchor === 'min' && ms < anchorMs) anchorMs = ms;
+  }
+
+  if (!Number.isFinite(anchorMs)) return 0;
+
+  return applyOffset(new Date(anchorMs), ref.offset, ref.unit);
+}
+
+/**
+ * Walk a predicate tree and resolve any RelativeTimeRef values to concrete numbers.
+ * Returns a new predicate tree (does not mutate the original).
  */
+function resolveRelativeRefs(data: DataRow[], predicate: FilterPredicate): FilterPredicate {
+  if ('and' in predicate) {
+    return { and: predicate.and.map((p) => resolveRelativeRefs(data, p)) };
+  }
+  if ('or' in predicate) {
+    return { or: predicate.or.map((p) => resolveRelativeRefs(data, p)) };
+  }
+  if ('not' in predicate) {
+    return { not: resolveRelativeRefs(data, predicate.not) };
+  }
+
+  // FieldPredicate: check each comparison property for RelativeTimeRef
+  if ('field' in predicate) {
+    const fp = predicate as FieldPredicate;
+    let needsCopy = false;
+    const resolved: Partial<FieldPredicate> = {};
 
-import type { DataRow, FilterPredicate } from '@opendata-ai/openchart-core';
-import { evaluatePredicate } from './predicates';
+    for (const prop of ['lt', 'lte', 'gt', 'gte'] as const) {
+      if (isRelativeTimeRef(fp[prop])) {
+        resolved[prop] = resolveRef(data, fp.field, fp[prop] as RelativeTimeRef);
+        needsCopy = true;
+      }
+    }
+
+    if (fp.range) {
+      const [lo, hi] = fp.range;
+      const loResolved = isRelativeTimeRef(lo) ? resolveRef(data, fp.field, lo) : lo;
+      const hiResolved = isRelativeTimeRef(hi) ? resolveRef(data, fp.field, hi) : hi;
+      if (isRelativeTimeRef(lo) || isRelativeTimeRef(hi)) {
+        resolved.range = [loResolved as number, hiResolved as number];
+        needsCopy = true;
+      }
+    }
+
+    if (needsCopy) {
+      return { ...fp, ...resolved };
+    }
+  }
+
+  return predicate;
+}
 
 /**
  * Filter data rows by a predicate.
  *
+ * If the predicate contains RelativeTimeRef values, they are resolved
+ * against the data extent first, then the standard evaluatePredicate
+ * logic runs per-row.
+ *
  * @param data - Input rows.
  * @param predicate - Filter predicate to evaluate per row.
  * @returns Rows that pass the predicate.
  */
 export function runFilter(data: DataRow[], predicate: FilterPredicate): DataRow[] {
-  return data.filter((datum) => evaluatePredicate(datum, predicate));
+  const resolved = resolveRelativeRefs(data, predicate);
+  return data.filter((datum) => evaluatePredicate(datum, resolved));
 }

package/src/transforms/index.ts

@@ -13,6 +13,7 @@ import { runCalculate } from './calculate';
 import { runFilter } from './filter';
 import { runFold } from './fold';
 import { runTimeUnit } from './timeunit';
+import { runWindow } from './window';
 
 export { runAggregate } from './aggregate';
 export { runBin } from './bin';
@@ -20,8 +21,9 @@ export { runCalculate } from './calculate';
 export { isConditionalValueDef, resolveConditionalValue } from './conditional';
 export { runFilter } from './filter';
 export { runFold } from './fold';
-export { evaluatePredicate } from './predicates';
+export { evaluatePredicate, isRelativeTimeRef } from './predicates';
 export { runTimeUnit } from './timeunit';
+export { runWindow } from './window';
 
 /**
  * Run a sequence of transforms against a data array.
@@ -49,6 +51,8 @@ export function runTransforms(data: DataRow[], transforms: Transform[]): DataRow
       result = runAggregate(result, transform);
     } else if ('fold' in transform) {
       result = runFold(result, transform);
+    } else if ('window' in transform) {
+      result = runWindow(result, transform);
     }
   }