@opendata-ai/openchart-engine 8.0.0-rc.12 → 8.0.0-rc.3

package/src/transforms/conditional.ts

@@ -0,0 +1,46 @@
+/**
+ * Conditional encoding evaluation.
+ *
+ * Resolves conditional value definitions per-datum by evaluating
+ * predicates and returning the first matching condition's value.
+ */
+
+import type { ConditionalValueDef, DataRow } from '@opendata-ai/openchart-core';
+import { evaluatePredicate } from './predicates';
+
+/**
+ * Resolve a conditional value definition for a datum.
+ *
+ * Evaluates conditions in order, returning the value from the first
+ * condition whose test passes. Falls back to the default value if
+ * no condition matches.
+ *
+ * @param datum - The data row to evaluate against.
+ * @param channelDef - The conditional value definition.
+ * @returns The resolved value, or undefined if no condition matches and no default.
+ */
+export function resolveConditionalValue(datum: DataRow, channelDef: ConditionalValueDef): unknown {
+  const conditions = Array.isArray(channelDef.condition)
+    ? channelDef.condition
+    : [channelDef.condition];
+
+  for (const cond of conditions) {
+    if (evaluatePredicate(datum, cond.test)) {
+      // If the condition specifies a field, return the datum's field value
+      if (cond.field !== undefined) {
+        return datum[cond.field];
+      }
+      return cond.value;
+    }
+  }
+
+  // No condition matched, return default
+  return channelDef.value;
+}
+
+/**
+ * Check if a channel definition is a ConditionalValueDef.
+ */
+export function isConditionalValueDef(def: unknown): def is ConditionalValueDef {
+  return def !== null && typeof def === 'object' && 'condition' in (def as Record<string, unknown>);
+}

package/src/transforms/filter.ts

@@ -0,0 +1,122 @@
+/**
+ * Filter transform: removes rows that don't match a predicate.
+ *
+ * Supports RelativeTimeRef values on comparison properties (lt, lte, gt, gte, range).
+ * These are resolved against the data extent before filtering.
+ */
+
+import type {
+  DataRow,
+  FieldPredicate,
+  FilterPredicate,
+  RelativeTimeRef,
+} from '@opendata-ai/openchart-core';
+import { evaluatePredicate, isRelativeTimeRef } from './predicates';
+
+/**
+ * Apply a time offset to a Date, returning the resulting timestamp.
+ */
+function applyOffset(anchor: Date, offset: number, unit: RelativeTimeRef['unit']): number {
+  const d = new Date(anchor.getTime());
+  switch (unit) {
+    case 'year':
+      d.setFullYear(d.getFullYear() + offset);
+      break;
+    case 'quarter':
+      d.setMonth(d.getMonth() + offset * 3);
+      break;
+    case 'month':
+      d.setMonth(d.getMonth() + offset);
+      break;
+    case 'week':
+      d.setDate(d.getDate() + offset * 7);
+      break;
+    case 'day':
+      d.setDate(d.getDate() + offset);
+      break;
+  }
+  return d.getTime();
+}
+
+/**
+ * Resolve a single RelativeTimeRef against a data array.
+ * Scans the field for min/max date values, then applies the offset.
+ */
+function resolveRef(data: DataRow[], field: string, ref: RelativeTimeRef): number {
+  let anchorMs = ref.anchor === 'max' ? -Infinity : Infinity;
+
+  for (const row of data) {
+    const val = row[field];
+    if (val == null) continue;
+    const ms = new Date(val as string | number).getTime();
+    if (Number.isNaN(ms)) continue;
+    if (ref.anchor === 'max' && ms > anchorMs) anchorMs = ms;
+    if (ref.anchor === 'min' && ms < anchorMs) anchorMs = ms;
+  }
+
+  if (!Number.isFinite(anchorMs)) return 0;
+
+  return applyOffset(new Date(anchorMs), ref.offset, ref.unit);
+}
+
+/**
+ * Walk a predicate tree and resolve any RelativeTimeRef values to concrete numbers.
+ * Returns a new predicate tree (does not mutate the original).
+ */
+function resolveRelativeRefs(data: DataRow[], predicate: FilterPredicate): FilterPredicate {
+  if ('and' in predicate) {
+    return { and: predicate.and.map((p) => resolveRelativeRefs(data, p)) };
+  }
+  if ('or' in predicate) {
+    return { or: predicate.or.map((p) => resolveRelativeRefs(data, p)) };
+  }
+  if ('not' in predicate) {
+    return { not: resolveRelativeRefs(data, predicate.not) };
+  }
+
+  // FieldPredicate: check each comparison property for RelativeTimeRef
+  if ('field' in predicate) {
+    const fp = predicate as FieldPredicate;
+    let needsCopy = false;
+    const resolved: Partial<FieldPredicate> = {};
+
+    for (const prop of ['lt', 'lte', 'gt', 'gte'] as const) {
+      if (isRelativeTimeRef(fp[prop])) {
+        resolved[prop] = resolveRef(data, fp.field, fp[prop] as RelativeTimeRef);
+        needsCopy = true;
+      }
+    }
+
+    if (fp.range) {
+      const [lo, hi] = fp.range;
+      const loResolved = isRelativeTimeRef(lo) ? resolveRef(data, fp.field, lo) : lo;
+      const hiResolved = isRelativeTimeRef(hi) ? resolveRef(data, fp.field, hi) : hi;
+      if (isRelativeTimeRef(lo) || isRelativeTimeRef(hi)) {
+        resolved.range = [loResolved as number, hiResolved as number];
+        needsCopy = true;
+      }
+    }
+
+    if (needsCopy) {
+      return { ...fp, ...resolved };
+    }
+  }
+
+  return predicate;
+}
+
+/**
+ * Filter data rows by a predicate.
+ *
+ * If the predicate contains RelativeTimeRef values, they are resolved
+ * against the data extent first, then the standard evaluatePredicate
+ * logic runs per-row.
+ *
+ * @param data - Input rows.
+ * @param predicate - Filter predicate to evaluate per row.
+ * @returns Rows that pass the predicate.
+ */
+export function runFilter(data: DataRow[], predicate: FilterPredicate): DataRow[] {
+  const resolved = resolveRelativeRefs(data, predicate);
+  return data.filter((datum) => evaluatePredicate(datum, resolved));
+}

package/src/transforms/fold.ts

@@ -0,0 +1,49 @@
+/**
+ * Fold transform: unpivots wide-format data into long format.
+ *
+ * Follows Vega-Lite fold transform conventions.
+ * For each input row, creates N new rows (one per fold field),
+ * copying all non-fold fields and adding key/value columns.
+ */
+
+import type { DataRow, FoldTransform } from '@opendata-ai/openchart-core';
+
+/**
+ * Apply a fold transform to data rows.
+ *
+ * For each input row, creates one output row per fold field. Each output
+ * row contains all fields from the original row (except the fold fields),
+ * plus a key field (the fold field name) and a value field (the fold field value).
+ *
+ * @param data - Input rows.
+ * @param transform - Fold transform definition.
+ * @returns Folded rows (N rows per input row, where N = fold fields count).
+ */
+export function runFold(data: DataRow[], transform: FoldTransform): DataRow[] {
+  const { fold } = transform;
+  const [keyAs, valueAs] = transform.as ?? ['key', 'value'];
+  const foldSet = new Set(fold);
+
+  const result: DataRow[] = [];
+
+  for (const row of data) {
+    // Copy all non-fold fields
+    const base: DataRow = {};
+    for (const [k, v] of Object.entries(row)) {
+      if (!foldSet.has(k)) {
+        base[k] = v;
+      }
+    }
+
+    // Create one row per fold field
+    for (const field of fold) {
+      result.push({
+        ...base,
+        [keyAs]: field,
+        [valueAs]: row[field],
+      });
+    }
+  }
+
+  return result;
+}

package/src/transforms/index.ts

@@ -0,0 +1,60 @@
+/**
+ * Data transform pipeline.
+ *
+ * Runs an ordered sequence of transforms (filter, bin, calculate, timeUnit)
+ * against a data array. Each transform produces a new array, feeding into
+ * the next transform in sequence.
+ */
+
+import type { DataRow, Transform } from '@opendata-ai/openchart-core';
+import { runAggregate } from './aggregate';
+import { runBin } from './bin';
+import { runCalculate } from './calculate';
+import { runFilter } from './filter';
+import { runFold } from './fold';
+import { runTimeUnit } from './timeunit';
+import { runWindow } from './window';
+
+export { runAggregate } from './aggregate';
+export { runBin } from './bin';
+export { runCalculate } from './calculate';
+export { isConditionalValueDef, resolveConditionalValue } from './conditional';
+export { runFilter } from './filter';
+export { runFold } from './fold';
+export { evaluatePredicate, isRelativeTimeRef } from './predicates';
+export { runTimeUnit } from './timeunit';
+export { runWindow } from './window';
+
+/**
+ * Run a sequence of transforms against a data array.
+ *
+ * Each transform is applied in order, producing a new data array
+ * that feeds into the next transform. The original data is not mutated.
+ *
+ * @param data - Input data rows.
+ * @param transforms - Ordered array of transform definitions.
+ * @returns Transformed data rows.
+ */
+export function runTransforms(data: DataRow[], transforms: Transform[]): DataRow[] {
+  let result = data;
+
+  for (const transform of transforms) {
+    if ('filter' in transform) {
+      result = runFilter(result, transform.filter);
+    } else if ('bin' in transform) {
+      result = runBin(result, transform);
+    } else if ('calculate' in transform) {
+      result = runCalculate(result, transform);
+    } else if ('timeUnit' in transform) {
+      result = runTimeUnit(result, transform);
+    } else if ('aggregate' in transform) {
+      result = runAggregate(result, transform);
+    } else if ('fold' in transform) {
+      result = runFold(result, transform);
+    } else if ('window' in transform) {
+      result = runWindow(result, transform);
+    }
+  }
+
+  return result;
+}

package/src/transforms/predicates.ts

@@ -0,0 +1,123 @@
+/**
+ * Predicate evaluation for filter transforms and conditional encoding.
+ *
+ * Supports field predicates (equal, lt, gt, range, oneOf, valid)
+ * and logical combinators (and, or, not).
+ */
+
+import type {
+  DataRow,
+  FieldPredicate,
+  FilterPredicate,
+  RelativeTimeRef,
+} from '@opendata-ai/openchart-core';
+
+/**
+ * Check if a predicate is a FieldPredicate (has a 'field' property).
+ */
+function isFieldPredicate(pred: FilterPredicate): pred is FieldPredicate {
+  return 'field' in pred;
+}
+
+/**
+ * Check if a value is a RelativeTimeRef (has anchor, offset, and unit properties).
+ */
+export function isRelativeTimeRef(value: unknown): value is RelativeTimeRef {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'anchor' in value &&
+    'offset' in value &&
+    'unit' in value
+  );
+}
+
+/**
+ * Coerce a comparison value to a number. RelativeTimeRef objects that weren't
+ * resolved upstream become NaN, which makes all comparisons return false (safe).
+ */
+function toNum(v: number | RelativeTimeRef): number {
+  return typeof v === 'number' ? v : NaN;
+}
+
+/**
+ * Evaluate a single field predicate against a datum.
+ */
+function evaluateFieldPredicate(datum: DataRow, pred: FieldPredicate): boolean {
+  const value = datum[pred.field];
+
+  // valid: check for non-null, non-undefined, non-NaN
+  if (pred.valid !== undefined) {
+    const isValid = value !== null && value !== undefined && !Number.isNaN(value);
+    return pred.valid ? isValid : !isValid;
+  }
+
+  // equal: use loose equality so "181" == 181 matches across type boundaries
+  // (data values may arrive as strings from CSV/JSON parsing)
+  if (pred.equal !== undefined) {
+    // biome-ignore lint/suspicious/noDoubleEquals: intentional loose equality for CSV/JSON type coercion
+    return value == pred.equal;
+  }
+
+  // Numeric comparisons (with date string fallback)
+  let numValue = Number(value);
+  if (Number.isNaN(numValue) && value != null) {
+    const ms = new Date(value as string | number).getTime();
+    if (!Number.isNaN(ms)) numValue = ms;
+  }
+
+  if (pred.lt !== undefined) {
+    return numValue < toNum(pred.lt);
+  }
+  if (pred.lte !== undefined) {
+    return numValue <= toNum(pred.lte);
+  }
+  if (pred.gt !== undefined) {
+    return numValue > toNum(pred.gt);
+  }
+  if (pred.gte !== undefined) {
+    return numValue >= toNum(pred.gte);
+  }
+
+  // range: inclusive [min, max]
+  if (pred.range !== undefined) {
+    const [lo, hi] = pred.range;
+    return numValue >= toNum(lo) && numValue <= toNum(hi);
+  }
+
+  // oneOf: use loose equality (same rationale as equal above)
+  if (pred.oneOf !== undefined) {
+    // biome-ignore lint/suspicious/noDoubleEquals: intentional loose equality for CSV/JSON type coercion
+    return pred.oneOf.some((v) => v == value);
+  }
+
+  // No condition specified, default to true
+  return true;
+}
+
+/**
+ * Evaluate a filter predicate (field predicate or logical combinator) against a datum.
+ *
+ * @param datum - The data row to test.
+ * @param predicate - The filter predicate to evaluate.
+ * @returns Whether the datum passes the predicate.
+ */
+export function evaluatePredicate(datum: DataRow, predicate: FilterPredicate): boolean {
+  if (isFieldPredicate(predicate)) {
+    return evaluateFieldPredicate(datum, predicate);
+  }
+
+  if ('and' in predicate) {
+    return predicate.and.every((p) => evaluatePredicate(datum, p));
+  }
+
+  if ('or' in predicate) {
+    return predicate.or.some((p) => evaluatePredicate(datum, p));
+  }
+
+  if ('not' in predicate) {
+    return !evaluatePredicate(datum, predicate.not);
+  }
+
+  return true;
+}

package/src/transforms/timeunit.ts

@@ -0,0 +1,88 @@
+/**
+ * Time unit transform: extracts temporal components from date fields.
+ *
+ * Follows Vega-Lite time unit conventions.
+ */
+
+import type { DataRow, TimeUnit, TimeUnitTransform } from '@opendata-ai/openchart-core';
+
+/**
+ * Extract a time unit value from a Date object.
+ */
+function extractTimeUnit(date: Date, unit: TimeUnit): number | string {
+  switch (unit) {
+    case 'year':
+      return date.getFullYear();
+    case 'quarter':
+      return Math.floor(date.getMonth() / 3) + 1;
+    case 'month':
+      return date.getMonth(); // 0-indexed like JS Date
+    case 'week': {
+      // ISO week number
+      const d = new Date(date.getTime());
+      d.setHours(0, 0, 0, 0);
+      d.setDate(d.getDate() + 3 - ((d.getDay() + 6) % 7));
+      const yearStart = new Date(d.getFullYear(), 0, 1);
+      return Math.ceil(((d.getTime() - yearStart.getTime()) / 86400000 + 1) / 7);
+    }
+    case 'day':
+      return date.getDay(); // 0 = Sunday
+    case 'dayofyear': {
+      const start = new Date(date.getFullYear(), 0, 0);
+      const diff = date.getTime() - start.getTime();
+      return Math.floor(diff / 86400000);
+    }
+    case 'date':
+      return date.getDate(); // 1-31
+    case 'hours':
+      return date.getHours();
+    case 'minutes':
+      return date.getMinutes();
+    case 'seconds':
+      return date.getSeconds();
+    case 'milliseconds':
+      return date.getMilliseconds();
+    // Compound units
+    case 'yearmonth':
+      return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`;
+    case 'yearmonthdate':
+      return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
+    case 'monthdate':
+      return `${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
+    case 'hoursminutes':
+      return `${String(date.getHours()).padStart(2, '0')}:${String(date.getMinutes()).padStart(2, '0')}`;
+  }
+}
+
+/**
+ * Parse a value into a Date object.
+ * Handles Date objects, ISO strings, and numeric timestamps.
+ */
+function toDate(value: unknown): Date | null {
+  if (value instanceof Date) return value;
+  if (typeof value === 'string' || typeof value === 'number') {
+    const d = new Date(value);
+    return Number.isNaN(d.getTime()) ? null : d;
+  }
+  return null;
+}
+
+/**
+ * Apply a time unit transform to data rows.
+ *
+ * Parses the field as a date and extracts the specified time unit,
+ * storing the result in a new field.
+ *
+ * @param data - Input rows.
+ * @param transform - Time unit transform definition.
+ * @returns New rows with the time unit field added.
+ */
+export function runTimeUnit(data: DataRow[], transform: TimeUnitTransform): DataRow[] {
+  return data.map((row) => {
+    const date = toDate(row[transform.field]);
+    return {
+      ...row,
+      [transform.as]: date ? extractTimeUnit(date, transform.timeUnit) : null,
+    };
+  });
+}

package/src/transforms/window.ts

@@ -0,0 +1,185 @@
+/**
+ * Window transform: computes values relative to other rows in sort order
+ * within a partition (lag, lead, diff, pct_change, cumsum, rank, first_value).
+ *
+ * Follows the same grouping pattern as aggregate.ts, but preserves all
+ * input rows and appends computed fields rather than collapsing groups.
+ */
+
+import type { DataRow, WindowTransform } from '@opendata-ai/openchart-core';
+
+/**
+ * Build a composite group key from a row's groupby field values.
+ * Uses null-char delimiter (same convention as aggregate.ts).
+ */
+function groupKey(row: DataRow, groupby: string[]): string {
+  return groupby.map((f) => String(row[f] ?? '')).join('\x00');
+}
+
+/**
+ * Try to parse a value as a date, returning its timestamp if valid.
+ * Only recognizes ISO-8601 strings (must contain '-' or 'T') and numeric timestamps.
+ * Bare numeric strings like "9" or "100" are not treated as dates.
+ */
+function tryParseDate(val: unknown): number {
+  if (val == null) return NaN;
+  if (typeof val === 'number') return new Date(val).getTime();
+  if (typeof val === 'string' && (val.includes('-') || val.includes('T'))) {
+    const ms = new Date(val).getTime();
+    return ms;
+  }
+  return NaN;
+}
+
+/**
+ * Type-aware comparison for sorting:
+ * 1. Try ISO date parsing first
+ * 2. If both are numeric, compare as numbers
+ * 3. Otherwise lexicographic string comparison
+ */
+function compareValues(a: unknown, b: unknown, order: 'ascending' | 'descending'): number {
+  const dir = order === 'descending' ? -1 : 1;
+
+  // Try dates first
+  const dateA = tryParseDate(a);
+  const dateB = tryParseDate(b);
+  if (!Number.isNaN(dateA) && !Number.isNaN(dateB)) {
+    return dir * (dateA - dateB);
+  }
+
+  // Try numeric comparison
+  const numA = Number(a);
+  const numB = Number(b);
+  if (Number.isFinite(numA) && Number.isFinite(numB)) {
+    return dir * (numA - numB);
+  }
+
+  // Fallback to string comparison
+  return dir * String(a ?? '').localeCompare(String(b ?? ''));
+}
+
+/**
+ * Apply a window transform to data rows.
+ *
+ * Groups rows by the groupby fields, sorts within each group, computes
+ * window operations, then returns all rows in their original input order
+ * with computed fields appended.
+ *
+ * @param data - Input rows.
+ * @param transform - Window transform definition.
+ * @returns Rows with computed window fields appended.
+ */
+export function runWindow(data: DataRow[], transform: WindowTransform): DataRow[] {
+  if (data.length === 0) return [];
+
+  const { window: windowDefs, sort, groupby = [] } = transform;
+
+  // Track original indices so we can restore input order
+  const indexed = data.map((row, i) => ({ row, originalIndex: i }));
+
+  // Group rows by groupby fields
+  const groups = new Map<string, { row: DataRow; originalIndex: number }[]>();
+  for (const entry of indexed) {
+    const key = groupby.length > 0 ? groupKey(entry.row, groupby) : '';
+    const existing = groups.get(key);
+    if (existing) {
+      existing.push(entry);
+    } else {
+      groups.set(key, [entry]);
+    }
+  }
+
+  // Build result array indexed by original position
+  const result: DataRow[] = new Array(data.length);
+
+  for (const groupEntries of groups.values()) {
+    // Sort the group entries by the sort fields
+    const sorted = [...groupEntries].sort((a, b) => {
+      for (const s of sort) {
+        const cmp = compareValues(a.row[s.field], b.row[s.field], s.order ?? 'ascending');
+        if (cmp !== 0) return cmp;
+      }
+      return 0;
+    });
+
+    // Compute window operations for each row in sorted order
+    for (let i = 0; i < sorted.length; i++) {
+      const entry = sorted[i];
+      const outRow: DataRow = { ...entry.row };
+
+      for (const def of windowDefs) {
+        const offset = def.offset ?? 1;
+        let computed: unknown = null;
+
+        switch (def.op) {
+          case 'lag': {
+            const lagIdx = i - offset;
+            computed = lagIdx >= 0 ? (sorted[lagIdx].row[def.field] ?? null) : null;
+            break;
+          }
+          case 'lead': {
+            const leadIdx = i + offset;
+            computed = leadIdx < sorted.length ? (sorted[leadIdx].row[def.field] ?? null) : null;
+            break;
+          }
+          case 'diff': {
+            const lagIdx = i - offset;
+            if (lagIdx >= 0) {
+              const current = Number(entry.row[def.field]);
+              const lagged = Number(sorted[lagIdx].row[def.field]);
+              computed =
+                Number.isFinite(current) && Number.isFinite(lagged) ? current - lagged : null;
+            }
+            break;
+          }
+          case 'pct_change': {
+            const lagIdx = i - offset;
+            if (lagIdx >= 0) {
+              const current = Number(entry.row[def.field]);
+              const lagged = Number(sorted[lagIdx].row[def.field]);
+              if (Number.isFinite(current) && Number.isFinite(lagged) && lagged !== 0) {
+                computed = (current - lagged) / lagged;
+              }
+            }
+            break;
+          }
+          case 'cumsum': {
+            const val = Number(entry.row[def.field]);
+            const addend = Number.isFinite(val) ? val : 0;
+            if (i === 0) {
+              computed = addend;
+            } else {
+              const prev = Number(result[sorted[i - 1].originalIndex]?.[def.as] ?? 0);
+              computed = prev + addend;
+            }
+            break;
+          }
+          case 'rank': {
+            let rank = i + 1;
+            for (let j = 0; j < i; j++) {
+              const isTie = sort.every(
+                (s) => String(sorted[j].row[s.field]) === String(entry.row[s.field]),
+              );
+              if (isTie) {
+                rank = result[sorted[j].originalIndex]?.[def.as] as number;
+                break;
+              }
+            }
+            computed = rank;
+            break;
+          }
+          case 'first_value': {
+            computed = sorted[0].row[def.field] ?? null;
+            break;
+          }
+        }
+
+        outRow[def.as] = computed;
+      }
+
+      result[entry.originalIndex] = outRow;
+    }
+  }
+
+  return result;
+}