ggaction 0.0.13 → 0.0.16

package/src/grammar/summary.js

@@ -5,9 +5,18 @@ import {
   validateAggregate,
   validateAggregateFieldValues
 } from "./aggregate.js";
+import {
+  calculateWeightedAggregate,
+  normalizeStatisticalWeight,
+  readStatisticalWeights,
+  summarizeStatisticalWeights,
+  validateWeightedAggregate,
+  validateWeightedNumericFields,
+  weightedRows
+} from "./weightedStatistics.js";
 
 const TRANSFORM_KEYS = Object.freeze([
-  "type", "groupBy", "aggregates", "members"
+  "type", "groupBy", "aggregates", "members", "weight"
 ]);
 const AGGREGATE_KEYS = Object.freeze(["op", "field", "as"]);
 const NOMINAL_OPERATIONS = new Set(["distinct", "valid", "missing"]);
@@ -42,14 +51,17 @@ function normalizeAggregate(value) {
   };
 }
 
-export function normalizeSummaryTransform({ groupBy, aggregates, members } = {}) {
+export function normalizeSummaryTransform({ groupBy, aggregates, members, weight } = {}) {
   const transform = {
     type: "summary",
     groupBy: normalizeGroupBy(groupBy),
     aggregates: Array.isArray(aggregates)
       ? aggregates.map(normalizeAggregate)
       : aggregates,
-    ...(members === undefined ? {} : { members })
+    ...(members === undefined ? {} : { members }),
+    ...(weight === undefined
+      ? {}
+      : { weight: normalizeStatisticalWeight(weight, "Summary weight") })
   };
   validateSummaryTransform(transform);
   return cloneAndFreeze(transform);
@@ -85,6 +97,7 @@ export function validateSummaryTransform(transform) {
     }
     rejectUnknownKeys(aggregate, AGGREGATE_KEYS, `summary aggregate ${index}`);
     const operation = validateAggregate(aggregate.op);
+    if (transform.weight !== undefined) validateWeightedAggregate(operation);
     if (operation === "count") {
       if (Object.hasOwn(aggregate, "field")) {
         throw new Error("Summary count does not accept a field.");
@@ -104,6 +117,9 @@ export function validateSummaryTransform(transform) {
       throw new Error(`Summary members field "${members}" collides with another output.`);
     }
   }
+  if (transform.weight !== undefined) {
+    normalizeStatisticalWeight(transform.weight, "Summary weight");
+  }
   return transform;
 }
 
@@ -136,6 +152,9 @@ function requireSourceFields(rows, transform) {
       throw new Error(`Summary source does not contain order field "${orderBy}".`);
     }
   }
+  if (transform.weight !== undefined && !fields.has(transform.weight.field)) {
+    throw new Error(`Summary source does not contain weight field "${transform.weight.field}".`);
+  }
 }
 
 function validateAggregateValues(rows, aggregate) {
@@ -150,21 +169,42 @@ function validateAggregateValues(rows, aggregate) {
 export function deriveSummaryRows(rows, transform) {
   validateSummaryTransform(transform);
   requireSourceFields(rows, transform);
-  for (const aggregate of transform.aggregates) {
-    validateAggregateValues(rows, aggregate);
+  let weightEntries;
+  if (transform.weight === undefined) {
+    for (const aggregate of transform.aggregates) {
+      validateAggregateValues(rows, aggregate);
+    }
+  } else {
+    weightEntries = readStatisticalWeights(rows, transform.weight, "Summary").entries;
+    validateWeightedNumericFields(
+      weightEntries,
+      [...new Set(transform.aggregates.flatMap(aggregate =>
+        aggregate.field === undefined ? [] : [aggregate.field]
+      ))],
+      "Summary"
+    );
   }
 
   const groups = new Map();
   if (transform.groupBy.length === 0) {
-    groups.set("all", { values: {}, rows });
+    groups.set("all", {
+      values: {},
+      rows,
+      ...(weightEntries === undefined ? {} : { entries: weightEntries })
+    });
   } else {
-    rows.forEach(row => {
+    rows.forEach((row, index) => {
       const values = Object.fromEntries(transform.groupBy.map(field => [field, row[field]]));
       const key = transform.groupBy.map(field =>
         scalarKey(row[field], `Summary group field "${field}"`)
       ).join("|");
-      const group = groups.get(key) ?? { values, rows: [] };
+      const group = groups.get(key) ?? {
+        values,
+        rows: [],
+        ...(weightEntries === undefined ? {} : { entries: [] })
+      };
       group.rows.push(row);
+      if (weightEntries !== undefined) group.entries.push(weightEntries[index]);
       groups.set(key, group);
     });
   }
@@ -172,12 +212,34 @@ export function deriveSummaryRows(rows, transform) {
     throw new RangeError(`Summary output cannot exceed ${MAX_OUTPUT_ROWS} groups.`);
   }
 
-  return [...groups.values()].map(group => ({
-    ...group.values,
-    ...Object.fromEntries(transform.aggregates.map(aggregate => [
-      aggregate.as,
-      aggregateRows(group.rows, aggregate.field ?? "__row", aggregate.op)
-    ])),
-    ...(transform.members === undefined ? {} : { [transform.members]: group.rows })
-  }));
+  return [...groups.values()].map(group => {
+    const weightSummary = transform.weight === undefined
+      ? undefined
+      : summarizeStatisticalWeights(
+          group.entries,
+          transform.weight.kind,
+          "Summary group"
+        );
+    return {
+      ...group.values,
+      ...Object.fromEntries(transform.aggregates.map(aggregate => [
+        aggregate.as,
+        weightSummary === undefined
+          ? aggregateRows(group.rows, aggregate.field ?? "__row", aggregate.op)
+          : calculateWeightedAggregate(
+              weightSummary,
+              aggregate.field,
+              aggregate.op,
+              `Summary aggregate "${aggregate.as}"`
+            )
+      ])),
+      ...(transform.members === undefined
+        ? {}
+        : {
+            [transform.members]: weightSummary === undefined
+              ? group.rows
+              : weightedRows(weightSummary)
+          })
+    };
+  });
 }

package/src/grammar/timeUnit.js

@@ -33,10 +33,47 @@ export const TIME_UNITS = cloneAndFreeze([
   "day",
   "hour",
   "minute",
-  "second"
+  "second",
+  "week",
+  "weekday"
 ]);
 
-const TRANSFORM_KEYS = Object.freeze(["type", "field", "unit", "as", "temporalUnit"]);
+const TRANSFORM_KEYS = Object.freeze([
+  "type", "field", "unit", "as", "temporalUnit", "timeZone",
+  "weekStartsOn", "weekRule"
+]);
+
+const CALENDAR_FORMATTERS = new Map();
+
+function formatter(timeZone) {
+  if (!CALENDAR_FORMATTERS.has(timeZone)) {
+    let value;
+    try {
+      value = new Intl.DateTimeFormat("en-US-u-ca-gregory-nu-latn", {
+        timeZone,
+        year: "numeric",
+        month: "2-digit",
+        day: "2-digit",
+        hour: "2-digit",
+        minute: "2-digit",
+        second: "2-digit",
+        hourCycle: "h23"
+      });
+    } catch {
+      throw new RangeError(`Unsupported time zone "${timeZone}".`);
+    }
+    CALENDAR_FORMATTERS.set(timeZone, value);
+  }
+  return CALENDAR_FORMATTERS.get(timeZone);
+}
+
+function validateTimeZone(timeZone) {
+  if (typeof timeZone !== "string" || timeZone.length === 0) {
+    throw new TypeError("Time-unit timeZone must be a non-empty IANA time-zone name.");
+  }
+  formatter(timeZone);
+  return timeZone;
+}
 
 function requireField(value, label) {
   if (typeof value !== "string" || value.length === 0) {
@@ -69,12 +106,38 @@ export function validateTimeUnitTransform(transform) {
   if (!TIME_UNITS.includes(transform.unit)) {
     throw new Error(`Unsupported time unit "${transform.unit}".`);
   }
+  if (Object.hasOwn(transform, "timeZone")) validateTimeZone(transform.timeZone);
+  const hasWeekStartsOn = Object.hasOwn(transform, "weekStartsOn");
+  const hasWeekRule = Object.hasOwn(transform, "weekRule");
+  if (transform.unit === "week") {
+    if (!Number.isInteger(transform.weekStartsOn) ||
+      transform.weekStartsOn < 0 || transform.weekStartsOn > 6) {
+      throw new RangeError("Time-unit weekStartsOn must be an integer from 0 through 6.");
+    }
+    if (!["calendar", "iso"].includes(transform.weekRule)) {
+      throw new Error(`Unsupported time-unit weekRule "${transform.weekRule}".`);
+    }
+    if (transform.weekRule === "iso" && transform.weekStartsOn !== 1) {
+      throw new Error("Time-unit ISO weeks require weekStartsOn 1 (Monday).");
+    }
+  } else if (hasWeekStartsOn || hasWeekRule) {
+    throw new Error("Time-unit weekStartsOn and weekRule require unit week.");
+  }
   return transform;
 }
 
-export function normalizeTimeUnitTransform({ field, unit, as, temporalUnit } = {}) {
+export function normalizeTimeUnitTransform(args = {}) {
+  const { field, unit, as, temporalUnit, timeZone } = args;
   const transform = { type: "timeUnit", field, unit, as,
-    ...(temporalUnit === undefined ? {} : { temporalUnit }) };
+    ...(temporalUnit === undefined ? {} : { temporalUnit }),
+    ...(timeZone === undefined ? {} : { timeZone }) };
+  if (unit === "week") {
+    transform.weekStartsOn = args.weekStartsOn ?? 1;
+    transform.weekRule = args.weekRule ?? "calendar";
+  } else {
+    if (Object.hasOwn(args, "weekStartsOn")) transform.weekStartsOn = args.weekStartsOn;
+    if (Object.hasOwn(args, "weekRule")) transform.weekRule = args.weekRule;
+  }
   validateTimeUnitTransform(transform);
   return cloneAndFreeze(transform);
 }
@@ -96,6 +159,10 @@ export function floorUtcTimeUnit(timestamp, unit) {
   if (unit === "quarter") return utcBucket(year, Math.floor(month / 3) * 3);
   if (unit === "month") return utcBucket(year, month);
   const day = date.getUTCDate();
+  if (unit === "weekday") return date.getUTCDay();
+  if (unit === "week") {
+    throw new Error("floorUtcTimeUnit requires a calendar week policy.");
+  }
   if (unit === "day") return utcBucket(year, month, day);
   const hour = date.getUTCHours();
   if (unit === "hour") return utcBucket(year, month, day, hour);
@@ -104,6 +171,150 @@ export function floorUtcTimeUnit(timestamp, unit) {
   return Math.floor(timestamp / 1000) * 1000;
 }
 
+function weekdayFromDate(year, month, day) {
+  return new Date(utcTimestamp(year, month - 1, day)).getUTCDay();
+}
+
+export function calendarParts(timestamp, timeZone) {
+  if (!validTimestamp(timestamp)) {
+    throw new TypeError("Time-unit timestamp must represent a valid date.");
+  }
+  validateTimeZone(timeZone);
+  const parts = Object.fromEntries(formatter(timeZone).formatToParts(timestamp)
+    .filter(part => part.type !== "literal")
+    .map(part => [part.type, Number(part.value)]));
+  const millisecond = ((timestamp % 1000) + 1000) % 1000;
+  return {
+    year: parts.year,
+    month: parts.month,
+    day: parts.day,
+    hour: parts.hour,
+    minute: parts.minute,
+    second: parts.second,
+    millisecond,
+    weekday: weekdayFromDate(parts.year, parts.month, parts.day)
+  };
+}
+
+function civilTimestamp(parts) {
+  const timestamp = utcTimestamp(
+    parts.year,
+    parts.month - 1,
+    parts.day,
+    parts.hour ?? 0,
+    parts.minute ?? 0
+  ) + (parts.second ?? 0) * 1000 + (parts.millisecond ?? 0);
+  if (!validTimestamp(timestamp)) {
+    throw new RangeError("Time-unit calendar boundary is outside the supported Date range.");
+  }
+  return timestamp;
+}
+
+function sameCivil(left, right) {
+  return ["year", "month", "day", "hour", "minute", "second", "millisecond"]
+    .every(key => left[key] === right[key]);
+}
+
+function sameBucket(candidate, target, unit) {
+  const keys = unit === "year" ? ["year"]
+    : unit === "quarter" ? ["year", "month"]
+      : unit === "month" ? ["year", "month"]
+        : unit === "day" || unit === "week" ? ["year", "month", "day"]
+          : unit === "hour" ? ["year", "month", "day", "hour"]
+            : unit === "minute" ? ["year", "month", "day", "hour", "minute"]
+              : ["year", "month", "day", "hour", "minute", "second"];
+  if (unit === "quarter") {
+    return candidate.year === target.year &&
+      Math.floor((candidate.month - 1) / 3) === Math.floor((target.month - 1) / 3);
+  }
+  return keys.every(key => candidate[key] === target[key]);
+}
+
+export function resolveCalendarBoundary(localParts, timeZone, unit) {
+  const naive = civilTimestamp(localParts);
+  const offsets = new Set();
+  for (let delta = -72; delta <= 72; delta += 6) {
+    const sample = naive + delta * 3_600_000;
+    if (!validTimestamp(sample)) continue;
+    offsets.add(civilTimestamp(calendarParts(sample, timeZone)) - sample);
+  }
+  const exact = [];
+  const gaps = [];
+  for (const offset of offsets) {
+    const candidate = naive - offset;
+    if (!validTimestamp(candidate)) continue;
+    const roundTrip = calendarParts(candidate, timeZone);
+    if (sameCivil(roundTrip, localParts)) exact.push(candidate);
+    else if (sameBucket(roundTrip, localParts, unit) &&
+      civilTimestamp(roundTrip) >= naive) {
+      gaps.push({ candidate, local: civilTimestamp(roundTrip) });
+    }
+  }
+  if (exact.length > 0) return Math.min(...exact);
+  if (gaps.length > 0) {
+    gaps.sort((left, right) => left.local - right.local || left.candidate - right.candidate);
+    return gaps[0].candidate;
+  }
+  throw new RangeError(`Time-unit ${unit} boundary does not exist in time zone "${timeZone}".`);
+}
+
+function addCalendarDays(parts, delta) {
+  const date = new Date(utcTimestamp(parts.year, parts.month - 1, parts.day));
+  date.setUTCDate(date.getUTCDate() + delta);
+  return {
+    ...parts,
+    year: date.getUTCFullYear(),
+    month: date.getUTCMonth() + 1,
+    day: date.getUTCDate(),
+    weekday: date.getUTCDay()
+  };
+}
+
+function boundaryParts(parts, transform) {
+  const result = {
+    year: parts.year,
+    month: parts.month,
+    day: parts.day,
+    hour: parts.hour,
+    minute: parts.minute,
+    second: parts.second,
+    millisecond: 0
+  };
+  if (transform.unit === "year") Object.assign(result, { month: 1, day: 1, hour: 0, minute: 0, second: 0 });
+  else if (transform.unit === "quarter") Object.assign(result, {
+    month: Math.floor((parts.month - 1) / 3) * 3 + 1,
+    day: 1, hour: 0, minute: 0, second: 0
+  });
+  else if (transform.unit === "month") Object.assign(result, { day: 1, hour: 0, minute: 0, second: 0 });
+  else if (["day", "week"].includes(transform.unit)) Object.assign(result, { hour: 0, minute: 0, second: 0 });
+  else if (transform.unit === "hour") Object.assign(result, { minute: 0, second: 0 });
+  else if (transform.unit === "minute") result.second = 0;
+  if (transform.unit === "week") {
+    const distance = (parts.weekday - transform.weekStartsOn + 7) % 7;
+    return addCalendarDays(result, -distance);
+  }
+  return result;
+}
+
+export function bucketTime(timestamp, transform) {
+  const timeZone = transform.timeZone ?? "UTC";
+  if (transform.unit === "weekday") {
+    return timeZone === "UTC"
+      ? new Date(timestamp).getUTCDay()
+      : calendarParts(timestamp, timeZone).weekday;
+  }
+  if (timeZone === "UTC") {
+    if (transform.unit === "week") {
+      const date = new Date(timestamp);
+      const distance = (date.getUTCDay() - transform.weekStartsOn + 7) % 7;
+      return utcBucket(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate() - distance);
+    }
+    return floorUtcTimeUnit(timestamp, transform.unit);
+  }
+  const parts = calendarParts(timestamp, timeZone);
+  return resolveCalendarBoundary(boundaryParts(parts, transform), timeZone, transform.unit);
+}
+
 export function deriveTimeUnitRows(rows, transform) {
   validateTimeUnitTransform(transform);
   if (!Array.isArray(rows) || !rows.every(isPlainObject)) {
@@ -121,7 +332,7 @@ export function deriveTimeUnitRows(rows, transform) {
     const timestamp = normalizeTemporalValue(row[transform.field], transform.field, index, transform.temporalUnit);
     return {
       ...row,
-      [transform.as]: floorUtcTimeUnit(timestamp, transform.unit)
+      [transform.as]: bucketTime(timestamp, transform)
     };
   });
   return cloneAndFreeze(values);

package/src/grammar/transformTopology.js

@@ -2,6 +2,7 @@ const TRANSFORM_TOPOLOGY = Object.freeze({
   bin: Object.freeze({ facetTopology: "statistical" }),
   bin2d: Object.freeze({ facetTopology: "statistical" }),
   computed: Object.freeze({ facetTopology: "rowPreserving" }),
+  complete: Object.freeze({ facetTopology: "statistical" }),
   boxOutlier: Object.freeze({ facetTopology: "statistical" }),
   boxSummary: Object.freeze({ facetTopology: "statistical" }),
   density: Object.freeze({ facetTopology: "statistical" }),
@@ -11,10 +12,13 @@ const TRANSFORM_TOPOLOGY = Object.freeze({
   gradientProfile: Object.freeze({ facetTopology: "statistical" }),
   horizon: Object.freeze({ facetTopology: "statistical" }),
   interval: Object.freeze({ facetTopology: "statistical" }),
+  impute: Object.freeze({ facetTopology: "statistical" }),
   markFilter: Object.freeze({ provenanceTransparent: true }),
+  normalize: Object.freeze({ facetTopology: "statistical" }),
   regression: Object.freeze({ facetTopology: "statistical" }),
   summary: Object.freeze({ facetTopology: "statistical" }),
   stack: Object.freeze({ facetTopology: "statistical" }),
+  statisticalReference: Object.freeze({ facetTopology: "statistical" }),
   timeUnit: Object.freeze({ facetTopology: "rowPreserving" }),
   window: Object.freeze({ facetTopology: "statistical" })
 });