@uwdata/mosaic-sql 0.10.0 → 0.12.0

package/src/index.js

@@ -1,156 +1,58 @@
-export {
-  Ref,
-  asColumn,
-  asRelation,
-  all,
-  column,
-  relation
-} from './ref.js';
-
-export {
-  isSQLExpression,
-  isParamLike,
-  sql,
-  SQLExpression
-} from './expression.js';
-
-export {
-  desc
-} from './desc.js';
-
-export {
-  literal
-} from './literal.js';
-
-export {
-  and,
-  or,
-  not,
-  eq,
-  neq,
-  lt,
-  gt,
-  lte,
-  gte,
-  isBetween,
-  isNotBetween,
-  isDistinct,
-  isNotDistinct,
-  isNull,
-  isNotNull
-} from './operators.js';
-
-export {
-  agg,
-  argmax,
-  argmin,
-  arrayAgg,
-  avg,
-  corr,
-  count,
-  covariance,
-  covarPop,
-  entropy,
-  first,
-  kurtosis,
-  mean,
-  mad,
-  max,
-  median,
-  min,
-  mode,
-  last,
-  product,
-  quantile,
-  regrAvgX,
-  regrAvgY,
-  regrCount,
-  regrIntercept,
-  regrR2,
-  regrSXX,
-  regrSXY,
-  regrSYY,
-  regrSlope,
-  skewness,
-  stddev,
-  stddevPop,
-  stringAgg,
-  sum,
-  variance,
-  varPop
-} from './aggregates.js';
-
-export {
-  cast,
-  castDouble,
-  castInteger
-} from './cast.js';
-
-export {
-  epoch_ms,
-  dateBin,
-  dateMonth,
-  dateMonthDay,
-  dateDay
-} from './datetime.js';
-
-export {
-  regexp_matches,
-  contains,
-  prefix,
-  suffix,
-  lower,
-  upper,
-  length,
-  isNaN,
-  isFinite,
-  isInfinite
-} from './functions.js';
-
-export {
-  centroid,
-  centroidX,
-  centroidY,
-  geojson,
-  x,
-  y
-} from './spatial.js';
-
-export {
-  row_number,
-  rank,
-  dense_rank,
-  percent_rank,
-  cume_dist,
-  ntile,
-  lag,
-  lead,
-  first_value,
-  last_value,
-  nth_value
-} from './windows.js';
-
-export {
-  Query,
-  isQuery,
-  isDescribeQuery
-} from './Query.js';
-
-export {
-  toSQL,
-  literalToSQL
-} from './to-sql.js';
-
-export {
-  scaleTransform
-} from './scales.js';
-
-export { create } from './load/create.js';
+export { AggregateNode } from './ast/aggregate.js';
+export { BetweenOpNode, NotBetweenOpNode } from './ast/between-op.js'
+export { BinaryOpNode } from './ast/binary-op.js';
+export { CaseNode, WhenNode } from './ast/case.js';
+export { CastNode } from './ast/cast.js';
+export { ColumnParamNode } from './ast/column-param.js';
+export { ColumnRefNode, ColumnNameRefNode, isColumnRef } from './ast/column-ref.js';
+export { FragmentNode } from './ast/fragment.js';
+export { FromClauseNode } from './ast/from.js';
+export { FunctionNode } from './ast/function.js';
+export { InOpNode } from './ast/in-op.js';
+export { IntervalNode } from './ast/interval.js';
+export { LiteralNode } from './ast/literal.js';
+export { AndNode, OrNode } from './ast/logical-op.js';
+export { SQLNode, ExprNode, isNode } from './ast/node.js';
+export { OrderByNode } from './ast/order-by.js';
+export { ParamNode } from './ast/param.js';
+export { DescribeQuery, Query, SelectQuery, SetOperation, isDescribeQuery, isQuery, isSelectQuery } from './ast/query.js';
+export { SampleClauseNode } from './ast/sample.js';
+export { SelectClauseNode } from './ast/select.js';
+export { TableRefNode, isTableRef } from './ast/table-ref.js';
+export { UnaryOpNode, UnaryPosftixOpNode } from './ast/unary-op.js';
+export { VerbatimNode } from './ast/verbatim.js';
+export { WindowClauseNode, WindowDefNode, WindowFrameNode, WindowFunctionNode, WindowNode } from './ast/window.js';
+export { WithClauseNode } from './ast/with.js';
+
+export { argmax, argmin, arrayAgg, avg, corr, count, covariance, covarPop, entropy, first, kurtosis, mad, max, median, min, mode, last, product, quantile, regrAvgX, regrAvgY, regrCount, regrIntercept, regrR2, regrSXX, regrSXY, regrSYY, regrSlope, skewness, stddev, stddevPop, stringAgg, sum, variance, varPop } from './functions/aggregate.js';
+export { cond } from './functions/case.js';
+export { cast, float32, float64, int32 } from './functions/cast.js';
+export { column } from './functions/column.js';
+export { dateBin, dateMonth, dateMonthDay, dateDay, epoch_ms, interval } from './functions/datetime.js';
+export { literal } from './functions/literal.js';
+export { abs, ceil, exp, floor, greatest, isFinite, isInfinite, isNaN, least, ln, log, round, sign, sqrt, trunc } from './functions/numeric.js';
+export { and, or, not, isNull, isNotNull, bitNot, bitAnd, bitOr, bitLeft, bitRight, add, sub, mul, div, idiv, mod, pow, eq, neq, lt, gt, lte, gte, isDistinct, isNotDistinct, isBetween, isNotBetween, isIn } from './functions/operators.js';
+export { asc, desc } from './functions/order-by.js';
+export { geojson, x, y, centroid, centroidX, centroidY } from './functions/spatial.js';
+export { sql } from './functions/sql-template-tag.js';
+export { regexp_matches, contains, prefix, suffix, lower, upper, length } from './functions/string.js';
+export { cume_dist, dense_rank, first_value, lag, last_value, lead, nth_value, ntile, percent_rank, rank, row_number } from './functions/window.js';
+
+export { rewrite } from './visit/rewrite.js';
+export { collectAggregates, collectColumns, collectParams, isAggregateExpression } from './visit/visitors.js';
+export { walk } from './visit/walk.js';
+
+export { createTable, createSchema } from './load/create.js';
 export { loadExtension } from './load/extension.js';
-export {
-  loadCSV,
-  loadJSON,
-  loadObjects,
-  loadParquet,
-  loadSpatial
-} from './load/load.js';
+export { loadCSV, loadJSON, loadObjects, loadParquet, loadSpatial } from './load/load.js';
+
+export { bin1d } from './transforms/bin-1d.js';
+export { bin2d } from './transforms/bin-2d.js';
+export { binLinear1d } from './transforms/bin-linear-1d.js';
+export { binLinear2d } from './transforms/bin-linear-2d.js';
+export { lineDensity } from './transforms/line-density.js';
+export { m4 } from './transforms/m4.js';
+export { scaleTransform } from './transforms/scales.js';
+
+export { asLiteral, asNode, asTableRef, asVerbatim, over } from './util/ast.js';
+export { isParamLike } from './util/type-check.js';

package/src/load/create.js

@@ -1,6 +1,6 @@
-export function create(name, query, {
+export function createTable(name, query, {
   replace = false,
-  temp = true,
+  temp = false,
   view = false
 } = {}) {
   return 'CREATE'
@@ -10,3 +10,11 @@ export function create(name, query, {
     + (replace ? ' ' : ' IF NOT EXISTS ')
     + name + ' AS ' + query;
 }
+
+export function createSchema(name, {
+  strict = false
+} = {}) {
+  return 'CREATE SCHEMA '
+    + (strict ? '' : 'IF NOT EXISTS ')
+    + name;
+}

package/src/load/load.js

@@ -1,4 +1,4 @@
-import { create } from './create.js';
+import { createTable } from './create.js';
 import { sqlFrom } from './sql-from.js';
 
 export function load(method, tableName, fileName, options = {}, defaults = {}) {
@@ -7,7 +7,7 @@ export function load(method, tableName, fileName, options = {}, defaults = {}) {
   const read = `${method}('${fileName}'${params ? ', ' + params : ''})`;
   const filter = where ? ` WHERE ${where}` : '';
   const query = `SELECT ${select.join(', ')} FROM ${read}${filter}`;
-  return create(tableName, query, { view, temp, replace });
+  return createTable(tableName, query, { view, temp, replace });
 }
 
 export function loadCSV(tableName, fileName, options) {
@@ -15,7 +15,7 @@ export function loadCSV(tableName, fileName, options) {
 }
 
 export function loadJSON(tableName, fileName, options) {
-  return load('read_json', tableName, fileName, options, { auto_detect: true, json_format: 'auto' });
+  return load('read_json', tableName, fileName, options, { auto_detect: true, format: 'auto' });
 }
 
 export function loadParquet(tableName, fileName, options) {
@@ -51,7 +51,7 @@ export function loadObjects(tableName, data, options = {}) {
   const query = select.length === 1 && select[0] === '*'
     ? values
     : `SELECT ${select} FROM ${values}`;
-  return create(tableName, query, opt);
+  return createTable(tableName, query, opt);
 }
 
 function parameters(options) {

package/src/load/sql-from.js

@@ -1,11 +1,12 @@
-import { literalToSQL } from '../to-sql.js';
+import { asLiteral } from '../util/ast.js';
 
 /**
  * Create a SQL query that embeds the given data for loading.
- * @param {*} data The dataset
- * @param {object} [options] Loading options
- * @param {string[]|object} [options.columns] The columns to include
- * @returns {string} SQL query string to load data
+ * @param {*} data The dataset as an array of objects.
+ * @param {object} [options] Loading options.
+ * @param {string[]|object} [options.columns] The columns to include.
+ *   If not specified, the keys of the first data object are used.
+ * @returns {string} SQL query string to load data.
  */
 export function sqlFrom(data, {
   columns = Object.keys(data?.[0] || {})
@@ -22,7 +23,7 @@ export function sqlFrom(data, {
   }
   const subq = [];
   for (const datum of data) {
-    const sel = keys.map(k => `${literalToSQL(datum[k])} AS "${columns[k]}"`);
+    const sel = keys.map(k => `${asLiteral(datum[k])} AS "${columns[k]}"`);
     subq.push(`(SELECT ${sel.join(', ')})`);
   }
   return subq.join(' UNION ALL ');

package/src/transforms/bin-1d.js

@@ -0,0 +1,21 @@
+import { float64 } from '../functions/cast.js';
+import { mul, sub } from '../functions/operators.js';
+
+/**
+ * Compute binned values over a one-dimensional extent.
+ * @param {import('../types.js').ExprValue} x The expression to bin.
+ *  The expression must return numeric values. For example, to bin
+ *  datetime values, the input expression might map them to numeric
+ *  values such as milliseconds since the epoch.
+ * @param {number} lo The low value of the bin extent.
+ * @param {number} hi The high value of the bin extent.
+ * @param {number} bins The integer number of bins to use within the
+ *  defined binning extent.
+ * @param {boolean} [reverse] Flag indicating if bins should be
+ *  produced in reverse order from *hi* to *lo* (default `false`).
+ */
+export function bin1d(x, lo, hi, bins, reverse) {
+  const diff = reverse ? sub(hi, float64(x)) : sub(float64(x), lo);
+  const scale = hi === lo ? 0 : bins / (hi - lo);
+  return scale ? mul(diff, float64(scale)) : diff;
+}

package/src/transforms/bin-2d.js

@@ -0,0 +1,29 @@
+import { SelectQuery } from '../ast/query.js';
+import { int32 } from '../functions/cast.js';
+import { floor } from '../functions/numeric.js';
+import { add, mul } from '../functions/operators.js';
+
+/**
+ * Perform aggregation over a binned 2D domain. This method takes expressions
+ * for the (non-truncated) x and y bin values; these expressions should be
+ * in units of grid indices, but can contain fractional components. The
+ * resulting query performs grouping and aggregation over the binned domain,
+ * and uses a 2D integer bin index of the form (xbin + num_xbins * ybin).
+ * @param {SelectQuery} q The input query. The FROM and WHERE clauses should
+ *  be added to the query separately, either before or after this method.
+ * @param {import('../types.js').ExprValue} xp The x bin expression.
+ * @param {import('../types.js').ExprValue} yp The y bin expression.
+ * @param {Record<string, import('../types.js').ExprValue>} aggs Named
+ *  aggregate expressions over bins.
+ * @param {number} xn The number of bins along the x dimension
+ * @param {string[]} groupby Group by expressions.
+ * @returns {SelectQuery} The input query, with binning expressions added.
+ */
+export function bin2d(q, xp, yp, aggs, xn, groupby) {
+  return q
+    .select({
+      index: add(int32(floor(xp)), mul(int32(floor(yp)), xn)),
+      ...aggs
+    })
+    .groupby('index', groupby);
+}

package/src/transforms/bin-linear-1d.js

@@ -0,0 +1,26 @@
+import { Query } from '../ast/query.js';
+import { sum } from '../functions/aggregate.js';
+import { int32 } from '../functions/cast.js';
+import { floor } from '../functions/numeric.js';
+import { add, mul, neq, sub } from '../functions/operators.js';
+
+/**
+ * Perform linear binning in one dimension.
+ * @param {import('../ast/query.js').SelectQuery} query The base query to bin.
+ * @param {import('../types.js').ExprValue} x The expression to bin.
+ * @param {import('../types.js').ExprValue} [weight] The expression to weight by.
+ * @returns {Query}
+ */
+export function binLinear1d(query, x, weight) {
+  const w = weight ? (x => mul(x, weight)) : (x => x);
+  const p0 = floor(x);
+  const p1 = add(p0, 1);
+  return Query
+    .from(Query.unionAll(
+      query.clone().select({ i: int32(p0), w: w(sub(p1, x)) }),
+      query.clone().select({ i: int32(p1), w: w(sub(x, p0)) })
+    ))
+    .select({ index: 'i', density: sum('w') })
+    .groupby('index')
+    .having(neq('density', 0));
+}

package/src/transforms/bin-linear-2d.js

@@ -0,0 +1,71 @@
+import { Query, SelectQuery } from '../ast/query.js';
+import { sum } from '../functions/aggregate.js';
+import { int32 } from '../functions/cast.js';
+import { floor } from '../functions/numeric.js';
+import { add, mul, neq, sub } from '../functions/operators.js';
+
+/**
+ * Identity function.
+ * @template T
+ * @param {T} x
+ * @returns {T}
+ */
+function identity(x) {
+  return x;
+}
+
+/**
+ * Compute densities over a 2D domain using linear binning. The weight of
+ * each data point is linearly distributed over adjacent bins, providing
+ * a better base for subsequent kernel density estimation. This method takes
+ * expressions for the (non-truncated) x and y bin values; these expressions
+ * should be in units of grid indices, but can contain fractional components.
+ * @param {SelectQuery} q The input query. The FROM and WHERE clauses should
+ *  be added to the query separately, before this method is invoked.
+ * @param {import('../types.js').ExprValue} xp The x grid bin expression
+ * @param {import('../types.js').ExprValue} yp The y grid bin expression
+ * @param {import('../types.js').ExprValue | undefined} weight Point weights.
+ * @param {number} xn The number of x grid bins.
+ * @param {string[]} groupby Group by expressions.
+ * @returns {SelectQuery} A linear binning query for bin `index` and
+ *  aggregate `density` columns, in addition to any group by expressions.
+ */
+export function binLinear2d(q, xp, yp, weight, xn, groupby) {
+
+  const w = weight ? x => mul(x, weight) : identity;
+
+  /**
+   * @param {import('../types.js').ExprValue} i
+   * @param {import('../types.js').ExprValue} w
+   */
+  const subq = (i, w) => q.clone().select({ xp, yp, i, w });
+  /**
+   * @param {import('../types.js').ExprValue} x
+   * @param {import('../types.js').ExprValue} y
+   */
+  const index = (x, y) => add(x, mul(y, xn));
+
+  const xu = int32(floor(xp));
+  const yu = int32(floor(yp));
+  const xv = add(xu, 1);
+  const yv = add(yu, 1);
+  const xpu = sub(xp, xu);
+  const xvp = sub(xv, xp);
+  const ypu = sub(yp, yu);
+  const yvp = sub(yv, yp);
+
+  return Query
+    .from(Query.unionAll(
+      // grid[xu + yu * xn] += (xv - xp) * (yv - yp) * wi
+      subq(index(xu, yu), w(mul(xvp, yvp))),
+      // grid[xu + yv * xn] += (xv - xp) * (yp - yu) * wi
+      subq(index(xu, yv), w(mul(xvp, ypu))),
+      // grid[xv + yu * xn] += (xp - xu) * (yv - yp) * wi
+      subq(index(xv, yu), w(mul(xpu, yvp))),
+      // grid[xv + yv * xn] += (xp - xu) * (yp - yu) * wi
+      subq(index(xv, yv), w(mul(xpu, ypu)))
+    ))
+    .select({ index: 'i', density: sum('w') }, groupby)
+    .groupby('index', groupby)
+    .having(neq('density', 0));
+}

package/src/transforms/line-density.js

@@ -0,0 +1,113 @@
+import { Query, SelectQuery } from '../ast/query.js';
+import { count, max, sum } from '../functions/aggregate.js';
+import { int32 } from '../functions/cast.js';
+import { abs, floor, greatest, round, sign } from '../functions/numeric.js';
+import { add, div, gt, isNull, lt, lte, mul, or, sub } from '../functions/operators.js';
+import { asc } from '../functions/order-by.js';
+import { sql } from '../functions/sql-template-tag.js';
+import { lead } from '../functions/window.js';
+import { over } from '../util/ast.js';
+
+/**
+ * Compute line segment densities over a gridded 2D domain. The returned
+ * query uses multiple subqueries (CTEs) to identify line segment end point
+ * pairs, perform line rasterization in-database, normalize arc lengths,
+ * and then sum results for all line series to produce a density map.
+ * Based on Moritz and Fisher's work: https://arxiv.org/abs/1808.06019
+ * @param {SelectQuery} q The base query over the data.
+ * @param {import('../types.js').ExprValue} x Bin expression for x dimension.
+ *  Provides gridded x coordinates, potentially with a fractional component.
+ * @param {import('../types.js').ExprValue} y Bin expression for x dimension.
+ *  Provides gridded y coordinates, potentially with a fractional component.
+ * @param {string[]} z Group by columns that segment data into individual line
+ *  series. An empty array indicates there is only a single line series.
+ * @param {number} xn The number of grid bins for the x dimension.
+ * @param {number} yn The number of grid bins for the y dimension.
+ * @param {string[]} [groupby] Additional group by expressions. Separate
+ *  line density maps are created for each of these groups.
+ * @param {boolean} [normalize=true] Flag toggling approximate arc-length
+ *  normalization to improve accuracy and reduce artifacts (default `true`).
+ * @returns {SelectQuery}
+ */
+export function lineDensity(
+  q, x, y, z, xn, yn,
+  groupby = [], normalize = true
+) {
+  // select x, y points binned to the grid
+  q.select({
+    x: int32(floor(x)),
+    y: int32(floor(y))
+  });
+
+  // select line segment end point pairs
+  // retain only segments within the grid region
+  const groups = groupby.concat(z);
+  const pairs = Query
+    .from(q)
+    .select(groups, {
+      x0: 'x',
+      y0: 'y',
+      dx: sub(lead('x').over('sw'), 'x'),
+      dy: sub(lead('y').over('sw'), 'y')
+    })
+    .window({
+      sw: over().partitionby(groups).orderby(asc('x'))
+    })
+    .qualify([
+      or(lt('x0', xn), lt(add('x0', 'dx'), xn)),
+      or(lt('y0', yn), lt(add('y0', 'dy'), yn)),
+      or(gt('x0', 0), gt(add('x0', 'dx'), 0)),
+      or(gt('y0', 0), gt(add('y0', 'dy'), 0))
+    ]);
+
+  // create indices to join against for rasterization
+  // generate the maximum number of indices needed
+  const num = Query
+    .select({ x: greatest(max(abs('dx')), max(abs('dy'))) })
+    .from('pairs');
+  const indices = Query.select({
+    i: int32(sql`UNNEST(range((${num})))`)
+  });
+
+  // rasterize line segments
+  const raster = Query.unionAll(
+    Query
+      .select(groups, {
+        x: add('x0', 'i'),
+        y: add('y0', int32(round(div(mul('i', 'dy'), 'dx'))))
+      })
+      .from('pairs', 'indices')
+      .where([lte(abs('dy'), abs('dx')), lt('i', abs('dx'))]),
+    Query
+      .select(groups, {
+        x: add('x0', int32(round(div(mul(mul(sign('dy'), 'i'), 'dx'), 'dy')))),
+        y: add('y0', mul(sign('dy'), 'i'))
+      })
+      .from('pairs', 'indices')
+      .where([gt(abs('dy'), abs('dx')), lt('i', abs('dy'))]),
+    Query
+      .select(groups, { x: 'x0', y: 'y0' })
+      .from('pairs')
+      .where(isNull('dx'))
+  );
+
+  // filter raster, normalize columns for each series
+  const points = Query
+    .from('raster')
+    .select(groups, 'x', 'y',
+      normalize
+        ? { w: div(1, count().partitionby(['x'].concat(groups))) }
+        : null
+    )
+    .where([lte(0, 'x'), lt('x', xn), lte(0, 'y'), lt('y', yn)]);
+
+  // sum normalized, rasterized series into output grids
+  return Query
+    .with({ pairs, indices, raster, points })
+    .from('points')
+    .select(groupby, {
+      index: add('x', mul('y', int32(xn))),
+      density: normalize ? sum('w') : count()
+    })
+    .groupby('index', groupby);
+}

package/src/transforms/m4.js

@@ -0,0 +1,38 @@
+import { Query } from '../ast/query.js';
+import { argmax, argmin, max, min } from '../functions/aggregate.js';
+import { int32 } from '../functions/cast.js';
+import { floor } from '../functions/numeric.js';
+
+/**
+ * M4 is an optimization for value-preserving time-series aggregation
+ * (https://www.vldb.org/pvldb/vol7/p797-jugel.pdf). This implementation uses
+ * an efficient version with a single scan and the aggregate functions
+ * argmin and argmax, following https://arxiv.org/pdf/2306.03714.pdf.
+ * This method can bin along either the *x* or *y* dimension, as determined
+ * by the caller-provided *bin* expression.
+ * @param {import('../types.js').FromExpr} input The base query or table.
+ * @param {import('../types.js').ExprValue} bin An expression that maps
+ *  time-series values to fractional pixel positions.
+ * @param {string} x The x dimension column name.
+ * @param {string} y The y dimension column name.
+ * @param {import('../ast/node.js').ExprNode[]} [groups] Additional
+ *  groupby columns, for example for faceted charts.
+ * @returns {Query} The resulting M4 query.
+ */
+export function m4(input, bin, x, y, groups = []) {
+  const pixel = int32(floor(bin));
+
+  const q = (sel) => Query
+    .from(input)
+    .select(sel)
+    .groupby(pixel, groups);
+
+  return Query
+    .union(
+      q([{ [x]: min(x), [y]: argmin(y, x) }, ...groups]),
+      q([{ [x]: max(x), [y]: argmax(y, x) }, ...groups]),
+      q([{ [x]: argmin(x, y), [y]: min(y) }, ...groups]),
+      q([{ [x]: argmax(x, y), [y]: max(y) }, ...groups])
+    )
+    .orderby(groups, x);
+}