@nshiab/simple-data-analysis-core 2.1.12 → 2.1.14

package/src/methods/addSummaryRows.ts

@@ -0,0 +1,209 @@
+import assertColumnsExist from "../helpers/assertColumnsExist.ts";
+import getStatExpression, { allStats, type Stat } from "../helpers/getStatExpression.ts";
+import keepNumericalColumns from "../helpers/keepNumericalColumns.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+import queueOp from "../helpers/queueOp.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+type StatRowOperation = Exclude<Stat, "count">;
+type StatRow = StatRowOperation | { stat: StatRowOperation; label?: string };
+type NormalizedStatRow = { stat: StatRowOperation; label: string };
+type AddSummaryRowsOptions = {
+  stats?: StatRow | StatRow[];
+  position?: "top" | "bottom";
+};
+
+const allStatRowOperations = allStats.filter((stat): stat is StatRowOperation => stat !== "count");
+
+export default function addSummaryRows(
+  simpleTable: SimpleTable,
+  columns: string | string[],
+  labelColumn: string,
+  options: AddSummaryRowsOptions = {},
+) {
+  options = structuredClone(options);
+  if (
+    options.position !== undefined &&
+    options.position !== "top" &&
+    options.position !== "bottom"
+  ) {
+    throw new Error(
+      `addSummaryRows() options.position must be "top" or "bottom". Received ${JSON.stringify(
+        options.position,
+      )}.`,
+    );
+  }
+  const selectedColumns = columns === "all" ? "all" : stringToArray(columns);
+  if (selectedColumns !== "all" && selectedColumns.length === 0) {
+    throw new Error('addSummaryRows() requires at least one column or "all".');
+  }
+  if (selectedColumns !== "all") {
+    const duplicates = selectedColumns.filter(
+      (column, index) => selectedColumns.indexOf(column) !== index,
+    );
+    if (duplicates.length > 0) {
+      throw new Error(
+        `addSummaryRows() received duplicate column${
+          new Set(duplicates).size === 1 ? "" : "s"
+        } ${[...new Set(duplicates)].map(quoteIdentifier).join(", ")}.`,
+      );
+    }
+    if (selectedColumns.includes(labelColumn)) {
+      throw new Error(
+        `addSummaryRows() label column ${quoteIdentifier(labelColumn)} cannot also be summarized.`,
+      );
+    }
+  }
+
+  const statRows = normalizeStatRows(options.stats);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "addSummaryRows()",
+    parameters: { columns, labelColumn, options },
+    needsSchema: true,
+    values: statRows.map((row) => row.label),
+    buildSelect: (input, schema) =>
+      addSummaryRowsSelect(
+        input,
+        schema,
+        selectedColumns,
+        labelColumn,
+        statRows,
+        options.position ?? "bottom",
+      ),
+  });
+}
+
+function normalizeStatRows(stats?: StatRow | StatRow[]): NormalizedStatRow[] {
+  const rows =
+    stats === undefined ? [...allStatRowOperations] : Array.isArray(stats) ? stats : [stats];
+  if (rows.length === 0) {
+    throw new Error(
+      "addSummaryRows() stats cannot be an empty array. Omit stats to add every supported stat.",
+    );
+  }
+
+  return rows.map((row) => {
+    if (typeof row === "string") {
+      assertStat(row);
+      return { stat: row, label: row };
+    }
+    if (row === null || typeof row !== "object") {
+      throw new Error(
+        "addSummaryRows() stats must contain stat names or objects with a stat property.",
+      );
+    }
+    assertStat(row.stat);
+    if (row.label !== undefined && typeof row.label !== "string") {
+      throw new Error("addSummaryRows() stat labels must be strings.");
+    }
+    return { stat: row.stat, label: row.label ?? row.stat };
+  });
+}
+
+function assertStat(stat: string): asserts stat is StatRowOperation {
+  if (!(allStatRowOperations as readonly string[]).includes(stat)) {
+    throw new Error(
+      `addSummaryRows() stat ${JSON.stringify(stat)} is not supported. Use ${allStatRowOperations
+        .map((value) => JSON.stringify(value))
+        .join(", ")}.`,
+    );
+  }
+}
+
+function addSummaryRowsSelect(
+  input: string,
+  schema: TableSchema,
+  columns: "all" | string[],
+  labelColumn: string,
+  statRows: NormalizedStatRow[],
+  position: "top" | "bottom",
+): string {
+  const method = "addSummaryRows()";
+  const tableColumns = Object.keys(schema);
+  assertColumnsExist(schema, [labelColumn], method);
+  if (schema[labelColumn] !== "VARCHAR") {
+    throw new Error(
+      `${method} label column ${quoteIdentifier(
+        labelColumn,
+      )} must have type VARCHAR, but has type ${schema[labelColumn]}.`,
+    );
+  }
+
+  const numericalColumns = keepNumericalColumns(schema);
+  const selectedColumns = columns === "all" ? numericalColumns : columns;
+  if (selectedColumns.length === 0) {
+    throw new Error(
+      `${method} could not find any numeric columns. Select numeric columns explicitly or convert columns to numeric types first.`,
+    );
+  }
+  if (columns !== "all") {
+    assertColumnsExist(schema, selectedColumns, method);
+    const numericalColumnSet = new Set(numericalColumns);
+    const nonNumericalColumns = selectedColumns.filter((column) => !numericalColumnSet.has(column));
+    if (nonNumericalColumns.length > 0) {
+      throw new Error(
+        `${method} requires numeric columns, but ${nonNumericalColumns
+          .map((column) => `${quoteIdentifier(column)} has type ${schema[column]}`)
+          .join(", ")}. Convert ${
+          nonNumericalColumns.length === 1 ? "it" : "them"
+        } to a numeric type first.`,
+      );
+    }
+  }
+
+  const selectedColumnSet = new Set(selectedColumns);
+  const kindColumn = unusedColumn(schema, "_sda_stat_kind");
+  const orderColumn = unusedColumn({ ...schema, [kindColumn]: "INTEGER" }, "_sda_stat_order");
+  const projectedColumns = tableColumns.map(quoteIdentifier).join(", ");
+  const dataKind = position === "top" ? 1 : 0;
+  const statKind = position === "top" ? 0 : 1;
+  const branches = [
+    `SELECT ${projectedColumns}, ${dataKind} AS ${quoteIdentifier(
+      kindColumn,
+    )}, ROW_NUMBER() OVER () AS ${quoteIdentifier(orderColumn)} FROM ${input}`,
+    ...statRows.map((row, index) => {
+      const projections = tableColumns.map((column) => {
+        if (column === labelColumn) {
+          return `CAST(? AS VARCHAR) AS ${quoteIdentifier(column)}`;
+        }
+        if (!selectedColumnSet.has(column)) {
+          return `NULL AS ${quoteIdentifier(column)}`;
+        }
+        const expression = getStatExpression(
+          row.stat,
+          schema[column],
+          quoteIdentifier(column),
+          undefined,
+        );
+        if (expression === null) {
+          throw new Error(
+            `${method} stat ${JSON.stringify(
+              row.stat,
+            )} cannot be computed for column ${quoteIdentifier(
+              column,
+            )} with type ${schema[column]}.`,
+          );
+        }
+        return `${expression} AS ${quoteIdentifier(column)}`;
+      });
+      return `SELECT ${projections.join(", ")}, ${statKind} AS ${quoteIdentifier(
+        kindColumn,
+      )}, ${index} AS ${quoteIdentifier(orderColumn)} FROM ${input}`;
+    }),
+  ];
+
+  return `SELECT ${projectedColumns}
+FROM (${branches.join("\nUNION ALL\n")}) AS ${quoteIdentifier("stats")}
+ORDER BY ${quoteIdentifier(kindColumn)}, ${quoteIdentifier(orderColumn)}`;
+}
+
+function unusedColumn(schema: TableSchema, base: string): string {
+  let column = base;
+  while (schema[column] !== undefined) {
+    column += "_";
+  }
+  return column;
+}

package/src/methods/addVertexCount.ts

@@ -0,0 +1,30 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function addVertexCount(
+  simpleTable: SimpleTable,
+  newColumn: string,
+  options: { column?: string } = {},
+) {
+  options = { ...options };
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "addVertexCount()",
+    parameters: { newColumn, options },
+    needsSchema: true,
+    needsSpatial: true,
+    buildSelect: (input, types) => {
+      assertNewColumns(types, [newColumn], "addVertexCount()");
+      const column =
+        typeof options.column === "string"
+          ? options.column
+          : findGeoColumnFromSchema(types, "addVertexCount()", simpleTable.name);
+      return `SELECT *, CAST(ST_NPoints(${quoteIdentifier(
+        column,
+      )}) AS BIGINT) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/addWordCount.ts

@@ -0,0 +1,19 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+
+export default function addWordCount(simpleTable: SimpleTable, column: string, newColumn: string) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "addWordCount()",
+    parameters: { column, newColumn },
+    needsSchema: true,
+    buildSelect: (input, types) => {
+      assertNewColumns(types, [newColumn], "addWordCount()");
+      return `SELECT *, ARRAY_LENGTH(REGEXP_EXTRACT_ALL(${quoteIdentifier(
+        column,
+      )}, '[^[:space:]]+')) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/aggregateGeo.ts

@@ -0,0 +1,105 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import findGeoColumn from "../helpers/findGeoColumn.ts";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import resolveOutputTable from "../helpers/resolveOutputTable.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function aggregateGeo(
+  simpleTable: SimpleTable,
+  method: "union" | "intersection",
+  options: {
+    column?: string;
+    by?: string | string[];
+    outputTable?: string | boolean;
+  } = {},
+): SimpleTable {
+  options = structuredClone(options);
+  options.outputTable = resolveOutputTable(simpleTable, options.outputTable);
+
+  if (typeof options.outputTable === "string") {
+    // The output table instance is created at call time so it can be
+    // returned synchronously and chained on right away.
+    const outputTable = simpleTable.sdb.newTable(options.outputTable);
+    queueOp(outputTable, {
+      kind: "barrier",
+      method: "aggregateGeo()",
+      parameters: { method, options },
+      execute: async () => {
+        const column =
+          typeof options.column === "string"
+            ? options.column
+            : await findGeoColumn(simpleTable, "aggregateGeo()");
+        await queryDB(
+          simpleTable,
+          `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable.name)} AS ${aggregateGeoSelect(
+            `${quoteIdentifier(simpleTable.name)}`,
+            column,
+            method,
+            options,
+          )}`,
+          mergeOptions(simpleTable, {
+            table: outputTable.name,
+            method: "aggregateGeo()",
+            parameters: { column, method, options },
+          }),
+        );
+      },
+    });
+    return outputTable;
+  }
+
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "aggregateGeo()",
+    parameters: { method, options },
+    needsSchema: true,
+    needsSpatial: true,
+    buildSelect: (input, types) => {
+      const column =
+        typeof options.column === "string"
+          ? options.column
+          : findGeoColumnFromSchema(types, "aggregateGeo()", simpleTable.name);
+      return aggregateGeoSelect(input, column, method, options);
+    },
+  });
+  return simpleTable;
+}
+
+function aggregateGeoSelect(
+  input: string,
+  column: string,
+  method: "union" | "intersection",
+  options: {
+    by?: string | string[];
+  } = {},
+) {
+  const categoriesOptions = options.by ?? [];
+  const by = stringToArray(categoriesOptions);
+
+  let query = `SELECT${
+    by.length > 0 ? ` ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")},` : ""
+  }`;
+
+  if (method === "union") {
+    query += ` CASE WHEN ST_IsEmpty(ST_Union_Agg(${quoteIdentifier(
+      column,
+    )})) THEN NULL ELSE ST_Union_Agg(${quoteIdentifier(column)}) END AS ${quoteIdentifier(column)}`;
+  } else if (method === "intersection") {
+    query += ` ST_Intersection_Agg(${quoteIdentifier(column)}) AS ${quoteIdentifier(column)}`;
+  } else {
+    throw new Error(`aggregateGeo() received an unknown method: ${String(method)}.`);
+  }
+
+  query += `\nFROM ${input}`;
+
+  if (by.length > 0) {
+    query += `\nGROUP BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}`;
+    query += `\nORDER BY ${by.map((d) => `${quoteIdentifier(d)} ASC`).join(", ")}`;
+  }
+
+  return query;
+}

package/src/methods/area.ts

@@ -0,0 +1,36 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function area(
+  simpleTable: SimpleTable,
+  newColumn: string,
+  options: { unit?: "m2" | "km2"; column?: string; decimals?: number } = {},
+) {
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "area()",
+    parameters: { newColumn, options },
+    needsSchema: true,
+    needsSpatial: true,
+    buildSelect: (input, types) => {
+      assertNewColumns(types, [newColumn], "area()");
+      const column =
+        typeof options.column === "string"
+          ? options.column
+          : findGeoColumnFromSchema(types, "area()", simpleTable.name);
+      let expression = `ST_Area_Spheroid(${quoteIdentifier(column)}) ${
+        options.unit === "km2" ? "/ 1000000" : ""
+      }`;
+      if (typeof options.decimals === "number") {
+        expression = `ROUND(${expression}, ${options.decimals})`;
+      }
+      return `SELECT *, CAST(${expression} AS DOUBLE) AS ${quoteIdentifier(
+        newColumn,
+      )} FROM ${input}`;
+    },
+  });
+}

package/src/methods/bins.ts

@@ -0,0 +1,150 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function bins(
+  simpleTable: SimpleTable,
+  column: string,
+  interval: number,
+  startColumn: string,
+  endColumn: string,
+  options: {
+    startValue?: number;
+  } = {},
+) {
+  // This validation does not need the database, so it stays at call time.
+  if (!Number.isFinite(interval) || interval <= 0) {
+    throw new Error("bins() interval must be a finite number greater than 0.");
+  }
+  if (typeof startColumn !== "string" || typeof endColumn !== "string") {
+    throw new Error("bins() requires start and end output column names.");
+  }
+  if (startColumn.toLowerCase() === endColumn.toLowerCase()) {
+    throw new Error("bins() start and end output column names must be distinct.");
+  }
+  if (options.startValue !== undefined && !Number.isFinite(options.startValue)) {
+    throw new Error("bins() startValue must be a finite number.");
+  }
+  // The intervals depend on the minimum and maximum values of the data, so
+  // bins can't be expressed as a single SELECT over its input: it executes
+  // as a barrier.
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "bins()",
+    parameters: { column, interval, startColumn, endColumn, options },
+    execute: async () => {
+      await queryDB(
+        simpleTable,
+        await binsQuery(simpleTable, column, interval, startColumn, endColumn, options),
+        mergeOptions(simpleTable, {
+          table: simpleTable.name,
+          method: "bins()",
+          parameters: {
+            column,
+            interval,
+            startColumn,
+            endColumn,
+            options,
+          },
+        }),
+      );
+    },
+  });
+}
+
+async function binsQuery(
+  SimpleTable: SimpleTable,
+  column: string,
+  interval: number,
+  startColumn: string,
+  endColumn: string,
+  options: {
+    startValue?: number;
+  } = {},
+) {
+  // A SELECT *, expr AS col colliding with an existing column would be
+  // silently renamed by DuckDB (col -> col_1) instead of erroring, unlike
+  // the ALTER TABLE ADD this used to run.
+  const types = await SimpleTable.getTypes();
+  const existingColumns = Object.keys(types);
+  assertNewColumns(
+    types,
+    [startColumn, endColumn].map(
+      (column) =>
+        existingColumns.find((existing) => existing.toLowerCase() === column.toLowerCase()) ??
+        column,
+    ),
+    "bins()",
+  );
+
+  // The minimum and maximum are computed in one scan instead of one
+  // getMin/getMax query each.
+  const minMax = await queryDB(
+    SimpleTable,
+    `SELECT MIN(${quoteIdentifier(column)}) AS "min", MAX(${quoteIdentifier(
+      column,
+    )}) AS "max" FROM ${quoteIdentifier(SimpleTable.name)}`,
+    mergeOptions(SimpleTable, {
+      table: SimpleTable.name,
+      method: "bins()",
+      parameters: { column, interval, startColumn, endColumn, options },
+      returnData: true,
+    }),
+  );
+  const minValue = minMax?.[0]?.min;
+  if (typeof minValue !== "number" || !Number.isFinite(minValue)) {
+    throw new Error(`minValue of ${column} is not a finite number`);
+  }
+
+  let startValue: number;
+  if (typeof options.startValue === "number") {
+    if (options.startValue > minValue) {
+      throw new Error(
+        `startValue ${options.startValue} can't be greater than minValue ${minValue}`,
+      );
+    }
+    startValue = options.startValue;
+  } else {
+    startValue = minValue;
+  }
+
+  const maxValue = minMax?.[0]?.max;
+  if (typeof maxValue !== "number" || !Number.isFinite(maxValue)) {
+    throw new Error(`maxValue of ${column} is not a finite number`);
+  }
+  const starts: string[] = [];
+  const ends: string[] = [];
+  // Include the bin beginning at the maximum, even if floating-point division
+  // places an exact boundary slightly below its integer index.
+  const lastBin = Math.ceil((maxValue - startValue) / interval);
+  if (!Number.isSafeInteger(lastBin)) {
+    throw new Error("bins() range contains too many intervals.");
+  }
+  for (let i = 0; i <= lastBin; i++) {
+    // Let SQL evaluate the boundaries from numeric literals, avoiding repeated
+    // floating-point addition in JavaScript for fractional intervals.
+    // Decimal indices also prevent integer-only expressions from overflowing
+    // DuckDB's inferred INT32 type when an end exceeds that range.
+    const start = `(${startValue} + ${i}.0 * ${interval})`;
+    const end = `(${startValue} + ${i + 1}.0 * ${interval})`;
+    const condition = `WHEN ${quoteIdentifier(column)} >= ${start} AND ${quoteIdentifier(
+      column,
+    )} < ${end}`;
+    starts.push(`${condition} THEN ${start}`);
+    ends.push(`${condition} THEN ${end}`);
+  }
+
+  // Keep both outputs numeric, including fractional boundaries. Null source
+  // values match no interval and yield null in both output columns.
+  const query = `CREATE OR REPLACE TABLE ${quoteIdentifier(SimpleTable.name)} AS
+    SELECT *,
+      CAST(CASE ${starts.join("\n")} END AS DOUBLE) AS ${quoteIdentifier(startColumn)},
+      CAST(CASE ${ends.join("\n")} END AS DOUBLE) AS ${quoteIdentifier(endColumn)}
+    FROM ${quoteIdentifier(SimpleTable.name)}`;
+
+  return query;
+}