@nshiab/simple-data-analysis-core 2.1.12 → 2.1.13

package/src/methods/normalize.ts

@@ -0,0 +1,65 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function normalize(
+  simpleTable: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: {
+    by?: string | string[];
+    decimals?: number;
+    range?: [number, number];
+  } = {},
+) {
+  if (
+    options.range !== undefined &&
+    (!Array.isArray(options.range) ||
+      options.range.length !== 2 ||
+      !Number.isFinite(options.range[0]) ||
+      !Number.isFinite(options.range[1]) ||
+      options.range[0] >= options.range[1])
+  ) {
+    throw new Error(
+      "normalize() options.range must contain two finite numbers in ascending order.",
+    );
+  }
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "normalize()",
+    parameters: { column, newColumn, options },
+    needsSchema: true,
+    buildSelect: (input, schema) => {
+      assertNewColumns(schema, [newColumn], "normalize()");
+
+      const by = options.by ? stringToArray(options.by) : [];
+      const partition =
+        by.length > 0 ? `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}` : "";
+
+      const normalizedQuery = `(${quoteIdentifier(column)} - MIN(${quoteIdentifier(
+        column,
+      )}) OVER(${partition}))
+    /
+    (MAX(${quoteIdentifier(column)}) OVER(${partition}) - MIN(${quoteIdentifier(
+      column,
+    )}) OVER(${partition}))`;
+      const [rangeMin, rangeMax] = options.range ?? [0, 1];
+      const scaledQuery =
+        options.range === undefined
+          ? normalizedQuery
+          : `${rangeMin} + (${normalizedQuery}) * ${rangeMax - rangeMin}`;
+
+      return `SELECT *, (
+        ${
+          typeof options.decimals === "number"
+            ? `ROUND(${scaledQuery}, ${options.decimals})`
+            : scaledQuery
+        }
+        ) AS ${quoteIdentifier(newColumn)},
+    FROM ${input}`;
+    },
+  });
+}

package/src/methods/normalizeString.ts

@@ -0,0 +1,36 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import queueOp from "../helpers/queueOp.ts";
+
+export default function normalizeString(
+  simpleTable: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: { stripPunctuation?: boolean } = {},
+): void {
+  const { stripPunctuation = true } = options;
+
+  const accentRemoved = `strip_accents(${quoteIdentifier(column)})`;
+
+  const lowercased = `lower(${accentRemoved})`;
+
+  const punctuationRemoved = stripPunctuation
+    ? `regexp_replace(${lowercased}, '[[:punct:]]', '', 'g')`
+    : lowercased;
+
+  const normalizedClause = `trim(regexp_replace(${punctuationRemoved}, '\\s+', ' ', 'g'))`;
+
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "normalizeString()",
+    parameters: { column, newColumn, stripPunctuation },
+    needsSchema: false,
+    buildSelect: (input) =>
+      `SELECT *,
+      CASE
+        WHEN ${quoteIdentifier(column)} IS NULL THEN NULL
+        ELSE ${normalizedClause}
+      END AS ${quoteIdentifier(newColumn)}
+    FROM ${input}`,
+  });
+}

package/src/methods/normalizeVector.ts

@@ -0,0 +1,202 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import foldIdentifier from "../helpers/foldIdentifier.ts";
+import prepareNumericFeatures from "../helpers/prepareNumericFeatures.ts";
+import publishPreparedColumns from "../helpers/publishPreparedColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+
+type Options = NonNullable<Parameters<SimpleTable["normalizeVector"]>[2]>;
+
+export default function normalizeVector(
+  table: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: Options = {},
+): void {
+  if (
+    options.normalization !== undefined &&
+    options.normalization !== "dimensionMinMax" &&
+    options.normalization !== "rowL2"
+  ) {
+    throw new Error(
+      'normalizeVector() options.normalization must be "dimensionMinMax" or "rowL2".',
+    );
+  }
+  const settings = structuredClone(options);
+  queueOp(table, {
+    kind: "barrier",
+    method: "normalizeVector()",
+    parameters: { column, newColumn, options: settings },
+    execute: () => execute(table, column, newColumn, settings),
+  });
+}
+
+async function execute(
+  table: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: Options,
+): Promise<void> {
+  const normalization = options.normalization ?? "dimensionMinMax";
+  const overwrite = foldIdentifier(column) === foldIdentifier(newColumn);
+  if (!overwrite) {
+    const sourceColumns = Object.keys(await table.getTypes());
+    if (sourceColumns.some((name) => foldIdentifier(name) === foldIdentifier(newColumn))) {
+      throw new Error(
+        `normalizeVector() cannot create ${quoteIdentifier(
+          newColumn,
+        )} because that column already exists. Remove it first or choose a different name.`,
+      );
+    }
+  }
+  const prepared = await prepareNumericFeatures(
+    table,
+    { kind: "vector", column },
+    { method: "normalizeVector()" },
+  );
+  const connection = table.connection!;
+  const q = quoteIdentifier;
+  const suffix = crypto.randomUUID().replaceAll("-", "");
+  const statsName = `__sda_normalize_vector_${suffix}_stats`;
+  const resultName = `__sda_normalize_vector_${suffix}_result`;
+  const dimension = q(`__sda_dimension_${suffix}`);
+  const minimum = q(`__sda_min_${suffix}`);
+  const maximum = q(`__sda_max_${suffix}`);
+  const normalized = `__sda_normalized_${suffix}`;
+
+  try {
+    if (prepared.rowCount === 0) {
+      throw new Error("normalizeVector() requires at least one row; the dataset is empty.");
+    }
+    const sourceColumn = prepared.sourceColumns.find(
+      (name) => foldIdentifier(name) === foldIdentifier(column),
+    )!;
+    if (normalization === "rowL2") {
+      // Scale before squaring so extreme finite magnitudes do not overflow
+      // or underflow the norm. A zero vector stays zero.
+      await connection.run(
+        `CREATE TEMP TABLE ${q(resultName)} AS
+         WITH magnitudes AS (
+           SELECT ${q(prepared.rowIdColumn)}, ${q(prepared.vectorColumn)} AS vec,
+             list_max(list_transform(${q(prepared.vectorColumn)}, value -> abs(value))) AS magnitude
+           FROM ${q(prepared.relation)}
+         ), scaled AS (
+           SELECT *, list_transform(vec, value -> value / nullif(magnitude, 0)) AS unit_scale
+           FROM magnitudes
+         ), norms AS (
+           SELECT *, sqrt(list_sum(list_transform(unit_scale, value -> value * value))) AS norm
+           FROM scaled
+         )
+         SELECT ${q(prepared.rowIdColumn)},
+           (CASE WHEN magnitude = 0 THEN vec::DOUBLE[]
+             ELSE list_transform(unit_scale, value -> value / norm)
+           END)::DOUBLE[${prepared.dimensions}] AS ${q(normalized)}
+         FROM norms`,
+      );
+    } else {
+      await connection.run(
+        `CREATE TEMP TABLE ${q(statsName)} AS
+         SELECT ordinal::BIGINT AS ${dimension}, min(value)::DOUBLE AS ${minimum},
+           max(value)::DOUBLE AS ${maximum}
+         FROM ${q(prepared.relation)} p,
+           UNNEST(p.${q(prepared.vectorColumn)}::DOUBLE[]) WITH ORDINALITY AS values(value, ordinal)
+         GROUP BY ordinal`,
+      );
+      const invalidStats = Number(
+        (
+          await connection.runAndReadAll(
+            `SELECT count(*) FROM ${q(statsName)}
+           WHERE NOT isfinite(${minimum}) OR NOT isfinite(${maximum})`,
+          )
+        ).getRowsJS()[0][0],
+      );
+      if (invalidStats > 0) {
+        throw new Error(
+          `normalizeVector() found ${invalidStats} dimension${
+            invalidStats === 1 ? "" : "s"
+          } with non-finite aggregate values after conversion to DOUBLE.`,
+        );
+      }
+      const constantDimensions = (
+        await connection.runAndReadAll(
+          `SELECT ${dimension} FROM ${q(statsName)}
+         WHERE ${minimum} = ${maximum} ORDER BY ${dimension}`,
+        )
+      )
+        .getRowsJS()
+        .map((row) => Number(row[0]));
+      if (constantDimensions.length > 0) {
+        throw new Error(
+          `normalizeVector() cannot scale constant dimension${
+            constantDimensions.length === 1 ? "" : "s"
+          } ${constantDimensions.join(", ")} in column ${q(
+            sourceColumn,
+          )} because the range is zero. Remove ${
+            constantDimensions.length === 1 ? "that dimension" : "those dimensions"
+          } before normalizing. Conversion to DOUBLE can also collapse distinct large integers or decimals at its precision limit.`,
+        );
+      }
+
+      const value = `p.${q(prepared.vectorColumn)}[s.${dimension}]`;
+      const min = `s.${minimum}`;
+      const max = `s.${maximum}`;
+      // Dividing both sides by the larger magnitude when a range crosses zero
+      // avoids overflowing max - min for extreme finite DOUBLE values.
+      const scaled = `CASE
+        WHEN ${min} < 0 AND ${max} > 0 AND -${min} >= ${max}
+          THEN ((${value} / -${min}) + 1.0) / ((${max} / -${min}) + 1.0)
+        WHEN ${min} < 0 AND ${max} > 0
+          THEN ((${value} / ${max}) + (-${min} / ${max})) /
+            (1.0 + (-${min} / ${max}))
+        ELSE (${value} - ${min}) / (${max} - ${min})
+      END`;
+      await connection.run(
+        `CREATE TEMP TABLE ${q(resultName)} AS
+         SELECT p.${q(prepared.rowIdColumn)},
+           list(${scaled} ORDER BY s.${dimension})::DOUBLE[${prepared.dimensions}]
+             AS ${q(normalized)}
+         FROM ${q(prepared.relation)} p CROSS JOIN ${q(statsName)} s
+         GROUP BY p.${q(prepared.rowIdColumn)}`,
+      );
+    }
+    const invalidResults = Number(
+      (
+        await connection.runAndReadAll(
+          `SELECT count(*) FROM ${q(resultName)}
+         WHERE ${q(normalized)} IS NULL
+           OR list_count(${q(normalized)}) != ${prepared.dimensions}
+           OR NOT list_bool_and(list_transform(${q(normalized)}, value -> isfinite(value)))
+           OR list_min(${q(normalized)}) < ${normalization === "rowL2" ? -1 : 0}
+           OR list_max(${q(normalized)}) > 1`,
+        )
+      ).getRowsJS()[0][0],
+    );
+    if (invalidResults > 0) {
+      throw new Error(
+        `normalizeVector() produced non-finite or out-of-range values for ${invalidResults} row${
+          invalidResults === 1 ? "" : "s"
+        }. Rescale the source values before normalizing.`,
+      );
+    }
+
+    await publishPreparedColumns(table, prepared, {
+      method: "normalizeVector()",
+      parameters: { column, newColumn, options },
+      result: {
+        relation: resultName,
+        rowIdColumn: prepared.rowIdColumn,
+      },
+      outputs: [
+        {
+          name: overwrite ? sourceColumn : newColumn,
+          expression: `r.${q(normalized)}`,
+          replace: overwrite ? sourceColumn : undefined,
+        },
+      ],
+    });
+  } finally {
+    await connection.run(`DROP TABLE IF EXISTS ${q(resultName)}`);
+    await connection.run(`DROP TABLE IF EXISTS ${q(statsName)}`);
+    await prepared.cleanup();
+  }
+}

package/src/methods/outliersIQR.ts

@@ -0,0 +1,55 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function outliersIQR(
+  simpleTable: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: {
+    by?: string | string[];
+  } = {},
+) {
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "outliersIQR()",
+    parameters: { column, newColumn, options },
+    needsSchema: false,
+    buildSelect: (input) => outliersIQRSelect(input, column, newColumn, options),
+  });
+}
+
+function outliersIQRSelect(
+  input: string,
+  column: string,
+  newColumn: string,
+  options: {
+    by?: string | string[];
+  } = {},
+) {
+  const by = options.by ? stringToArray(options.by).map((d) => `${quoteIdentifier(d)}`) : [];
+
+  const partition = by.length > 0 ? `PARTITION BY ${by.join(", ")}` : "";
+  const quantile = (fraction: number) =>
+    `IF(COUNT(${quoteIdentifier(column)}) OVER (${partition}) % 2 = 0,
+      quantile_disc(${quoteIdentifier(column)}, ${fraction}) OVER (${partition}),
+      quantile_cont(${quoteIdentifier(column)}, ${fraction}) OVER (${partition})
+    )`;
+
+  // q1/q3 are computed as window values (one pass) instead of a per-category
+  // CTE joined back through a correlated subquery per row.
+  return `SELECT * EXCLUDE ("_sda_q1", "_sda_q3"), CASE
+        WHEN ${quoteIdentifier(column)} > "_sda_q3" + ("_sda_q3" - "_sda_q1") * 1.5
+          OR ${quoteIdentifier(column)} < "_sda_q1" - ("_sda_q3" - "_sda_q1") * 1.5
+        THEN TRUE
+        ELSE FALSE
+    END AS ${quoteIdentifier(newColumn)}
+    FROM (
+        SELECT *,
+            ${quantile(0.25)} AS "_sda_q1",
+            ${quantile(0.75)} AS "_sda_q3"
+        FROM ${input}
+    ) "_sda"`;
+}

package/src/methods/pad.ts

@@ -0,0 +1,88 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function pad(
+  simpleTable: SimpleTable,
+  columns: string | string[],
+  length: number,
+  options: { side?: "left" | "right"; character?: string } = {},
+) {
+  if (!Number.isFinite(length) || !Number.isInteger(length) || length < 0) {
+    throw new Error("pad() length must be a finite integer greater than or equal to 0.");
+  }
+
+  columns = Array.isArray(columns) ? [...columns] : columns;
+  options = structuredClone(options);
+  const columnList = stringToArray(columns);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "pad()",
+    parameters: { columns, length, options },
+    needsSchema: true,
+    preservesSchema: true,
+    values: padValues(columnList, length, options),
+    buildSelect: (input, schema) => padSelect(input, schema, columnList, length, options),
+  });
+}
+
+function padSelect(
+  input: string,
+  schema: TableSchema,
+  columns: string[],
+  length: number,
+  options: { side?: "left" | "right"; character?: string },
+): string {
+  for (const column of columns) {
+    if (schema[column] !== "VARCHAR") {
+      throw new Error(
+        `The column ${quoteIdentifier(column)} is of type ${
+          schema[column]
+        }. The pad() method only works with string (VARCHAR) columns. Please convert the column to string first with the .convert() method.`,
+      );
+    }
+  }
+
+  const usedNames = new Set(Object.keys(schema));
+  const overflowAliases = columns.map((_, index) => {
+    let alias = `__sda_pad_overflow_${index}`;
+    while (usedNames.has(alias)) {
+      alias += "_";
+    }
+    usedNames.add(alias);
+    return alias;
+  });
+  const func = options.side === "right" ? "RPAD" : "LPAD";
+  const counts = columns
+    .map(
+      (column, index) =>
+        `COUNT(*) FILTER (WHERE LENGTH(${quoteIdentifier(
+          column,
+        )}) > ${length}) OVER () AS ${quoteIdentifier(overflowAliases[index])}`,
+    )
+    .join(", ");
+  const replacements = columns
+    .map((column, index) => {
+      const alias = quoteIdentifier(overflowAliases[index]);
+      return `CASE WHEN ${alias} > 0 THEN error(CONCAT(?, CAST(${alias} AS VARCHAR), ?)) ELSE ${func}(${quoteIdentifier(
+        column,
+      )}, ${length}, ?) END AS ${quoteIdentifier(column)}`;
+    })
+    .join(", ");
+  const excluded = overflowAliases.map(quoteIdentifier).join(", ");
+  return `SELECT * EXCLUDE (${excluded}) REPLACE (${replacements})
+FROM (
+  SELECT *, ${counts} FROM ${input}
+) AS ${quoteIdentifier("__sda_pad_source")}`;
+}
+
+function padValues(columns: string[], length: number, options: { character?: string }): string[] {
+  const character = options.character ?? "0";
+  return columns.flatMap((column) => [
+    `The column ${quoteIdentifier(column)} has `,
+    ` string(s) exceeding the target length of ${length}. The pad() method does not truncate. Shorten the strings first or use a larger target length.`,
+    character,
+  ]);
+}