@nshiab/simple-data-analysis-core 2.1.12 → 2.1.14

package/src/helpers/prepareNumericFeatures.ts

@@ -0,0 +1,354 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import foldIdentifier from "./foldIdentifier.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+const NUMERIC_SCALAR_TYPE =
+  /^(?:TINYINT|SMALLINT|INTEGER|BIGINT|HUGEINT|UTINYINT|USMALLINT|UINTEGER|UBIGINT|UHUGEINT|BIGNUM|FLOAT|REAL|DOUBLE|DECIMAL\(\d+,\d+\))$/;
+
+/** Returns whether a DuckDB type is a scalar numeric type. */
+export function isNumericScalarType(type: string): boolean {
+  return NUMERIC_SCALAR_TYPE.test(type);
+}
+
+export type NumericFeatureInput =
+  | { kind: "scalars"; columns: string[] }
+  | { kind: "vector"; column: string };
+
+export type PreparedNumericFeatures = {
+  /** Raw temporary-relation name. Quote it before embedding it in SQL. */
+  relation: string;
+  /** Raw private row-identity column name. */
+  rowIdColumn: string;
+  /** Raw private `DOUBLE[d]` feature column name. */
+  vectorColumn: string;
+  dimensions: number;
+  rowCount: number;
+  /** Source column names in their original order and spelling. */
+  sourceColumns: string[];
+  sourceTemporary: boolean;
+  /** Drops all scratch state owned by this preparation. Safe to call twice. */
+  cleanup: () => Promise<void>;
+};
+
+/**
+ * Prepares scalar columns or a vector column as a private `DOUBLE[d]` feature
+ * vector while retaining every source column in DuckDB. Converting large
+ * integers or exact decimals to DOUBLE can lose precision. The source columns
+ * remain unchanged, so callers can publish results from the returned snapshot.
+ *
+ * The private row identity is positional and distinguishes duplicate rows. A
+ * caller owns the returned relation and must invoke `cleanup()` in `finally`.
+ * Preparation failures clean up their own scratch state.
+ */
+export default async function prepareNumericFeatures(
+  table: SimpleTable,
+  input: NumericFeatureInput,
+  options: {
+    method: string;
+    rejectNulls?: boolean;
+    rejectNonFinite?: boolean;
+  },
+): Promise<PreparedNumericFeatures> {
+  const rejectNulls = options.rejectNulls ?? true;
+  const rejectNonFinite = options.rejectNonFinite ?? true;
+  const types = await table.getTypes();
+  // Object.keys(types) sorts integer-like identifiers, which would reorder
+  // source columns when algorithms publish from this snapshot.
+  const sourceColumns = await table.getColumns();
+  const resolveColumn = (requested: string): string => {
+    const resolved = sourceColumns.find(
+      (column) => foldIdentifier(column) === foldIdentifier(requested),
+    );
+    if (resolved === undefined) {
+      throw new Error(
+        `${options.method} could not find column ${quoteIdentifier(
+          requested,
+        )}. Available columns: ${sourceColumns.map(quoteIdentifier).join(", ")}.`,
+      );
+    }
+    return resolved;
+  };
+
+  let selectedColumns: string[];
+  let vectorType: { elementType: string; fixedDimensions?: number } | undefined;
+  if (input.kind === "scalars") {
+    if (input.columns.length === 0) {
+      throw new Error(`${options.method} requires at least one numeric scalar column.`);
+    }
+    const duplicate = findDuplicate(input.columns);
+    if (duplicate !== undefined) {
+      throw new Error(
+        `${options.method} received duplicate input column ${quoteIdentifier(
+          duplicate,
+        )}. Each numeric feature column must be listed once.`,
+      );
+    }
+    selectedColumns = input.columns.map(resolveColumn);
+    const nonNumeric = selectedColumns.filter((column) => !isNumericScalarType(types[column]));
+    if (nonNumeric.length > 0) {
+      throw new Error(
+        `${options.method} requires numeric scalar columns. The following input columns are not numeric scalars: ${nonNumeric
+          .map((column) => `${quoteIdentifier(column)} (${types[column]})`)
+          .join(", ")}. Convert them to numeric scalar types first.`,
+      );
+    }
+  } else {
+    const column = resolveColumn(input.column);
+    selectedColumns = [column];
+    vectorType = parseNumericVectorType(types[column]);
+    if (vectorType === undefined) {
+      throw new Error(
+        `${options.method} requires a one-dimensional numeric LIST or ARRAY column, but ${quoteIdentifier(
+          column,
+        )} has type ${types[column]}.`,
+      );
+    }
+  }
+
+  const connection = table.connection!;
+  const source = quoteIdentifier(table.name);
+  const suffix = crypto.randomUUID().replaceAll("-", "");
+  const relation = `__sda_features_${suffix}`;
+  const rowIdColumn = `__sda_row_${suffix}`;
+  const vectorColumn = `__sda_vector_${suffix}`;
+  const preparedRelation = quoteIdentifier(relation);
+  let cleaned = false;
+  const cleanup = async () => {
+    if (cleaned) return;
+    await connection.run(`DROP TABLE IF EXISTS ${preparedRelation}`);
+    cleaned = true;
+  };
+
+  try {
+    const tableDetails = (
+      await connection.runAndReadAll(
+        `SELECT table_oid, temporary FROM duckdb_tables()
+       WHERE translate(table_name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') = $1
+         AND schema_name = current_schema()
+         AND database_name IN (current_database(), 'temp')
+       ORDER BY temporary DESC LIMIT 1`,
+        [foldIdentifier(table.name)],
+      )
+    ).getRowsJS()[0];
+    if (tableDetails === undefined) {
+      throw new Error(`${options.method} requires a materialized source table.`);
+    }
+    const sourceTemporary = Boolean(tableDetails[1]);
+    const rowCount = Number(
+      (await connection.runAndReadAll(`SELECT count(*) FROM ${source}`)).getRowsJS()[0][0],
+    );
+
+    let dimensions: number;
+    let vectorExpression: string;
+    if (input.kind === "scalars") {
+      dimensions = selectedColumns.length;
+      await rejectInvalidScalarRows(connection, source, selectedColumns, types, options.method, {
+        rejectNulls,
+        rejectNonFinite,
+      });
+      vectorExpression = `[${selectedColumns
+        .map((column) => castToDouble(quoteIdentifier(column), types[column]))
+        .join(", ")}]::DOUBLE[${dimensions}]`;
+    } else {
+      const column = selectedColumns[0];
+      const quotedColumn = quoteIdentifier(column);
+      const emptyRows = Number(
+        (
+          await connection.runAndReadAll(
+            `SELECT count(*) FROM ${source}
+           WHERE ${quotedColumn} IS NOT NULL AND len(${quotedColumn}) = 0`,
+          )
+        ).getRowsJS()[0][0],
+      );
+      if (emptyRows > 0) {
+        throw new Error(
+          `${options.method} found ${emptyRows} row${
+            emptyRows === 1 ? "" : "s"
+          } with an empty vector in column ${quoteIdentifier(
+            column,
+          )}. Vectors must have a nonzero dimension.`,
+        );
+      }
+      await rejectInvalidVectorRows(
+        connection,
+        source,
+        column,
+        vectorType!.elementType,
+        options.method,
+        { rejectNulls, rejectNonFinite },
+      );
+
+      if (vectorType?.fixedDimensions !== undefined) {
+        dimensions = vectorType.fixedDimensions;
+      } else {
+        const dimensionRows = (
+          await connection.runAndReadAll(
+            `SELECT len(${quotedColumn}) AS dimensions
+           FROM ${source} WHERE ${quotedColumn} IS NOT NULL LIMIT 1`,
+          )
+        ).getRowsJS();
+        if (dimensionRows.length === 0) {
+          throw new Error(
+            `${options.method} could not determine a vector dimension for ${quoteIdentifier(
+              column,
+            )}. Provide at least one non-null, nonempty vector.`,
+          );
+        }
+        dimensions = Number(dimensionRows[0][0]);
+      }
+
+      const distinctDimensions = Number(
+        (
+          await connection.runAndReadAll(
+            `SELECT count(DISTINCT len(${quotedColumn})) FROM ${source}
+           WHERE ${quotedColumn} IS NOT NULL`,
+          )
+        ).getRowsJS()[0][0],
+      );
+      if (distinctDimensions > 1) {
+        throw new Error(
+          `${options.method} requires vectors with equal dimensions, but column ${quoteIdentifier(
+            column,
+          )} contains ${distinctDimensions} different lengths.`,
+        );
+      }
+      vectorExpression =
+        vectorType!.elementType === "BIGNUM"
+          ? `list_transform(${quotedColumn}, value -> ${castToDouble(
+              "value",
+              "BIGNUM",
+            )})::DOUBLE[${dimensions}]`
+          : `${quotedColumn}::DOUBLE[${dimensions}]`;
+    }
+
+    await connection.run(
+      `CREATE TEMP TABLE ${preparedRelation} AS
+       SELECT *, row_number() OVER () - 1 AS ${quoteIdentifier(
+         rowIdColumn,
+       )}, ${vectorExpression} AS ${quoteIdentifier(vectorColumn)}
+       FROM ${source}`,
+    );
+
+    return {
+      relation,
+      rowIdColumn,
+      vectorColumn,
+      dimensions,
+      rowCount,
+      sourceColumns,
+      sourceTemporary,
+      cleanup,
+    };
+  } catch (error) {
+    await cleanup();
+    throw error;
+  }
+}
+
+function parseNumericVectorType(
+  type: string,
+): { elementType: string; fixedDimensions?: number } | undefined {
+  const match = /^(.+)\[(\d*)\]$/.exec(type);
+  if (match === null || !isNumericScalarType(match[1])) return undefined;
+  const elementType = match[1];
+  if (match[2] === "") return { elementType };
+  const fixedDimensions = Number(match[2]);
+  return fixedDimensions > 0 ? { elementType, fixedDimensions } : undefined;
+}
+
+function findDuplicate(columns: string[]): string | undefined {
+  const seen = new Set<string>();
+  for (const column of columns) {
+    const folded = foldIdentifier(column);
+    if (seen.has(folded)) return column;
+    seen.add(folded);
+  }
+  return undefined;
+}
+
+async function rejectInvalidScalarRows(
+  connection: SimpleTable["connection"],
+  source: string,
+  columns: string[],
+  types: { [column: string]: string },
+  method: string,
+  validation: { rejectNulls: boolean; rejectNonFinite: boolean },
+) {
+  const predicates = columns.map((column) => {
+    const value = quoteIdentifier(column);
+    const converted = castToDouble(value, types[column], true);
+    const checks: string[] = [];
+    if (validation.rejectNulls) checks.push(`${value} IS NULL`);
+    if (validation.rejectNonFinite) {
+      checks.push(
+        `(${value} IS NOT NULL AND (${converted} IS NULL OR NOT isfinite(${converted})))`,
+      );
+    }
+    return checks.length === 0 ? "FALSE" : `(${checks.join(" OR ")})`;
+  });
+  if (predicates.every((predicate) => predicate === "FALSE")) return;
+  const aliases = columns.map((_, index) => `invalid_${index}`);
+  const query = `SELECT
+    count(*) FILTER (WHERE ${predicates.join(" OR ")}) AS invalid_rows,
+    ${predicates
+      .map((predicate, index) => `count(*) FILTER (WHERE ${predicate}) AS ${aliases[index]}`)
+      .join(", ")}
+    FROM ${source}`;
+  const row = (await connection.runAndReadAll(query)).getRowsJS()[0];
+  const invalidRows = Number(row[0]);
+  if (invalidRows === 0) return;
+  const affected = columns.flatMap((column, index) => {
+    const count = Number(row[index + 1]);
+    return count === 0
+      ? []
+      : [`${quoteIdentifier(column)} (${count} row${count === 1 ? "" : "s"})`];
+  });
+  throw new Error(
+    `${method} requires finite, non-null numeric features. Found ${invalidRows} invalid row${
+      invalidRows === 1 ? "" : "s"
+    }. Affected columns: ${affected.join(", ")}.`,
+  );
+}
+
+async function rejectInvalidVectorRows(
+  connection: SimpleTable["connection"],
+  source: string,
+  column: string,
+  elementType: string,
+  method: string,
+  validation: { rejectNulls: boolean; rejectNonFinite: boolean },
+) {
+  const vector = quoteIdentifier(column);
+  const checks: string[] = [];
+  if (validation.rejectNulls) {
+    checks.push(`${vector} IS NULL`);
+    checks.push(`(${vector} IS NOT NULL AND list_count(${vector}) != len(${vector}))`);
+  }
+  if (validation.rejectNonFinite) {
+    checks.push(
+      `(${vector} IS NOT NULL AND COALESCE(NOT list_bool_and(list_transform(${vector}, value -> value IS NULL OR COALESCE(isfinite(${castToDouble(
+        "value",
+        elementType,
+        true,
+      )}), FALSE))), FALSE))`,
+    );
+  }
+  if (checks.length === 0) return;
+  const invalidRows = Number(
+    (
+      await connection.runAndReadAll(`SELECT count(*) FROM ${source} WHERE ${checks.join(" OR ")}`)
+    ).getRowsJS()[0][0],
+  );
+  if (invalidRows === 0) return;
+  throw new Error(
+    `${method} requires finite, non-null vector values. Column ${quoteIdentifier(
+      column,
+    )} has ${invalidRows} invalid row${invalidRows === 1 ? "" : "s"}.`,
+  );
+}
+
+/** BIGNUM's direct DOUBLE conversion can throw even inside DuckDB TRY_CAST. */
+function castToDouble(value: string, type: string, tryCast = false): string {
+  const convertible = type === "BIGNUM" ? `CAST(${value} AS VARCHAR)` : value;
+  return `${tryCast ? "TRY_CAST" : "CAST"}(${convertible} AS DOUBLE)`;
+}

package/src/helpers/prepareUmapLayoutOptions.ts

@@ -0,0 +1,38 @@
+import fitUmapCurve from "./fitUmapCurve.ts";
+
+// Resolve and validate settings before database work, fitting the curve once.
+export default function prepareUmapLayoutOptions(
+  options: {
+    epochs?: number;
+    seed?: number;
+    minDistance?: number;
+    learningRate?: number;
+    negativeSamples?: number;
+  } = {},
+) {
+  const {
+    epochs = 200,
+    seed = 42,
+    minDistance = 0.1,
+    learningRate = 1,
+    negativeSamples = 5,
+  } = options;
+  for (const [key, value] of Object.entries({ epochs, negativeSamples })) {
+    if (!Number.isSafeInteger(value) || value < 1) {
+      throw new Error(`${key} must be a positive safe integer.`);
+    }
+  }
+  if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) {
+    throw new Error("seed must be an integer between 0 and 4294967295.");
+  }
+  if (!Number.isFinite(learningRate) || learningRate <= 0) {
+    throw new Error("learningRate must be finite and positive.");
+  }
+  return {
+    epochs,
+    seed,
+    learningRate,
+    negativeSamples,
+    curve: fitUmapCurve(minDistance),
+  };
+}

package/src/helpers/prepareValueFilters.ts

@@ -0,0 +1,22 @@
+import type { DuckDBValue } from "@duckdb/node-api";
+import toDuckDBValue from "./toDuckDBValue.ts";
+
+export type ValueFilter = {
+  column: string;
+  values: DuckDBValue[];
+  includesNull: boolean;
+};
+
+/** Converts column filter values to bindable values while tracking nulls. */
+export default function prepareValueFilters(columnsAndValues: {
+  [key: string]: unknown;
+}): ValueFilter[] {
+  return Object.entries(columnsAndValues).map(([column, value]) => {
+    const boundValues = (Array.isArray(value) ? value : [value]).map(toDuckDBValue);
+    return {
+      column,
+      values: boundValues.filter((boundValue) => boundValue !== null),
+      includesNull: boundValues.some((boundValue) => boundValue === null),
+    };
+  });
+}

package/src/helpers/prettyDuration.ts

@@ -0,0 +1,164 @@
+/**
+ * Formats a duration into a human-readable string, breaking it down into years, months, days, hours, minutes, seconds, and milliseconds. This function is useful for displaying elapsed time in a user-friendly format, such as for performance logging, task completion times, or general time tracking.
+ *
+ * The function can calculate the duration from a given start time to the current time, or between a specified start and end time. It provides options for logging the output directly to the console and adding custom prefixes or suffixes to the formatted string. Note that for simplicity, months are approximated as 30 days and years as 365 days.
+ *
+ * @example
+ * ```ts
+ * // A starting Date somewhere in your code.
+ * const startDate = new Date(); // or Date.now()
+ *
+ * // When you want to know the elapsed duration, pass the start date.
+ * const duration = prettyDuration(startDate);
+ * console.log(duration); // Returns something like "22 days, 6 h, 3 min, 15 sec, 3 ms"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // If you want to console.log it directly, set the `log` option to `true`.
+ * // This will print the duration to the console and also return the string.
+ * const startDateForLog = new Date();
+ * // ... some operations ...
+ * prettyDuration(startDateForLog, { log: true });
+ * ```
+ *
+ * @example
+ * ```ts
+ * // You can also use a prefix and/or suffix for the output string.
+ * const startDateWithPrefixSuffix = new Date();
+ * // ... some operations ...
+ * prettyDuration(startDateWithPrefixSuffix, { log: true, prefix: "Elapsed time: ", suffix: " (Main function)" });
+ * // Returns and logs something like "Total duration: 3 min, 15 sec, 3 ms (Main function)"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // If you want to format the duration between two specific dates, use the `end` option.
+ * const start = new Date("2024-01-01T17:00:00Z");
+ * const end = new Date("2024-01-23T23:03:15Z");
+ * const specificDuration = prettyDuration(start, { end });
+ * console.log(specificDuration); // Returns "22 days, 6 h, 3 min, 15 sec, 0 ms"
+ * ```
+ *
+ * @param start - The starting point of the duration. This can be a `Date` object or a Unix timestamp (number of milliseconds since epoch).
+ *
+ * @param options - Optional settings to customize the duration formatting and output.
+ *   @param options.log - If `true`, the formatted duration string will be logged to the console. Defaults to `false`.
+ *   @param options.end - The ending point of the duration. This can be a `Date` object or a Unix timestamp. If omitted, the current time (`Date.now()`) will be used as the end point.
+ *   @param options.prefix - A string to prepend to the formatted duration string (e.g., "Elapsed time: ").
+ *   @param options.suffix - A string to append to the formatted duration string (e.g., " (Task completed)").
+ *
+ * @returns A human-readable string representing the duration.
+ *
+ * @category Formatting
+ */
+export default function prettyDuration(
+  start: Date | number,
+  options: {
+    log?: boolean;
+    end?: Date | number;
+    prefix?: string;
+    suffix?: string;
+  } = {},
+): string {
+  if (start instanceof Date) {
+    start = start.getTime();
+  }
+
+  let end;
+  if (options.end instanceof Date) {
+    end = options.end.getTime();
+  } else if (typeof options.end === "number") {
+    end = options.end;
+  } else {
+    end = Date.now();
+  }
+
+  const differenceInMs = end - start;
+
+  let prettyDuration = "";
+
+  if (differenceInMs < 1000) {
+    // Less than a second
+    prettyDuration = `${differenceInMs} ms`;
+  } else if (differenceInMs < 60_000) {
+    // Less than a minute
+    const sec = Math.floor(differenceInMs / 1000);
+    const ms = differenceInMs % 1000;
+    prettyDuration = `${sec} sec, ${ms} ms`;
+  } else if (differenceInMs < 3_600_000) {
+    // Less than an hour
+    const min = Math.floor(differenceInMs / 60_000);
+    const remainingMs = differenceInMs % 60_000;
+    const sec = Math.floor(remainingMs / 1000);
+    const ms = remainingMs % 1000;
+    prettyDuration = `${min} min, ${sec} sec, ${ms} ms`;
+  } else if (differenceInMs < 86_400_000) {
+    // Less than a day
+    const hours = Math.floor(differenceInMs / 3_600_000);
+    const remainingMsAfterHours = differenceInMs % 3_600_000;
+    const min = Math.floor(remainingMsAfterHours / 60_000);
+    const remainingMsAfterMinutes = remainingMsAfterHours % 60_000;
+    const sec = Math.floor(remainingMsAfterMinutes / 1000);
+    const ms = remainingMsAfterMinutes % 1000;
+    prettyDuration = `${hours} h, ${min} min, ${sec} sec, ${ms} ms`;
+  } else if (differenceInMs < 2_592_000_000) {
+    // Less than a month
+    const days = Math.floor(differenceInMs / 86_400_000);
+    const remainingMsAfterDays = differenceInMs % 86_400_000;
+    const hours = Math.floor(remainingMsAfterDays / 3_600_000);
+    const remainingMsAfterHours = remainingMsAfterDays % 3_600_000;
+    const min = Math.floor(remainingMsAfterHours / 60_000);
+    const remainingMsAfterMin = remainingMsAfterHours % 60_000;
+    const sec = Math.floor(remainingMsAfterMin / 1000);
+    const ms = remainingMsAfterMin % 1000;
+    prettyDuration = `${days} ${
+      days <= 1 ? "day" : "days"
+    }, ${hours} h, ${min} min, ${sec} sec, ${ms} ms`;
+  } else if (differenceInMs < 31_536_000_000) {
+    // Less than a year
+    const months = Math.floor(differenceInMs / 2_592_000_000);
+    const remainingMsAfterMonths = differenceInMs % 2_592_000_000;
+    const days = Math.floor(remainingMsAfterMonths / 86_400_000);
+    const remainingMsAfterDays = differenceInMs % 86_400_000;
+    const hours = Math.floor(remainingMsAfterDays / 3_600_000);
+    const remainingMsAfterHours = remainingMsAfterDays % 3_600_000;
+    const min = Math.floor(remainingMsAfterHours / 60_000);
+    const remainingMsAfterMin = remainingMsAfterHours % 60_000;
+    const sec = Math.floor(remainingMsAfterMin / 1000);
+    const ms = remainingMsAfterMin % 1000;
+    prettyDuration = `${months} ${months <= 1 ? "month" : "months"}, ${days} ${
+      days <= 1 ? "day" : "days"
+    }, ${hours} h, ${min} min, ${sec} sec, ${ms} ms`;
+  } else {
+    // More than a year
+    const years = Math.floor(differenceInMs / 31_536_000_000);
+    const remainingMsAfterYears = differenceInMs % 31_536_000_000;
+    const months = Math.floor(remainingMsAfterYears / 2_592_000_000);
+    const remainingMsAfterMonths = differenceInMs % 2_592_000_000;
+    const days = Math.floor(remainingMsAfterMonths / 86_400_000);
+    const remainingMsAfterDays = differenceInMs % 86_400_000;
+    const hours = Math.floor(remainingMsAfterDays / 3_600_000);
+    const remainingMsAfterHours = remainingMsAfterDays % 3_600_000;
+    const min = Math.floor(remainingMsAfterHours / 60_000);
+    const remainingMsAfterMin = remainingMsAfterHours % 60_000;
+    const sec = Math.floor(remainingMsAfterMin / 1000);
+    const ms = remainingMsAfterMin % 1000;
+    prettyDuration = `${years} ${years <= 1 ? "year" : "years"}, ${months} ${
+      months <= 1 ? "month" : "months"
+    }, ${days} ${days <= 1 ? "day" : "days"}, ${hours} h, ${min} min, ${sec} sec, ${ms} ms`;
+  }
+
+  if (typeof options.prefix === "string") {
+    prettyDuration = `${options.prefix}${prettyDuration}`;
+  }
+  if (typeof options.suffix === "string") {
+    prettyDuration = `${prettyDuration}${options.suffix}`;
+  }
+
+  if (options.log === true) {
+    console.log(prettyDuration);
+  }
+
+  return prettyDuration;
+}