@nshiab/simple-data-analysis-core 2.1.11 → 2.1.13

package/src/helpers/formatNumber.ts

@@ -0,0 +1,166 @@
+/**
+ * Rounds a number based on specified criteria: a fixed number of decimal places or to a specific number of significant digits.
+ *
+ * @example
+ * ```ts
+ * // Round to one decimal place.
+ * const result = round(1234.567, { decimals: 1 });
+ * console.log(result); // 1234.6
+ * ```
+ *
+ * @example
+ * ```ts
+ * // Round to 3 significant digits.
+ * const result = round(0.004622, { significantDigits: 3 });
+ * console.log(result); // 0.00462
+ * ```
+ *
+ * @param number - The number to be rounded.
+ *
+ * @param options - An object containing options for rounding.
+ *
+ * @param options.decimals - The number of decimal places to keep when rounding.
+ *
+ * @param options.significantDigits - The number of significant digits to retain.
+ *
+ * @returns The rounded number.
+ *
+ * @throws {Error} If more than one rounding option is provided.
+ */
+function round(
+  number: number,
+  options: {
+    decimals?: number;
+    significantDigits?: number;
+  } = {},
+): number {
+  if (typeof options.decimals === "number" && typeof options.significantDigits === "number") {
+    throw new Error("You can't use options decimals and significantDigits together. Pick one.");
+  }
+
+  if (typeof options.decimals === "number") {
+    return parseFloat(number.toFixed(options.decimals));
+  } else if (typeof options.significantDigits === "number") {
+    return parseFloat(number.toPrecision(options.significantDigits));
+  } else {
+    return Math.round(number);
+  }
+}
+
+/**
+ * Formats a number with thousands separators, decimals, abbreviations, and more.
+ *
+ * @example
+ * ```ts
+ * // Basic usage: Format a number with thousands separator.
+ * const num1 = formatNumber(1234567.89);
+ * console.log(num1); // "1,234,567.89"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // With significant digits and suffix.
+ * const num2 = formatNumber(66.666, { significantDigits: 3, suffix: "%" });
+ * console.log(num2); // "66.7%"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // Abbreviation.
+ * const num3 = formatNumber(12000, { abbreviation: true });
+ * console.log(num3); // "12K"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // With decimal places.
+ * const num4 = formatNumber(123.456, { decimals: 1 });
+ * console.log(num4); // "123.5"
+ * ```
+ *
+ * @param number - The number to be formatted.
+ *
+ * @param options - An object containing various formatting options.
+ *
+ * @param options.decimals - The number of decimal places to round to.
+ *
+ * @param options.significantDigits - The number of significant digits to round to.
+ *
+ * @param options.abbreviation - If `true`, the number will be abbreviated (e.g., 1,200,000 becomes "1.2M").
+ *
+ * @param options.prefix - A string to prepend before the formatted number.
+ *
+ * @param options.suffix - A string to append after the formatted number.
+ *
+ * @returns The formatted number as a string.
+ */
+export default function formatNumber(
+  number: number,
+  options: {
+    decimals?: number;
+    significantDigits?: number;
+    abbreviation?: boolean;
+    prefix?: string;
+    suffix?: string;
+  } = {},
+): string {
+  if (typeof number !== "number") {
+    throw new Error("Not a number");
+  }
+
+  const mergedOptions: {
+    decimals?: number;
+    significantDigits?: number;
+    abbreviation?: boolean;
+    prefix: string;
+    suffix: string;
+  } = {
+    prefix: "",
+    suffix: "",
+    ...options,
+  };
+
+  // Handle abbreviation (K/M/B/T)
+  let abbreviation = "";
+  if (mergedOptions.abbreviation && number !== 0) {
+    const abbreviations = ["", "K", "M", "B", "T"];
+    const index = Math.floor(Math.log10(Math.abs(number)) / 3);
+    abbreviation = abbreviations[index] ?? "";
+    number = number / Math.pow(10, index * 3);
+  }
+
+  // Round if needed
+  if (
+    typeof mergedOptions.decimals === "number" ||
+    typeof mergedOptions.significantDigits === "number"
+  ) {
+    number = round(number, {
+      decimals: mergedOptions.decimals,
+      significantDigits: mergedOptions.significantDigits,
+    });
+  }
+
+  // Format with thousands separator
+  const regex = /\B(?=(\d{3})+(?!\d))/g;
+  const [integers, decimals] = number.toString().split(".");
+
+  const formattedIntegers = integers.replace(regex, ",");
+  const formattedNumber = decimals ? `${formattedIntegers}.${decimals}` : formattedIntegers;
+
+  // Handle dollar sign prefix placement (after minus sign for negative numbers)
+  if (mergedOptions.prefix === "$") {
+    if (formattedNumber.startsWith("-")) {
+      return `-${mergedOptions.prefix}${formattedNumber.slice(
+        1,
+      )}${abbreviation}${mergedOptions.suffix}`;
+    } else if (formattedNumber.startsWith("+")) {
+      return `+${mergedOptions.prefix}${formattedNumber.slice(
+        1,
+      )}${abbreviation}${mergedOptions.suffix}`;
+    } else {
+      return `${mergedOptions.prefix}${formattedNumber}${abbreviation}${mergedOptions.suffix}`;
+    }
+  }
+
+  return `${mergedOptions.prefix}${formattedNumber}${abbreviation}${mergedOptions.suffix}`;
+}

package/src/helpers/geometryFromJSON.ts

@@ -0,0 +1,4 @@
+/** Converts validated GeoJSON SQL text to an explicitly typed WGS84 geometry. */
+export default function geometryFromJSON(expression: string): string {
+  return `ST_GeomFromGeoJSON(${expression})::GEOMETRY('EPSG:4326')`;
+}

package/src/helpers/getCombinations.ts

@@ -0,0 +1,27 @@
+// from https://github.com/simple-statistics/simple-statistics/blob/main/src/combinations.js
+
+function getCombinations<T>(x: T[], k: 2): [T, T][];
+function getCombinations<T>(x: T[], k: number): T[][];
+function getCombinations<T>(x: T[], k: number): T[][] {
+  let i;
+  let subI;
+  const combinationList: T[][] = [];
+  let subsetCombinations;
+  let next;
+
+  for (i = 0; i < x.length; i++) {
+    if (k === 1) {
+      combinationList.push([x[i]]);
+    } else {
+      subsetCombinations = getCombinations(x.slice(i + 1, x.length), k - 1);
+      for (subI = 0; subI < subsetCombinations.length; subI++) {
+        next = subsetCombinations[subI];
+        next.unshift(x[i]);
+        combinationList.push(next);
+      }
+    }
+  }
+  return combinationList;
+}
+
+export default getCombinations;

package/src/helpers/getCurrentDatabase.ts

@@ -0,0 +1,9 @@
+import queryDbFile from "./queryDbFile.ts";
+import type SimpleDB from "../class/SimpleDB.ts";
+
+export default async function getCurrentDatabase(sdb: SimpleDB): Promise<string> {
+  const rows = await queryDbFile(sdb, "SELECT current_database() AS name;", {
+    returnData: true,
+  });
+  return rows![0].name as string;
+}

package/src/helpers/getDbFileType.ts

@@ -0,0 +1,10 @@
+import { extname } from "node:path";
+
+export default function getDbFileType(file: string): "duckdb" | "sqlite" {
+  const extension = extname(file).slice(1).toLowerCase();
+  if (extension === "db" || extension === "duckdb") return "duckdb";
+  if (extension === "sqlite") return "sqlite";
+  throw new Error(
+    `The extension ${extension} is not supported. Please use .db, .duckdb or .sqlite instead.`,
+  );
+}

package/src/helpers/getDbIndexes.ts

@@ -0,0 +1,11 @@
+import type SimpleDB from "../class/SimpleDB.ts";
+import type { IndexDefinition } from "./indexDefinitions.ts";
+
+export default function getDbIndexes(sdb: SimpleDB): Record<string, IndexDefinition[]> {
+  return Object.fromEntries(
+    sdb
+      .getTables()
+      .filter((table) => table.indexes.length > 0)
+      .map((table) => [table.name, structuredClone(table.indexes)]),
+  );
+}

package/src/helpers/getExtension.ts

@@ -0,0 +1,6 @@
+export default function getExtension(path: string) {
+  const extensionSplit = path.replace(".gz", "").replace(".zstd", "").split(".");
+  const extension = extensionSplit[extensionSplit.length - 1].toLocaleLowerCase();
+
+  return extension;
+}

package/src/helpers/getGraphEdgeIdColumn.ts

@@ -0,0 +1,45 @@
+import foldIdentifier from "./foldIdentifier.ts";
+import getGraphEndpointColumns from "./getGraphEndpointColumns.ts";
+import type { TableSchema } from "./pendingOps.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+export type GraphEdgeIdColumn = {
+  column: string;
+  family: "string" | "numeric";
+  idType: string;
+};
+
+/** Resolves a route edge-ID column and its exact comparison type. */
+export default function getGraphEdgeIdColumn(
+  schema: TableSchema,
+  requestedEdgeId: string,
+  method: string,
+): GraphEdgeIdColumn {
+  const folded = foldIdentifier(requestedEdgeId);
+  const column = Object.keys(schema).find((name) => foldIdentifier(name) === folded);
+  if (column === undefined) {
+    throw new Error(
+      `${method} the column ${quoteIdentifier(
+        requestedEdgeId,
+      )} does not exist. Check for typos, or load the data first.`,
+    );
+  }
+
+  try {
+    const resolved = getGraphEndpointColumns({ [column]: schema[column] }, column, column, method);
+    return {
+      column,
+      family: resolved.family,
+      idType: resolved.idType,
+    };
+  } catch (error) {
+    if (error instanceof TypeError) {
+      throw new TypeError(
+        `${method} requires string or whole-number edge IDs, but column ${quoteIdentifier(
+          column,
+        )} has type ${schema[column]}.`,
+      );
+    }
+    throw error;
+  }
+}

package/src/helpers/getGraphEndpointColumns.ts

@@ -0,0 +1,205 @@
+import foldIdentifier from "./foldIdentifier.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+import type { TableSchema } from "./pendingOps.ts";
+
+export type GraphEndpointColumns = {
+  source: string;
+  target: string;
+  idType: string;
+  family: "string" | "numeric";
+  floating: boolean;
+};
+
+type IntegerType = {
+  signed: boolean;
+  bits: 8 | 16 | 32 | 64 | 128;
+  digits: number;
+};
+
+const integerTypes: Record<string, IntegerType> = {
+  TINYINT: { signed: true, bits: 8, digits: 3 },
+  SMALLINT: { signed: true, bits: 16, digits: 5 },
+  INTEGER: { signed: true, bits: 32, digits: 10 },
+  BIGINT: { signed: true, bits: 64, digits: 19 },
+  HUGEINT: { signed: true, bits: 128, digits: 39 },
+  UTINYINT: { signed: false, bits: 8, digits: 3 },
+  USMALLINT: { signed: false, bits: 16, digits: 5 },
+  UINTEGER: { signed: false, bits: 32, digits: 10 },
+  UBIGINT: { signed: false, bits: 64, digits: 20 },
+  UHUGEINT: { signed: false, bits: 128, digits: 39 },
+};
+
+/** Resolves and validates a graph's endpoint columns from an existing schema. */
+export default function getGraphEndpointColumns(
+  schema: TableSchema,
+  requestedSource: string,
+  requestedTarget: string,
+  method: string,
+): GraphEndpointColumns {
+  const source = findColumn(schema, requestedSource);
+  const target = findColumn(schema, requestedTarget);
+  const missing = [
+    source === undefined ? requestedSource : undefined,
+    target === undefined ? requestedTarget : undefined,
+  ].filter((column): column is string => column !== undefined);
+  if (missing.length > 0) {
+    const plural = missing.length > 1;
+    throw new Error(
+      `${method} the column${plural ? "s" : ""} ${missing.map(quoteIdentifier).join(", ")} ${
+        plural ? "do" : "does"
+      } not exist. Check for typos, or load the data first.`,
+    );
+  }
+
+  const sourceType = normalizeType(schema[source!]);
+  const targetType = normalizeType(schema[target!]);
+  const sourceKind = typeKind(sourceType);
+  const targetKind = typeKind(targetType);
+  if (sourceKind === undefined || targetKind === undefined) {
+    const unsupported =
+      sourceKind === undefined
+        ? { column: source!, type: sourceType }
+        : { column: target!, type: targetType };
+    throw new TypeError(
+      `${method} requires string or numeric endpoint IDs, but column ${quoteIdentifier(
+        unsupported.column,
+      )} has type ${unsupported.type}.`,
+    );
+  }
+  if (sourceKind.family !== targetKind.family) {
+    throw new TypeError(
+      `${method} requires compatible source and target ID types, but ${quoteIdentifier(
+        source!,
+      )} has type ${sourceType} and ${quoteIdentifier(target!)} has type ${targetType}.`,
+    );
+  }
+
+  if (sourceKind.family === "string") {
+    return {
+      source: source!,
+      target: target!,
+      idType: "VARCHAR",
+      family: "string",
+      floating: false,
+    };
+  }
+
+  const idType = compatibleNumericType(sourceType, targetType);
+  if (idType === undefined) {
+    throw new TypeError(
+      `${method} cannot combine endpoint types ${sourceType} and ${targetType} without losing ID precision. Convert both columns to one compatible exact numeric type first.`,
+    );
+  }
+  return {
+    source: source!,
+    target: target!,
+    idType,
+    family: "numeric",
+    floating: isFloating(sourceType) || isFloating(targetType),
+  };
+}
+
+function findColumn(schema: TableSchema, requested: string): string | undefined {
+  const folded = foldIdentifier(requested);
+  return Object.keys(schema).find((column) => foldIdentifier(column) === folded);
+}
+
+function normalizeType(type: string): string {
+  return type.toUpperCase().replace(/^REAL$/, "FLOAT");
+}
+
+function typeKind(type: string): { family: "string" } | { family: "numeric" } | undefined {
+  if (type === "VARCHAR") return { family: "string" };
+  if (
+    integerTypes[type] !== undefined ||
+    isDecimal(type) ||
+    isFloating(type) ||
+    type === "BIGNUM"
+  ) {
+    return { family: "numeric" };
+  }
+  return undefined;
+}
+
+function compatibleNumericType(left: string, right: string): string | undefined {
+  if (left === right) return left;
+  if (isFloating(left) || isFloating(right)) {
+    return isFloating(left) && isFloating(right) ? "DOUBLE" : undefined;
+  }
+  if (left === "BIGNUM" || right === "BIGNUM") {
+    const other = left === "BIGNUM" ? right : left;
+    return integerTypes[other] !== undefined || decimalScale(other) === 0 ? "BIGNUM" : undefined;
+  }
+
+  const leftInteger = integerTypes[left];
+  const rightInteger = integerTypes[right];
+  if (leftInteger !== undefined && rightInteger !== undefined) {
+    return commonIntegerType(leftInteger, rightInteger);
+  }
+
+  const leftDecimal = parseDecimal(left);
+  const rightDecimal = parseDecimal(right);
+  if (leftDecimal === undefined && rightDecimal === undefined) return undefined;
+  const scale = Math.max(leftDecimal?.scale ?? 0, rightDecimal?.scale ?? 0);
+  const integerDigits = Math.max(
+    leftDecimal === undefined ? leftInteger!.digits : leftDecimal.precision - leftDecimal.scale,
+    rightDecimal === undefined ? rightInteger!.digits : rightDecimal.precision - rightDecimal.scale,
+  );
+  const precision = integerDigits + scale;
+  return precision <= 38 ? `DECIMAL(${precision},${scale})` : undefined;
+}
+
+function commonIntegerType(left: IntegerType, right: IntegerType): string | undefined {
+  if (left.signed === right.signed) {
+    const bits = Math.max(left.bits, right.bits);
+    return integerTypeName(left.signed, bits);
+  }
+  const signed = left.signed ? left : right;
+  const unsigned = left.signed ? right : left;
+  const requiredBits = Math.max(signed.bits, nextBits(unsigned.bits));
+  return requiredBits > 128 ? undefined : integerTypeName(true, requiredBits);
+}
+
+function nextBits(bits: number): number {
+  if (bits <= 8) return 16;
+  if (bits <= 16) return 32;
+  if (bits <= 32) return 64;
+  if (bits <= 64) return 128;
+  return 256;
+}
+
+function integerTypeName(signed: boolean, bits: number): string | undefined {
+  const names = signed
+    ? {
+        8: "TINYINT",
+        16: "SMALLINT",
+        32: "INTEGER",
+        64: "BIGINT",
+        128: "HUGEINT",
+      }
+    : {
+        8: "UTINYINT",
+        16: "USMALLINT",
+        32: "UINTEGER",
+        64: "UBIGINT",
+        128: "UHUGEINT",
+      };
+  return names[bits as keyof typeof names];
+}
+
+function isFloating(type: string): boolean {
+  return type === "FLOAT" || type === "DOUBLE";
+}
+
+function isDecimal(type: string): boolean {
+  return /^DECIMAL\(\d+,\d+\)$/.test(type);
+}
+
+function parseDecimal(type: string): { precision: number; scale: number } | undefined {
+  const match = /^DECIMAL\((\d+),(\d+)\)$/.exec(type);
+  return match === null ? undefined : { precision: Number(match[1]), scale: Number(match[2]) };
+}
+
+function decimalScale(type: string): number | undefined {
+  return parseDecimal(type)?.scale;
+}

package/src/helpers/getGraphWeightColumn.ts

@@ -0,0 +1,64 @@
+import foldIdentifier from "./foldIdentifier.ts";
+import type { TableSchema } from "./pendingOps.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+type GraphWeightColumn = {
+  column: string;
+  distanceType: string;
+  type: string;
+};
+
+const integerTypes = new Set([
+  "TINYINT",
+  "SMALLINT",
+  "INTEGER",
+  "BIGINT",
+  "UTINYINT",
+  "USMALLINT",
+  "UINTEGER",
+  "UBIGINT",
+]);
+const wideIntegerTypes = new Set(["HUGEINT", "UHUGEINT", "BIGNUM"]);
+
+/** Resolves a numeric graph weight and a precision-preserving sum type. */
+export default function getGraphWeightColumn(
+  schema: TableSchema,
+  requestedWeight: string,
+  method: string,
+): GraphWeightColumn {
+  const folded = foldIdentifier(requestedWeight);
+  const column = Object.keys(schema).find((name) => foldIdentifier(name) === folded);
+  if (column === undefined) {
+    throw new Error(
+      `${method} the column ${quoteIdentifier(
+        requestedWeight,
+      )} does not exist. Check for typos, or load the data first.`,
+    );
+  }
+
+  const type = schema[column].toUpperCase().replace(/^REAL$/, "FLOAT");
+  if (integerTypes.has(type)) {
+    return { column, type, distanceType: "HUGEINT" };
+  }
+  if (wideIntegerTypes.has(type)) {
+    return { column, type, distanceType: "BIGNUM" };
+  }
+  if (type === "FLOAT" || type === "DOUBLE") {
+    return { column, type, distanceType: type };
+  }
+
+  const decimal = /^DECIMAL\((\d+),(\d+)\)$/.exec(type);
+  if (decimal !== null) {
+    return {
+      column,
+      type,
+      distanceType: `DECIMAL(38,${Number(decimal[2])})`,
+    };
+  }
+
+  throw new TypeError(
+    `${method} requires a numeric weight column, but column ${quoteIdentifier(
+      column,
+    )} has type ${type}.`,
+  );
+}

package/src/helpers/getIdenticalColumns.ts

@@ -0,0 +1,3 @@
+export default function getIdenticalColumns(table1Columns: string[], table2Columns: string[]) {
+  return table1Columns.filter((column) => table2Columns.includes(column));
+}

package/src/helpers/getName.ts

@@ -0,0 +1,13 @@
+export default function getName(file: string) {
+  if (file === ":memory:") {
+    return "memory";
+  } else {
+    const nameSplit = file.split("/");
+    const name = nameSplit[nameSplit.length - 1];
+    if (!name.includes(".")) {
+      return name; // Return the name directly if there's no extension
+    }
+    const nameWithoutExtension = name.split(".").slice(0, -1).join(".");
+    return nameWithoutExtension;
+  }
+}

package/src/helpers/getStatExpression.ts

@@ -0,0 +1,90 @@
+export type Stat =
+  | "count"
+  | "countDistinct"
+  | "countNull"
+  | "min"
+  | "max"
+  | "mean"
+  | "median"
+  | "sum"
+  | "skew"
+  | "stdDev"
+  | "variance";
+
+export const allStats: readonly Stat[] = [
+  "count",
+  "countDistinct",
+  "countNull",
+  "min",
+  "max",
+  "mean",
+  "median",
+  "sum",
+  "skew",
+  "stdDev",
+  "variance",
+];
+
+const aggregateFunctions: Partial<Record<Stat, string>> = {
+  min: "MIN",
+  max: "MAX",
+  mean: "AVG",
+  median: "MEDIAN",
+  sum: "SUM",
+  skew: "SKEWNESS",
+  stdDev: "STDDEV",
+  variance: "VARIANCE",
+};
+
+const temporalStatTypes: readonly string[] = [
+  "DATE",
+  "TIME",
+  "TIMESTAMP",
+  "TIMESTAMP_MS",
+  "TIMESTAMP WITH TIME ZONE",
+];
+
+/** Returns whether a DuckDB type is temporal for statistic compatibility. */
+export function isTemporalStatType(type: string | undefined): boolean {
+  return typeof type === "string" && temporalStatTypes.includes(type);
+}
+
+/**
+ * Returns the aggregate expression for one statistic of one column, or `null`
+ * when the statistic is not supported for the column type.
+ */
+export default function getStatExpression(
+  stat: Stat,
+  type: string | undefined,
+  reference: string,
+  decimals: number | undefined,
+): string | null {
+  if (typeof type === "string" && type.toLowerCase().includes("geometry")) {
+    return null;
+  }
+  if (stat === "count") {
+    return `CAST(COUNT(*) AS INTEGER)`;
+  }
+  if (stat === "countDistinct") {
+    return `CAST(COUNT(DISTINCT ${reference}) AS INTEGER)`;
+  }
+  if (stat === "countNull") {
+    return `CAST(COUNT(CASE WHEN ${reference} IS NULL THEN 1 END) AS INTEGER)`;
+  }
+  if (type === "VARCHAR") {
+    return null;
+  }
+  if (isTemporalStatType(type) && ["mean", "sum", "skew", "stdDev", "variance"].includes(stat)) {
+    return null;
+  }
+  const aggregateFunction = aggregateFunctions[stat];
+  if (aggregateFunction === undefined) {
+    return null;
+  }
+  const aggregate = `${aggregateFunction}(${reference})`;
+  return typeof decimals === "number" &&
+    typeof type === "string" &&
+    !["VARCHAR", "DATE", "TIME", "TIMESTAMP", "TIMESTAMP WITH TIME ZONE"].includes(type)
+    ? `ROUND(${aggregate}, ${decimals})`
+    : aggregate;
+}

package/src/helpers/graphRouteResultSchema.ts

@@ -0,0 +1,26 @@
+import type { TableSchema } from "./pendingOps.ts";
+import validateGraphRouteResultSchema from "./validateGraphRouteResultSchema.ts";
+
+/** Derives the route result schema while preserving input columns and types. */
+export default function graphRouteResultSchema(
+  schema: TableSchema,
+  distanceType: string,
+  method: string,
+  elapsedTime = false,
+  includeTotal = true,
+): TableSchema {
+  validateGraphRouteResultSchema(
+    schema,
+    method,
+    elapsedTime ? ["elapsedTimeMs"] : [],
+    includeTotal,
+  );
+  return {
+    pathId: "BIGINT",
+    step: "BIGINT",
+    weight: distanceType,
+    ...(includeTotal ? { total: distanceType } : {}),
+    ...(elapsedTime ? { elapsedTimeMs: "DOUBLE" } : {}),
+    ...schema,
+  };
+}