@nshiab/simple-data-analysis-core 2.1.12 → 2.1.13

package/src/methods/convert.ts

@@ -0,0 +1,228 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import parseType from "../helpers/parseTypes.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+
+export default function convert(
+  simpleTable: SimpleTable,
+  types: {
+    [key: string]:
+      | "integer"
+      | "float"
+      | "number"
+      | "string"
+      | "date"
+      | "time"
+      | "datetime"
+      | "datetimeTz"
+      | "bigint"
+      | "double"
+      | "varchar"
+      | "timestamp"
+      | "timestamp with time zone"
+      | "boolean"
+      | "json"
+      | "JSON"
+      | `float[${number}]`
+      | `FLOAT[${number}]`
+      | `geometry('${string}')`
+      | `GEOMETRY('${string}')`;
+  },
+  options: {
+    strict?: boolean;
+    datetimeFormat?: string;
+  } = {},
+) {
+  types = structuredClone(types);
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "convert()",
+    parameters: { types, options },
+    needsSchema: true,
+    needsSpatial: Object.values(types).some((type) => type.toLowerCase().startsWith("geometry")),
+    outputSchema: (allTypes) => ({
+      ...allTypes,
+      ...Object.fromEntries(
+        Object.entries(types).map(([column, type]) => [column, parseType(type)]),
+      ),
+    }),
+    values: (allTypes) =>
+      getDatetimeFormatValues(
+        Object.keys(types),
+        Object.values(types),
+        Object.keys(allTypes),
+        allTypes,
+        options,
+      ),
+    buildSelect: (input, allTypes) => {
+      const allColumns = Object.keys(allTypes);
+
+      for (const col in types) {
+        if (!allColumns.includes(col)) {
+          throw new Error(`The column ${col} does not exist in the table.`);
+        }
+      }
+
+      return convertSelect(
+        input,
+        Object.keys(types),
+        Object.values(types),
+        allColumns,
+        allTypes,
+        options,
+      );
+    },
+  });
+}
+
+export function convertSelect(
+  input: string,
+  columns: string[],
+  columnsTypes: (
+    | "integer"
+    | "float"
+    | "number"
+    | "string"
+    | "date"
+    | "time"
+    | "datetime"
+    | "datetimeTz"
+    | "bigint"
+    | "double"
+    | "varchar"
+    | "timestamp"
+    | "timestamp with time zone"
+    | "boolean"
+    | "json"
+    | "JSON"
+    | `float[${number}]`
+    | `FLOAT[${number}]`
+    | `geometry('${string}')`
+    | `GEOMETRY('${string}')`
+  )[],
+  allColumns: string[],
+  allTypes: TableSchema,
+  options: { datetimeFormat?: string; strict?: boolean },
+) {
+  let query = `SELECT`;
+
+  const cast = options.strict === false ? "TRY_CAST" : "CAST";
+
+  for (const column of allColumns) {
+    const indexOf = columns.indexOf(column);
+    if (indexOf === -1) {
+      query += ` ${quoteIdentifier(column)},`;
+    } else {
+      const expectedType = parseType(columnsTypes[indexOf]);
+      const currentType = allTypes[column];
+      const datetimeFormatExist = typeof options.datetimeFormat === "string";
+
+      const stringToNumber =
+        currentType === "VARCHAR" && ["DOUBLE", "BIGINT"].includes(expectedType);
+      const stringToDate =
+        currentType === "VARCHAR" &&
+        (expectedType.includes("TIME") || expectedType.includes("DATE"));
+      const dateToString =
+        (currentType.includes("DATE") || currentType.includes("TIME")) &&
+        expectedType === "VARCHAR";
+      const timeToMs = currentType.includes("TIME") && ["DOUBLE", "BIGINT"].includes(expectedType);
+      const dateToMs = currentType.includes("DATE") && ["DOUBLE", "BIGINT"].includes(expectedType);
+      const msToTime = ["DOUBLE", "BIGINT"].includes(currentType) && expectedType === "TIME";
+      const msToDate = ["DOUBLE", "BIGINT"].includes(currentType) && expectedType.includes("DATE");
+      const msToTimestamp =
+        ["DOUBLE", "BIGINT"].includes(currentType) && expectedType.includes("TIMESTAMP");
+
+      if (datetimeFormatExist && stringToDate) {
+        query += ` strptime(${quoteIdentifier(column)}, ?) AS ${quoteIdentifier(column)},`;
+      } else if (datetimeFormatExist && dateToString) {
+        query += ` strftime(${quoteIdentifier(column)}, ?) AS ${quoteIdentifier(column)},`;
+      } else if (timeToMs) {
+        query += ` date_part('epoch', ${quoteIdentifier(column)}) * 1000 AS ${quoteIdentifier(
+          column,
+        )},`;
+      } else if (dateToMs) {
+        query += ` epoch(${quoteIdentifier(column)}) * 1000 AS ${quoteIdentifier(column)},`;
+      } else if (msToTime) {
+        query += ` TIME '00:00:00' + to_milliseconds(${quoteIdentifier(
+          column,
+        )}) AS ${quoteIdentifier(column)},`;
+      } else if (msToDate) {
+        query += ` DATE '1970-01-01' + to_milliseconds(${quoteIdentifier(
+          column,
+        )}) AS ${quoteIdentifier(column)},`;
+      } else if (msToTimestamp) {
+        query += ` TIMESTAMP '1970-01-01 00:00:00' + to_milliseconds(${quoteIdentifier(
+          column,
+        )}) AS ${quoteIdentifier(column)},`;
+      } else if (options.strict === false && expectedType.startsWith("GEOMETRY")) {
+        // DuckDB's geometry parser can throw even inside TRY_CAST.
+        query += ` TRY(CAST(${quoteIdentifier(
+          column,
+        )} AS ${expectedType})) AS ${quoteIdentifier(column)},`;
+      } else if (stringToNumber) {
+        // Thousand separators would make the cast fail.
+        query += ` ${cast}(REPLACE(${quoteIdentifier(column)}, ',', '') AS ${parseType(
+          columnsTypes[indexOf],
+        )}) AS ${quoteIdentifier(column)},`;
+      } else {
+        query += ` ${cast}(${quoteIdentifier(columns[indexOf])} AS ${parseType(
+          columnsTypes[indexOf],
+        )}) AS ${quoteIdentifier(columns[indexOf])},`;
+      }
+    }
+  }
+
+  query = query.slice(0, query.length - 1);
+  query += ` FROM ${input}`;
+
+  return query;
+}
+
+function getDatetimeFormatValues(
+  columns: string[],
+  columnsTypes: (
+    | "integer"
+    | "float"
+    | "number"
+    | "string"
+    | "date"
+    | "time"
+    | "datetime"
+    | "datetimeTz"
+    | "bigint"
+    | "double"
+    | "varchar"
+    | "timestamp"
+    | "timestamp with time zone"
+    | "boolean"
+    | "json"
+    | "JSON"
+    | `float[${number}]`
+    | `FLOAT[${number}]`
+    | `geometry('${string}')`
+    | `GEOMETRY('${string}')`
+  )[],
+  allColumns: string[],
+  allTypes: TableSchema,
+  options: { datetimeFormat?: string },
+): string[] {
+  const datetimeFormat = options.datetimeFormat;
+  if (typeof datetimeFormat !== "string") {
+    return [];
+  }
+  return allColumns.flatMap((column) => {
+    const index = columns.indexOf(column);
+    if (index === -1) {
+      return [];
+    }
+    const currentType = allTypes[column];
+    const expectedType = parseType(columnsTypes[index]);
+    const stringToDate =
+      currentType === "VARCHAR" && (expectedType.includes("TIME") || expectedType.includes("DATE"));
+    const dateToString =
+      (currentType.includes("DATE") || currentType.includes("TIME")) && expectedType === "VARCHAR";
+    return stringToDate || dateToString ? [datetimeFormat] : [];
+  });
+}

package/src/methods/correlations.ts

@@ -0,0 +1,136 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import getCombinations from "../helpers/getCombinations.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import resolveOutputTable from "../helpers/resolveOutputTable.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+import validateNumericalAnalysis from "../helpers/validateNumericalAnalysis.ts";
+
+export default function correlations(
+  simpleTable: SimpleTable,
+  options: {
+    x?: string;
+    y?: string;
+    by?: string | string[];
+    decimals?: number;
+    outputTable?: string | boolean;
+  } = {},
+): SimpleTable {
+  options = structuredClone(options);
+  options.outputTable = resolveOutputTable(simpleTable, options.outputTable);
+
+  if (typeof options.outputTable === "string") {
+    // The output table instance is created at call time so it can be
+    // returned synchronously and chained on right away.
+    const outputTable = simpleTable.sdb.newTable(options.outputTable);
+    queueOp(outputTable, {
+      kind: "barrier",
+      method: "correlations()",
+      parameters: { options },
+      execute: async () => {
+        const combinations = getCorrelationsCombinations(
+          await simpleTable.getTypes(),
+          options,
+          simpleTable.name,
+        );
+        await queryDB(
+          simpleTable,
+          `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable.name)} AS ${correlationsSelect(
+            `${quoteIdentifier(simpleTable.name)}`,
+            combinations,
+            options,
+          )}`,
+          mergeOptions(simpleTable, {
+            table: outputTable.name,
+            method: "correlations()",
+            parameters: {
+              options,
+              "combinations (computed)": combinations,
+            },
+            values: correlationsValues(combinations),
+          }),
+        );
+      },
+    });
+    return outputTable;
+  }
+
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "correlations()",
+    parameters: { options },
+    needsSchema: true,
+    values: (types) =>
+      correlationsValues(getCorrelationsCombinations(types, options, simpleTable.name)),
+    buildSelect: (input, types) =>
+      correlationsSelect(
+        input,
+        getCorrelationsCombinations(types, options, simpleTable.name),
+        options,
+      ),
+  });
+  return simpleTable;
+}
+
+function correlationsValues(combinations: [string, string][]): string[] {
+  return combinations.flatMap(([x, y]) => [x, y]);
+}
+
+function getCorrelationsCombinations(
+  types: TableSchema,
+  options: { x?: string; y?: string },
+  tableName: string,
+): [string, string][] {
+  const columns = validateNumericalAnalysis(types, options, "correlations()", tableName);
+  let combinations: [string, string][] = [];
+  if (options.x === undefined) {
+    combinations = getCombinations(columns, 2);
+  } else if (options.y === undefined) {
+    combinations = [];
+    for (const col of columns) {
+      if (col !== options.x) {
+        combinations.push([options.x, col]);
+      }
+    }
+  } else {
+    combinations = [[options.x, options.y]];
+  }
+  return combinations;
+}
+
+function correlationsSelect(
+  input: string,
+  combinations: [string, string][],
+  options: {
+    by?: string | string[];
+    decimals?: number;
+  },
+) {
+  const by = options.by ? stringToArray(options.by) : [];
+
+  const groupBy =
+    by.length === 0 ? "" : ` GROUP BY ${by.map((d) => `${quoteIdentifier(d)}`).join(",")}`;
+
+  const catSelect = by.length > 0 ? `${by.map((d) => `${quoteIdentifier(d)}`).join(",")}, ` : "";
+
+  // One UNION ALL branch per pair (each with a distinct pair of literals, so
+  // deduplication would never remove anything). DuckDB runs the branches as
+  // concurrent pipelines, which parallelizes the aggregates better than
+  // computing them all in one scan.
+  const branches = combinations.map((comb) => {
+    const expression =
+      typeof options.decimals === "number"
+        ? `ROUND(corr(${quoteIdentifier(comb[0])}, ${quoteIdentifier(
+            comb[1],
+          )}), ${options.decimals})`
+        : `corr(${quoteIdentifier(comb[0])}, ${quoteIdentifier(comb[1])})`;
+    return `SELECT ${catSelect}? AS ${quoteIdentifier("x")}, ? AS ${quoteIdentifier(
+      "y",
+    )}, ${expression} AS ${quoteIdentifier("corr")} FROM ${input}${groupBy}`;
+  });
+
+  return branches.join("\nUNION ALL\n");
+}

package/src/methods/coveredBy.ts

@@ -0,0 +1,25 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function coveredBy(
+  simpleTable: SimpleTable,
+  column: string,
+  containerColumn: string,
+  newColumn: string,
+) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "coveredBy()",
+    parameters: { column, containerColumn, newColumn },
+    needsSchema: true,
+    needsSpatial: true,
+    buildSelect: (input, types) => {
+      assertNewColumns(types, [newColumn], "coveredBy()");
+      return `SELECT *, CAST(ST_Covers(${quoteIdentifier(containerColumn)}, ${quoteIdentifier(
+        column,
+      )}) AS BOOLEAN) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/createFtsIndex.ts

@@ -0,0 +1,139 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import camelCase from "../helpers/camelCase.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { FtsIndexDefinition } from "../helpers/indexDefinitions.ts";
+import buildCreateIndexQuery from "../helpers/indexQueries.ts";
+
+export default function createFtsIndex(
+  simpleTable: SimpleTable,
+  idColumn: string,
+  textColumn: string,
+  options: {
+    stemmer?:
+      | "arabic"
+      | "basque"
+      | "catalan"
+      | "danish"
+      | "dutch"
+      | "english"
+      | "finnish"
+      | "french"
+      | "german"
+      | "greek"
+      | "hindi"
+      | "hungarian"
+      | "indonesian"
+      | "irish"
+      | "italian"
+      | "lithuanian"
+      | "nepali"
+      | "norwegian"
+      | "porter"
+      | "portuguese"
+      | "romanian"
+      | "russian"
+      | "serbian"
+      | "spanish"
+      | "swedish"
+      | "tamil"
+      | "turkish"
+      | "none";
+    stopwords?: string;
+    ignore?: string;
+    stripAccents?: boolean;
+    lower?: boolean;
+    overwrite?: boolean;
+    verbose?: boolean;
+  } = {},
+) {
+  // Index creation is multi-statement by nature: it executes as a barrier.
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "createFtsIndex()",
+    parameters: { idColumn, textColumn, options },
+    execute: () => executeCreateFtsIndex(simpleTable, idColumn, textColumn, options),
+  });
+}
+
+/**
+ * Creates the index immediately, without queueing. For use inside a
+ * barrier's execute, which runs during a flush.
+ */
+export async function executeCreateFtsIndex(
+  simpleTable: SimpleTable,
+  idColumn: string,
+  textColumn: string,
+  options: {
+    stemmer?: string;
+    stopwords?: string;
+    ignore?: string;
+    stripAccents?: boolean;
+    lower?: boolean;
+    overwrite?: boolean;
+    verbose?: boolean;
+  },
+): Promise<void> {
+  const indexName = `fts_index_${camelCase(simpleTable.name)}`;
+  const indexPosition = simpleTable.indexes.findIndex(({ name }) => name === indexName);
+  const indexExists = indexPosition >= 0;
+
+  if (indexExists && options.overwrite) {
+    if (options.verbose) {
+      console.log(`\nDropping existing FTS index on ${quoteIdentifier(textColumn)} column...`);
+    }
+
+    await simpleTable.sdb.customQuery(
+      `PRAGMA drop_fts_index(${quoteIdentifier(simpleTable.name)});`,
+    );
+
+    if (options.verbose) {
+      console.log("FTS index dropped.");
+    }
+  }
+
+  if (!indexExists || options.overwrite) {
+    if (options.verbose) {
+      console.log(`\nCreating FTS index on ${quoteIdentifier(textColumn)} column...`);
+    }
+
+    const indexOptions: FtsIndexDefinition["options"] = {};
+    if (options.stemmer !== undefined) {
+      indexOptions.stemmer = options.stemmer;
+    }
+    if (options.stopwords !== undefined) {
+      indexOptions.stopwords = options.stopwords;
+    }
+    if (options.ignore !== undefined) {
+      indexOptions.ignore = options.ignore;
+    }
+    if (options.stripAccents !== undefined) {
+      indexOptions.stripAccents = options.stripAccents;
+    }
+    if (options.lower !== undefined) {
+      indexOptions.lower = options.lower;
+    }
+    const definition: FtsIndexDefinition = {
+      kind: "fts",
+      name: indexName,
+      idColumn,
+      textColumn,
+      options: indexOptions,
+    };
+    await simpleTable.sdb.customQuery(buildCreateIndexQuery(simpleTable.name, definition));
+    if (indexExists) {
+      simpleTable.indexes[indexPosition] = definition;
+    } else {
+      simpleTable.indexes.push(definition);
+    }
+
+    if (options.verbose) {
+      console.log("FTS index created successfully.");
+    }
+  } else {
+    if (options.verbose) {
+      console.log("FTS index already exists.");
+    }
+  }
+}

package/src/methods/createPoints.ts

@@ -0,0 +1,33 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import parseValue from "../helpers/parseValue.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function createPoints(
+  simpleTable: SimpleTable,
+  latColumn: string,
+  lonColumn: string,
+  newColumn: string,
+  options: { projection?: string } = {},
+) {
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "createPoints()",
+    parameters: { latColumn, lonColumn, newColumn, options },
+    needsSchema: true,
+    needsSpatial: true,
+    outputSchema: (types) => ({
+      ...types,
+      [newColumn]: `GEOMETRY('${options.projection ?? "EPSG:4326"}')`,
+    }),
+    buildSelect: (input, types) => {
+      const expression = `ST_Point(${quoteIdentifier(lonColumn)}, ${quoteIdentifier(
+        latColumn,
+      )})::GEOMETRY(${parseValue(options.projection ?? "EPSG:4326")})`;
+      return Object.keys(types).includes(newColumn)
+        ? `SELECT * REPLACE (${expression} AS ${quoteIdentifier(newColumn)}) FROM ${input}`
+        : `SELECT *, ${expression} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/createVssIndex.ts

@@ -0,0 +1,99 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import camelCase from "../helpers/camelCase.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { VssIndexDefinition } from "../helpers/indexDefinitions.ts";
+import buildCreateIndexQuery from "../helpers/indexQueries.ts";
+
+export default function createVssIndex(
+  simpleTable: SimpleTable,
+  column: string,
+  options: {
+    overwrite?: boolean;
+    verbose?: boolean;
+    efConstruction?: number;
+    efSearch?: number;
+    M?: number;
+  } = {},
+) {
+  // Index creation is multi-statement by nature: it executes as a barrier.
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "createVssIndex()",
+    parameters: { column, options },
+    execute: () => executeCreateVssIndex(simpleTable, column, options),
+  });
+}
+
+async function executeCreateVssIndex(
+  simpleTable: SimpleTable,
+  column: string,
+  options: {
+    overwrite?: boolean;
+    verbose?: boolean;
+    efConstruction?: number;
+    efSearch?: number;
+    M?: number;
+  },
+): Promise<void> {
+  const indexName = `vss_cosine_index_${camelCase(simpleTable.name)}`;
+  const indexPosition = simpleTable.indexes.findIndex(({ name }) => name === indexName);
+  const indexExists = indexPosition >= 0;
+
+  if (indexExists && options.overwrite) {
+    if (options.verbose) {
+      console.log(`\nDropping existing VSS index on ${quoteIdentifier(column)} column...`);
+    }
+
+    await simpleTable.sdb.customQuery(`DROP INDEX IF EXISTS ${indexName};`);
+
+    if (options.verbose) {
+      console.log("VSS index dropped.");
+    }
+  }
+
+  if (!indexExists || options.overwrite) {
+    if (options.verbose) {
+      console.log(`\nCreating VSS index on ${quoteIdentifier(column)} column...`);
+    }
+
+    const indexOptions: VssIndexDefinition["options"] = {};
+    if (options.efConstruction !== undefined) {
+      indexOptions.efConstruction = options.efConstruction;
+    }
+    if (options.efSearch !== undefined) {
+      indexOptions.efSearch = options.efSearch;
+    }
+    if (options.M !== undefined) {
+      indexOptions.M = options.M;
+    }
+    const definition: VssIndexDefinition = {
+      kind: "vss",
+      name: indexName,
+      column,
+      options: indexOptions,
+    };
+    await simpleTable.sdb.customQuery(
+      `INSTALL vss; LOAD vss;${
+        simpleTable.sdb.file !== ":memory:"
+          ? "\nSET hnsw_enable_experimental_persistence=true;"
+          : ""
+      }
+      ${buildCreateIndexQuery(simpleTable.name, definition)}`,
+    );
+    if (indexExists) {
+      simpleTable.indexes[indexPosition] = definition;
+    } else {
+      simpleTable.indexes.push(definition);
+    }
+
+    if (options.verbose) {
+      console.log("VSS index created successfully.");
+    }
+  } else {
+    if (options.verbose) {
+      console.log("VSS index already exists.");
+    }
+  }
+}