@nshiab/simple-data-analysis-core 2.1.12 → 2.1.13

package/src/methods/similarityMahalanobis.ts

@@ -0,0 +1,166 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import computeCovariance from "../helpers/computeCovariance.ts";
+import computeMahalanobisDistances from "../helpers/computeMahalanobisDistances.ts";
+import foldIdentifier from "../helpers/foldIdentifier.ts";
+import prepareNumericFeatures from "../helpers/prepareNumericFeatures.ts";
+import publishPreparedColumns from "../helpers/publishPreparedColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+
+type Options = NonNullable<Parameters<SimpleTable["similarityMahalanobis"]>[3]>;
+
+export default function similarityMahalanobis(
+  table: SimpleTable,
+  columns: string | string[],
+  referencePoint: number[] | { [key: string]: unknown },
+  newColumn: string,
+  options: Options = {},
+): void {
+  const selected = typeof columns === "string" ? columns : [...columns];
+  if (
+    !Array.isArray(referencePoint) &&
+    referencePoint !== null &&
+    typeof referencePoint === "object"
+  ) {
+    if (typeof selected === "string") {
+      throw new Error(
+        "similarityMahalanobis() an object referencePoint requires an array of scalar feature column names; use a numeric array for a vector column.",
+      );
+    }
+    const referenceObject = referencePoint;
+    referencePoint = selected.map((column) => {
+      if (!Object.hasOwn(referenceObject, column)) {
+        throw new Error(
+          `similarityMahalanobis() referencePoint is missing its own value for feature ${quoteIdentifier(
+            column,
+          )}.`,
+        );
+      }
+      const value = referenceObject[column];
+      if (typeof value !== "number" || !Number.isFinite(value)) {
+        throw new Error(
+          `similarityMahalanobis() referencePoint feature ${quoteIdentifier(
+            column,
+          )} must be a finite number.`,
+        );
+      }
+      return value;
+    });
+  }
+  if (
+    !Array.isArray(referencePoint) ||
+    referencePoint.length === 0 ||
+    Array.from(referencePoint).some((value) => typeof value !== "number" || !Number.isFinite(value))
+  ) {
+    throw new Error(
+      "similarityMahalanobis() referencePoint must be a nonempty array of finite numbers in feature-dimension order.",
+    );
+  }
+  const reference = [...referencePoint];
+  const settings = { ...options };
+  queueOp(table, {
+    kind: "barrier",
+    method: "similarityMahalanobis()",
+    parameters: {
+      columns: selected,
+      referencePoint: reference,
+      newColumn,
+      options: settings,
+    },
+    execute: () => execute(table, selected, reference, newColumn, settings),
+  });
+}
+
+async function execute(
+  table: SimpleTable,
+  columns: string | string[],
+  referencePoint: number[],
+  newColumn: string,
+  options: Options,
+): Promise<void> {
+  const sourceColumns = Object.keys(await table.getTypes());
+  const scoreColumn =
+    options.similarityColumn === true
+      ? "similarity"
+      : options.similarityColumn === false
+        ? undefined
+        : options.similarityColumn;
+  const outputNames = scoreColumn === undefined ? [newColumn] : [newColumn, scoreColumn];
+  const seen = new Set<string>();
+  for (const name of outputNames) {
+    if (typeof name !== "string" || name.length === 0 || name.includes("\0")) {
+      throw new Error(
+        "similarityMahalanobis() output column names must be nonempty strings without null characters.",
+      );
+    }
+    const folded = foldIdentifier(name);
+    if (seen.has(folded)) {
+      throw new Error(
+        "similarityMahalanobis() distance and similarity score columns must have different names (case-insensitive).",
+      );
+    }
+    seen.add(folded);
+    if (sourceColumns.some((column) => foldIdentifier(column) === folded)) {
+      throw new Error(
+        `similarityMahalanobis() cannot create ${quoteIdentifier(
+          name,
+        )} because that column already exists. Remove it first or choose a different name.`,
+      );
+    }
+  }
+
+  const prepared = await prepareNumericFeatures(
+    table,
+    typeof columns === "string"
+      ? { kind: "vector", column: columns }
+      : { kind: "scalars", columns },
+    { method: "similarityMahalanobis()" },
+  );
+  try {
+    if (referencePoint.length !== prepared.dimensions) {
+      throw new Error(
+        `similarityMahalanobis() referencePoint must contain ${prepared.dimensions} values in feature-dimension order; received ${referencePoint.length}.`,
+      );
+    }
+    const model = await computeCovariance(table.connection!, {
+      relation: prepared.relation,
+      rowIdColumn: prepared.rowIdColumn,
+      vectorColumn: prepared.vectorColumn,
+      observations: prepared.rowCount,
+      dimensions: prepared.dimensions,
+    });
+    const distances = await computeMahalanobisDistances(
+      table.connection!,
+      {
+        relation: prepared.relation,
+        rowIdColumn: prepared.rowIdColumn,
+        vectorColumn: prepared.vectorColumn,
+      },
+      model,
+      referencePoint,
+    );
+    try {
+      const distance = `r.${quoteIdentifier(distances.distanceColumn)}`;
+      const outputs = [{ name: newColumn, expression: distance }];
+      if (scoreColumn !== undefined) {
+        outputs.push({
+          name: scoreColumn,
+          expression: `CASE WHEN max(${distance}) OVER () = 0 THEN 1::DOUBLE ELSE 1 - ${distance} / max(${distance}) OVER () END`,
+        });
+      }
+      await publishPreparedColumns(table, prepared, {
+        method: "similarityMahalanobis()",
+        parameters: { columns, referencePoint, newColumn, options },
+        result: {
+          relation: distances.relation,
+          rowIdColumn: distances.rowIdColumn,
+        },
+        outputs,
+      });
+    } finally {
+      await distances.cleanup();
+    }
+  } finally {
+    await prepared.cleanup();
+  }
+}

package/src/methods/simplify.ts

@@ -0,0 +1,64 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import ensureSpatial from "../helpers/ensureSpatial.ts";
+import findGeoColumn from "../helpers/findGeoColumn.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function simplify(
+  simpleTable: SimpleTable,
+  tolerance: number,
+  options: { column?: string; simplifyBoundary?: boolean } = {},
+) {
+  // The coverage simplification groups by rowid, which only exists on the
+  // materialized table: it executes as a barrier.
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "simplify()",
+    parameters: { tolerance, options },
+    execute: () => executeSimplify(simpleTable, tolerance, options),
+  });
+}
+
+async function executeSimplify(
+  simpleTable: SimpleTable,
+  tolerance: number,
+  options: { column?: string; simplifyBoundary?: boolean },
+): Promise<void> {
+  await ensureSpatial(simpleTable);
+
+  const column =
+    typeof options.column === "string"
+      ? options.column
+      : await findGeoColumn(simpleTable, "simplify()");
+
+  const geoType = await simpleTable.getProjection(column);
+
+  // Each row is its own coverage group: grouping by rowid directly avoids
+  // materializing a temporary row-number column. rowid is not part of
+  // SELECT *, so the other columns are listed explicitly in the GROUP BY.
+  const groupBy = [
+    "rowid",
+    ...(await simpleTable.getColumns())
+      .filter((d) => d !== column)
+      .map((d) => `${quoteIdentifier(d)}`),
+  ].join(", ");
+
+  await queryDB(
+    simpleTable,
+    `CREATE OR REPLACE TABLE ${quoteIdentifier(
+      simpleTable.name,
+    )} AS SELECT * REPLACE(ST_CoverageSimplify(ARRAY_AGG(${quoteIdentifier(column)}), ${tolerance}${
+      options.simplifyBoundary === false ? ", FAlSE" : ""
+    })::${geoType} AS ${quoteIdentifier(column)}) FROM ${quoteIdentifier(
+      simpleTable.name,
+    )} GROUP BY ${groupBy};`,
+    mergeOptions(simpleTable, {
+      table: simpleTable.name,
+      method: "simplify()",
+      parameters: { column, tolerance },
+    }),
+  );
+}

package/src/methods/skip.ts

@@ -0,0 +1,13 @@
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function skip(simpleTable: SimpleTable, count: number) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "skip()",
+    parameters: { count },
+    needsSchema: false,
+    preservesSchema: true,
+    buildSelect: (input) => `SELECT * FROM ${input} OFFSET ${count} ROWS`,
+  });
+}

package/src/methods/sort.ts

@@ -0,0 +1,52 @@
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+
+export default function sort(
+  simpleTable: SimpleTable,
+  order: { [key: string]: "asc" | "desc" } | null = null,
+  options: {
+    lang?: { [key: string]: string };
+  } = {},
+) {
+  const capturedOrder = order === null ? null : { ...order };
+  const capturedOptions = {
+    ...options,
+    lang: options.lang === undefined ? undefined : { ...options.lang },
+  };
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "sort()",
+    parameters: { order: capturedOrder, options: capturedOptions },
+    needsSchema: false,
+    preservesSchema: true,
+    buildSelect: (input) => sortSelect(input, capturedOrder, capturedOptions),
+  });
+}
+
+function sortSelect(
+  input: string,
+  order: { [key: string]: "asc" | "desc" } | null,
+  options: {
+    lang?: { [key: string]: string };
+  } = {},
+) {
+  let query = `SELECT * FROM ${input}
+    ORDER BY`;
+
+  if (order === null) {
+    query += " ALL,";
+  } else {
+    for (const column of Object.keys(order)) {
+      if (options.lang && options.lang[column]) {
+        query += `\n${quoteIdentifier(column)} COLLATE ${options.lang[column]} ${order[
+          column
+        ].toUpperCase()},`;
+      } else {
+        query += `\n${quoteIdentifier(column)} ${order[column].toUpperCase()},`;
+      }
+    }
+  }
+
+  return query.slice(0, query.length - 1);
+}

package/src/methods/splitExtract.ts

@@ -0,0 +1,29 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function splitExtract(
+  simpleTable: SimpleTable,
+  column: string,
+  separator: string,
+  index: number,
+  newColumn: string,
+) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "splitExtract()",
+    parameters: { column, separator, index, newColumn },
+    values: [separator],
+    needsSchema: true,
+    buildSelect: (input, types) => {
+      // Writing back to the split column itself is allowed (in-place); any
+      // other existing column is a duplicate, like v1's ALTER TABLE ADD.
+      assertNewColumns(types, [newColumn], "splitExtract()", [column]);
+      const expression = `SPLIT_PART(${quoteIdentifier(column)}, ?, ${index + 1})`;
+      return column === newColumn
+        ? `SELECT * REPLACE (${expression} AS ${quoteIdentifier(newColumn)}) FROM ${input}`
+        : `SELECT *, ${expression} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/splitSpread.ts

@@ -0,0 +1,140 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function splitSpread(
+  simpleTable: SimpleTable,
+  column: string,
+  separator: string,
+  newColumns: string[],
+  options: {
+    strict?: boolean;
+  } = {},
+) {
+  newColumns = [...newColumns];
+  if (newColumns.length === 0) {
+    throw new Error("splitSpread() newColumns must contain at least one column.");
+  }
+  options = structuredClone(options);
+  // The pre-validation queries the data, so splitSpread can't be expressed
+  // as a single SELECT over its input: it executes as a barrier.
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "splitSpread()",
+    parameters: { column, separator, newColumns, options },
+    execute: () => executeSplitSpread(simpleTable, column, separator, newColumns, options),
+  });
+}
+
+async function executeSplitSpread(
+  simpleTable: SimpleTable,
+  column: string,
+  separator: string,
+  newColumns: string[],
+  options: {
+    strict?: boolean;
+  },
+): Promise<void> {
+  const partCount = newColumns.length;
+  const strict = options.strict !== false;
+
+  // A SELECT *, expr AS col colliding with an existing column would be
+  // silently renamed by DuckDB (col -> col_1) instead of erroring, unlike
+  // the ALTER TABLE ADD this used to run.
+  assertNewColumns(await simpleTable.getTypes(), newColumns, "splitSpread()");
+
+  if (strict) {
+    // Both bounds come from the same scan instead of one query each.
+    const partsResult = await queryDB(
+      simpleTable,
+      `SELECT
+        MAX(ARRAY_LENGTH(STRING_SPLIT(${quoteIdentifier(column)}, ?))) AS max_parts,
+        MIN(ARRAY_LENGTH(STRING_SPLIT(${quoteIdentifier(column)}, ?))) AS min_parts
+      FROM ${quoteIdentifier(simpleTable.name)}`,
+      mergeOptions(simpleTable, {
+        table: simpleTable.name,
+        method: "splitSpread()",
+        parameters: { column, separator, newColumns },
+        returnData: true,
+        values: [separator, separator],
+      }),
+    );
+
+    if (partsResult && partsResult.length > 0) {
+      const maxParts = partsResult[0].max_parts as number;
+      const minParts = partsResult[0].min_parts as number;
+
+      if (maxParts > partCount) {
+        // Get the first 5 rows with more parts than expected
+        const problematicRows = await queryDB(
+          simpleTable,
+          `SELECT ${quoteIdentifier(column)}
+         FROM ${quoteIdentifier(simpleTable.name)}
+         WHERE ARRAY_LENGTH(STRING_SPLIT(${quoteIdentifier(column)}, ?)) > ${partCount}
+         LIMIT 5`,
+          mergeOptions(simpleTable, {
+            table: simpleTable.name,
+            method: "splitSpread()",
+            parameters: { column, separator, newColumns },
+            returnData: true,
+            values: [separator],
+          }),
+        );
+
+        const exampleRows = problematicRows
+          ? problematicRows.map((row) => row[column]).join("\n  - ")
+          : "";
+
+        throw new Error(
+          `Some rows contain more values after splitting (${maxParts}) than the number of new columns specified (${partCount}).
+When splitting by '${separator}', each row must produce at most ${partCount} value(s) to fit in the columns: ${newColumns.join(
+            ", ",
+          )}.
+
+First 5 rows with too many values:\n  - ${exampleRows}`,
+        );
+      }
+
+      if (minParts < partCount) {
+        console.warn(
+          `splitSpread() warning: Some rows contain fewer values after splitting (${minParts}) than the number of new columns (${partCount}). Empty strings will be used for missing values.`,
+        );
+      }
+    }
+  }
+
+  const split = splitSpreadQuery(simpleTable.name, column, separator, newColumns);
+  await queryDB(
+    simpleTable,
+    split.query,
+    mergeOptions(simpleTable, {
+      table: simpleTable.name,
+      method: "splitSpread()",
+      parameters: { column, separator, newColumns },
+      values: split.values,
+    }),
+  );
+}
+
+function splitSpreadQuery(
+  table: string,
+  column: string,
+  separator: string,
+  newColumns: string[],
+): { query: string; values: string[] } {
+  // All columns are added in a single rewrite, so the table is scanned once
+  // instead of once per ALTER + once per UPDATE.
+  const splitColumns = newColumns
+    .map(
+      (col, i) => `SPLIT_PART(${quoteIdentifier(column)}, ?, ${i + 1}) AS ${quoteIdentifier(col)}`,
+    )
+    .join(", ");
+
+  const query = `CREATE OR REPLACE TABLE ${quoteIdentifier(
+    table,
+  )} AS SELECT *, ${splitColumns} FROM ${quoteIdentifier(table)}`;
+  return { query, values: newColumns.map(() => separator) };
+}

package/src/methods/stream.ts

@@ -0,0 +1,79 @@
+import cleanSQL from "../helpers/cleanSQL.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import { makeConverter } from "../helpers/runQuery.ts";
+import SDAError from "../class/SDAError.ts";
+import flushAllTables from "../helpers/flushAllTables.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import observeQuery from "../helpers/observeQuery.ts";
+
+export default async function* stream(
+  simpleTable: SimpleTable,
+  options: {
+    columns?: string | string[];
+    conditions?: string;
+  } = {},
+): AsyncGenerator<{ [key: string]: unknown }, void, undefined> {
+  if (simpleTable.connection === undefined) {
+    await simpleTable.sdb.start();
+    simpleTable.db = simpleTable.sdb.db;
+    simpleTable.connection = simpleTable.sdb.connection;
+  }
+
+  // stream() reads directly from the connection instead of going through
+  // queryDB, so it must flush the pending chains itself before yielding.
+  await flushAllTables(simpleTable.sdb);
+
+  const columns = options.columns
+    ? typeof options.columns === "string"
+      ? [options.columns]
+      : options.columns
+    : undefined;
+  const query = cleanSQL(
+    `SELECT ${
+      columns ? columns.map((d) => `${quoteIdentifier(d)}`).join(", ") : "*"
+    } FROM ${quoteIdentifier(simpleTable.name)}${
+      options.conditions ? ` WHERE ${options.conditions}` : ""
+    };`,
+    simpleTable.sdb.expressionSyntax,
+  );
+  try {
+    await observeQuery(simpleTable.connection, query, [], {
+      logSQL: simpleTable.sdb.logSQL,
+      explainSQL: simpleTable.sdb.explainSQL,
+    });
+    const result = await simpleTable.connection.stream(query);
+    const columnNames = result.deduplicatedColumnNames();
+    const columnTypes = result.columnTypes();
+    if (columnTypes.some((type) => type.toString().toLowerCase().includes("geometry"))) {
+      throw new Error("The query returns geometry columns. Use getGeoData() instead.");
+    }
+    const converters = columnTypes.map((type, i) =>
+      makeConverter(type, columnNames[i], simpleTable.name),
+    );
+    const columnCount = columnNames.length;
+
+    while (true) {
+      const chunk = await result.fetchChunk();
+      if (!chunk || chunk.rowCount === 0) {
+        break;
+      }
+      for (const rawRow of chunk.getRows()) {
+        const row: { [key: string]: unknown } = {};
+        for (let i = 0; i < columnCount; i++) {
+          row[columnNames[i]] = converters[i](rawRow[i]);
+        }
+        yield row;
+      }
+    }
+  } catch (error) {
+    if (error instanceof SDAError) {
+      throw error;
+    }
+    throw new SDAError({
+      method: "stream()",
+      parameters: { options },
+      query,
+      cause: error,
+    });
+  }
+}