@nshiab/simple-data-analysis-core 2.1.12 → 2.1.14

package/src/helpers/updateColumnsWithJS.ts

@@ -0,0 +1,199 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+import readMutationRows from "./readMutationRows.ts";
+import { retainRegisteredTables } from "./tableRegistry.ts";
+import { prepareArray } from "../methods/loadArray.ts";
+import appendColumnBatches from "./appendColumnBatches.ts";
+import parseDuckDBType from "./parseDuckDBType.ts";
+
+/**
+ * Generates columns from selected input columns, keeping other SQL values in
+ * DuckDB. Call from an asynchronous extension barrier. The callback must return
+ * one row per input row in the same order; only the declared output columns are
+ * stored. Existing output columns are replaced, with types inferred from the
+ * generated values. Generation runs sequentially in bounded batches (1,000 rows
+ * by default); callbacks must not depend on receiving the whole table. Non-null
+ * output types must agree across batches. All-null batches defer type inference.
+ * Only one batch of inputs and outputs is retained by this helper in JavaScript;
+ * callers must also bound their own buffering. Reads finish before callbacks or
+ * writes use the shared connection. The original table is replaced only after
+ * staging succeeds. Empty tables are unchanged and do not invoke the callback;
+ * outputs that remain null in every batch use VARCHAR.
+ *
+ * Untouched geometry columns remain in DuckDB, preserving their values, types
+ * and CRS without GeoJSON conversion. Geometry columns cannot be selected as
+ * inputs or overwritten as outputs; use updateWithJS() for editable GeoJSON.
+ * This SQL-only preservation path accepts any source CRS.
+ *
+ * @example
+ * ```ts
+ * queueAsyncBarrier(table, {
+ *   method: "label()",
+ *   parameters: null,
+ *   execute: () => updateColumnsWithJS(table, ["name"], ["label"], async (rows) =>
+ *     rows.map((row) => ({ label: String(row.name).toUpperCase() })),
+ *     { batchSize: 500 }),
+ * });
+ * await table.log();
+ * ```
+ *
+ * @param table - Table to enrich.
+ *
+ * @param inputColumns - Columns available to the callback.
+ *
+ * @param outputColumns - Generated columns to add or replace.
+ *
+ * @param generate - Produces one result per input row, in input order.
+ *
+ * @param options - Controls the maximum rows read, generated, and staged at once.
+ *
+ * @returns Resolves after merging the generated columns into the table.
+ */
+export default async function updateColumnsWithJS(
+  table: SimpleTable,
+  inputColumns: string[],
+  outputColumns: string[],
+  generate: (rows: { [key: string]: unknown }[]) => Promise<{ [key: string]: unknown }[]>,
+  options: {
+    /** Maximum rows per callback and transfer batch. Defaults to 1,000.
+     * @example
+     * ```ts
+     * { batchSize: 500 }
+     * ```
+     */
+    batchSize?: number;
+  } = {},
+): Promise<void> {
+  const batchSize = options.batchSize ?? 1000;
+  if (!Number.isSafeInteger(batchSize) || batchSize < 1) {
+    throw new Error("batchSize must be a positive safe integer.");
+  }
+  if (inputColumns.length === 0 || outputColumns.length === 0) {
+    throw new Error("Input and output columns must not be empty.");
+  }
+  if (new Set(outputColumns.map((name) => name.toLowerCase())).size !== outputColumns.length) {
+    throw new Error("Output columns must be unique.");
+  }
+  const types = await table.getTypes();
+  for (const [name, type] of Object.entries(types)) {
+    if (
+      type.toUpperCase().startsWith("GEOMETRY") &&
+      [...inputColumns, ...outputColumns].some(
+        (column) => column.toLowerCase() === name.toLowerCase(),
+      )
+    ) {
+      throw new Error(
+        `Geometry column ${JSON.stringify(
+          name,
+        )} cannot be an input or output of updateColumnsWithJS(); use updateWithJS() for editable GeoJSON.`,
+      );
+    }
+  }
+  const suffix = crypto.randomUUID().replaceAll("-", "");
+  const id = `__sda_id_${suffix}`;
+  const snapshot = `__sda_source_${suffix}`;
+  const staged = table.sdb.newTable(`__sda_generated_${suffix}`);
+  const q = quoteIdentifier;
+  try {
+    // A SQL snapshot gives every row a stable identity, including duplicate rows
+    // and user columns named rowid. No untouched values cross the JS boundary.
+    await table.sdb.customQuery(
+      `CREATE TEMP TABLE ${q(snapshot)} AS SELECT *, row_number() OVER () AS ${q(
+        id,
+      )} FROM ${q(table.name)}`,
+    );
+    const knownTypes = new Map<string, string>();
+    let lastId = "0";
+    let hasRows = false;
+    while (true) {
+      const upperId = BigInt(lastId) + BigInt(batchSize);
+      const { rows } = await readMutationRows(
+        table.connection!,
+        `SELECT ${q(id)}, ${inputColumns.map(q).join(", ")} FROM ${q(
+          snapshot,
+        )} WHERE ${q(id)} > ${lastId} AND ${q(id)} <= ${upperId}
+        ORDER BY ${q(id)} LIMIT ${batchSize}`,
+      );
+      if (rows.length === 0) break;
+      const ids = rows.map((row) => String(row[id]));
+      lastId = ids[ids.length - 1];
+      for (const row of rows) delete row[id];
+      const generated = await generate(rows);
+      if (generated.length !== ids.length) {
+        throw new Error("Column generation must return one row per input row.");
+      }
+      // Generated vectors retain this helper's existing inference contract.
+      const prepared = prepareArray(
+        generated.map((row, i) => ({
+          ...Object.fromEntries(outputColumns.map((name) => [name, row[name]])),
+          [id]: ids[i],
+        })),
+      );
+      const batchTypes = new Map(prepared.keys.map((name, i) => [name, prepared.types[i]]));
+      for (const name of outputColumns) {
+        const nonNull = generated.some((row) => row[name] !== null && row[name] !== undefined);
+        if (!nonNull) continue;
+        const previous = knownTypes.get(name);
+        const current = batchTypes.get(name)!;
+        if (previous !== undefined && previous !== current) {
+          throw new Error(
+            `Generated column ${JSON.stringify(name)} changed type from ${previous} to ${current}.`,
+          );
+        }
+        if (previous === undefined) {
+          knownTypes.set(name, current);
+          if (hasRows) {
+            await table.sdb.customQuery(
+              `ALTER TABLE ${q(staged.name)} ALTER COLUMN ${q(name)} TYPE ${current} USING NULL`,
+            );
+          }
+        }
+      }
+      if (!hasRows) {
+        await table.sdb.customQuery(
+          `CREATE TEMP TABLE ${q(staged.name)} (${prepared.keys
+            .map((name, i) => `${q(name)} ${prepared.types[i]}`)
+            .join(", ")})`,
+        );
+        hasRows = true;
+      }
+      // All-null batches use the column's established type, avoiding a separate
+      // batch table and SQL copy just to cast nulls into the staged schema.
+      await appendColumnBatches(
+        table.connection!,
+        staged.name,
+        prepared.keys.map((name, i) => parseDuckDBType(knownTypes.get(name) ?? prepared.types[i])),
+        prepared.rowCount,
+        (column, start, end) => prepared.columnsData[column].slice(start, end),
+      );
+    }
+    if (!hasRows) return;
+    const outputs = new Map(outputColumns.map((name) => [name.toLowerCase(), name]));
+    const columns = Object.keys(types).map((name) => {
+      const replacement = outputs.get(name.toLowerCase());
+      return replacement === undefined
+        ? `original.${q(name)}`
+        : `generated.${q(replacement)} AS ${q(name)}`;
+    });
+    const existing = new Set(Object.keys(types).map((name) => name.toLowerCase()));
+    for (const name of outputColumns) {
+      if (!existing.has(name.toLowerCase())) {
+        columns.push(`generated.${q(name)}`);
+      }
+    }
+    await table.sdb.customQuery(
+      `CREATE OR REPLACE TABLE ${q(table.name)} AS SELECT ${columns.join(
+        ", ",
+      )} FROM ${q(snapshot)} original JOIN ${q(
+        staged.name,
+      )} generated ON original.${q(id)} = CAST(generated.${q(
+        id,
+      )} AS BIGINT) ORDER BY original.${q(id)}`,
+    );
+  } finally {
+    await table.sdb.customQuery(
+      `DROP TABLE IF EXISTS ${q(snapshot)}; DROP TABLE IF EXISTS ${q(staged.name)};`,
+    );
+    retainRegisteredTables(table.sdb, (item) => item !== staged);
+  }
+}

package/src/helpers/validateGraphRouteResultSchema.ts

@@ -0,0 +1,30 @@
+import foldIdentifier from "./foldIdentifier.ts";
+import type { TableSchema } from "./pendingOps.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+const resultColumns = ["pathId", "step", "weight", "total"] as const;
+
+/** Validates that route result columns do not replace original input columns. */
+export default function validateGraphRouteResultSchema(
+  schema: TableSchema,
+  method: string,
+  additionalResultColumns: readonly string[] = [],
+  includeTotal = true,
+): void {
+  for (const resultColumn of [...resultColumns, ...additionalResultColumns]) {
+    if (resultColumn === "total" && !includeTotal) continue;
+    const folded = foldIdentifier(resultColumn);
+    const inputColumn = Object.keys(schema).find((column) => foldIdentifier(column) === folded);
+    if (inputColumn !== undefined) {
+      throw new Error(
+        `${method} cannot add ${quoteIdentifier(
+          resultColumn,
+        )} because input column ${quoteIdentifier(
+          inputColumn,
+        )} conflicts with that result column. Rename ${quoteIdentifier(
+          inputColumn,
+        )} with renameColumns() before calculating the graph.`,
+      );
+    }
+  }
+}

package/src/helpers/validateGraphTemporalEvents.ts

@@ -0,0 +1,63 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import mergeOptions from "./mergeOptions.ts";
+import prepareGraphTemporalSql, {
+  type PreparedGraphTemporalOptions,
+} from "./prepareGraphTemporalSql.ts";
+import queryDB from "./queryDB.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+/** Rejects invalid selected timestamps before a chronological graph runs. */
+export default async function validateGraphTemporalEvents(
+  simpleTable: SimpleTable,
+  options: PreparedGraphTemporalOptions,
+  method: string,
+  parameters: { [key: string]: unknown },
+): Promise<void> {
+  const schema = await simpleTable.getTypes();
+  const temporal = prepareGraphTemporalSql(schema, options, method);
+  const q = quoteIdentifier;
+  const eventsName = "__graph_temporal_events";
+  const events = q(eventsName);
+  const eventStart = `${events}.${q("__event_start")}`;
+  const eventEnd = `${events}.${q("__event_end")}`;
+  const rows = await queryDB(
+    simpleTable,
+    `SELECT CASE
+      WHEN ${eventStart} IS NULL THEN 1
+      WHEN ${eventEnd} IS NULL THEN 2
+      WHEN NOT isfinite(${eventStart}) THEN 3
+      WHEN NOT isfinite(${eventEnd}) THEN 4
+      ELSE 5
+    END AS ${q("reason")}
+    FROM (
+      SELECT ${temporal.eventSelections().join(", ")}
+      FROM ${q(simpleTable.name)}
+    ) AS ${events}
+    WHERE (${temporal.eventValidity(eventsName)}) IS NOT TRUE
+    LIMIT 1`,
+    mergeOptions(simpleTable, {
+      table: simpleTable.name,
+      method,
+      parameters,
+      returnData: true,
+    }),
+  );
+  const reason = rows?.[0]?.reason;
+  if (reason === undefined) return;
+
+  const startColumn = temporal.startTimeColumn ?? temporal.endTimeColumn!;
+  const endColumn = temporal.endTimeColumn ?? temporal.startTimeColumn!;
+  const message =
+    reason === 1
+      ? `selected start-time column ${q(startColumn)} contains a null timestamp.`
+      : reason === 2
+        ? `selected end-time column ${q(endColumn)} contains a null timestamp.`
+        : reason === 3
+          ? `selected start-time column ${q(startColumn)} contains an infinite timestamp.`
+          : reason === 4
+            ? `selected end-time column ${q(endColumn)} contains an infinite timestamp.`
+            : `selected end-time column ${q(
+                endColumn,
+              )} contains a timestamp before its start in ${q(startColumn)}.`;
+  throw new TypeError(`${method} ${message}`);
+}

package/src/helpers/validateNumericalAnalysis.ts

@@ -0,0 +1,83 @@
+import quoteIdentifier from "./quoteIdentifier.ts";
+import keepNumericalColumns from "./keepNumericalColumns.ts";
+import type { TableSchema } from "./pendingOps.ts";
+
+/**
+ * Validates the column options shared by numerical pair analyses.
+ *
+ * @example
+ * ```ts
+ * const columns = validateNumericalAnalysis(
+ *   { x: "DOUBLE", y: "BIGINT" },
+ *   { x: "x" },
+ *   "correlations()",
+ *   "measurements",
+ * );
+ * ```
+ *
+ * @param types - The schema of the table being analyzed.
+ *
+ * @param options - The optional x and y columns selected by the caller.
+ *
+ * @param method - The SDA method performing the analysis.
+ *
+ * @param tableName - The name of the table being analyzed.
+ *
+ * @returns All numeric columns in the table schema.
+ */
+export default function validateNumericalAnalysis(
+  types: TableSchema,
+  options: { x?: string; y?: string },
+  method: string,
+  tableName: string,
+): string[] {
+  if (options.x === undefined && options.y !== undefined) {
+    throw new Error(
+      `${method} options.y cannot be used without options.x. Provide options.x, or omit options.y to analyze every numeric column pair.`,
+    );
+  }
+
+  const numericalColumns = keepNumericalColumns(types);
+  const requestedColumns = [options.x, options.y].filter(
+    (column): column is string => column !== undefined,
+  );
+  for (const column of requestedColumns) {
+    const type = types[column];
+    if (type === undefined) {
+      const availableColumns = Object.keys(types).sort();
+      throw new Error(
+        `${method} could not find column ${quoteIdentifier(column)} in table ${quoteIdentifier(
+          tableName,
+        )}. Available columns: ${availableColumns.map(quoteIdentifier).join(", ")}.`,
+      );
+    }
+    if (!numericalColumns.includes(column)) {
+      throw new Error(
+        `${method} requires numeric columns, but column ${quoteIdentifier(
+          column,
+        )} in table ${quoteIdentifier(tableName)} has type ${type}. Convert ${quoteIdentifier(
+          column,
+        )} to a numeric type first.`,
+      );
+    }
+  }
+
+  if (options.y === undefined && numericalColumns.length < 2) {
+    const found =
+      numericalColumns.length === 0 ? "" : `: ${numericalColumns.map(quoteIdentifier).join(", ")}`;
+    const missingCount = 2 - numericalColumns.length;
+    const columnsToConvert =
+      numericalColumns.length === 0
+        ? `${missingCount} columns`
+        : `${missingCount} more column${missingCount === 1 ? "" : "s"}`;
+    throw new Error(
+      `${method} requires at least 2 numeric columns in table ${quoteIdentifier(
+        tableName,
+      )}, but found ${numericalColumns.length}${found}. Convert at least ${columnsToConvert} to ${
+        missingCount === 1 ? "a numeric type" : "numeric types"
+      } first.`,
+    );
+  }
+
+  return numericalColumns;
+}

package/src/helpers/validateVectorRowIds.ts

@@ -0,0 +1,29 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+
+/** Validate the positional identity contract before bounded vector scans. */
+export default async function validateVectorRowIds(
+  connection: DuckDBConnection,
+  rows: string,
+  count: number,
+): Promise<void> {
+  // Graph state and DuckDB scratch schemas use signed INTEGER vertex ids.
+  if (!Number.isSafeInteger(count) || count < 1 || count > 2147483647) {
+    throw new Error("Vector row count must fit a positive INTEGER vertex count.");
+  }
+  const [actual, distinct, minimum, maximum, invalid] = (
+    await connection.runAndReadAll(`SELECT count(*),count(DISTINCT vertex),
+      min(vertex),max(vertex),count(*) FILTER (
+        WHERE vertex IS NULL OR vertex!=trunc(vertex)) FROM ${rows}`)
+  ).getRowsJS()[0];
+  if (
+    Number(actual) !== count ||
+    Number(distinct) !== count ||
+    Number(minimum) !== 0 ||
+    Number(maximum) !== count - 1 ||
+    Number(invalid) !== 0
+  ) {
+    throw new Error(
+      `Vector rows must contain exactly ${count} unique integer vertex ids from 0 to ${count - 1}.`,
+    );
+  }
+}

package/src/helpers/vectorDistanceExpression.ts

@@ -0,0 +1,14 @@
+/**
+ * Build a DOUBLE-precision distance expression for prepared vector aliases.
+ * Cosine callers first stabilize their private rows with stabilizeCosineVectors.
+ */
+export default function vectorDistanceExpression(
+  left: string,
+  right: string,
+  metric: "euclidean" | "cosine",
+): string {
+  if (metric === "euclidean") return `array_distance(${left},${right})`;
+  // DuckDB can return tiny positive or negative errors for equal/cosine-aligned
+  // vectors. Exact equality also makes duplicate behavior deterministic.
+  return `CASE WHEN ${left}=${right} THEN 0 ELSE greatest(0,array_cosine_distance(${left},${right})) END`;
+}

package/src/helpers/wrapString.ts

@@ -0,0 +1,102 @@
+import splitGraphemes from "./splitGraphemes.ts";
+
+/**
+ * Wraps a string to a specified maximum width, attempting to break at word
+ * boundaries when possible. If a single word exceeds the maximum width, it
+ * will be broken at a grapheme boundary, keeping combining accents with their
+ * letters. This function is primarily used for preparing text to be displayed
+ * in console tables. Widths count graphemes, not terminal columns for wide
+ * characters or emoji.
+ *
+ * @example
+ * ```typescript
+ * const text = "This is a very long sentence that needs to be wrapped";
+ * const wrapped = wrapString(text, 20);
+ * console.log(wrapped);
+ * // Output:
+ * // This is a very long
+ * // sentence that needs
+ * // to be wrapped
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Character-based wrapping (no word boundaries)
+ * const text = "Thisisaverylongword";
+ * const wrapped = wrapString(text, 10, false);
+ * // Output: Thisisaver
+ * //         ylongword
+ * ```
+ *
+ * @param str - The string to wrap.
+ *
+ * @param maxWidth - The maximum number of graphemes in each line.
+ *
+ * @param wordWrap - If true, attempts to break at word boundaries. If false,
+ *   breaks at grapheme boundaries. Defaults to `true`.
+ *
+ * @returns The wrapped string with newline characters inserted at appropriate
+ *   positions.
+ */
+export default function wrapString(str: string, maxWidth: number, wordWrap = true): string {
+  const graphemes = splitGraphemes(str);
+  if (graphemes.length <= maxWidth) {
+    return str;
+  }
+
+  if (!wordWrap) {
+    // Simple grapheme-based wrapping
+    const lines: string[] = [];
+    for (let i = 0; i < graphemes.length; i += maxWidth) {
+      lines.push(graphemes.slice(i, i + maxWidth).join(""));
+    }
+    return lines.join("\n");
+  }
+
+  // Word-aware wrapping
+  const lines: string[] = [];
+  let currentLine = "";
+  let currentWidth = 0;
+
+  const words = str.split(/(\s+)/); // Split but keep whitespace
+
+  for (const word of words) {
+    const wordGraphemes = splitGraphemes(word);
+    const wordWidth = wordGraphemes.length;
+    // If adding this word would exceed maxWidth
+    if (currentWidth + wordWidth > maxWidth) {
+      // If current line is not empty, save it
+      if (currentLine.length > 0) {
+        lines.push(currentLine.trimEnd());
+        currentLine = "";
+        currentWidth = 0;
+      }
+
+      // If the word itself is longer than maxWidth, break it
+      if (wordWidth > maxWidth) {
+        for (let i = 0; i < wordWidth; i += maxWidth) {
+          const chunk = wordGraphemes.slice(i, i + maxWidth).join("");
+          if (i + maxWidth < wordWidth) {
+            lines.push(chunk);
+          } else {
+            currentLine = chunk;
+            currentWidth = wordWidth - i;
+          }
+        }
+      } else {
+        currentLine = word;
+        currentWidth = wordWidth;
+      }
+    } else {
+      currentLine += word;
+      currentWidth += wordWidth;
+    }
+  }
+
+  // Add any remaining text
+  if (currentLine.length > 0) {
+    lines.push(currentLine.trimEnd());
+  }
+
+  return lines.join("\n");
+}

package/src/helpers/writeDataAsArrays.ts

@@ -0,0 +1,14 @@
+import dataToArrays from "./dataToArrays.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import getExtension from "./getExtension.ts";
+import { writeFileSync } from "node:fs";
+
+export default async function writeDataAsArrays(simpleTable: SimpleTable, file: string) {
+  const fileExtension = getExtension(file);
+  if (fileExtension === "json") {
+    const data = await simpleTable.getData();
+    writeFileSync(file, JSON.stringify(dataToArrays(data)));
+  } else {
+    throw new Error("The option dataAsArrays works only with json files.");
+  }
+}

package/src/helpers/writeDbMetadata.ts

@@ -0,0 +1,19 @@
+import queryDbFile from "./queryDbFile.ts";
+import type SimpleDB from "../class/SimpleDB.ts";
+import type { IndexDefinition } from "./indexDefinitions.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+import parseValue from "./parseValue.ts";
+
+export default async function writeDbMetadata(
+  sdb: SimpleDB,
+  database: string,
+  indexes: Record<string, IndexDefinition[]>,
+): Promise<void> {
+  const catalog = quoteIdentifier(database);
+  await queryDbFile(
+    sdb,
+    `CREATE SCHEMA IF NOT EXISTS ${catalog}.__sda;
+     CREATE OR REPLACE TABLE ${catalog}.__sda.metadata AS
+     SELECT 1 AS version, ${parseValue(JSON.stringify(indexes))}::VARCHAR AS indexes;`,
+  );
+}