@nshiab/simple-data-analysis-core 2.1.9 → 2.1.11

package/script/helpers/wrapString.js

@@ -1,11 +1,17 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = wrapString;
+const splitGraphemes_js_1 = __importDefault(require("./splitGraphemes.js"));
 /**
  * Wraps a string to a specified maximum width, attempting to break at word
  * boundaries when possible. If a single word exceeds the maximum width, it
- * will be broken at the character boundary. This function is primarily used
- * for preparing text to be displayed in console tables.
+ * will be broken at a grapheme boundary, keeping combining accents with their
+ * letters. This function is primarily used for preparing text to be displayed
+ * in console tables. Widths count graphemes, not terminal columns for wide
+ * characters or emoji.
  *
  * @example
  * ```typescript
@@ -29,56 +35,64 @@ exports.default = wrapString;
  *
  * @param str - The string to wrap.
  *
- * @param maxWidth - The maximum width of each line.
+ * @param maxWidth - The maximum number of graphemes in each line.
  *
  * @param wordWrap - If true, attempts to break at word boundaries. If false,
- *   breaks at character boundaries. Defaults to `true`.
+ *   breaks at grapheme boundaries. Defaults to `true`.
  *
  * @returns The wrapped string with newline characters inserted at appropriate
  *   positions.
  */
 function wrapString(str, maxWidth, wordWrap = true) {
-    if (str.length <= maxWidth) {
+    const graphemes = (0, splitGraphemes_js_1.default)(str);
+    if (graphemes.length <= maxWidth) {
         return str;
     }
     if (!wordWrap) {
-        // Simple character-based wrapping
+        // Simple grapheme-based wrapping
         const lines = [];
-        for (let i = 0; i < str.length; i += maxWidth) {
-            lines.push(str.slice(i, i + maxWidth));
+        for (let i = 0; i < graphemes.length; i += maxWidth) {
+            lines.push(graphemes.slice(i, i + maxWidth).join(""));
         }
         return lines.join("\n");
     }
     // Word-aware wrapping
     const lines = [];
     let currentLine = "";
+    let currentWidth = 0;
     const words = str.split(/(\s+)/); // Split but keep whitespace
     for (const word of words) {
+        const wordGraphemes = (0, splitGraphemes_js_1.default)(word);
+        const wordWidth = wordGraphemes.length;
         // If adding this word would exceed maxWidth
-        if (currentLine.length + word.length > maxWidth) {
+        if (currentWidth + wordWidth > maxWidth) {
             // If current line is not empty, save it
             if (currentLine.length > 0) {
                 lines.push(currentLine.trimEnd());
                 currentLine = "";
+                currentWidth = 0;
             }
             // If the word itself is longer than maxWidth, break it
-            if (word.length > maxWidth) {
-                for (let i = 0; i < word.length; i += maxWidth) {
-                    const chunk = word.slice(i, i + maxWidth);
-                    if (i + maxWidth < word.length) {
+            if (wordWidth > maxWidth) {
+                for (let i = 0; i < wordWidth; i += maxWidth) {
+                    const chunk = wordGraphemes.slice(i, i + maxWidth).join("");
+                    if (i + maxWidth < wordWidth) {
                         lines.push(chunk);
                     }
                     else {
                         currentLine = chunk;
+                        currentWidth = wordWidth - i;
                     }
                 }
             }
             else {
                 currentLine = word;
+                currentWidth = wordWidth;
             }
         }
         else {
             currentLine += word;
+            currentWidth += wordWidth;
         }
     }
     // Add any remaining text

package/script/methods/umap.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 type Options = NonNullable<Parameters<SimpleTable["umap"]>[1]>;
-export default function umap(table: SimpleTable, column: string, options?: Options): void;
+export default function umap(table: SimpleTable, columns: string | string[], options?: Options): void;
 export {};
 //# sourceMappingURL=umap.d.ts.map

package/script/methods/umap.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"umap.d.ts","sourceRoot":"","sources":["../../src/methods/umap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAavD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,OAAY,QAStB"}
+{"version":3,"file":"umap.d.ts","sourceRoot":"","sources":["../../src/methods/umap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAcvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE,OAAY,QAUtB"}

package/script/methods/umap.js

@@ -9,22 +9,24 @@ const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const buildUmapGraph_js_1 = __importDefault(require("../helpers/buildUmapGraph.js"));
+const prepareNumericFeatures_js_1 = __importDefault(require("../helpers/prepareNumericFeatures.js"));
 const prepareUmapLayoutOptions_js_1 = __importDefault(require("../helpers/prepareUmapLayoutOptions.js"));
 const readScalarNumber_js_1 = __importDefault(require("../helpers/readScalarNumber.js"));
 const optimizeUmapLayout_js_1 = __importDefault(require("../helpers/optimizeUmapLayout.js"));
 const initialUmapCoordinates_js_1 = __importDefault(require("../helpers/initialUmapCoordinates.js"));
 const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
 const node_api_1 = require("@duckdb/node-api");
-function umap(table, column, options = {}) {
+function umap(table, columns, options = {}) {
+    const selected = typeof columns === "string" ? columns : [...columns];
     const settings = structuredClone(options);
     (0, queueOp_js_1.default)(table, {
         kind: "barrier",
         method: "umap()",
-        parameters: { column, options: settings },
-        execute: () => execute(table, column, settings),
+        parameters: { columns: selected, options: settings },
+        execute: () => execute(table, selected, settings),
     });
 }
-async function execute(table, column, options) {
+async function execute(table, columns, options) {
     const { neighbors = 15, metric = "euclidean" } = options;
     if (!Number.isSafeInteger(neighbors) || neighbors < 2) {
         throw new Error("neighbors must be a safe integer of at least 2.");
@@ -33,22 +35,10 @@ async function execute(table, column, options) {
     if (!["euclidean", "cosine"].includes(metric)) {
         throw new Error("metric must be euclidean or cosine.");
     }
-    const types = await table.getTypes();
-    const columns = Object.keys(types);
-    const resolve = (name) => {
-        const found = columns.find((c) => c.toLowerCase() === name.toLowerCase());
-        if (found === undefined)
-            throw new Error(`Column ${name} does not exist.`);
-        return found;
-    };
-    const vector = resolve(column);
-    if (columns.some((c) => ["umapx", "umapy"].includes(c.toLowerCase()))) {
+    const sourceColumns = await table.getColumns();
+    if (sourceColumns.some((c) => ["umapx", "umapy"].includes(c.toLowerCase()))) {
         throw new Error("UMAP output columns umapX and umapY must not already exist.");
     }
-    if (!/^(FLOAT|DOUBLE|TINYINT|SMALLINT|INTEGER|BIGINT|HUGEINT|UTINYINT|USMALLINT|UINTEGER|UBIGINT|UHUGEINT|DECIMAL\(\d+,\d+\))\[\d*\]$/
-        .test(types[vector])) {
-        throw new Error("UMAP requires a one-dimensional numeric LIST or ARRAY column.");
-    }
     const connection = table.connection;
     const [sourceOid, sourceTemporary] = (await connection.runAndReadAll(`SELECT table_oid,temporary FROM duckdb_tables()
       WHERE lower(table_name)=lower($1) AND schema_name=current_schema()
@@ -58,10 +48,18 @@ async function execute(table, column, options) {
         throw new Error("UMAP requires a materialized source table.");
     }
     const q = quoteIdentifier_js_1.default;
+    const count = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${q(table.name)}`);
+    if (count < 3 || count > 0x7fffffff) {
+        throw new Error("UMAP requires between 3 and 2147483647 rows.");
+    }
+    const prepared = await (0, prepareNumericFeatures_js_1.default)(table, typeof columns === "string"
+        ? { kind: "vector", column: columns }
+        : { kind: "scalars", columns }, { method: "umap()" });
+    const dimensions = prepared.dimensions;
     const suffix = crypto.randomUUID().replaceAll("-", "");
     const name = (part) => `__sda_umap_${suffix}_${part}`;
-    const ordinal = q(name("ordinal"));
-    const source = q(table.name), snapshot = q(name("snapshot"));
+    const ordinal = q(prepared.rowIdColumn);
+    const source = q(table.name), snapshot = q(prepared.relation);
     const layoutName = name("layout"), layout = q(layoutName);
     const names = {
         rows: q(name("rows")),
@@ -74,33 +72,15 @@ async function execute(table, column, options) {
         graph: q(name("graph")),
     };
     const scratch = [
-        snapshot,
         layout,
         ...Object.entries(names)
             .filter(([key]) => key !== "hnsw").map(([, value]) => value),
     ];
     try {
-        // Retain payloads in native DuckDB types. A private ordinal distinguishes
-        // duplicate rows and is independent of any user column named rowid.
-        await connection.run(`CREATE TEMP TABLE ${snapshot} AS
-      SELECT *, row_number() OVER () - 1 AS ${ordinal} FROM ${source}`);
-        const count = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${snapshot}`);
-        if (count < 3 || count > 0x7fffffff) {
-            throw new Error("UMAP requires between 3 and 2147483647 rows.");
-        }
-        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${snapshot}
-      WHERE ${q(vector)} IS NULL OR len(${q(vector)})=0
-        OR list_count(${q(vector)}) != len(${q(vector)})
-        OR NOT list_bool_and(list_transform(${q(vector)}, x -> isfinite(x::DOUBLE)))`)) {
-            throw new Error("Vectors must be nonempty, non-null and contain finite numbers only.");
-        }
-        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(DISTINCT len(${q(vector)})) FROM ${snapshot}`) !== 1) {
-            throw new Error("Vectors must have equal dimensions.");
-        }
-        const dimensions = await (0, readScalarNumber_js_1.default)(connection, `SELECT len(${q(vector)}) FROM ${snapshot} LIMIT 1`);
         await connection.run(`CREATE TEMP TABLE ${names.rows} AS
       SELECT (row_number() OVER (ORDER BY ${ordinal})-1)::INTEGER AS vertex,
-        ${ordinal} AS ordinal, ${q(vector)}::DOUBLE[${dimensions}] AS vec FROM ${snapshot}`);
+        ${ordinal} AS ordinal, ${q(prepared.vectorColumn)} AS vec FROM ${snapshot}`);
+        await connection.run(`ALTER TABLE ${snapshot} DROP COLUMN ${q(prepared.vectorColumn)}`);
         if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.rows}
       WHERE NOT isfinite(array_inner_product(vec,vec)) OR array_inner_product(vec,vec)>1e300
       ${metric === "cosine" ? "OR array_inner_product(vec,vec)=0" : ""}`)) {
@@ -137,12 +117,12 @@ async function execute(table, column, options) {
         await connection.run("BEGIN TRANSACTION");
         try {
             await (0, queryDB_js_1.default)(table, `CREATE OR REPLACE ${sourceTemporary ? "TEMP " : ""}TABLE ${source} AS
-        SELECT ${columns.map((c) => `s.${q(c)}`).join(",")}, l.x AS "umapX", l.y AS "umapY"
+        SELECT ${prepared.sourceColumns.map((c) => `s.${q(c)}`).join(",")}, l.x AS "umapX", l.y AS "umapY"
         FROM ${snapshot} s JOIN ${names.rows} r ON s.${ordinal}=r.ordinal
         JOIN ${layout} l ON r.vertex=l.vertex ORDER BY s.${ordinal}`, (0, mergeOptions_js_1.default)(table, {
                 table: table.name,
                 method: "umap()",
-                parameters: { column, options },
+                parameters: { columns, options },
                 noClean: true,
             }));
             for (const sql of indexes)
@@ -158,5 +138,6 @@ async function execute(table, column, options) {
         for (const relation of scratch) {
             await connection.run(`DROP TABLE IF EXISTS ${relation}`);
         }
+        await prepared.cleanup();
     }
 }