@nshiab/simple-data-analysis-core 2.0.8 → 2.0.9

package/esm/helpers/optimizeUmapLayout.d.ts

@@ -0,0 +1,7 @@
+import type prepareUmapLayoutOptions from "./prepareUmapLayoutOptions.js";
+export default function optimizeUmapLayout(graph: {
+    source: Uint32Array;
+    target: Uint32Array;
+    weight: Float64Array;
+}, initial: Float64Array, options: ReturnType<typeof prepareUmapLayoutOptions>): Float64Array;
+//# sourceMappingURL=optimizeUmapLayout.d.ts.map

package/esm/helpers/optimizeUmapLayout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimizeUmapLayout.d.ts","sourceRoot":"","sources":["../../src/helpers/optimizeUmapLayout.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,wBAAwB,MAAM,+BAA+B,CAAC;AAS1E,MAAM,CAAC,OAAO,UAAU,kBAAkB,CACxC,KAAK,EAAE;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,YAAY,CAAA;CAAE,EACzE,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,UAAU,CAAC,OAAO,wBAAwB,CAAC,GACnD,YAAY,CA2Fd"}

package/esm/helpers/optimizeUmapLayout.js

@@ -0,0 +1,93 @@
+import umapRandom from "./umapRandom.js";
+// Internal interface. Graph entries are directed, in a stable order; the fuzzy
+// union graph supplies both orientations. Coordinates are interleaved x,y.
+// Inputs are borrowed read-only. State is O(edges + vertices), never O(n²).
+// Implements reference Euclidean UMAP SGD: scheduled attractive edges, sampled
+// repulsion, clipped gradients, symmetric attractive updates, linear decay.
+// Algorithm: https://umap-learn.readthedocs.io/en/latest/how_umap_works.html
+export default function optimizeUmapLayout(graph, initial, options) {
+    const { epochs, seed, learningRate, negativeSamples, curve: [a, b] } = options;
+    const count = initial.length / 2;
+    if (!Number.isInteger(count) || count < 3 || count > 0xffffffff) {
+        throw new Error("Expected interleaved coordinates for at least three vertices.");
+    }
+    for (const value of initial) {
+        if (!Number.isFinite(value)) {
+            throw new Error("Initial coordinates must be finite.");
+        }
+    }
+    const { source, target, weight } = graph;
+    if (!weight.length || source.length !== weight.length ||
+        target.length !== weight.length)
+        throw new Error("Invalid graph column lengths.");
+    let maximum = 0;
+    for (let edge = 0; edge < weight.length; edge++) {
+        if (source[edge] >= count || target[edge] >= count ||
+            source[edge] === target[edge])
+            throw new Error("Graph contains an invalid vertex or self edge.");
+        if (!Number.isFinite(weight[edge]) || weight[edge] < 0 || weight[edge] > 1)
+            throw new Error("Graph weights must be between zero and one.");
+        maximum = Math.max(maximum, weight[edge]);
+    }
+    if (!maximum)
+        throw new Error("Graph must contain a positive edge.");
+    const period = new Float64Array(weight.length);
+    const nextPositive = new Float64Array(weight.length);
+    const nextNegative = new Float64Array(weight.length);
+    // Reference pruning uses 500 when explicitly requesting at most ten epochs.
+    const threshold = maximum / (epochs > 10 ? epochs : 500);
+    for (let edge = 0; edge < weight.length; edge++) {
+        period[edge] = weight[edge] > 0 && weight[edge] >= threshold
+            ? epochs / ((weight[edge] / maximum) * epochs)
+            : Infinity;
+        nextPositive[edge] = period[edge];
+        nextNegative[edge] = period[edge] / negativeSamples;
+    }
+    const coordinates = initial.slice();
+    const random = umapRandom(seed ^ 0x7f4a7c15);
+    const clip = (gradient) => Math.max(-4, Math.min(4, gradient));
+    let alpha = learningRate;
+    for (let epoch = 0; epoch < epochs; epoch++) {
+        for (let edge = 0; edge < weight.length; edge++) {
+            if (nextPositive[edge] > epoch)
+                continue;
+            const u = source[edge] * 2, v = target[edge] * 2;
+            let dx = coordinates[u] - coordinates[v], dy = coordinates[u + 1] - coordinates[v + 1];
+            let squared = dx * dx + dy * dy;
+            if (squared > 0) {
+                const attraction = -2 * a * b * Math.pow(squared, b - 1) /
+                    (1 + a * Math.pow(squared, b));
+                const gx = clip(attraction * dx) * alpha, gy = clip(attraction * dy) * alpha;
+                coordinates[u] += gx;
+                coordinates[u + 1] += gy;
+                coordinates[v] -= gx;
+                coordinates[v + 1] -= gy;
+            }
+            nextPositive[edge] += period[edge];
+            const negativePeriod = period[edge] / negativeSamples;
+            const draws = Math.floor((epoch - nextNegative[edge]) / negativePeriod);
+            for (let draw = 0; draw < draws; draw++) {
+                const other = Math.floor(random() * count) * 2;
+                dx = coordinates[u] - coordinates[other];
+                dy = coordinates[u + 1] - coordinates[other + 1];
+                squared = dx * dx + dy * dy;
+                // Coincident points get zero repulsive gradient, as in current Python
+                // UMAP; this intentionally differs from UMAP-JS's constant +4 fallback.
+                if (squared > 0) {
+                    const repulsion = 2 * b /
+                        ((0.001 + squared) * (1 + a * Math.pow(squared, b)));
+                    coordinates[u] += clip(repulsion * dx) * alpha;
+                    coordinates[u + 1] += clip(repulsion * dy) * alpha;
+                }
+            }
+            nextNegative[edge] += draws * negativePeriod;
+        }
+        alpha = learningRate * (1 - epoch / epochs);
+    }
+    for (const value of coordinates) {
+        if (!Number.isFinite(value)) {
+            throw new Error("UMAP optimization produced a non-finite coordinate.");
+        }
+    }
+    return coordinates;
+}

package/esm/helpers/prepareUmapLayoutOptions.d.ts

@@ -0,0 +1,14 @@
+export default function prepareUmapLayoutOptions(options?: {
+    epochs?: number;
+    seed?: number;
+    minDistance?: number;
+    learningRate?: number;
+    negativeSamples?: number;
+}): {
+    epochs: number;
+    seed: number;
+    learningRate: number;
+    negativeSamples: number;
+    curve: [number, number];
+};
+//# sourceMappingURL=prepareUmapLayoutOptions.d.ts.map

package/esm/helpers/prepareUmapLayoutOptions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prepareUmapLayoutOptions.d.ts","sourceRoot":"","sources":["../../src/helpers/prepareUmapLayoutOptions.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,wBAAwB,CAC9C,OAAO,GAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;CACrB;;;;;;EA2BP"}

package/esm/helpers/prepareUmapLayoutOptions.js

@@ -0,0 +1,23 @@
+import fitUmapCurve from "./fitUmapCurve.js";
+// Resolve and validate settings before database work, fitting the curve once.
+export default function prepareUmapLayoutOptions(options = {}) {
+    const { epochs = 200, seed = 42, minDistance = 0.1, learningRate = 1, negativeSamples = 5, } = options;
+    for (const [key, value] of Object.entries({ epochs, negativeSamples })) {
+        if (!Number.isSafeInteger(value) || value < 1) {
+            throw new Error(`${key} must be a positive safe integer.`);
+        }
+    }
+    if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) {
+        throw new Error("seed must be an integer between 0 and 4294967295.");
+    }
+    if (!Number.isFinite(learningRate) || learningRate <= 0) {
+        throw new Error("learningRate must be finite and positive.");
+    }
+    return {
+        epochs,
+        seed,
+        learningRate,
+        negativeSamples,
+        curve: fitUmapCurve(minDistance),
+    };
+}

package/esm/helpers/readScalarNumber.d.ts

@@ -0,0 +1,3 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+export default function readScalarNumber(connection: DuckDBConnection, sql: string): Promise<number>;
+//# sourceMappingURL=readScalarNumber.d.ts.map

package/esm/helpers/readScalarNumber.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readScalarNumber.d.ts","sourceRoot":"","sources":["../../src/helpers/readScalarNumber.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,wBAA8B,gBAAgB,CAC5C,UAAU,EAAE,gBAAgB,EAC5B,GAAG,EAAE,MAAM,mBAGZ"}

package/esm/helpers/readScalarNumber.js

@@ -0,0 +1,4 @@
+// For internal queries returning a single numeric value, such as row counts.
+export default async function readScalarNumber(connection, sql) {
+    return Number((await connection.runAndReadAll(sql)).getRowsJS()[0][0]);
+}

package/esm/helpers/umapRandom.d.ts

@@ -0,0 +1,2 @@
+export default function umapRandom(seed: number): () => number;
+//# sourceMappingURL=umapRandom.d.ts.map

package/esm/helpers/umapRandom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"umapRandom.d.ts","sourceRoot":"","sources":["../../src/helpers/umapRandom.ts"],"names":[],"mappings":"AACA,MAAM,CAAC,OAAO,UAAU,UAAU,CAAC,IAAI,EAAE,MAAM,gBAQ9C"}

package/esm/helpers/umapRandom.js

@@ -0,0 +1,10 @@
+// Mulberry32: separate initialization and sampling streams, stable per seed.
+export default function umapRandom(seed) {
+    let state = seed >>> 0;
+    return () => {
+        state = (state + 0x6D2B79F5) | 0;
+        let value = Math.imul(state ^ state >>> 15, 1 | state);
+        value ^= value + Math.imul(value ^ value >>> 7, 61 | value);
+        return ((value ^ value >>> 14) >>> 0) / 4294967296;
+    };
+}

package/esm/methods/cache.js

@@ -7,7 +7,7 @@ import serializeCacheInputs, { getCacheInputLabel, } from "../helpers/serializeC
 import { createTableGenerationId, peekTableGeneration, restoreTableGeneration, } from "../helpers/tableGeneration.js";
 import { loadCacheDatabase, writeCacheDatabase, } from "../helpers/cacheDatabase.js";
 import { assertCacheTableMutation, cacheTableDependenciesMatch, captureCacheTableDependencies, changedCacheTableDependenciesMessage, recordCacheTableAccess, unchangedCacheTableDependenciesMessage, } from "../helpers/cacheTableDependencies.js";
-const CACHE_FORMAT_VERSION = "duckdb-v4";
+const CACHE_FORMAT_VERSION = "duckdb-v5";
 export default async function cache(table, compute, options = {}) {
     // If cache() is itself called from another cache computation, the outer
     // computation depends on the table produced here.

package/esm/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA4ExB"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAM1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+DxB"}

package/esm/methods/loadArray.js

@@ -1,4 +1,5 @@
-import { arrayValue, DuckDBDataChunk, DuckDBDateValue, DuckDBTimestampTZValue, DuckDBTimestampValue, DuckDBTimeValue, } from "@duckdb/node-api";
+import { arrayValue, DuckDBDateValue, DuckDBTimestampTZValue, DuckDBTimestampValue, DuckDBTimeValue, } from "@duckdb/node-api";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
 import prepareGeometry from "../helpers/prepareGeometry.js";
 import prepareJSON from "../helpers/prepareJSON.js";
 import geometryFromJSON from "../helpers/geometryFromJSON.js";
@@ -136,25 +137,7 @@ export async function executePreparedArray(simpleTable, prepared) {
     }
     try {
         await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${quoteIdentifier(staged)}(${keys.map((key, i) => `${quoteIdentifier(key)} ${storageTypes[i]}`).join(", ")})`);
-        const appender = await simpleTable.connection
-            .createAppender(staged);
-        try {
-            const duckDBTypes = storageTypes.map((d) => parseDuckDBType(d));
-            // The maximum capacity of a DuckDB data chunk is 2048 rows.
-            const chunkSize = 2000;
-            for (let start = 0; start < rowCount; start += chunkSize) {
-                const end = Math.min(start + chunkSize, rowCount);
-                const dataChunk = DuckDBDataChunk.create(duckDBTypes, end - start);
-                for (let i = 0; i < keys.length; i++) {
-                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-                }
-                appender.appendDataChunk(dataChunk);
-            }
-            appender.flushSync();
-        }
-        finally {
-            appender.closeSync();
-        }
+        await appendColumnBatches(simpleTable.connection, staged, storageTypes.map((type) => parseDuckDBType(type)), rowCount, (column, start, end) => columnsData[column].slice(start, end));
         if (!needsConversion)
             return;
         // CTAS is atomic: conversion must finish before the destination is replaced.

package/esm/methods/umap.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["umap"]>[1]>;
+export default function umap(table: SimpleTable, column: string, options?: Options): void;
+export {};
+//# sourceMappingURL=umap.d.ts.map

package/esm/methods/umap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"umap.d.ts","sourceRoot":"","sources":["../../src/methods/umap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAavD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,OAAY,QAStB"}

package/esm/methods/umap.js

@@ -0,0 +1,156 @@
+import queueOp from "../helpers/queueOp.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import queryDB from "../helpers/queryDB.js";
+import mergeOptions from "../helpers/mergeOptions.js";
+import buildUmapGraph from "../helpers/buildUmapGraph.js";
+import prepareUmapLayoutOptions from "../helpers/prepareUmapLayoutOptions.js";
+import readScalarNumber from "../helpers/readScalarNumber.js";
+import optimizeUmapLayout from "../helpers/optimizeUmapLayout.js";
+import initialUmapCoordinates from "../helpers/initialUmapCoordinates.js";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
+import { DOUBLE, INTEGER } from "@duckdb/node-api";
+export default function umap(table, column, options = {}) {
+    const settings = structuredClone(options);
+    queueOp(table, {
+        kind: "barrier",
+        method: "umap()",
+        parameters: { column, options: settings },
+        execute: () => execute(table, column, settings),
+    });
+}
+async function execute(table, column, options) {
+    const { neighbors = 15, metric = "euclidean" } = options;
+    if (!Number.isSafeInteger(neighbors) || neighbors < 2) {
+        throw new Error("neighbors must be a safe integer of at least 2.");
+    }
+    const layoutOptions = prepareUmapLayoutOptions(options);
+    if (!["euclidean", "cosine"].includes(metric)) {
+        throw new Error("metric must be euclidean or cosine.");
+    }
+    const types = await table.getTypes();
+    const columns = Object.keys(types);
+    const resolve = (name) => {
+        const found = columns.find((c) => c.toLowerCase() === name.toLowerCase());
+        if (found === undefined)
+            throw new Error(`Column ${name} does not exist.`);
+        return found;
+    };
+    const vector = resolve(column);
+    if (columns.some((c) => ["umapx", "umapy"].includes(c.toLowerCase()))) {
+        throw new Error("UMAP output columns umapX and umapY must not already exist.");
+    }
+    if (!/^(FLOAT|DOUBLE|TINYINT|SMALLINT|INTEGER|BIGINT|HUGEINT|UTINYINT|USMALLINT|UINTEGER|UBIGINT|UHUGEINT|DECIMAL\(\d+,\d+\))\[\d*\]$/
+        .test(types[vector])) {
+        throw new Error("UMAP requires a one-dimensional numeric LIST or ARRAY column.");
+    }
+    const connection = table.connection;
+    const [sourceOid, sourceTemporary] = (await connection.runAndReadAll(`SELECT table_oid,temporary FROM duckdb_tables()
+      WHERE lower(table_name)=lower($1) AND schema_name=current_schema()
+        AND database_name IN (current_database(),'temp')
+      ORDER BY temporary DESC LIMIT 1`, [table.name])).getRowsJS()[0] ?? [];
+    if (typeof sourceOid !== "number" && typeof sourceOid !== "bigint") {
+        throw new Error("UMAP requires a materialized source table.");
+    }
+    const q = quoteIdentifier;
+    const suffix = crypto.randomUUID().replaceAll("-", "");
+    const name = (part) => `__sda_umap_${suffix}_${part}`;
+    const ordinal = q(name("ordinal"));
+    const source = q(table.name), snapshot = q(name("snapshot"));
+    const layoutName = name("layout"), layout = q(layoutName);
+    const names = {
+        rows: q(name("rows")),
+        knn: q(name("knn")),
+        search: q(name("search")),
+        hnsw: q(name("hnsw")),
+        candidates: q(name("candidates")),
+        scales: q(name("scales")),
+        directed: q(name("directed")),
+        graph: q(name("graph")),
+    };
+    const scratch = [
+        snapshot,
+        layout,
+        ...Object.entries(names)
+            .filter(([key]) => key !== "hnsw").map(([, value]) => value),
+    ];
+    try {
+        // Retain payloads in native DuckDB types. A private ordinal distinguishes
+        // duplicate rows and is independent of any user column named rowid.
+        await connection.run(`CREATE TEMP TABLE ${snapshot} AS
+      SELECT *, row_number() OVER () - 1 AS ${ordinal} FROM ${source}`);
+        const count = await readScalarNumber(connection, `SELECT count(*) FROM ${snapshot}`);
+        if (count < 3 || count > 0x7fffffff) {
+            throw new Error("UMAP requires between 3 and 2147483647 rows.");
+        }
+        if (await readScalarNumber(connection, `SELECT count(*) FROM ${snapshot}
+      WHERE ${q(vector)} IS NULL OR len(${q(vector)})=0
+        OR list_count(${q(vector)}) != len(${q(vector)})
+        OR NOT list_bool_and(list_transform(${q(vector)}, x -> isfinite(x::DOUBLE)))`)) {
+            throw new Error("Vectors must be nonempty, non-null and contain finite numbers only.");
+        }
+        if (await readScalarNumber(connection, `SELECT count(DISTINCT len(${q(vector)})) FROM ${snapshot}`) !== 1) {
+            throw new Error("Vectors must have equal dimensions.");
+        }
+        const dimensions = await readScalarNumber(connection, `SELECT len(${q(vector)}) FROM ${snapshot} LIMIT 1`);
+        await connection.run(`CREATE TEMP TABLE ${names.rows} AS
+      SELECT (row_number() OVER (ORDER BY ${ordinal})-1)::INTEGER AS vertex,
+        ${ordinal} AS ordinal, ${q(vector)}::DOUBLE[${dimensions}] AS vec FROM ${snapshot}`);
+        if (await readScalarNumber(connection, `SELECT count(*) FROM ${names.rows}
+      WHERE NOT isfinite(array_inner_product(vec,vec)) OR array_inner_product(vec,vec)>1e300
+      ${metric === "cosine" ? "OR array_inner_product(vec,vec)=0" : ""}`)) {
+            throw new Error("Vector norms overflow, or cosine distance received a zero vector.");
+        }
+        const graph = await buildUmapGraph(connection, {
+            count,
+            dimensions,
+            neighbors: Math.min(neighbors, count - 1),
+        }, {
+            metric,
+            search: count <= 1000 ? "exact" : "hnsw",
+        }, names);
+        // Release index and vector scratch before allocating optimizer state.
+        for (const relation of [
+            names.search,
+            names.candidates,
+            names.knn,
+            names.directed,
+            names.scales,
+            names.graph,
+        ]) {
+            await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+        }
+        await connection.run(`ALTER TABLE ${names.rows} DROP COLUMN vec`);
+        const coordinates = optimizeUmapLayout(graph, initialUmapCoordinates(count, layoutOptions.seed), layoutOptions);
+        await connection.run(`CREATE TEMP TABLE ${layout} (vertex INTEGER,x DOUBLE,y DOUBLE)`);
+        await appendColumnBatches(connection, layoutName, [INTEGER, DOUBLE, DOUBLE], count, (column, start, end) => Array.from({ length: end - start }, (_, row) => column === 0
+            ? start + row
+            : coordinates[2 * (start + row) + column - 1]));
+        // Publish in one statement: failed fits never add partial columns. Existing
+        // native indexes must be restored because CREATE OR REPLACE drops them.
+        const indexes = (await connection.runAndReadAll("SELECT sql FROM duckdb_indexes() WHERE table_oid=$1", [sourceOid])).getRowsJS().map((row) => String(row[0]));
+        await connection.run("BEGIN TRANSACTION");
+        try {
+            await queryDB(table, `CREATE OR REPLACE ${sourceTemporary ? "TEMP " : ""}TABLE ${source} AS
+        SELECT ${columns.map((c) => `s.${q(c)}`).join(",")}, l.x AS "umapX", l.y AS "umapY"
+        FROM ${snapshot} s JOIN ${names.rows} r ON s.${ordinal}=r.ordinal
+        JOIN ${layout} l ON r.vertex=l.vertex ORDER BY s.${ordinal}`, mergeOptions(table, {
+                table: table.name,
+                method: "umap()",
+                parameters: { column, options },
+                noClean: true,
+            }));
+            for (const sql of indexes)
+                await connection.run(sql);
+            await connection.run("COMMIT");
+        }
+        catch (error) {
+            await connection.run("ROLLBACK");
+            throw error;
+        }
+    }
+    finally {
+        for (const relation of scratch) {
+            await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+        }
+    }
+}

package/esm/methods/updateWithJS.js

@@ -4,7 +4,7 @@ import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import { retainRegisteredTables } from "../helpers/tableRegistry.js";
 import queueOp from "../helpers/queueOp.js";
 import { prepareArray } from "./loadArray.js";
-import { DuckDBDataChunk } from "@duckdb/node-api";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
 import readMutationRows from "../helpers/readMutationRows.js";
 import parseDuckDBType from "../helpers/parseDuckDBType.js";
 export default function updateWithJS(simpleTable, dataModifier, options = {}) {
@@ -133,26 +133,14 @@ async function executeUpdateWithJS(simpleTable, dataModifier, options) {
                     parseDuckDBType(added.types[newKeys.indexOf(key)]));
             await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(scratch.name)} (${keys.map((key, i) => `${quoteIdentifier(key)} ${columnTypes[i]}`)
                 .join(", ")})`);
-            const appender = await simpleTable.connection.createAppender(scratch.name);
-            try {
-                for (let start = 0; start < modified.length; start += 2000) {
-                    const end = Math.min(start + 2000, modified.length);
-                    const chunk = DuckDBDataChunk.create(columnTypes, end - start);
-                    for (let i = 0; i < keys.length; i++) {
-                        const key = keys[i];
-                        chunk.setColumnValues(i, outputGeometry.includes(key)
-                            ? geometryData.get(key).slice(start, end)
-                            : source.types.has(key)
-                                ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[i]))
-                                : added.columnsData[newKeys.indexOf(key)].slice(start, end));
-                    }
-                    appender.appendDataChunk(chunk);
-                }
-                appender.flushSync();
-            }
-            finally {
-                appender.closeSync();
-            }
+            await appendColumnBatches(simpleTable.connection, scratch.name, columnTypes, modified.length, (column, start, end) => {
+                const key = keys[column];
+                return outputGeometry.includes(key)
+                    ? geometryData.get(key).slice(start, end)
+                    : source.types.has(key)
+                        ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[column]))
+                        : added.columnsData[newKeys.indexOf(key)].slice(start, end);
+            });
             const projection = keys.map((key) => outputGeometry.includes(key)
                 ? `${geometryFromJSON(quoteIdentifier(key))} AS ${quoteIdentifier(key)}`
                 : outputJSON.includes(key)

package/llm.md

@@ -1,7 +1,7 @@
 # The Simple Data Analysis (Core) Library
 
 - Package: `@nshiab/simple-data-analysis-core`
-- Version: `2.0.8`
+- Version: `2.0.9`
 
 To install the library with Deno, use:
 
@@ -1451,6 +1451,85 @@ await table.createVssIndex("embedding_column", {
 }).log();
 ```
 
+#### `umap`
+
+Reduces numeric vectors, such as embeddings, to a two-dimensional UMAP
+projection. The resulting coordinates are added as `umapX` and `umapY`, while
+all existing columns (including the source vector column), their values and
+types, and the input row order are preserved. DuckDB computes neighbors and the
+fuzzy graph; TypeScript optimizes the coordinates without copying the input
+vectors into JavaScript. Neighbor search is selected automatically.
+
+The defaults are a starting point for exploration. To adjust the projection:
+
+- `neighbors` (default `15`): How many nearby points influence the layout.
+  Smaller values emphasize local detail but can fragment groups. Larger values
+  emphasize broader structure, can hide local detail, and generally require more
+  time and memory.
+- `metric` (default `"euclidean"`): How similarity is measured. Euclidean
+  compares distance, including differences in vector magnitude. Cosine compares
+  direction, ignoring magnitude. Choose according to what makes vectors similar
+  in your data; neither is universally better.
+- `epochs` (default `200`): How many passes refine the coordinates. Fewer passes
+  finish sooner but may leave the layout unfinished. More passes allow further
+  refinement and take longer, with diminishing returns.
+- `minDistance` (default `0.1`): How tightly points can group in the projection.
+  Smaller values allow tighter clumps; larger values spread points out. This
+  changes the layout's appearance, not an accuracy level or the number of
+  optimization passes.
+- `learningRate` (default `1`): The initial size of coordinate adjustments.
+  Smaller values make gentler adjustments and may need more epochs. Larger
+  values make bigger adjustments but can overshoot useful positions. This does
+  not change the number of passes.
+- `negativeSamples` (default `5`): How many random points are sampled for
+  repulsion per attractive update. Smaller values reduce work and repulsion;
+  larger values increase both, tending to separate unrelated points more
+  strongly. More samples do not guarantee a better projection.
+
+##### Signature
+
+```typescript
+umap(column: string, options?: { neighbors?: number; metric?: "euclidean" | "cosine"; epochs?: number; seed?: number; minDistance?: number; learningRate?: number; negativeSamples?: number }): this;
+```
+
+##### Parameters
+
+- **`column`**: The column containing numeric vector embeddings.
+- **`options`**: Optional projection settings.
+- **`options.neighbors`**: Neighborhood size, including the point itself.
+  Integer of at least 2, clamped to row count minus one. Defaults to 15.
+- **`options.metric`**: Input distance metric: "euclidean" or "cosine". Defaults
+  to "euclidean".
+- **`options.epochs`**: Integer number of refinement passes, at least 1.
+  Defaults to 200.
+- **`options.seed`**: Integer used to initialize random choices. Defaults to 42.
+- **`options.minDistance`**: Grouping distance parameter between 0 and 1.
+  Defaults to 0.1.
+- **`options.learningRate`**: Finite, positive initial learning rate. Defaults
+  to 1.
+- **`options.negativeSamples`**: Integer number of samples used to separate
+  unrelated points, at least 1. Defaults to 5.
+
+##### Returns
+
+The table, so methods can be chained.
+
+##### Examples
+
+```ts
+await table.umap("embedding", {
+  metric: "cosine",
+  seed: 42,
+}).log();
+```
+
+```ts
+await table.umap("embedding", {
+  neighbors: 30,
+  minDistance: 0.25,
+}).selectColumns(["label", "umapX", "umapY"]).log();
+```
+
 #### `bm25`
 
 Searches a text column using DuckDB's BM25 ranking function, which scores

package/llms.txt

@@ -3,7 +3,7 @@
 > Lightweight DuckDB-powered TypeScript library for tabular, SQL, CSV, Parquet, and geospatial data analysis on Deno, Node.js, and Bun.
 
 - Package: `@nshiab/simple-data-analysis-core`
-- Version: `2.0.8`
+- Version: `2.0.9`
 
 Install with Deno: `deno add jsr:@nshiab/simple-data-analysis-core`
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nshiab/simple-data-analysis-core",
-  "version": "2.0.8",
+  "version": "2.0.9",
   "description": "Lightweight DuckDB-powered TypeScript library for tabular, SQL, CSV, Parquet, and geospatial data analysis on Deno, Node.js, and Bun.",
   "keywords": [
     "data-analysis",

package/script/class/SimpleTable.d.ts

@@ -757,6 +757,78 @@ export default class SimpleTable extends Simple {
         efSearch?: number;
         M?: number;
     }): this;
+    /**
+     * Reduces numeric vectors, such as embeddings, to a two-dimensional UMAP
+     * projection. The resulting coordinates are added as `umapX` and `umapY`,
+     * while all existing columns (including the source vector column), their
+     * values and types, and the input row order are preserved. DuckDB computes
+     * neighbors and the fuzzy graph; TypeScript optimizes the coordinates without
+     * copying the input vectors into JavaScript. Neighbor search is selected
+     * automatically.
+     *
+     * The defaults are a starting point for exploration. To adjust the projection:
+     *
+     * - `neighbors` (default `15`): How many nearby points influence the layout.
+     *   Smaller values emphasize local detail but can fragment groups. Larger
+     *   values emphasize broader structure, can hide local detail, and generally
+     *   require more time and memory.
+     * - `metric` (default `"euclidean"`): How similarity is measured. Euclidean
+     *   compares distance, including differences in vector magnitude. Cosine
+     *   compares direction, ignoring magnitude. Choose according to what makes
+     *   vectors similar in your data; neither is universally better.
+     * - `epochs` (default `200`): How many passes refine the coordinates. Fewer
+     *   passes finish sooner but may leave the layout unfinished. More passes
+     *   allow further refinement and take longer, with diminishing returns.
+     * - `minDistance` (default `0.1`): How tightly points can group in the
+     *   projection. Smaller values allow tighter clumps; larger values spread
+     *   points out. This changes the layout's appearance, not an accuracy level
+     *   or the number of optimization passes.
+     * - `learningRate` (default `1`): The initial size of coordinate adjustments.
+     *   Smaller values make gentler adjustments and may need more epochs. Larger
+     *   values make bigger adjustments but can overshoot useful positions. This
+     *   does not change the number of passes.
+     * - `negativeSamples` (default `5`): How many random points are sampled for
+     *   repulsion per attractive update. Smaller values reduce work and
+     *   repulsion; larger values increase both, tending to separate unrelated
+     *   points more strongly. More samples do not guarantee a better projection.
+     *
+     * @param column - The column containing numeric vector embeddings.
+     * @param options - Optional projection settings.
+     * @param options.neighbors - Neighborhood size, including the point itself. Integer of at least 2, clamped to row count minus one. Defaults to 15.
+     * @param options.metric - Input distance metric: "euclidean" or "cosine". Defaults to "euclidean".
+     * @param options.epochs - Integer number of refinement passes, at least 1. Defaults to 200.
+     * @param options.seed - Integer used to initialize random choices. Defaults to 42.
+     * @param options.minDistance - Grouping distance parameter between 0 and 1. Defaults to 0.1.
+     * @param options.learningRate - Finite, positive initial learning rate. Defaults to 1.
+     * @param options.negativeSamples - Integer number of samples used to separate unrelated points, at least 1. Defaults to 5.
+     * @returns The table, so methods can be chained.
+     * @category Vector Search
+     *
+     * @example
+     * ```ts
+     * await table.umap("embedding", {
+     *   metric: "cosine",
+     *   seed: 42,
+     * }).log();
+     * ```
+     *
+     * @example
+     * ```ts
+     * await table.umap("embedding", {
+     *   neighbors: 30,
+     *   minDistance: 0.25,
+     * }).selectColumns(["label", "umapX", "umapY"]).log();
+     * ```
+     */
+    umap(column: string, options?: {
+        neighbors?: number;
+        metric?: "euclidean" | "cosine";
+        epochs?: number;
+        seed?: number;
+        minDistance?: number;
+        learningRate?: number;
+        negativeSamples?: number;
+    }): this;
     /**
      * Searches a text column using DuckDB's BM25 ranking function, which scores
      * matches using factors including term frequency and document length.