@nshiab/simple-data-analysis-core 2.0.7 → 2.0.9

package/script/helpers/initialUmapCoordinates.js

@@ -0,0 +1,11 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = initialUmapCoordinates;
+const umapRandom_js_1 = __importDefault(require("./umapRandom.js"));
+function initialUmapCoordinates(count, seed) {
+    const random = (0, umapRandom_js_1.default)(seed);
+    return Float64Array.from({ length: count * 2 }, () => random() * 20 - 10);
+}

package/script/helpers/optimizeUmapLayout.d.ts

@@ -0,0 +1,7 @@
+import type prepareUmapLayoutOptions from "./prepareUmapLayoutOptions.js";
+export default function optimizeUmapLayout(graph: {
+    source: Uint32Array;
+    target: Uint32Array;
+    weight: Float64Array;
+}, initial: Float64Array, options: ReturnType<typeof prepareUmapLayoutOptions>): Float64Array;
+//# sourceMappingURL=optimizeUmapLayout.d.ts.map

package/script/helpers/optimizeUmapLayout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimizeUmapLayout.d.ts","sourceRoot":"","sources":["../../src/helpers/optimizeUmapLayout.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,wBAAwB,MAAM,+BAA+B,CAAC;AAS1E,MAAM,CAAC,OAAO,UAAU,kBAAkB,CACxC,KAAK,EAAE;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,YAAY,CAAA;CAAE,EACzE,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,UAAU,CAAC,OAAO,wBAAwB,CAAC,GACnD,YAAY,CA2Fd"}

package/script/helpers/optimizeUmapLayout.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = optimizeUmapLayout;
+const umapRandom_js_1 = __importDefault(require("./umapRandom.js"));
+// Internal interface. Graph entries are directed, in a stable order; the fuzzy
+// union graph supplies both orientations. Coordinates are interleaved x,y.
+// Inputs are borrowed read-only. State is O(edges + vertices), never O(n²).
+// Implements reference Euclidean UMAP SGD: scheduled attractive edges, sampled
+// repulsion, clipped gradients, symmetric attractive updates, linear decay.
+// Algorithm: https://umap-learn.readthedocs.io/en/latest/how_umap_works.html
+function optimizeUmapLayout(graph, initial, options) {
+    const { epochs, seed, learningRate, negativeSamples, curve: [a, b] } = options;
+    const count = initial.length / 2;
+    if (!Number.isInteger(count) || count < 3 || count > 0xffffffff) {
+        throw new Error("Expected interleaved coordinates for at least three vertices.");
+    }
+    for (const value of initial) {
+        if (!Number.isFinite(value)) {
+            throw new Error("Initial coordinates must be finite.");
+        }
+    }
+    const { source, target, weight } = graph;
+    if (!weight.length || source.length !== weight.length ||
+        target.length !== weight.length)
+        throw new Error("Invalid graph column lengths.");
+    let maximum = 0;
+    for (let edge = 0; edge < weight.length; edge++) {
+        if (source[edge] >= count || target[edge] >= count ||
+            source[edge] === target[edge])
+            throw new Error("Graph contains an invalid vertex or self edge.");
+        if (!Number.isFinite(weight[edge]) || weight[edge] < 0 || weight[edge] > 1)
+            throw new Error("Graph weights must be between zero and one.");
+        maximum = Math.max(maximum, weight[edge]);
+    }
+    if (!maximum)
+        throw new Error("Graph must contain a positive edge.");
+    const period = new Float64Array(weight.length);
+    const nextPositive = new Float64Array(weight.length);
+    const nextNegative = new Float64Array(weight.length);
+    // Reference pruning uses 500 when explicitly requesting at most ten epochs.
+    const threshold = maximum / (epochs > 10 ? epochs : 500);
+    for (let edge = 0; edge < weight.length; edge++) {
+        period[edge] = weight[edge] > 0 && weight[edge] >= threshold
+            ? epochs / ((weight[edge] / maximum) * epochs)
+            : Infinity;
+        nextPositive[edge] = period[edge];
+        nextNegative[edge] = period[edge] / negativeSamples;
+    }
+    const coordinates = initial.slice();
+    const random = (0, umapRandom_js_1.default)(seed ^ 0x7f4a7c15);
+    const clip = (gradient) => Math.max(-4, Math.min(4, gradient));
+    let alpha = learningRate;
+    for (let epoch = 0; epoch < epochs; epoch++) {
+        for (let edge = 0; edge < weight.length; edge++) {
+            if (nextPositive[edge] > epoch)
+                continue;
+            const u = source[edge] * 2, v = target[edge] * 2;
+            let dx = coordinates[u] - coordinates[v], dy = coordinates[u + 1] - coordinates[v + 1];
+            let squared = dx * dx + dy * dy;
+            if (squared > 0) {
+                const attraction = -2 * a * b * Math.pow(squared, b - 1) /
+                    (1 + a * Math.pow(squared, b));
+                const gx = clip(attraction * dx) * alpha, gy = clip(attraction * dy) * alpha;
+                coordinates[u] += gx;
+                coordinates[u + 1] += gy;
+                coordinates[v] -= gx;
+                coordinates[v + 1] -= gy;
+            }
+            nextPositive[edge] += period[edge];
+            const negativePeriod = period[edge] / negativeSamples;
+            const draws = Math.floor((epoch - nextNegative[edge]) / negativePeriod);
+            for (let draw = 0; draw < draws; draw++) {
+                const other = Math.floor(random() * count) * 2;
+                dx = coordinates[u] - coordinates[other];
+                dy = coordinates[u + 1] - coordinates[other + 1];
+                squared = dx * dx + dy * dy;
+                // Coincident points get zero repulsive gradient, as in current Python
+                // UMAP; this intentionally differs from UMAP-JS's constant +4 fallback.
+                if (squared > 0) {
+                    const repulsion = 2 * b /
+                        ((0.001 + squared) * (1 + a * Math.pow(squared, b)));
+                    coordinates[u] += clip(repulsion * dx) * alpha;
+                    coordinates[u + 1] += clip(repulsion * dy) * alpha;
+                }
+            }
+            nextNegative[edge] += draws * negativePeriod;
+        }
+        alpha = learningRate * (1 - epoch / epochs);
+    }
+    for (const value of coordinates) {
+        if (!Number.isFinite(value)) {
+            throw new Error("UMAP optimization produced a non-finite coordinate.");
+        }
+    }
+    return coordinates;
+}

package/script/helpers/prepareUmapLayoutOptions.d.ts

@@ -0,0 +1,14 @@
+export default function prepareUmapLayoutOptions(options?: {
+    epochs?: number;
+    seed?: number;
+    minDistance?: number;
+    learningRate?: number;
+    negativeSamples?: number;
+}): {
+    epochs: number;
+    seed: number;
+    learningRate: number;
+    negativeSamples: number;
+    curve: [number, number];
+};
+//# sourceMappingURL=prepareUmapLayoutOptions.d.ts.map

package/script/helpers/prepareUmapLayoutOptions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prepareUmapLayoutOptions.d.ts","sourceRoot":"","sources":["../../src/helpers/prepareUmapLayoutOptions.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,wBAAwB,CAC9C,OAAO,GAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;CACrB;;;;;;EA2BP"}

package/script/helpers/prepareUmapLayoutOptions.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = prepareUmapLayoutOptions;
+const fitUmapCurve_js_1 = __importDefault(require("./fitUmapCurve.js"));
+// Resolve and validate settings before database work, fitting the curve once.
+function prepareUmapLayoutOptions(options = {}) {
+    const { epochs = 200, seed = 42, minDistance = 0.1, learningRate = 1, negativeSamples = 5, } = options;
+    for (const [key, value] of Object.entries({ epochs, negativeSamples })) {
+        if (!Number.isSafeInteger(value) || value < 1) {
+            throw new Error(`${key} must be a positive safe integer.`);
+        }
+    }
+    if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) {
+        throw new Error("seed must be an integer between 0 and 4294967295.");
+    }
+    if (!Number.isFinite(learningRate) || learningRate <= 0) {
+        throw new Error("learningRate must be finite and positive.");
+    }
+    return {
+        epochs,
+        seed,
+        learningRate,
+        negativeSamples,
+        curve: (0, fitUmapCurve_js_1.default)(minDistance),
+    };
+}

package/script/helpers/readScalarNumber.d.ts

@@ -0,0 +1,3 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+export default function readScalarNumber(connection: DuckDBConnection, sql: string): Promise<number>;
+//# sourceMappingURL=readScalarNumber.d.ts.map

package/script/helpers/readScalarNumber.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readScalarNumber.d.ts","sourceRoot":"","sources":["../../src/helpers/readScalarNumber.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,wBAA8B,gBAAgB,CAC5C,UAAU,EAAE,gBAAgB,EAC5B,GAAG,EAAE,MAAM,mBAGZ"}

package/script/helpers/readScalarNumber.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = readScalarNumber;
+// For internal queries returning a single numeric value, such as row counts.
+async function readScalarNumber(connection, sql) {
+    return Number((await connection.runAndReadAll(sql)).getRowsJS()[0][0]);
+}

package/script/helpers/umapRandom.d.ts

@@ -0,0 +1,2 @@
+export default function umapRandom(seed: number): () => number;
+//# sourceMappingURL=umapRandom.d.ts.map

package/script/helpers/umapRandom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"umapRandom.d.ts","sourceRoot":"","sources":["../../src/helpers/umapRandom.ts"],"names":[],"mappings":"AACA,MAAM,CAAC,OAAO,UAAU,UAAU,CAAC,IAAI,EAAE,MAAM,gBAQ9C"}

package/script/helpers/umapRandom.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = umapRandom;
+// Mulberry32: separate initialization and sampling streams, stable per seed.
+function umapRandom(seed) {
+    let state = seed >>> 0;
+    return () => {
+        state = (state + 0x6D2B79F5) | 0;
+        let value = Math.imul(state ^ state >>> 15, 1 | state);
+        value ^= value + Math.imul(value ^ value >>> 7, 61 | value);
+        return ((value ^ value >>> 14) >>> 0) / 4294967296;
+    };
+}

package/script/methods/cache.js

@@ -46,7 +46,7 @@ const serializeCacheInputs_js_1 = __importStar(require("../helpers/serializeCach
 const tableGeneration_js_1 = require("../helpers/tableGeneration.js");
 const cacheDatabase_js_1 = require("../helpers/cacheDatabase.js");
 const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
-const CACHE_FORMAT_VERSION = "duckdb-v4";
+const CACHE_FORMAT_VERSION = "duckdb-v5";
 async function cache(table, compute, options = {}) {
     // If cache() is itself called from another cache computation, the outer
     // computation depends on the table produced here.

package/script/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA4ExB"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAM1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+DxB"}

package/script/methods/loadArray.js

@@ -7,6 +7,7 @@ exports.default = loadArray;
 exports.prepareArray = prepareArray;
 exports.executePreparedArray = executePreparedArray;
 const node_api_1 = require("@duckdb/node-api");
+const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
 const prepareGeometry_js_1 = __importDefault(require("../helpers/prepareGeometry.js"));
 const prepareJSON_js_1 = __importDefault(require("../helpers/prepareJSON.js"));
 const geometryFromJSON_js_1 = __importDefault(require("../helpers/geometryFromJSON.js"));
@@ -144,25 +145,7 @@ async function executePreparedArray(simpleTable, prepared) {
     }
     try {
         await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${(0, quoteIdentifier_js_1.default)(staged)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${storageTypes[i]}`).join(", ")})`);
-        const appender = await simpleTable.connection
-            .createAppender(staged);
-        try {
-            const duckDBTypes = storageTypes.map((d) => (0, parseDuckDBType_js_1.default)(d));
-            // The maximum capacity of a DuckDB data chunk is 2048 rows.
-            const chunkSize = 2000;
-            for (let start = 0; start < rowCount; start += chunkSize) {
-                const end = Math.min(start + chunkSize, rowCount);
-                const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
-                for (let i = 0; i < keys.length; i++) {
-                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-                }
-                appender.appendDataChunk(dataChunk);
-            }
-            appender.flushSync();
-        }
-        finally {
-            appender.closeSync();
-        }
+        await (0, appendColumnBatches_js_1.default)(simpleTable.connection, staged, storageTypes.map((type) => (0, parseDuckDBType_js_1.default)(type)), rowCount, (column, start, end) => columnsData[column].slice(start, end));
         if (!needsConversion)
             return;
         // CTAS is atomic: conversion must finish before the destination is replaced.

package/script/methods/log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../../src/methods/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,GAAG,CAC/B,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,KAAK,GAAG,MAAM,GAAG;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,iBA4EP"}
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../../src/methods/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,GAAG,CAC/B,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,KAAK,GAAG,MAAM,GAAG;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,iBAyEP"}

package/script/methods/log.js

@@ -60,10 +60,7 @@ async function log(simpleTable, options = {}) {
         const showing = displayedRows < rowCount
             ? ` / showing ${(0, formatNumber_js_1.default)(displayedRows)} rows`
             : "";
-        const charsToLog = typeof simpleTable.charsToLog === "number"
-            ? ` (charsToLog: ${simpleTable.charsToLog})`
-            : "";
-        console.log(`${(0, formatNumber_js_1.default)(rowCount)} rows in total${showing}${charsToLog}`);
+        console.log(`${(0, formatNumber_js_1.default)(rowCount)} rows in total${showing}`);
     }
     return;
 }

package/script/methods/umap.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["umap"]>[1]>;
+export default function umap(table: SimpleTable, column: string, options?: Options): void;
+export {};
+//# sourceMappingURL=umap.d.ts.map

package/script/methods/umap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"umap.d.ts","sourceRoot":"","sources":["../../src/methods/umap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAavD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,OAAY,QAStB"}

package/script/methods/umap.js

@@ -0,0 +1,162 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = umap;
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
+const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
+const buildUmapGraph_js_1 = __importDefault(require("../helpers/buildUmapGraph.js"));
+const prepareUmapLayoutOptions_js_1 = __importDefault(require("../helpers/prepareUmapLayoutOptions.js"));
+const readScalarNumber_js_1 = __importDefault(require("../helpers/readScalarNumber.js"));
+const optimizeUmapLayout_js_1 = __importDefault(require("../helpers/optimizeUmapLayout.js"));
+const initialUmapCoordinates_js_1 = __importDefault(require("../helpers/initialUmapCoordinates.js"));
+const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
+const node_api_1 = require("@duckdb/node-api");
+function umap(table, column, options = {}) {
+    const settings = structuredClone(options);
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "umap()",
+        parameters: { column, options: settings },
+        execute: () => execute(table, column, settings),
+    });
+}
+async function execute(table, column, options) {
+    const { neighbors = 15, metric = "euclidean" } = options;
+    if (!Number.isSafeInteger(neighbors) || neighbors < 2) {
+        throw new Error("neighbors must be a safe integer of at least 2.");
+    }
+    const layoutOptions = (0, prepareUmapLayoutOptions_js_1.default)(options);
+    if (!["euclidean", "cosine"].includes(metric)) {
+        throw new Error("metric must be euclidean or cosine.");
+    }
+    const types = await table.getTypes();
+    const columns = Object.keys(types);
+    const resolve = (name) => {
+        const found = columns.find((c) => c.toLowerCase() === name.toLowerCase());
+        if (found === undefined)
+            throw new Error(`Column ${name} does not exist.`);
+        return found;
+    };
+    const vector = resolve(column);
+    if (columns.some((c) => ["umapx", "umapy"].includes(c.toLowerCase()))) {
+        throw new Error("UMAP output columns umapX and umapY must not already exist.");
+    }
+    if (!/^(FLOAT|DOUBLE|TINYINT|SMALLINT|INTEGER|BIGINT|HUGEINT|UTINYINT|USMALLINT|UINTEGER|UBIGINT|UHUGEINT|DECIMAL\(\d+,\d+\))\[\d*\]$/
+        .test(types[vector])) {
+        throw new Error("UMAP requires a one-dimensional numeric LIST or ARRAY column.");
+    }
+    const connection = table.connection;
+    const [sourceOid, sourceTemporary] = (await connection.runAndReadAll(`SELECT table_oid,temporary FROM duckdb_tables()
+      WHERE lower(table_name)=lower($1) AND schema_name=current_schema()
+        AND database_name IN (current_database(),'temp')
+      ORDER BY temporary DESC LIMIT 1`, [table.name])).getRowsJS()[0] ?? [];
+    if (typeof sourceOid !== "number" && typeof sourceOid !== "bigint") {
+        throw new Error("UMAP requires a materialized source table.");
+    }
+    const q = quoteIdentifier_js_1.default;
+    const suffix = crypto.randomUUID().replaceAll("-", "");
+    const name = (part) => `__sda_umap_${suffix}_${part}`;
+    const ordinal = q(name("ordinal"));
+    const source = q(table.name), snapshot = q(name("snapshot"));
+    const layoutName = name("layout"), layout = q(layoutName);
+    const names = {
+        rows: q(name("rows")),
+        knn: q(name("knn")),
+        search: q(name("search")),
+        hnsw: q(name("hnsw")),
+        candidates: q(name("candidates")),
+        scales: q(name("scales")),
+        directed: q(name("directed")),
+        graph: q(name("graph")),
+    };
+    const scratch = [
+        snapshot,
+        layout,
+        ...Object.entries(names)
+            .filter(([key]) => key !== "hnsw").map(([, value]) => value),
+    ];
+    try {
+        // Retain payloads in native DuckDB types. A private ordinal distinguishes
+        // duplicate rows and is independent of any user column named rowid.
+        await connection.run(`CREATE TEMP TABLE ${snapshot} AS
+      SELECT *, row_number() OVER () - 1 AS ${ordinal} FROM ${source}`);
+        const count = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${snapshot}`);
+        if (count < 3 || count > 0x7fffffff) {
+            throw new Error("UMAP requires between 3 and 2147483647 rows.");
+        }
+        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${snapshot}
+      WHERE ${q(vector)} IS NULL OR len(${q(vector)})=0
+        OR list_count(${q(vector)}) != len(${q(vector)})
+        OR NOT list_bool_and(list_transform(${q(vector)}, x -> isfinite(x::DOUBLE)))`)) {
+            throw new Error("Vectors must be nonempty, non-null and contain finite numbers only.");
+        }
+        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(DISTINCT len(${q(vector)})) FROM ${snapshot}`) !== 1) {
+            throw new Error("Vectors must have equal dimensions.");
+        }
+        const dimensions = await (0, readScalarNumber_js_1.default)(connection, `SELECT len(${q(vector)}) FROM ${snapshot} LIMIT 1`);
+        await connection.run(`CREATE TEMP TABLE ${names.rows} AS
+      SELECT (row_number() OVER (ORDER BY ${ordinal})-1)::INTEGER AS vertex,
+        ${ordinal} AS ordinal, ${q(vector)}::DOUBLE[${dimensions}] AS vec FROM ${snapshot}`);
+        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.rows}
+      WHERE NOT isfinite(array_inner_product(vec,vec)) OR array_inner_product(vec,vec)>1e300
+      ${metric === "cosine" ? "OR array_inner_product(vec,vec)=0" : ""}`)) {
+            throw new Error("Vector norms overflow, or cosine distance received a zero vector.");
+        }
+        const graph = await (0, buildUmapGraph_js_1.default)(connection, {
+            count,
+            dimensions,
+            neighbors: Math.min(neighbors, count - 1),
+        }, {
+            metric,
+            search: count <= 1000 ? "exact" : "hnsw",
+        }, names);
+        // Release index and vector scratch before allocating optimizer state.
+        for (const relation of [
+            names.search,
+            names.candidates,
+            names.knn,
+            names.directed,
+            names.scales,
+            names.graph,
+        ]) {
+            await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+        }
+        await connection.run(`ALTER TABLE ${names.rows} DROP COLUMN vec`);
+        const coordinates = (0, optimizeUmapLayout_js_1.default)(graph, (0, initialUmapCoordinates_js_1.default)(count, layoutOptions.seed), layoutOptions);
+        await connection.run(`CREATE TEMP TABLE ${layout} (vertex INTEGER,x DOUBLE,y DOUBLE)`);
+        await (0, appendColumnBatches_js_1.default)(connection, layoutName, [node_api_1.INTEGER, node_api_1.DOUBLE, node_api_1.DOUBLE], count, (column, start, end) => Array.from({ length: end - start }, (_, row) => column === 0
+            ? start + row
+            : coordinates[2 * (start + row) + column - 1]));
+        // Publish in one statement: failed fits never add partial columns. Existing
+        // native indexes must be restored because CREATE OR REPLACE drops them.
+        const indexes = (await connection.runAndReadAll("SELECT sql FROM duckdb_indexes() WHERE table_oid=$1", [sourceOid])).getRowsJS().map((row) => String(row[0]));
+        await connection.run("BEGIN TRANSACTION");
+        try {
+            await (0, queryDB_js_1.default)(table, `CREATE OR REPLACE ${sourceTemporary ? "TEMP " : ""}TABLE ${source} AS
+        SELECT ${columns.map((c) => `s.${q(c)}`).join(",")}, l.x AS "umapX", l.y AS "umapY"
+        FROM ${snapshot} s JOIN ${names.rows} r ON s.${ordinal}=r.ordinal
+        JOIN ${layout} l ON r.vertex=l.vertex ORDER BY s.${ordinal}`, (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "umap()",
+                parameters: { column, options },
+                noClean: true,
+            }));
+            for (const sql of indexes)
+                await connection.run(sql);
+            await connection.run("COMMIT");
+        }
+        catch (error) {
+            await connection.run("ROLLBACK");
+            throw error;
+        }
+    }
+    finally {
+        for (const relation of scratch) {
+            await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+        }
+    }
+}

package/script/methods/updateWithJS.js

@@ -10,7 +10,7 @@ const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier
 const tableRegistry_js_1 = require("../helpers/tableRegistry.js");
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const loadArray_js_1 = require("./loadArray.js");
-const node_api_1 = require("@duckdb/node-api");
+const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
 const readMutationRows_js_1 = __importDefault(require("../helpers/readMutationRows.js"));
 const parseDuckDBType_js_1 = __importDefault(require("../helpers/parseDuckDBType.js"));
 function updateWithJS(simpleTable, dataModifier, options = {}) {
@@ -139,26 +139,14 @@ async function executeUpdateWithJS(simpleTable, dataModifier, options) {
                     (0, parseDuckDBType_js_1.default)(added.types[newKeys.indexOf(key)]));
             await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(scratch.name)} (${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${columnTypes[i]}`)
                 .join(", ")})`);
-            const appender = await simpleTable.connection.createAppender(scratch.name);
-            try {
-                for (let start = 0; start < modified.length; start += 2000) {
-                    const end = Math.min(start + 2000, modified.length);
-                    const chunk = node_api_1.DuckDBDataChunk.create(columnTypes, end - start);
-                    for (let i = 0; i < keys.length; i++) {
-                        const key = keys[i];
-                        chunk.setColumnValues(i, outputGeometry.includes(key)
-                            ? geometryData.get(key).slice(start, end)
-                            : source.types.has(key)
-                                ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[i]))
-                                : added.columnsData[newKeys.indexOf(key)].slice(start, end));
-                    }
-                    appender.appendDataChunk(chunk);
-                }
-                appender.flushSync();
-            }
-            finally {
-                appender.closeSync();
-            }
+            await (0, appendColumnBatches_js_1.default)(simpleTable.connection, scratch.name, columnTypes, modified.length, (column, start, end) => {
+                const key = keys[column];
+                return outputGeometry.includes(key)
+                    ? geometryData.get(key).slice(start, end)
+                    : source.types.has(key)
+                        ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[column]))
+                        : added.columnsData[newKeys.indexOf(key)].slice(start, end);
+            });
             const projection = keys.map((key) => outputGeometry.includes(key)
                 ? `${(0, geometryFromJSON_js_1.default)((0, quoteIdentifier_js_1.default)(key))} AS ${(0, quoteIdentifier_js_1.default)(key)}`
                 : outputJSON.includes(key)