@nshiab/simple-data-analysis-core 2.0.8 → 2.1.0

package/esm/methods/fuzzyClean.js

@@ -1,12 +1,13 @@
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import { DuckDBListValue } from "@duckdb/node-api";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
 import queueOp from "../helpers/queueOp.js";
 import buildFuzzyMatchSql from "../helpers/buildFuzzyMatchSql.js";
+import selectFuzzyReplacements from "../helpers/selectFuzzyReplacements.js";
 export default function fuzzyClean(table, column, newColumn, threshold, options = {}) {
-    // The clustering runs in JS over the fuzzy pairs read from the table, so
-    // fuzzyClean can't be expressed as a single SELECT over its input: it
-    // executes as a barrier.
+    // This implementation clusters fuzzy pairs in JS and writes the mapping
+    // back to DuckDB, so it executes as a barrier.
     options = structuredClone(options);
     queueOp(table, {
         kind: "barrier",
@@ -57,149 +58,29 @@ async function executeFuzzyClean(table, column, newColumn, threshold, options =
         }
         return;
     }
-    // Build count map from the pairs — no separate query needed.
-    const countMap = new Map();
-    for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {
-        countMap.set(left_value, Number(left_cnt));
-        countMap.set(right_value, Number(right_cnt));
-    }
-    // Union-Find — only over values that actually participate in a pair.
-    const parent = new Map();
-    const find = (x) => {
-        if (!parent.has(x))
-            parent.set(x, x);
-        if (parent.get(x) !== x)
-            parent.set(x, find(parent.get(x)));
-        return parent.get(x);
-    };
-    const union = (a, b) => {
-        const ra = find(a);
-        const rb = find(b);
-        if (ra !== rb)
-            parent.set(ra, rb);
-    };
-    for (const v of countMap.keys())
-        find(v);
-    for (const { left_value, right_value } of pairs) {
-        union(left_value, right_value);
-    }
-    // Group values by their cluster root.
-    const clusters = new Map();
-    for (const v of countMap.keys()) {
-        const root = find(v);
-        if (!clusters.has(root))
-            clusters.set(root, []);
-        clusters.get(root).push(v);
-    }
-    // Pick the canonical value for each cluster.
-    const replacement = new Map();
-    for (const members of clusters.values()) {
-        if (members.length === 1)
-            continue; // All pairs are already canonical — skip.
-        // Score sums and per-member max scores are computed lazily — only when
-        // actually needed as a primary criterion or tie-breaker.
-        //
-        // scoreSum: total similarity to all other cluster members (used by
-        //   "mostCentral" as primary, and as tie-breaker for all strategies).
-        // scoreMax: highest single pairwise score (used by "maxScore" as primary).
-        let scoreSum = null;
-        const getScoreSum = () => {
-            if (scoreSum === null) {
-                scoreSum = new Map(members.map((m) => [m, 0]));
-                for (const { left_value, right_value, score } of pairs) {
-                    if (scoreSum.has(left_value) && scoreSum.has(right_value)) {
-                        scoreSum.set(left_value, (scoreSum.get(left_value) ?? 0) + score);
-                        scoreSum.set(right_value, (scoreSum.get(right_value) ?? 0) + score);
-                    }
-                }
-            }
-            return scoreSum;
-        };
-        let scoreMax = null;
-        const getScoreMax = () => {
-            if (scoreMax === null) {
-                scoreMax = new Map(members.map((m) => [m, 0]));
-                for (const { left_value, right_value, score } of pairs) {
-                    if (scoreMax.has(left_value) && scoreMax.has(right_value)) {
-                        scoreMax.set(left_value, Math.max(scoreMax.get(left_value) ?? 0, score));
-                        scoreMax.set(right_value, Math.max(scoreMax.get(right_value) ?? 0, score));
-                    }
-                }
-            }
-            return scoreMax;
-        };
-        // Sum-based comparator: highest total similarity, then alphabetical.
-        const byScore = (a, b) => {
-            const sum = getScoreSum();
-            const sa = sum.get(a) ?? 0;
-            const sb = sum.get(b) ?? 0;
-            if (sa !== sb)
-                return sa > sb ? a : b;
-            return a <= b ? a : b; // alphabetical tie-break
-        };
-        // Max-based comparator: highest single-pair score, then sum, then alphabetical.
-        const byMaxScore = (a, b) => {
-            const mx = getScoreMax();
-            const ma = mx.get(a) ?? 0;
-            const mb = mx.get(b) ?? 0;
-            if (ma !== mb)
-                return ma > mb ? a : b;
-            return byScore(a, b); // sum then alphabetical tie-break
-        };
-        let canonical;
-        if (strategy === "longestString") {
-            canonical = members.reduce((a, b) => {
-                if (a.length !== b.length)
-                    return a.length > b.length ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "shortestString") {
-            canonical = members.reduce((a, b) => {
-                if (a.length !== b.length)
-                    return a.length < b.length ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "mostCommon") {
-            canonical = members.reduce((a, b) => {
-                const ca = countMap.get(a) ?? 0;
-                const cb = countMap.get(b) ?? 0;
-                if (ca !== cb)
-                    return ca > cb ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "mostCentral") {
-            // Most central string — highest total similarity to all other cluster members.
-            canonical = members.reduce((a, b) => byScore(a, b));
-        }
-        else {
-            // "maxScore": the string participating in the single highest-scoring pair.
-            canonical = members.reduce((a, b) => byMaxScore(a, b));
-        }
-        for (const m of members) {
-            if (m !== canonical)
-                replacement.set(m, canonical);
-        }
-    }
+    const replacement = selectFuzzyReplacements(pairs, strategy);
     if (replacement.size === 0)
         return;
-    // Use a VALUES-based CTE for the UPDATE so DuckDB can use a hash join
-    // instead of evaluating a potentially huge CASE WHEN expression.
-    const replacementEntries = [...replacement.entries()];
-    const valuesList = replacementEntries.map(() => "(?, ?)").join(",\n     ");
-    const values = replacementEntries.flatMap(([from, to]) => [from, to]);
+    // Two bound lists keep the SQL and parameter count fixed as the mapping
+    // grows. Side-by-side UNNEST zips the equally sized lists into mapping rows,
+    // which DuckDB can hash-join to the original table.
+    const values = [
+        new DuckDBListValue([...replacement.keys()]),
+        new DuckDBListValue([...replacement.values()]),
+    ];
+    const mapping = `WITH mapping(original, canonical) AS (
+    SELECT UNNEST(?::VARCHAR[]), UNNEST(?::VARCHAR[])
+  )`;
     const query = newColumn !== column
         ? `ALTER TABLE ${quoteIdentifier(table.name)} ADD ${quoteIdentifier(newColumn)} VARCHAR;
        UPDATE ${quoteIdentifier(table.name)}
          SET ${quoteIdentifier(newColumn)} = ${quoteIdentifier(column)};
-       WITH mapping(original, canonical) AS (VALUES ${valuesList})
+       ${mapping}
        UPDATE ${quoteIdentifier(table.name)}
          SET ${quoteIdentifier(newColumn)} = m.canonical
          FROM mapping m
          WHERE ${quoteIdentifier(table.name)}.${quoteIdentifier(newColumn)} = m.original`
-        : `WITH mapping(original, canonical) AS (VALUES ${valuesList})
+        : `${mapping}
        UPDATE ${quoteIdentifier(table.name)}
          SET ${quoteIdentifier(column)} = m.canonical
          FROM mapping m

package/esm/methods/fuzzyJoin.js

@@ -48,17 +48,7 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
     }
     const method = options.method ?? "ratio";
     const similarityColumn = options.similarityColumn;
-    // The right table's copy of a column shared with the left table (only
-    // rightColumn can be shared, checked above) is excluded from the SELECT
-    // directly, instead of dropping its _1 duplicate with a rewrite after the
-    // join.
-    const rightSelect = rightCols
-        .filter((d) => !leftCols.includes(d))
-        .map((d) => `${quoteIdentifier(rightTable.name)}.${quoteIdentifier(d)}`)
-        .join(", ");
-    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength);
-    await queryDB(leftTable, sql, mergeOptions(leftTable, {
+    const queryOptions = mergeOptions(leftTable, {
         table: outputTable.name,
         method: "fuzzyJoin()",
         parameters: {
@@ -68,26 +58,90 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
             threshold,
             options,
         },
-    }));
+    });
+    const deduplicate = await shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions);
+    // The right table's copy of a column shared with the left table (only
+    // rightColumn can be shared, checked above) is excluded from the SELECT
+    // directly, instead of dropping its _1 duplicate with a rewrite after the
+    // join.
+    const rightSelect = rightCols
+        .filter((d) => !leftCols.includes(d))
+        .map((d) => `${quoteIdentifier(rightTable.name)}.${quoteIdentifier(d)}`)
+        .join(", ");
+    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
+        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength, deduplicate);
+    await queryDB(leftTable, sql, queryOptions);
 }
-function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength) {
+async function shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions) {
+    const left = quoteIdentifier(leftTable.name);
+    const right = quoteIdentifier(rightTable.name);
+    const counts = (await queryDB(leftTable, `SELECT (SELECT COUNT(*) FROM ${left}) AS left_count,
+      (SELECT COUNT(*) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    const leftCount = Number(counts.left_count);
+    const rightCount = Number(counts.right_count);
+    const comparisons = leftCount * rightCount;
+    // Avoid distinct scans for small joins and short lookup tables. For larger
+    // joins, require at least a twofold reduction to pay for grouping and rejoining.
+    if (comparisons < 100_000 || Math.min(leftCount, rightCount) < 32) {
+        return false;
+    }
+    const distinct = (await queryDB(leftTable, `SELECT
+      (SELECT COUNT(DISTINCT ENCODE(${quoteIdentifier(leftColumn)})) FROM ${left}) AS left_count,
+      (SELECT COUNT(DISTINCT ENCODE(${quoteIdentifier(rightColumn)})) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    return Number(distinct.left_count) * Number(distinct.right_count) <=
+        comparisons / 2;
+}
+function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength, deduplicate = false) {
     const leftExpression = `${quoteIdentifier(leftTable)}.${quoteIdentifier(leftColumn)}`;
     const rightExpression = `${quoteIdentifier(rightTable)}.${quoteIdentifier(rightColumn)}`;
     const { scoreExpression, condition } = buildFuzzyMatchSql(leftExpression, rightExpression, method, threshold, { decimals: 2, prefilterPrefixLength });
+    let cte = "";
+    let from = `FROM ${quoteIdentifier(leftTable)} LEFT JOIN ${quoteIdentifier(rightTable)} ON ${condition}`;
+    let score = scoreExpression;
+    if (deduplicate) {
+        const matches = quoteIdentifier(`__sda_fuzzy_matches_${crypto.randomUUID().replaceAll("-", "")}`);
+        const match = buildFuzzyMatchSql("l.value", "r.value", method, threshold, { decimals: 2, prefilterPrefixLength });
+        // Keep the original value's collation for prefix matching, but use byte
+        // keys for deduplication and expansion: fuzzy scores distinguish case even
+        // when an input column has a case-insensitive collation.
+        cte = `WITH ${matches} AS MATERIALIZED (
+      SELECT l.key AS left_key, r.key AS right_key,
+        ${match.scoreExpression} AS score
+      FROM (
+        SELECT DISTINCT ${quoteIdentifier(leftColumn)} AS value,
+          ENCODE(${quoteIdentifier(leftColumn)}) AS key
+        FROM ${quoteIdentifier(leftTable)}
+        WHERE ${quoteIdentifier(leftColumn)} IS NOT NULL
+      ) l
+      INNER JOIN (
+        SELECT DISTINCT ${quoteIdentifier(rightColumn)} AS value,
+          ENCODE(${quoteIdentifier(rightColumn)}) AS key
+        FROM ${quoteIdentifier(rightTable)}
+        WHERE ${quoteIdentifier(rightColumn)} IS NOT NULL
+      ) r ON ${match.condition}
+    )`;
+        from = `FROM ${quoteIdentifier(leftTable)}
+      LEFT JOIN ${matches} ON ENCODE(${leftExpression}) = ${matches}.left_key
+      LEFT JOIN ${quoteIdentifier(rightTable)}
+        ON ENCODE(${rightExpression}) = ${matches}.right_key`;
+        score = `${matches}.score`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable)} AS
+${cte}
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS ${quoteIdentifier(similarityColumn)}
 FROM (
-  SELECT ${quoteIdentifier(leftTable)}.*, ${rightSelect}, ${scoreExpression} AS "_sda_score"
-  FROM ${quoteIdentifier(leftTable)} LEFT JOIN ${quoteIdentifier(rightTable)} ON ${condition}
+  SELECT ${quoteIdentifier(leftTable)}.*, ${rightSelect}, ${score} AS "_sda_score"
+  ${from}
 ) _sda
 ORDER BY ${quoteIdentifier(leftColumn)}, "_sda_score" DESC;\n`;
     }
     return `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable)} AS
+${cte}
 SELECT *
 FROM (
   SELECT ${quoteIdentifier(leftTable)}.*, ${rightSelect}
-  FROM ${quoteIdentifier(leftTable)} LEFT JOIN ${quoteIdentifier(rightTable)} ON ${condition}
+  ${from}
 ) _sda
 ORDER BY ${quoteIdentifier(leftColumn)}, ${quoteIdentifier(rightColumn)};\n`;
 }

package/esm/methods/getTypes.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function getTypes(simpleTable: SimpleTable): Promise<{
-    [key: string]: string;
+    [k: string]: string;
 }>;
 //# sourceMappingURL=getTypes.d.ts.map

package/esm/methods/hdbscan.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["hdbscan"]>[2]>;
+export default function hdbscan(table: SimpleTable, columns: string | string[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=hdbscan.d.ts.map

package/esm/methods/hdbscan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hdbscan.d.ts","sourceRoot":"","sources":["../../src/methods/hdbscan.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElE,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CASN"}

package/esm/methods/hdbscan.js

@@ -0,0 +1,208 @@
+import { DOUBLE, INTEGER, VARCHAR } from "@duckdb/node-api";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
+import buildApproximateMutualReachabilityMst from "../helpers/buildApproximateMutualReachabilityMst.js";
+import buildExactMutualReachabilityMst from "../helpers/buildExactMutualReachabilityMst.js";
+import clusterHdbscan from "../helpers/clusterHdbscan.js";
+import foldIdentifier from "../helpers/foldIdentifier.js";
+import prepareNumericFeatures from "../helpers/prepareNumericFeatures.js";
+import publishPreparedColumns from "../helpers/publishPreparedColumns.js";
+import queueOp from "../helpers/queueOp.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+export default function hdbscan(table, columns, newColumn, options = {}) {
+    const selected = typeof columns === "string" ? columns : [...columns];
+    const settings = structuredClone(options);
+    queueOp(table, {
+        kind: "barrier",
+        method: "hdbscan()",
+        parameters: { columns: selected, newColumn, options: settings },
+        execute: () => execute(table, selected, newColumn, settings),
+    });
+}
+async function execute(table, columns, newColumn, options) {
+    const { minClusterSize = 5, metric = "euclidean", allowSingleCluster = false, approximate = false, membershipScoreColumn, outlierScoreColumn, } = options;
+    const minSamples = options.minSamples ?? minClusterSize;
+    if (typeof newColumn !== "string") {
+        throw new Error("hdbscan() newColumn must be a string.");
+    }
+    if (!Number.isSafeInteger(minClusterSize) || minClusterSize < 2) {
+        throw new Error("hdbscan() minClusterSize must be a safe integer of at least 2.");
+    }
+    if (!Number.isSafeInteger(minSamples) || minSamples < 1) {
+        throw new Error("hdbscan() minSamples must be a safe integer of at least 1.");
+    }
+    if (!["euclidean", "cosine"].includes(metric)) {
+        throw new Error('hdbscan() metric must be "euclidean" or "cosine".');
+    }
+    if (typeof allowSingleCluster !== "boolean") {
+        throw new Error("hdbscan() allowSingleCluster must be a boolean.");
+    }
+    if (typeof approximate !== "boolean") {
+        throw new Error("hdbscan() approximate must be a boolean.");
+    }
+    if (membershipScoreColumn !== undefined &&
+        typeof membershipScoreColumn !== "string") {
+        throw new Error("hdbscan() membershipScoreColumn must be a string.");
+    }
+    if (outlierScoreColumn !== undefined &&
+        typeof outlierScoreColumn !== "string") {
+        throw new Error("hdbscan() outlierScoreColumn must be a string.");
+    }
+    const sourceColumns = Object.keys(await table.getTypes());
+    const outputNames = [newColumn, membershipScoreColumn, outlierScoreColumn]
+        .filter((name) => name !== undefined);
+    validateOutputNames(sourceColumns, outputNames);
+    const prepared = await prepareNumericFeatures(table, typeof columns === "string"
+        ? { kind: "vector", column: columns }
+        : { kind: "scalars", columns }, { method: "hdbscan()" });
+    const connection = table.connection;
+    const suffix = crypto.randomUUID().replaceAll("-", "");
+    const name = (part) => `__sda_hdbscan_${suffix}_${part}`;
+    const names = {
+        rows: name("rows"),
+        neighbors: name("neighbors"),
+        search: name("search"),
+        hnsw: name("hnsw"),
+        candidates: name("candidates"),
+        coreDistances: name("core_distances"),
+        frontier: name("frontier"),
+        edges: name("edges"),
+        components: name("components"),
+        representatives: name("representatives"),
+        componentPairs: name("component_pairs"),
+        bridgeCandidates: name("bridge_candidates"),
+        mst: name("mst"),
+        result: name("result"),
+    };
+    try {
+        if (prepared.rowCount < 2) {
+            throw new Error(`hdbscan() requires at least 2 rows, but the input has ${prepared.rowCount}.`);
+        }
+        if (prepared.rowCount > 0x40000000) {
+            throw new Error("hdbscan() supports at most 1073741824 rows because its 2*n-1 hierarchy nodes use signed INTEGER identifiers.");
+        }
+        if (minSamples >= prepared.rowCount) {
+            const defaulted = options.minSamples === undefined
+                ? " (defaulted from minClusterSize)"
+                : "";
+            throw new Error(`hdbscan() minSamples is ${minSamples}${defaulted}, but ${prepared.rowCount} rows provide at most ${prepared.rowCount - 1} other point${prepared.rowCount === 2 ? "" : "s"}. Set minSamples between 1 and ${prepared.rowCount - 1}, or provide at least ${minSamples + 1} rows.`);
+        }
+        if (minClusterSize > prepared.rowCount) {
+            throw new Error(`hdbscan() minClusterSize is ${minClusterSize}, but the input has ${prepared.rowCount} rows. Set minClusterSize between 2 and ${prepared.rowCount}, or provide more rows.`);
+        }
+        const q = quoteIdentifier;
+        await connection.run(`CREATE TEMP TABLE ${q(names.rows)} AS
+       SELECT ${q(prepared.rowIdColumn)}::INTEGER AS vertex,
+         ${q(prepared.vectorColumn)} AS vec
+       FROM ${q(prepared.relation)}
+       ORDER BY ${q(prepared.rowIdColumn)}`);
+        if (approximate) {
+            await buildApproximateMutualReachabilityMst(connection, {
+                count: prepared.rowCount,
+                dimensions: prepared.dimensions,
+                minSamples,
+            }, { metric }, {
+                rows: names.rows,
+                neighbors: names.neighbors,
+                search: names.search,
+                hnsw: names.hnsw,
+                candidates: names.candidates,
+                coreDistances: names.coreDistances,
+                edges: names.edges,
+                components: names.components,
+                representatives: names.representatives,
+                componentPairs: names.componentPairs,
+                bridgeCandidates: names.bridgeCandidates,
+                mst: names.mst,
+            });
+        }
+        else {
+            await buildExactMutualReachabilityMst(connection, { count: prepared.rowCount, minSamples }, { metric }, {
+                rows: q(names.rows),
+                coreDistances: q(names.coreDistances),
+                frontier: q(names.frontier),
+                mst: q(names.mst),
+            });
+        }
+        const result = await clusterHdbscan(connection, q(names.mst), {
+            count: prepared.rowCount,
+            minClusterSize,
+            allowSingleCluster,
+        });
+        const labelColumn = "label";
+        const membershipColumn = "membership";
+        const gloshColumn = "glosh";
+        const resultColumns = [
+            "row_id INTEGER",
+            `${labelColumn} VARCHAR`,
+            membershipScoreColumn === undefined
+                ? undefined
+                : `${membershipColumn} DOUBLE`,
+            outlierScoreColumn === undefined ? undefined : `${gloshColumn} DOUBLE`,
+        ].filter((definition) => definition !== undefined);
+        await connection.run(`CREATE TEMP TABLE ${q(names.result)} (${resultColumns.join(",")})`);
+        const outputTypes = [
+            INTEGER,
+            VARCHAR,
+            ...(membershipScoreColumn === undefined ? [] : [DOUBLE]),
+            ...(outlierScoreColumn === undefined ? [] : [DOUBLE]),
+        ];
+        await appendColumnBatches(connection, names.result, outputTypes, prepared.rowCount, (column, start, end) => {
+            if (column === 0) {
+                return Array.from({ length: end - start }, (_, offset) => start + offset);
+            }
+            if (column === 1) {
+                return Array.from(result.labels.subarray(start, end), (label) => label < 0 ? "noise" : `cluster-${label}`);
+            }
+            const membershipIndex = membershipScoreColumn === undefined ? -1 : 2;
+            return Array.from(column === membershipIndex
+                ? result.probabilities.subarray(start, end)
+                : result.outlierScores.subarray(start, end));
+        });
+        const publicationOutputs = [{
+                name: newColumn,
+                expression: `r.${q(labelColumn)}`,
+            }];
+        if (membershipScoreColumn !== undefined) {
+            publicationOutputs.push({
+                name: membershipScoreColumn,
+                expression: `r.${q(membershipColumn)}`,
+            });
+        }
+        if (outlierScoreColumn !== undefined) {
+            publicationOutputs.push({
+                name: outlierScoreColumn,
+                expression: `r.${q(gloshColumn)}`,
+            });
+        }
+        await publishPreparedColumns(table, prepared, {
+            method: "hdbscan()",
+            parameters: { columns, newColumn, options },
+            result: { relation: names.result, rowIdColumn: "row_id" },
+            outputs: publicationOutputs,
+        });
+    }
+    finally {
+        for (const relation of Object.entries(names).reverse().flatMap(([key, value]) => key === "hnsw" ? [] : [value])) {
+            await connection.run(`DROP TABLE IF EXISTS ${quoteIdentifier(relation)}`);
+        }
+        await prepared.cleanup();
+    }
+}
+function validateOutputNames(sourceColumns, outputNames) {
+    const seen = new Map();
+    for (const name of outputNames) {
+        if (name.length === 0 || name.includes("\0")) {
+            throw new Error("hdbscan() output column names must be non-empty strings without null characters.");
+        }
+        const folded = foldIdentifier(name);
+        const source = sourceColumns.find((column) => foldIdentifier(column) === folded);
+        if (source !== undefined) {
+            throw new Error(`hdbscan() cannot create ${quoteIdentifier(name)} because that source column already exists. Remove it first or choose different output names.`);
+        }
+        const duplicate = seen.get(folded);
+        if (duplicate !== undefined) {
+            throw new Error(`hdbscan() output columns ${quoteIdentifier(duplicate)} and ${quoteIdentifier(name)} refer to the same column. Choose distinct output names.`);
+        }
+        seen.set(folded, name);
+    }
+}

package/esm/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA4ExB"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAM1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+DxB"}

package/esm/methods/loadArray.js

@@ -1,4 +1,5 @@
-import { arrayValue, DuckDBDataChunk, DuckDBDateValue, DuckDBTimestampTZValue, DuckDBTimestampValue, DuckDBTimeValue, } from "@duckdb/node-api";
+import { arrayValue, DuckDBDateValue, DuckDBTimestampTZValue, DuckDBTimestampValue, DuckDBTimeValue, } from "@duckdb/node-api";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
 import prepareGeometry from "../helpers/prepareGeometry.js";
 import prepareJSON from "../helpers/prepareJSON.js";
 import geometryFromJSON from "../helpers/geometryFromJSON.js";
@@ -136,25 +137,7 @@ export async function executePreparedArray(simpleTable, prepared) {
     }
     try {
         await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${quoteIdentifier(staged)}(${keys.map((key, i) => `${quoteIdentifier(key)} ${storageTypes[i]}`).join(", ")})`);
-        const appender = await simpleTable.connection
-            .createAppender(staged);
-        try {
-            const duckDBTypes = storageTypes.map((d) => parseDuckDBType(d));
-            // The maximum capacity of a DuckDB data chunk is 2048 rows.
-            const chunkSize = 2000;
-            for (let start = 0; start < rowCount; start += chunkSize) {
-                const end = Math.min(start + chunkSize, rowCount);
-                const dataChunk = DuckDBDataChunk.create(duckDBTypes, end - start);
-                for (let i = 0; i < keys.length; i++) {
-                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-                }
-                appender.appendDataChunk(dataChunk);
-            }
-            appender.flushSync();
-        }
-        finally {
-            appender.closeSync();
-        }
+        await appendColumnBatches(simpleTable.connection, staged, storageTypes.map((type) => parseDuckDBType(type)), rowCount, (column, start, end) => columnsData[column].slice(start, end));
         if (!needsConversion)
             return;
         // CTAS is atomic: conversion must finish before the destination is replaced.

package/esm/methods/mahalanobis.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["mahalanobis"]>[3]>;
+export default function mahalanobis(table: SimpleTable, columns: string | string[], referencePoint: number[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=mahalanobis.d.ts.map

package/esm/methods/mahalanobis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mahalanobis.d.ts","sourceRoot":"","sources":["../../src/methods/mahalanobis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEtE,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,cAAc,EAAE,MAAM,EAAE,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CAyBN"}