@nshiab/simple-data-analysis-core 2.0.9 → 2.1.1

package/script/methods/fuzzyClean.js

@@ -5,14 +5,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = fuzzyClean;
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const node_api_1 = require("@duckdb/node-api");
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const buildFuzzyMatchSql_js_1 = __importDefault(require("../helpers/buildFuzzyMatchSql.js"));
+const selectFuzzyReplacements_js_1 = __importDefault(require("../helpers/selectFuzzyReplacements.js"));
 function fuzzyClean(table, column, newColumn, threshold, options = {}) {
-    // The clustering runs in JS over the fuzzy pairs read from the table, so
-    // fuzzyClean can't be expressed as a single SELECT over its input: it
-    // executes as a barrier.
+    // This implementation clusters fuzzy pairs in JS and writes the mapping
+    // back to DuckDB, so it executes as a barrier.
     options = structuredClone(options);
     (0, queueOp_js_1.default)(table, {
         kind: "barrier",
@@ -63,149 +64,29 @@ async function executeFuzzyClean(table, column, newColumn, threshold, options =
         }
         return;
     }
-    // Build count map from the pairs — no separate query needed.
-    const countMap = new Map();
-    for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {
-        countMap.set(left_value, Number(left_cnt));
-        countMap.set(right_value, Number(right_cnt));
-    }
-    // Union-Find — only over values that actually participate in a pair.
-    const parent = new Map();
-    const find = (x) => {
-        if (!parent.has(x))
-            parent.set(x, x);
-        if (parent.get(x) !== x)
-            parent.set(x, find(parent.get(x)));
-        return parent.get(x);
-    };
-    const union = (a, b) => {
-        const ra = find(a);
-        const rb = find(b);
-        if (ra !== rb)
-            parent.set(ra, rb);
-    };
-    for (const v of countMap.keys())
-        find(v);
-    for (const { left_value, right_value } of pairs) {
-        union(left_value, right_value);
-    }
-    // Group values by their cluster root.
-    const clusters = new Map();
-    for (const v of countMap.keys()) {
-        const root = find(v);
-        if (!clusters.has(root))
-            clusters.set(root, []);
-        clusters.get(root).push(v);
-    }
-    // Pick the canonical value for each cluster.
-    const replacement = new Map();
-    for (const members of clusters.values()) {
-        if (members.length === 1)
-            continue; // All pairs are already canonical — skip.
-        // Score sums and per-member max scores are computed lazily — only when
-        // actually needed as a primary criterion or tie-breaker.
-        //
-        // scoreSum: total similarity to all other cluster members (used by
-        //   "mostCentral" as primary, and as tie-breaker for all strategies).
-        // scoreMax: highest single pairwise score (used by "maxScore" as primary).
-        let scoreSum = null;
-        const getScoreSum = () => {
-            if (scoreSum === null) {
-                scoreSum = new Map(members.map((m) => [m, 0]));
-                for (const { left_value, right_value, score } of pairs) {
-                    if (scoreSum.has(left_value) && scoreSum.has(right_value)) {
-                        scoreSum.set(left_value, (scoreSum.get(left_value) ?? 0) + score);
-                        scoreSum.set(right_value, (scoreSum.get(right_value) ?? 0) + score);
-                    }
-                }
-            }
-            return scoreSum;
-        };
-        let scoreMax = null;
-        const getScoreMax = () => {
-            if (scoreMax === null) {
-                scoreMax = new Map(members.map((m) => [m, 0]));
-                for (const { left_value, right_value, score } of pairs) {
-                    if (scoreMax.has(left_value) && scoreMax.has(right_value)) {
-                        scoreMax.set(left_value, Math.max(scoreMax.get(left_value) ?? 0, score));
-                        scoreMax.set(right_value, Math.max(scoreMax.get(right_value) ?? 0, score));
-                    }
-                }
-            }
-            return scoreMax;
-        };
-        // Sum-based comparator: highest total similarity, then alphabetical.
-        const byScore = (a, b) => {
-            const sum = getScoreSum();
-            const sa = sum.get(a) ?? 0;
-            const sb = sum.get(b) ?? 0;
-            if (sa !== sb)
-                return sa > sb ? a : b;
-            return a <= b ? a : b; // alphabetical tie-break
-        };
-        // Max-based comparator: highest single-pair score, then sum, then alphabetical.
-        const byMaxScore = (a, b) => {
-            const mx = getScoreMax();
-            const ma = mx.get(a) ?? 0;
-            const mb = mx.get(b) ?? 0;
-            if (ma !== mb)
-                return ma > mb ? a : b;
-            return byScore(a, b); // sum then alphabetical tie-break
-        };
-        let canonical;
-        if (strategy === "longestString") {
-            canonical = members.reduce((a, b) => {
-                if (a.length !== b.length)
-                    return a.length > b.length ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "shortestString") {
-            canonical = members.reduce((a, b) => {
-                if (a.length !== b.length)
-                    return a.length < b.length ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "mostCommon") {
-            canonical = members.reduce((a, b) => {
-                const ca = countMap.get(a) ?? 0;
-                const cb = countMap.get(b) ?? 0;
-                if (ca !== cb)
-                    return ca > cb ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "mostCentral") {
-            // Most central string — highest total similarity to all other cluster members.
-            canonical = members.reduce((a, b) => byScore(a, b));
-        }
-        else {
-            // "maxScore": the string participating in the single highest-scoring pair.
-            canonical = members.reduce((a, b) => byMaxScore(a, b));
-        }
-        for (const m of members) {
-            if (m !== canonical)
-                replacement.set(m, canonical);
-        }
-    }
+    const replacement = (0, selectFuzzyReplacements_js_1.default)(pairs, strategy);
     if (replacement.size === 0)
         return;
-    // Use a VALUES-based CTE for the UPDATE so DuckDB can use a hash join
-    // instead of evaluating a potentially huge CASE WHEN expression.
-    const replacementEntries = [...replacement.entries()];
-    const valuesList = replacementEntries.map(() => "(?, ?)").join(",\n     ");
-    const values = replacementEntries.flatMap(([from, to]) => [from, to]);
+    // Two bound lists keep the SQL and parameter count fixed as the mapping
+    // grows. Side-by-side UNNEST zips the equally sized lists into mapping rows,
+    // which DuckDB can hash-join to the original table.
+    const values = [
+        new node_api_1.DuckDBListValue([...replacement.keys()]),
+        new node_api_1.DuckDBListValue([...replacement.values()]),
+    ];
+    const mapping = `WITH mapping(original, canonical) AS (
+    SELECT UNNEST(?::VARCHAR[]), UNNEST(?::VARCHAR[])
+  )`;
     const query = newColumn !== column
         ? `ALTER TABLE ${(0, quoteIdentifier_js_1.default)(table.name)} ADD ${(0, quoteIdentifier_js_1.default)(newColumn)} VARCHAR;
        UPDATE ${(0, quoteIdentifier_js_1.default)(table.name)}
          SET ${(0, quoteIdentifier_js_1.default)(newColumn)} = ${(0, quoteIdentifier_js_1.default)(column)};
-       WITH mapping(original, canonical) AS (VALUES ${valuesList})
+       ${mapping}
        UPDATE ${(0, quoteIdentifier_js_1.default)(table.name)}
          SET ${(0, quoteIdentifier_js_1.default)(newColumn)} = m.canonical
          FROM mapping m
          WHERE ${(0, quoteIdentifier_js_1.default)(table.name)}.${(0, quoteIdentifier_js_1.default)(newColumn)} = m.original`
-        : `WITH mapping(original, canonical) AS (VALUES ${valuesList})
+        : `${mapping}
        UPDATE ${(0, quoteIdentifier_js_1.default)(table.name)}
          SET ${(0, quoteIdentifier_js_1.default)(column)} = m.canonical
          FROM mapping m

package/script/methods/fuzzyJoin.js

@@ -54,17 +54,7 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
     }
     const method = options.method ?? "ratio";
     const similarityColumn = options.similarityColumn;
-    // The right table's copy of a column shared with the left table (only
-    // rightColumn can be shared, checked above) is excluded from the SELECT
-    // directly, instead of dropping its _1 duplicate with a rewrite after the
-    // join.
-    const rightSelect = rightCols
-        .filter((d) => !leftCols.includes(d))
-        .map((d) => `${(0, quoteIdentifier_js_1.default)(rightTable.name)}.${(0, quoteIdentifier_js_1.default)(d)}`)
-        .join(", ");
-    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength);
-    await (0, queryDB_js_1.default)(leftTable, sql, (0, mergeOptions_js_1.default)(leftTable, {
+    const queryOptions = (0, mergeOptions_js_1.default)(leftTable, {
         table: outputTable.name,
         method: "fuzzyJoin()",
         parameters: {
@@ -74,26 +64,90 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
             threshold,
             options,
         },
-    }));
+    });
+    const deduplicate = await shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions);
+    // The right table's copy of a column shared with the left table (only
+    // rightColumn can be shared, checked above) is excluded from the SELECT
+    // directly, instead of dropping its _1 duplicate with a rewrite after the
+    // join.
+    const rightSelect = rightCols
+        .filter((d) => !leftCols.includes(d))
+        .map((d) => `${(0, quoteIdentifier_js_1.default)(rightTable.name)}.${(0, quoteIdentifier_js_1.default)(d)}`)
+        .join(", ");
+    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
+        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength, deduplicate);
+    await (0, queryDB_js_1.default)(leftTable, sql, queryOptions);
 }
-function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength) {
+async function shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions) {
+    const left = (0, quoteIdentifier_js_1.default)(leftTable.name);
+    const right = (0, quoteIdentifier_js_1.default)(rightTable.name);
+    const counts = (await (0, queryDB_js_1.default)(leftTable, `SELECT (SELECT COUNT(*) FROM ${left}) AS left_count,
+      (SELECT COUNT(*) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    const leftCount = Number(counts.left_count);
+    const rightCount = Number(counts.right_count);
+    const comparisons = leftCount * rightCount;
+    // Avoid distinct scans for small joins and short lookup tables. For larger
+    // joins, require at least a twofold reduction to pay for grouping and rejoining.
+    if (comparisons < 100_000 || Math.min(leftCount, rightCount) < 32) {
+        return false;
+    }
+    const distinct = (await (0, queryDB_js_1.default)(leftTable, `SELECT
+      (SELECT COUNT(DISTINCT ENCODE(${(0, quoteIdentifier_js_1.default)(leftColumn)})) FROM ${left}) AS left_count,
+      (SELECT COUNT(DISTINCT ENCODE(${(0, quoteIdentifier_js_1.default)(rightColumn)})) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    return Number(distinct.left_count) * Number(distinct.right_count) <=
+        comparisons / 2;
+}
+function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength, deduplicate = false) {
     const leftExpression = `${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(leftColumn)}`;
     const rightExpression = `${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(rightColumn)}`;
     const { scoreExpression, condition } = (0, buildFuzzyMatchSql_js_1.default)(leftExpression, rightExpression, method, threshold, { decimals: 2, prefilterPrefixLength });
+    let cte = "";
+    let from = `FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)} ON ${condition}`;
+    let score = scoreExpression;
+    if (deduplicate) {
+        const matches = (0, quoteIdentifier_js_1.default)(`__sda_fuzzy_matches_${crypto.randomUUID().replaceAll("-", "")}`);
+        const match = (0, buildFuzzyMatchSql_js_1.default)("l.value", "r.value", method, threshold, { decimals: 2, prefilterPrefixLength });
+        // Keep the original value's collation for prefix matching, but use byte
+        // keys for deduplication and expansion: fuzzy scores distinguish case even
+        // when an input column has a case-insensitive collation.
+        cte = `WITH ${matches} AS MATERIALIZED (
+      SELECT l.key AS left_key, r.key AS right_key,
+        ${match.scoreExpression} AS score
+      FROM (
+        SELECT DISTINCT ${(0, quoteIdentifier_js_1.default)(leftColumn)} AS value,
+          ENCODE(${(0, quoteIdentifier_js_1.default)(leftColumn)}) AS key
+        FROM ${(0, quoteIdentifier_js_1.default)(leftTable)}
+        WHERE ${(0, quoteIdentifier_js_1.default)(leftColumn)} IS NOT NULL
+      ) l
+      INNER JOIN (
+        SELECT DISTINCT ${(0, quoteIdentifier_js_1.default)(rightColumn)} AS value,
+          ENCODE(${(0, quoteIdentifier_js_1.default)(rightColumn)}) AS key
+        FROM ${(0, quoteIdentifier_js_1.default)(rightTable)}
+        WHERE ${(0, quoteIdentifier_js_1.default)(rightColumn)} IS NOT NULL
+      ) r ON ${match.condition}
+    )`;
+        from = `FROM ${(0, quoteIdentifier_js_1.default)(leftTable)}
+      LEFT JOIN ${matches} ON ENCODE(${leftExpression}) = ${matches}.left_key
+      LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)}
+        ON ENCODE(${rightExpression}) = ${matches}.right_key`;
+        score = `${matches}.score`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable)} AS
+${cte}
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS ${(0, quoteIdentifier_js_1.default)(similarityColumn)}
 FROM (
-  SELECT ${(0, quoteIdentifier_js_1.default)(leftTable)}.*, ${rightSelect}, ${scoreExpression} AS "_sda_score"
-  FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)} ON ${condition}
+  SELECT ${(0, quoteIdentifier_js_1.default)(leftTable)}.*, ${rightSelect}, ${score} AS "_sda_score"
+  ${from}
 ) _sda
 ORDER BY ${(0, quoteIdentifier_js_1.default)(leftColumn)}, "_sda_score" DESC;\n`;
     }
     return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable)} AS
+${cte}
 SELECT *
 FROM (
   SELECT ${(0, quoteIdentifier_js_1.default)(leftTable)}.*, ${rightSelect}
-  FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)} ON ${condition}
+  ${from}
 ) _sda
 ORDER BY ${(0, quoteIdentifier_js_1.default)(leftColumn)}, ${(0, quoteIdentifier_js_1.default)(rightColumn)};\n`;
 }

package/script/methods/getTypes.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function getTypes(simpleTable: SimpleTable): Promise<{
-    [key: string]: string;
+    [k: string]: string;
 }>;
 //# sourceMappingURL=getTypes.d.ts.map

package/script/methods/hdbscan.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["hdbscan"]>[2]>;
+export default function hdbscan(table: SimpleTable, columns: string | string[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=hdbscan.d.ts.map

package/script/methods/hdbscan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hdbscan.d.ts","sourceRoot":"","sources":["../../src/methods/hdbscan.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElE,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CASN"}

package/script/methods/hdbscan.js

@@ -0,0 +1,214 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = hdbscan;
+const node_api_1 = require("@duckdb/node-api");
+const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
+const buildApproximateMutualReachabilityMst_js_1 = __importDefault(require("../helpers/buildApproximateMutualReachabilityMst.js"));
+const buildExactMutualReachabilityMst_js_1 = __importDefault(require("../helpers/buildExactMutualReachabilityMst.js"));
+const clusterHdbscan_js_1 = __importDefault(require("../helpers/clusterHdbscan.js"));
+const foldIdentifier_js_1 = __importDefault(require("../helpers/foldIdentifier.js"));
+const prepareNumericFeatures_js_1 = __importDefault(require("../helpers/prepareNumericFeatures.js"));
+const publishPreparedColumns_js_1 = __importDefault(require("../helpers/publishPreparedColumns.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+function hdbscan(table, columns, newColumn, options = {}) {
+    const selected = typeof columns === "string" ? columns : [...columns];
+    const settings = structuredClone(options);
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "hdbscan()",
+        parameters: { columns: selected, newColumn, options: settings },
+        execute: () => execute(table, selected, newColumn, settings),
+    });
+}
+async function execute(table, columns, newColumn, options) {
+    const { minClusterSize = 5, metric = "euclidean", allowSingleCluster = false, approximate = false, membershipScoreColumn, outlierScoreColumn, } = options;
+    const minSamples = options.minSamples ?? minClusterSize;
+    if (typeof newColumn !== "string") {
+        throw new Error("hdbscan() newColumn must be a string.");
+    }
+    if (!Number.isSafeInteger(minClusterSize) || minClusterSize < 2) {
+        throw new Error("hdbscan() minClusterSize must be a safe integer of at least 2.");
+    }
+    if (!Number.isSafeInteger(minSamples) || minSamples < 1) {
+        throw new Error("hdbscan() minSamples must be a safe integer of at least 1.");
+    }
+    if (!["euclidean", "cosine"].includes(metric)) {
+        throw new Error('hdbscan() metric must be "euclidean" or "cosine".');
+    }
+    if (typeof allowSingleCluster !== "boolean") {
+        throw new Error("hdbscan() allowSingleCluster must be a boolean.");
+    }
+    if (typeof approximate !== "boolean") {
+        throw new Error("hdbscan() approximate must be a boolean.");
+    }
+    if (membershipScoreColumn !== undefined &&
+        typeof membershipScoreColumn !== "string") {
+        throw new Error("hdbscan() membershipScoreColumn must be a string.");
+    }
+    if (outlierScoreColumn !== undefined &&
+        typeof outlierScoreColumn !== "string") {
+        throw new Error("hdbscan() outlierScoreColumn must be a string.");
+    }
+    const sourceColumns = Object.keys(await table.getTypes());
+    const outputNames = [newColumn, membershipScoreColumn, outlierScoreColumn]
+        .filter((name) => name !== undefined);
+    validateOutputNames(sourceColumns, outputNames);
+    const prepared = await (0, prepareNumericFeatures_js_1.default)(table, typeof columns === "string"
+        ? { kind: "vector", column: columns }
+        : { kind: "scalars", columns }, { method: "hdbscan()" });
+    const connection = table.connection;
+    const suffix = crypto.randomUUID().replaceAll("-", "");
+    const name = (part) => `__sda_hdbscan_${suffix}_${part}`;
+    const names = {
+        rows: name("rows"),
+        neighbors: name("neighbors"),
+        search: name("search"),
+        hnsw: name("hnsw"),
+        candidates: name("candidates"),
+        coreDistances: name("core_distances"),
+        frontier: name("frontier"),
+        edges: name("edges"),
+        components: name("components"),
+        representatives: name("representatives"),
+        componentPairs: name("component_pairs"),
+        bridgeCandidates: name("bridge_candidates"),
+        mst: name("mst"),
+        result: name("result"),
+    };
+    try {
+        if (prepared.rowCount < 2) {
+            throw new Error(`hdbscan() requires at least 2 rows, but the input has ${prepared.rowCount}.`);
+        }
+        if (prepared.rowCount > 0x40000000) {
+            throw new Error("hdbscan() supports at most 1073741824 rows because its 2*n-1 hierarchy nodes use signed INTEGER identifiers.");
+        }
+        if (minSamples >= prepared.rowCount) {
+            const defaulted = options.minSamples === undefined
+                ? " (defaulted from minClusterSize)"
+                : "";
+            throw new Error(`hdbscan() minSamples is ${minSamples}${defaulted}, but ${prepared.rowCount} rows provide at most ${prepared.rowCount - 1} other point${prepared.rowCount === 2 ? "" : "s"}. Set minSamples between 1 and ${prepared.rowCount - 1}, or provide at least ${minSamples + 1} rows.`);
+        }
+        if (minClusterSize > prepared.rowCount) {
+            throw new Error(`hdbscan() minClusterSize is ${minClusterSize}, but the input has ${prepared.rowCount} rows. Set minClusterSize between 2 and ${prepared.rowCount}, or provide more rows.`);
+        }
+        const q = quoteIdentifier_js_1.default;
+        await connection.run(`CREATE TEMP TABLE ${q(names.rows)} AS
+       SELECT ${q(prepared.rowIdColumn)}::INTEGER AS vertex,
+         ${q(prepared.vectorColumn)} AS vec
+       FROM ${q(prepared.relation)}
+       ORDER BY ${q(prepared.rowIdColumn)}`);
+        if (approximate) {
+            await (0, buildApproximateMutualReachabilityMst_js_1.default)(connection, {
+                count: prepared.rowCount,
+                dimensions: prepared.dimensions,
+                minSamples,
+            }, { metric }, {
+                rows: names.rows,
+                neighbors: names.neighbors,
+                search: names.search,
+                hnsw: names.hnsw,
+                candidates: names.candidates,
+                coreDistances: names.coreDistances,
+                edges: names.edges,
+                components: names.components,
+                representatives: names.representatives,
+                componentPairs: names.componentPairs,
+                bridgeCandidates: names.bridgeCandidates,
+                mst: names.mst,
+            });
+        }
+        else {
+            await (0, buildExactMutualReachabilityMst_js_1.default)(connection, { count: prepared.rowCount, minSamples }, { metric }, {
+                rows: q(names.rows),
+                coreDistances: q(names.coreDistances),
+                frontier: q(names.frontier),
+                mst: q(names.mst),
+            });
+        }
+        const result = await (0, clusterHdbscan_js_1.default)(connection, q(names.mst), {
+            count: prepared.rowCount,
+            minClusterSize,
+            allowSingleCluster,
+        });
+        const labelColumn = "label";
+        const membershipColumn = "membership";
+        const gloshColumn = "glosh";
+        const resultColumns = [
+            "row_id INTEGER",
+            `${labelColumn} VARCHAR`,
+            membershipScoreColumn === undefined
+                ? undefined
+                : `${membershipColumn} DOUBLE`,
+            outlierScoreColumn === undefined ? undefined : `${gloshColumn} DOUBLE`,
+        ].filter((definition) => definition !== undefined);
+        await connection.run(`CREATE TEMP TABLE ${q(names.result)} (${resultColumns.join(",")})`);
+        const outputTypes = [
+            node_api_1.INTEGER,
+            node_api_1.VARCHAR,
+            ...(membershipScoreColumn === undefined ? [] : [node_api_1.DOUBLE]),
+            ...(outlierScoreColumn === undefined ? [] : [node_api_1.DOUBLE]),
+        ];
+        await (0, appendColumnBatches_js_1.default)(connection, names.result, outputTypes, prepared.rowCount, (column, start, end) => {
+            if (column === 0) {
+                return Array.from({ length: end - start }, (_, offset) => start + offset);
+            }
+            if (column === 1) {
+                return Array.from(result.labels.subarray(start, end), (label) => label < 0 ? "noise" : `cluster-${label}`);
+            }
+            const membershipIndex = membershipScoreColumn === undefined ? -1 : 2;
+            return Array.from(column === membershipIndex
+                ? result.probabilities.subarray(start, end)
+                : result.outlierScores.subarray(start, end));
+        });
+        const publicationOutputs = [{
+                name: newColumn,
+                expression: `r.${q(labelColumn)}`,
+            }];
+        if (membershipScoreColumn !== undefined) {
+            publicationOutputs.push({
+                name: membershipScoreColumn,
+                expression: `r.${q(membershipColumn)}`,
+            });
+        }
+        if (outlierScoreColumn !== undefined) {
+            publicationOutputs.push({
+                name: outlierScoreColumn,
+                expression: `r.${q(gloshColumn)}`,
+            });
+        }
+        await (0, publishPreparedColumns_js_1.default)(table, prepared, {
+            method: "hdbscan()",
+            parameters: { columns, newColumn, options },
+            result: { relation: names.result, rowIdColumn: "row_id" },
+            outputs: publicationOutputs,
+        });
+    }
+    finally {
+        for (const relation of Object.entries(names).reverse().flatMap(([key, value]) => key === "hnsw" ? [] : [value])) {
+            await connection.run(`DROP TABLE IF EXISTS ${(0, quoteIdentifier_js_1.default)(relation)}`);
+        }
+        await prepared.cleanup();
+    }
+}
+function validateOutputNames(sourceColumns, outputNames) {
+    const seen = new Map();
+    for (const name of outputNames) {
+        if (name.length === 0 || name.includes("\0")) {
+            throw new Error("hdbscan() output column names must be non-empty strings without null characters.");
+        }
+        const folded = (0, foldIdentifier_js_1.default)(name);
+        const source = sourceColumns.find((column) => (0, foldIdentifier_js_1.default)(column) === folded);
+        if (source !== undefined) {
+            throw new Error(`hdbscan() cannot create ${(0, quoteIdentifier_js_1.default)(name)} because that source column already exists. Remove it first or choose different output names.`);
+        }
+        const duplicate = seen.get(folded);
+        if (duplicate !== undefined) {
+            throw new Error(`hdbscan() output columns ${(0, quoteIdentifier_js_1.default)(duplicate)} and ${(0, quoteIdentifier_js_1.default)(name)} refer to the same column. Choose distinct output names.`);
+        }
+        seen.set(folded, name);
+    }
+}

package/script/methods/mahalanobis.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["mahalanobis"]>[3]>;
+export default function mahalanobis(table: SimpleTable, columns: string | string[], referencePoint: number[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=mahalanobis.d.ts.map

package/script/methods/mahalanobis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mahalanobis.d.ts","sourceRoot":"","sources":["../../src/methods/mahalanobis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEtE,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,cAAc,EAAE,MAAM,EAAE,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CAyBN"}

package/script/methods/mahalanobis.js

@@ -0,0 +1,98 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = mahalanobis;
+const computeCovariance_js_1 = __importDefault(require("../helpers/computeCovariance.js"));
+const computeMahalanobisDistances_js_1 = __importDefault(require("../helpers/computeMahalanobisDistances.js"));
+const foldIdentifier_js_1 = __importDefault(require("../helpers/foldIdentifier.js"));
+const prepareNumericFeatures_js_1 = __importDefault(require("../helpers/prepareNumericFeatures.js"));
+const publishPreparedColumns_js_1 = __importDefault(require("../helpers/publishPreparedColumns.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+function mahalanobis(table, columns, referencePoint, newColumn, options = {}) {
+    if (!Array.isArray(referencePoint) || referencePoint.length === 0 ||
+        Array.from(referencePoint).some((value) => typeof value !== "number" || !Number.isFinite(value))) {
+        throw new Error("mahalanobis() referencePoint must be a nonempty array of finite numbers in feature-dimension order.");
+    }
+    const selected = typeof columns === "string" ? columns : [...columns];
+    const reference = [...referencePoint];
+    const settings = { ...options };
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "mahalanobis()",
+        parameters: {
+            columns: selected,
+            referencePoint: reference,
+            newColumn,
+            options: settings,
+        },
+        execute: () => execute(table, selected, reference, newColumn, settings),
+    });
+}
+async function execute(table, columns, referencePoint, newColumn, options) {
+    const sourceColumns = Object.keys(await table.getTypes());
+    const outputNames = options.similarityScoreColumn === undefined
+        ? [newColumn]
+        : [newColumn, options.similarityScoreColumn];
+    const seen = new Set();
+    for (const name of outputNames) {
+        if (typeof name !== "string" || name.length === 0 || name.includes("\0")) {
+            throw new Error("mahalanobis() output column names must be nonempty strings without null characters.");
+        }
+        const folded = (0, foldIdentifier_js_1.default)(name);
+        if (seen.has(folded)) {
+            throw new Error("mahalanobis() distance and similarity score columns must have different names (case-insensitive).");
+        }
+        seen.add(folded);
+        if (sourceColumns.some((column) => (0, foldIdentifier_js_1.default)(column) === folded)) {
+            throw new Error(`mahalanobis() cannot create ${(0, quoteIdentifier_js_1.default)(name)} because that column already exists. Remove it first or choose a different name.`);
+        }
+    }
+    const prepared = await (0, prepareNumericFeatures_js_1.default)(table, typeof columns === "string"
+        ? { kind: "vector", column: columns }
+        : { kind: "scalars", columns }, { method: "mahalanobis()" });
+    try {
+        if (referencePoint.length !== prepared.dimensions) {
+            throw new Error(`mahalanobis() referencePoint must contain ${prepared.dimensions} values in feature-dimension order; received ${referencePoint.length}.`);
+        }
+        const model = await (0, computeCovariance_js_1.default)(table.connection, {
+            relation: prepared.relation,
+            rowIdColumn: prepared.rowIdColumn,
+            vectorColumn: prepared.vectorColumn,
+            observations: prepared.rowCount,
+            dimensions: prepared.dimensions,
+        });
+        const distances = await (0, computeMahalanobisDistances_js_1.default)(table.connection, {
+            relation: prepared.relation,
+            rowIdColumn: prepared.rowIdColumn,
+            vectorColumn: prepared.vectorColumn,
+        }, model, referencePoint);
+        try {
+            const distance = `r.${(0, quoteIdentifier_js_1.default)(distances.distanceColumn)}`;
+            const outputs = [{ name: newColumn, expression: distance }];
+            if (options.similarityScoreColumn !== undefined) {
+                outputs.push({
+                    name: options.similarityScoreColumn,
+                    expression: `CASE WHEN max(${distance}) OVER () = 0 THEN 1::DOUBLE ELSE 1 - ${distance} / max(${distance}) OVER () END`,
+                });
+            }
+            await (0, publishPreparedColumns_js_1.default)(table, prepared, {
+                method: "mahalanobis()",
+                parameters: { columns, referencePoint, newColumn, options },
+                result: {
+                    relation: distances.relation,
+                    rowIdColumn: distances.rowIdColumn,
+                },
+                outputs,
+            });
+        }
+        finally {
+            await distances.cleanup();
+        }
+    }
+    finally {
+        await prepared.cleanup();
+    }
+}

package/script/methods/neighbors.d.ts

@@ -0,0 +1,9 @@
+import { type GraphId } from "../helpers/prepareGraphStarts.js";
+import type SimpleTable from "../class/SimpleTable.js";
+type NeighborsOptions = {
+    direction?: "outgoing" | "incoming" | "both";
+    outputTable?: string | boolean;
+};
+export default function neighbors(simpleTable: SimpleTable, sourceColumn: string, targetColumn: string, startNodes: GraphId | GraphId[], options?: NeighborsOptions): SimpleTable;
+export {};
+//# sourceMappingURL=neighbors.d.ts.map

package/script/methods/neighbors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"neighbors.d.ts","sourceRoot":"","sources":["../../src/methods/neighbors.ts"],"names":[],"mappings":"AAAA,OAA2B,EACzB,KAAK,OAAO,EAEb,MAAM,kCAAkC,CAAC;AAO1C,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,KAAK,gBAAgB,GAAG;IACtB,SAAS,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IAC7C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,OAAO,GAAG,OAAO,EAAE,EAC/B,OAAO,GAAE,gBAAqB,GAC7B,WAAW,CAuEb"}