@nshiab/simple-data-analysis-core 2.0.9 → 2.1.0

package/esm/methods/findCycles.js

@@ -0,0 +1,330 @@
+import getGraphEdgeIdColumn from "../helpers/getGraphEdgeIdColumn.js";
+import getGraphWeightColumn from "../helpers/getGraphWeightColumn.js";
+import graphRouteResultSchema from "../helpers/graphRouteResultSchema.js";
+import graphRouteResultSelect from "../helpers/graphRouteResultSelect.js";
+import prepareGraphTemporalSql, { prepareGraphTemporalOptions, } from "../helpers/prepareGraphTemporalSql.js";
+import prepareGraphStarts from "../helpers/prepareGraphStarts.js";
+import prepareGraphTraversal, { validateGraphStarts, } from "../helpers/prepareGraphTraversal.js";
+import queueGraphResult from "../helpers/queueGraphResult.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import validateGraphTemporalEvents from "../helpers/validateGraphTemporalEvents.js";
+import validateGraphRouteResultSchema from "../helpers/validateGraphRouteResultSchema.js";
+export default function findCycles(simpleTable, sourceColumn, targetColumn, edgeId, startNodes, options = {}) {
+    if (typeof sourceColumn !== "string") {
+        throw new TypeError("findCycles() sourceColumn must be a string.");
+    }
+    if (typeof targetColumn !== "string") {
+        throw new TypeError("findCycles() targetColumn must be a string.");
+    }
+    if (typeof edgeId !== "string") {
+        throw new TypeError("findCycles() edgeId must be a string.");
+    }
+    const preparedStarts = prepareGraphStarts(startNodes, "findCycles()", "startNodes");
+    if (options === null || typeof options !== "object" || Array.isArray(options)) {
+        throw new TypeError("findCycles() options must be an object.");
+    }
+    if (options.direction !== undefined &&
+        !["outgoing", "incoming", "both"].includes(options.direction)) {
+        throw new TypeError('findCycles() options.direction must be "outgoing", "incoming", or "both".');
+    }
+    if (options.weight !== undefined && typeof options.weight !== "string") {
+        throw new TypeError("findCycles() options.weight must be a string.");
+    }
+    if (options.outputTable !== undefined &&
+        typeof options.outputTable !== "string" &&
+        typeof options.outputTable !== "boolean") {
+        throw new TypeError("findCycles() options.outputTable must be a string or boolean.");
+    }
+    const direction = options.direction ?? "outgoing";
+    const temporalOptions = prepareGraphTemporalOptions(options, direction, "findCycles()");
+    options = structuredClone(options);
+    const parameters = {
+        sourceColumn,
+        targetColumn,
+        edgeId,
+        startNodes: structuredClone(startNodes),
+        options,
+    };
+    return queueGraphResult(simpleTable, {
+        method: "findCycles()",
+        parameters,
+        outputTable: options.outputTable,
+        preflight: temporalOptions === undefined
+            ? undefined
+            : (input) => validateGraphTemporalEvents(input, temporalOptions, "findCycles()", parameters),
+        values: (schema) => {
+            validateFindCyclesInputs(schema, sourceColumn, targetColumn, edgeId, preparedStarts, options.weight);
+            const temporal = temporalOptions === undefined
+                ? undefined
+                : prepareGraphTemporalSql(schema, temporalOptions, "findCycles()");
+            return temporal === undefined
+                ? preparedStarts.values
+                : [temporal.gapParameter, ...preparedStarts.values];
+        },
+        buildSelect: (input, schema) => findCyclesSelect(input, schema, sourceColumn, targetColumn, edgeId, preparedStarts, direction, options.weight, temporalOptions),
+        outputSchema: (schema) => {
+            const validated = validateFindCyclesInputs(schema, sourceColumn, targetColumn, edgeId, preparedStarts, options.weight);
+            if (temporalOptions !== undefined) {
+                prepareGraphTemporalSql(schema, temporalOptions, "findCycles()");
+            }
+            return {
+                start: validated.nodeIdType,
+                ...graphRouteResultSchema(schema, validated.distanceType, "findCycles()"),
+            };
+        },
+    });
+}
+function validateFindCyclesInputs(schema, source, target, edgeId, starts, weight) {
+    const endpoints = validateGraphStarts(schema, source, target, starts, "findCycles()", "startNodes");
+    const edgeIdColumn = getGraphEdgeIdColumn(schema, edgeId, "findCycles()");
+    const weightColumn = weight === undefined
+        ? undefined
+        : getGraphWeightColumn(schema, weight, "findCycles()");
+    validateGraphRouteResultSchema(schema, "findCycles()", ["start"]);
+    return {
+        distanceType: weightColumn?.distanceType ?? "BIGINT",
+        edgeIdType: edgeIdColumn.idType,
+        nodeIdType: endpoints.idType,
+    };
+}
+function findCyclesSelect(input, schema, source, target, edgeId, starts, direction, weight, temporalOptions) {
+    const prepared = prepareGraphTraversal(input, schema, source, target, starts, "findCycles()", "startNodes");
+    const edgeIdColumn = getGraphEdgeIdColumn(schema, edgeId, "findCycles()");
+    const weightColumn = weight === undefined
+        ? undefined
+        : getGraphWeightColumn(schema, weight, "findCycles()");
+    const distanceType = weightColumn?.distanceType ?? "BIGINT";
+    const q = quoteIdentifier;
+    const edgeIdReference = `${q("edges")}.${q(edgeIdColumn.column)}`;
+    const typedEdgeId = `CAST(${edgeIdReference} AS ${edgeIdColumn.idType})`;
+    const edgeKey = edgeIdColumn.family === "string"
+        ? `ENCODE(${typedEdgeId})`
+        : typedEdgeId;
+    const edgeWeight = weightColumn === undefined
+        ? `CAST(1 AS ${distanceType})`
+        : `CAST(${q("edges")}.${q(weightColumn.column)} AS ${distanceType})`;
+    const stepType = `STRUCT(${q("edgeId")} ${edgeIdColumn.idType}, ${q("source")} ${prepared.endpoints.idType}, ${q("target")} ${prepared.endpoints.idType}, ${q("weight")} ${distanceType}, ${q("distance")} ${distanceType})`;
+    if (temporalOptions !== undefined) {
+        const temporal = prepareGraphTemporalSql(schema, temporalOptions, "findCycles()");
+        return temporalFindCyclesSelect(prepared, direction, temporal, typedEdgeId, edgeKey, edgeWeight, distanceType, stepType, input, edgeIdColumn);
+    }
+    const relations = prepared.relationNames([
+        "graph_start_values",
+        "graph_starts",
+        "graph_edges",
+        "graph_cycle_walks",
+        "graph_complete_cycles",
+        "graph_ranked_cycles",
+    ]);
+    const startValuesRelation = relations.graph_start_values;
+    const startsRelation = relations.graph_starts;
+    const edgesRelation = relations.graph_edges;
+    const walksRelation = relations.graph_cycle_walks;
+    const completeRelation = relations.graph_complete_cycles;
+    const rankedRelation = relations.graph_ranked_cycles;
+    const edgeFrom = `${q("edges")}.${q("__from")}`;
+    const edgeTo = `${q("edges")}.${q("__to")}`;
+    const edgeFromKey = `${q("edges")}.${q("__from_key")}`;
+    const edgeToKey = `${q("edges")}.${q("__to_key")}`;
+    const edgeIdentity = `${q("edges")}.${q("__edge_id")}`;
+    const edgeIdentityKey = `${q("edges")}.${q("__edge_key")}`;
+    const edgeCost = `${q("edges")}.${q("__weight")}`;
+    const walkDistance = `${q("walks")}.${q("distance")}`;
+    const candidateDistance = `CAST(${walkDistance} + ${edgeCost} AS ${distanceType})`;
+    return `WITH RECURSIVE ${startValuesRelation}(${q("start")}) AS (
+      VALUES ${prepared.startValues}
+    ), ${startsRelation} AS (
+      SELECT ${q("start")},
+        ${prepared.key(q("start"))} AS ${q("__start_key")}
+      FROM ${startValuesRelation}
+    ), ${edgesRelation} AS (
+      ${prepared.edges(direction, [
+        `${typedEdgeId} AS ${q("__edge_id")}`,
+        `${edgeKey} AS ${q("__edge_key")}`,
+        `${edgeWeight} AS ${q("__weight")}`,
+    ])}
+    ), ${walksRelation}(
+      ${q("start")}, ${q("__start_key")}, ${q("node")},
+      ${q("__node_key")}, ${q("__visited")}, ${q("__edge_keys")},
+      ${q("steps")}, ${q("distance")}, ${q("closed")}
+    ) AS (
+      SELECT ${q("starts")}.${q("start")},
+        ${q("starts")}.${q("__start_key")}, ${edgeTo}, ${edgeToKey},
+        CASE WHEN ${edgeFromKey} = ${edgeToKey}
+          THEN [${edgeFromKey}]
+          ELSE [${edgeFromKey}, ${edgeToKey}]
+        END,
+        [${edgeIdentityKey}],
+        [struct_pack(
+          ${q("edgeId")} := ${edgeIdentity},
+          ${q("source")} := ${edgeFrom},
+          ${q("target")} := ${edgeTo},
+          ${q("weight")} := ${edgeCost},
+          ${q("distance")} := ${edgeCost}
+        )]::${stepType}[],
+        ${edgeCost}, ${edgeFromKey} = ${edgeToKey}
+      FROM ${edgesRelation} AS ${q("edges")}
+      INNER JOIN ${startsRelation} AS ${q("starts")}
+        ON ${q("starts")}.${q("__start_key")} = ${edgeFromKey}
+      UNION ALL
+      SELECT ${q("walks")}.${q("start")},
+        ${q("walks")}.${q("__start_key")}, ${edgeTo}, ${edgeToKey},
+        list_append(${q("walks")}.${q("__visited")}, ${edgeToKey}),
+        list_append(${q("walks")}.${q("__edge_keys")}, ${edgeIdentityKey}),
+        list_append(${q("walks")}.${q("steps")}, struct_pack(
+          ${q("edgeId")} := ${edgeIdentity},
+          ${q("source")} := ${edgeFrom},
+          ${q("target")} := ${edgeTo},
+          ${q("weight")} := ${edgeCost},
+          ${q("distance")} := ${candidateDistance}
+        )), ${candidateDistance},
+        ${edgeToKey} = ${q("walks")}.${q("__start_key")}
+      FROM ${walksRelation} AS ${q("walks")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${q("walks")}.${q("__node_key")} = ${edgeFromKey}
+      WHERE NOT ${q("walks")}.${q("closed")}
+        AND NOT list_contains(
+          ${q("walks")}.${q("__edge_keys")}, ${edgeIdentityKey}
+        )
+        AND (
+          ${edgeToKey} = ${q("walks")}.${q("__start_key")}
+          OR NOT list_contains(
+            ${q("walks")}.${q("__visited")}, ${edgeToKey}
+          )
+        )
+    ), ${completeRelation} AS (
+      SELECT DISTINCT ${q("start")}, ${q("__start_key")},
+        ${q("__edge_keys")}, ${q("steps")}
+      FROM ${walksRelation}
+      WHERE ${q("closed")}${direction === "both"
+        ? `
+        AND ${q("__edge_keys")} <= list_reverse(${q("__edge_keys")})`
+        : ""}
+    ), ${rankedRelation} AS (
+      SELECT ${q("start")}, ${q("__start_key")},
+        CAST(row_number() OVER (
+          PARTITION BY ${q("__start_key")}
+          ORDER BY ${q("__edge_keys")}) - 1
+          AS BIGINT) AS ${q("pathId")}, ${q("steps")}
+      FROM ${completeRelation}
+    )
+    ${graphRouteResultSelect(rankedRelation, input, edgeIdColumn, true)}`;
+}
+function temporalFindCyclesSelect(prepared, direction, temporal, typedEdgeId, edgeKey, edgeWeight, distanceType, stepType, input, edgeIdColumn) {
+    const q = quoteIdentifier;
+    const relations = prepared.relationNames([
+        "graph_start_values",
+        "graph_starts",
+        "graph_temporal_settings",
+        "graph_event_rows",
+        "graph_edges",
+        "graph_cycle_walks",
+        "graph_complete_cycles",
+        "graph_ranked_cycles",
+    ]);
+    const startValuesRelation = relations.graph_start_values;
+    const startsRelation = relations.graph_starts;
+    const settingsRelation = relations.graph_temporal_settings;
+    const eventRowsRelation = relations.graph_event_rows;
+    const edgesRelation = relations.graph_edges;
+    const walksRelation = relations.graph_cycle_walks;
+    const completeRelation = relations.graph_complete_cycles;
+    const rankedRelation = relations.graph_ranked_cycles;
+    const edgeFrom = `${q("edges")}.${q("__from")}`;
+    const edgeTo = `${q("edges")}.${q("__to")}`;
+    const edgeFromKey = `${q("edges")}.${q("__from_key")}`;
+    const edgeToKey = `${q("edges")}.${q("__to_key")}`;
+    const edgeIdentity = `${q("edges")}.${q("__edge_id")}`;
+    const edgeIdentityKey = `${q("edges")}.${q("__edge_key")}`;
+    const edgeCost = `${q("edges")}.${q("__weight")}`;
+    const walkDistance = `${q("walks")}.${q("distance")}`;
+    const candidateDistance = `CAST(${walkDistance} + ${edgeCost} AS ${distanceType})`;
+    const gap = `${q("settings")}.${q("__gap")}`;
+    const step = (distance) => `struct_pack(
+          ${q("edgeId")} := ${edgeIdentity},
+          ${q("source")} := ${edgeFrom},
+          ${q("target")} := ${edgeTo},
+          ${q("weight")} := ${edgeCost},
+          ${q("distance")} := ${distance}
+        )`;
+    return `WITH RECURSIVE ${settingsRelation} AS MATERIALIZED (
+      SELECT CAST(? AS HUGEINT) AS ${q("__gap")}
+    ), ${startValuesRelation}(${q("start")}) AS (
+      VALUES ${prepared.startValues}
+    ), ${startsRelation} AS (
+      SELECT ${q("start")},
+        ${prepared.key(q("start"))} AS ${q("__start_key")}
+      FROM ${startValuesRelation}
+    ), ${eventRowsRelation} AS MATERIALIZED (
+      ${prepared.edges(direction, [
+        `${typedEdgeId} AS ${q("__edge_id")}`,
+        `${edgeKey} AS ${q("__edge_key")}`,
+        `${edgeWeight} AS ${q("__weight")}`,
+        ...temporal.eventSelections("edges"),
+    ])}
+    ), ${edgesRelation} AS MATERIALIZED (
+      SELECT *
+      FROM ${eventRowsRelation} AS ${q("events")}
+      WHERE ${temporal.eventValidity("events")}
+    ), ${walksRelation}(
+      ${q("start")}, ${q("__start_key")}, ${q("node")},
+      ${q("__node_key")}, ${q("__visited")}, ${q("__event_ids")},
+      ${q("__edge_keys")}, ${q("steps")}, ${q("distance")},
+      ${q("closed")}, ${q("__event_id")}, ${q("__event_start")},
+      ${q("__event_end")}
+    ) AS (
+      SELECT ${q("starts")}.${q("start")},
+        ${q("starts")}.${q("__start_key")}, ${edgeTo}, ${edgeToKey},
+        CASE WHEN ${edgeFromKey} = ${edgeToKey}
+          THEN [${edgeFromKey}]
+          ELSE [${edgeFromKey}, ${edgeToKey}]
+        END,
+        [${q("edges")}.${q("__event_id")}], [${edgeIdentityKey}],
+        [${step(edgeCost)}]::${stepType}[], ${edgeCost},
+        ${edgeFromKey} = ${edgeToKey},
+        ${q("edges")}.${q("__event_id")},
+        ${q("edges")}.${q("__event_start")},
+        ${q("edges")}.${q("__event_end")}
+      FROM ${edgesRelation} AS ${q("edges")}
+      INNER JOIN ${startsRelation} AS ${q("starts")}
+        ON ${q("starts")}.${q("__start_key")} = ${edgeFromKey}
+      UNION ALL
+      SELECT ${q("walks")}.${q("start")},
+        ${q("walks")}.${q("__start_key")}, ${edgeTo}, ${edgeToKey},
+        list_append(${q("walks")}.${q("__visited")}, ${edgeToKey}),
+        list_append(${q("walks")}.${q("__event_ids")},
+          ${q("edges")}.${q("__event_id")}),
+        list_append(${q("walks")}.${q("__edge_keys")}, ${edgeIdentityKey}),
+        list_append(${q("walks")}.${q("steps")}, ${step(candidateDistance)}),
+        ${candidateDistance},
+        ${edgeToKey} = ${q("walks")}.${q("__start_key")},
+        ${q("edges")}.${q("__event_id")},
+        ${q("edges")}.${q("__event_start")},
+        ${q("edges")}.${q("__event_end")}
+      FROM ${walksRelation} AS ${q("walks")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${q("walks")}.${q("__node_key")} = ${edgeFromKey}
+      CROSS JOIN ${settingsRelation} AS ${q("settings")}
+      WHERE NOT ${q("walks")}.${q("closed")}
+        AND NOT list_contains(${q("walks")}.${q("__event_ids")},
+          ${q("edges")}.${q("__event_id")})
+        AND (
+          ${edgeToKey} = ${q("walks")}.${q("__start_key")}
+          OR NOT list_contains(${q("walks")}.${q("__visited")}, ${edgeToKey})
+        )
+        AND ${temporal.transition("walks", "edges", direction, gap)}
+    ), ${completeRelation} AS (
+      SELECT DISTINCT ${q("start")}, ${q("__start_key")},
+        ${q("__edge_keys")}, ${q("steps")}
+      FROM ${walksRelation}
+      WHERE ${q("closed")}
+    ), ${rankedRelation} AS (
+      SELECT ${q("start")}, ${q("__start_key")},
+        CAST(row_number() OVER (
+          PARTITION BY ${q("__start_key")}
+          ORDER BY ${q("__edge_keys")}) - 1
+          AS BIGINT) AS ${q("pathId")}, ${q("steps")}
+      FROM ${completeRelation}
+    )
+    ${graphRouteResultSelect(rankedRelation, input, edgeIdColumn, true)}`;
+}

package/esm/methods/fuzzyClean.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,QAAQ,CAAC,EACL,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAC3B,GACL,IAAI,CAYN"}
+{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,QAAQ,CAAC,EACL,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAC3B,GACL,IAAI,CAWN"}

package/esm/methods/fuzzyClean.js

@@ -1,12 +1,13 @@
 import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import { DuckDBListValue } from "@duckdb/node-api";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
 import queueOp from "../helpers/queueOp.js";
 import buildFuzzyMatchSql from "../helpers/buildFuzzyMatchSql.js";
+import selectFuzzyReplacements from "../helpers/selectFuzzyReplacements.js";
 export default function fuzzyClean(table, column, newColumn, threshold, options = {}) {
-    // The clustering runs in JS over the fuzzy pairs read from the table, so
-    // fuzzyClean can't be expressed as a single SELECT over its input: it
-    // executes as a barrier.
+    // This implementation clusters fuzzy pairs in JS and writes the mapping
+    // back to DuckDB, so it executes as a barrier.
     options = structuredClone(options);
     queueOp(table, {
         kind: "barrier",
@@ -57,149 +58,29 @@ async function executeFuzzyClean(table, column, newColumn, threshold, options =
         }
         return;
     }
-    // Build count map from the pairs — no separate query needed.
-    const countMap = new Map();
-    for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {
-        countMap.set(left_value, Number(left_cnt));
-        countMap.set(right_value, Number(right_cnt));
-    }
-    // Union-Find — only over values that actually participate in a pair.
-    const parent = new Map();
-    const find = (x) => {
-        if (!parent.has(x))
-            parent.set(x, x);
-        if (parent.get(x) !== x)
-            parent.set(x, find(parent.get(x)));
-        return parent.get(x);
-    };
-    const union = (a, b) => {
-        const ra = find(a);
-        const rb = find(b);
-        if (ra !== rb)
-            parent.set(ra, rb);
-    };
-    for (const v of countMap.keys())
-        find(v);
-    for (const { left_value, right_value } of pairs) {
-        union(left_value, right_value);
-    }
-    // Group values by their cluster root.
-    const clusters = new Map();
-    for (const v of countMap.keys()) {
-        const root = find(v);
-        if (!clusters.has(root))
-            clusters.set(root, []);
-        clusters.get(root).push(v);
-    }
-    // Pick the canonical value for each cluster.
-    const replacement = new Map();
-    for (const members of clusters.values()) {
-        if (members.length === 1)
-            continue; // All pairs are already canonical — skip.
-        // Score sums and per-member max scores are computed lazily — only when
-        // actually needed as a primary criterion or tie-breaker.
-        //
-        // scoreSum: total similarity to all other cluster members (used by
-        //   "mostCentral" as primary, and as tie-breaker for all strategies).
-        // scoreMax: highest single pairwise score (used by "maxScore" as primary).
-        let scoreSum = null;
-        const getScoreSum = () => {
-            if (scoreSum === null) {
-                scoreSum = new Map(members.map((m) => [m, 0]));
-                for (const { left_value, right_value, score } of pairs) {
-                    if (scoreSum.has(left_value) && scoreSum.has(right_value)) {
-                        scoreSum.set(left_value, (scoreSum.get(left_value) ?? 0) + score);
-                        scoreSum.set(right_value, (scoreSum.get(right_value) ?? 0) + score);
-                    }
-                }
-            }
-            return scoreSum;
-        };
-        let scoreMax = null;
-        const getScoreMax = () => {
-            if (scoreMax === null) {
-                scoreMax = new Map(members.map((m) => [m, 0]));
-                for (const { left_value, right_value, score } of pairs) {
-                    if (scoreMax.has(left_value) && scoreMax.has(right_value)) {
-                        scoreMax.set(left_value, Math.max(scoreMax.get(left_value) ?? 0, score));
-                        scoreMax.set(right_value, Math.max(scoreMax.get(right_value) ?? 0, score));
-                    }
-                }
-            }
-            return scoreMax;
-        };
-        // Sum-based comparator: highest total similarity, then alphabetical.
-        const byScore = (a, b) => {
-            const sum = getScoreSum();
-            const sa = sum.get(a) ?? 0;
-            const sb = sum.get(b) ?? 0;
-            if (sa !== sb)
-                return sa > sb ? a : b;
-            return a <= b ? a : b; // alphabetical tie-break
-        };
-        // Max-based comparator: highest single-pair score, then sum, then alphabetical.
-        const byMaxScore = (a, b) => {
-            const mx = getScoreMax();
-            const ma = mx.get(a) ?? 0;
-            const mb = mx.get(b) ?? 0;
-            if (ma !== mb)
-                return ma > mb ? a : b;
-            return byScore(a, b); // sum then alphabetical tie-break
-        };
-        let canonical;
-        if (strategy === "longestString") {
-            canonical = members.reduce((a, b) => {
-                if (a.length !== b.length)
-                    return a.length > b.length ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "shortestString") {
-            canonical = members.reduce((a, b) => {
-                if (a.length !== b.length)
-                    return a.length < b.length ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "mostCommon") {
-            canonical = members.reduce((a, b) => {
-                const ca = countMap.get(a) ?? 0;
-                const cb = countMap.get(b) ?? 0;
-                if (ca !== cb)
-                    return ca > cb ? a : b;
-                return byScore(a, b);
-            });
-        }
-        else if (strategy === "mostCentral") {
-            // Most central string — highest total similarity to all other cluster members.
-            canonical = members.reduce((a, b) => byScore(a, b));
-        }
-        else {
-            // "maxScore": the string participating in the single highest-scoring pair.
-            canonical = members.reduce((a, b) => byMaxScore(a, b));
-        }
-        for (const m of members) {
-            if (m !== canonical)
-                replacement.set(m, canonical);
-        }
-    }
+    const replacement = selectFuzzyReplacements(pairs, strategy);
     if (replacement.size === 0)
         return;
-    // Use a VALUES-based CTE for the UPDATE so DuckDB can use a hash join
-    // instead of evaluating a potentially huge CASE WHEN expression.
-    const replacementEntries = [...replacement.entries()];
-    const valuesList = replacementEntries.map(() => "(?, ?)").join(",\n     ");
-    const values = replacementEntries.flatMap(([from, to]) => [from, to]);
+    // Two bound lists keep the SQL and parameter count fixed as the mapping
+    // grows. Side-by-side UNNEST zips the equally sized lists into mapping rows,
+    // which DuckDB can hash-join to the original table.
+    const values = [
+        new DuckDBListValue([...replacement.keys()]),
+        new DuckDBListValue([...replacement.values()]),
+    ];
+    const mapping = `WITH mapping(original, canonical) AS (
+    SELECT UNNEST(?::VARCHAR[]), UNNEST(?::VARCHAR[])
+  )`;
     const query = newColumn !== column
         ? `ALTER TABLE ${quoteIdentifier(table.name)} ADD ${quoteIdentifier(newColumn)} VARCHAR;
        UPDATE ${quoteIdentifier(table.name)}
          SET ${quoteIdentifier(newColumn)} = ${quoteIdentifier(column)};
-       WITH mapping(original, canonical) AS (VALUES ${valuesList})
+       ${mapping}
        UPDATE ${quoteIdentifier(table.name)}
          SET ${quoteIdentifier(newColumn)} = m.canonical
          FROM mapping m
          WHERE ${quoteIdentifier(table.name)}.${quoteIdentifier(newColumn)} = m.original`
-        : `WITH mapping(original, canonical) AS (VALUES ${valuesList})
+        : `${mapping}
        UPDATE ${quoteIdentifier(table.name)}
          SET ${quoteIdentifier(column)} = m.canonical
          FROM mapping m

package/esm/methods/fuzzyJoin.js

@@ -48,17 +48,7 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
     }
     const method = options.method ?? "ratio";
     const similarityColumn = options.similarityColumn;
-    // The right table's copy of a column shared with the left table (only
-    // rightColumn can be shared, checked above) is excluded from the SELECT
-    // directly, instead of dropping its _1 duplicate with a rewrite after the
-    // join.
-    const rightSelect = rightCols
-        .filter((d) => !leftCols.includes(d))
-        .map((d) => `${quoteIdentifier(rightTable.name)}.${quoteIdentifier(d)}`)
-        .join(", ");
-    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength);
-    await queryDB(leftTable, sql, mergeOptions(leftTable, {
+    const queryOptions = mergeOptions(leftTable, {
         table: outputTable.name,
         method: "fuzzyJoin()",
         parameters: {
@@ -68,26 +58,90 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
             threshold,
             options,
         },
-    }));
+    });
+    const deduplicate = await shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions);
+    // The right table's copy of a column shared with the left table (only
+    // rightColumn can be shared, checked above) is excluded from the SELECT
+    // directly, instead of dropping its _1 duplicate with a rewrite after the
+    // join.
+    const rightSelect = rightCols
+        .filter((d) => !leftCols.includes(d))
+        .map((d) => `${quoteIdentifier(rightTable.name)}.${quoteIdentifier(d)}`)
+        .join(", ");
+    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
+        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength, deduplicate);
+    await queryDB(leftTable, sql, queryOptions);
 }
-function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength) {
+async function shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions) {
+    const left = quoteIdentifier(leftTable.name);
+    const right = quoteIdentifier(rightTable.name);
+    const counts = (await queryDB(leftTable, `SELECT (SELECT COUNT(*) FROM ${left}) AS left_count,
+      (SELECT COUNT(*) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    const leftCount = Number(counts.left_count);
+    const rightCount = Number(counts.right_count);
+    const comparisons = leftCount * rightCount;
+    // Avoid distinct scans for small joins and short lookup tables. For larger
+    // joins, require at least a twofold reduction to pay for grouping and rejoining.
+    if (comparisons < 100_000 || Math.min(leftCount, rightCount) < 32) {
+        return false;
+    }
+    const distinct = (await queryDB(leftTable, `SELECT
+      (SELECT COUNT(DISTINCT ENCODE(${quoteIdentifier(leftColumn)})) FROM ${left}) AS left_count,
+      (SELECT COUNT(DISTINCT ENCODE(${quoteIdentifier(rightColumn)})) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    return Number(distinct.left_count) * Number(distinct.right_count) <=
+        comparisons / 2;
+}
+function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength, deduplicate = false) {
     const leftExpression = `${quoteIdentifier(leftTable)}.${quoteIdentifier(leftColumn)}`;
     const rightExpression = `${quoteIdentifier(rightTable)}.${quoteIdentifier(rightColumn)}`;
     const { scoreExpression, condition } = buildFuzzyMatchSql(leftExpression, rightExpression, method, threshold, { decimals: 2, prefilterPrefixLength });
+    let cte = "";
+    let from = `FROM ${quoteIdentifier(leftTable)} LEFT JOIN ${quoteIdentifier(rightTable)} ON ${condition}`;
+    let score = scoreExpression;
+    if (deduplicate) {
+        const matches = quoteIdentifier(`__sda_fuzzy_matches_${crypto.randomUUID().replaceAll("-", "")}`);
+        const match = buildFuzzyMatchSql("l.value", "r.value", method, threshold, { decimals: 2, prefilterPrefixLength });
+        // Keep the original value's collation for prefix matching, but use byte
+        // keys for deduplication and expansion: fuzzy scores distinguish case even
+        // when an input column has a case-insensitive collation.
+        cte = `WITH ${matches} AS MATERIALIZED (
+      SELECT l.key AS left_key, r.key AS right_key,
+        ${match.scoreExpression} AS score
+      FROM (
+        SELECT DISTINCT ${quoteIdentifier(leftColumn)} AS value,
+          ENCODE(${quoteIdentifier(leftColumn)}) AS key
+        FROM ${quoteIdentifier(leftTable)}
+        WHERE ${quoteIdentifier(leftColumn)} IS NOT NULL
+      ) l
+      INNER JOIN (
+        SELECT DISTINCT ${quoteIdentifier(rightColumn)} AS value,
+          ENCODE(${quoteIdentifier(rightColumn)}) AS key
+        FROM ${quoteIdentifier(rightTable)}
+        WHERE ${quoteIdentifier(rightColumn)} IS NOT NULL
+      ) r ON ${match.condition}
+    )`;
+        from = `FROM ${quoteIdentifier(leftTable)}
+      LEFT JOIN ${matches} ON ENCODE(${leftExpression}) = ${matches}.left_key
+      LEFT JOIN ${quoteIdentifier(rightTable)}
+        ON ENCODE(${rightExpression}) = ${matches}.right_key`;
+        score = `${matches}.score`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable)} AS
+${cte}
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS ${quoteIdentifier(similarityColumn)}
 FROM (
-  SELECT ${quoteIdentifier(leftTable)}.*, ${rightSelect}, ${scoreExpression} AS "_sda_score"
-  FROM ${quoteIdentifier(leftTable)} LEFT JOIN ${quoteIdentifier(rightTable)} ON ${condition}
+  SELECT ${quoteIdentifier(leftTable)}.*, ${rightSelect}, ${score} AS "_sda_score"
+  ${from}
 ) _sda
 ORDER BY ${quoteIdentifier(leftColumn)}, "_sda_score" DESC;\n`;
     }
     return `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable)} AS
+${cte}
 SELECT *
 FROM (
   SELECT ${quoteIdentifier(leftTable)}.*, ${rightSelect}
-  FROM ${quoteIdentifier(leftTable)} LEFT JOIN ${quoteIdentifier(rightTable)} ON ${condition}
+  ${from}
 ) _sda
 ORDER BY ${quoteIdentifier(leftColumn)}, ${quoteIdentifier(rightColumn)};\n`;
 }

package/esm/methods/getTypes.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function getTypes(simpleTable: SimpleTable): Promise<{
-    [key: string]: string;
+    [k: string]: string;
 }>;
 //# sourceMappingURL=getTypes.d.ts.map

package/esm/methods/hdbscan.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["hdbscan"]>[2]>;
+export default function hdbscan(table: SimpleTable, columns: string | string[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=hdbscan.d.ts.map

package/esm/methods/hdbscan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hdbscan.d.ts","sourceRoot":"","sources":["../../src/methods/hdbscan.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElE,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CASN"}