@nshiab/simple-data-analysis-core 2.0.8 → 2.1.0

package/script/methods/fuzzyJoin.js

@@ -54,17 +54,7 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
     }
     const method = options.method ?? "ratio";
     const similarityColumn = options.similarityColumn;
-    // The right table's copy of a column shared with the left table (only
-    // rightColumn can be shared, checked above) is excluded from the SELECT
-    // directly, instead of dropping its _1 duplicate with a rewrite after the
-    // join.
-    const rightSelect = rightCols
-        .filter((d) => !leftCols.includes(d))
-        .map((d) => `${(0, quoteIdentifier_js_1.default)(rightTable.name)}.${(0, quoteIdentifier_js_1.default)(d)}`)
-        .join(", ");
-    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength);
-    await (0, queryDB_js_1.default)(leftTable, sql, (0, mergeOptions_js_1.default)(leftTable, {
+    const queryOptions = (0, mergeOptions_js_1.default)(leftTable, {
         table: outputTable.name,
         method: "fuzzyJoin()",
         parameters: {
@@ -74,26 +64,90 @@ async function executeFuzzyJoin(leftTable, rightTable, outputTable, leftColumn,
             threshold,
             options,
         },
-    }));
+    });
+    const deduplicate = await shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions);
+    // The right table's copy of a column shared with the left table (only
+    // rightColumn can be shared, checked above) is excluded from the SELECT
+    // directly, instead of dropping its _1 duplicate with a rewrite after the
+    // join.
+    const rightSelect = rightCols
+        .filter((d) => !leftCols.includes(d))
+        .map((d) => `${(0, quoteIdentifier_js_1.default)(rightTable.name)}.${(0, quoteIdentifier_js_1.default)(d)}`)
+        .join(", ");
+    const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
+        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTable.name, similarityColumn, rightSelect, options.prefilterPrefixLength, deduplicate);
+    await (0, queryDB_js_1.default)(leftTable, sql, queryOptions);
 }
-function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength) {
+async function shouldDeduplicateFuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, queryOptions) {
+    const left = (0, quoteIdentifier_js_1.default)(leftTable.name);
+    const right = (0, quoteIdentifier_js_1.default)(rightTable.name);
+    const counts = (await (0, queryDB_js_1.default)(leftTable, `SELECT (SELECT COUNT(*) FROM ${left}) AS left_count,
+      (SELECT COUNT(*) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    const leftCount = Number(counts.left_count);
+    const rightCount = Number(counts.right_count);
+    const comparisons = leftCount * rightCount;
+    // Avoid distinct scans for small joins and short lookup tables. For larger
+    // joins, require at least a twofold reduction to pay for grouping and rejoining.
+    if (comparisons < 100_000 || Math.min(leftCount, rightCount) < 32) {
+        return false;
+    }
+    const distinct = (await (0, queryDB_js_1.default)(leftTable, `SELECT
+      (SELECT COUNT(DISTINCT ENCODE(${(0, quoteIdentifier_js_1.default)(leftColumn)})) FROM ${left}) AS left_count,
+      (SELECT COUNT(DISTINCT ENCODE(${(0, quoteIdentifier_js_1.default)(rightColumn)})) FROM ${right}) AS right_count`, { ...queryOptions, returnData: true }))[0];
+    return Number(distinct.left_count) * Number(distinct.right_count) <=
+        comparisons / 2;
+}
+function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, rightSelect, prefilterPrefixLength, deduplicate = false) {
     const leftExpression = `${(0, quoteIdentifier_js_1.default)(leftTable)}.${(0, quoteIdentifier_js_1.default)(leftColumn)}`;
     const rightExpression = `${(0, quoteIdentifier_js_1.default)(rightTable)}.${(0, quoteIdentifier_js_1.default)(rightColumn)}`;
     const { scoreExpression, condition } = (0, buildFuzzyMatchSql_js_1.default)(leftExpression, rightExpression, method, threshold, { decimals: 2, prefilterPrefixLength });
+    let cte = "";
+    let from = `FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)} ON ${condition}`;
+    let score = scoreExpression;
+    if (deduplicate) {
+        const matches = (0, quoteIdentifier_js_1.default)(`__sda_fuzzy_matches_${crypto.randomUUID().replaceAll("-", "")}`);
+        const match = (0, buildFuzzyMatchSql_js_1.default)("l.value", "r.value", method, threshold, { decimals: 2, prefilterPrefixLength });
+        // Keep the original value's collation for prefix matching, but use byte
+        // keys for deduplication and expansion: fuzzy scores distinguish case even
+        // when an input column has a case-insensitive collation.
+        cte = `WITH ${matches} AS MATERIALIZED (
+      SELECT l.key AS left_key, r.key AS right_key,
+        ${match.scoreExpression} AS score
+      FROM (
+        SELECT DISTINCT ${(0, quoteIdentifier_js_1.default)(leftColumn)} AS value,
+          ENCODE(${(0, quoteIdentifier_js_1.default)(leftColumn)}) AS key
+        FROM ${(0, quoteIdentifier_js_1.default)(leftTable)}
+        WHERE ${(0, quoteIdentifier_js_1.default)(leftColumn)} IS NOT NULL
+      ) l
+      INNER JOIN (
+        SELECT DISTINCT ${(0, quoteIdentifier_js_1.default)(rightColumn)} AS value,
+          ENCODE(${(0, quoteIdentifier_js_1.default)(rightColumn)}) AS key
+        FROM ${(0, quoteIdentifier_js_1.default)(rightTable)}
+        WHERE ${(0, quoteIdentifier_js_1.default)(rightColumn)} IS NOT NULL
+      ) r ON ${match.condition}
+    )`;
+        from = `FROM ${(0, quoteIdentifier_js_1.default)(leftTable)}
+      LEFT JOIN ${matches} ON ENCODE(${leftExpression}) = ${matches}.left_key
+      LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)}
+        ON ENCODE(${rightExpression}) = ${matches}.right_key`;
+        score = `${matches}.score`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable)} AS
+${cte}
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS ${(0, quoteIdentifier_js_1.default)(similarityColumn)}
 FROM (
-  SELECT ${(0, quoteIdentifier_js_1.default)(leftTable)}.*, ${rightSelect}, ${scoreExpression} AS "_sda_score"
-  FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)} ON ${condition}
+  SELECT ${(0, quoteIdentifier_js_1.default)(leftTable)}.*, ${rightSelect}, ${score} AS "_sda_score"
+  ${from}
 ) _sda
 ORDER BY ${(0, quoteIdentifier_js_1.default)(leftColumn)}, "_sda_score" DESC;\n`;
     }
     return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(outputTable)} AS
+${cte}
 SELECT *
 FROM (
   SELECT ${(0, quoteIdentifier_js_1.default)(leftTable)}.*, ${rightSelect}
-  FROM ${(0, quoteIdentifier_js_1.default)(leftTable)} LEFT JOIN ${(0, quoteIdentifier_js_1.default)(rightTable)} ON ${condition}
+  ${from}
 ) _sda
 ORDER BY ${(0, quoteIdentifier_js_1.default)(leftColumn)}, ${(0, quoteIdentifier_js_1.default)(rightColumn)};\n`;
 }

package/script/methods/getTypes.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function getTypes(simpleTable: SimpleTable): Promise<{
-    [key: string]: string;
+    [k: string]: string;
 }>;
 //# sourceMappingURL=getTypes.d.ts.map

package/script/methods/hdbscan.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["hdbscan"]>[2]>;
+export default function hdbscan(table: SimpleTable, columns: string | string[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=hdbscan.d.ts.map

package/script/methods/hdbscan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hdbscan.d.ts","sourceRoot":"","sources":["../../src/methods/hdbscan.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAWvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElE,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CASN"}

package/script/methods/hdbscan.js

@@ -0,0 +1,214 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = hdbscan;
+const node_api_1 = require("@duckdb/node-api");
+const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
+const buildApproximateMutualReachabilityMst_js_1 = __importDefault(require("../helpers/buildApproximateMutualReachabilityMst.js"));
+const buildExactMutualReachabilityMst_js_1 = __importDefault(require("../helpers/buildExactMutualReachabilityMst.js"));
+const clusterHdbscan_js_1 = __importDefault(require("../helpers/clusterHdbscan.js"));
+const foldIdentifier_js_1 = __importDefault(require("../helpers/foldIdentifier.js"));
+const prepareNumericFeatures_js_1 = __importDefault(require("../helpers/prepareNumericFeatures.js"));
+const publishPreparedColumns_js_1 = __importDefault(require("../helpers/publishPreparedColumns.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+function hdbscan(table, columns, newColumn, options = {}) {
+    const selected = typeof columns === "string" ? columns : [...columns];
+    const settings = structuredClone(options);
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "hdbscan()",
+        parameters: { columns: selected, newColumn, options: settings },
+        execute: () => execute(table, selected, newColumn, settings),
+    });
+}
+async function execute(table, columns, newColumn, options) {
+    const { minClusterSize = 5, metric = "euclidean", allowSingleCluster = false, approximate = false, membershipScoreColumn, outlierScoreColumn, } = options;
+    const minSamples = options.minSamples ?? minClusterSize;
+    if (typeof newColumn !== "string") {
+        throw new Error("hdbscan() newColumn must be a string.");
+    }
+    if (!Number.isSafeInteger(minClusterSize) || minClusterSize < 2) {
+        throw new Error("hdbscan() minClusterSize must be a safe integer of at least 2.");
+    }
+    if (!Number.isSafeInteger(minSamples) || minSamples < 1) {
+        throw new Error("hdbscan() minSamples must be a safe integer of at least 1.");
+    }
+    if (!["euclidean", "cosine"].includes(metric)) {
+        throw new Error('hdbscan() metric must be "euclidean" or "cosine".');
+    }
+    if (typeof allowSingleCluster !== "boolean") {
+        throw new Error("hdbscan() allowSingleCluster must be a boolean.");
+    }
+    if (typeof approximate !== "boolean") {
+        throw new Error("hdbscan() approximate must be a boolean.");
+    }
+    if (membershipScoreColumn !== undefined &&
+        typeof membershipScoreColumn !== "string") {
+        throw new Error("hdbscan() membershipScoreColumn must be a string.");
+    }
+    if (outlierScoreColumn !== undefined &&
+        typeof outlierScoreColumn !== "string") {
+        throw new Error("hdbscan() outlierScoreColumn must be a string.");
+    }
+    const sourceColumns = Object.keys(await table.getTypes());
+    const outputNames = [newColumn, membershipScoreColumn, outlierScoreColumn]
+        .filter((name) => name !== undefined);
+    validateOutputNames(sourceColumns, outputNames);
+    const prepared = await (0, prepareNumericFeatures_js_1.default)(table, typeof columns === "string"
+        ? { kind: "vector", column: columns }
+        : { kind: "scalars", columns }, { method: "hdbscan()" });
+    const connection = table.connection;
+    const suffix = crypto.randomUUID().replaceAll("-", "");
+    const name = (part) => `__sda_hdbscan_${suffix}_${part}`;
+    const names = {
+        rows: name("rows"),
+        neighbors: name("neighbors"),
+        search: name("search"),
+        hnsw: name("hnsw"),
+        candidates: name("candidates"),
+        coreDistances: name("core_distances"),
+        frontier: name("frontier"),
+        edges: name("edges"),
+        components: name("components"),
+        representatives: name("representatives"),
+        componentPairs: name("component_pairs"),
+        bridgeCandidates: name("bridge_candidates"),
+        mst: name("mst"),
+        result: name("result"),
+    };
+    try {
+        if (prepared.rowCount < 2) {
+            throw new Error(`hdbscan() requires at least 2 rows, but the input has ${prepared.rowCount}.`);
+        }
+        if (prepared.rowCount > 0x40000000) {
+            throw new Error("hdbscan() supports at most 1073741824 rows because its 2*n-1 hierarchy nodes use signed INTEGER identifiers.");
+        }
+        if (minSamples >= prepared.rowCount) {
+            const defaulted = options.minSamples === undefined
+                ? " (defaulted from minClusterSize)"
+                : "";
+            throw new Error(`hdbscan() minSamples is ${minSamples}${defaulted}, but ${prepared.rowCount} rows provide at most ${prepared.rowCount - 1} other point${prepared.rowCount === 2 ? "" : "s"}. Set minSamples between 1 and ${prepared.rowCount - 1}, or provide at least ${minSamples + 1} rows.`);
+        }
+        if (minClusterSize > prepared.rowCount) {
+            throw new Error(`hdbscan() minClusterSize is ${minClusterSize}, but the input has ${prepared.rowCount} rows. Set minClusterSize between 2 and ${prepared.rowCount}, or provide more rows.`);
+        }
+        const q = quoteIdentifier_js_1.default;
+        await connection.run(`CREATE TEMP TABLE ${q(names.rows)} AS
+       SELECT ${q(prepared.rowIdColumn)}::INTEGER AS vertex,
+         ${q(prepared.vectorColumn)} AS vec
+       FROM ${q(prepared.relation)}
+       ORDER BY ${q(prepared.rowIdColumn)}`);
+        if (approximate) {
+            await (0, buildApproximateMutualReachabilityMst_js_1.default)(connection, {
+                count: prepared.rowCount,
+                dimensions: prepared.dimensions,
+                minSamples,
+            }, { metric }, {
+                rows: names.rows,
+                neighbors: names.neighbors,
+                search: names.search,
+                hnsw: names.hnsw,
+                candidates: names.candidates,
+                coreDistances: names.coreDistances,
+                edges: names.edges,
+                components: names.components,
+                representatives: names.representatives,
+                componentPairs: names.componentPairs,
+                bridgeCandidates: names.bridgeCandidates,
+                mst: names.mst,
+            });
+        }
+        else {
+            await (0, buildExactMutualReachabilityMst_js_1.default)(connection, { count: prepared.rowCount, minSamples }, { metric }, {
+                rows: q(names.rows),
+                coreDistances: q(names.coreDistances),
+                frontier: q(names.frontier),
+                mst: q(names.mst),
+            });
+        }
+        const result = await (0, clusterHdbscan_js_1.default)(connection, q(names.mst), {
+            count: prepared.rowCount,
+            minClusterSize,
+            allowSingleCluster,
+        });
+        const labelColumn = "label";
+        const membershipColumn = "membership";
+        const gloshColumn = "glosh";
+        const resultColumns = [
+            "row_id INTEGER",
+            `${labelColumn} VARCHAR`,
+            membershipScoreColumn === undefined
+                ? undefined
+                : `${membershipColumn} DOUBLE`,
+            outlierScoreColumn === undefined ? undefined : `${gloshColumn} DOUBLE`,
+        ].filter((definition) => definition !== undefined);
+        await connection.run(`CREATE TEMP TABLE ${q(names.result)} (${resultColumns.join(",")})`);
+        const outputTypes = [
+            node_api_1.INTEGER,
+            node_api_1.VARCHAR,
+            ...(membershipScoreColumn === undefined ? [] : [node_api_1.DOUBLE]),
+            ...(outlierScoreColumn === undefined ? [] : [node_api_1.DOUBLE]),
+        ];
+        await (0, appendColumnBatches_js_1.default)(connection, names.result, outputTypes, prepared.rowCount, (column, start, end) => {
+            if (column === 0) {
+                return Array.from({ length: end - start }, (_, offset) => start + offset);
+            }
+            if (column === 1) {
+                return Array.from(result.labels.subarray(start, end), (label) => label < 0 ? "noise" : `cluster-${label}`);
+            }
+            const membershipIndex = membershipScoreColumn === undefined ? -1 : 2;
+            return Array.from(column === membershipIndex
+                ? result.probabilities.subarray(start, end)
+                : result.outlierScores.subarray(start, end));
+        });
+        const publicationOutputs = [{
+                name: newColumn,
+                expression: `r.${q(labelColumn)}`,
+            }];
+        if (membershipScoreColumn !== undefined) {
+            publicationOutputs.push({
+                name: membershipScoreColumn,
+                expression: `r.${q(membershipColumn)}`,
+            });
+        }
+        if (outlierScoreColumn !== undefined) {
+            publicationOutputs.push({
+                name: outlierScoreColumn,
+                expression: `r.${q(gloshColumn)}`,
+            });
+        }
+        await (0, publishPreparedColumns_js_1.default)(table, prepared, {
+            method: "hdbscan()",
+            parameters: { columns, newColumn, options },
+            result: { relation: names.result, rowIdColumn: "row_id" },
+            outputs: publicationOutputs,
+        });
+    }
+    finally {
+        for (const relation of Object.entries(names).reverse().flatMap(([key, value]) => key === "hnsw" ? [] : [value])) {
+            await connection.run(`DROP TABLE IF EXISTS ${(0, quoteIdentifier_js_1.default)(relation)}`);
+        }
+        await prepared.cleanup();
+    }
+}
+function validateOutputNames(sourceColumns, outputNames) {
+    const seen = new Map();
+    for (const name of outputNames) {
+        if (name.length === 0 || name.includes("\0")) {
+            throw new Error("hdbscan() output column names must be non-empty strings without null characters.");
+        }
+        const folded = (0, foldIdentifier_js_1.default)(name);
+        const source = sourceColumns.find((column) => (0, foldIdentifier_js_1.default)(column) === folded);
+        if (source !== undefined) {
+            throw new Error(`hdbscan() cannot create ${(0, quoteIdentifier_js_1.default)(name)} because that source column already exists. Remove it first or choose different output names.`);
+        }
+        const duplicate = seen.get(folded);
+        if (duplicate !== undefined) {
+            throw new Error(`hdbscan() output columns ${(0, quoteIdentifier_js_1.default)(duplicate)} and ${(0, quoteIdentifier_js_1.default)(name)} refer to the same column. Choose distinct output names.`);
+        }
+        seen.set(folded, name);
+    }
+}

package/script/methods/loadArray.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAQL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA4ExB"}
+{"version":3,"file":"loadArray.d.ts","sourceRoot":"","sources":["../../src/methods/loadArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAM1B,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,OAAO,GAAE,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAM,QAiBnE;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,wBAAgB,YAAY,CAC1B,IAAI,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,EAClC,WAAW,GAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAO,EAC3C,0BAA0B,UAAQ,GACjC,aAAa,CAsHf;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,aAAa,iBA+DxB"}

package/script/methods/loadArray.js

@@ -7,6 +7,7 @@ exports.default = loadArray;
 exports.prepareArray = prepareArray;
 exports.executePreparedArray = executePreparedArray;
 const node_api_1 = require("@duckdb/node-api");
+const appendColumnBatches_js_1 = __importDefault(require("../helpers/appendColumnBatches.js"));
 const prepareGeometry_js_1 = __importDefault(require("../helpers/prepareGeometry.js"));
 const prepareJSON_js_1 = __importDefault(require("../helpers/prepareJSON.js"));
 const geometryFromJSON_js_1 = __importDefault(require("../helpers/geometryFromJSON.js"));
@@ -144,25 +145,7 @@ async function executePreparedArray(simpleTable, prepared) {
     }
     try {
         await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${(0, quoteIdentifier_js_1.default)(staged)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${storageTypes[i]}`).join(", ")})`);
-        const appender = await simpleTable.connection
-            .createAppender(staged);
-        try {
-            const duckDBTypes = storageTypes.map((d) => (0, parseDuckDBType_js_1.default)(d));
-            // The maximum capacity of a DuckDB data chunk is 2048 rows.
-            const chunkSize = 2000;
-            for (let start = 0; start < rowCount; start += chunkSize) {
-                const end = Math.min(start + chunkSize, rowCount);
-                const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
-                for (let i = 0; i < keys.length; i++) {
-                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-                }
-                appender.appendDataChunk(dataChunk);
-            }
-            appender.flushSync();
-        }
-        finally {
-            appender.closeSync();
-        }
+        await (0, appendColumnBatches_js_1.default)(simpleTable.connection, staged, storageTypes.map((type) => (0, parseDuckDBType_js_1.default)(type)), rowCount, (column, start, end) => columnsData[column].slice(start, end));
         if (!needsConversion)
             return;
         // CTAS is atomic: conversion must finish before the destination is replaced.

package/script/methods/mahalanobis.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["mahalanobis"]>[3]>;
+export default function mahalanobis(table: SimpleTable, columns: string | string[], referencePoint: number[], newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=mahalanobis.d.ts.map

package/script/methods/mahalanobis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mahalanobis.d.ts","sourceRoot":"","sources":["../../src/methods/mahalanobis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AASvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEtE,MAAM,CAAC,OAAO,UAAU,WAAW,CACjC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,cAAc,EAAE,MAAM,EAAE,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CAyBN"}

package/script/methods/mahalanobis.js

@@ -0,0 +1,98 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = mahalanobis;
+const computeCovariance_js_1 = __importDefault(require("../helpers/computeCovariance.js"));
+const computeMahalanobisDistances_js_1 = __importDefault(require("../helpers/computeMahalanobisDistances.js"));
+const foldIdentifier_js_1 = __importDefault(require("../helpers/foldIdentifier.js"));
+const prepareNumericFeatures_js_1 = __importDefault(require("../helpers/prepareNumericFeatures.js"));
+const publishPreparedColumns_js_1 = __importDefault(require("../helpers/publishPreparedColumns.js"));
+const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+function mahalanobis(table, columns, referencePoint, newColumn, options = {}) {
+    if (!Array.isArray(referencePoint) || referencePoint.length === 0 ||
+        Array.from(referencePoint).some((value) => typeof value !== "number" || !Number.isFinite(value))) {
+        throw new Error("mahalanobis() referencePoint must be a nonempty array of finite numbers in feature-dimension order.");
+    }
+    const selected = typeof columns === "string" ? columns : [...columns];
+    const reference = [...referencePoint];
+    const settings = { ...options };
+    (0, queueOp_js_1.default)(table, {
+        kind: "barrier",
+        method: "mahalanobis()",
+        parameters: {
+            columns: selected,
+            referencePoint: reference,
+            newColumn,
+            options: settings,
+        },
+        execute: () => execute(table, selected, reference, newColumn, settings),
+    });
+}
+async function execute(table, columns, referencePoint, newColumn, options) {
+    const sourceColumns = Object.keys(await table.getTypes());
+    const outputNames = options.similarityScoreColumn === undefined
+        ? [newColumn]
+        : [newColumn, options.similarityScoreColumn];
+    const seen = new Set();
+    for (const name of outputNames) {
+        if (typeof name !== "string" || name.length === 0 || name.includes("\0")) {
+            throw new Error("mahalanobis() output column names must be nonempty strings without null characters.");
+        }
+        const folded = (0, foldIdentifier_js_1.default)(name);
+        if (seen.has(folded)) {
+            throw new Error("mahalanobis() distance and similarity score columns must have different names (case-insensitive).");
+        }
+        seen.add(folded);
+        if (sourceColumns.some((column) => (0, foldIdentifier_js_1.default)(column) === folded)) {
+            throw new Error(`mahalanobis() cannot create ${(0, quoteIdentifier_js_1.default)(name)} because that column already exists. Remove it first or choose a different name.`);
+        }
+    }
+    const prepared = await (0, prepareNumericFeatures_js_1.default)(table, typeof columns === "string"
+        ? { kind: "vector", column: columns }
+        : { kind: "scalars", columns }, { method: "mahalanobis()" });
+    try {
+        if (referencePoint.length !== prepared.dimensions) {
+            throw new Error(`mahalanobis() referencePoint must contain ${prepared.dimensions} values in feature-dimension order; received ${referencePoint.length}.`);
+        }
+        const model = await (0, computeCovariance_js_1.default)(table.connection, {
+            relation: prepared.relation,
+            rowIdColumn: prepared.rowIdColumn,
+            vectorColumn: prepared.vectorColumn,
+            observations: prepared.rowCount,
+            dimensions: prepared.dimensions,
+        });
+        const distances = await (0, computeMahalanobisDistances_js_1.default)(table.connection, {
+            relation: prepared.relation,
+            rowIdColumn: prepared.rowIdColumn,
+            vectorColumn: prepared.vectorColumn,
+        }, model, referencePoint);
+        try {
+            const distance = `r.${(0, quoteIdentifier_js_1.default)(distances.distanceColumn)}`;
+            const outputs = [{ name: newColumn, expression: distance }];
+            if (options.similarityScoreColumn !== undefined) {
+                outputs.push({
+                    name: options.similarityScoreColumn,
+                    expression: `CASE WHEN max(${distance}) OVER () = 0 THEN 1::DOUBLE ELSE 1 - ${distance} / max(${distance}) OVER () END`,
+                });
+            }
+            await (0, publishPreparedColumns_js_1.default)(table, prepared, {
+                method: "mahalanobis()",
+                parameters: { columns, referencePoint, newColumn, options },
+                result: {
+                    relation: distances.relation,
+                    rowIdColumn: distances.rowIdColumn,
+                },
+                outputs,
+            });
+        }
+        finally {
+            await distances.cleanup();
+        }
+    }
+    finally {
+        await prepared.cleanup();
+    }
+}

package/script/methods/neighbors.d.ts

@@ -0,0 +1,9 @@
+import { type GraphId } from "../helpers/prepareGraphStarts.js";
+import type SimpleTable from "../class/SimpleTable.js";
+type NeighborsOptions = {
+    direction?: "outgoing" | "incoming" | "both";
+    outputTable?: string | boolean;
+};
+export default function neighbors(simpleTable: SimpleTable, sourceColumn: string, targetColumn: string, startNodes: GraphId | GraphId[], options?: NeighborsOptions): SimpleTable;
+export {};
+//# sourceMappingURL=neighbors.d.ts.map

package/script/methods/neighbors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"neighbors.d.ts","sourceRoot":"","sources":["../../src/methods/neighbors.ts"],"names":[],"mappings":"AAAA,OAA2B,EACzB,KAAK,OAAO,EAEb,MAAM,kCAAkC,CAAC;AAO1C,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,KAAK,gBAAgB,GAAG;IACtB,SAAS,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IAC7C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,OAAO,GAAG,OAAO,EAAE,EAC/B,OAAO,GAAE,gBAAqB,GAC7B,WAAW,CAuEb"}

package/script/methods/neighbors.js

@@ -0,0 +1,120 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = neighbors;
+const prepareGraphStarts_js_1 = __importDefault(require("../helpers/prepareGraphStarts.js"));
+const prepareGraphTraversal_js_1 = __importStar(require("../helpers/prepareGraphTraversal.js"));
+const queueGraphResult_js_1 = __importDefault(require("../helpers/queueGraphResult.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+function neighbors(simpleTable, sourceColumn, targetColumn, startNodes, options = {}) {
+    if (typeof sourceColumn !== "string") {
+        throw new TypeError("neighbors() sourceColumn must be a string.");
+    }
+    if (typeof targetColumn !== "string") {
+        throw new TypeError("neighbors() targetColumn must be a string.");
+    }
+    if (options === null || typeof options !== "object" || Array.isArray(options)) {
+        throw new TypeError("neighbors() options must be an object.");
+    }
+    if (options.direction !== undefined &&
+        !["outgoing", "incoming", "both"].includes(options.direction)) {
+        throw new TypeError('neighbors() options.direction must be "outgoing", "incoming", or "both".');
+    }
+    if (options.outputTable !== undefined &&
+        typeof options.outputTable !== "string" &&
+        typeof options.outputTable !== "boolean") {
+        throw new TypeError("neighbors() options.outputTable must be a string or boolean.");
+    }
+    const preparedStarts = (0, prepareGraphStarts_js_1.default)(startNodes, "neighbors()", "startNodes");
+    options = structuredClone(options);
+    const direction = options.direction ?? "outgoing";
+    const parameters = {
+        sourceColumn,
+        targetColumn,
+        startNodes: structuredClone(startNodes),
+        options,
+    };
+    return (0, queueGraphResult_js_1.default)(simpleTable, {
+        method: "neighbors()",
+        parameters,
+        outputTable: options.outputTable,
+        values: (schema) => {
+            validateStarts(schema, sourceColumn, targetColumn, preparedStarts);
+            return preparedStarts.values;
+        },
+        buildSelect: (input, schema) => neighborsSelect(input, schema, sourceColumn, targetColumn, preparedStarts, direction),
+        outputSchema: (schema) => {
+            const endpoints = validateStarts(schema, sourceColumn, targetColumn, preparedStarts);
+            return { start: endpoints.idType, node: endpoints.idType };
+        },
+    });
+}
+function validateStarts(schema, source, target, starts) {
+    return (0, prepareGraphTraversal_js_1.validateGraphStarts)(schema, source, target, starts, "neighbors()", "startNodes");
+}
+function neighborsSelect(input, schema, source, target, starts, direction) {
+    const prepared = (0, prepareGraphTraversal_js_1.default)(input, schema, source, target, starts, "neighbors()", "startNodes");
+    const relations = prepared.relationNames([
+        "graph_starts",
+        "graph_edges",
+        "neighbor_matches",
+    ]);
+    const startsRelation = relations.graph_starts;
+    const edgesRelation = relations.graph_edges;
+    const matchesRelation = relations.neighbor_matches;
+    return `WITH ${startsRelation}(${(0, quoteIdentifier_js_1.default)("start")}) AS (
+      VALUES ${prepared.startValues}
+    ), ${edgesRelation} AS (
+      ${prepared.edges(direction)}
+    ), ${matchesRelation} AS (
+      SELECT ${(0, quoteIdentifier_js_1.default)("starts")}.${(0, quoteIdentifier_js_1.default)("start")},
+        ${(0, quoteIdentifier_js_1.default)("edges")}.${(0, quoteIdentifier_js_1.default)("__to")} AS ${(0, quoteIdentifier_js_1.default)("node")},
+        ${(0, quoteIdentifier_js_1.default)("edges")}.${(0, quoteIdentifier_js_1.default)("__from_key")} AS ${(0, quoteIdentifier_js_1.default)("__start_key")},
+        ${(0, quoteIdentifier_js_1.default)("edges")}.${(0, quoteIdentifier_js_1.default)("__to_key")} AS ${(0, quoteIdentifier_js_1.default)("__node_key")}
+      FROM ${edgesRelation} AS ${(0, quoteIdentifier_js_1.default)("edges")}
+      INNER JOIN ${startsRelation} AS ${(0, quoteIdentifier_js_1.default)("starts")}
+        ON ${(0, quoteIdentifier_js_1.default)("edges")}.${(0, quoteIdentifier_js_1.default)("__from_key")} =
+          ${prepared.key(`${(0, quoteIdentifier_js_1.default)("starts")}.${(0, quoteIdentifier_js_1.default)("start")}`)}
+    )
+    SELECT ${(0, quoteIdentifier_js_1.default)("start")}, ${(0, quoteIdentifier_js_1.default)("node")}
+    FROM (
+      SELECT DISTINCT ${(0, quoteIdentifier_js_1.default)("start")}, ${(0, quoteIdentifier_js_1.default)("node")}, ${(0, quoteIdentifier_js_1.default)("__start_key")}, ${(0, quoteIdentifier_js_1.default)("__node_key")}
+      FROM ${matchesRelation}
+    ) AS ${(0, quoteIdentifier_js_1.default)("distinct_neighbors")}
+    ORDER BY ${(0, quoteIdentifier_js_1.default)("__start_key")}, ${(0, quoteIdentifier_js_1.default)("__node_key")}`;
+}

package/script/methods/normalizeVector.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["normalizeVector"]>[2]>;
+export default function normalizeVector(table: SimpleTable, column: string, newColumn: string, options?: Options): void;
+export {};
+//# sourceMappingURL=normalizeVector.d.ts.map

package/script/methods/normalizeVector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalizeVector.d.ts","sourceRoot":"","sources":["../../src/methods/normalizeVector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAOvD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE1E,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAY,GACpB,IAAI,CAiBN"}