@nshiab/simple-data-analysis-core 1.0.0 → 2.0.0

package/esm/methods/distance.js

@@ -0,0 +1,48 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import queueOp from "../helpers/queueOp.js";
+export default function distance(simpleTable, column1, column2, newColumn, options = {}) {
+    // Building the expression doesn't need the database, so invalid options
+    // throw at call time.
+    const expression = distanceExpression(column1, column2, options);
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "distance()",
+        parameters: { column1, column2, newColumn },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            assertNewColumns(types, [newColumn], "distance()");
+            return `SELECT *, CAST(${expression} AS DOUBLE) AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+        },
+    });
+}
+function distanceExpression(column1, column2, options = {}) {
+    options = structuredClone(options);
+    options.method = options.method ?? "srs";
+    if (options.method === "srs" && typeof options.unit === "string") {
+        throw new Error("Using the SRS unit. You can't specify options.unit unless you set options.method to 'spheroid' or 'haversine'.");
+    }
+    else if (["spheroid", "haversine"].includes(options.method)) {
+        options.unit = options.unit ?? "m";
+        if (!["m", "km"].includes(options.unit)) {
+            throw new Error(`Unknown unit ${options.unit}. Choose between 'm' and 'km'.`);
+        }
+    }
+    let expression = "";
+    if (options.method === "srs") {
+        expression = `ST_Distance(${quoteIdentifier(column1)}, ${quoteIdentifier(column2)})`;
+    }
+    else if (options.method === "haversine") {
+        expression = `ST_Distance_Sphere(${quoteIdentifier(column1)}, ${quoteIdentifier(column2)}) ${options.unit === "km" ? "/ 1000" : ""}`;
+    }
+    else if (options.method === "spheroid") {
+        expression = `ST_Distance_Spheroid(${quoteIdentifier(column1)}::GEOMETRY, ${quoteIdentifier(column2)}::GEOMETRY) ${options.unit === "km" ? "/ 1000" : ""}`;
+    }
+    else {
+        throw new Error(`distance() received an unknown method: ${options.method}. Choose "srs", "haversine", or "spheroid".`);
+    }
+    return typeof options.decimals === "number"
+        ? `ROUND(${expression}, ${options.decimals})`
+        : expression;
+}

package/esm/methods/extractDatePart.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type DatePart = "year" | "quarter" | "month" | "week" | "day" | "dayOfWeek" | "dayOfYear" | "hour" | "minute" | "second";
+export default function extractDatePart(simpleTable: SimpleTable, column: string, parts: DatePart | {
+    [newColumn: string]: DatePart;
+}): void;
+export {};
+//# sourceMappingURL=extractDatePart.d.ts.map

package/esm/methods/extractDatePart.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractDatePart.d.ts","sourceRoot":"","sources":["../../src/methods/extractDatePart.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,KAAK,QAAQ,GACT,MAAM,GACN,SAAS,GACT,OAAO,GACP,MAAM,GACN,KAAK,GACL,WAAW,GACX,WAAW,GACX,MAAM,GACN,QAAQ,GACR,QAAQ,CAAC;AAeb,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,QAAQ,GAAG;IAAE,CAAC,SAAS,EAAE,MAAM,GAAG,QAAQ,CAAA;CAAE,QAiCpD"}

package/esm/methods/extractDatePart.js

@@ -0,0 +1,38 @@
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import queueOp from "../helpers/queueOp.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+const sqlParts = {
+    year: "year",
+    quarter: "quarter",
+    month: "month",
+    week: "week",
+    day: "day",
+    dayOfWeek: "dayofweek",
+    dayOfYear: "dayofyear",
+    hour: "hour",
+    minute: "minute",
+    second: "second",
+};
+export default function extractDatePart(simpleTable, column, parts) {
+    const capturedParts = typeof parts === "string"
+        ? parts
+        : structuredClone(parts);
+    const entries = typeof capturedParts === "string"
+        ? [[capturedParts, capturedParts]]
+        : Object.entries(capturedParts);
+    if (entries.length === 0) {
+        throw new Error("extractDatePart() requires at least one date part.");
+    }
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "extractDatePart()",
+        parameters: { column, parts: capturedParts },
+        needsSchema: true,
+        buildSelect: (input, schema) => {
+            assertNewColumns(schema, entries.map(([newColumn]) => newColumn), "extractDatePart()");
+            const source = quoteIdentifier(column);
+            const extractions = entries.map(([newColumn, part]) => `date_part('${sqlParts[part]}', ${source}) AS ${quoteIdentifier(newColumn)}`);
+            return `SELECT *, ${extractions.join(", ")} FROM ${input}`;
+        },
+    });
+}

package/esm/methods/extractLatLon.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function extractLatLon(simpleTable: SimpleTable, column: string, latColumn: string, lonColumn: string): void;
+//# sourceMappingURL=extractLatLon.d.ts.map

package/esm/methods/extractLatLon.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractLatLon.d.ts","sourceRoot":"","sources":["../../src/methods/extractLatLon.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,QAiBlB"}

package/esm/methods/extractLatLon.js

@@ -0,0 +1,16 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import assertNewColumns from "../helpers/assertNewColumns.js";
+import queueOp from "../helpers/queueOp.js";
+export default function extractLatLon(simpleTable, column, latColumn, lonColumn) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "extractLatLon()",
+        parameters: { column, latColumn, lonColumn },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            assertNewColumns(types, [latColumn, lonColumn], "extractLatLon()");
+            return `SELECT *, CAST(ST_Y(${quoteIdentifier(column)}) AS DOUBLE) AS ${quoteIdentifier(latColumn)}, CAST(ST_X(${quoteIdentifier(column)}) AS DOUBLE) AS ${quoteIdentifier(lonColumn)} FROM ${input}`;
+        },
+    });
+}

package/esm/methods/fill.d.ts

@@ -1,7 +1,7 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function fill(simpleTable: SimpleTable, columns: string | string[], options?: {
-    categories?: string | string[];
+    by?: string | string[];
     interpolate?: boolean;
     interpolateBy?: string;
-}): Promise<void>;
+}): void;
 //# sourceMappingURL=fill.d.ts.map

package/esm/methods/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/methods/fill.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,iBAmFP"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/methods/fill.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,QAmBP"}

package/esm/methods/fill.js

@@ -1,60 +1,57 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
+import queueOp from "../helpers/queueOp.js";
 import stringToArray from "../helpers/stringToArray.js";
-export default async function fill(simpleTable, columns, options = {}) {
-    const categories = options.categories
-        ? stringToArray(options.categories)
-        : [];
-    const tempRowCol = `rowNumberForFill`;
+export default function fill(simpleTable, columns, options = {}) {
+    // This validation doesn't need the database, so it stays at call time.
     if (options.interpolateBy && options.interpolate === false) {
         throw new Error(`interpolate cannot be false when interpolateBy is set.`);
     }
+    // The fill order is based on rowid, which only exists on the materialized
+    // table: it executes as a barrier.
+    columns = Array.isArray(columns) ? [...columns] : columns;
+    options = structuredClone(options);
+    queueOp(simpleTable, {
+        kind: "barrier",
+        method: "fill()",
+        parameters: { columns, options },
+        execute: () => executeFill(simpleTable, columns, options),
+    });
+}
+async function executeFill(simpleTable, columns, options) {
+    const by = options.by ? stringToArray(options.by) : [];
+    const cols = stringToArray(columns);
+    const excludeList = cols.map((col) => `${quoteIdentifier(col)}`).join(", ");
+    let selectList;
     if (options.interpolate || options.interpolateBy) {
-        const cols = stringToArray(columns);
-        // Always add temp row number to preserve original row order in output
-        await simpleTable.addRowNumber(tempRowCol);
-        // Use interpolateBy for the window function's ORDER BY (correct interpolation math),
-        // but always order final output by tempRowCol to preserve input row order
-        const windowOrderCol = options.interpolateBy ?? tempRowCol;
-        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
-        const overClause = categories.length > 0
-            ? `(PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")})`
+        // Use interpolateBy for the window function's ORDER BY (correct
+        // interpolation math), but always order the final output by rowid to
+        // preserve input row order.
+        const windowOrder = options.interpolateBy
+            ? `${quoteIdentifier(options.interpolateBy)}`
+            : "rowid";
+        const overClause = by.length > 0
+            ? `(PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")})`
             : `()`;
-        const selectList = cols
-            .map((col) => `fill(${col} ORDER BY "${windowOrderCol}") OVER ${overClause} as ${col}`)
+        selectList = cols
+            .map((col) => `fill(${quoteIdentifier(col)} ORDER BY ${windowOrder}) OVER ${overClause} as ${quoteIdentifier(col)}`)
             .join(", ");
-        await queryDB(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, mergeOptions(simpleTable, {
-            table: simpleTable.name,
-            method: "fill()",
-            parameters: { columns, ...options },
-        }));
     }
-    else if (categories.length > 0) {
-        const partition = `PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")}`;
-        await simpleTable.addRowNumber(tempRowCol);
-        const cols = stringToArray(columns);
-        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
-        const selectList = cols
-            .map((col) => `COALESCE(${col}, LAG(${col} IGNORE NULLS) OVER(${partition} ORDER BY "${tempRowCol}")) as ${col}`)
+    else if (by.length > 0) {
+        const partition = `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}`;
+        selectList = cols
+            .map((col) => `COALESCE(${quoteIdentifier(col)}, LAG(${quoteIdentifier(col)} IGNORE NULLS) OVER(${partition} ORDER BY rowid)) as ${quoteIdentifier(col)}`)
             .join(", ");
-        await queryDB(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, mergeOptions(simpleTable, {
-            table: simpleTable.name,
-            method: "fill()",
-            parameters: { columns, ...options },
-        }));
     }
     else {
-        // Simple path: add temp row number and order by it to preserve input row order
-        await simpleTable.addRowNumber(tempRowCol);
-        const cols = stringToArray(columns);
-        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
-        const selectList = cols
-            .map((col) => `COALESCE(${col}, LAG(${col} IGNORE NULLS) OVER()) as ${col}`)
+        selectList = cols
+            .map((col) => `COALESCE(${quoteIdentifier(col)}, LAG(${quoteIdentifier(col)} IGNORE NULLS) OVER(ORDER BY rowid)) as ${quoteIdentifier(col)}`)
             .join(", ");
-        await queryDB(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, mergeOptions(simpleTable, {
-            table: simpleTable.name,
-            method: "fill()",
-            parameters: { columns, ...options },
-        }));
     }
+    await queryDB(simpleTable, `CREATE OR REPLACE TABLE ${quoteIdentifier(simpleTable.name)} AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM ${quoteIdentifier(simpleTable.name)} ORDER BY rowid;`, mergeOptions(simpleTable, {
+        table: simpleTable.name,
+        method: "fill()",
+        parameters: { columns, ...options },
+    }));
 }

package/esm/methods/fillHoles.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function fillHoles(simpleTable: SimpleTable, column?: string): void;
+//# sourceMappingURL=fillHoles.d.ts.map

package/esm/methods/fillHoles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fillHoles.d.ts","sourceRoot":"","sources":["../../src/methods/fillHoles.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,WAAW,EAAE,WAAW,EACxB,MAAM,CAAC,EAAE,MAAM,QAkBhB"}

package/esm/methods/fillHoles.js

@@ -0,0 +1,19 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.js";
+import queueOp from "../helpers/queueOp.js";
+export default function fillHoles(simpleTable, column) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "fillHoles()",
+        parameters: { column },
+        needsSchema: true,
+        needsSpatial: true,
+        buildSelect: (input, types) => {
+            const col = column ??
+                findGeoColumnFromSchema(types, "fillHoles()", simpleTable.name);
+            // Like the previous UPDATE-based implementation, the result is always
+            // stored in the "geom" column, and the assignment cast keeps its type.
+            return `SELECT * REPLACE (ST_MakePolygon(ST_ExteriorRing(${quoteIdentifier(col)}))${types["geom"] ? `::${types["geom"]}` : ""} AS "geom") FROM ${input}`;
+        },
+    });
+}

package/esm/methods/filter.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function filter(simpleTable: SimpleTable, conditions: string): void;
+//# sourceMappingURL=filter.d.ts.map

package/esm/methods/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../src/methods/filter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,MAAM,CAAC,OAAO,UAAU,MAAM,CAC5B,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,QAcnB"}

package/esm/methods/filter.js

@@ -0,0 +1,14 @@
+import queueOp from "../helpers/queueOp.js";
+export default function filter(simpleTable, conditions) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "filter()",
+        parameters: { conditions },
+        needsSchema: false,
+        rawSQL: [conditions],
+        preservesSchema: true,
+        buildSelect: (input) => `SELECT *
+        FROM ${input}
+        WHERE ${conditions}`,
+    });
+}

package/esm/methods/firstChars.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function firstChars(simpleTable: SimpleTable, column: string, count: number): void;
+//# sourceMappingURL=firstChars.d.ts.map

package/esm/methods/firstChars.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"firstChars.d.ts","sourceRoot":"","sources":["../../src/methods/firstChars.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,QAYd"}

package/esm/methods/firstChars.js

@@ -0,0 +1,11 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import queueOp from "../helpers/queueOp.js";
+export default function firstChars(simpleTable, column, count) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "firstChars()",
+        parameters: { column, count },
+        needsSchema: false,
+        buildSelect: (input) => `SELECT * REPLACE (LEFT(${quoteIdentifier(column)}, ${count}) AS ${quoteIdentifier(column)}) FROM ${input}`,
+    });
+}

package/esm/methods/fixGeo.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function fixGeo(simpleTable: SimpleTable, column?: string): void;
+//# sourceMappingURL=fixGeo.d.ts.map

package/esm/methods/fixGeo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixGeo.d.ts","sourceRoot":"","sources":["../../src/methods/fixGeo.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,MAAM,CAC5B,WAAW,EAAE,WAAW,EACxB,MAAM,CAAC,EAAE,MAAM,QAqBhB"}

package/esm/methods/fixGeo.js

@@ -0,0 +1,22 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.js";
+import queueOp from "../helpers/queueOp.js";
+export default function fixGeo(simpleTable, column) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "fixGeo()",
+        parameters: { column },
+        needsSchema: true,
+        needsSpatial: true,
+        // The cast pins the geometry column back to its existing type, so the
+        // schema is unchanged.
+        preservesSchema: true,
+        buildSelect: (input, types) => {
+            const col = column ??
+                findGeoColumnFromSchema(types, "fixGeo()", simpleTable.name);
+            // The schema type carries the projection (e.g. GEOMETRY('EPSG:4326')),
+            // so the cast keeps it on the new geometries.
+            return `SELECT * REPLACE (ST_MakeValid(${quoteIdentifier(col)})::${types[col]} AS ${quoteIdentifier(col)}) FROM ${input}`;
+        },
+    });
+}

package/esm/methods/flipCoordinates.d.ts

@@ -0,0 +1,3 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function flipCoordinates(simpleTable: SimpleTable, column?: string): void;
+//# sourceMappingURL=flipCoordinates.d.ts.map

package/esm/methods/flipCoordinates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flipCoordinates.d.ts","sourceRoot":"","sources":["../../src/methods/flipCoordinates.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,eAAe,CACrC,WAAW,EAAE,WAAW,EACxB,MAAM,CAAC,EAAE,MAAM,QAqBhB"}

package/esm/methods/flipCoordinates.js

@@ -0,0 +1,22 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.js";
+import queueOp from "../helpers/queueOp.js";
+export default function flipCoordinates(simpleTable, column) {
+    queueOp(simpleTable, {
+        kind: "fusable",
+        method: "flipCoordinates()",
+        parameters: { column },
+        needsSchema: true,
+        needsSpatial: true,
+        // The cast pins the geometry column back to its existing type, so the
+        // schema is unchanged.
+        preservesSchema: true,
+        buildSelect: (input, types) => {
+            const col = column ??
+                findGeoColumnFromSchema(types, "flipCoordinates()", simpleTable.name);
+            // The schema type carries the projection (e.g. GEOMETRY('EPSG:4326')),
+            // so the cast keeps it on the new geometries.
+            return `SELECT * REPLACE (ST_FlipCoordinates(${quoteIdentifier(col)})::${types[col]} AS ${quoteIdentifier(col)}) FROM ${input}`;
+        },
+    });
+}

package/esm/methods/fuzzyClean.d.ts

@@ -1,7 +1,7 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    keep?: "mostCommon" | "longestString" | "shortestString" | "mostCentral" | "maxScore";
-    preFilterPrefixLen?: number;
-}): Promise<void>;
+    strategy?: "mostCommon" | "longestString" | "shortestString" | "mostCentral" | "maxScore";
+    prefilterPrefixLength?: number;
+}): void;
 //# sourceMappingURL=fuzzyClean.d.ts.map

package/esm/methods/fuzzyClean.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,GACL,OAAO,CAAC,IAAI,CAAC,CAmPf"}
+{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,MAAM,CAAC,OAAO,UAAU,UAAU,CAChC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,QAAQ,CAAC,EACL,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAC3B,GACL,IAAI,CAYN"}

package/esm/methods/fuzzyClean.js

@@ -1,49 +1,55 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
-export default async function fuzzyClean(table, column, newColumn, threshold, options = {}) {
+import queueOp from "../helpers/queueOp.js";
+import buildFuzzyMatchSql from "../helpers/buildFuzzyMatchSql.js";
+export default function fuzzyClean(table, column, newColumn, threshold, options = {}) {
+    // The clustering runs in JS over the fuzzy pairs read from the table, so
+    // fuzzyClean can't be expressed as a single SELECT over its input: it
+    // executes as a barrier.
+    options = structuredClone(options);
+    queueOp(table, {
+        kind: "barrier",
+        method: "fuzzyClean()",
+        parameters: { column, newColumn, threshold, options },
+        execute: () => executeFuzzyClean(table, column, newColumn, threshold, options),
+    });
+}
+async function executeFuzzyClean(table, column, newColumn, threshold, options = {}) {
     const method = options.method ?? "ratio";
-    const keep = options.keep ?? "mostCommon";
-    let onClause = `rapidfuzz_${method}(a.value, b.value) >= ${threshold}`;
-    if (method === "ratio") {
-        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
-        onClause +=
-            ` AND ABS(LENGTH(a.value) - LENGTH(b.value)) <= ${maxDiffMultiplier} * GREATEST(LENGTH(a.value), LENGTH(b.value))`;
-    }
-    if (options.preFilterPrefixLen !== undefined) {
-        onClause +=
-            ` AND SUBSTR(a.value, 1, ${options.preFilterPrefixLen}) = SUBSTR(b.value, 1, ${options.preFilterPrefixLen})`;
-    }
-    // Single round trip: compute fuzzy pairs and embed counts for both sides.
-    // Only values that appear in at least one pair above the threshold can be
-    // normalized — singletons need no processing at all.
+    const strategy = options.strategy ?? "mostCommon";
+    const { scoreExpression, condition } = buildFuzzyMatchSql("a.value", "b.value", method, threshold, { prefilterPrefixLength: options.prefilterPrefixLength });
+    // Compute fuzzy pairs and embed counts for both sides. Only values that
+    // appear in at least one pair above the threshold can be normalized —
+    // singletons need no processing at all.
     const pairsData = await queryDB(table, `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;
      WITH uniques AS (
-       SELECT "${column}" AS value, COUNT(*) AS cnt
-       FROM "${table.name}"
-       WHERE "${column}" IS NOT NULL
-       GROUP BY "${column}"
+       SELECT ${quoteIdentifier(column)} AS value, COUNT(*) AS cnt
+       FROM ${quoteIdentifier(table.name)}
+       WHERE ${quoteIdentifier(column)} IS NOT NULL
+       GROUP BY ${quoteIdentifier(column)}
      )
      SELECT
        a.value AS left_value,
        b.value AS right_value,
        a.cnt   AS left_cnt,
        b.cnt   AS right_cnt,
-       rapidfuzz_${method}(a.value, b.value) AS score
+       ${scoreExpression} AS score
      FROM uniques a
      JOIN uniques b
-       ON ${onClause}
+       ON ${condition}
        AND a.value < b.value`, mergeOptions(table, {
         table: table.name,
         method: "fuzzyClean()",
         parameters: { column, newColumn, threshold, options },
-        returnDataFrom: "query",
+        returnData: true,
     }));
     const pairs = pairsData ?? [];
     if (pairs.length === 0) {
         if (newColumn !== column) {
-            await queryDB(table, `ALTER TABLE "${table.name}" ADD "${newColumn}" VARCHAR;
-         UPDATE "${table.name}"
-           SET "${newColumn}" = "${column}";`, mergeOptions(table, {
+            await queryDB(table, `ALTER TABLE ${quoteIdentifier(table.name)} ADD ${quoteIdentifier(newColumn)} VARCHAR;
+         UPDATE ${quoteIdentifier(table.name)}
+           SET ${quoteIdentifier(newColumn)} = ${quoteIdentifier(column)};`, mergeOptions(table, {
                 table: table.name,
                 method: "fuzzyClean()",
                 parameters: { column, newColumn, threshold, options },
@@ -141,21 +147,21 @@ export default async function fuzzyClean(table, column, newColumn, threshold, op
             return byScore(a, b); // sum then alphabetical tie-break
         };
         let canonical;
-        if (keep === "longestString") {
+        if (strategy === "longestString") {
             canonical = members.reduce((a, b) => {
                 if (a.length !== b.length)
                     return a.length > b.length ? a : b;
                 return byScore(a, b);
             });
         }
-        else if (keep === "shortestString") {
+        else if (strategy === "shortestString") {
             canonical = members.reduce((a, b) => {
                 if (a.length !== b.length)
                     return a.length < b.length ? a : b;
                 return byScore(a, b);
             });
         }
-        else if (keep === "mostCommon") {
+        else if (strategy === "mostCommon") {
             canonical = members.reduce((a, b) => {
                 const ca = countMap.get(a) ?? 0;
                 const cb = countMap.get(b) ?? 0;
@@ -164,7 +170,7 @@ export default async function fuzzyClean(table, column, newColumn, threshold, op
                 return byScore(a, b);
             });
         }
-        else if (keep === "mostCentral") {
+        else if (strategy === "mostCentral") {
             // Most central string — highest total similarity to all other cluster members.
             canonical = members.reduce((a, b) => byScore(a, b));
         }
@@ -181,33 +187,27 @@ export default async function fuzzyClean(table, column, newColumn, threshold, op
         return;
     // Use a VALUES-based CTE for the UPDATE so DuckDB can use a hash join
     // instead of evaluating a potentially huge CASE WHEN expression.
-    const escape = (s) => s.replace(/'/g, "''");
-    const valuesList = [...replacement.entries()]
-        .map(([from, to]) => `('${escape(from)}', '${escape(to)}')`)
-        .join(",\n     ");
-    if (newColumn !== column) {
-        await queryDB(table, `ALTER TABLE "${table.name}" ADD "${newColumn}" VARCHAR;
-       UPDATE "${table.name}"
-         SET "${newColumn}" = "${column}";
+    const replacementEntries = [...replacement.entries()];
+    const valuesList = replacementEntries.map(() => "(?, ?)").join(",\n     ");
+    const values = replacementEntries.flatMap(([from, to]) => [from, to]);
+    const query = newColumn !== column
+        ? `ALTER TABLE ${quoteIdentifier(table.name)} ADD ${quoteIdentifier(newColumn)} VARCHAR;
+       UPDATE ${quoteIdentifier(table.name)}
+         SET ${quoteIdentifier(newColumn)} = ${quoteIdentifier(column)};
        WITH mapping(original, canonical) AS (VALUES ${valuesList})
-       UPDATE "${table.name}"
-         SET "${newColumn}" = m.canonical
+       UPDATE ${quoteIdentifier(table.name)}
+         SET ${quoteIdentifier(newColumn)} = m.canonical
          FROM mapping m
-         WHERE "${table.name}"."${newColumn}" = m.original`, mergeOptions(table, {
-            table: table.name,
-            method: "fuzzyClean()",
-            parameters: { column, newColumn, options },
-        }));
-    }
-    else {
-        await queryDB(table, `WITH mapping(original, canonical) AS (VALUES ${valuesList})
-       UPDATE "${table.name}"
-         SET "${column}" = m.canonical
+         WHERE ${quoteIdentifier(table.name)}.${quoteIdentifier(newColumn)} = m.original`
+        : `WITH mapping(original, canonical) AS (VALUES ${valuesList})
+       UPDATE ${quoteIdentifier(table.name)}
+         SET ${quoteIdentifier(column)} = m.canonical
          FROM mapping m
-         WHERE "${table.name}"."${column}" = m.original`, mergeOptions(table, {
-            table: table.name,
-            method: "fuzzyClean()",
-            parameters: { column, newColumn, options },
-        }));
-    }
+         WHERE ${quoteIdentifier(table.name)}.${quoteIdentifier(column)} = m.original`;
+    await queryDB(table, query, mergeOptions(table, {
+        table: table.name,
+        method: "fuzzyClean()",
+        parameters: { column, newColumn, options },
+        values,
+    }));
 }

package/esm/methods/fuzzyJoin.d.ts

@@ -3,6 +3,6 @@ export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTabl
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
     similarityColumn?: string;
     outputTable?: string | boolean;
-    preFilterPrefixLen?: number;
-}): Promise<SimpleTable>;
+    prefilterPrefixLength?: number;
+}): SimpleTable;
 //# sourceMappingURL=fuzzyJoin.d.ts.map

package/esm/methods/fuzzyJoin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,wBAiFP"}
+{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAUvD,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAC3B,GACL,WAAW,CAyCb"}