@nshiab/simple-data-analysis-core 0.0.20 → 0.0.22

package/script/methods/fuzzyClean.js

@@ -6,10 +6,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = fuzzyClean;
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
-async function fuzzyClean(table, column, newColumn, options = {}) {
+async function fuzzyClean(table, column, newColumn, threshold, options = {}) {
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const keep = options.keep ?? "mostCommon";
+    let onClause = `rapidfuzz_${method}(a.value, b.value) >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH(a.value) - LENGTH(b.value)) <= ${maxDiffMultiplier} * GREATEST(LENGTH(a.value), LENGTH(b.value))`;
+    }
+    if (options.preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR(a.value, 1, ${options.preFilterPrefixLen}) = SUBSTR(b.value, 1, ${options.preFilterPrefixLen})`;
+    }
     // Single round trip: compute fuzzy pairs and embed counts for both sides.
     // Only values that appear in at least one pair above the threshold can be
     // normalized — singletons need no processing at all.
@@ -28,16 +37,26 @@ async function fuzzyClean(table, column, newColumn, options = {}) {
        rapidfuzz_${method}(a.value, b.value) AS score
      FROM uniques a
      JOIN uniques b
-       ON rapidfuzz_${method}(a.value, b.value) >= ${threshold}
+       ON ${onClause}
        AND a.value < b.value`, (0, mergeOptions_js_1.default)(table, {
         table: table.name,
         method: "fuzzyClean()",
-        parameters: { column, newColumn, options },
+        parameters: { column, newColumn, threshold, options },
         returnDataFrom: "query",
     }));
     const pairs = pairsData ?? [];
-    if (pairs.length === 0)
-        return; // Nothing to normalize.
+    if (pairs.length === 0) {
+        if (newColumn !== column) {
+            await (0, queryDB_js_1.default)(table, `ALTER TABLE "${table.name}" ADD "${newColumn}" VARCHAR;
+         UPDATE "${table.name}"
+           SET "${newColumn}" = "${column}";`, (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "fuzzyClean()",
+                parameters: { column, newColumn, threshold, options },
+            }));
+        }
+        return;
+    }
     // Build count map from the pairs — no separate query needed.
     const countMap = new Map();
     for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {

package/script/methods/fuzzyJoin.d.ts

@@ -1,8 +1,8 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, options?: {
+export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     similarityColumn?: string;
     outputTable?: string | boolean;
+    preFilterPrefixLen?: number;
 }): Promise<SimpleTable>;
 //# sourceMappingURL=fuzzyJoin.d.ts.map

package/script/methods/fuzzyJoin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBAuFP"}
+{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,wBAwFP"}

package/script/methods/fuzzyJoin.js

@@ -8,7 +8,7 @@ const getIdenticalColumns_js_1 = __importDefault(require("../helpers/getIdentica
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const fuzzyJoinQuery_js_1 = __importDefault(require("./fuzzyJoinQuery.js"));
-async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options = {}) {
+async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, threshold, options = {}) {
     if (leftColumn === rightColumn) {
         throw new Error(`The leftColumn and rightColumn have the same name "${leftColumn}". Rename one of them before doing the fuzzy join.`);
     }
@@ -27,13 +27,12 @@ async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options
         }
     }
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const similarityColumn = options.similarityColumn;
     const outputTableName = typeof options.outputTable === "string"
         ? options.outputTable
         : leftTable.name;
     const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        (0, fuzzyJoinQuery_js_1.default)(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTableName, similarityColumn);
+        (0, fuzzyJoinQuery_js_1.default)(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTableName, similarityColumn, options.preFilterPrefixLen);
     await (0, queryDB_js_1.default)(leftTable, sql, (0, mergeOptions_js_1.default)(leftTable, {
         table: outputTableName,
         method: "fuzzyJoin()",
@@ -41,6 +40,7 @@ async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options
             leftColumn,
             rightColumn,
             rightTable: rightTable.name,
+            threshold,
             options,
         },
     }));

package/script/methods/fuzzyJoinQuery.d.ts

@@ -1,2 +1,2 @@
-export default function fuzzyJoinQuery(leftTable: string, leftColumn: string, rightTable: string, rightColumn: string, method: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio", threshold: number, outputTable: string, similarityColumn: string | undefined): string;
+export default function fuzzyJoinQuery(leftTable: string, leftColumn: string, rightTable: string, rightColumn: string, method: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio", threshold: number, outputTable: string, similarityColumn: string | undefined, preFilterPrefixLen?: number): string;
 //# sourceMappingURL=fuzzyJoinQuery.d.ts.map

package/script/methods/fuzzyJoinQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoinQuery.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoinQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EACF,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,EACrB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GAAG,SAAS,UAsBrC"}
+{"version":3,"file":"fuzzyJoinQuery.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoinQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EACF,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,EACrB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GAAG,SAAS,EACpC,kBAAkB,CAAC,EAAE,MAAM,UAkC5B"}

package/script/methods/fuzzyJoinQuery.js

@@ -1,14 +1,24 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = fuzzyJoinQuery;
-function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn) {
+function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, preFilterPrefixLen) {
     const fn = `ROUND(rapidfuzz_${method}("${leftTable}"."${leftColumn}", "${rightTable}"."${rightColumn}"), 2)`;
+    let onClause = `${fn} >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH("${leftTable}"."${leftColumn}") - LENGTH("${rightTable}"."${rightColumn}")) <= ${maxDiffMultiplier} * GREATEST(LENGTH("${leftTable}"."${leftColumn}"), LENGTH("${rightTable}"."${rightColumn}"))`;
+    }
+    if (preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR("${leftTable}"."${leftColumn}", 1, ${preFilterPrefixLen}) = SUBSTR("${rightTable}"."${rightColumn}", 1, ${preFilterPrefixLen})`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE "${outputTable}" AS
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS "${similarityColumn}"
 FROM (
   SELECT "${leftTable}".*, "${rightTable}".*, ${fn} AS "_sda_score"
-  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${fn} >= ${threshold}
+  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${onClause}
 ) _sda
 ORDER BY "${leftColumn}", "_sda_score" DESC;\n`;
     }
@@ -16,7 +26,7 @@ ORDER BY "${leftColumn}", "_sda_score" DESC;\n`;
 SELECT *
 FROM (
   SELECT "${leftTable}".*, "${rightTable}".*
-  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${fn} >= ${threshold}
+  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${onClause}
 ) _sda
 ORDER BY "${leftColumn}", "${rightColumn}";\n`;
 }

package/script/methods/loadSample.d.ts

@@ -0,0 +1,9 @@
+import type SimpleTable from "../class/SimpleTable.js";
+/**
+ * Fetches sample data from the simple-data-analysis-core GitHub repository.
+ *
+ * @param table - The SimpleTable instance to load the data into.
+ * @param sample - The name of the sample to load.
+ */
+export default function loadSample(table: SimpleTable, sample: "fires" | "recipes" | "temperatures" | "temperaturesCities" | "canada" | "firesGeo"): Promise<SimpleTable>;
+//# sourceMappingURL=loadSample.d.ts.map

package/script/methods/loadSample.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadSample.d.ts","sourceRoot":"","sources":["../../src/methods/loadSample.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD;;;;;GAKG;AACH,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EACF,OAAO,GACP,SAAS,GACT,cAAc,GACd,oBAAoB,GACpB,QAAQ,GACR,UAAU,GACb,OAAO,CAAC,WAAW,CAAC,CA6CtB"}

package/script/methods/loadSample.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = loadSample;
+/**
+ * Fetches sample data from the simple-data-analysis-core GitHub repository.
+ *
+ * @param table - The SimpleTable instance to load the data into.
+ * @param sample - The name of the sample to load.
+ */
+async function loadSample(table, sample) {
+    const samples = {
+        fires: {
+            url: "https://raw.githubusercontent.com/nshiab/simple-data-analysis-core/refs/heads/main/test/geodata/files/firesCanada2023.csv",
+            geo: false,
+        },
+        recipes: {
+            url: "https://github.com/nshiab/simple-data-analysis-core/raw/refs/heads/main/test/data/files/recipes.parquet",
+            geo: false,
+        },
+        temperatures: {
+            url: "https://raw.githubusercontent.com/nshiab/simple-data-analysis-core/refs/heads/main/test/data/files/dailyTemperatures.csv",
+            geo: false,
+        },
+        temperaturesCities: {
+            url: "https://raw.githubusercontent.com/nshiab/simple-data-analysis-core/refs/heads/main/test/data/files/cities.csv",
+            geo: false,
+        },
+        canada: {
+            url: "https://raw.githubusercontent.com/nshiab/simple-data-analysis-core/refs/heads/main/test/geodata/files/CanadianProvincesAndTerritories.json",
+            geo: true,
+        },
+        firesGeo: {
+            url: "https://raw.githubusercontent.com/nshiab/simple-data-analysis-core/refs/heads/main/test/geodata/files/firesCanada2023.geojson",
+            geo: true,
+        },
+    };
+    const sampleToLoad = samples[sample];
+    if (!sampleToLoad) {
+        throw new Error(`Unknown sample: ${sample}`);
+    }
+    if (sampleToLoad.geo) {
+        return await table.loadGeoData(sampleToLoad.url);
+    }
+    else {
+        return await table.loadData(sampleToLoad.url);
+    }
+}

package/script/methods/writeGeoDataQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writeGeoDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/writeGeoDataQuery.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,UAqBrC"}
+{"version":3,"file":"writeGeoDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/writeGeoDataQuery.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,UAyBrC"}

package/script/methods/writeGeoDataQuery.js

@@ -12,10 +12,13 @@ function writeGeoDataQuery(table, file, fileExtension, options = {}) {
             layerOptions.push(`COORDINATE_PRECISION=${options.precision}`);
         }
         layerOptions.push(`RFC7946=YES`);
-        return `COPY "${table}" to '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
+        return `INSTALL spatial; LOAD spatial; COPY "${table}" to '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
             ? `, LAYER_CREATION_OPTIONS ('WRITE_NAME=NO', ${layerOptions.map((d) => `'${d}'`).join(", ")})`
             : ""})`;
     }
+    else if (fileExtension === "shp") {
+        return `INSTALL spatial; LOAD spatial; COPY "${table}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
+    }
     else {
         throw new Error(`Unknown extension ${fileExtension}`);
     }