@nshiab/simple-data-analysis-core 0.0.19 → 0.0.21

package/script/helpers/writeGeoData.js

@@ -0,0 +1,116 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = writeGeoData;
+const node_fs_1 = require("node:fs");
+const createDirectory_js_1 = __importDefault(require("./createDirectory.js"));
+const getExtension_js_1 = __importDefault(require("./getExtension.js"));
+const findGeoColumn_js_1 = __importDefault(require("./findGeoColumn.js"));
+const hasGeometryColumn_js_1 = __importDefault(require("./hasGeometryColumn.js"));
+const shouldFlipBeforeExport_js_1 = __importDefault(require("./shouldFlipBeforeExport.js"));
+const queryDB_js_1 = __importDefault(require("./queryDB.js"));
+const writeGeoDataQuery_js_1 = __importDefault(require("../methods/writeGeoDataQuery.js"));
+const mergeOptions_js_1 = __importDefault(require("./mergeOptions.js"));
+const rewind_js_1 = __importDefault(require("./rewind.js"));
+const stringifyDates_js_1 = __importDefault(require("./stringifyDates.js"));
+const stringifyDatesInvert_js_1 = __importDefault(require("./stringifyDatesInvert.js"));
+const cleanPath_js_1 = __importDefault(require("./cleanPath.js"));
+async function writeGeoData(table, file, options = {}) {
+    (0, createDirectory_js_1.default)(file);
+    if (!(await (0, hasGeometryColumn_js_1.default)(table))) {
+        throw new Error("Table contains no geometry columns. Use writeData() instead.");
+    }
+    const fileExtension = (0, getExtension_js_1.default)(file);
+    if (fileExtension === "geojson" || fileExtension === "json") {
+        let types;
+        if (options.formatDates === true) {
+            types = await table.getTypes();
+            if (Object.values(types).includes("DATE") ||
+                Object.values(types).includes("TIMESTAMP")) {
+                await (0, stringifyDates_js_1.default)(table, types);
+            }
+        }
+        if (typeof options.compression === "boolean") {
+            throw new Error("The compression option is not supported for writing GeoJSON files.");
+        }
+        const geoColumn = await (0, findGeoColumn_js_1.default)(table);
+        const flip = (0, shouldFlipBeforeExport_js_1.default)(table.projections[geoColumn]);
+        if (flip) {
+            await table.flipCoordinates(geoColumn);
+            await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+            await table.flipCoordinates(geoColumn);
+        }
+        else {
+            await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+        }
+        if (options.metadata) {
+            const fileData = JSON.parse((0, node_fs_1.readFileSync)(file, "utf-8"));
+            fileData.metadata = options.metadata;
+            (0, node_fs_1.writeFileSync)(file, JSON.stringify(fileData));
+        }
+        if (options.rewind) {
+            const fileData = JSON.parse((0, node_fs_1.readFileSync)(file, "utf-8"));
+            const fileRewinded = (0, rewind_js_1.default)(fileData);
+            (0, node_fs_1.writeFileSync)(file, JSON.stringify(fileRewinded));
+        }
+        if (types && (Object.values(types).includes("DATE") ||
+            Object.values(types).includes("TIMESTAMP"))) {
+            await (0, stringifyDatesInvert_js_1.default)(table, types);
+        }
+    }
+    else if (fileExtension === "shp") {
+        if (typeof options.precision === "number" ||
+            typeof options.compression === "boolean" ||
+            typeof options.rewind === "boolean" ||
+            options.metadata ||
+            options.formatDates === true) {
+            throw new Error("The following options are not supported for writing SHAPEFILE files: precision, compression, rewind, metadata, and formatDates.");
+        }
+        const geoColumn = await (0, findGeoColumn_js_1.default)(table);
+        const flip = (0, shouldFlipBeforeExport_js_1.default)(table.projections[geoColumn]);
+        if (flip) {
+            await table.flipCoordinates(geoColumn);
+            await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+            await table.flipCoordinates(geoColumn);
+        }
+        else {
+            await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+        }
+    }
+    else if (fileExtension === "geoparquet") {
+        if (typeof options.precision === "number") {
+            throw new Error("The precision option is not supported for writing PARQUET files. Use the .reducePrecision() method.");
+        }
+        if (typeof options.rewind === "boolean") {
+            throw new Error("The rewind option is not supported for writing PARQUET files.");
+        }
+        await (0, queryDB_js_1.default)(table, `COPY "${table.name}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""}, KV_METADATA {
+             projections: '${JSON.stringify(table.projections)}'
+        });`, (0, mergeOptions_js_1.default)(table, {
+            table: table.name,
+            method: "writeGeoData()",
+            parameters: { file, options },
+        }));
+    }
+    else {
+        throw new Error(`Unknown extension ${fileExtension}`);
+    }
+}

package/script/methods/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/methods/fill.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,iBAiFP"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/methods/fill.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,iBAmFP"}

package/script/methods/fill.js

@@ -17,24 +17,19 @@ async function fill(simpleTable, columns, options = {}) {
     }
     if (options.interpolate || options.interpolateBy) {
         const cols = (0, stringToArray_js_1.default)(columns);
-        let orderCol;
-        let excludeList;
-        if (options.interpolateBy) {
-            orderCol = options.interpolateBy;
-            excludeList = cols.join(", ");
-        }
-        else {
-            await simpleTable.addRowNumber(tempRowCol);
-            orderCol = tempRowCol;
-            excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
-        }
+        // Always add temp row number to preserve original row order in output
+        await simpleTable.addRowNumber(tempRowCol);
+        // Use interpolateBy for the window function's ORDER BY (correct interpolation math),
+        // but always order final output by tempRowCol to preserve input row order
+        const windowOrderCol = options.interpolateBy ?? tempRowCol;
+        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
         const overClause = categories.length > 0
             ? `(PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")})`
             : `()`;
         const selectList = cols
-            .map((col) => `fill(${col} ORDER BY "${orderCol}") OVER ${overClause} as ${col}`)
+            .map((col) => `fill(${col} ORDER BY "${windowOrderCol}") OVER ${overClause} as ${col}`)
             .join(", ");
-        await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${orderCol}";`, (0, mergeOptions_js_1.default)(simpleTable, {
+        await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, (0, mergeOptions_js_1.default)(simpleTable, {
             table: simpleTable.name,
             method: "fill()",
             parameters: { columns, ...options },
@@ -55,9 +50,14 @@ async function fill(simpleTable, columns, options = {}) {
         }));
     }
     else {
-        await (0, queryDB_js_1.default)(simpleTable, (0, stringToArray_js_1.default)(columns)
-            .map((col) => `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${col}), COALESCE(${col}, LAG(${col} IGNORE NULLS) OVER()) as ${col} FROM "${simpleTable.name}";`)
-            .join("\n"), (0, mergeOptions_js_1.default)(simpleTable, {
+        // Simple path: add temp row number and order by it to preserve input row order
+        await simpleTable.addRowNumber(tempRowCol);
+        const cols = (0, stringToArray_js_1.default)(columns);
+        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
+        const selectList = cols
+            .map((col) => `COALESCE(${col}, LAG(${col} IGNORE NULLS) OVER()) as ${col}`)
+            .join(", ");
+        await (0, queryDB_js_1.default)(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, (0, mergeOptions_js_1.default)(simpleTable, {
             table: simpleTable.name,
             method: "fill()",
             parameters: { columns, ...options },

package/script/methods/fuzzyClean.d.ts

@@ -1,7 +1,7 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, options?: {
+export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     keep?: "mostCommon" | "longestString" | "shortestString" | "mostCentral" | "maxScore";
+    preFilterPrefixLen?: number;
 }): Promise<void>;
 //# sourceMappingURL=fuzzyClean.d.ts.map

package/script/methods/fuzzyClean.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;CACX,GACL,OAAO,CAAC,IAAI,CAAC,CAyNf"}
+{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,GACL,OAAO,CAAC,IAAI,CAAC,CAmPf"}

package/script/methods/fuzzyClean.js

@@ -6,10 +6,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = fuzzyClean;
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
-async function fuzzyClean(table, column, newColumn, options = {}) {
+async function fuzzyClean(table, column, newColumn, threshold, options = {}) {
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const keep = options.keep ?? "mostCommon";
+    let onClause = `rapidfuzz_${method}(a.value, b.value) >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH(a.value) - LENGTH(b.value)) <= ${maxDiffMultiplier} * GREATEST(LENGTH(a.value), LENGTH(b.value))`;
+    }
+    if (options.preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR(a.value, 1, ${options.preFilterPrefixLen}) = SUBSTR(b.value, 1, ${options.preFilterPrefixLen})`;
+    }
     // Single round trip: compute fuzzy pairs and embed counts for both sides.
     // Only values that appear in at least one pair above the threshold can be
     // normalized — singletons need no processing at all.
@@ -28,16 +37,26 @@ async function fuzzyClean(table, column, newColumn, options = {}) {
        rapidfuzz_${method}(a.value, b.value) AS score
      FROM uniques a
      JOIN uniques b
-       ON rapidfuzz_${method}(a.value, b.value) >= ${threshold}
+       ON ${onClause}
        AND a.value < b.value`, (0, mergeOptions_js_1.default)(table, {
         table: table.name,
         method: "fuzzyClean()",
-        parameters: { column, newColumn, options },
+        parameters: { column, newColumn, threshold, options },
         returnDataFrom: "query",
     }));
     const pairs = pairsData ?? [];
-    if (pairs.length === 0)
-        return; // Nothing to normalize.
+    if (pairs.length === 0) {
+        if (newColumn !== column) {
+            await (0, queryDB_js_1.default)(table, `ALTER TABLE "${table.name}" ADD "${newColumn}" VARCHAR;
+         UPDATE "${table.name}"
+           SET "${newColumn}" = "${column}";`, (0, mergeOptions_js_1.default)(table, {
+                table: table.name,
+                method: "fuzzyClean()",
+                parameters: { column, newColumn, threshold, options },
+            }));
+        }
+        return;
+    }
     // Build count map from the pairs — no separate query needed.
     const countMap = new Map();
     for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {

package/script/methods/fuzzyJoin.d.ts

@@ -1,8 +1,8 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, options?: {
+export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     similarityColumn?: string;
     outputTable?: string | boolean;
+    preFilterPrefixLen?: number;
 }): Promise<SimpleTable>;
 //# sourceMappingURL=fuzzyJoin.d.ts.map

package/script/methods/fuzzyJoin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBAuFP"}
+{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,wBAwFP"}

package/script/methods/fuzzyJoin.js

@@ -8,7 +8,7 @@ const getIdenticalColumns_js_1 = __importDefault(require("../helpers/getIdentica
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const fuzzyJoinQuery_js_1 = __importDefault(require("./fuzzyJoinQuery.js"));
-async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options = {}) {
+async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, threshold, options = {}) {
     if (leftColumn === rightColumn) {
         throw new Error(`The leftColumn and rightColumn have the same name "${leftColumn}". Rename one of them before doing the fuzzy join.`);
     }
@@ -27,13 +27,12 @@ async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options
         }
     }
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const similarityColumn = options.similarityColumn;
     const outputTableName = typeof options.outputTable === "string"
         ? options.outputTable
         : leftTable.name;
     const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        (0, fuzzyJoinQuery_js_1.default)(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTableName, similarityColumn);
+        (0, fuzzyJoinQuery_js_1.default)(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTableName, similarityColumn, options.preFilterPrefixLen);
     await (0, queryDB_js_1.default)(leftTable, sql, (0, mergeOptions_js_1.default)(leftTable, {
         table: outputTableName,
         method: "fuzzyJoin()",
@@ -41,6 +40,7 @@ async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options
             leftColumn,
             rightColumn,
             rightTable: rightTable.name,
+            threshold,
             options,
         },
     }));

package/script/methods/fuzzyJoinQuery.d.ts

@@ -1,2 +1,2 @@
-export default function fuzzyJoinQuery(leftTable: string, leftColumn: string, rightTable: string, rightColumn: string, method: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio", threshold: number, outputTable: string, similarityColumn: string | undefined): string;
+export default function fuzzyJoinQuery(leftTable: string, leftColumn: string, rightTable: string, rightColumn: string, method: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio", threshold: number, outputTable: string, similarityColumn: string | undefined, preFilterPrefixLen?: number): string;
 //# sourceMappingURL=fuzzyJoinQuery.d.ts.map

package/script/methods/fuzzyJoinQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoinQuery.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoinQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EACF,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,EACrB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GAAG,SAAS,UAsBrC"}
+{"version":3,"file":"fuzzyJoinQuery.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoinQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EACF,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,EACrB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GAAG,SAAS,EACpC,kBAAkB,CAAC,EAAE,MAAM,UAkC5B"}

package/script/methods/fuzzyJoinQuery.js

@@ -1,14 +1,24 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = fuzzyJoinQuery;
-function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn) {
+function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, preFilterPrefixLen) {
     const fn = `ROUND(rapidfuzz_${method}("${leftTable}"."${leftColumn}", "${rightTable}"."${rightColumn}"), 2)`;
+    let onClause = `${fn} >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH("${leftTable}"."${leftColumn}") - LENGTH("${rightTable}"."${rightColumn}")) <= ${maxDiffMultiplier} * GREATEST(LENGTH("${leftTable}"."${leftColumn}"), LENGTH("${rightTable}"."${rightColumn}"))`;
+    }
+    if (preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR("${leftTable}"."${leftColumn}", 1, ${preFilterPrefixLen}) = SUBSTR("${rightTable}"."${rightColumn}", 1, ${preFilterPrefixLen})`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE "${outputTable}" AS
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS "${similarityColumn}"
 FROM (
   SELECT "${leftTable}".*, "${rightTable}".*, ${fn} AS "_sda_score"
-  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${fn} >= ${threshold}
+  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${onClause}
 ) _sda
 ORDER BY "${leftColumn}", "_sda_score" DESC;\n`;
     }
@@ -16,7 +26,7 @@ ORDER BY "${leftColumn}", "_sda_score" DESC;\n`;
 SELECT *
 FROM (
   SELECT "${leftTable}".*, "${rightTable}".*
-  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${fn} >= ${threshold}
+  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${onClause}
 ) _sda
 ORDER BY "${leftColumn}", "${rightColumn}";\n`;
 }

package/script/methods/loadDataQuery.d.ts

@@ -7,6 +7,7 @@ export default function loadDataQuery(table: string, files: string[], options?:
     columnTypes?: {
         [key: string]: string;
     };
+    columns?: string[];
     header?: boolean;
     allText?: boolean;
     delim?: string;

package/script/methods/loadDataQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/loadDataQuery.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAExC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,UAAU,CAAC,EAAE,cAAc,GAAG,kBAAkB,GAAG,OAAO,CAAC;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,UAwFP"}
+{"version":3,"file":"loadDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/loadDataQuery.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAExC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,UAAU,CAAC,EAAE,cAAc,GAAG,kBAAkB,GAAG,OAAO,CAAC;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,UAmGP"}

package/script/methods/loadDataQuery.js

@@ -9,6 +9,14 @@ const getExtension_js_1 = __importDefault(require("../helpers/getExtension.js"))
 function loadDataQuery(table, files, options = {}) {
     const fileExtension = (0, getExtension_js_1.default)(files[0]);
     const filesAsString = JSON.stringify(files.map((d) => (0, cleanPath_js_1.default)(d)));
+    // Column selection: throw for Excel, build SELECT list for others
+    const isExcel = options.fileType === "excel" || fileExtension === "xlsx";
+    if (options.columns && options.columns.length > 0 && isExcel) {
+        throw new Error("The 'columns' option is not supported for Excel files.");
+    }
+    const selectColumns = options.columns && options.columns.length > 0
+        ? options.columns.map((c) => `"${c}"`).join(", ")
+        : "*";
     // General options, except for parquet
     const autoDetect = typeof options.autoDetect === "boolean"
         ? `, auto_detect=${String(options.autoDetect).toUpperCase()}`
@@ -50,7 +58,7 @@ function loadDataQuery(table, files, options = {}) {
         const encoding = options.encoding ? `, encoding='${options.encoding}'` : "";
         const strict = options.strict === false ? `, strict_mode=FALSE` : "";
         return `CREATE OR REPLACE TABLE "${table}"
-            AS SELECT * FROM read_csv_auto(${filesAsString}${generalOptions}${header}${allText}${delim}${skip}${compression}${encoding}${strict}${nullPadding}${ignoreErrors})${limit};`;
+            AS SELECT ${selectColumns} FROM read_csv_auto(${filesAsString}${generalOptions}${header}${allText}${delim}${skip}${compression}${encoding}${strict}${nullPadding}${ignoreErrors})${limit};`;
     }
     else if (options.fileType === "json" || fileExtension === "json") {
         const jsonFormat = options.jsonFormat
@@ -60,10 +68,10 @@ function loadDataQuery(table, files, options = {}) {
             ? `, records=${String(options.records).toUpperCase()}`
             : "";
         return `CREATE OR REPLACE TABLE "${table}"
-            AS SELECT * FROM read_json_auto(${filesAsString}${generalOptions}${jsonFormat}${records})${limit};`;
+            AS SELECT ${selectColumns} FROM read_json_auto(${filesAsString}${generalOptions}${jsonFormat}${records})${limit};`;
     }
     else if (options.fileType === "parquet" || fileExtension === "parquet") {
-        return `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM read_parquet(${filesAsString}${fileName}${unifyColumns})${limit};`;
+        return `CREATE OR REPLACE TABLE "${table}" AS SELECT ${selectColumns} FROM read_parquet(${filesAsString}${fileName}${unifyColumns})${limit};`;
     }
     else if (options.fileType === "excel" || fileExtension === "xlsx") {
         if (files.length > 1) {
@@ -75,7 +83,7 @@ function loadDataQuery(table, files, options = {}) {
         const allText = typeof options.allText === "boolean"
             ? `, all_varchar=${String(options.allText).toUpperCase()}`
             : "";
-        return `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM read_xlsx('${files[0]}'${options.sheet ? `, sheet='${options.sheet}'` : ""}${header}${allText});`;
+        return `CREATE OR REPLACE TABLE "${table}" AS SELECT ${selectColumns} FROM read_xlsx('${files[0]}'${options.sheet ? `, sheet='${options.sheet}'` : ""}${header}${allText});`;
     }
     else {
         throw new Error(`Unknown options.fileType ${options.fileType} or fileExtension ${fileExtension}`);

package/script/methods/padQuery.d.ts

@@ -0,0 +1,5 @@
+export default function padQuery(table: string, columns: string[], length: number, options: {
+    method?: "left" | "right";
+    char?: string;
+}): string;
+//# sourceMappingURL=padQuery.d.ts.map

package/script/methods/padQuery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"padQuery.d.ts","sourceRoot":"","sources":["../../src/methods/padQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,UAiBtD"}

package/script/methods/padQuery.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = padQuery;
+function padQuery(table, columns, length, options) {
+    const method = options.method ?? "left";
+    const char = options.char ?? "0";
+    // Escape single quotes and wrap in single quotes for SQL
+    const escapedChar = char.replace(/'/g, "''");
+    const paddedChar = `'${escapedChar}'`;
+    const func = method === "left" ? "LPAD" : "RPAD";
+    let query = "";
+    for (const column of columns) {
+        query +=
+            `\nUPDATE "${table}" SET "${column}" = ${func}("${column}", ${length}, ${paddedChar});`;
+    }
+    return query;
+}

package/script/methods/writeGeoDataQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writeGeoDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/writeGeoDataQuery.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,UAqBrC"}
+{"version":3,"file":"writeGeoDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/writeGeoDataQuery.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,UAyBrC"}

package/script/methods/writeGeoDataQuery.js

@@ -12,10 +12,13 @@ function writeGeoDataQuery(table, file, fileExtension, options = {}) {
             layerOptions.push(`COORDINATE_PRECISION=${options.precision}`);
         }
         layerOptions.push(`RFC7946=YES`);
-        return `COPY "${table}" to '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
+        return `INSTALL spatial; LOAD spatial; COPY "${table}" to '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
             ? `, LAYER_CREATION_OPTIONS ('WRITE_NAME=NO', ${layerOptions.map((d) => `'${d}'`).join(", ")})`
             : ""})`;
     }
+    else if (fileExtension === "shp") {
+        return `INSTALL spatial; LOAD spatial; COPY "${table}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
+    }
     else {
         throw new Error(`Unknown extension ${fileExtension}`);
     }