@nshiab/simple-data-analysis-core 0.0.19 → 0.0.21

package/esm/helpers/writeGeoData.js

@@ -0,0 +1,110 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import createDirectory from "./createDirectory.js";
+import getExtension from "./getExtension.js";
+import findGeoColumn from "./findGeoColumn.js";
+import hasGeometryColumn from "./hasGeometryColumn.js";
+import shouldFlipBeforeExport from "./shouldFlipBeforeExport.js";
+import queryDB from "./queryDB.js";
+import writeGeoDataQuery from "../methods/writeGeoDataQuery.js";
+import mergeOptions from "./mergeOptions.js";
+import rewind from "./rewind.js";
+import stringifyDates from "./stringifyDates.js";
+import stringifyDatesInvert from "./stringifyDatesInvert.js";
+import cleanPath from "./cleanPath.js";
+export default async function writeGeoData(table, file, options = {}) {
+    createDirectory(file);
+    if (!(await hasGeometryColumn(table))) {
+        throw new Error("Table contains no geometry columns. Use writeData() instead.");
+    }
+    const fileExtension = getExtension(file);
+    if (fileExtension === "geojson" || fileExtension === "json") {
+        let types;
+        if (options.formatDates === true) {
+            types = await table.getTypes();
+            if (Object.values(types).includes("DATE") ||
+                Object.values(types).includes("TIMESTAMP")) {
+                await stringifyDates(table, types);
+            }
+        }
+        if (typeof options.compression === "boolean") {
+            throw new Error("The compression option is not supported for writing GeoJSON files.");
+        }
+        const geoColumn = await findGeoColumn(table);
+        const flip = shouldFlipBeforeExport(table.projections[geoColumn]);
+        if (flip) {
+            await table.flipCoordinates(geoColumn);
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+            await table.flipCoordinates(geoColumn);
+        }
+        else {
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+        }
+        if (options.metadata) {
+            const fileData = JSON.parse(readFileSync(file, "utf-8"));
+            fileData.metadata = options.metadata;
+            writeFileSync(file, JSON.stringify(fileData));
+        }
+        if (options.rewind) {
+            const fileData = JSON.parse(readFileSync(file, "utf-8"));
+            const fileRewinded = rewind(fileData);
+            writeFileSync(file, JSON.stringify(fileRewinded));
+        }
+        if (types && (Object.values(types).includes("DATE") ||
+            Object.values(types).includes("TIMESTAMP"))) {
+            await stringifyDatesInvert(table, types);
+        }
+    }
+    else if (fileExtension === "shp") {
+        if (typeof options.precision === "number" ||
+            typeof options.compression === "boolean" ||
+            typeof options.rewind === "boolean" ||
+            options.metadata ||
+            options.formatDates === true) {
+            throw new Error("The following options are not supported for writing SHAPEFILE files: precision, compression, rewind, metadata, and formatDates.");
+        }
+        const geoColumn = await findGeoColumn(table);
+        const flip = shouldFlipBeforeExport(table.projections[geoColumn]);
+        if (flip) {
+            await table.flipCoordinates(geoColumn);
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+            await table.flipCoordinates(geoColumn);
+        }
+        else {
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+        }
+    }
+    else if (fileExtension === "geoparquet") {
+        if (typeof options.precision === "number") {
+            throw new Error("The precision option is not supported for writing PARQUET files. Use the .reducePrecision() method.");
+        }
+        if (typeof options.rewind === "boolean") {
+            throw new Error("The rewind option is not supported for writing PARQUET files.");
+        }
+        await queryDB(table, `COPY "${table.name}" TO '${cleanPath(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""}, KV_METADATA {
+             projections: '${JSON.stringify(table.projections)}'
+        });`, mergeOptions(table, {
+            table: table.name,
+            method: "writeGeoData()",
+            parameters: { file, options },
+        }));
+    }
+    else {
+        throw new Error(`Unknown extension ${fileExtension}`);
+    }
+}

package/esm/methods/fill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/methods/fill.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,iBAiFP"}
+{"version":3,"file":"fill.d.ts","sourceRoot":"","sources":["../../src/methods/fill.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,OAAO,GAAE;IACP,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACnB,iBAmFP"}

package/esm/methods/fill.js

@@ -11,24 +11,19 @@ export default async function fill(simpleTable, columns, options = {}) {
     }
     if (options.interpolate || options.interpolateBy) {
         const cols = stringToArray(columns);
-        let orderCol;
-        let excludeList;
-        if (options.interpolateBy) {
-            orderCol = options.interpolateBy;
-            excludeList = cols.join(", ");
-        }
-        else {
-            await simpleTable.addRowNumber(tempRowCol);
-            orderCol = tempRowCol;
-            excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
-        }
+        // Always add temp row number to preserve original row order in output
+        await simpleTable.addRowNumber(tempRowCol);
+        // Use interpolateBy for the window function's ORDER BY (correct interpolation math),
+        // but always order final output by tempRowCol to preserve input row order
+        const windowOrderCol = options.interpolateBy ?? tempRowCol;
+        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
         const overClause = categories.length > 0
             ? `(PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")})`
             : `()`;
         const selectList = cols
-            .map((col) => `fill(${col} ORDER BY "${orderCol}") OVER ${overClause} as ${col}`)
+            .map((col) => `fill(${col} ORDER BY "${windowOrderCol}") OVER ${overClause} as ${col}`)
             .join(", ");
-        await queryDB(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${orderCol}";`, mergeOptions(simpleTable, {
+        await queryDB(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, mergeOptions(simpleTable, {
             table: simpleTable.name,
             method: "fill()",
             parameters: { columns, ...options },
@@ -49,9 +44,14 @@ export default async function fill(simpleTable, columns, options = {}) {
         }));
     }
     else {
-        await queryDB(simpleTable, stringToArray(columns)
-            .map((col) => `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${col}), COALESCE(${col}, LAG(${col} IGNORE NULLS) OVER()) as ${col} FROM "${simpleTable.name}";`)
-            .join("\n"), mergeOptions(simpleTable, {
+        // Simple path: add temp row number and order by it to preserve input row order
+        await simpleTable.addRowNumber(tempRowCol);
+        const cols = stringToArray(columns);
+        const excludeList = [`"${tempRowCol}"`, ...cols].join(", ");
+        const selectList = cols
+            .map((col) => `COALESCE(${col}, LAG(${col} IGNORE NULLS) OVER()) as ${col}`)
+            .join(", ");
+        await queryDB(simpleTable, `CREATE OR REPLACE TABLE "${simpleTable.name}" AS SELECT * EXCLUDE(${excludeList}), ${selectList} FROM "${simpleTable.name}" ORDER BY "${tempRowCol}";`, mergeOptions(simpleTable, {
             table: simpleTable.name,
             method: "fill()",
             parameters: { columns, ...options },

package/esm/methods/fuzzyClean.d.ts

@@ -1,7 +1,7 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, options?: {
+export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     keep?: "mostCommon" | "longestString" | "shortestString" | "mostCentral" | "maxScore";
+    preFilterPrefixLen?: number;
 }): Promise<void>;
 //# sourceMappingURL=fuzzyClean.d.ts.map

package/esm/methods/fuzzyClean.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;CACX,GACL,OAAO,CAAC,IAAI,CAAC,CAyNf"}
+{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,GACL,OAAO,CAAC,IAAI,CAAC,CAmPf"}

package/esm/methods/fuzzyClean.js

@@ -1,9 +1,18 @@
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
-export default async function fuzzyClean(table, column, newColumn, options = {}) {
+export default async function fuzzyClean(table, column, newColumn, threshold, options = {}) {
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const keep = options.keep ?? "mostCommon";
+    let onClause = `rapidfuzz_${method}(a.value, b.value) >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH(a.value) - LENGTH(b.value)) <= ${maxDiffMultiplier} * GREATEST(LENGTH(a.value), LENGTH(b.value))`;
+    }
+    if (options.preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR(a.value, 1, ${options.preFilterPrefixLen}) = SUBSTR(b.value, 1, ${options.preFilterPrefixLen})`;
+    }
     // Single round trip: compute fuzzy pairs and embed counts for both sides.
     // Only values that appear in at least one pair above the threshold can be
     // normalized — singletons need no processing at all.
@@ -22,16 +31,26 @@ export default async function fuzzyClean(table, column, newColumn, options = {})
        rapidfuzz_${method}(a.value, b.value) AS score
      FROM uniques a
      JOIN uniques b
-       ON rapidfuzz_${method}(a.value, b.value) >= ${threshold}
+       ON ${onClause}
        AND a.value < b.value`, mergeOptions(table, {
         table: table.name,
         method: "fuzzyClean()",
-        parameters: { column, newColumn, options },
+        parameters: { column, newColumn, threshold, options },
         returnDataFrom: "query",
     }));
     const pairs = pairsData ?? [];
-    if (pairs.length === 0)
-        return; // Nothing to normalize.
+    if (pairs.length === 0) {
+        if (newColumn !== column) {
+            await queryDB(table, `ALTER TABLE "${table.name}" ADD "${newColumn}" VARCHAR;
+         UPDATE "${table.name}"
+           SET "${newColumn}" = "${column}";`, mergeOptions(table, {
+                table: table.name,
+                method: "fuzzyClean()",
+                parameters: { column, newColumn, threshold, options },
+            }));
+        }
+        return;
+    }
     // Build count map from the pairs — no separate query needed.
     const countMap = new Map();
     for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {

package/esm/methods/fuzzyJoin.d.ts

@@ -1,8 +1,8 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, options?: {
+export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     similarityColumn?: string;
     outputTable?: string | boolean;
+    preFilterPrefixLen?: number;
 }): Promise<SimpleTable>;
 //# sourceMappingURL=fuzzyJoin.d.ts.map

package/esm/methods/fuzzyJoin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBAuFP"}
+{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,wBAwFP"}

package/esm/methods/fuzzyJoin.js

@@ -2,7 +2,7 @@ import getIdenticalColumns from "../helpers/getIdenticalColumns.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
 import fuzzyJoinQuery from "./fuzzyJoinQuery.js";
-export default async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, options = {}) {
+export default async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, threshold, options = {}) {
     if (leftColumn === rightColumn) {
         throw new Error(`The leftColumn and rightColumn have the same name "${leftColumn}". Rename one of them before doing the fuzzy join.`);
     }
@@ -21,13 +21,12 @@ export default async function fuzzyJoin(leftTable, rightTable, leftColumn, right
         }
     }
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const similarityColumn = options.similarityColumn;
     const outputTableName = typeof options.outputTable === "string"
         ? options.outputTable
         : leftTable.name;
     const sql = `INSTALL rapidfuzz FROM community; LOAD rapidfuzz;\n` +
-        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTableName, similarityColumn);
+        fuzzyJoinQuery(leftTable.name, leftColumn, rightTable.name, rightColumn, method, threshold, outputTableName, similarityColumn, options.preFilterPrefixLen);
     await queryDB(leftTable, sql, mergeOptions(leftTable, {
         table: outputTableName,
         method: "fuzzyJoin()",
@@ -35,6 +34,7 @@ export default async function fuzzyJoin(leftTable, rightTable, leftColumn, right
             leftColumn,
             rightColumn,
             rightTable: rightTable.name,
+            threshold,
             options,
         },
     }));

package/esm/methods/fuzzyJoinQuery.d.ts

@@ -1,2 +1,2 @@
-export default function fuzzyJoinQuery(leftTable: string, leftColumn: string, rightTable: string, rightColumn: string, method: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio", threshold: number, outputTable: string, similarityColumn: string | undefined): string;
+export default function fuzzyJoinQuery(leftTable: string, leftColumn: string, rightTable: string, rightColumn: string, method: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio", threshold: number, outputTable: string, similarityColumn: string | undefined, preFilterPrefixLen?: number): string;
 //# sourceMappingURL=fuzzyJoinQuery.d.ts.map

package/esm/methods/fuzzyJoinQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyJoinQuery.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoinQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EACF,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,EACrB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GAAG,SAAS,UAsBrC"}
+{"version":3,"file":"fuzzyJoinQuery.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoinQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EACF,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,EACrB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,gBAAgB,EAAE,MAAM,GAAG,SAAS,EACpC,kBAAkB,CAAC,EAAE,MAAM,UAkC5B"}

package/esm/methods/fuzzyJoinQuery.js

@@ -1,11 +1,21 @@
-export default function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn) {
+export default function fuzzyJoinQuery(leftTable, leftColumn, rightTable, rightColumn, method, threshold, outputTable, similarityColumn, preFilterPrefixLen) {
     const fn = `ROUND(rapidfuzz_${method}("${leftTable}"."${leftColumn}", "${rightTable}"."${rightColumn}"), 2)`;
+    let onClause = `${fn} >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH("${leftTable}"."${leftColumn}") - LENGTH("${rightTable}"."${rightColumn}")) <= ${maxDiffMultiplier} * GREATEST(LENGTH("${leftTable}"."${leftColumn}"), LENGTH("${rightTable}"."${rightColumn}"))`;
+    }
+    if (preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR("${leftTable}"."${leftColumn}", 1, ${preFilterPrefixLen}) = SUBSTR("${rightTable}"."${rightColumn}", 1, ${preFilterPrefixLen})`;
+    }
     if (similarityColumn) {
         return `CREATE OR REPLACE TABLE "${outputTable}" AS
 SELECT * EXCLUDE ("_sda_score"), "_sda_score" AS "${similarityColumn}"
 FROM (
   SELECT "${leftTable}".*, "${rightTable}".*, ${fn} AS "_sda_score"
-  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${fn} >= ${threshold}
+  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${onClause}
 ) _sda
 ORDER BY "${leftColumn}", "_sda_score" DESC;\n`;
     }
@@ -13,7 +23,7 @@ ORDER BY "${leftColumn}", "_sda_score" DESC;\n`;
 SELECT *
 FROM (
   SELECT "${leftTable}".*, "${rightTable}".*
-  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${fn} >= ${threshold}
+  FROM "${leftTable}" LEFT JOIN "${rightTable}" ON ${onClause}
 ) _sda
 ORDER BY "${leftColumn}", "${rightColumn}";\n`;
 }

package/esm/methods/loadDataQuery.d.ts

@@ -7,6 +7,7 @@ export default function loadDataQuery(table: string, files: string[], options?:
     columnTypes?: {
         [key: string]: string;
     };
+    columns?: string[];
     header?: boolean;
     allText?: boolean;
     delim?: string;

package/esm/methods/loadDataQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loadDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/loadDataQuery.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAExC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,UAAU,CAAC,EAAE,cAAc,GAAG,kBAAkB,GAAG,OAAO,CAAC;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,UAwFP"}
+{"version":3,"file":"loadDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/loadDataQuery.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAExC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,UAAU,CAAC,EAAE,cAAc,GAAG,kBAAkB,GAAG,OAAO,CAAC;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,UAmGP"}

package/esm/methods/loadDataQuery.js

@@ -3,6 +3,14 @@ import getExtension from "../helpers/getExtension.js";
 export default function loadDataQuery(table, files, options = {}) {
     const fileExtension = getExtension(files[0]);
     const filesAsString = JSON.stringify(files.map((d) => cleanPath(d)));
+    // Column selection: throw for Excel, build SELECT list for others
+    const isExcel = options.fileType === "excel" || fileExtension === "xlsx";
+    if (options.columns && options.columns.length > 0 && isExcel) {
+        throw new Error("The 'columns' option is not supported for Excel files.");
+    }
+    const selectColumns = options.columns && options.columns.length > 0
+        ? options.columns.map((c) => `"${c}"`).join(", ")
+        : "*";
     // General options, except for parquet
     const autoDetect = typeof options.autoDetect === "boolean"
         ? `, auto_detect=${String(options.autoDetect).toUpperCase()}`
@@ -44,7 +52,7 @@ export default function loadDataQuery(table, files, options = {}) {
         const encoding = options.encoding ? `, encoding='${options.encoding}'` : "";
         const strict = options.strict === false ? `, strict_mode=FALSE` : "";
         return `CREATE OR REPLACE TABLE "${table}"
-            AS SELECT * FROM read_csv_auto(${filesAsString}${generalOptions}${header}${allText}${delim}${skip}${compression}${encoding}${strict}${nullPadding}${ignoreErrors})${limit};`;
+            AS SELECT ${selectColumns} FROM read_csv_auto(${filesAsString}${generalOptions}${header}${allText}${delim}${skip}${compression}${encoding}${strict}${nullPadding}${ignoreErrors})${limit};`;
     }
     else if (options.fileType === "json" || fileExtension === "json") {
         const jsonFormat = options.jsonFormat
@@ -54,10 +62,10 @@ export default function loadDataQuery(table, files, options = {}) {
             ? `, records=${String(options.records).toUpperCase()}`
             : "";
         return `CREATE OR REPLACE TABLE "${table}"
-            AS SELECT * FROM read_json_auto(${filesAsString}${generalOptions}${jsonFormat}${records})${limit};`;
+            AS SELECT ${selectColumns} FROM read_json_auto(${filesAsString}${generalOptions}${jsonFormat}${records})${limit};`;
     }
     else if (options.fileType === "parquet" || fileExtension === "parquet") {
-        return `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM read_parquet(${filesAsString}${fileName}${unifyColumns})${limit};`;
+        return `CREATE OR REPLACE TABLE "${table}" AS SELECT ${selectColumns} FROM read_parquet(${filesAsString}${fileName}${unifyColumns})${limit};`;
     }
     else if (options.fileType === "excel" || fileExtension === "xlsx") {
         if (files.length > 1) {
@@ -69,7 +77,7 @@ export default function loadDataQuery(table, files, options = {}) {
         const allText = typeof options.allText === "boolean"
             ? `, all_varchar=${String(options.allText).toUpperCase()}`
             : "";
-        return `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM read_xlsx('${files[0]}'${options.sheet ? `, sheet='${options.sheet}'` : ""}${header}${allText});`;
+        return `CREATE OR REPLACE TABLE "${table}" AS SELECT ${selectColumns} FROM read_xlsx('${files[0]}'${options.sheet ? `, sheet='${options.sheet}'` : ""}${header}${allText});`;
     }
     else {
         throw new Error(`Unknown options.fileType ${options.fileType} or fileExtension ${fileExtension}`);

package/esm/methods/padQuery.d.ts

@@ -0,0 +1,5 @@
+export default function padQuery(table: string, columns: string[], length: number, options: {
+    method?: "left" | "right";
+    char?: string;
+}): string;
+//# sourceMappingURL=padQuery.d.ts.map

package/esm/methods/padQuery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"padQuery.d.ts","sourceRoot":"","sources":["../../src/methods/padQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,UAiBtD"}

package/esm/methods/padQuery.js

@@ -0,0 +1,14 @@
+export default function padQuery(table, columns, length, options) {
+    const method = options.method ?? "left";
+    const char = options.char ?? "0";
+    // Escape single quotes and wrap in single quotes for SQL
+    const escapedChar = char.replace(/'/g, "''");
+    const paddedChar = `'${escapedChar}'`;
+    const func = method === "left" ? "LPAD" : "RPAD";
+    let query = "";
+    for (const column of columns) {
+        query +=
+            `\nUPDATE "${table}" SET "${column}" = ${func}("${column}", ${length}, ${paddedChar});`;
+    }
+    return query;
+}

package/esm/methods/writeGeoDataQuery.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writeGeoDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/writeGeoDataQuery.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,UAqBrC"}
+{"version":3,"file":"writeGeoDataQuery.d.ts","sourceRoot":"","sources":["../../src/methods/writeGeoDataQuery.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,iBAAiB,CACvC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,UAyBrC"}

package/esm/methods/writeGeoDataQuery.js

@@ -6,10 +6,13 @@ export default function writeGeoDataQuery(table, file, fileExtension, options =
             layerOptions.push(`COORDINATE_PRECISION=${options.precision}`);
         }
         layerOptions.push(`RFC7946=YES`);
-        return `COPY "${table}" to '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
+        return `INSTALL spatial; LOAD spatial; COPY "${table}" to '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
             ? `, LAYER_CREATION_OPTIONS ('WRITE_NAME=NO', ${layerOptions.map((d) => `'${d}'`).join(", ")})`
             : ""})`;
     }
+    else if (fileExtension === "shp") {
+        return `INSTALL spatial; LOAD spatial; COPY "${table}" TO '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
+    }
     else {
         throw new Error(`Unknown extension ${fileExtension}`);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nshiab/simple-data-analysis-core",
-  "version": "0.0.19",
+  "version": "0.0.21",
   "description": "The core functionalities of the simple-data-analysis library, with only one dependency: DuckDB.",
   "repository": {
     "type": "git",