@nshiab/simple-data-analysis-core 0.0.20 → 0.0.21

package/esm/class/SimpleTable.js

@@ -47,6 +47,7 @@ import normalizeQuery from "../methods/normalizeQuery.js";
 import rollingQuery from "../methods/rollingQuery.js";
 import distanceQuery from "../methods/distanceQuery.js";
 import getGeoData from "../methods/getGeoData.js";
+import writeGeoData from "../helpers/writeGeoData.js";
 import splitSpread from "../methods/splitSpread.js";
 import { readdirSync } from "node:fs";
 import stringToArray from "../helpers/stringToArray.js";
@@ -54,24 +55,20 @@ import loadDataQuery from "../methods/loadDataQuery.js";
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
 import writeDataQuery from "../methods/writeDataQuery.js";
-import writeGeoDataQuery from "../methods/writeGeoDataQuery.js";
 import runQuery from "../helpers/runQuery.js";
 import aggregateGeoQuery from "../methods/aggregateGeoQuery.js";
 import summarize from "../methods/summarize.js";
 import correlations from "../methods/correlations.js";
 import linearRegressions from "../methods/linearRegressions.js";
 import joinGeo from "../methods/joinGeo.js";
-import shouldFlipBeforeExport from "../helpers/shouldFlipBeforeExport.js";
 import getProjection from "../helpers/getProjection.js";
 import cache from "../methods/cache.js";
 import camelCase from "../helpers/camelCase.js";
 import formatNumber from "../helpers/formatNumber.js";
 import createDirectory from "../helpers/createDirectory.js";
-import rewind from "../helpers/rewind.js";
 import writeDataAsArrays from "../helpers/writeDataAsArrays.js";
 import logData from "../helpers/logData.js";
 import fill from "../methods/fill.js";
-import { readFileSync, writeFileSync } from "node:fs";
 import loadArray from "../methods/loadArray.js";
 import cleanPath from "../helpers/cleanPath.js";
 import Simple from "./Simple.js";
@@ -88,10 +85,9 @@ import capitalizeQuery from "../methods/capitalizeQuery.js";
 import truncateQuery from "../methods/truncateQuery.js";
 import padQuery from "../methods/padQuery.js";
 import getProjectionParquet from "../helpers/getProjectionParquet.js";
+import hasGeometryColumn from "../helpers/hasGeometryColumn.js";
 import unifyColumns from "../helpers/unifyColumns.js";
 import accumulateQuery from "../helpers/accumulateQuery.js";
-import stringifyDates from "../helpers/stringifyDates.js";
-import stringifyDatesInvert from "../helpers/stringifyDatesInvert.js";
 import unnestQuery from "../helpers/unnestQuery.js";
 import nestQuery from "../helpers/nestQuery.js";
 import concatenateRowQuery from "../helpers/concatenateRowQuery.js";
@@ -422,7 +418,13 @@ export default class SimpleTable extends Simple {
      *
      * @example
      * ```ts
-     * // Load geospatial data from a shapefile and reproject to WGS84
+     * // Load geospatial data from a shapefile (with relevant files in the same folder) and reproject to WGS84
+     * await table.loadGeoData("./some-data/some-data.shp", { toWGS84: true });
+     * ```
+     *
+     * @example
+     * ```ts
+     * // Load geospatial data from a zipped shapefile and reproject to WGS84
      * await table.loadGeoData("./some-data.shp.zip", { toWGS84: true });
      * ```
      */
@@ -1903,48 +1905,57 @@ export default class SimpleTable extends Simple {
      * @param rightTable - The SimpleTable instance to be joined with this table.
      * @param leftColumn - The name of the column in this (left) table containing the text to compare.
      * @param rightColumn - The name of the column in the right table containing the text to compare.
+     * @param threshold - The minimum similarity score (0–100) required for two rows to be joined. For `method: "ratio"`, a length-based pre-filter is automatically applied based on the threshold to improve performance without losing accuracy.
      * @param options - An optional object with configuration options:
      * @param options.method - The rapidfuzz similarity algorithm to use. Defaults to `"ratio"`.
      *   - `"ratio"`: Overall similarity (Levenshtein-based).
      *   - `"partial_ratio"`: Best partial/substring similarity.
      *   - `"token_sort_ratio"`: Similarity after sorting tokens (words), useful for reordered words.
      *   - `"token_set_ratio"`: Similarity based on sets of tokens, ignoring duplicates and word order.
-     * @param options.threshold - The minimum similarity score (0–100) required for two rows to be joined. Defaults to `80`.
      * @param options.similarityColumn - If provided, a column with this name is added to the result containing the similarity score (0–100). If omitted, the score is not included in the output.
      * @param options.outputTable - If `true`, the results will be stored in a new table with a generated name. If a string, it will be used as the name for the new table. If `false` or omitted, the current table will be overwritten. Defaults to `false`.
+     * @param options.preFilterPrefixLen - An optional prefix length. Only strings sharing the same first N characters are compared. Note that prefix filtering is lossy (e.g. "John" vs. "Phon" will not match despite high similarity).
      * @returns A promise that resolves to a table instance containing the fuzzy-joined data (either the modified current table or a new table).
      * @category Table Operations
      *
      * @example
      * ```ts
-     * // Fuzzy left join tableA with tableB on 'name' (left) and 'standardName' (right) (ratio >= 80)
-     * await tableA.fuzzyJoin(tableB, "name", "standardName");
+     * // Fuzzy left join tableA with tableB on 'name' (left) and 'standardName' (right) with a threshold of 80
+     * // A length-based pre-filter is automatically applied.
+     * await tableA.fuzzyJoin(tableB, "name", "standardName", 80);
+     * ```
+     *
+     * @example
+     * ```ts
+     * // Fuzzy join with a prefix-based pre-filter and a threshold of 80
+     * await tableA.fuzzyJoin(tableB, "name", "standardName", 80, {
+     *   preFilterPrefixLen: 3, // Must share the same first 3 characters
+     * });
      * ```
      *
      * @example
      * ```ts
      * // Fuzzy join with a custom threshold and method, storing results in a new table
-     * const tableC = await tableA.fuzzyJoin(tableB, "name", "standardName", {
+     * const tableC = await tableA.fuzzyJoin(tableB, "name", "standardName", 90, {
      *   method: "token_sort_ratio",
-     *   threshold: 90,
      *   outputTable: "tableC",
      * });
      * ```
      *
      * @example
      * ```ts
-     * // Fuzzy join with a custom similarity column name
-     * await tableA.fuzzyJoin(tableB, "name", "standardName", {
+     * // Fuzzy join with a custom similarity column name and a threshold of 80
+     * await tableA.fuzzyJoin(tableB, "name", "standardName", 80, {
      *   similarityColumn: "matchScore",
      * });
      * ```
      */
-    async fuzzyJoin(rightTable, leftColumn, rightColumn, options = {}) {
+    async fuzzyJoin(rightTable, leftColumn, rightColumn, threshold, options = {}) {
         if (options.outputTable === true) {
             options.outputTable = `table${this.sdb.tableIncrement}`;
             this.sdb.tableIncrement += 1;
         }
-        return await fuzzyJoin(this, rightTable, leftColumn, rightColumn, options);
+        return await fuzzyJoin(this, rightTable, leftColumn, rightColumn, threshold, options);
     }
     /**
      * Normalizes string values in a column by detecting fuzzy duplicates and replacing them with a single canonical value.
@@ -1958,42 +1969,52 @@ export default class SimpleTable extends Simple {
      *
      * @param column - The name of the column containing the strings to normalize.
      * @param newColumn - The name of the column to write the normalized values to. Use the same name as `column` to normalize in-place.
+     * @param threshold - The minimum similarity score (0–100) for two strings to be considered duplicates. For `method: "ratio"`, a length-based pre-filter is automatically applied based on the threshold to improve performance without losing accuracy.
      * @param options - An optional object with configuration options:
      * @param options.method - The rapidfuzz similarity algorithm to use. Defaults to `"ratio"`.
      *   - `"ratio"`: Overall similarity.
      *   - `"partial_ratio"`: Best partial/substring similarity.
      *   - `"token_sort_ratio"`: Similarity after sorting tokens (words), useful for reordered words.
      *   - `"token_set_ratio"`: Similarity based on sets of tokens, ignoring duplicates and word order.
-     * @param options.threshold - The minimum similarity score (0–100) for two strings to be considered duplicates. Defaults to `80`.
      * @param options.keep - The strategy for choosing the canonical value within each cluster of similar strings. Defaults to `"mostCommon"`.
      *   - `"mostCommon"`: Keep the value that appears most frequently in the original column.
      *   - `"longestString"`: Keep the longest string in the cluster.
      *   - `"shortestString"`: Keep the shortest string in the cluster.
      *   - `"mostCentral"`: Keep the string with the highest total similarity score to all other cluster members (the most "central" string).
      *   - `"maxScore"`: Keep the string that participates in the single highest-scoring pairwise match within the cluster.
+     * @param options.preFilterPrefixLen - An optional prefix length. Only strings sharing the same first N characters are compared. Note that prefix filtering is lossy (e.g. "John" vs. "Phon" will not match despite high similarity).
      * @returns A promise that resolves when the column has been normalized.
      * @category Updating Data
      *
      * @example
      * ```ts
-     * // Normalize 'city' into a new 'cityClean' column, keeping the most common string per cluster
-     * await table.fuzzyClean("city", "cityClean");
+     * // Normalize 'city' into a new 'cityClean' column, keeping the most common string per cluster with a threshold of 80
+     * // A length-based pre-filter is automatically applied.
+     * await table.fuzzyClean("city", "cityClean", 80);
      * ```
      *
      * @example
      * ```ts
-     * // Normalize 'companyName' into a new column using token_sort_ratio and a stricter threshold
-     * await table.fuzzyClean("companyName", "companyNameClean", { method: "token_sort_ratio", threshold: 90 });
+     * // Normalize with a prefix-based pre-filter and a threshold of 80
+     * await table.fuzzyClean("city", "cityClean", 80, {
+     *   preFilterPrefixLen: 5, // Must share the same first 5 characters
+     * });
      * ```
      *
      * @example
      * ```ts
-     * // Normalize 'category' in-place, keeping the longest string in each cluster
-     * await table.fuzzyClean("category", "category", { keep: "longestString" });
+     * // Normalize 'companyName' into a new column using token_sort_ratio and a threshold of 90
+     * await table.fuzzyClean("companyName", "companyNameClean", 90, { method: "token_sort_ratio" });
+     * ```
+     *
+     * @example
+     * ```ts
+     * // Normalize 'category' in-place, keeping the longest string in each cluster and a threshold of 80
+     * await table.fuzzyClean("category", "category", 80, { keep: "longestString" });
      * ```
      */
-    async fuzzyClean(column, newColumn, options = {}) {
-        await fuzzyClean(this, column, newColumn, options);
+    async fuzzyClean(column, newColumn, threshold, options = {}) {
+        await fuzzyClean(this, column, newColumn, threshold, options);
     }
     /**
      * Replaces specified strings in the selected columns.
@@ -3925,6 +3946,9 @@ export default class SimpleTable extends Simple {
      * ```
      */
     async getData(options = {}) {
+        if (await hasGeometryColumn(this)) {
+            throw new Error("Table contains geometry columns. Use getGeoData() instead.");
+        }
         const columns = options.columns
             ? (typeof options.columns === "string"
                 ? [options.columns]
@@ -3989,8 +4013,9 @@ export default class SimpleTable extends Simple {
      * ```
      */
     async points(columnLat, columnLon, newColumn) {
-        await queryDB(this, `INSTALL spatial; LOAD spatial;
-            ALTER TABLE "${this.name}" ADD COLUMN "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Point2D("${columnLat}", "${columnLon}")`, mergeOptions(this, {
+        await queryDB(this, (await this.getColumns()).includes(newColumn)
+            ? `INSTALL spatial; LOAD spatial; UPDATE "${this.name}" SET "${newColumn}" = ST_Point2D("${columnLat}", "${columnLon}")`
+            : `INSTALL spatial; LOAD spatial; ALTER TABLE "${this.name}" ADD COLUMN "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Point2D("${columnLat}", "${columnLon}")`, mergeOptions(this, {
             table: this.name,
             method: "points()",
             parameters: { columnLat, columnLon, newColumn },
@@ -4406,7 +4431,9 @@ export default class SimpleTable extends Simple {
         const column = typeof options.column === "string"
             ? options.column
             : await findGeoColumn(this);
-        await queryDB(this, `ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" =  ST_Buffer("${column}", ${distance});`, mergeOptions(this, {
+        await queryDB(this, (await this.getColumns()).includes(newColumn)
+            ? `INSTALL spatial; LOAD spatial; UPDATE "${this.name}" SET "${newColumn}" = ST_Buffer("${column}", ${distance})`
+            : `INSTALL spatial; LOAD spatial; ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Buffer("${column}", ${distance})`, mergeOptions(this, {
             table: this.name,
             method: "buffer()",
             parameters: { column, newColumn, distance },
@@ -4492,7 +4519,9 @@ export default class SimpleTable extends Simple {
         if (this.projections[column1] !== this.projections[column2]) {
             throw new Error(`${column1} and ${column2} don't have the same projection.\n${column1}: ${this.projections[column1]}\n${column2}: ${this.projections[column2]}`);
         }
-        await queryDB(this, `ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Intersection("${column1}", "${column2}")`, mergeOptions(this, {
+        await queryDB(this, (await this.getColumns()).includes(newColumn)
+            ? `INSTALL spatial; LOAD spatial; UPDATE "${this.name}" SET "${newColumn}" = ST_Intersection("${column1}", "${column2}")`
+            : `INSTALL spatial; LOAD spatial; ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Intersection("${column1}", "${column2}")`, mergeOptions(this, {
             table: this.name,
             method: "intersection()",
             parameters: { column1, column2, newColumn },
@@ -4617,7 +4646,9 @@ export default class SimpleTable extends Simple {
         if (this.projections[column1] !== this.projections[column2]) {
             throw new Error(`${column1} and ${column2} don't have the same projection.\n${column1}: ${this.projections[column1]}\n${column2}: ${this.projections[column2]}`);
         }
-        await queryDB(this, `ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Union("${column1}", "${column2}")`, mergeOptions(this, {
+        await queryDB(this, (await this.getColumns()).includes(newColumn)
+            ? `INSTALL spatial; LOAD spatial; UPDATE "${this.name}" SET "${newColumn}" = ST_Union("${column1}", "${column2}")`
+            : `INSTALL spatial; LOAD spatial; ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Union("${column1}", "${column2}")`, mergeOptions(this, {
             table: this.name,
             method: "union()",
             parameters: { column1, column2, newColumn },
@@ -4708,7 +4739,9 @@ export default class SimpleTable extends Simple {
         const column = typeof options.column === "string"
             ? options.column
             : await findGeoColumn(this);
-        await queryDB(this, `ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" =  ST_Centroid("${column}");`, mergeOptions(this, {
+        await queryDB(this, (await this.getColumns()).includes(newColumn)
+            ? `INSTALL spatial; LOAD spatial; UPDATE "${this.name}" SET "${newColumn}" = ST_Centroid("${column}")`
+            : `INSTALL spatial; LOAD spatial; ALTER TABLE "${this.name}" ADD "${newColumn}" GEOMETRY; UPDATE "${this.name}" SET "${newColumn}" = ST_Centroid("${column}")`, mergeOptions(this, {
             table: this.name,
             method: "centroid()",
             parameters: { column, newColumn },
@@ -4993,6 +5026,9 @@ export default class SimpleTable extends Simple {
      * ```
      */
     async writeData(file, options = {}) {
+        if (await hasGeometryColumn(this)) {
+            throw new Error("Table contains geometry columns. Use writeGeoData() instead.");
+        }
         createDirectory(file);
         const extension = getExtension(file);
         if (options.dataAsArrays) {
@@ -5007,12 +5043,12 @@ export default class SimpleTable extends Simple {
         }
     }
     /**
-     * Writes the table's geospatial data to a file in GeoJSON or GeoParquet format.
+     * Writes the table's geospatial data to a file in GeoJSON, GeoParquet, or Shapefile format.
      * If the specified path does not exist, it will be created.
      *
      * For GeoJSON files (`.geojson` or `.json`), if the projection is WGS84 or EPSG:4326 (`[latitude, longitude]` axis order), the coordinates will be flipped to follow the RFC7946 standard (`[longitude, latitude]` axis order) in the output.
      *
-     * @param file - The absolute path to the output file (e.g., `"./output.geojson"`, `"./output.geoparquet"`).
+     * @param file - The absolute path to the output file (e.g., `"./output.geojson"`, `"./output.geoparquet"`, `"./shapefile-folder/output.shp"`).
      * @param options - An optional object with configuration options:
      * @param options.precision - For GeoJSON, the maximum number of figures after the decimal separator to write in coordinates. Defaults to `undefined` (full precision).
      * @param options.compression - For GeoParquet, if `true`, the output will be ZSTD compressed. Defaults to `false`.
@@ -5036,6 +5072,12 @@ export default class SimpleTable extends Simple {
      *
      * @example
      * ```ts
+     * // Write geospatial data to a Shapefile with all relevant files  in the same folder
+     * await table.writeGeoData("./shapefile-folder/output.shp");
+     * ```
+     *
+     * @example
+     * ```ts
      * // Write GeoJSON with specific precision and metadata
      * await table.writeGeoData("./output_high_precision.geojson", {
      *   precision: 6,
@@ -5044,71 +5086,7 @@ export default class SimpleTable extends Simple {
      * ```
      */
     async writeGeoData(file, options = {}) {
-        createDirectory(file);
-        const fileExtension = getExtension(file);
-        if (fileExtension === "geojson" || fileExtension === "json") {
-            let types;
-            if (options.formatDates === true) {
-                types = await this.getTypes();
-                if (Object.values(types).includes("DATE") ||
-                    Object.values(types).includes("TIMESTAMP")) {
-                    await stringifyDates(this, types);
-                }
-            }
-            if (typeof options.compression === "boolean") {
-                throw new Error("The compression option is not supported for writing GeoJSON files.");
-            }
-            const geoColumn = await findGeoColumn(this);
-            const flip = shouldFlipBeforeExport(this.projections[geoColumn]);
-            if (flip) {
-                await this.flipCoordinates(geoColumn);
-                await queryDB(this, writeGeoDataQuery(this.name, file, fileExtension, options), mergeOptions(this, {
-                    table: this.name,
-                    method: "writeGeoData()",
-                    parameters: { file, options },
-                }));
-                await this.flipCoordinates(geoColumn);
-            }
-            else {
-                await queryDB(this, writeGeoDataQuery(this.name, file, fileExtension, options), mergeOptions(this, {
-                    table: this.name,
-                    method: "writeGeoData()",
-                    parameters: { file, options },
-                }));
-            }
-            if (options.metadata) {
-                const fileData = JSON.parse(readFileSync(file, "utf-8"));
-                fileData.metadata = options.metadata;
-                writeFileSync(file, JSON.stringify(fileData));
-            }
-            if (options.rewind) {
-                const fileData = JSON.parse(readFileSync(file, "utf-8"));
-                const fileRewinded = rewind(fileData);
-                writeFileSync(file, JSON.stringify(fileRewinded));
-            }
-            if (types && (Object.values(types).includes("DATE") ||
-                Object.values(types).includes("TIMESTAMP"))) {
-                await stringifyDatesInvert(this, types);
-            }
-        }
-        else if (fileExtension === "geoparquet") {
-            if (typeof options.precision === "number") {
-                throw new Error("The precision option is not supported for writing PARQUET files. Use the .reducePrecision() method.");
-            }
-            if (typeof options.rewind === "boolean") {
-                throw new Error("The rewind option is not supported for writing PARQUET files.");
-            }
-            await queryDB(this, `COPY "${this.name}" TO '${cleanPath(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""}, KV_METADATA {
-             projections: '${JSON.stringify(this.projections)}'
-        });`, mergeOptions(this, {
-                table: this.name,
-                method: "writeGeoData()",
-                parameters: { file, options },
-            }));
-        }
-        else {
-            throw new Error(`Unknown extension ${fileExtension}`);
-        }
+        await writeGeoData(this, file, options);
     }
     /**
      * Caches the results of computations in `./.sda-cache`.

package/esm/helpers/findGeoColumn.js

@@ -3,7 +3,7 @@ export default async function findGeoColumn(SimpleTable) {
     const types = await SimpleTable.getTypes();
     const geometries = Object.values(types).filter((d) => d.toLowerCase() === "geometry");
     if (geometries.length === 0) {
-        throw new Error("No column storing geometries");
+        throw new Error("Table contains no geometry columns.");
     }
     else if (geometries.length > 1) {
         throw new Error("More than one column storing geometries. If the method allows to specify one, do it. Otherwise, use the selectColumns methods beforehand.");

package/esm/helpers/getProjection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"getProjection.d.ts","sourceRoot":"","sources":["../../src/helpers/getProjection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAGjD,wBAA8B,aAAa,CACzC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,mBA8Bb"}
+{"version":3,"file":"getProjection.d.ts","sourceRoot":"","sources":["../../src/helpers/getProjection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAGjD,wBAA8B,aAAa,CACzC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,mBAiCb"}

package/esm/helpers/getProjection.js

@@ -15,6 +15,9 @@ export default async function getProjection(simpleDB, file) {
         throw new Error("No queryResults");
     }
     const proj4 = queryResult[0].proj4;
+    if (proj4 === null) {
+        return "UNKNOWN";
+    }
     if (typeof proj4 !== "string") {
         throw new Error(`Expected proj4 to be a string, got ${typeof proj4}`);
     }

package/esm/helpers/hasGeometryColumn.d.ts

@@ -0,0 +1,20 @@
+import type SimpleTable from "../class/SimpleTable.js";
+/**
+ * Returns `true` if the table has one or more columns of type geometry.
+ *
+ * Uses `getTypes()` to inspect column types and checks for any column
+ * whose type normalizes to `"geometry"`.
+ *
+ * @param table - The SimpleTable instance to inspect.
+ * @returns `true` if at least one geometry column exists, `false` otherwise.
+ *
+ * @example
+ * ```ts
+ * const hasGeo = await hasGeometryColumn(table);
+ * if (hasGeo) {
+ *   console.log("This table contains geometry columns");
+ * }
+ * ```
+ */
+export default function hasGeometryColumn(table: SimpleTable): Promise<boolean>;
+//# sourceMappingURL=hasGeometryColumn.d.ts.map

package/esm/helpers/hasGeometryColumn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hasGeometryColumn.d.ts","sourceRoot":"","sources":["../../src/helpers/hasGeometryColumn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAA8B,iBAAiB,CAC7C,KAAK,EAAE,WAAW,GACjB,OAAO,CAAC,OAAO,CAAC,CAGlB"}

package/esm/helpers/hasGeometryColumn.js

@@ -0,0 +1,21 @@
+/**
+ * Returns `true` if the table has one or more columns of type geometry.
+ *
+ * Uses `getTypes()` to inspect column types and checks for any column
+ * whose type normalizes to `"geometry"`.
+ *
+ * @param table - The SimpleTable instance to inspect.
+ * @returns `true` if at least one geometry column exists, `false` otherwise.
+ *
+ * @example
+ * ```ts
+ * const hasGeo = await hasGeometryColumn(table);
+ * if (hasGeo) {
+ *   console.log("This table contains geometry columns");
+ * }
+ * ```
+ */
+export default async function hasGeometryColumn(table) {
+    const types = await table.getTypes();
+    return Object.values(types).some((t) => t.toLowerCase() === "geometry");
+}

package/esm/helpers/writeGeoData.d.ts

@@ -0,0 +1,9 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function writeGeoData(table: SimpleTable, file: string, options?: {
+    precision?: number;
+    compression?: boolean;
+    rewind?: boolean;
+    metadata?: unknown;
+    formatDates?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=writeGeoData.d.ts.map

package/esm/helpers/writeGeoData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writeGeoData.d.ts","sourceRoot":"","sources":["../../src/helpers/writeGeoData.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CAClB,GACL,OAAO,CAAC,IAAI,CAAC,CAqIf"}

package/esm/helpers/writeGeoData.js

@@ -0,0 +1,110 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import createDirectory from "./createDirectory.js";
+import getExtension from "./getExtension.js";
+import findGeoColumn from "./findGeoColumn.js";
+import hasGeometryColumn from "./hasGeometryColumn.js";
+import shouldFlipBeforeExport from "./shouldFlipBeforeExport.js";
+import queryDB from "./queryDB.js";
+import writeGeoDataQuery from "../methods/writeGeoDataQuery.js";
+import mergeOptions from "./mergeOptions.js";
+import rewind from "./rewind.js";
+import stringifyDates from "./stringifyDates.js";
+import stringifyDatesInvert from "./stringifyDatesInvert.js";
+import cleanPath from "./cleanPath.js";
+export default async function writeGeoData(table, file, options = {}) {
+    createDirectory(file);
+    if (!(await hasGeometryColumn(table))) {
+        throw new Error("Table contains no geometry columns. Use writeData() instead.");
+    }
+    const fileExtension = getExtension(file);
+    if (fileExtension === "geojson" || fileExtension === "json") {
+        let types;
+        if (options.formatDates === true) {
+            types = await table.getTypes();
+            if (Object.values(types).includes("DATE") ||
+                Object.values(types).includes("TIMESTAMP")) {
+                await stringifyDates(table, types);
+            }
+        }
+        if (typeof options.compression === "boolean") {
+            throw new Error("The compression option is not supported for writing GeoJSON files.");
+        }
+        const geoColumn = await findGeoColumn(table);
+        const flip = shouldFlipBeforeExport(table.projections[geoColumn]);
+        if (flip) {
+            await table.flipCoordinates(geoColumn);
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+            await table.flipCoordinates(geoColumn);
+        }
+        else {
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+        }
+        if (options.metadata) {
+            const fileData = JSON.parse(readFileSync(file, "utf-8"));
+            fileData.metadata = options.metadata;
+            writeFileSync(file, JSON.stringify(fileData));
+        }
+        if (options.rewind) {
+            const fileData = JSON.parse(readFileSync(file, "utf-8"));
+            const fileRewinded = rewind(fileData);
+            writeFileSync(file, JSON.stringify(fileRewinded));
+        }
+        if (types && (Object.values(types).includes("DATE") ||
+            Object.values(types).includes("TIMESTAMP"))) {
+            await stringifyDatesInvert(table, types);
+        }
+    }
+    else if (fileExtension === "shp") {
+        if (typeof options.precision === "number" ||
+            typeof options.compression === "boolean" ||
+            typeof options.rewind === "boolean" ||
+            options.metadata ||
+            options.formatDates === true) {
+            throw new Error("The following options are not supported for writing SHAPEFILE files: precision, compression, rewind, metadata, and formatDates.");
+        }
+        const geoColumn = await findGeoColumn(table);
+        const flip = shouldFlipBeforeExport(table.projections[geoColumn]);
+        if (flip) {
+            await table.flipCoordinates(geoColumn);
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+            await table.flipCoordinates(geoColumn);
+        }
+        else {
+            await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
+                table: table.name,
+                method: "writeGeoData()",
+                parameters: { file, options },
+            }));
+        }
+    }
+    else if (fileExtension === "geoparquet") {
+        if (typeof options.precision === "number") {
+            throw new Error("The precision option is not supported for writing PARQUET files. Use the .reducePrecision() method.");
+        }
+        if (typeof options.rewind === "boolean") {
+            throw new Error("The rewind option is not supported for writing PARQUET files.");
+        }
+        await queryDB(table, `COPY "${table.name}" TO '${cleanPath(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""}, KV_METADATA {
+             projections: '${JSON.stringify(table.projections)}'
+        });`, mergeOptions(table, {
+            table: table.name,
+            method: "writeGeoData()",
+            parameters: { file, options },
+        }));
+    }
+    else {
+        throw new Error(`Unknown extension ${fileExtension}`);
+    }
+}

package/esm/methods/fuzzyClean.d.ts

@@ -1,7 +1,7 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, options?: {
+export default function fuzzyClean(table: SimpleTable, column: string, newColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     keep?: "mostCommon" | "longestString" | "shortestString" | "mostCentral" | "maxScore";
+    preFilterPrefixLen?: number;
 }): Promise<void>;
 //# sourceMappingURL=fuzzyClean.d.ts.map

package/esm/methods/fuzzyClean.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;CACX,GACL,OAAO,CAAC,IAAI,CAAC,CAyNf"}
+{"version":3,"file":"fuzzyClean.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyClean.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,UAAU,CACtC,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,IAAI,CAAC,EACD,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,aAAa,GACb,UAAU,CAAC;IACf,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,GACL,OAAO,CAAC,IAAI,CAAC,CAmPf"}

package/esm/methods/fuzzyClean.js

@@ -1,9 +1,18 @@
 import mergeOptions from "../helpers/mergeOptions.js";
 import queryDB from "../helpers/queryDB.js";
-export default async function fuzzyClean(table, column, newColumn, options = {}) {
+export default async function fuzzyClean(table, column, newColumn, threshold, options = {}) {
     const method = options.method ?? "ratio";
-    const threshold = options.threshold ?? 80;
     const keep = options.keep ?? "mostCommon";
+    let onClause = `rapidfuzz_${method}(a.value, b.value) >= ${threshold}`;
+    if (method === "ratio") {
+        const maxDiffMultiplier = (200 - 2 * threshold) / (200 - threshold);
+        onClause +=
+            ` AND ABS(LENGTH(a.value) - LENGTH(b.value)) <= ${maxDiffMultiplier} * GREATEST(LENGTH(a.value), LENGTH(b.value))`;
+    }
+    if (options.preFilterPrefixLen !== undefined) {
+        onClause +=
+            ` AND SUBSTR(a.value, 1, ${options.preFilterPrefixLen}) = SUBSTR(b.value, 1, ${options.preFilterPrefixLen})`;
+    }
     // Single round trip: compute fuzzy pairs and embed counts for both sides.
     // Only values that appear in at least one pair above the threshold can be
     // normalized — singletons need no processing at all.
@@ -22,16 +31,26 @@ export default async function fuzzyClean(table, column, newColumn, options = {})
        rapidfuzz_${method}(a.value, b.value) AS score
      FROM uniques a
      JOIN uniques b
-       ON rapidfuzz_${method}(a.value, b.value) >= ${threshold}
+       ON ${onClause}
        AND a.value < b.value`, mergeOptions(table, {
         table: table.name,
         method: "fuzzyClean()",
-        parameters: { column, newColumn, options },
+        parameters: { column, newColumn, threshold, options },
         returnDataFrom: "query",
     }));
     const pairs = pairsData ?? [];
-    if (pairs.length === 0)
-        return; // Nothing to normalize.
+    if (pairs.length === 0) {
+        if (newColumn !== column) {
+            await queryDB(table, `ALTER TABLE "${table.name}" ADD "${newColumn}" VARCHAR;
+         UPDATE "${table.name}"
+           SET "${newColumn}" = "${column}";`, mergeOptions(table, {
+                table: table.name,
+                method: "fuzzyClean()",
+                parameters: { column, newColumn, threshold, options },
+            }));
+        }
+        return;
+    }
     // Build count map from the pairs — no separate query needed.
     const countMap = new Map();
     for (const { left_value, left_cnt, right_value, right_cnt } of pairs) {

package/esm/methods/fuzzyJoin.d.ts

@@ -1,8 +1,8 @@
 import type SimpleTable from "../class/SimpleTable.js";
-export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, options?: {
+export default function fuzzyJoin(leftTable: SimpleTable, rightTable: SimpleTable, leftColumn: string, rightColumn: string, threshold: number, options?: {
     method?: "ratio" | "partial_ratio" | "token_sort_ratio" | "token_set_ratio";
-    threshold?: number;
     similarityColumn?: string;
     outputTable?: string | boolean;
+    preFilterPrefixLen?: number;
 }): Promise<SimpleTable>;
 //# sourceMappingURL=fuzzyJoin.d.ts.map