@nshiab/simple-data-analysis-core 2.0.5 → 2.0.6

package/script/methods/loadArray.js

@@ -7,6 +7,9 @@ exports.default = loadArray;
 exports.prepareArray = prepareArray;
 exports.executePreparedArray = executePreparedArray;
 const node_api_1 = require("@duckdb/node-api");
+const prepareGeometry_js_1 = __importDefault(require("../helpers/prepareGeometry.js"));
+const prepareJSON_js_1 = __importDefault(require("../helpers/prepareJSON.js"));
+const geometryFromJSON_js_1 = __importDefault(require("../helpers/geometryFromJSON.js"));
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const parseTypes_js_1 = __importDefault(require("../helpers/parseTypes.js"));
 const parseDuckDBType_js_1 = __importDefault(require("../helpers/parseDuckDBType.js"));
@@ -16,7 +19,7 @@ function loadArray(simpleTable, rows, options = {}) {
     if (rows.length === 0) {
         throw new Error("The array is empty. loadArray needs at least one object to infer the column types.");
     }
-    const prepared = prepareArray(rows, options.columnTypes);
+    const prepared = prepareArray(rows, options.columnTypes, true);
     (0, queueOp_js_1.default)(simpleTable, {
         kind: "barrier",
         method: "loadArray()",
@@ -24,17 +27,26 @@ function loadArray(simpleTable, rows, options = {}) {
         execute: () => executePreparedArray(simpleTable, prepared),
     });
 }
-function prepareArray(rows, columnTypes = {}) {
+function prepareArray(rows, columnTypes = {}, requireExplicitNestedTypes = false) {
     const keys = Object.keys(rows[0]);
     const overrides = new Map();
     for (const [key, type] of Object.entries(columnTypes)) {
         if (!keys.includes(key)) {
             throw new Error(`Unknown column ${JSON.stringify(key)} in loadArray columnTypes.`);
         }
+        if (type.toUpperCase() === "JSON") {
+            overrides.set(key, "JSON");
+            continue;
+        }
+        if (/^geometry/i.test(type)) {
+            if (!/^geometry\('EPSG:4326'\)$/i.test(type)) {
+                throw new Error(`Unsupported type ${JSON.stringify(type)} for column ${JSON.stringify(key)}. GeoJSON ingestion requires GEOMETRY('EPSG:4326'); reproject input to WGS84 first.`);
+            }
+            overrides.set(key, "GEOMETRY('EPSG:4326')");
+            continue;
+        }
         try {
-            const normalized = /^FLOAT\[[1-9]\d*\]$/i.test(type)
-                ? type.toUpperCase()
-                : (0, parseTypes_js_1.default)(type);
+            const normalized = (0, parseTypes_js_1.default)(type);
             (0, parseDuckDBType_js_1.default)(normalized);
             overrides.set(key, normalized);
         }
@@ -56,6 +68,14 @@ function prepareArray(rows, columnTypes = {}) {
             }
             continue;
         }
+        if (requireExplicitNestedTypes) {
+            for (let j = 0; j < rows.length; j++) {
+                const cell = rows[j][key];
+                if (cell !== null && typeof cell === "object" && !(cell instanceof Date)) {
+                    throw new Error(`Column ${JSON.stringify(key)}, row ${j + 1}: array and object cells require an explicit columnTypes entry. Use JSON for nested data, FLOAT[n] for fixed-size vectors, or GEOMETRY('EPSG:4326') for GeoJSON geometries.`);
+                }
+            }
+        }
         const value = firstNonNullValue[i];
         const type = typeof value;
         if (type === "symbol" || type === "function") {
@@ -112,24 +132,59 @@ async function executePreparedArray(simpleTable, prepared) {
         simpleTable.connection = simpleTable.sdb.connection;
     }
     const { keys, types, columnsData, rowCount } = prepared;
-    await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${types[i]}`).join(", ")})`);
-    const appender = await simpleTable.connection
-        .createAppender(simpleTable.name);
-    const duckDBTypes = types.map((d) => (0, parseDuckDBType_js_1.default)(d));
-    // The maximum capacity of a DuckDB data chunk is 2048 rows.
-    const chunkSize = 2000;
-    for (let start = 0; start < rowCount; start += chunkSize) {
-        const end = Math.min(start + chunkSize, rowCount);
-        const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
-        for (let i = 0; i < keys.length; i++) {
-            dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
-        }
-        appender.appendDataChunk(dataChunk);
+    const hasGeometry = types.includes("GEOMETRY('EPSG:4326')");
+    const needsConversion = hasGeometry || types.includes("JSON");
+    const staged = needsConversion
+        ? `__sda_array_${crypto.randomUUID().replaceAll("-", "")}`
+        : simpleTable.name;
+    const storageTypes = types.map((type) => type.startsWith("GEOMETRY") || type === "JSON" ? "VARCHAR" : type);
+    if (hasGeometry && !simpleTable.sdb.spatialLoaded) {
+        await simpleTable.sdb.customQuery("INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;");
+        simpleTable.sdb.spatialLoaded = true;
+    }
+    try {
+        await simpleTable.sdb.customQuery(`CREATE ${needsConversion ? "TEMP" : "OR REPLACE"} TABLE ${(0, quoteIdentifier_js_1.default)(staged)}(${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${storageTypes[i]}`).join(", ")})`);
+        const appender = await simpleTable.connection
+            .createAppender(staged);
+        try {
+            const duckDBTypes = storageTypes.map((d) => (0, parseDuckDBType_js_1.default)(d));
+            // The maximum capacity of a DuckDB data chunk is 2048 rows.
+            const chunkSize = 2000;
+            for (let start = 0; start < rowCount; start += chunkSize) {
+                const end = Math.min(start + chunkSize, rowCount);
+                const dataChunk = node_api_1.DuckDBDataChunk.create(duckDBTypes, end - start);
+                for (let i = 0; i < keys.length; i++) {
+                    dataChunk.setColumnValues(i, columnsData[i].slice(start, end));
+                }
+                appender.appendDataChunk(dataChunk);
+            }
+            appender.flushSync();
+        }
+        finally {
+            appender.closeSync();
+        }
+        if (!needsConversion)
+            return;
+        // CTAS is atomic: conversion must finish before the destination is replaced.
+        await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} AS SELECT ${keys.map((key, i) => types[i].startsWith("GEOMETRY")
+            ? `${(0, geometryFromJSON_js_1.default)((0, quoteIdentifier_js_1.default)(key))} AS ${(0, quoteIdentifier_js_1.default)(key)}`
+            : types[i] === "JSON"
+                ? `${(0, quoteIdentifier_js_1.default)(key)}::JSON AS ${(0, quoteIdentifier_js_1.default)(key)}`
+                : (0, quoteIdentifier_js_1.default)(key)).join(", ")} FROM ${(0, quoteIdentifier_js_1.default)(staged)}`);
+    }
+    finally {
+        if (needsConversion) {
+            await simpleTable.sdb.customQuery(`DROP TABLE IF EXISTS ${(0, quoteIdentifier_js_1.default)(staged)}`);
+        }
     }
-    appender.flushSync();
 }
 /** Validates and snapshots a value for an explicitly declared destination type. */
 function prepareTypedValue(value, type, column, row) {
+    if (type === "JSON")
+        return (0, prepareJSON_js_1.default)(value, column, row);
+    if (type === "GEOMETRY('EPSG:4326')") {
+        return (0, prepareGeometry_js_1.default)(value, column, row);
+    }
     if (value === null || value === undefined || Number.isNaN(value))
         return null;
     let compatible = false;

package/script/methods/loadData.js

@@ -54,6 +54,9 @@ function loadDataQuery(table, files, options = {}) {
     return `CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(table)} AS ${loadDataSelect(files, options)};`;
 }
 function loadDataSelect(files, options = {}) {
+    if (Object.values(options.columnTypes ?? {}).some((type) => /^\s*geometry\b/i.test(type))) {
+        throw new Error("Geometry column types are not supported by loadData(). Use loadGeoData() instead.");
+    }
     const fileExtension = (0, getExtension_js_1.default)(files[0]);
     if (options.fileType !== undefined &&
         !["csv", "dsv", "json", "parquet", "excel"].includes(options.fileType)) {

package/script/methods/log.d.ts

@@ -0,0 +1,7 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function log(simpleTable: SimpleTable, options?: "all" | number | {
+    count?: number | "all";
+    types?: boolean;
+    conditions?: string;
+}): Promise<void>;
+//# sourceMappingURL=log.d.ts.map

package/script/methods/log.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../../src/methods/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,GAAG,CAC/B,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE,KAAK,GAAG,MAAM,GAAG;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,iBAyEP"}

package/script/methods/log.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = log;
+const logData_js_1 = __importDefault(require("../helpers/logData.js"));
+const getData_js_1 = __importDefault(require("./getData.js"));
+const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
+const formatNumber_js_1 = __importDefault(require("../helpers/formatNumber.js"));
+async function log(simpleTable, options = {}) {
+    if (simpleTable.connection === undefined) {
+        await simpleTable.sdb.start();
+        simpleTable.db = simpleTable.sdb.db;
+        simpleTable.connection = simpleTable.sdb.connection;
+    }
+    if (simpleTable.connection === undefined) {
+        throw new Error("simpleTable.connection is undefined");
+    }
+    let count;
+    if (typeof options === "number") {
+        count = options;
+    }
+    else if (options === "all") {
+        count = await simpleTable.getRowCount();
+    }
+    else if (typeof options === "object") {
+        if (options.count === "all") {
+            count = await simpleTable.getRowCount();
+        }
+        else if (typeof options.count === "number") {
+            count = options.count;
+        }
+        else {
+            count = simpleTable.rowsToLog;
+        }
+    }
+    else {
+        count = simpleTable.rowsToLog;
+    }
+    const types = typeof options === "object"
+        ? options.types ?? simpleTable.typesToLog
+        : simpleTable.typesToLog;
+    const conditions = typeof options === "object"
+        ? options.conditions ?? undefined
+        : undefined;
+    if (simpleTable.connection === undefined ||
+        !(await simpleTable.sdb.hasTable(simpleTable.name))) {
+        console.log(`\nTable ${simpleTable.name}: no data`);
+    }
+    else {
+        console.log(`\nTable ${simpleTable.name}:`);
+        conditions && console.log(`Conditions: ${conditions}`);
+        const data = await (0, getData_js_1.default)(simpleTable, { limit: count, conditions }, "log()");
+        (0, logData_js_1.default)(types ? await simpleTable.getTypes() : null, data, simpleTable.charsToLog);
+        const rowCount = conditions
+            ? parseInt((await simpleTable.sdb.customQuery(`select count(*) as count from ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} where ${conditions}`, { returnData: true }))[0].count)
+            : await simpleTable.getRowCount();
+        console.log(`${(0, formatNumber_js_1.default)(rowCount)} rows in total ${`(count: ${count}${typeof simpleTable.charsToLog === "number"
+            ? `, charsToLog: ${simpleTable.charsToLog}`
+            : ""})`}`);
+    }
+    return;
+}

package/script/methods/logBottom.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+export default function logBottom(simpleTable: SimpleTable, count?: number, options?: {
+    originalOrder?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=logBottom.d.ts.map

package/script/methods/logBottom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logBottom.d.ts","sourceRoot":"","sources":["../../src/methods/logBottom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,SAAS,CACrC,WAAW,EAAE,WAAW,EACxB,KAAK,CAAC,EAAE,MAAM,EACd,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAO,iBAW1C"}

package/script/methods/logBottom.js

@@ -0,0 +1,15 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = logBottom;
+const logData_js_1 = __importDefault(require("../helpers/logData.js"));
+const getBottom_js_1 = __importDefault(require("./getBottom.js"));
+async function logBottom(simpleTable, count, options = {}) {
+    const rows = count ?? simpleTable.rowsToLog;
+    console.log(`\nTable ${simpleTable.name} (${rows} bottom rows):`);
+    const data = await (0, getBottom_js_1.default)(simpleTable, rows, options, "logBottom()");
+    (0, logData_js_1.default)(null, data, simpleTable.charsToLog);
+    return;
+}

package/script/methods/removeTable.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"removeTable.d.ts","sourceRoot":"","sources":["../../src/methods/removeTable.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,iBAgBzB"}
+{"version":3,"file":"removeTable.d.ts","sourceRoot":"","sources":["../../src/methods/removeTable.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,iBAiBzB"}

package/script/methods/removeTable.js

@@ -4,11 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = removeTable;
+const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const tableRegistry_js_1 = require("../helpers/tableRegistry.js");
 async function removeTable(simpleTable) {
+    (0, cacheTableDependencies_js_1.assertCacheTableMutation)(simpleTable);
     await (0, queryDB_js_1.default)(simpleTable, `DROP TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)};`, (0, mergeOptions_js_1.default)(simpleTable, {
         table: null,
         method: "removeTable()",

package/script/methods/removeTables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"removeTables.d.ts","sourceRoot":"","sources":["../../src/methods/removeTables.ts"],"names":[],"mappings":"AAGA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAIjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBAkCxD"}
+{"version":3,"file":"removeTables.d.ts","sourceRoot":"","sources":["../../src/methods/removeTables.ts"],"names":[],"mappings":"AAIA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAIjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBA2CxD"}

package/script/methods/removeTables.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = removeTables;
+const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
@@ -17,6 +18,12 @@ async function removeTables(simpleDB, tables) {
             ? tables
             : [tables];
     (0, assertSameDatabase_js_1.default)(simpleDB, tablesToBeRemoved.filter((table) => table instanceof SimpleTable_js_1.default), "removeTables()");
+    const names = tablesToBeRemoved.map((table) => (table instanceof SimpleTable_js_1.default ? table.name : table).toLowerCase());
+    for (const table of simpleDB.getTables()) {
+        if (names.includes(table.name.toLowerCase())) {
+            (0, cacheTableDependencies_js_1.assertCacheTableMutation)(table);
+        }
+    }
     await (0, queryDB_js_1.default)(simpleDB, tablesToBeRemoved.map((d) => `DROP TABLE ${(0, quoteIdentifier_js_1.default)(d instanceof SimpleTable_js_1.default ? d.name : d)};`).join("\n"), (0, mergeOptions_js_1.default)(simpleDB, {
         table: null,
         method: "removeTable()",

package/script/methods/renameTable.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"renameTable.d.ts","sourceRoot":"","sources":["../../src/methods/renameTable.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,iBAcb"}
+{"version":3,"file":"renameTable.d.ts","sourceRoot":"","sources":["../../src/methods/renameTable.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,WAAW,CACvC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,iBAgBb"}

package/script/methods/renameTable.js

@@ -4,11 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = renameTable;
+const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
 const tableGeneration_js_1 = require("../helpers/tableGeneration.js");
+const tableRegistry_js_1 = require("../helpers/tableRegistry.js");
 async function renameTable(simpleTable, name) {
+    (0, cacheTableDependencies_js_1.assertCacheTableMutation)(simpleTable);
+    (0, tableRegistry_js_1.assertTableNameAvailable)(simpleTable.sdb, name, simpleTable);
     await (0, queryDB_js_1.default)(simpleTable, `ALTER TABLE ${(0, quoteIdentifier_js_1.default)(simpleTable.name)} RENAME TO ${(0, quoteIdentifier_js_1.default)(name)};`, (0, mergeOptions_js_1.default)(simpleTable, {
         table: simpleTable.name,
         method: "renameTable()",

package/script/methods/rowToText.js

@@ -25,8 +25,8 @@ function rowToText(simpleTable, columns, newColumn) {
                     throw new Error(`The column ${col} is of type ${allTypes[col]}. The rowToText() method only works with string columns. Please convert the column to string first with the .convert() method.`);
                 }
             }
-            const parts = columns.map((col) => `? || COALESCE(${(0, quoteIdentifier_js_1.default)(col)}, ?)`);
-            const concatenatedExpression = parts.join(" || ");
+            const parts = columns.map((col) => `concat(?, COALESCE(${(0, quoteIdentifier_js_1.default)(col)}, ?))`);
+            const concatenatedExpression = `concat(${parts.join(", ")})`;
             return `SELECT *, CAST(${concatenatedExpression} AS VARCHAR) AS ${(0, quoteIdentifier_js_1.default)(newColumn)} FROM ${input}`;
         },
     });

package/script/methods/selectTables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selectTables.d.ts","sourceRoot":"","sources":["../../src/methods/selectTables.ts"],"names":[],"mappings":"AAGA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAKjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBA+CxD"}
+{"version":3,"file":"selectTables.d.ts","sourceRoot":"","sources":["../../src/methods/selectTables.ts"],"names":[],"mappings":"AAIA,OAAO,WAAW,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAKjD,wBAA8B,YAAY,CACxC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC,EAAE,iBAqDxD"}

package/script/methods/selectTables.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = selectTables;
+const cacheTableDependencies_js_1 = require("../helpers/cacheTableDependencies.js");
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
 const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
@@ -15,6 +16,11 @@ async function selectTables(simpleDB, tables) {
     const selected = Array.isArray(tables) ? tables : [tables];
     (0, assertSameDatabase_js_1.default)(simpleDB, selected.filter((table) => table instanceof SimpleTable_js_1.default), "selectTables()");
     const tablesToBeSelected = selected.map((t) => t instanceof SimpleTable_js_1.default ? t.name : t);
+    for (const table of simpleDB.getTables()) {
+        if (!tablesToBeSelected.includes(table.name)) {
+            (0, cacheTableDependencies_js_1.assertCacheTableMutation)(table);
+        }
+    }
     const existingTables = await simpleDB.getTableNames();
     const missingTables = tablesToBeSelected.filter((table) => !existingTables.includes(table));
     if (missingTables.length > 0) {

package/script/methods/setTypes.d.ts

@@ -1,5 +1,5 @@
 import type SimpleTable from "../class/SimpleTable.js";
 export default function setTypes(simpleTable: SimpleTable, types: {
-    [key: string]: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | `geometry('${string}')` | `GEOMETRY('${string}')`;
+    [key: string]: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | `float[${number}]` | `FLOAT[${number}]` | `geometry('${string}')` | `GEOMETRY('${string}')`;
 }): void;
 //# sourceMappingURL=setTypes.d.ts.map

package/script/methods/setTypes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"setTypes.d.ts","sourceRoot":"","sources":["../../src/methods/setTypes.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,WAAW,EAAE,WAAW,EACxB,KAAK,EAAE;IACL,CAAC,GAAG,EAAE,MAAM,GACR,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,aAAa,MAAM,IAAI,GACvB,aAAa,MAAM,IAAI,CAAC;CAC7B,QAkCF"}
+{"version":3,"file":"setTypes.d.ts","sourceRoot":"","sources":["../../src/methods/setTypes.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,UAAU,QAAQ,CAC9B,WAAW,EAAE,WAAW,EACxB,KAAK,EAAE;IACL,CAAC,GAAG,EAAE,MAAM,GACR,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,MAAM,GACN,MAAM,GACN,SAAS,MAAM,GAAG,GAClB,SAAS,MAAM,GAAG,GAClB,aAAa,MAAM,IAAI,GACvB,aAAa,MAAM,IAAI,CAAC;CAC7B,QAkCF"}

package/script/methods/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/methods/stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA+B,MAAM,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IACP,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,GACL,cAAc,CAAC;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CAmE7D"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/methods/stream.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAGvD,wBAA+B,MAAM,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IACP,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CAChB,GACL,cAAc,CAAC;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CAsE7D"}

package/script/methods/stream.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = stream;
+const cleanSQL_js_1 = __importDefault(require("../helpers/cleanSQL.js"));
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const runQuery_js_1 = require("../helpers/runQuery.js");
 const SDAError_js_1 = __importDefault(require("../class/SDAError.js"));
@@ -23,7 +24,7 @@ async function* stream(simpleTable, options = {}) {
             ? [options.columns]
             : options.columns)
         : undefined;
-    const query = `SELECT ${columns ? columns.map((d) => `${(0, quoteIdentifier_js_1.default)(d)}`).join(", ") : "*"} FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""};`;
+    const query = (0, cleanSQL_js_1.default)(`SELECT ${columns ? columns.map((d) => `${(0, quoteIdentifier_js_1.default)(d)}`).join(", ") : "*"} FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}${options.conditions ? ` WHERE ${options.conditions}` : ""};`, simpleTable.sdb.expressionSyntax);
     try {
         await (0, observeQuery_js_1.default)(simpleTable.connection, query, [], {
             logSQL: simpleTable.sdb.logSQL,

package/script/methods/updateWithJS.d.ts

@@ -3,9 +3,7 @@ type Row = {
     [key: string]: unknown;
 };
 type DataModifier = (rows: Row[]) => Row[] | Promise<Row[]>;
-type UpdateWithJSOptions = {
-    batchSize?: number;
-};
+type UpdateWithJSOptions = NonNullable<Parameters<SimpleTable["updateWithJS"]>[1]>;
 export default function updateWithJS(simpleTable: SimpleTable, dataModifier: DataModifier, options?: UpdateWithJSOptions): void;
 export {};
 //# sourceMappingURL=updateWithJS.d.ts.map

package/script/methods/updateWithJS.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"updateWithJS.d.ts","sourceRoot":"","sources":["../../src/methods/updateWithJS.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,KAAK,GAAG,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEtC,KAAK,YAAY,GAAG,CAClB,IAAI,EAAE,GAAG,EAAE,KACR,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAE5B,KAAK,mBAAmB,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,WAAW,EAAE,WAAW,EACxB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,mBAAwB,GAChC,IAAI,CAeN"}
+{"version":3,"file":"updateWithJS.d.ts","sourceRoot":"","sources":["../../src/methods/updateWithJS.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAQvD,KAAK,GAAG,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEtC,KAAK,YAAY,GAAG,CAClB,IAAI,EAAE,GAAG,EAAE,KACR,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;AAE5B,KAAK,mBAAmB,GAAG,WAAW,CACpC,UAAU,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3C,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,WAAW,EAAE,WAAW,EACxB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,mBAAwB,GAChC,IAAI,CAeN"}

package/script/methods/updateWithJS.js

@@ -4,10 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = updateWithJS;
+const prepareGeometry_js_1 = __importDefault(require("../helpers/prepareGeometry.js"));
+const geometryFromJSON_js_1 = __importDefault(require("../helpers/geometryFromJSON.js"));
 const quoteIdentifier_js_1 = __importDefault(require("../helpers/quoteIdentifier.js"));
 const tableRegistry_js_1 = require("../helpers/tableRegistry.js");
 const queueOp_js_1 = __importDefault(require("../helpers/queueOp.js"));
 const loadArray_js_1 = require("./loadArray.js");
+const node_api_1 = require("@duckdb/node-api");
+const readMutationRows_js_1 = __importDefault(require("../helpers/readMutationRows.js"));
+const parseDuckDBType_js_1 = __importDefault(require("../helpers/parseDuckDBType.js"));
 function updateWithJS(simpleTable, dataModifier, options = {}) {
     if (options.batchSize !== undefined &&
         (!Number.isInteger(options.batchSize) || options.batchSize < 1)) {
@@ -23,39 +28,35 @@ function updateWithJS(simpleTable, dataModifier, options = {}) {
 }
 async function executeUpdateWithJS(simpleTable, dataModifier, options) {
     const types = await simpleTable.getTypes();
-    // Geometry columns are typed as GEOMETRY('<crs>'), not a bare "GEOMETRY".
-    if (Object.values(types).some((d) => d.toUpperCase().startsWith("GEOMETRY"))) {
-        throw new Error("updateWithJS doesn't work with tables containing geometries.");
-    }
-    if (options.batchSize === undefined) {
-        const oldData = await simpleTable.getData();
-        if (!oldData) {
-            throw new Error("No data from getData.");
+    const geometryColumns = Object.keys(types).filter((key) => types[key].toUpperCase().startsWith("GEOMETRY"));
+    for (const column of geometryColumns) {
+        if (types[column].toUpperCase() !== "GEOMETRY('EPSG:4326')") {
+            throw new Error(`Column ${JSON.stringify(column)} has CRS ${types[column]}. Call .reproject("EPSG:4326", { column: ${JSON.stringify(column)} }) before updateWithJS().`);
         }
-        const newData = await dataModifier(oldData);
-        if (newData.length === 0) {
-            if (oldData.length === 0) {
-                // Empty table in, empty data out: nothing to update.
-                return;
-            }
-            throw new Error("The dataModifier returned no rows. updateWithJS can't infer the table schema from zero rows.");
+        const name = (0, quoteIdentifier_js_1.default)(column);
+        // Compare the actual binary representation, including dimensions and double
+        // precision, rather than assuming GeoJSON serialization is lossless.
+        const result = await simpleTable.connection.runAndReadAll(`SELECT count(*) AS invalid FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}
+       WHERE ${name} IS NOT NULL AND
+       ST_AsWKB(${name}) != ST_AsWKB(${(0, geometryFromJSON_js_1.default)(`ST_AsGeoJSON(${name})`)})`);
+        if (result.getRows()[0][0] !== 0n) {
+            throw new Error(`Column ${JSON.stringify(column)} contains geometry that cannot round-trip through GeoJSON without losing precision or dimensions; unsupported representations cannot be updated with updateWithJS().`);
         }
-        await (0, loadArray_js_1.executePreparedArray)(simpleTable, (0, loadArray_js_1.prepareArray)(newData));
-        return;
     }
-    const batchSize = options.batchSize;
-    if (Object.keys(types).includes("__sda_rowid")) {
-        throw new Error('The table has a column named "__sda_rowid", which conflicts with the internal column used by the batchSize option. Rename it or run updateWithJS without batchSize.');
+    for (const key of Object.keys(options.columnTypes ?? {})) {
+        if (Object.keys(types).some((name) => name.toLowerCase() === key.toLowerCase())) {
+            throw new Error(`updateWithJS() columnTypes may only declare new columns; ${JSON.stringify(key)} already exists.`);
+        }
     }
-    const rowCount = await simpleTable.getRowCount();
-    if (rowCount === 0) {
-        // Same behavior as the non-batched path on an empty table.
-        const newData = await dataModifier([]);
-        if (newData.length === 0) {
-            return;
+    const batchSize = options.batchSize;
+    const geometryBatch = batchSize !== undefined && geometryColumns.length > 0;
+    if (batchSize !== undefined) {
+        // DuckDB resolves identifiers case-insensitively, and a real rowid column
+        // shadows its hidden row identifier used for pagination.
+        const conflictingColumn = Object.keys(types).find((column) => ["rowid", "__sda_rowid"].includes(column.toLowerCase()));
+        if (conflictingColumn !== undefined) {
+            throw new Error(`The table has a column named "${conflictingColumn}", which conflicts with the internal column used by the batchSize option. Rename it or run updateWithJS without batchSize.`);
         }
-        await (0, loadArray_js_1.executePreparedArray)(simpleTable, (0, loadArray_js_1.prepareArray)(newData));
-        return;
     }
     // Rows are pulled in batches by rowid, passed through the modifier, and
     // accumulated in a temporary table, so only one batch of rows is
@@ -66,27 +67,112 @@ async function executeUpdateWithJS(simpleTable, dataModifier, options) {
     try {
         let first = true;
         let lastRowid = null;
+        let sawRows = false;
+        let outputOffset = 0;
+        let inputOffset = 0;
         while (true) {
-            const batch = (await simpleTable.sdb.customQuery(`SELECT *, rowid AS __sda_rowid FROM ${(0, quoteIdentifier_js_1.default)(simpleTable.name)}${lastRowid === null ? "" : ` WHERE rowid > ${lastRowid}`} ORDER BY rowid LIMIT ${batchSize}`, { returnData: true }));
-            if (batch.length === 0) {
+            const page = `${(0, quoteIdentifier_js_1.default)(simpleTable.name)}${lastRowid === null ? "" : ` WHERE rowid > ${lastRowid}`}${batchSize === undefined ? "" : ` ORDER BY rowid LIMIT ${batchSize}`}`;
+            // Select the batch before serializing geometry. Otherwise DuckDB can
+            // evaluate ST_AsGeoJSON on remaining rows before applying the LIMIT,
+            // repeatedly converting polygons that belong to later batches.
+            const from = geometryBatch
+                ? `(SELECT *, rowid AS __sda_rowid FROM ${page}) AS __sda_batch ORDER BY __sda_rowid`
+                : page;
+            const source = await (0, readMutationRows_js_1.default)(simpleTable.connection, `SELECT ${Object.keys(types).map((key) => geometryColumns.includes(key)
+                ? `ST_AsGeoJSON(${(0, quoteIdentifier_js_1.default)(key)})::VARCHAR AS ${(0, quoteIdentifier_js_1.default)(key)}`
+                : (0, quoteIdentifier_js_1.default)(key)).join(", ")}${batchSize === undefined
+                ? ""
+                : geometryBatch
+                    ? ", __sda_rowid"
+                    : ", rowid AS __sda_rowid"} FROM ${from}`, batchSize === undefined ? undefined : "__sda_rowid");
+            const batch = source.rows;
+            const inputCount = batch.length;
+            if (inputCount === 0 && sawRows)
                 break;
+            sawRows ||= inputCount > 0;
+            if (batchSize !== undefined && batch.length > 0) {
+                // Keep the cursor separate: adding and deleting it on every row also
+                // makes ordinary callback operations such as object spread slower.
+                lastRowid = source.lastCursorValue;
             }
-            lastRowid = batch[batch.length - 1].__sda_rowid;
-            for (const row of batch) {
-                delete row.__sda_rowid;
+            for (const [index, row] of batch.entries()) {
+                for (const key of geometryColumns) {
+                    row[key] = row[key] === null ? null : JSON.parse(row[key]);
+                    (0, prepareGeometry_js_1.default)(row[key], key, inputOffset + index + 1);
+                }
             }
+            inputOffset += inputCount;
             const modified = await dataModifier(batch);
             if (modified.length === 0) {
+                if (!sawRows)
+                    return;
+                if (batchSize === undefined)
+                    break;
                 continue;
             }
-            await (0, loadArray_js_1.executePreparedArray)(scratch, (0, loadArray_js_1.prepareArray)(modified));
+            const keys = Object.keys(modified[0]);
+            for (const key of Object.keys(options.columnTypes ?? {})) {
+                if (!keys.includes(key)) {
+                    throw new Error(`Unknown column ${JSON.stringify(key)} in updateWithJS columnTypes.`);
+                }
+            }
+            const newKeys = keys.filter((key) => !source.types.has(key));
+            const added = newKeys.length === 0 ? undefined : (0, loadArray_js_1.prepareArray)(modified.map((row) => Object.fromEntries(newKeys.map((key) => [key, row[key]]))), options.columnTypes);
+            const outputGeometry = keys.filter((key) => geometryColumns.includes(key) ||
+                (added && added.types[newKeys.indexOf(key)] === "GEOMETRY('EPSG:4326')"));
+            const outputJSON = keys.filter((key) => types[key] === "JSON" ||
+                (added && added.types[newKeys.indexOf(key)] === "JSON"));
+            if (outputGeometry.length && !simpleTable.sdb.spatialLoaded) {
+                await simpleTable.sdb.customQuery("INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;");
+                simpleTable.sdb.spatialLoaded = true;
+            }
+            const geometryData = new Map(outputGeometry.map((key) => [
+                key,
+                geometryColumns.includes(key)
+                    ? modified.map((row, i) => (0, prepareGeometry_js_1.default)(row[key], key, outputOffset + i + 1))
+                    : added.columnsData[newKeys.indexOf(key)],
+            ]));
+            outputOffset += modified.length;
+            const columnTypes = keys.map((key) => outputGeometry.includes(key) || outputJSON.includes(key)
+                ? (0, parseDuckDBType_js_1.default)("VARCHAR")
+                : source.types.get(key) ??
+                    (0, parseDuckDBType_js_1.default)(added.types[newKeys.indexOf(key)]));
+            await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(scratch.name)} (${keys.map((key, i) => `${(0, quoteIdentifier_js_1.default)(key)} ${columnTypes[i]}`)
+                .join(", ")})`);
+            const appender = await simpleTable.connection.createAppender(scratch.name);
+            try {
+                for (let start = 0; start < modified.length; start += 2000) {
+                    const end = Math.min(start + 2000, modified.length);
+                    const chunk = node_api_1.DuckDBDataChunk.create(columnTypes, end - start);
+                    for (let i = 0; i < keys.length; i++) {
+                        const key = keys[i];
+                        chunk.setColumnValues(i, outputGeometry.includes(key)
+                            ? geometryData.get(key).slice(start, end)
+                            : source.types.has(key)
+                                ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[i]))
+                                : added.columnsData[newKeys.indexOf(key)].slice(start, end));
+                    }
+                    appender.appendDataChunk(chunk);
+                }
+                appender.flushSync();
+            }
+            finally {
+                appender.closeSync();
+            }
+            const projection = keys.map((key) => outputGeometry.includes(key)
+                ? `${(0, geometryFromJSON_js_1.default)((0, quoteIdentifier_js_1.default)(key))} AS ${(0, quoteIdentifier_js_1.default)(key)}`
+                : outputJSON.includes(key)
+                    ? `${(0, quoteIdentifier_js_1.default)(key)}::JSON AS ${(0, quoteIdentifier_js_1.default)(key)}`
+                    : (0, quoteIdentifier_js_1.default)(key)).join(", ");
             if (first) {
-                await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(accumulator)} AS SELECT * FROM ${(0, quoteIdentifier_js_1.default)(scratch.name)}`);
+                await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${(0, quoteIdentifier_js_1.default)(accumulator)} AS SELECT ${projection} FROM ${(0, quoteIdentifier_js_1.default)(scratch.name)}`);
                 first = false;
             }
             else {
-                await simpleTable.sdb.customQuery(`INSERT INTO ${(0, quoteIdentifier_js_1.default)(accumulator)} BY NAME SELECT * FROM ${(0, quoteIdentifier_js_1.default)(scratch.name)}`);
+                await simpleTable.sdb.customQuery(`INSERT INTO ${(0, quoteIdentifier_js_1.default)(accumulator)} BY NAME SELECT ${projection} FROM ${(0, quoteIdentifier_js_1.default)(scratch.name)}`);
             }
+            if (batchSize === undefined || inputCount === 0)
+                break;
         }
         if (first) {
             throw new Error("The dataModifier returned no rows. updateWithJS can't infer the table schema from zero rows.");